diff --git a/src/common/mem.c b/src/common/mem.c index e473cb8bfe5..8dd7254bca3 100644 --- a/src/common/mem.c +++ b/src/common/mem.c @@ -806,6 +806,8 @@ pmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned i pflags |= POBJ_FLAG_ZERO; if (flags & UMEM_FLAG_NO_FLUSH) pflags |= POBJ_FLAG_NO_FLUSH; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= POBJ_FLAG_TX_NO_ABORT; return umem_id2off(umm, pmemobj_tx_xalloc(size, type_num, pflags)); } @@ -828,6 +830,8 @@ pmem_tx_xadd(struct umem_instance *umm, umem_off_t umoff, uint64_t offset, if (flags & UMEM_XADD_NO_SNAPSHOT) pflags |= POBJ_XADD_NO_SNAPSHOT; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= POBJ_FLAG_TX_NO_ABORT; rc = pmemobj_tx_xadd_range(umem_off2id(umm, umoff), offset, size, pflags); @@ -997,6 +1001,16 @@ pmem_defer_free(struct umem_instance *umm, umem_off_t off, void *act) pmemobj_defer_free(pop, id, (struct pobj_action *)act); } +static int +pmem_tx_set_snapbuf(struct umem_instance *umm, umem_off_t snapbuf, size_t size) +{ + void *buf = umem_off2ptr(umm, snapbuf); + int rc; + + rc = pmemobj_tx_log_append_buffer(TX_LOG_TYPE_SNAPSHOT, buf, size); + return rc ? umem_tx_errno(rc) : 0; +} + static int pmem_tx_stage(void) { @@ -1135,28 +1149,28 @@ umem_tx_add_cb(struct umem_instance *umm, struct umem_tx_stage_data *txd, return 0; } -static umem_ops_t pmem_ops = { - .mo_tx_free = pmem_tx_free, - .mo_tx_alloc = pmem_tx_alloc, - .mo_tx_add = pmem_tx_add, - .mo_tx_xadd = pmem_tx_xadd, - .mo_tx_add_ptr = pmem_tx_add_ptr, - .mo_tx_abort = pmem_tx_abort, - .mo_tx_begin = pmem_tx_begin, - .mo_tx_commit = pmem_tx_commit, - .mo_tx_stage = pmem_tx_stage, - .mo_reserve = pmem_reserve, - .mo_defer_free = pmem_defer_free, - .mo_cancel = pmem_cancel, - .mo_tx_publish = pmem_tx_publish, - .mo_atomic_copy = pmem_atomic_copy, - .mo_atomic_alloc = pmem_atomic_alloc, - .mo_atomic_free = pmem_atomic_free, - .mo_atomic_flush = pmem_atomic_flush, - .mo_tx_add_callback = umem_tx_add_cb, +static umem_ops_t pmem_ops = { + .mo_tx_free = pmem_tx_free, + .mo_tx_alloc = pmem_tx_alloc, + .mo_tx_add = pmem_tx_add, + .mo_tx_xadd = pmem_tx_xadd, + .mo_tx_add_ptr = pmem_tx_add_ptr, + .mo_tx_abort = pmem_tx_abort, + .mo_tx_begin = pmem_tx_begin, + .mo_tx_commit = pmem_tx_commit, + .mo_tx_set_snapbuf = pmem_tx_set_snapbuf, + .mo_tx_stage = pmem_tx_stage, + .mo_reserve = pmem_reserve, + .mo_defer_free = pmem_defer_free, + .mo_cancel = pmem_cancel, + .mo_tx_publish = pmem_tx_publish, + .mo_atomic_copy = pmem_atomic_copy, + .mo_atomic_alloc = pmem_atomic_alloc, + .mo_atomic_free = pmem_atomic_free, + .mo_atomic_flush = pmem_atomic_flush, + .mo_tx_add_callback = umem_tx_add_cb, }; - /** BMEM operations (depends on dav) */ static int @@ -1199,6 +1213,8 @@ bmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned i pflags |= DAV_FLAG_ZERO; if (flags & UMEM_FLAG_NO_FLUSH) pflags |= DAV_FLAG_NO_FLUSH; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= DAV_FLAG_TX_NO_ABORT; return dav_tx_xalloc(size, type_num, pflags); } @@ -1221,6 +1237,8 @@ bmem_tx_xadd(struct umem_instance *umm, umem_off_t umoff, uint64_t offset, if (flags & UMEM_XADD_NO_SNAPSHOT) pflags |= DAV_XADD_NO_SNAPSHOT; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= DAV_FLAG_TX_NO_ABORT; rc = dav_tx_xadd_range(umem_off2offset(umoff), size, pflags); return rc ? umem_tx_errno(rc) : 0; @@ -1438,6 +1456,8 @@ bmem_tx_alloc_v2(struct umem_instance *umm, size_t size, uint64_t flags, unsigne pflags |= DAV_FLAG_ZERO; if (flags & UMEM_FLAG_NO_FLUSH) pflags |= DAV_FLAG_NO_FLUSH; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= DAV_FLAG_TX_NO_ABORT; if (mbkt_id != 0) pflags |= DAV_EZONE_ID(mbkt_id); return dav_tx_alloc_v2(size, type_num, pflags); @@ -1462,6 +1482,8 @@ bmem_tx_xadd_v2(struct umem_instance *umm, umem_off_t umoff, uint64_t offset, if (flags & UMEM_XADD_NO_SNAPSHOT) pflags |= DAV_XADD_NO_SNAPSHOT; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= DAV_FLAG_TX_NO_ABORT; rc = dav_tx_xadd_range_v2(umem_off2offset(umoff), size, pflags); return rc ? umem_tx_errno(rc) : 0; diff --git a/src/common/tests/btree.c b/src/common/tests/btree.c index c60c0bdebc8..05eddec9fe1 100644 --- a/src/common/tests/btree.c +++ b/src/common/tests/btree.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2016-2022 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -69,8 +69,7 @@ struct btr_test_state { }; #define IK_TREE_CLASS 100 -#define POOL_NAME "/mnt/daos/btree-test" -#define POOL_SIZE ((1024 * 1024 * 1024ULL)) +#define POOL_NAME "/mnt/daos/btree-test" /** customized functions for btree */ static int diff --git a/src/common/tests/btree_direct.c b/src/common/tests/btree_direct.c index d399bbd1681..bf24c83d272 100644 --- a/src/common/tests/btree_direct.c +++ b/src/common/tests/btree_direct.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2018-2022 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -55,8 +55,7 @@ struct btr_test_state { }; #define SK_TREE_CLASS 100 -#define POOL_NAME "/mnt/daos/btree-direct-test" -#define POOL_SIZE ((1024 * 1024 * 1024ULL)) +#define POOL_NAME "/mnt/daos/btree-direct-test" #define SK_ORDER_DEF 16 diff --git a/src/common/tests/umem_test.c b/src/common/tests/umem_test.c index 6080843f51c..b916c96a9f9 100644 --- a/src/common/tests/umem_test.c +++ b/src/common/tests/umem_test.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2019-2024 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -27,7 +28,6 @@ #include #include "utest_common.h" -#define POOL_SIZE ((1024 * 1024 * 1024ULL)) #define MAX_CHUNKS 8192 struct chunk { @@ -207,7 +207,7 @@ teardown_pmem(void **state) return rc; } -int +static int setup_pmem(void **state) { struct test_arg *arg = *state; @@ -749,6 +749,16 @@ test_p2_evict(void **state) umem_cache_free(&arg->ta_store); } +static void +test_tx_fail_no_abort(void **state) +{ + struct test_arg *arg = *state; + int rc; + + rc = utest_tx_fail_no_abort(arg->ta_utx); + assert_int_equal(rc, 0); +} + int main(int argc, char **argv) { @@ -762,6 +772,8 @@ main(int argc, char **argv) {"UMEM007: Test page cache many writes", test_many_writes, NULL, NULL}, {"UMEM008: Test phase2 APIs", test_p2_basic, NULL, NULL}, {"UMEM009: Test phase2 eviction", test_p2_evict, NULL, NULL}, + {"UMEM010: Test TX failure with NO_ABORT on pmem", test_tx_fail_no_abort, setup_pmem, + teardown_pmem}, {NULL, NULL, NULL, NULL}}; d_register_alt_assert(mock_assert); diff --git a/src/common/tests/umem_test_bmem.c b/src/common/tests/umem_test_bmem.c index a62fe3814a4..7d6442c6c28 100644 --- a/src/common/tests/umem_test_bmem.c +++ b/src/common/tests/umem_test_bmem.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2019-2024 Intel Corporation. - * (C) Copyright 2023-2025 Hewlett Packard Enterprise Development LP. + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -28,7 +28,6 @@ #include #include "utest_common.h" -#define POOL_SIZE ((256 * 1024 * 1024ULL)) #define NEMB_RATIO (0.8) #define MB_SIZE (16 * 1024 * 1024) @@ -2669,6 +2668,16 @@ test_umempobj_heap_mb_stats(void **state) D_FREE(name); } +static void +test_tx_fail_no_abort(void **state) +{ + struct test_arg *arg = *state; + int rc; + + rc = utest_tx_fail_no_abort(arg->ta_utx); + assert_int_equal(rc, 0); +} + int main(int argc, char **argv) { @@ -2702,6 +2711,8 @@ main(int argc, char **argv) teardown_pmem}, {"BMEM015a: Test tx defer free publish/cancel", test_tx_bucket_dfree_publish_cancel, setup_pmem, teardown_pmem}, + {"BMEM016: Test TX failure with NO_ABORT on bmem", test_tx_fail_no_abort, setup_pmem, + teardown_pmem}, {NULL, NULL, NULL, NULL}}; static const struct CMUnitTest v2_tests[] = { @@ -2747,6 +2758,8 @@ main(int argc, char **argv) NULL}, {"BMEM022: Test umempobj non_evictable MB usage", test_umempobj_nemb_usage, NULL, NULL}, {"BMEM023: Test umempobj get MB stats", test_umempobj_heap_mb_stats, NULL, NULL}, + {"BMEM024: Test TX failure with NO_ABORT on bmem_v2", test_tx_fail_no_abort, + setup_pmem_v2, teardown_pmem}, {NULL, NULL, NULL, NULL}}; rc = daos_debug_init(DAOS_LOG_DEFAULT); diff --git a/src/common/tests/utest_common.c b/src/common/tests/utest_common.c index 472039c74a8..1c9d52d0c6a 100644 --- a/src/common/tests/utest_common.c +++ b/src/common/tests/utest_common.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -375,3 +376,62 @@ utest_check_mem_initial_status(struct utest_context *utx) } return 0; } + +/* + * A failed allocation with UMEM_FLAG_NO_ABORT must leave the transaction + * alive so in-tx fallbacks (e.g. vos_dtx_reuse_cmt_blob) keep working; + * if without the flag, the failure aborts the TX and any further tx_add + * is a fatal usage error in the allocator. + */ +int +utest_tx_fail_no_abort(struct utest_context *utx) +{ + struct umem_instance *umm = utest_utx2umm(utx); + int *value; + umem_off_t umoff = UMOFF_NULL; + umem_off_t huge; + int rc; + + rc = utest_tx_begin(utx); + if (rc != 0) + goto done; + + /* Larger than the pool: must fail, but must NOT abort the TX */ + huge = umem_alloc_verb(umm, UMEM_FLAG_ZERO | UMEM_FLAG_NO_ABORT, POOL_SIZE, + UMEM_DEFAULT_MBKT_ID); + if (!UMOFF_IS_NULL(huge)) { + D_ERROR("huge allocation unexpectedly succeeded\n"); + D_GOTO(end, rc = 1); + } + + /* TX still alive: alloc + snapshot + write must all work */ + umoff = umem_zalloc(umm, 4); + if (UMOFF_IS_NULL(umoff)) { + D_ERROR("small alloc failed after NO_ABORT failure\n"); + D_GOTO(end, rc = 2); + } + + value = umem_off2ptr(umm, umoff); + + rc = umem_tx_xadd_ptr(umm, value, (uint64_t)(-2), UMEM_FLAG_NO_ABORT); + if (rc == 0) { + D_ERROR("large sized add_ptr() should fail\n"); + D_GOTO(end, rc = 3); + } + + rc = umem_tx_add_ptr(umm, value, 4); + if (rc != 0) { + D_ERROR("tx_add_ptr failed after NO_ABORT failure\n"); + D_GOTO(end, rc = 4); + } + + *value = 42; + + rc = umem_free(umm, umoff); + +end: + rc = utest_tx_end(utx, rc); + +done: + return rc; +} diff --git a/src/common/tests/utest_common.h b/src/common/tests/utest_common.h index 2fd3328234e..71e7826e768 100644 --- a/src/common/tests/utest_common.h +++ b/src/common/tests/utest_common.h @@ -1,11 +1,13 @@ /** * (C) Copyright 2019-2021 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ #include #define UTEST_POOL_NAME_MAX 255 +#define POOL_SIZE (1024 * 1024 * 1024ULL) struct utest_context; @@ -174,3 +176,10 @@ int utest_check_mem_increase(struct utest_context *utx); */ int utest_check_mem_initial_status(struct utest_context *utx); +/** Check transaction failure with NO_ABROT flag + * \param utx[IN] utest_context + * + * \return 0 on success + */ +int +utest_tx_fail_no_abort(struct utest_context *utx); diff --git a/src/dtx/dtx_common.c b/src/dtx/dtx_common.c index 169c2b2d6a2..d27963b5bf8 100644 --- a/src/dtx/dtx_common.c +++ b/src/dtx/dtx_common.c @@ -1032,6 +1032,34 @@ dtx_sub_init(struct dtx_handle *dth, daos_unit_oid_t *oid, uint64_t dkey_hash) return rc; } +int +dtx_commit_large(daos_handle_t coh, struct dtx_id *dtis, int cnt, bool keep_act, bool *rm_cos) +{ + int step = DTX_YIELD_CYCLE; + int committed = 0; + int rc = 0; + int i = 0; + + while (i < cnt) { + if (i + step > cnt) + step = cnt - i; + + rc = vos_dtx_commit(coh, dtis + i, step, keep_act, rm_cos); + if (rc >= 0) { + committed += rc; + i += step; + } else { + if ((rc != -DER_NOSPACE && rc != -DER_OVERFLOW) || step <= 1) + return rc; + + /* If out of space, reduce TX size and retry. */ + step >>= 1; + } + } + + return committed; +} + /** * Prepare the leader DTX handle in DRAM. * diff --git a/src/dtx/dtx_internal.h b/src/dtx/dtx_internal.h index 6930976d08f..65a311b2a41 100644 --- a/src/dtx/dtx_internal.h +++ b/src/dtx/dtx_internal.h @@ -101,7 +101,7 @@ CRT_RPC_DECLARE(dtx, DAOS_ISEQ_DTX, DAOS_OSEQ_DTX); CRT_RPC_DECLARE(dtx_coll, DAOS_ISEQ_COLL_DTX, DAOS_OSEQ_COLL_DTX); -#define DTX_YIELD_CYCLE (DTX_THRESHOLD_COUNT >> 3) +#define DTX_YIELD_CYCLE DTX_PIGGYBACK_COUNT /* The count threshold (per pool) for triggering DTX aggregation. */ #define DTX_AGG_THD_CNT_MAX (1 << 24) @@ -263,6 +263,7 @@ extern btr_ops_t dtx_btr_cos_ops; /* dtx_common.c */ int dtx_handle_reinit(struct dtx_handle *dth); void dtx_batched_commit(void *arg); +int dtx_commit_large(daos_handle_t coh, struct dtx_id *dtis, int cnt, bool keep_act, bool *rm_cos); void dtx_aggregation_main(void *arg); int start_dtx_reindex_ult(struct ds_cont_child *cont); void dtx_merge_check_result(int *tgt, int src); diff --git a/src/dtx/dtx_rpc.c b/src/dtx/dtx_rpc.c index e6be4616a38..b2bcc0cc602 100644 --- a/src/dtx/dtx_rpc.c +++ b/src/dtx/dtx_rpc.c @@ -878,11 +878,13 @@ dtx_commit(struct ds_cont_child *cont, struct dtx_entry **dtes, } /* - * Some DTX entries may have been committed on some participants. Then mark all - * the DTX entries (in the dtis) as "PARTIAL_COMMITTED" and re-commit them later. - * It is harmless to re-commit the DTX that has ever been committed. + * Some DTX entries may have been committed on parts of (remote) participants. + * It is no way to revert related partially committed DTX entries since we do + * not know whether someone has already read related data for those partially + * committed DTX entries. Then let's mark all the DTX entries in the @dtis as + * "PARTIAL_COMMITTED" and re-commit them later. It is safe to re-commit them. */ - rc1 = vos_dtx_commit(cont->sc_hdl, dca.dca_dtis, count, rc != 0, rm_cos); + rc1 = dtx_commit_large(cont->sc_hdl, dca.dca_dtis, count, rc != 0, rm_cos); if (rc1 > 0) { dra->dra_committed += rc1; rc1 = 0; diff --git a/src/dtx/dtx_srv.c b/src/dtx/dtx_srv.c index 0a04c47f2e9..db6c73a7b32 100644 --- a/src/dtx/dtx_srv.c +++ b/src/dtx/dtx_srv.c @@ -196,19 +196,17 @@ dtx_handler(crt_rpc_t *rpc) if (unlikely(din->di_epoch == 1)) D_GOTO(out, rc = -DER_IO); - while (i < din->di_dtx_array.ca_count) { - if (i + count > din->di_dtx_array.ca_count) - count = din->di_dtx_array.ca_count - i; - - dtis = (struct dtx_id *)din->di_dtx_array.ca_arrays + i; - rc1 = vos_dtx_commit(cont->sc_hdl, dtis, count, false, NULL); - if (rc1 > 0) - committed += rc1; - else if (rc == 0 && rc1 < 0) - rc = rc1; + /* + * The count of DTX entries will not exceed DTX_THRESHOLD_COUNT, that + * is guaranteed by the caller. Even if some wrong number was offered + * (via network), dtx_commit_large will handle related cases properly. + */ + rc1 = dtx_commit_large(cont->sc_hdl, (struct dtx_id *)din->di_dtx_array.ca_arrays, + (int)din->di_dtx_array.ca_count, false, NULL); + if (rc1 < 0) + D_GOTO(out, rc = rc1); - i += count; - } + committed += rc1; if (din->di_flags.ca_count > 0) flags = din->di_flags.ca_arrays; diff --git a/src/dtx/tests/dts_aggregate.c b/src/dtx/tests/dts_aggregate.c index 89a8db7bacf..4eebd13a321 100644 --- a/src/dtx/tests/dts_aggregate.c +++ b/src/dtx/tests/dts_aggregate.c @@ -156,6 +156,14 @@ tx_add_ptr(struct umem_instance *umm, void *ptr, size_t ptr_size) return mock_type(int); } +static int +tx_xadd(struct umem_instance *umm, umem_off_t umoff, uint64_t offset, size_t size, uint64_t flags) +{ + void *ptr = umem_off2ptr(umm, umoff); + + return tx_add_ptr(umm, ptr, size); +} + static int tx_free(struct umem_instance *umm, umem_off_t umoff) { @@ -227,6 +235,7 @@ static umem_ops_t umm_ops = {.mo_tx_begin = tx_begin, .mo_tx_commit = tx_commit, .mo_tx_abort = tx_abort, .mo_tx_add_ptr = tx_add_ptr, + .mo_tx_xadd = tx_xadd, .mo_tx_free = tx_free}; static int diff --git a/src/include/daos/common.h b/src/include/daos/common.h index e8e9b792238..9c1906abba5 100644 --- a/src/include/daos/common.h +++ b/src/include/daos/common.h @@ -860,6 +860,7 @@ enum { #define DAOS_OBJ_COLL_SPARSE (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x4d) #define DAOS_DTX_RESEND_NONLEADER (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x4e) +#define DAOS_DTX_NOSPACE_NOREFRESH (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x4f) #define DAOS_NVME_FAULTY (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x50) #define DAOS_NVME_WRITE_ERR (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x51) diff --git a/src/include/daos/dtx.h b/src/include/daos/dtx.h index b91e5a1084c..db175503646 100644 --- a/src/include/daos/dtx.h +++ b/src/include/daos/dtx.h @@ -1,6 +1,6 @@ /** * (C) Copyright 2019-2023 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -25,6 +25,8 @@ */ #define DTX_THRESHOLD_COUNT (1 << 9) +#define DTX_PIGGYBACK_COUNT (1 << 6) + /* The time (in second) threshold for batched DTX commit. */ #define DTX_COMMIT_THRESHOLD_AGE 10 diff --git a/src/include/daos/mem.h b/src/include/daos/mem.h index d451f26704d..2f798dc0327 100644 --- a/src/include/daos/mem.h +++ b/src/include/daos/mem.h @@ -677,6 +677,13 @@ struct umem_instance; #define UMEM_FLAG_NO_FLUSH (((uint64_t)1) << 1) #define UMEM_XADD_NO_SNAPSHOT (((uint64_t)1) << 2) +/* + * Do NOT abort current TX if failed to operate (such as allocation) something under current + * transaction context, instead, return some errno# to the caller who can handle the failure + * and further use such TX. + */ +#define UMEM_FLAG_NO_ABORT (((uint64_t)1) << 3) + /* Macros associated with Memory buckets */ #define UMEM_DEFAULT_MBKT_ID 0 @@ -751,6 +758,8 @@ typedef struct { int (*mo_tx_commit)(struct umem_instance *umm, void *data); #ifdef DAOS_PMEM_BUILD + /** Set emergency buffer for transaction snapshot */ + int (*mo_tx_set_snapbuf)(struct umem_instance *umm, umem_off_t snap_buf, size_t size); /** get TX stage */ int (*mo_tx_stage)(void); @@ -1080,6 +1089,15 @@ bool umem_tx_none(struct umem_instance *umm); int umem_tx_errno(int err); +static inline int +umem_tx_set_snapbuf(struct umem_instance *umm, umem_off_t snap_buf, size_t size) +{ + if (umm->umm_ops->mo_tx_set_snapbuf) + return umm->umm_ops->mo_tx_set_snapbuf(umm, snap_buf, size); + else + return 0; +} + static inline int umem_tx_stage(struct umem_instance *umm) { diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index a0cebabc9f8..d22a746e976 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -2101,14 +2101,18 @@ obj_local_rw_internal_wrap(crt_rpc_t *rpc, struct obj_io_context *ioc, struct dt static int obj_local_rw(crt_rpc_t *rpc, struct obj_io_context *ioc, struct dtx_handle *dth) { - struct obj_rw_in *orw = crt_req_get(rpc); - struct dtx_share_peer *dsp; - uint32_t retry = 0; - int rc; + struct obj_rw_in *orw = crt_req_get(rpc); + struct dtx_share_peer *dsp; + uint32_t retry = 0; + uint32_t opc = opc_get(rpc->cr_opc); + int rc; again: rc = obj_local_rw_internal_wrap(rpc, ioc, dth); if (dth != NULL && obj_dtx_need_refresh(dth, rc)) { + if (opc == DAOS_OBJ_RPC_FETCH && DAOS_FAIL_CHECK(DAOS_DTX_NOSPACE_NOREFRESH)) + return -DER_NONEXIST; + if (++retry < 3) { rc = dtx_refresh(dth, ioc->ioc_coc); if (rc == 0) @@ -3184,7 +3188,7 @@ ds_obj_rw_handler(crt_rpc_t *rpc) */ D_FREE(dti_cos); dti_cos_cnt = dtx_cos_get_piggyback(ioc.ioc_coc, &orw->orw_oid, orw->orw_dkey_hash, - DTX_THRESHOLD_COUNT, &dti_cos); + DTX_PIGGYBACK_COUNT, &dti_cos); if (dti_cos_cnt < 0) D_GOTO(out, rc = dti_cos_cnt); @@ -4112,7 +4116,7 @@ ds_obj_punch_handler(crt_rpc_t *rpc) */ D_FREE(dti_cos); dti_cos_cnt = dtx_cos_get_piggyback(ioc.ioc_coc, &opi->opi_oid, opi->opi_dkey_hash, - DTX_THRESHOLD_COUNT, &dti_cos); + DTX_PIGGYBACK_COUNT, &dti_cos); if (dti_cos_cnt < 0) D_GOTO(out, rc = dti_cos_cnt); diff --git a/src/tests/suite/daos_base_tx.c b/src/tests/suite/daos_base_tx.c index 0c6bd80d71c..299c1e3d397 100644 --- a/src/tests/suite/daos_base_tx.c +++ b/src/tests/suite/daos_base_tx.c @@ -1024,6 +1024,203 @@ dtx_23(void **state) ioreq_fini(&req); } +#define TSIZE 8 +#define CONTS 5 + +static void +dtx_24(void **state) +{ + test_arg_t *arg = *state; + const char *dkey = dts_dtx_dkey; + char akey[TSIZE] = {0}; + char wbuf[TSIZE]; + char rbuf[TSIZE]; + daos_obj_id_t oid; + struct ioreq req; + char i; + char j; + + FAULT_INJECTION_REQUIRED(); + + print_message("DTX24: DTX commit under space pressure with single container\n"); + + if (!test_runable(arg, dts_dtx_replica_cnt)) + return; + + oid = daos_test_oid_gen(arg->coh, dts_dtx_class, 0, 0, arg->myrank); + ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); + + print_message("Filling DTX committed table...\n"); + + for (i = 'a'; i <= 'z'; i++) { + snprintf(akey, TSIZE, "akey-%c", i); + for (j = 'A'; j <= 'Z'; j++) { + memset(wbuf, j, TSIZE); + insert_single(dkey, akey, (j - 'A') * TSIZE, wbuf, TSIZE, DAOS_TX_NONE, + &req); + } + } + + /* Wait for batched commit. */ + sleep(DTX_COMMIT_THRESHOLD_AGE + 3); + + /* Simulate DER_NOSPACE when DTX commit via DAOS_DTX_NOSPACE_NOREFRESH. */ + dtx_set_fail_loc(arg, DAOS_DTX_NOSPACE_NOREFRESH | DAOS_FAIL_ALWAYS); + + print_message("Writing more after space pressure...\n"); + + for (i = 'N'; i <= 'Z'; i++) { + snprintf(akey, TSIZE, "akey-%c", i); + for (j = 'a'; j <= 'm'; j++) { + memset(wbuf, j, TSIZE); + insert_single(dkey, akey, (j - 'a') * TSIZE, wbuf, TSIZE, DAOS_TX_NONE, + &req); + } + } + + /* Wait for batched commit. */ + sleep(DTX_COMMIT_THRESHOLD_AGE + 3); + + print_message("Verifying all written data...\n"); + + /* + * DAOS_DTX_NOSPACE_NOREFRESH will prevent DTX refresh. If former batched commit failed + * to commit some DTX because of simulated space exhaustion, then related lookup will + * return IO failure and the fetch result will be different from former write one. + */ + for (i = 'N'; i <= 'Z'; i++) { + snprintf(akey, TSIZE, "akey-%c", i); + for (j = 'a'; j <= 'm'; j++) { + memset(wbuf, j, TSIZE); + lookup_single(dkey, akey, (j - 'a') * TSIZE, rbuf, TSIZE, DAOS_TX_NONE, + &req); + assert_memory_equal(wbuf, rbuf, TSIZE); + } + } + + dtx_set_fail_loc(arg, 0); + ioreq_fini(&req); +} + +static void +dtx_25(void **state) +{ + test_arg_t *arg = *state; + struct test_cont conts[CONTS] = {0}; + daos_obj_id_t oids[CONTS] = {0}; + struct ioreq reqs[CONTS] = {0}; + char dkey[TSIZE] = {0}; + char akey[TSIZE] = {0}; + char wbuf[TSIZE]; + char rbuf[TSIZE]; + daos_prop_t *redun_prop; + int rc; + int m; + char n; + char i; + char j; + + FAULT_INJECTION_REQUIRED(); + + print_message("DTX25: DTX commit under space pressure with multiple containers\n"); + + if (!test_runable(arg, 3)) + return; + + redun_prop = daos_prop_alloc(1); + assert_non_null(redun_prop); + + redun_prop->dpp_entries[0].dpe_type = DAOS_PROP_CO_REDUN_LVL; + redun_prop->dpp_entries[0].dpe_val = DAOS_PROP_CO_REDUN_RANK; + + for (m = 0; m < CONTS; m++) { + rc = daos_cont_create(arg->pool.poh, &conts[m].uuid, redun_prop, NULL); + assert_rc_equal(rc, 0); + + uuid_unparse_lower(conts[m].uuid, conts[m].label); + rc = daos_cont_open(arg->pool.poh, conts[m].label, DAOS_COO_RW, &conts[m].coh, NULL, + NULL); + assert_rc_equal(rc, 0); + + oids[m] = daos_test_oid_gen(conts[m].coh, OC_RP_3GX, 0, 0, arg->myrank); + ioreq_init(&reqs[m], conts[m].coh, oids[m], DAOS_IOD_ARRAY, arg); + + print_message("Filling DTX committed table for the container " DF_UUID "\n", + DP_UUID(conts[m].uuid)); + + for (n = '0'; n < '8'; n++) { + snprintf(dkey, TSIZE, "dkey-%c", n); + for (i = 'a'; i <= 'm'; i++) { + snprintf(akey, TSIZE, "akey-%c", i); + for (j = 'A'; j <= 'Z'; j++) { + memset(wbuf, j, TSIZE); + insert_single(dkey, akey, (j - 'A') * TSIZE, wbuf, TSIZE, + DAOS_TX_NONE, &reqs[m]); + } + } + } + } + + /* Wait for batched commit. */ + sleep(DTX_COMMIT_THRESHOLD_AGE + 3); + + for (m = 0; m < CONTS; m++) { + ioreq_fini(&reqs[m]); + oids[m] = daos_test_oid_gen(conts[m].coh, OC_RP_3G1, 0, 0, arg->myrank); + ioreq_init(&reqs[m], conts[m].coh, oids[m], DAOS_IOD_ARRAY, arg); + } + + /* Simulate DER_NOSPACE when DTX commit via DAOS_DTX_NOSPACE_NOREFRESH. */ + dtx_set_fail_loc(arg, DAOS_DTX_NOSPACE_NOREFRESH | DAOS_FAIL_ALWAYS); + + for (m = 0; m < CONTS; m++) { + print_message("Writing more to cont " DF_UUID " after space pressure...\n", + DP_UUID(conts[m].uuid)); + + for (i = 'N'; i <= 'Z'; i++) { + snprintf(akey, TSIZE, "akey-%c", i); + for (j = 'a'; j <= 'h'; j++) { + memset(wbuf, j, TSIZE); + insert_single(dkey, akey, (j - 'a') * TSIZE, wbuf, TSIZE, + DAOS_TX_NONE, &reqs[m]); + } + } + } + + /* Wait for batched commit. */ + sleep(DTX_COMMIT_THRESHOLD_AGE + 3); + + print_message("Verifying all written data...\n"); + + /* + * DAOS_DTX_NOSPACE_NOREFRESH will prevent DTX refresh. If former batched commit failed + * to commit some DTX because of simulated space exhaustion, then related lookup will + * return IO failure and the fetch result will be different from former write one. + */ + for (m = 0; m < CONTS; m++) { + for (i = 'N'; i <= 'Z'; i++) { + snprintf(akey, TSIZE, "akey-%c", i); + for (j = 'a'; j <= 'h'; j++) { + memset(wbuf, j, TSIZE); + lookup_single(dkey, akey, (j - 'a') * TSIZE, rbuf, TSIZE, + DAOS_TX_NONE, &reqs[m]); + assert_memory_equal(wbuf, rbuf, TSIZE); + } + } + } + + daos_prop_free(redun_prop); + dtx_set_fail_loc(arg, 0); + + for (m = 0; m < CONTS; m++) { + ioreq_fini(&reqs[m]); + if (daos_handle_is_valid(conts[m].coh)) + daos_cont_close(conts[m].coh, NULL); + if (daos_is_valid_uuid_string(conts[m].label, UUID_SST_NONE)) + daos_cont_destroy(arg->pool.poh, conts[m].label, 0, NULL); + } +} + static int dtx_base_rf0_setup(void **state) { @@ -1092,6 +1289,10 @@ static const struct CMUnitTest dtx_tests[] = { dtx_22, NULL, test_case_teardown}, {"DTX23: Resend with lost reply from non-leader", dtx_23, NULL, test_case_teardown}, + {"DTX24: DTX under space pressure with single container", + dtx_24, NULL, test_case_teardown}, + {"DTX25: DTX under space pressure with multiple containers", + dtx_25, NULL, test_case_teardown}, }; /* clang-format on */ diff --git a/src/vos/tests/evt_ctl.c b/src/vos/tests/evt_ctl.c index 6fc061aba6f..14e27866f65 100644 --- a/src/vos/tests/evt_ctl.c +++ b/src/vos/tests/evt_ctl.c @@ -1,6 +1,7 @@ /** * (C) Copyright 2017-2023 Intel Corporation. * (C) Copyright 2026 Google LLC + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -76,8 +77,7 @@ static daos_handle_t ts_toh; #define D_1M_SIZE (1024 * 1024) #define D_256M_SIZE (256 * 1024 * 1024) -#define POOL_NAME "/mnt/daos/evtree-utest" -#define POOL_SIZE ((1024 * 1024 * 1024ULL)) +#define POOL_NAME "/mnt/daos/evtree-utest" struct test_arg { struct utest_context *ta_utx; diff --git a/src/vos/vos_container.c b/src/vos/vos_container.c index 8c63f7c0f09..388a065d605 100644 --- a/src/vos/vos_container.c +++ b/src/vos/vos_container.c @@ -240,9 +240,8 @@ struct d_ulink_ops co_hdl_uh_ops = { .uop_cmp = cont_cmp, }; -int -cont_insert(struct vos_container *cont, struct d_uuid *key, struct d_uuid *pkey, - daos_handle_t *coh) +static int +cont_insert(struct vos_container *cont, struct d_uuid *key, struct d_uuid *pkey, daos_handle_t *coh) { int rc = 0; @@ -497,6 +496,8 @@ vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) goto exit; } + d_list_add_tail(&cont->vc_pool_link, &pool->vp_cont_list); + cont->vc_open_count = 1; D_DEBUG(DB_TRACE, "Inert cont "DF_UUID" into hash table.\n", DP_UUID(cont->vc_id)); @@ -527,8 +528,10 @@ vos_cont_close(daos_handle_t coh) DP_UUID(cont->vc_id), cont->vc_open_count); cont->vc_open_count--; - if (cont->vc_open_count == 0) + if (cont->vc_open_count == 0) { + d_list_del_init(&cont->vc_pool_link); vos_obj_cache_evict(cont); + } D_DEBUG(DB_TRACE, "Close cont "DF_UUID", open count: %d\n", DP_UUID(cont->vc_id), cont->vc_open_count); diff --git a/src/vos/vos_dtx.c b/src/vos/vos_dtx.c index 3789bb11451..cd3e599f0f1 100644 --- a/src/vos/vos_dtx.c +++ b/src/vos/vos_dtx.c @@ -428,6 +428,30 @@ static btr_ops_t dtx_committed_btr_ops = { .to_rec_update = dtx_cmt_ent_update, }; +static inline int +vos_dtx_add_ptr(struct vos_pool *pool, void *ptr, size_t size) +{ + struct umem_instance *umm = &pool->vp_umm; + int rc; + + rc = umem_tx_xadd_ptr(umm, ptr, size, UMEM_FLAG_NO_ABORT); +#ifdef DAOS_PMEM_BUILD + if (unlikely(rc == -DER_NOSPACE)) { + struct vos_pool_ext_df *ext_df = umem_off2ptr(umm, pool->vp_pool_df->pd_ext); + + if (ext_df != NULL && !UMOFF_IS_NULL(ext_df->ped_emerg_buf)) { + rc = + umem_tx_set_snapbuf(umm, ext_df->ped_emerg_buf, VOS_SNAPBUF_EMERG_SIZE); + if (rc == 0) + rc = umem_tx_add_ptr(umm, ptr, size); + else + rc = -DER_NOSPACE; + } + } +#endif + return rc; +} + int vos_dtx_table_register(void) { @@ -448,18 +472,19 @@ vos_dtx_table_register(void) } int -vos_dtx_table_destroy(struct umem_instance *umm, struct vos_cont_df *cont_df) +vos_dtx_table_destroy(struct vos_pool *pool, struct vos_cont_df *cont_df) { - struct vos_dtx_blob_df *dbd; - struct vos_dtx_act_ent_df *dae_df; - umem_off_t dbd_off; - int i; - int rc; + struct umem_instance *umm = &pool->vp_umm; + struct vos_dtx_blob_df *dbd; + struct vos_dtx_act_ent_df *dae_df; + umem_off_t dbd_off; + int i; + int rc; /* cd_dtx_committed_tail is next to cd_dtx_committed_head */ - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_committed_head, + rc = vos_dtx_add_ptr(pool, &cont_df->cd_dtx_committed_head, sizeof(cont_df->cd_dtx_committed_head) + - sizeof(cont_df->cd_dtx_committed_tail)); + sizeof(cont_df->cd_dtx_committed_tail)); if (rc != 0) return rc; @@ -479,9 +504,9 @@ vos_dtx_table_destroy(struct umem_instance *umm, struct vos_cont_df *cont_df) cont_df->cd_dtx_committed_tail = UMOFF_NULL; /* cd_dtx_active_tail is next to cd_dtx_active_head */ - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_active_head, + rc = vos_dtx_add_ptr(pool, &cont_df->cd_dtx_active_head, sizeof(cont_df->cd_dtx_active_head) + - sizeof(cont_df->cd_dtx_active_tail)); + sizeof(cont_df->cd_dtx_active_tail)); if (rc != 0) return rc; @@ -579,7 +604,7 @@ do_dtx_rec_release(struct umem_instance *umm, struct vos_container *cont, } if (abort) { if (DAE_INDEX(dae) != DTX_INDEX_INVAL) { - rc = umem_tx_add_ptr(umm, &svt->ir_dtx, + rc = vos_dtx_add_ptr(cont->vc_pool, &svt->ir_dtx, sizeof(svt->ir_dtx)); if (rc != 0) return rc; @@ -587,8 +612,7 @@ do_dtx_rec_release(struct umem_instance *umm, struct vos_container *cont, dtx_set_aborted(&svt->ir_dtx); } else { - rc = umem_tx_add_ptr(umm, &svt->ir_dtx, - sizeof(svt->ir_dtx)); + rc = vos_dtx_add_ptr(cont->vc_pool, &svt->ir_dtx, sizeof(svt->ir_dtx)); if (rc != 0) return rc; @@ -608,7 +632,7 @@ do_dtx_rec_release(struct umem_instance *umm, struct vos_container *cont, if (abort) { if (DAE_INDEX(dae) != DTX_INDEX_INVAL) { - rc = umem_tx_add_ptr(umm, &evt->dc_dtx, + rc = vos_dtx_add_ptr(cont->vc_pool, &evt->dc_dtx, sizeof(evt->dc_dtx)); if (rc != 0) return rc; @@ -616,8 +640,7 @@ do_dtx_rec_release(struct umem_instance *umm, struct vos_container *cont, dtx_set_aborted(&evt->dc_dtx); } else { - rc = umem_tx_add_ptr(umm, &evt->dc_dtx, - sizeof(evt->dc_dtx)); + rc = vos_dtx_add_ptr(cont->vc_pool, &evt->dc_dtx, sizeof(evt->dc_dtx)); if (rc != 0) return rc; @@ -716,7 +739,8 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab return rc; if (!invalid && keep_act) { - rc = umem_tx_add_ptr(umm, &dae_df->dae_rec_off, sizeof(dae_df->dae_rec_off)); + rc = vos_dtx_add_ptr(cont->vc_pool, &dae_df->dae_rec_off, + sizeof(dae_df->dae_rec_off)); if (rc != 0) return rc; dae_df->dae_rec_off = UMOFF_NULL; @@ -726,7 +750,7 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab if (!invalid && keep_act) { /* When re-commit partial committed DTX, the count can be zero. */ if (dae_df->dae_rec_cnt > 0) { - rc = umem_tx_add_ptr(umm, &dae_df->dae_rec_cnt, + rc = vos_dtx_add_ptr(cont->vc_pool, &dae_df->dae_rec_cnt, sizeof(dae_df->dae_rec_cnt)); if (rc != 0) return rc; @@ -741,7 +765,7 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab if (DAE_FLAGS(dae) & DTE_PARTIAL_COMMITTED) return 0; - rc = umem_tx_add_ptr(umm, &dae_df->dae_flags, sizeof(dae_df->dae_flags)); + rc = vos_dtx_add_ptr(cont->vc_pool, &dae_df->dae_flags, sizeof(dae_df->dae_flags)); if (rc != 0) return rc; @@ -761,11 +785,11 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab } if (dbd->dbd_count > 1 || dbd->dbd_index < dbd->dbd_cap) { - rc = umem_tx_add_ptr(umm, &dae_df->dae_flags, sizeof(dae_df->dae_flags)); + rc = vos_dtx_add_ptr(cont->vc_pool, &dae_df->dae_flags, sizeof(dae_df->dae_flags)); if (rc != 0) return rc; - rc = umem_tx_add_ptr(umm, &dbd->dbd_count, sizeof(dbd->dbd_count)); + rc = vos_dtx_add_ptr(cont->vc_pool, &dbd->dbd_count, sizeof(dbd->dbd_count)); if (rc != 0) return rc; @@ -780,8 +804,7 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab dbd_off = umem_ptr2off(umm, dbd); tmp = umem_off2ptr(umm, dbd->dbd_prev); if (tmp != NULL) { - rc = umem_tx_add_ptr(umm, &tmp->dbd_next, - sizeof(tmp->dbd_next)); + rc = vos_dtx_add_ptr(cont->vc_pool, &tmp->dbd_next, sizeof(tmp->dbd_next)); if (rc != 0) return rc; @@ -790,8 +813,7 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab tmp = umem_off2ptr(umm, dbd->dbd_next); if (tmp != NULL) { - rc = umem_tx_add_ptr(umm, &tmp->dbd_prev, - sizeof(tmp->dbd_prev)); + rc = vos_dtx_add_ptr(cont->vc_pool, &tmp->dbd_prev, sizeof(tmp->dbd_prev)); if (rc != 0) return rc; @@ -799,8 +821,8 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab } if (cont_df->cd_dtx_active_head == dbd_off) { - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_active_head, - sizeof(cont_df->cd_dtx_active_head)); + rc = vos_dtx_add_ptr(cont->vc_pool, &cont_df->cd_dtx_active_head, + sizeof(cont_df->cd_dtx_active_head)); if (rc != 0) return rc; @@ -808,8 +830,8 @@ dtx_rec_release(struct vos_container *cont, struct vos_dtx_act_ent *dae, bool ab } if (cont_df->cd_dtx_active_tail == dbd_off) { - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_active_tail, - sizeof(cont_df->cd_dtx_active_tail)); + rc = vos_dtx_add_ptr(cont->vc_pool, &cont_df->cd_dtx_active_tail, + sizeof(cont_df->cd_dtx_active_tail)); if (rc != 0) return rc; @@ -1026,6 +1048,299 @@ vos_dtx_extend_act_table(struct vos_container *cont) return rc; } +static inline bool +vos_dtx_cmt_victim_candidate(struct vos_container *cont, uint64_t *time_l, uint64_t *time_h) +{ + struct vos_cont_df *cont_df = cont->vc_cont_df; + struct vos_dtx_blob_df *dbd; + + /* + * Keep at least one blob for each container committed DTX table. + * For cases of "cd_dtx_committed_head == cd_dtx_committed_tail", + * either the committed DTX table is empty, or only has one blob. + */ + if (cont_df->cd_dtx_committed_head == cont_df->cd_dtx_committed_tail) + return false; + + dbd = umem_off2ptr(vos_cont2umm(cont), cont_df->cd_dtx_committed_head); + D_ASSERT(dbd != NULL); + + if ((*time_h > dbd->dbd_committed_data[dbd->dbd_count - 1].dce_cmt_time) || + (*time_h == dbd->dbd_committed_data[dbd->dbd_count - 1].dce_cmt_time && + *time_l > dbd->dbd_committed_data[0].dce_cmt_time)) { + *time_l = dbd->dbd_committed_data[0].dce_cmt_time; + *time_h = dbd->dbd_committed_data[dbd->dbd_count - 1].dce_cmt_time; + return true; + } + + return false; +} + +/* + * Space is almost exhausted. Under such case, we must reclaim space to make current + * DTX commit to be proceed; otherwise, uncommitted DTX may block VOS aggregation as + * to prevent further space release. The most direct approach is to reclaim some old + * blob from current container's committed DTX table. It may be unfair because other + * containers could hold older committed DTX entries. However, it maybe not worth to + * scan all pools/containers on the target to find the globally oldest committed DTX + * blob under space pressure. So we make some tradeoff: firstly try to choose victim + * from the neighbors of current container, including prov one, next one and itself + * on the containers list belong to the same pool. If cannot find sitable candidate, + * choose the (more) next one that has more than one committed DTX blob. DAOS-18690. + */ +static struct vos_container * +vos_dtx_choose_victim(struct vos_container *cont) +{ + struct vos_pool *pool = cont->vc_pool; + struct vos_container *victim = NULL; + struct vos_container *prev = NULL; + struct vos_container *next = NULL; + uint64_t time_l = (uint64_t)(-1); + uint64_t time_h = (uint64_t)(-1); + + D_ASSERT(!d_list_empty(&pool->vp_cont_list)); + + if (vos_dtx_cmt_victim_candidate(cont, &time_l, &time_h)) + victim = cont; + + if (cont->vc_pool_link.prev == &pool->vp_cont_list) + prev = d_list_entry(pool->vp_cont_list.prev, struct vos_container, vc_pool_link); + else + prev = d_list_entry(cont->vc_pool_link.prev, struct vos_container, vc_pool_link); + + if (prev == cont) + goto out; + + if (vos_dtx_cmt_victim_candidate(prev, &time_l, &time_h)) + victim = prev; + + if (cont->vc_pool_link.next == &pool->vp_cont_list) + next = d_list_entry(pool->vp_cont_list.next, struct vos_container, vc_pool_link); + else + next = d_list_entry(cont->vc_pool_link.next, struct vos_container, vc_pool_link); + + if (next == prev) + goto out; + + if (vos_dtx_cmt_victim_candidate(next, &time_l, &time_h)) + victim = next; + +out: + if (unlikely(victim == NULL && next != NULL)) { + do { + if (next->vc_pool_link.next == &pool->vp_cont_list) + next = d_list_entry(pool->vp_cont_list.next, struct vos_container, + vc_pool_link); + else + next = d_list_entry(next->vc_pool_link.next, struct vos_container, + vc_pool_link); + if (next == prev || next == cont) + break; + + if (vos_dtx_cmt_victim_candidate(next, &time_l, &time_h)) + victim = next; + } while (victim == NULL); + } + + return victim; +} + +static int +vos_dtx_reuse_cmt_blob(struct vos_container *cont) +{ + struct vos_pool *pool = cont->vc_pool; + struct umem_instance *umm = vos_pool2umm(pool); + struct vos_container *vcm_cont; + struct vos_cont_df *cur_cont_df; + struct vos_cont_df *vcm_cont_df; + struct vos_dtx_blob_df *vcm_dbd; + struct vos_dtx_blob_df *new_head; + struct vos_dtx_blob_df *cur_tail; + struct vos_dtx_cmt_ent_df *dce_df; + d_iov_t kiov; + umem_off_t vcm_dbd_off; + umem_off_t new_head_off; + umem_off_t cur_tail_off; + daos_epoch_t epoch = 0; + uint32_t count = 0; + int rc; + int i; + + vcm_cont = vos_dtx_choose_victim(cont); + if (unlikely(vcm_cont == NULL)) { + D_ERROR("No enough space to extend commit DTX table for cont " DF_UUID "\n", + DP_UUID(cont->vc_id)); + return -DER_NOSPACE; + } + + cur_cont_df = cont->vc_cont_df; + vcm_cont_df = vcm_cont->vc_cont_df; + vcm_dbd_off = vcm_cont_df->cd_dtx_committed_head; + vcm_dbd = umem_off2ptr(umm, vcm_dbd_off); + D_ASSERT(vcm_dbd != NULL); + + /* P1: discard related DTX entries for the victim DTX blob. */ + for (i = 0; i < vcm_dbd->dbd_count; i++) { + dce_df = &vcm_dbd->dbd_committed_data[i]; + d_iov_set(&kiov, &dce_df->dce_xid, sizeof(dce_df->dce_xid)); + rc = dbtree_delete(vcm_cont->vc_dtx_committed_hdl, BTR_PROBE_EQ, &kiov, NULL); + if (rc == 0) + count++; + if (rc == -DER_NONEXIST) + rc = 0; + if (unlikely(rc != 0)) { + D_ERROR("Failed to discard DTX entry " DF_DTI " when purge committed DTX " + "blob on victim " DF_UUID " with dbd_off " UMOFF_PF ": " DF_RC "\n", + DP_DTI(&dce_df->dce_xid), DP_UUID(vcm_cont->vc_id), + UMOFF_P(vcm_dbd_off), DP_RC(rc)); + goto out; + } + + if (epoch < dce_df->dce_epoch) + epoch = dce_df->dce_epoch; + } + + /* P2: prepare to delete victim blob from victim container committed DTX table (head). */ + new_head_off = vcm_dbd->dbd_next; + new_head = umem_off2ptr(umm, new_head_off); + D_ASSERT(new_head != NULL); + + rc = vos_dtx_add_ptr(pool, &vcm_cont_df->cd_dtx_committed_head, + sizeof(vcm_cont_df->cd_dtx_committed_head)); + if (rc != 0) + goto out; + + rc = vos_dtx_add_ptr(pool, &new_head->dbd_prev, sizeof(new_head->dbd_prev)); + if (rc != 0) + goto out; + + /* P3: prepare to insert victim blob to current container committed DTX table (tail). */ + cur_tail_off = cur_cont_df->cd_dtx_committed_tail; + cur_tail = umem_off2ptr(umm, cur_tail_off); + + if (UMOFF_IS_NULL(cur_cont_df->cd_dtx_committed_head)) + /* cd_dtx_committed_tail is next to cd_dtx_committed_head */ + rc = vos_dtx_add_ptr(pool, &cur_cont_df->cd_dtx_committed_head, + sizeof(cur_cont_df->cd_dtx_committed_head) + + sizeof(cur_cont_df->cd_dtx_committed_tail)); + else + rc = vos_dtx_add_ptr(pool, &cur_cont_df->cd_dtx_committed_tail, + sizeof(cur_cont_df->cd_dtx_committed_tail)); + if (rc != 0) + goto out; + + if (cur_tail != NULL) { + rc = vos_dtx_add_ptr(pool, &cur_tail->dbd_next, sizeof(cur_tail->dbd_next)); + if (rc != 0) + goto out; + } + + /* dbd_next is next to dbd_prev */ + rc = vos_dtx_add_ptr(pool, &vcm_dbd->dbd_prev, + sizeof(vcm_dbd->dbd_prev) + sizeof(vcm_dbd->dbd_next)); + if (rc != 0) + goto out; + + /* P4: prepare to reset victim DTX blob conuter, dbd_index is next to dbd_count. */ + rc = vos_dtx_add_ptr(pool, &vcm_dbd->dbd_count, + sizeof(vcm_dbd->dbd_count) + sizeof(vcm_dbd->dbd_index)); + if (rc != 0) + goto out; + + /* P5: delete victim blob from victim container committed DTX table (head). */ + vcm_cont_df->cd_dtx_committed_head = new_head_off; + new_head->dbd_prev = UMOFF_NULL; + + /* P6: insert victim blob to current container committed DTX table (tail). */ + cur_cont_df->cd_dtx_committed_tail = vcm_dbd_off; + vcm_dbd->dbd_prev = cur_tail_off; + vcm_dbd->dbd_next = UMOFF_NULL; + if (UMOFF_IS_NULL(cur_cont_df->cd_dtx_committed_head)) + cur_cont_df->cd_dtx_committed_head = vcm_dbd_off; + if (cur_tail != NULL) + cur_tail->dbd_next = vcm_dbd_off; + + /* P7: reset victim DTX blob counter. */ + vcm_dbd->dbd_count = 0; + vcm_dbd->dbd_index = 0; + + /* P8: refresh DTX related metrics. */ + if (count > 0) { + D_ASSERTF(vcm_cont->vc_dtx_committed_count >= count, + "Unexpected committed DTX entries count for " DF_UUID ": %u vs %u\n", + DP_UUID(vcm_cont->vc_id), vcm_cont->vc_dtx_committed_count, count); + + vcm_cont->vc_dtx_committed_count -= count; + pool->vp_dtx_committed_count -= count; + } + +out: + DL_CDEBUG(rc == 0, DLOG_WARN, DLOG_ERR, rc, + "Reused committed DTX blob %p (" UMOFF_PF ") from cont " DF_UUID + " to cont " DF_UUID " under space pressure in pool " DF_UUID, + vcm_dbd, UMOFF_P(vcm_dbd_off), DP_UUID(vcm_cont->vc_id), DP_UUID(cont->vc_id), + DP_UUID(cont->vc_pool->vp_id)); + return rc; +} + +static int +vos_dtx_extend_cmt_table(struct vos_container *cont) +{ + struct umem_instance *umm = vos_cont2umm(cont); + struct vos_cont_df *cont_df = cont->vc_cont_df; + struct vos_dtx_blob_df *dbd = NULL; + struct vos_dtx_blob_df *tail; + umem_off_t dbd_off = UMOFF_NULL; + int rc; + + if (!DAOS_FAIL_CHECK(DAOS_DTX_NOSPACE_NOREFRESH)) + dbd_off = umem_alloc_verb(umm, UMEM_FLAG_ZERO | UMEM_FLAG_NO_ABORT, + DTX_CMT_BLOB_SIZE, UMEM_DEFAULT_MBKT_ID); + + if (UMOFF_IS_NULL(dbd_off)) + return vos_dtx_reuse_cmt_blob(cont); + + dbd = umem_off2ptr(umm, dbd_off); + tail = umem_off2ptr(umm, cont_df->cd_dtx_committed_tail); + if (tail == NULL) { + D_ASSERT(UMOFF_IS_NULL(cont_df->cd_dtx_committed_head)); + + /* cd_dtx_committed_tail is next to cd_dtx_committed_head */ + rc = vos_dtx_add_ptr(cont->vc_pool, &cont_df->cd_dtx_committed_head, + sizeof(cont_df->cd_dtx_committed_head) + + sizeof(cont_df->cd_dtx_committed_tail)); + if (rc != 0) + goto out; + + cont_df->cd_dtx_committed_head = dbd_off; + } else { + rc = vos_dtx_add_ptr(cont->vc_pool, &tail->dbd_next, sizeof(tail->dbd_next)); + if (rc != 0) + goto out; + + rc = vos_dtx_add_ptr(cont->vc_pool, &cont_df->cd_dtx_committed_tail, + sizeof(cont_df->cd_dtx_committed_tail)); + if (rc != 0) + goto out; + + tail->dbd_next = dbd_off; + dbd->dbd_prev = cont_df->cd_dtx_committed_tail; + } + cont_df->cd_dtx_committed_tail = dbd_off; + + dbd->dbd_magic = DTX_CMT_BLOB_MAGIC; + dbd->dbd_cap = (DTX_CMT_BLOB_SIZE - sizeof(struct vos_dtx_blob_df)) / + sizeof(struct vos_dtx_cmt_ent_df); + dbd->dbd_count = 0; + dbd->dbd_index = 0; + +out: + DL_CDEBUG(rc == 0, DB_IO, DLOG_ERR, rc, + "Allocated DTX committed blob %p (" UMOFF_PF ") for " DF_UUID "/" DF_UUID, dbd, + UMOFF_P(dbd_off), DP_UUID(cont->vc_pool->vp_id), DP_UUID(cont->vc_id)); + return rc; +} + static int vos_dtx_alloc(struct umem_instance *umm, struct dtx_handle *dth) { @@ -2236,19 +2551,17 @@ vos_dtx_commit_internal(struct vos_container *cont, struct dtx_id dtis[], int count, daos_epoch_t epoch, bool keep_act, bool rm_cos[], struct vos_dtx_act_ent **daes, struct vos_dtx_cmt_ent **dces) { - struct vos_cont_df *cont_df = cont->vc_cont_df; - struct umem_instance *umm = vos_cont2umm(cont); - struct vos_dtx_blob_df *dbd; - struct vos_dtx_blob_df *dbd_prev; - umem_off_t dbd_off; - uint64_t cmt_time = daos_wallclock_secs(); - int committed = 0; - int rc = 0; - int p = 0; - int i = 0; - int j; - int k; - bool allocated = false; + struct vos_dtx_blob_df *dbd; + struct vos_cont_df *cont_df = cont->vc_cont_df; + struct umem_instance *umm = vos_cont2umm(cont); + uint64_t cmt_time = daos_wallclock_secs(); + int committed = 0; + int rc = 0; + int p = 0; + int i = 0; + int j; + int k; + bool allocated = false; dbd = umem_off2ptr(umm, cont_df->cd_dtx_committed_tail); if (dbd == NULL) @@ -2285,12 +2598,14 @@ vos_dtx_commit_internal(struct vos_container *cont, struct dtx_id dtis[], if (!allocated) { rc = umem_tx_xadd_ptr(umm, &dbd->dbd_committed_data[dbd->dbd_count], sizeof(struct vos_dtx_cmt_ent_df) * - (j - dbd->dbd_count), UMEM_XADD_NO_SNAPSHOT); + (j - dbd->dbd_count), + UMEM_XADD_NO_SNAPSHOT | UMEM_FLAG_NO_ABORT); if (rc != 0) goto out; /* Only need to add range for the first partial blob. */ - rc = umem_tx_add_ptr(umm, &dbd->dbd_count, sizeof(dbd->dbd_count)); + rc = + vos_dtx_add_ptr(cont->vc_pool, &dbd->dbd_count, sizeof(dbd->dbd_count)); if (rc != 0) goto out; } @@ -2310,52 +2625,17 @@ vos_dtx_commit_internal(struct vos_container *cont, struct dtx_id dtis[], goto out; new_blob: - dbd_prev = dbd; - /* Need new @dbd */ - dbd_off = umem_zalloc(umm, DTX_CMT_BLOB_SIZE); - if (UMOFF_IS_NULL(dbd_off)) { - D_ERROR("No space to store committed DTX %d "DF_DTI"\n", - count, DP_DTI(&dtis[i])); - D_GOTO(out, rc = -DER_NOSPACE); - } - - dbd = umem_off2ptr(umm, dbd_off); - dbd->dbd_magic = DTX_CMT_BLOB_MAGIC; - dbd->dbd_cap = (DTX_CMT_BLOB_SIZE - sizeof(struct vos_dtx_blob_df)) / - sizeof(struct vos_dtx_cmt_ent_df); - dbd->dbd_prev = umem_ptr2off(umm, dbd_prev); - - if (dbd_prev == NULL) { - D_ASSERT(UMOFF_IS_NULL(cont_df->cd_dtx_committed_head)); - D_ASSERT(UMOFF_IS_NULL(cont_df->cd_dtx_committed_tail)); - - /* cd_dtx_committed_tail is next to cd_dtx_committed_head */ - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_committed_head, - sizeof(cont_df->cd_dtx_committed_head) + - sizeof(cont_df->cd_dtx_committed_tail)); - if (rc != 0) - goto out; + if (unlikely(allocated)) + D_GOTO(out, rc = -DER_OVERFLOW); - cont_df->cd_dtx_committed_head = dbd_off; - } else { - rc = umem_tx_add_ptr(umm, &dbd_prev->dbd_next, - sizeof(dbd_prev->dbd_next)); - if (rc != 0) - goto out; - - dbd_prev->dbd_next = dbd_off; + rc = vos_dtx_extend_cmt_table(cont); + if (rc != 0) + goto out; - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_committed_tail, - sizeof(cont_df->cd_dtx_committed_tail)); - if (rc != 0) - goto out; - } + allocated = true; - D_DEBUG(DB_IO, "Allocated DTX committed blob %p ("UMOFF_PF") for cont "DF_UUID"\n", - dbd, UMOFF_P(dbd_off), DP_UUID(cont->vc_id)); + dbd = umem_off2ptr(umm, cont_df->cd_dtx_committed_tail); - cont_df->cd_dtx_committed_tail = dbd_off; - allocated = true; goto again; out: @@ -3127,7 +3407,7 @@ dtx_blob_aggregate(struct umem_instance *umm, struct vos_tls *tls, struct vos_co } if (epoch != cont_df->cd_newest_aggregated) { - rc = umem_tx_add_ptr(umm, &cont_df->cd_newest_aggregated, + rc = vos_dtx_add_ptr(cont->vc_pool, &cont_df->cd_newest_aggregated, sizeof(cont_df->cd_newest_aggregated)); if (unlikely(rc != 0)) { D_ERROR("Failed to refresh epoch for DTX aggregation " UMOFF_PF ": " DF_RC @@ -3141,7 +3421,7 @@ dtx_blob_aggregate(struct umem_instance *umm, struct vos_tls *tls, struct vos_co if (dbd->dbd_count - dtx_aggr_count > 0) { size_t buf_len; - rc = umem_tx_add_ptr(umm, &dbd->dbd_committed_data[0], + rc = vos_dtx_add_ptr(cont->vc_pool, &dbd->dbd_committed_data[0], sizeof(dbd->dbd_committed_data[0]) * dbd->dbd_count); if (unlikely(rc != 0)) { D_ERROR("Failed update committed DTX blob " UMOFF_PF ": " DF_RC "\n", @@ -3152,7 +3432,7 @@ dtx_blob_aggregate(struct umem_instance *umm, struct vos_tls *tls, struct vos_co memmove(&dbd->dbd_committed_data[0], &dbd->dbd_committed_data[dtx_aggr_count], buf_len); - rc = umem_tx_add_ptr(umm, &dbd->dbd_count, sizeof(dbd->dbd_count)); + rc = vos_dtx_add_ptr(cont->vc_pool, &dbd->dbd_count, sizeof(dbd->dbd_count)); if (unlikely(rc != 0)) { D_ERROR("Failed update committed DTX count " UMOFF_PF ": " DF_RC "\n", UMOFF_P(dbd_off), DP_RC(rc)); @@ -3169,7 +3449,7 @@ dtx_blob_aggregate(struct umem_instance *umm, struct vos_tls *tls, struct vos_co D_ASSERT(UMOFF_IS_NULL(dbd_prev_off)); D_ASSERT(dbd_off == cont_df->cd_dtx_committed_head); - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_committed_head, + rc = vos_dtx_add_ptr(cont->vc_pool, &cont_df->cd_dtx_committed_head, sizeof(cont_df->cd_dtx_committed_head)); if (unlikely(rc != 0)) { D_ERROR("Failed to update head for DTX aggregation " UMOFF_PF ": " DF_RC @@ -3181,7 +3461,7 @@ dtx_blob_aggregate(struct umem_instance *umm, struct vos_tls *tls, struct vos_co if (dbd_next == NULL) { D_ASSERT(dbd_off == cont_df->cd_dtx_committed_tail); - rc = umem_tx_add_ptr(umm, &cont_df->cd_dtx_committed_tail, + rc = vos_dtx_add_ptr(cont->vc_pool, &cont_df->cd_dtx_committed_tail, sizeof(cont_df->cd_dtx_committed_tail)); if (unlikely(rc != 0)) { D_ERROR("Failed to update tail for DTX aggregation " UMOFF_PF @@ -3191,7 +3471,8 @@ dtx_blob_aggregate(struct umem_instance *umm, struct vos_tls *tls, struct vos_co } cont_df->cd_dtx_committed_tail = dbd_prev_off; } else { - rc = umem_tx_add_ptr(umm, &dbd_next->dbd_prev, sizeof(dbd_next->dbd_prev)); + rc = vos_dtx_add_ptr(cont->vc_pool, &dbd_next->dbd_prev, + sizeof(dbd_next->dbd_prev)); if (unlikely(rc != 0)) { D_ERROR("Failed to update previous DTXs blob for DTX " "aggregation " UMOFF_PF ": " DF_RC "\n", @@ -3325,32 +3606,12 @@ vos_dtx_stat(daos_handle_t coh, struct dtx_stat *stat, uint32_t flags) stat->dtx_newest_aggregated = cont_df->cd_newest_aggregated; if (!UMOFF_IS_NULL(cont_df->cd_dtx_committed_head)) { - struct umem_instance *umm = vos_cont2umm(cont); - struct vos_dtx_blob_df *dbd; - struct vos_dtx_cmt_ent_df *dce; - int i; - - dbd = umem_off2ptr(umm, cont_df->cd_dtx_committed_head); - - for (i = 0; i < dbd->dbd_count; i++) { - dce = &dbd->dbd_committed_data[i]; + struct vos_dtx_blob_df *dbd = + umem_off2ptr(vos_cont2umm(cont), cont_df->cd_dtx_committed_head); - if (!daos_is_zero_dti(&dce->dce_xid) && - dce->dce_cmt_time != 0) { - stat->dtx_first_cmt_blob_time_up = dce->dce_cmt_time; - break; - } - } - - for (i = dbd->dbd_count - 1; i > 0; i--) { - dce = &dbd->dbd_committed_data[i]; - - if (!daos_is_zero_dti(&dce->dce_xid) && - dce->dce_cmt_time != 0) { - stat->dtx_first_cmt_blob_time_lo = dce->dce_cmt_time; - break; - } - } + stat->dtx_first_cmt_blob_time_up = dbd->dbd_committed_data[0].dce_cmt_time; + stat->dtx_first_cmt_blob_time_lo = + dbd->dbd_committed_data[dbd->dbd_count - 1].dce_cmt_time; } } diff --git a/src/vos/vos_gc.c b/src/vos/vos_gc.c index 8e1cb8790d0..2c4e54e11bb 100644 --- a/src/vos/vos_gc.c +++ b/src/vos/vos_gc.c @@ -307,7 +307,7 @@ gc_drain_cont(struct vos_gc *gc, struct vos_pool *pool, daos_handle_t coh, * Then destroy DTX table firstly to avoid dangling DXT records during drain * the container (that may yield). */ - rc = vos_dtx_table_destroy(&pool->vp_umm, cont); + rc = vos_dtx_table_destroy(pool, cont); if (rc != 0) { DL_ERROR(rc, "Failed to destroy DTX table."); return rc; diff --git a/src/vos/vos_internal.h b/src/vos/vos_internal.h index b257c38f824..6832f9a2334 100644 --- a/src/vos/vos_internal.h +++ b/src/vos/vos_internal.h @@ -332,6 +332,10 @@ struct vos_pool { d_list_t vp_gc_link; /** List of open containers with objects in gc pool */ d_list_t vp_gc_cont; + + /** List of open containers */ + d_list_t vp_cont_list; + /** address of durable-format pool in SCM */ struct vos_pool_df *vp_pool_df; /** Dummy data I/O context */ @@ -390,6 +394,10 @@ struct vos_container { d_list_t vc_dtx_unsorted_list; /* The list for the active DTX entries that are re-indexed when open the container. */ d_list_t vc_dtx_reindex_list; + + /* Link into vos_pool::vp_cont_list */ + d_list_t vc_pool_link; + /* The largest epoch difference for re-indexed DTX entries max/min pairs. */ uint64_t vc_dtx_reindex_eph_diff; /* The latest calculated local stable epoch. */ @@ -717,13 +725,13 @@ vos_obj_tab_register(); * DTX table destroy * Called from vos_cont_destroy * - * \param umm [IN] Instance of an unified memory class. + * \param pool [IN] The pool that holds the container. * \param cont_df [IN] Pointer to the on-disk VOS container. * * \return 0 on success and negative on failure. */ int -vos_dtx_table_destroy(struct umem_instance *umm, struct vos_cont_df *cont_df); +vos_dtx_table_destroy(struct vos_pool *pool, struct vos_cont_df *cont_df); /** * Register dbtree class for DTX table. diff --git a/src/vos/vos_pool.c b/src/vos/vos_pool.c index 3d40c89684f..e856a84e004 100644 --- a/src/vos/vos_pool.c +++ b/src/vos/vos_pool.c @@ -1226,6 +1226,7 @@ pool_alloc(uuid_t uuid, struct vos_pool **pool_p) d_uhash_ulink_init(&pool->vp_hlink, &pool_uuid_hops); D_INIT_LIST_HEAD(&pool->vp_gc_link); D_INIT_LIST_HEAD(&pool->vp_gc_cont); + D_INIT_LIST_HEAD(&pool->vp_cont_list); uuid_copy(pool->vp_id, uuid); *pool_p = pool;