From 6c4dd320e72b7db5248f0efb2bb26aa2ccf9b382 Mon Sep 17 00:00:00 2001 From: hgichon Date: Sat, 4 Jul 2026 15:54:17 +0900 Subject: [PATCH] DAOS-18690 vos: fix engine abort in DTX commit blob fallback on ENOSPC When the pool is physically full, vos_dtx_extend_cmt_table() fails to allocate a new commit blob and falls back to vos_dtx_reuse_cmt_blob() inside the same transaction. The failing allocation is a plain (non-NO_ABORT) alloc inside the regular object modification transaction started by vos_tx_begin(), whose failure behavior is left at the default, so the allocator aborts the whole TX on failure. The fallback then calls umem_tx_add_ptr() on the aborted TX, which is a fatal usage error in the allocator: CRIT pmdk tx.c:1403 pmemobj_tx_add_range_direct() called in invalid stage 3 *** Process (daos_engine) received signal 6 (Aborted) *** vos_dtx_commit_internal <- vos_dtx_prepared <- vos_tx_end <- vos_obj_punch <- ds_obj_punch_handler The reuse fallback introduced by #18141 is only exercised in tests via the DAOS_DTX_NOSPACE_NOREFRESH fail-check, which skips the allocation entirely and therefore never reproduces the TX abort of a real ENOSPC. Fix: add a UMEM_FLAG_NO_ABORT alloc flag (1 << 3, distinct from UMEM_XADD_NO_SNAPSHOT), map it to POBJ_XALLOC_NO_ABORT / DAV_XALLOC_NO_ABORT in the pmem/dav/dav_v2 backends, and use it for the commit blob allocation so the transaction survives the failure and the reuse fallback can run as designed. Add a umem regression test (UMEM010): a failed NO_ABORT allocation must leave the TX alive so a subsequent alloc/tx_add_ptr/commit succeeds. Verified on a 400 MB tmpfs-SCM pool: fill the pool to ENOSPC, then run a create/unlink storm. Before the fix the engine aborts within seconds with the backtrace above; after the fix it stays up and the operations fail cleanly with -DER_NOSPACE. Signed-off-by: hgichon --- src/common/mem.c | 6 ++++ src/common/tests/umem_test.c | 53 ++++++++++++++++++++++++++++++++++++ src/include/daos/mem.h | 4 +++ src/vos/vos_dtx.c | 8 +++++- 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/common/mem.c b/src/common/mem.c index f64d744be72..be19df506bd 100644 --- a/src/common/mem.c +++ b/src/common/mem.c @@ -806,6 +806,8 @@ pmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned i pflags |= POBJ_FLAG_ZERO; if (flags & UMEM_FLAG_NO_FLUSH) pflags |= POBJ_FLAG_NO_FLUSH; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= POBJ_XALLOC_NO_ABORT; return umem_id2off(umm, pmemobj_tx_xalloc(size, type_num, pflags)); } @@ -1244,6 +1246,8 @@ bmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned i pflags |= DAV_FLAG_ZERO; if (flags & UMEM_FLAG_NO_FLUSH) pflags |= DAV_FLAG_NO_FLUSH; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= DAV_XALLOC_NO_ABORT; return dav_tx_xalloc(size, type_num, pflags); } @@ -1483,6 +1487,8 @@ bmem_tx_alloc_v2(struct umem_instance *umm, size_t size, uint64_t flags, unsigne pflags |= DAV_FLAG_ZERO; if (flags & UMEM_FLAG_NO_FLUSH) pflags |= DAV_FLAG_NO_FLUSH; + if (flags & UMEM_FLAG_NO_ABORT) + pflags |= DAV_XALLOC_NO_ABORT; if (mbkt_id != 0) pflags |= DAV_EZONE_ID(mbkt_id); return dav_tx_alloc_v2(size, type_num, pflags); diff --git a/src/common/tests/umem_test.c b/src/common/tests/umem_test.c index 6080843f51c..bd25f257eca 100644 --- a/src/common/tests/umem_test.c +++ b/src/common/tests/umem_test.c @@ -360,6 +360,58 @@ test_alloc(void **state) assert_int_equal(rc, 0); } +/* A failed allocation with UMEM_FLAG_NO_ABORT must leave the transaction + * alive so in-tx fallbacks (e.g. vos_dtx_reuse_cmt_blob) keep working; + * without the flag the failure aborts the TX and any further tx_add is a + * fatal usage error in the allocator. + */ +static void +test_alloc_no_abort(void **state) +{ + struct test_arg *arg = *state; + struct umem_instance *umm = utest_utx2umm(arg->ta_utx); + int *value; + umem_off_t umoff = 0; + umem_off_t huge; + int rc; + + rc = utest_tx_begin(arg->ta_utx); + if (rc != 0) + goto done; + + /* Larger than the pool: must fail, but must NOT abort the TX */ + huge = umem_alloc_verb(umm, UMEM_FLAG_ZERO | UMEM_FLAG_NO_ABORT, + POOL_SIZE, UMEM_DEFAULT_MBKT_ID); + if (!UMOFF_IS_NULL(huge)) { + print_message("huge allocation unexpectedly succeeded\n"); + rc = 1; + goto end; + } + + /* TX still alive: alloc + snapshot + write must all work */ + umoff = umem_zalloc(umm, 4); + if (UMOFF_IS_NULL(umoff)) { + print_message("small alloc failed after NO_ABORT failure\n"); + rc = 1; + goto end; + } + + value = umem_off2ptr(umm, umoff); + rc = umem_tx_add_ptr(umm, value, 4); + if (rc != 0) { + print_message("tx_add_ptr failed after NO_ABORT failure\n"); + rc = 1; + goto end; + } + *value = 42; + + rc = umem_free(umm, umoff); +end: + rc = utest_tx_end(arg->ta_utx, rc); +done: + assert_int_equal(rc, 0); +} + static int flush_prep(struct umem_store *store, struct umem_store_iod *iod, daos_handle_t *fh) { @@ -757,6 +809,7 @@ main(int argc, char **argv) {"UMEM002: Test null flags vmem", test_invalid_flags, setup_vmem, teardown_vmem}, {"UMEM003: Test alloc pmem", test_alloc, setup_pmem, teardown_pmem}, {"UMEM004: Test alloc vmem", test_alloc, setup_vmem, teardown_vmem}, + {"UMEM010: Test alloc NO_ABORT pmem", test_alloc_no_abort, setup_pmem, teardown_pmem}, {"UMEM005: Test page cache", test_page_cache, NULL, NULL}, {"UMEM006: Test page cache many pages", test_many_pages, NULL, NULL}, {"UMEM007: Test page cache many writes", test_many_writes, NULL, NULL}, diff --git a/src/include/daos/mem.h b/src/include/daos/mem.h index eb4bdbf3552..6b275afea68 100644 --- a/src/include/daos/mem.h +++ b/src/include/daos/mem.h @@ -681,6 +681,10 @@ struct umem_instance; #define UMEM_FLAG_ZERO (((uint64_t)1) << 0) #define UMEM_FLAG_NO_FLUSH (((uint64_t)1) << 1) #define UMEM_XADD_NO_SNAPSHOT (((uint64_t)1) << 2) +/* On allocation failure, return UMOFF_NULL without aborting the current + * transaction, so the caller can fall back to another strategy in-tx. + */ +#define UMEM_FLAG_NO_ABORT (((uint64_t)1) << 3) /* Macros associated with Memory buckets */ #define UMEM_DEFAULT_MBKT_ID 0 diff --git a/src/vos/vos_dtx.c b/src/vos/vos_dtx.c index a7e830bff7a..636810ee88e 100644 --- a/src/vos/vos_dtx.c +++ b/src/vos/vos_dtx.c @@ -1293,8 +1293,14 @@ vos_dtx_extend_cmt_table(struct vos_container *cont) umem_off_t dbd_off = UMOFF_NULL; int rc; + /* NO_ABORT: on ENOSPC the fallback below (vos_dtx_reuse_cmt_blob) + * keeps working inside the same transaction; a plain alloc failure + * would have aborted the tx and any further tx_add would be a fatal + * usage error in the allocator (engine abort). + */ if (!DAOS_FAIL_CHECK(DAOS_DTX_NOSPACE_NOREFRESH)) - dbd_off = umem_zalloc(umm, DTX_CMT_BLOB_SIZE); + dbd_off = umem_alloc_verb(umm, UMEM_FLAG_ZERO | UMEM_FLAG_NO_ABORT, + DTX_CMT_BLOB_SIZE, UMEM_DEFAULT_MBKT_ID); if (UMOFF_IS_NULL(dbd_off)) return vos_dtx_reuse_cmt_blob(cont);