DAOS-18690 vos: fix engine crash when extend committed DTX table

src/common/mem.c

@@ -806,6 +806,8 @@ pmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned i
 		pflags |= POBJ_FLAG_ZERO;
 	if (flags & UMEM_FLAG_NO_FLUSH)
 		pflags |= POBJ_FLAG_NO_FLUSH;
+	if (flags & UMEM_FLAG_NO_ABORT)
+		pflags |= POBJ_FLAG_TX_NO_ABORT;
 	return umem_id2off(umm, pmemobj_tx_xalloc(size, type_num, pflags));
 }
 
@@ -828,6 +830,8 @@ pmem_tx_xadd(struct umem_instance *umm, umem_off_t umoff, uint64_t offset,
 
 	if (flags & UMEM_XADD_NO_SNAPSHOT)
 		pflags |= POBJ_XADD_NO_SNAPSHOT;
+	if (flags & UMEM_FLAG_NO_ABORT)
+		pflags |= POBJ_FLAG_TX_NO_ABORT;
 
 	rc = pmemobj_tx_xadd_range(umem_off2id(umm, umoff), offset, size,
 				   pflags);
@@ -997,39 +1001,6 @@ pmem_defer_free(struct umem_instance *umm, umem_off_t off, void *act)
 	pmemobj_defer_free(pop, id, (struct pobj_action *)act);
 }
 
-static void
-pmem_tx_set_failure_behavior(enum umem_tx_failure_behavior behavior)
-{
-	switch (behavior) {
-	case TX_FAILURE_ABORT:
-		pmemobj_tx_set_failure_behavior(POBJ_TX_FAILURE_ABORT);
-		break;
-	case TX_FAILURE_RETURN:
-		pmemobj_tx_set_failure_behavior(POBJ_TX_FAILURE_RETURN);
-		break;
-	default:
-		D_ASSERTF(0, "Unknown TX failure behavior %d\n", behavior);
-	}
-}
-
-static int
-pmem_tx_get_failure_behavior(void)
-{
-	enum pobj_tx_failure_behavior behavior;
-
-	behavior = pmemobj_tx_get_failure_behavior();
-
-	switch (behavior) {
-	case POBJ_TX_FAILURE_ABORT:
-		return TX_FAILURE_ABORT;
-	case POBJ_TX_FAILURE_RETURN:
-		return TX_FAILURE_RETURN;
-	default:
-		D_ASSERTF(0, "Unknown TX failure behavior %d\n", behavior);
-		return -DER_INVAL;
-	}
-}
-
 static int
 pmem_tx_set_snapbuf(struct umem_instance *umm, umem_off_t snapbuf, size_t size)
 {
@@ -1179,27 +1150,25 @@ umem_tx_add_cb(struct umem_instance *umm, struct umem_tx_stage_data *txd,
 }
 
 static umem_ops_t pmem_ops = {
-    .mo_tx_free                 = pmem_tx_free,
-    .mo_tx_alloc                = pmem_tx_alloc,
-    .mo_tx_add                  = pmem_tx_add,
-    .mo_tx_xadd                 = pmem_tx_xadd,
-    .mo_tx_add_ptr              = pmem_tx_add_ptr,
-    .mo_tx_abort                = pmem_tx_abort,
-    .mo_tx_begin                = pmem_tx_begin,
-    .mo_tx_commit               = pmem_tx_commit,
-    .mo_tx_set_failure_behavior = pmem_tx_set_failure_behavior,
-    .mo_tx_get_failure_behavior = pmem_tx_get_failure_behavior,
-    .mo_tx_set_snapbuf          = pmem_tx_set_snapbuf,
-    .mo_tx_stage                = pmem_tx_stage,
-    .mo_reserve                 = pmem_reserve,
-    .mo_defer_free              = pmem_defer_free,
-    .mo_cancel                  = pmem_cancel,
-    .mo_tx_publish              = pmem_tx_publish,
-    .mo_atomic_copy             = pmem_atomic_copy,
-    .mo_atomic_alloc            = pmem_atomic_alloc,
-    .mo_atomic_free             = pmem_atomic_free,
-    .mo_atomic_flush            = pmem_atomic_flush,
-    .mo_tx_add_callback         = umem_tx_add_cb,
+    .mo_tx_free         = pmem_tx_free,
+    .mo_tx_alloc        = pmem_tx_alloc,
+    .mo_tx_add          = pmem_tx_add,
+    .mo_tx_xadd         = pmem_tx_xadd,
+    .mo_tx_add_ptr      = pmem_tx_add_ptr,
+    .mo_tx_abort        = pmem_tx_abort,
+    .mo_tx_begin        = pmem_tx_begin,
+    .mo_tx_commit       = pmem_tx_commit,
+    .mo_tx_set_snapbuf  = pmem_tx_set_snapbuf,
+    .mo_tx_stage        = pmem_tx_stage,
+    .mo_reserve         = pmem_reserve,
+    .mo_defer_free      = pmem_defer_free,
+    .mo_cancel          = pmem_cancel,
+    .mo_tx_publish      = pmem_tx_publish,
+    .mo_atomic_copy     = pmem_atomic_copy,
+    .mo_atomic_alloc    = pmem_atomic_alloc,
+    .mo_atomic_free     = pmem_atomic_free,
+    .mo_atomic_flush    = pmem_atomic_flush,
+    .mo_tx_add_callback = umem_tx_add_cb,
 };
 
 /** BMEM operations (depends on dav) */
@@ -1244,6 +1213,8 @@ bmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned i
 		pflags |= DAV_FLAG_ZERO;
 	if (flags & UMEM_FLAG_NO_FLUSH)
 		pflags |= DAV_FLAG_NO_FLUSH;
+	if (flags & UMEM_FLAG_NO_ABORT)
+		pflags |= DAV_FLAG_TX_NO_ABORT;
 	return dav_tx_xalloc(size, type_num, pflags);
 }
 
@@ -1266,6 +1237,8 @@ bmem_tx_xadd(struct umem_instance *umm, umem_off_t umoff, uint64_t offset,
 
 	if (flags & UMEM_XADD_NO_SNAPSHOT)
 		pflags |= DAV_XADD_NO_SNAPSHOT;
+	if (flags & UMEM_FLAG_NO_ABORT)
+		pflags |= DAV_FLAG_TX_NO_ABORT;
 
 	rc = dav_tx_xadd_range(umem_off2offset(umoff), size, pflags);
 	return rc ? umem_tx_errno(rc) : 0;
@@ -1483,6 +1456,8 @@ bmem_tx_alloc_v2(struct umem_instance *umm, size_t size, uint64_t flags, unsigne
 		pflags |= DAV_FLAG_ZERO;
 	if (flags & UMEM_FLAG_NO_FLUSH)
 		pflags |= DAV_FLAG_NO_FLUSH;
+	if (flags & UMEM_FLAG_NO_ABORT)
+		pflags |= DAV_FLAG_TX_NO_ABORT;
 	if (mbkt_id != 0)
 		pflags |= DAV_EZONE_ID(mbkt_id);
 	return dav_tx_alloc_v2(size, type_num, pflags);
@@ -1507,6 +1482,8 @@ bmem_tx_xadd_v2(struct umem_instance *umm, umem_off_t umoff, uint64_t offset,
 
 	if (flags & UMEM_XADD_NO_SNAPSHOT)
 		pflags |= DAV_XADD_NO_SNAPSHOT;
+	if (flags & UMEM_FLAG_NO_ABORT)
+		pflags |= DAV_FLAG_TX_NO_ABORT;
 
 	rc = dav_tx_xadd_range_v2(umem_off2offset(umoff), size, pflags);
 	return rc ? umem_tx_errno(rc) : 0;

src/common/tests/btree.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2016-2022 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -69,8 +69,7 @@ struct btr_test_state {
 };
 
 #define IK_TREE_CLASS	100
-#define POOL_NAME "/mnt/daos/btree-test"
-#define POOL_SIZE ((1024 * 1024 * 1024ULL))
+#define POOL_NAME       "/mnt/daos/btree-test"
 
 /** customized functions for btree */
 static int

src/common/tests/btree_direct.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2018-2022 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -55,8 +55,7 @@ struct btr_test_state {
 };
 
 #define SK_TREE_CLASS	100
-#define POOL_NAME "/mnt/daos/btree-direct-test"
-#define POOL_SIZE ((1024 * 1024  * 1024ULL))
+#define POOL_NAME       "/mnt/daos/btree-direct-test"
 
 #define SK_ORDER_DEF	16
 

src/common/tests/umem_test.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2019-2024 Intel Corporation.
+ * (C) Copyright 2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -27,7 +28,6 @@
 #include <daos/tests_lib.h>
 #include "utest_common.h"
 
-#define POOL_SIZE ((1024 * 1024  * 1024ULL))
 #define MAX_CHUNKS 8192
 
 struct chunk {
@@ -207,7 +207,7 @@ teardown_pmem(void **state)
 
 	return rc;
 }
-int
+static int
 setup_pmem(void **state)
 {
 	struct test_arg		*arg = *state;
@@ -749,6 +749,16 @@ test_p2_evict(void **state)
 	umem_cache_free(&arg->ta_store);
 }
 
+static void
+test_tx_fail_no_abort(void **state)
+{
+	struct test_arg *arg = *state;
+	int              rc;
+
+	rc = utest_tx_fail_no_abort(arg->ta_utx);
+	assert_int_equal(rc, 0);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -762,6 +772,8 @@ main(int argc, char **argv)
 	    {"UMEM007: Test page cache many writes", test_many_writes, NULL, NULL},
 	    {"UMEM008: Test phase2 APIs", test_p2_basic, NULL, NULL},
 	    {"UMEM009: Test phase2 eviction", test_p2_evict, NULL, NULL},
+	    {"UMEM010: Test TX failure with NO_ABORT on pmem", test_tx_fail_no_abort, setup_pmem,
+	     teardown_pmem},
 	    {NULL, NULL, NULL, NULL}};
 
 	d_register_alt_assert(mock_assert);

src/common/tests/umem_test_bmem.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2019-2024 Intel Corporation.
- * (C) Copyright 2023-2025 Hewlett Packard Enterprise Development LP.
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -28,7 +28,6 @@
 #include <daos/tests_lib.h>
 #include "utest_common.h"
 
-#define POOL_SIZE  ((256 * 1024 * 1024ULL))
 #define NEMB_RATIO (0.8)
 #define MB_SIZE    (16 * 1024 * 1024)
 
@@ -2669,6 +2668,16 @@ test_umempobj_heap_mb_stats(void **state)
 	D_FREE(name);
 }
 
+static void
+test_tx_fail_no_abort(void **state)
+{
+	struct test_arg *arg = *state;
+	int              rc;
+
+	rc = utest_tx_fail_no_abort(arg->ta_utx);
+	assert_int_equal(rc, 0);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -2702,6 +2711,8 @@ main(int argc, char **argv)
 	     teardown_pmem},
 	    {"BMEM015a: Test tx defer free publish/cancel", test_tx_bucket_dfree_publish_cancel,
 	     setup_pmem, teardown_pmem},
+	    {"BMEM016: Test TX failure with NO_ABORT on bmem", test_tx_fail_no_abort, setup_pmem,
+	     teardown_pmem},
 	    {NULL, NULL, NULL, NULL}};
 
 	static const struct CMUnitTest v2_tests[] = {
@@ -2747,6 +2758,8 @@ main(int argc, char **argv)
 	     NULL},
 	    {"BMEM022: Test umempobj non_evictable MB usage", test_umempobj_nemb_usage, NULL, NULL},
 	    {"BMEM023: Test umempobj get MB stats", test_umempobj_heap_mb_stats, NULL, NULL},
+	    {"BMEM024: Test TX failure with NO_ABORT on bmem_v2", test_tx_fail_no_abort,
+	     setup_pmem_v2, teardown_pmem},
 	    {NULL, NULL, NULL, NULL}};
 
 	rc = daos_debug_init(DAOS_LOG_DEFAULT);

src/common/tests/utest_common.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2019-2023 Intel Corporation.
+ * (C) Copyright 2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -375,3 +376,62 @@ utest_check_mem_initial_status(struct utest_context *utx)
 	}
 	return 0;
 }
+
+/*
+ * A failed allocation with UMEM_FLAG_NO_ABORT must leave the transaction
+ * alive so in-tx fallbacks (e.g. vos_dtx_reuse_cmt_blob) keep working;
+ * if without the flag, the failure aborts the TX and any further tx_add
+ * is a fatal usage error in the allocator.
+ */
+int
+utest_tx_fail_no_abort(struct utest_context *utx)
+{
+	struct umem_instance *umm = utest_utx2umm(utx);
+	int                  *value;
+	umem_off_t            umoff = UMOFF_NULL;
+	umem_off_t            huge;
+	int                   rc;
+
+	rc = utest_tx_begin(utx);
+	if (rc != 0)
+		goto done;
+
+	/* Larger than the pool: must fail, but must NOT abort the TX */
+	huge = umem_alloc_verb(umm, UMEM_FLAG_ZERO | UMEM_FLAG_NO_ABORT, POOL_SIZE,
+			       UMEM_DEFAULT_MBKT_ID);
+	if (!UMOFF_IS_NULL(huge)) {
+		D_ERROR("huge allocation unexpectedly succeeded\n");
+		D_GOTO(end, rc = 1);
+	}
+
+	/* TX still alive: alloc + snapshot + write must all work */
+	umoff = umem_zalloc(umm, 4);
+	if (UMOFF_IS_NULL(umoff)) {
+		D_ERROR("small alloc failed after NO_ABORT failure\n");
+		D_GOTO(end, rc = 2);
+	}
+
+	value = umem_off2ptr(umm, umoff);
+
+	rc = umem_tx_xadd_ptr(umm, value, (uint64_t)(-2), UMEM_FLAG_NO_ABORT);
+	if (rc == 0) {
+		D_ERROR("large sized add_ptr() should fail\n");
+		D_GOTO(end, rc = 3);
+	}
+
+	rc = umem_tx_add_ptr(umm, value, 4);
+	if (rc != 0) {
+		D_ERROR("tx_add_ptr failed after NO_ABORT failure\n");
+		D_GOTO(end, rc = 4);
+	}
+
+	*value = 42;
+
+	rc = umem_free(umm, umoff);
+
+end:
+	rc = utest_tx_end(utx, rc);
+
+done:
+	return rc;
+}

src/common/tests/utest_common.h

@@ -1,11 +1,13 @@
 /**
  * (C) Copyright 2019-2021 Intel Corporation.
+ * (C) Copyright 2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
 #include <daos/mem.h>
 
 #define UTEST_POOL_NAME_MAX	255
+#define POOL_SIZE               (1024 * 1024 * 1024ULL)
 
 struct utest_context;
 
@@ -174,3 +176,10 @@ int utest_check_mem_increase(struct utest_context *utx);
  */
 int utest_check_mem_initial_status(struct utest_context *utx);
 
+/** Check transaction failure with NO_ABROT flag
+ *  \param	utx[IN]	utest_context
+ *
+ *  \return 0 on success
+ */
+int
+utest_tx_fail_no_abort(struct utest_context *utx);

src/dtx/tests/dts_aggregate.c

@@ -156,6 +156,14 @@ tx_add_ptr(struct umem_instance *umm, void *ptr, size_t ptr_size)
 	return mock_type(int);
 }
 
+static int
+tx_xadd(struct umem_instance *umm, umem_off_t umoff, uint64_t offset, size_t size, uint64_t flags)
+{
+	void *ptr = umem_off2ptr(umm, umoff);
+
+	return tx_add_ptr(umm, ptr, size);
+}
+
 static int
 tx_free(struct umem_instance *umm, umem_off_t umoff)
 {
@@ -227,6 +235,7 @@ static umem_ops_t umm_ops = {.mo_tx_begin   = tx_begin,
 			     .mo_tx_commit  = tx_commit,
 			     .mo_tx_abort   = tx_abort,
 			     .mo_tx_add_ptr = tx_add_ptr,
+			     .mo_tx_xadd    = tx_xadd,
 			     .mo_tx_free    = tx_free};
 
 static int