backport: bitcoin#18628, #19393, #25204, #25689, #25785, bitcoin-core/gui#613

src/Makefile.bench.include

@@ -31,6 +31,7 @@ bench_bench_dash_SOURCES = \
   bench/crypto_hash.cpp \
   bench/data.cpp \
   bench/data.h \
+  bench/descriptors.cpp \
   bench/duplicate_inputs.cpp \
   bench/ecdsa.cpp \
   bench/ellswift.cpp \

src/bench/descriptors.cpp

@@ -0,0 +1,30 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <key.h>
+#include <script/descriptor.h>
+#include <script/standard.h>
+
+#include <string>
+#include <utility>
+
+static void ExpandDescriptor(benchmark::Bench& bench)
+{
+    const auto desc_str = "sh(multi(15,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06))";
+    const std::pair<int64_t, int64_t> range = {0, 1000};
+    FlatSigningProvider provider;
+    std::string error;
+    auto desc = Parse(desc_str, provider, error);
+
+    bench.run([&] {
+        for (int i = range.first; i <= range.second; ++i) {
+            std::vector<CScript> scripts;
+            bool success = desc->Expand(i, provider, scripts, provider);
+            assert(success);
+        }
+    });
+}
+
+BENCHMARK(ExpandDescriptor, benchmark::PriorityLevel::HIGH);

src/net_processing.cpp

@@ -2102,6 +2102,10 @@ void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler)
  */
 void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex)
 {
+    // Orphans included in or conflicted by the block can never be accepted, so drop them before
+    // reconsidering the ones the block may have just made acceptable.
+    m_orphanage.EraseForBlock(*pblock);
+
     // Candidates are sourced from a block and therefore cannot be attributed to a peer, we use -1 as the identifier
     bool have_candidates{true};
     {
@@ -2117,7 +2121,6 @@ void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock
         }
     }
 
-    m_orphanage.EraseForBlock(*pblock);
     m_last_tip_update = GetTime<std::chrono::seconds>();
 
     {

src/qt/rpcconsole.cpp

@@ -36,10 +36,6 @@
 #include <univalue.h>
 
 #ifdef ENABLE_WALLET
-#ifdef USE_BDB
-#include <wallet/bdb.h>
-#endif
-#include <wallet/db.h>
 #include <wallet/walletutil.h>
 #endif
 

src/rpc/mempool.cpp

@@ -245,23 +245,12 @@ static std::vector<RPCResult> MempoolEntryDescription()
 {
     return {
         RPCResult{RPCResult::Type::NUM, "vsize", "Transaction size."},
-        RPCResult{RPCResult::Type::STR_AMOUNT, "fee", /*optional=*/true,
-                  "transaction fee, denominated in " + CURRENCY_UNIT + " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"},
-        RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", /*optional=*/true,
-                  "transaction fee with fee deltas used for mining priority, denominated in " + CURRENCY_UNIT +
-                      " (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"},
         RPCResult{RPCResult::Type::NUM_TIME, "time", "local time transaction entered pool in " + UNIX_EPOCH_TIME},
         RPCResult{RPCResult::Type::NUM, "height", "block height when transaction entered pool"},
         RPCResult{RPCResult::Type::NUM, "descendantcount", "number of in-mempool descendant transactions (including this one)"},
         RPCResult{RPCResult::Type::NUM, "descendantsize", "size of in-mempool descendants (including this one)"},
-        RPCResult{RPCResult::Type::STR_AMOUNT, "descendantfees", /*optional=*/true,
-                  "transaction fees of in-mempool descendants (including this one) with fee deltas used for mining priority, denominated in " +
-                      CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"},
         RPCResult{RPCResult::Type::NUM, "ancestorcount", "number of in-mempool ancestor transactions (including this one)"},
         RPCResult{RPCResult::Type::NUM, "ancestorsize", "size of in-mempool ancestors (including this one)"},
-        RPCResult{RPCResult::Type::STR_AMOUNT, "ancestorfees", /*optional=*/true,
-                  "transaction fees of in-mempool ancestors (including this one) with fee deltas used for mining priority, denominated in " +
-                      CURRENCY_ATOM + "s (DEPRECATED, returned only if config option -deprecatedrpc=fees is passed)"},
         RPCResult{RPCResult::Type::OBJ, "fees", "",
             {
                 RPCResult{RPCResult::Type::STR_AMOUNT, "base", "transaction fee, denominated in " + CURRENCY_UNIT},
@@ -283,24 +272,12 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool
     AssertLockHeld(pool.cs);
 
     info.pushKV("vsize", (int)e.GetTxSize());
-    // TODO: top-level fee fields are deprecated. deprecated_fee_fields_enabled blocks should be removed in v24
-    const bool deprecated_fee_fields_enabled{IsDeprecatedRPCEnabled("fees")};
-    if (deprecated_fee_fields_enabled) {
-        info.pushKV("fee", ValueFromAmount(e.GetFee()));
-        info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee()));
-    }
     info.pushKV("time", count_seconds(e.GetTime()));
     info.pushKV("height", (int)e.GetHeight());
     info.pushKV("descendantcount", e.GetCountWithDescendants());
     info.pushKV("descendantsize", e.GetSizeWithDescendants());
-    if (deprecated_fee_fields_enabled) {
-        info.pushKV("descendantfees", e.GetModFeesWithDescendants());
-    }
     info.pushKV("ancestorcount", e.GetCountWithAncestors());
     info.pushKV("ancestorsize", e.GetSizeWithAncestors());
-    if (deprecated_fee_fields_enabled) {
-        info.pushKV("ancestorfees", e.GetModFeesWithAncestors());
-    }
 
     UniValue fees(UniValue::VOBJ);
     fees.pushKV("base", ValueFromAmount(e.GetFee()));

src/test/fuzz/tx_pool.cpp

@@ -138,7 +138,6 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool)
     auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())};
 
     MockTime(fuzzed_data_provider, chainstate);
-    SetMempoolConstraints(*node.args, fuzzed_data_provider);
 
     // All RBF-spendable outpoints
     std::set<COutPoint> outpoints_rbf;
@@ -152,6 +151,7 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool)
     // The sum of the values of all spendable outpoints
     constexpr CAmount SUPPLY_TOTAL{COINBASE_MATURITY * 50 * COIN};
 
+    SetMempoolConstraints(*node.args, fuzzed_data_provider);
     CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)};
     MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_);
 
@@ -220,9 +220,6 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool)
         if (fuzzed_data_provider.ConsumeBool()) {
             MockTime(fuzzed_data_provider, chainstate);
         }
-        if (fuzzed_data_provider.ConsumeBool()) {
-            SetMempoolConstraints(*node.args, fuzzed_data_provider);
-        }
         if (fuzzed_data_provider.ConsumeBool()) {
             tx_pool.RollingFeeUpdate();
         }
@@ -315,7 +312,6 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool)
     auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())};
 
     MockTime(fuzzed_data_provider, chainstate);
-    SetMempoolConstraints(*node.args, fuzzed_data_provider);
 
     std::vector<uint256> txids;
     txids.reserve(g_outpoints_coinbase_init_mature.size());
@@ -328,6 +324,7 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool)
         txids.push_back(ConsumeUInt256(fuzzed_data_provider));
     }
 
+    SetMempoolConstraints(*node.args, fuzzed_data_provider);
     CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)};
     MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_);
 
@@ -340,9 +337,6 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool)
         if (fuzzed_data_provider.ConsumeBool()) {
             MockTime(fuzzed_data_provider, chainstate);
         }
-        if (fuzzed_data_provider.ConsumeBool()) {
-            SetMempoolConstraints(*node.args, fuzzed_data_provider);
-        }
         if (fuzzed_data_provider.ConsumeBool()) {
             tx_pool.RollingFeeUpdate();
         }

test/functional/p2p_invalid_tx.py

@@ -72,7 +72,6 @@ def run_test(self):
         block.solve()
         # Save the coinbase for later
         block2 = block
-        tip = block.sha256
         node.p2ps[0].send_blocks_and_test([block1, block2], node, success=True)
 
         self.log.info("Mature the block.")
@@ -116,24 +115,24 @@ def test_orphan_tx_handling(self, base_tx, resolve_via_block):
         SCRIPT_PUB_KEY_OP_TRUE = b'\x51\x75' * 15 + b'\x51'
         tx_withhold = CTransaction()
         tx_withhold.vin.append(CTxIn(outpoint=COutPoint(base_tx, 0)))
-        tx_withhold.vout.append(CTxOut(nValue=50 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
+        tx_withhold.vout = [CTxOut(nValue=25 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2
         tx_withhold.calc_sha256()
 
         # Our first orphan tx with some outputs to create further orphan txs
         tx_orphan_1 = CTransaction()
         tx_orphan_1.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 0)))
-        tx_orphan_1.vout = [CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3
+        tx_orphan_1.vout = [CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3
         tx_orphan_1.calc_sha256()
 
         # A valid transaction with low fee
         tx_orphan_2_no_fee = CTransaction()
         tx_orphan_2_no_fee.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 0)))
-        tx_orphan_2_no_fee.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
+        tx_orphan_2_no_fee.vout.append(CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
 
         # A valid transaction with sufficient fee
         tx_orphan_2_valid = CTransaction()
         tx_orphan_2_valid.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 1)))
-        tx_orphan_2_valid.vout.append(CTxOut(nValue=10 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
+        tx_orphan_2_valid.vout.append(CTxOut(nValue=8 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
         tx_orphan_2_valid.calc_sha256()
 
         # An invalid transaction with negative fee
@@ -188,7 +187,8 @@ def test_orphan_tx_handling(self, base_tx, resolve_via_block):
         # this test is different with bitcoin due to dashpay/dash#3121
         # we have a limit based on size in megabytes, not by amount of txes
         # one tx is 91byte; 1Mb / 4451byte = 224; need to send at least 225
-        orphan_tx_pool = [CTransaction() for _ in range(225)]
+        tx_count_to_orphan = 224
+        orphan_tx_pool = [CTransaction() for _ in range(tx_count_to_orphan + 1)]
         for i in range(len(orphan_tx_pool)):
             orphan_tx_pool[i].vin.append(CTxIn(outpoint=COutPoint(i, 333)))
             for j in range(110):
@@ -197,13 +197,86 @@ def test_orphan_tx_handling(self, base_tx, resolve_via_block):
         with node.assert_debug_log(['orphanage overflow, removed 1 tx']):
             node.p2ps[0].send_txs_and_test(orphan_tx_pool, node, success=False)
 
+        self.log.info('Test orphan with rejected parents')
         rejected_parent = CTransaction()
         rejected_parent.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_2_invalid.sha256, 0)))
         rejected_parent.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
         rejected_parent.rehash()
-        # TODO: somehow it fails on `block` stage without 'not keeping orphan'
-        #with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(rejected_parent.hash)]):
-        node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False)
+        if resolve_via_block:
+            # connecting the block advanced the tip, which resets the recent rejects filter, so the
+            # parent is no longer known to be rejected here and this orphan is kept after all
+            expected_orphans = tx_count_to_orphan + 1
+            node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False)
+        else:
+            expected_orphans = tx_count_to_orphan
+            with node.assert_debug_log([f'not keeping orphan with rejected parents {rejected_parent.hash}']):
+                node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False)
+
+        self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool')
+        # Unlike upstream, this helper runs twice with a node restart in between, so the id of
+        # the last remaining peer (the one holding the orphans) is not a fixed number.
+        orphan_peer_id = node.getpeerinfo()[0]['id']
+        with node.assert_debug_log([f'Erased {expected_orphans} orphan tx from peer={orphan_peer_id}']):
+            self.reconnect_p2p(num_connections=1)
+
+        self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool')
+        tx_withhold_until_block_A = CTransaction()
+        tx_withhold_until_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 1)))
+        tx_withhold_until_block_A.vout = [CTxOut(nValue=12 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2
+        tx_withhold_until_block_A.calc_sha256()
+
+        tx_orphan_include_by_block_A = CTransaction()
+        tx_orphan_include_by_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 0)))
+        tx_orphan_include_by_block_A.vout.append(CTxOut(nValue=12 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
+        tx_orphan_include_by_block_A.calc_sha256()
+
+        self.log.info('Send the orphan ... ')
+        node.p2ps[0].send_txs_and_test([tx_orphan_include_by_block_A], node, success=False)
+
+        tip = int(node.getbestblockhash(), 16)
+        height = node.getblockcount() + 1
+        prev_block = node.getblockheader(node.getbestblockhash())
+        block_A = create_block(tip, create_coinbase(height), prev_block["time"] + 1)
+        if not resolve_via_block:
+            # in the block-resolution pass tx_withhold is already part of the chain
+            block_A.vtx.append(tx_withhold)
+        block_A.vtx.extend([tx_withhold_until_block_A, tx_orphan_include_by_block_A])
+        block_A.hashMerkleRoot = block_A.calc_merkle_root()
+        block_A.solve()
+
+        self.log.info('Send the block that includes the previous orphan ... ')
+        with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]):
+            node.p2ps[0].send_blocks_and_test([block_A], node, success=True)
+
+        self.log.info('Test that a transaction in the orphan pool conflicts with a new tip block causes erase this transaction from the orphan pool')
+        tx_withhold_until_block_B = CTransaction()
+        tx_withhold_until_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 1)))
+        tx_withhold_until_block_B.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
+        tx_withhold_until_block_B.calc_sha256()
+
+        tx_orphan_include_by_block_B = CTransaction()
+        tx_orphan_include_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0)))
+        tx_orphan_include_by_block_B.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
+        tx_orphan_include_by_block_B.calc_sha256()
+
+        tx_orphan_conflict_by_block_B = CTransaction()
+        tx_orphan_conflict_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0)))
+        tx_orphan_conflict_by_block_B.vout.append(CTxOut(nValue=9 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE))
+        tx_orphan_conflict_by_block_B.calc_sha256()
+        self.log.info('Send the orphan ... ')
+        node.p2ps[0].send_txs_and_test([tx_orphan_conflict_by_block_B], node, success=False)
+
+        tip = int(node.getbestblockhash(), 16)
+        height = node.getblockcount() + 1
+        block_time = self.nodes[0].getblock(node.getbestblockhash())['time'] + 1
+        block_B = create_block(tip, create_coinbase(height), block_time)
+        block_B.vtx.extend([tx_withhold_until_block_B, tx_orphan_include_by_block_B])
+        block_B.hashMerkleRoot = block_B.calc_merkle_root()
+        block_B.solve()
+
+        self.log.info('Send the block that includes a transaction which conflicts with the previous orphan ... ')
+        with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]):
+            node.p2ps[0].send_blocks_and_test([block_B], node, success=True)
 
 
 if __name__ == '__main__':