backport: Merge bitcoin#25077, 26533

src/bench/duplicate_inputs.cpp

@@ -25,7 +25,7 @@ static void DuplicateInputs(benchmark::Bench& bench)
     CMutableTransaction coinbaseTx{};
     CMutableTransaction naughtyTx{};
 
-    CBlockIndex* pindexPrev = testing_setup->m_node.chainman->ActiveChain().Tip();
+    CBlockIndex* pindexPrev = WITH_LOCK(testing_setup->m_node.chainman->GetMutex(), return testing_setup->m_node.chainman->ActiveChain().Tip());
     assert(pindexPrev != nullptr);
     block.nBits = GetNextWorkRequired(pindexPrev, &block, chainparams.GetConsensus());
     block.nNonce = 0;

src/bitcoin-chainstate.cpp

@@ -153,12 +153,14 @@ int main(int argc, char* argv[])
     // Main program logic starts here
     std::cout
         << "Hello! I'm going to print out some information about your datadir." << std::endl
-        << "\t" << "Path: " << gArgs.GetDataDirNet() << std::endl
+        << "\t" << "Path: " << gArgs.GetDataDirNet() << std::endl;
+    {
+        LOCK(chainman.GetMutex());
+        std::cout
         << "\t" << "Reindexing: " << std::boolalpha << node::fReindex.load() << std::noboolalpha << std::endl
         << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl
         << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl
         << "\t" << "Active IBD: " << std::boolalpha << chainman.ActiveChainstate().IsInitialBlockDownload() << std::noboolalpha << std::endl;
-    {
         CBlockIndex* tip = chainman.ActiveTip();
         if (tip) {
             std::cout << "\t" << tip->ToString() << std::endl;

src/chainlock/clsig.cpp

@@ -27,4 +27,13 @@ llmq::VerifyRecSigStatus VerifyChainLock(const Consensus::Params& params, const
     return llmq::VerifyRecoveredSig(llmqType, chain, qman, clsig.getHeight(), request_id, clsig.getBlockHash(),
                                     clsig.getSig());
 }
+
+llmq::VerifyRecSigStatus VerifyChainLock(const Consensus::Params& params, const llmq::CQuorumManager& qman,
+                                         const chainlock::ChainLockSig& clsig, const CBlockIndex* pindexStart)
+{
+    const auto llmqType = params.llmqTypeChainLocks;
+    const uint256 request_id = GenSigRequestId(clsig.getHeight());
+
+    return llmq::VerifyRecoveredSig(llmqType, qman, pindexStart, request_id, clsig.getBlockHash(), clsig.getSig());
+}
 } // namespace chainlock

src/chainlock/clsig.h

@@ -8,6 +8,7 @@
 #include <cstdint>
 
 class CChain;
+class CBlockIndex;
 class uint256;
 
 namespace Consensus {
@@ -27,6 +28,8 @@ uint256 GenSigRequestId(const int32_t nHeight);
 
 llmq::VerifyRecSigStatus VerifyChainLock(const Consensus::Params& params, const CChain& chain,
                                          const llmq::CQuorumManager& qman, const ChainLockSig& clsig);
+llmq::VerifyRecSigStatus VerifyChainLock(const Consensus::Params& params, const llmq::CQuorumManager& qman,
+                                         const ChainLockSig& clsig, const CBlockIndex* pindexStart);
 } // namespace chainlock
 
 #endif // BITCOIN_CHAINLOCK_CLSIG_H

src/chainlock/handler.cpp

@@ -102,7 +102,9 @@ MessageProcessingResult ChainlockHandler::ProcessNewChainLock(const NodeId from,
         }
     }
 
-    if (const auto ret = chainlock::VerifyChainLock(Params().GetConsensus(), m_chainman.ActiveChain(), qman, clsig);
+    const CBlockIndex* pindex_start = WITH_LOCK(::cs_main,
+        return llmq::SelectQuorumForSigningStartBlock(m_chainman.ActiveChain(), clsig.getHeight()));
+    if (const auto ret = chainlock::VerifyChainLock(Params().GetConsensus(), qman, clsig, pindex_start);
         ret != llmq::VerifyRecSigStatus::Valid) {
         LogPrint(BCLog::CHAINLOCKS, "ChainlockHandler::%s -- invalid CLSIG (%s), status=%d peer=%d\n", __func__,
                  clsig.ToString(), std23::to_underlying(ret), from);

src/governance/signing.cpp

@@ -60,7 +60,7 @@ std::optional<const CSuperblock> GovernanceSigner::CreateSuperblockCandidate(int
     CSuperblock::GetNearestSuperblocksHeights(nHeight, nLastSuperblock, nNextSuperblock);
     auto SBEpochTime = static_cast<int64_t>(GetTime<std::chrono::seconds>().count() +
                                             (nNextSuperblock - nHeight) * 2.62 * 60);
-    auto governanceBudget = CSuperblock::GetPaymentsLimit(m_chainman.ActiveChain(), nNextSuperblock);
+    auto governanceBudget = WITH_LOCK(::cs_main, return CSuperblock::GetPaymentsLimit(m_chainman.ActiveChain(), nNextSuperblock));
 
     CAmount budgetAllocated{};
     for (const auto& proposal : approvedProposals) {

src/init.cpp

@@ -913,7 +913,7 @@ static void PeriodicStats(NodeContext& node)
         LogPrintf("%s: GetUTXOStats failed\n", __func__);
     }
 
-    CBlockIndex *tip = chainman.ActiveChain().Tip();
+    CBlockIndex *tip = WITH_LOCK(::cs_main, return chainman.ActiveChain().Tip());
     double nNetworkHashPS = [&]() {
         // Short version of GetNetworkHashPS(120, -1);
         CBlockIndex *pindex = tip;
@@ -2298,7 +2298,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         RegisterValidationInterface(node.cj_walletman.get());
     }
 
-    bool fLoadCacheFiles = !(fReindex || fReindexChainState) && (chainman.ActiveChain().Tip() != nullptr);
+    bool fLoadCacheFiles = !(fReindex || fReindexChainState) && WITH_LOCK(::cs_main, return chainman.ActiveChain().Tip() != nullptr);
 
     if (!node.netfulfilledman->LoadCache(fLoadCacheFiles)) {
         auto file_path = fs::PathToString(gArgs.GetDataDirNet() / "netfulfilled.dat");
@@ -2470,7 +2470,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly.
     // No locking, as this happens before any background thread is started.
     boost::signals2::connection block_notify_genesis_wait_connection;
-    if (chainman.ActiveChain().Tip() == nullptr) {
+    if (WITH_LOCK(chainman.GetMutex(), return chainman.ActiveChain().Tip() == nullptr)) {
         block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(std::bind(BlockNotifyGenesisWait, std::placeholders::_2));
     } else {
         fHaveGenesis = true;
@@ -2528,7 +2528,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         // Seed InstantSend tip-height cache; NetInstantSend receives future
         // updates via CValidationInterface but misses InitializeCurrentBlockTip.
         // TODO: move cache updates from NetInstantSend to g_ds_notification due to specific of Tip's processing
-        node.llmq_ctx->isman->CacheTipHeight(chainman.ActiveChain().Tip());
+        node.llmq_ctx->isman->CacheTipHeight(WITH_LOCK(::cs_main, return chainman.ActiveChain().Tip()));
 
         {
             // Get all UTXOs for each MN collateral in one go so that we can fill coin cache early
@@ -2592,7 +2592,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         }
 
         if (node.active_ctx) {
-            node.active_ctx->nodeman->Init(chainman.ActiveTip());
+            node.active_ctx->nodeman->Init(WITH_LOCK(::cs_main, return chainman.ActiveTip()));
             // Now that nodeman->Init has set proTxHash, fan out the
             // startup tip to all CValidationInterface subscribers.
             // The earlier call only kicked CDSNotificationInterface
@@ -2804,12 +2804,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     // At this point, the RPC is "started", but still in warmup, which means it
     // cannot yet be called. Before we make it callable, we need to make sure
     // that the RPC's view of the best block is valid and consistent with
-    // ChainstateManager's ActiveTip.
+    // ChainstateManager's active tip.
     //
     // If we do not do this, RPC's view of the best block will be height=0 and
     // hash=0x0. This will lead to erroroneous responses for things like
     // waitforblockheight.
-    RPCNotifyBlockChange(chainman.ActiveTip());
+    RPCNotifyBlockChange(WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()));
     SetRPCWarmupFinished();
 
     uiInterface.InitMessage(_("Done loading").translated);

src/llmq/ehf_signals.cpp

@@ -67,7 +67,8 @@ void CEHFSignalsHandler::trySignEHFSignal(int bit, const CBlockIndex* const pind
         return;
     }
 
-    const auto quorum = llmq::SelectQuorumForSigning(llmq_params_opt.value(), m_chainman.ActiveChain(), qman, requestId);
+    const CChain& active_chain = *WITH_LOCK(::cs_main, return &m_chainman.ActiveChain());
+    const auto quorum = llmq::SelectQuorumForSigning(llmq_params_opt.value(), active_chain, qman, requestId);
     if (!quorum) {
         LogPrintf("CEHFSignalsHandler::trySignEHFSignal no quorum for id=%s\n", requestId.ToString());
         return;

src/llmq/quorumsman.cpp

@@ -518,23 +518,26 @@ void CQuorumManager::MigrateOldQuorumDB(CEvoDB& evoDb) const
     LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- done\n", __func__);
 }
 
-CQuorumCPtr SelectQuorumForSigning(const Consensus::LLMQParams& llmq_params, const CChain& active_chain, const CQuorumManager& qman,
-                                   const uint256& selectionHash, int signHeight, int signOffset)
+CBlockIndex* SelectQuorumForSigningStartBlock(const CChain& active_chain, int signHeight, int signOffset)
 {
-    size_t poolSize = llmq_params.signingActiveQuorumCount;
+    AssertLockHeld(::cs_main);
+    if (signHeight == -1) {
+        signHeight = active_chain.Height();
+    }
+    const int startBlockHeight = signHeight - signOffset;
+    if (startBlockHeight > active_chain.Height() || startBlockHeight < 0) {
+        return nullptr;
+    }
+    return active_chain[startBlockHeight];
+}
 
-    CBlockIndex* pindexStart;
-    {
-        LOCK(::cs_main);
-        if (signHeight == -1) {
-            signHeight = active_chain.Height();
-        }
-        int startBlockHeight = signHeight - signOffset;
-        if (startBlockHeight > active_chain.Height() || startBlockHeight < 0) {
-            return {};
-        }
-        pindexStart = active_chain[startBlockHeight];
+CQuorumCPtr SelectQuorumForSigning(const Consensus::LLMQParams& llmq_params, const CQuorumManager& qman,
+                                   const uint256& selectionHash, const CBlockIndex* pindexStart)
+{
+    if (pindexStart == nullptr) {
+        return nullptr;
     }
+    size_t poolSize = llmq_params.signingActiveQuorumCount;
 
     // don't remove connections for the currently in-progress DKG round
     if (IsQuorumRotationEnabled(llmq_params, pindexStart)) {
@@ -581,13 +584,20 @@ CQuorumCPtr SelectQuorumForSigning(const Consensus::LLMQParams& llmq_params, con
     }
 }
 
-VerifyRecSigStatus VerifyRecoveredSig(Consensus::LLMQType llmqType, const CChain& active_chain, const CQuorumManager& qman,
-                        int signedAtHeight, const uint256& id, const uint256& msgHash, const CBLSSignature& sig,
-                        const int signOffset)
+CQuorumCPtr SelectQuorumForSigning(const Consensus::LLMQParams& llmq_params, const CChain& active_chain, const CQuorumManager& qman,
+                                   const uint256& selectionHash, int signHeight, int signOffset)
+{
+    const CBlockIndex* pindexStart = WITH_LOCK(::cs_main, return SelectQuorumForSigningStartBlock(active_chain, signHeight, signOffset));
+    return SelectQuorumForSigning(llmq_params, qman, selectionHash, pindexStart);
+}
+
+VerifyRecSigStatus VerifyRecoveredSig(Consensus::LLMQType llmqType, const CQuorumManager& qman,
+                                      const CBlockIndex* pindexStart, const uint256& id, const uint256& msgHash,
+                                      const CBLSSignature& sig)
 {
     const auto& llmq_params_opt = Params().GetLLMQ(llmqType);
     assert(llmq_params_opt.has_value());
-    auto quorum = SelectQuorumForSigning(llmq_params_opt.value(), active_chain, qman, id, signedAtHeight, signOffset);
+    auto quorum = SelectQuorumForSigning(llmq_params_opt.value(), qman, id, pindexStart);
     if (!quorum) {
         return VerifyRecSigStatus::NoQuorum;
     }
@@ -597,4 +607,12 @@ VerifyRecSigStatus VerifyRecoveredSig(Consensus::LLMQType llmqType, const CChain
     return ret ? VerifyRecSigStatus::Valid : VerifyRecSigStatus::Invalid;
 }
 
+VerifyRecSigStatus VerifyRecoveredSig(Consensus::LLMQType llmqType, const CChain& active_chain, const CQuorumManager& qman,
+                        int signedAtHeight, const uint256& id, const uint256& msgHash, const CBLSSignature& sig,
+                        const int signOffset)
+{
+    const CBlockIndex* pindexStart = WITH_LOCK(::cs_main, return SelectQuorumForSigningStartBlock(active_chain, signedAtHeight, signOffset));
+    return VerifyRecoveredSig(llmqType, qman, pindexStart, id, msgHash, sig);
+}
+
 } // namespace llmq

src/llmq/quorumsman.h

@@ -31,6 +31,7 @@ class CDeterministicMNManager;
 class CDBWrapper;
 class CEvoDB;
 class ChainstateManager;
+extern RecursiveMutex cs_main; // NOLINT(readability-redundant-declaration)
 namespace util {
 struct DbWrapperParams;
 } // namespace util
@@ -175,9 +176,19 @@ class CQuorumManager final
 // which are not 100% at the chain tip.
 static constexpr int SIGN_HEIGHT_OFFSET{8};
 
+CBlockIndex* SelectQuorumForSigningStartBlock(const CChain& active_chain, int signHeight = -1 /*chain tip*/,
+                                              int signOffset = SIGN_HEIGHT_OFFSET) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+CQuorumCPtr SelectQuorumForSigning(const Consensus::LLMQParams& llmq_params, const CQuorumManager& qman,
+                                   const uint256& selectionHash, const CBlockIndex* pindexStart);
+
 CQuorumCPtr SelectQuorumForSigning(const Consensus::LLMQParams& llmq_params, const CChain& active_chain, const CQuorumManager& qman,
                                    const uint256& selectionHash, int signHeight = -1 /*chain tip*/, int signOffset = SIGN_HEIGHT_OFFSET);
 
+VerifyRecSigStatus VerifyRecoveredSig(Consensus::LLMQType llmqType, const CQuorumManager& qman,
+                                      const CBlockIndex* pindexStart, const uint256& id, const uint256& msgHash,
+                                      const CBLSSignature& sig);
+
 // Verifies a recovered sig that was signed while the chain tip was at signedAtTip
 VerifyRecSigStatus VerifyRecoveredSig(Consensus::LLMQType llmqType, const CChain& active_chain, const CQuorumManager& qman,
                                       int signedAtHeight, const uint256& id, const uint256& msgHash, const CBLSSignature& sig,

src/llmq/signing_shares.cpp

@@ -744,7 +744,8 @@ bool CSigSharesManager::AsyncSignIfMember(Consensus::LLMQType llmqType, CSigning
             // the quorum list and no recovered signature has been created in the mean time
             const auto& llmq_params_opt = Params().GetLLMQ(llmqType);
             assert(llmq_params_opt.has_value());
-            return SelectQuorumForSigning(llmq_params_opt.value(), m_chainman.ActiveChain(), qman, id);
+            CChain& active_chain = *WITH_LOCK(::cs_main, return &m_chainman.ActiveChain());
+            return SelectQuorumForSigning(llmq_params_opt.value(), active_chain, qman, id);
         } else {
             return qman.GetQuorum(llmqType, quorumHash);
         }

src/net_processing.cpp

@@ -3322,7 +3322,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer,
     const std::string msg_type = uses_compressed ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS;
     if (nCount == GetHeadersLimit(pfrom, uses_compressed)) {
         // Headers message had its maximum size; the peer may have more headers.
-        if (MaybeSendGetHeaders(pfrom, msg_type, m_chainman.ActiveChain().GetLocator(pindexLast), peer)) {
+        if (MaybeSendGetHeaders(pfrom, msg_type, WITH_LOCK(m_chainman.GetMutex(), return m_chainman.ActiveChain().GetLocator(pindexLast)), peer)) {
             LogPrint(BCLog::NET, "more %s (%d) to end to peer=%d (startheight:%d)\n",
                     msg_type, pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height);
         }
@@ -4707,7 +4707,7 @@ void PeerManagerImpl::ProcessMessage(
                 pindex = m_chainman.ActiveChain().Next(pindex);
         }
 
-        const auto send_headers = [this /* for m_connman */, &hashStop, &pindex, &nodestate, &pfrom, &msgMaker](auto msg_type_internal, auto& v_headers, auto callback) {
+        const auto send_headers = [this /* for m_connman */, &hashStop, &pindex, &nodestate, &pfrom, &msgMaker](auto msg_type_internal, auto& v_headers, auto callback) EXCLUSIVE_LOCKS_REQUIRED(cs_main) {
             int nLimit = GetHeadersLimit(pfrom, msg_type_internal == NetMsgType::HEADERS2);
             for (; pindex; pindex = m_chainman.ActiveChain().Next(pindex)) {
                 v_headers.emplace_back(callback(pindex));

src/node/blockstorage.cpp

@@ -400,6 +400,23 @@ bool BlockManager::LoadBlockIndexDB()
     return true;
 }
 
+void BlockManager::ScanAndUnlinkAlreadyPrunedFiles()
+{
+    AssertLockHeld(::cs_main);
+    if (!m_have_pruned) {
+        return;
+    }
+
+    std::set<int> block_files_to_prune;
+    for (int file_number = 0; file_number < m_last_blockfile; file_number++) {
+        if (m_blockfile_info[file_number].nSize == 0) {
+            block_files_to_prune.insert(file_number);
+        }
+    }
+
+    UnlinkPrunedFiles(block_files_to_prune);
+}
+
 const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data)
 {
     const MapCheckpoints& checkpoints = data.mapCheckpoints;
@@ -575,11 +592,14 @@ uint64_t BlockManager::CalculateCurrentUsage()
 
 void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
 {
+    std::error_code ec;
     for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
         FlatFilePos pos(*it, 0);
-        fs::remove(BlockFileSeq().FileName(pos));
-        fs::remove(UndoFileSeq().FileName(pos));
-        LogPrint(BCLog::BLOCKSTORE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
+        const bool removed_blockfile{fs::remove(BlockFileSeq().FileName(pos), ec)};
+        const bool removed_undofile{fs::remove(UndoFileSeq().FileName(pos), ec)};
+        if (removed_blockfile || removed_undofile) {
+            LogPrint(BCLog::BLOCKSTORE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
+        }
     }
 }
 

src/node/blockstorage.h

@@ -169,6 +169,13 @@ class BlockManager
     bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
 
+    /**
+     * Remove any pruned block & undo files that are still on disk.
+     * This could happen on some systems if the file was still being read while unlinked,
+     * or if we crash before unlinking.
+     */
+    void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
     CBlockIndex* AddToBlockIndex(const CBlockHeader& block, const uint256& hash, CBlockIndex*& best_header,
                                  enum BlockStatus nStatus = BLOCK_VALID_TREE)
         EXCLUSIVE_LOCKS_REQUIRED(cs_main);

src/node/interfaces.cpp

@@ -324,7 +324,7 @@ class GOVImpl : public GOV
     std::optional<int32_t> getProposalFundedHeight(const uint256& proposal_hash) override
     {
         if (context().chain_helper != nullptr && context().chainman != nullptr) {
-            const int32_t nTipHeight = context().chainman->ActiveHeight();
+            const int32_t nTipHeight = WITH_LOCK(::cs_main, return context().chainman->ActiveHeight());
             for (const auto& trigger : context().chain_helper->superblocks->GetActiveTriggers()) {
                 if (!trigger || trigger->GetBlockHeight() > nTipHeight) continue;
                 for (const auto& hash : trigger->GetProposalHashes()) {
@@ -343,8 +343,12 @@ class GOVImpl : public GOV
             const auto tip_mn_list{context().dmnman->GetListAtChainTip()};
             if (const auto proposals{context().govman->GetApprovedProposals(tip_mn_list)}; !proposals.empty()) {
                 int32_t last_sb{0}, next_sb{0};
-                CSuperblock::GetNearestSuperblocksHeights(context().chainman->ActiveHeight(), last_sb, next_sb);
-                const CAmount budget{CSuperblock::GetPaymentsLimit(context().chainman->ActiveChain(), next_sb)};
+                CAmount budget{0};
+                {
+                    LOCK(::cs_main);
+                    CSuperblock::GetNearestSuperblocksHeights(context().chainman->ActiveHeight(), last_sb, next_sb);
+                    budget = CSuperblock::GetPaymentsLimit(context().chainman->ActiveChain(), next_sb);
+                }
                 for (const auto& proposal : proposals) {
                     UniValue json = proposal->GetJSONObject();
                     CAmount payment_amount{0};