Fix overlay only mode

ci-build.sh

@@ -212,7 +212,7 @@ echo "Using RND_SEED: $RND_SEED"
 ulimit -n 65536
 export INTERACTIVE=0
 
-export TEST_SPEC='[overlay-ipc]'
+export TEST_SPEC='[overlay-ipc],[loadgen]'
 export SKIP_SOROBAN_TESTS=true
 export STELLAR_OVERLAY_BINARY="${SRC_DIR}/build-${CC}-${PROTOCOL}/stellar-overlay"
 time make check

src/herder/HerderImpl.cpp

@@ -666,12 +666,6 @@ HerderImpl::recvTransaction(TransactionFrameBasePtr tx, bool submittedFromSelf,
     CLOG_TRACE(Herder, "recv transaction {} for {}",
                hexAbbrev(tx->getFullHash()),
                KeyUtils::toShortString(tx->getSourceID()));
-#ifdef BUILD_TESTS
-    if (submittedFromSelf)
-    {
-        mLedgerManager.recordTxSubmission(tx->getContentsHash());
-    }
-#endif
 
     auto const& env = tx->getEnvelope();
     mApp.getOverlayManager().broadcastTransaction(env, tx->getFullFee(),

src/herder/PendingEnvelopes.h

@@ -198,7 +198,6 @@ class PendingEnvelopes
     // Returns true if every tx set referenced by `env` is available locally
     bool areTxSetsFetched(SCPEnvelope const& env) const;
 
-
     SCPEnvelopeWrapperPtr pop(uint64 slotIndex);
 
     // erases data for all slots outside the range [minSlot, maxSlot].

src/ledger/LedgerManagerImpl.cpp

@@ -247,6 +247,10 @@ LedgerManagerImpl::TxLatencyMetrics::TxLatencyMetrics(MetricsRegistry& registry)
     : mTxsSubmitted(registry.NewCounter({"loadgen", "tx-latency", "submitted"}))
     , mTxsExternalized(
           registry.NewCounter({"loadgen", "tx-latency", "externalized"}))
+    , mPendingTxsSelfCount(
+          registry.NewCounter({"herder", "pending-txs", "self-count"}))
+    , mPendingSorobanTxsSelfCount(
+          registry.NewCounter({"herder", "pending-soroban-txs", "self-count"}))
     , mRunMin(registry.NewCounter({"loadgen", "tx-latency-run", "min-ms"}))
     , mRunMax(registry.NewCounter({"loadgen", "tx-latency-run", "max-ms"}))
     , mRunMean(registry.NewCounter({"loadgen", "tx-latency-run", "mean-ms"}))
@@ -1344,10 +1348,20 @@ LedgerManagerImpl::emitNextMeta()
 }
 
 #ifdef BUILD_TESTS
+bool
+LedgerManagerImpl::txSelfTrackingActive() const
+{
+    // Overlay-only loadgen's completion check relies on the self-count
+    // counters this tracking maintains, so it is active there regardless of
+    // whether latency measurement was requested.
+    return mApp.getConfig().LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING ||
+           mApp.getRunInOverlayOnlyMode();
+}
+
 void
 LedgerManagerImpl::recordTxSubmission(Hash const& contentsHash)
 {
-    if (!mApp.getConfig().LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING)
+    if (!txSelfTrackingActive())
     {
         return;
     }
@@ -1368,10 +1382,12 @@ LedgerManagerImpl::recordTxSubmission(Hash const& contentsHash)
 void
 LedgerManagerImpl::recordTxE2eLatency(ApplicableTxSetFrame const& txSet)
 {
-    if (!mApp.getConfig().LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING)
+    if (!txSelfTrackingActive())
     {
         return;
     }
+    bool const recordLatency =
+        mApp.getConfig().LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING;
     VirtualClock::time_point const applyEndTime = mApp.getClock().now();
     MutexLocker guard(mTxLatencyMetrics.mMutex);
     for (auto const& phase : txSet.getPhases())
@@ -1382,14 +1398,20 @@ LedgerManagerImpl::recordTxE2eLatency(ApplicableTxSetFrame const& txSet)
                     tx->getContentsHash());
                 submitted != mTxLatencyMetrics.mTxSubmitTimes.end())
             {
-                auto const latency = applyEndTime - submitted->second;
                 mTxLatencyMetrics.mTxsExternalized.inc();
-                int64_t const ms =
-                    std::chrono::duration_cast<std::chrono::milliseconds>(
-                        latency)
-                        .count();
-                mTxLatencyMetrics.mSamples.push_back(std::clamp<int64_t>(
-                    ms, 0, std::numeric_limits<uint32_t>::max()));
+                (tx->isSoroban() ? mTxLatencyMetrics.mPendingSorobanTxsSelfCount
+                                 : mTxLatencyMetrics.mPendingTxsSelfCount)
+                    .inc();
+                if (recordLatency)
+                {
+                    auto const latency = applyEndTime - submitted->second;
+                    int64_t const ms =
+                        std::chrono::duration_cast<std::chrono::milliseconds>(
+                            latency)
+                            .count();
+                    mTxLatencyMetrics.mSamples.push_back(std::clamp<int64_t>(
+                        ms, 0, std::numeric_limits<uint32_t>::max()));
+                }
                 mTxLatencyMetrics.mTxSubmitTimes.erase(submitted);
             }
         }
@@ -1399,7 +1421,7 @@ LedgerManagerImpl::recordTxE2eLatency(ApplicableTxSetFrame const& txSet)
 void
 LedgerManagerImpl::beginTxLatencyMeasurement(uint32_t expectedTxCount)
 {
-    if (!mApp.getConfig().LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING)
+    if (!txSelfTrackingActive())
     {
         return;
     }

src/ledger/LedgerManagerImpl.h

@@ -439,13 +439,15 @@ class LedgerManagerImpl : public LedgerManager
     // Local prng for OP_APPLY_SLEEP_TIME_*_FOR_TESTING.
     stellar_default_random_engine mApplySleepRng;
 
-    // Metrics for measuring tx e2e latency. Active only when
-    // Config::LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING is set.
     struct TxLatencyMetrics
     {
         // Lifetime totals (cumulative; not reset between runs).
         medida::Counter& mTxsSubmitted;
         medida::Counter& mTxsExternalized;
+        // Self-submitted txs externalized, split by phase; loadgen's
+        // completion check in overlay-only mode reads these.
+        medida::Counter& mPendingTxsSelfCount;
+        medida::Counter& mPendingSorobanTxsSelfCount;
         // Per-run "loadgen.tx-latency-run.*" statistics (ms), reset by
         // beginTxLatencyMeasurement.
         medida::Counter& mRunMin;
@@ -463,8 +465,7 @@ class LedgerManagerImpl : public LedgerManager
         TxLatencyMetrics(MetricsRegistry& registry);
     } mTxLatencyMetrics;
 
-    // End point of the tx-latency metric: records the submission to post-apply
-    // latency for each externalized transaction.
+    bool txSelfTrackingActive() const;
     void recordTxE2eLatency(ApplicableTxSetFrame const& txSet);
 #endif
 

src/simulation/test/LoadGeneratorTests.cpp

@@ -113,6 +113,61 @@ TEST_CASE("loadgen in overlay-only mode", "[loadgen]")
         500 * simulation->getExpectedLedgerCloseTime(), false);
 }
 
+TEST_CASE("multiple loadgen nodes in overlay-only mode", "[loadgen]")
+{
+    // Regression test: each node's completion check must only count its own
+    // externalized transactions, even while other nodes concurrently generate
+    // load from disjoint account ranges (offsets).
+    Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
+    Simulation::pointer simulation = Topologies::pair(networkID, [&](int i) {
+        auto cfg = getTestConfig(i);
+        configureOverlayV2Pair(cfg, i);
+        cfg.ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING = true;
+        cfg.ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING = true;
+        cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION =
+            Config::CURRENT_LEDGER_PROTOCOL_VERSION;
+        cfg.GENESIS_TEST_ACCOUNT_COUNT = 1000;
+        return cfg;
+    });
+
+    simulation->startAllNodes();
+    simulation->crankUntil(
+        [&]() { return simulation->haveAllExternalized(3, 1); },
+        10 * simulation->getExpectedLedgerCloseTime(), false);
+    auto nodes = simulation->getNodes();
+
+    for (auto& node : nodes)
+    {
+        node->setRunInOverlayOnlyMode(true);
+    }
+
+    uint32_t const nAccountsPerNode = 500;
+    uint32_t const nTxs = 100;
+
+    auto completedRuns = [&](Application& app) {
+        return app.getMetrics()
+            .NewMeter({"loadgen", "run", "complete"}, "run")
+            .count();
+    };
+    auto prev0 = completedRuns(*nodes[0]);
+    auto prev1 = completedRuns(*nodes[1]);
+
+    // Both nodes generate load concurrently, from disjoint account ranges.
+    nodes[0]->getLoadGenerator().generateLoad(
+        GeneratedLoadConfig::txLoad(LoadGenMode::PAY, nAccountsPerNode, nTxs,
+                                    /* txRate */ 100, /* offset */ 0));
+    nodes[1]->getLoadGenerator().generateLoad(GeneratedLoadConfig::txLoad(
+        LoadGenMode::PAY, nAccountsPerNode, nTxs,
+        /* txRate */ 100, /* offset */ nAccountsPerNode));
+
+    simulation->crankUntil(
+        [&]() {
+            return completedRuns(*nodes[0]) == prev0 + 1 &&
+                   completedRuns(*nodes[1]) == prev1 + 1;
+        },
+        500 * simulation->getExpectedLedgerCloseTime(), false);
+}
+
 TEST_CASE("mixed pregen and synthetic soroban in overlay-only mode",
           "[loadgen]")
 {