Extend load generator to produce high volume traffic

docs/software/commands.md

@@ -320,24 +320,33 @@ format.
   is started
 
 ### The following HTTP commands are exposed on test instances
-* **generateload**
-  `generateload[?mode=(create|pay|pretend)&accounts=N&offset=K&txs=M&txrate=R&batchsize=L&spikesize=S&spikeinterval=I]`<br>
-  Artificially generate load for testing; must be used with
-  `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING` set to true.
-  * `create` mode creates new accounts.
-    Additionally, allows batching up to 100 account creations per transaction via 'batchsize'.
+* **generateload** `generateload[?mode=
+    (create|pay|pretend)&accounts=N&offset=K&txs=M&txrate=R&batchsize=L&spikesize=S&spikeinterval=I&maxfeerate=F&skiplowfeetxs=(0|1)]`
+
+    Artificially generate load for testing; must be used with
+    `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING` set to true.
+  * `create` mode creates new accounts. Additionally, allows batching up to 100
+    account creations per transaction via `batchsize`.
   * `pay` mode generates `PaymentOp` transactions on accounts specified
     (where the number of accounts can be offset).
-  * `pretend` mode generates transactions on accounts specified
-    (where the number of accounts can be offset). Operations in `pretend` mode are
-    designed to have a realistic size to help users "pretend" that they have real traffic.
+  * `pretend` mode generates transactions on accounts specified(where the number
+    of accounts can be offset). Operations in `pretend` mode are designed to
+    have a realistic size to help users "pretend" that they have real traffic.
     You can add optional configs `LOADGEN_OP_COUNT_FOR_TESTING` and
     `LOADGEN_OP_COUNT_DISTRIBUTION_FOR_TESTING` in the config file to specify
-    the # of ops / tx and how often they appear. More specifically, the probability
-    that a transaction contains `COUNT[i]` ops is
-    `DISTRIBUTION[i] / (DISTRIBUTION[0] + DISTRIBUTION[1] + ...)`.
-
-  For `pay` and `pretend`, when a nonzero I is given, a spike will occur every I seconds injecting S transactions on top of `txrate`.
+    the # of ops / tx and how often they appear. More specifically, the
+    probability that a transaction contains `COUNT[i]` ops is `DISTRIBUTION
+    [i] / (DISTRIBUTION[0] + DISTRIBUTION[1] + ...)`.
+
+  Non-`create` load generation makes use of the additional parameters:
+  * when a nonzero `spikeinterval` is given, a spike will occur every
+    `spikeinterval` seconds injecting `spikesize` transactions on top of
+    `txrate`
+  * `maxfeerate` defines the maximum per-operation fee for generated
+    transactions (when not specified only minimum base fee is used)
+  * when `skiplowfeetxs` is set to `true` the transactions that are not accepted by
+    the node due to having too low fee to pass the rate limiting are silently
+    skipped. Otherwise (by default), such transactions would cause load generation to fail.
 
 * **manualclose**
   If MANUAL_CLOSE is set to true in the .cfg file, this will cause the current

src/herder/Herder.h

@@ -141,6 +141,8 @@ class Herder
                      std::optional<SecretKey> skToSignValue = std::nullopt) = 0;
 
     virtual VirtualTimer const& getTriggerTimer() const = 0;
+
+    virtual TransactionQueue& getTransactionQueue() = 0;
 #endif
     // a peer needs our SCP state
     virtual void sendSCPStateToPeer(uint32 ledgerSeq, Peer::pointer peer) = 0;

src/herder/HerderImpl.h

@@ -168,7 +168,7 @@ class HerderImpl : public Herder
     // used for testing
     PendingEnvelopes& getPendingEnvelopes();
 
-    TransactionQueue& getTransactionQueue();
+    TransactionQueue& getTransactionQueue() override;
 #endif
 
     // helper function to verify envelopes are signed

src/herder/TransactionQueue.cpp

@@ -1272,6 +1272,18 @@ TransactionQueue::getQueueSizeOps() const
 {
     return mTxQueueLimiter->size();
 }
+
+std::optional<int64_t>
+TransactionQueue::getInQueueSeqNum(AccountID const& account) const
+{
+    auto stateIter = mAccountStates.find(account);
+    if (stateIter == mAccountStates.end())
+    {
+        return std::nullopt;
+    }
+    auto& transactions = stateIter->second.mTransactions;
+    return transactions.back().mTx->getSeqNum();
+}
 #endif
 
 size_t

src/herder/TransactionQueue.h

@@ -237,6 +237,7 @@ class TransactionQueue
 #ifdef BUILD_TESTS
   public:
     size_t getQueueSizeOps() const;
+    std::optional<int64_t> getInQueueSeqNum(AccountID const& account) const;
     std::function<void(TransactionFrameBasePtr&)> mTxBroadcastedEvent;
 #endif
 };

src/herder/test/UpgradesTests.cpp

@@ -2537,9 +2537,8 @@ TEST_CASE("upgrade to generalized tx set in network", "[upgrades][overlay]")
 
     auto& loadGen = nodes[0]->getLoadGenerator();
     // Generate 8 ledgers worth of txs (40 / 5).
-    loadGen.generateLoad(LoadGenMode::CREATE, /* nAccounts */ 40, 0, 0,
-                         /*txRate*/ 1,
-                         /*batchSize*/ 1, std::chrono::seconds(0), 0);
+    loadGen.generateLoad(GeneratedLoadConfig::createAccountsLoad(
+        /* nAccounts */ 40, /* txRate */ 1, /* batchSize */ 1));
     auto& loadGenDone =
         nodes[0]->getMetrics().NewMeter({"loadgen", "run", "complete"}, "run");
     auto currLoadGenCount = loadGenDone.count();

src/main/Application.h

@@ -45,6 +45,7 @@ class StatusManager;
 class AbstractLedgerTxnParent;
 class BasicWork;
 enum class LoadGenMode;
+struct GeneratedLoadConfig;
 
 #ifdef BUILD_TESTS
 class LoadGenerator;
@@ -260,11 +261,7 @@ class Application
 #ifdef BUILD_TESTS
     // If config.ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true, generate some load
     // against the current application.
-    virtual void generateLoad(LoadGenMode mode, uint32_t nAccounts,
-                              uint32_t offset, uint32_t nTxs, uint32_t txRate,
-                              uint32_t batchSize,
-                              std::chrono::seconds spikeInterval,
-                              uint32_t spikeSize) = 0;
+    virtual void generateLoad(GeneratedLoadConfig cfg) = 0;
 
     // Access the load generator for manual operation.
     virtual LoadGenerator& getLoadGenerator() = 0;

src/main/ApplicationImpl.cpp

@@ -1004,15 +1004,10 @@ ApplicationImpl::advanceToLedgerBeforeManualCloseTarget(
 
 #ifdef BUILD_TESTS
 void
-ApplicationImpl::generateLoad(LoadGenMode mode, uint32_t nAccounts,
-                              uint32_t offset, uint32_t nTxs, uint32_t txRate,
-                              uint32_t batchSize,
-                              std::chrono::seconds spikeInterval,
-                              uint32_t spikeSize)
+ApplicationImpl::generateLoad(GeneratedLoadConfig cfg)
 {
     getMetrics().NewMeter({"loadgen", "run", "start"}, "run").Mark();
-    getLoadGenerator().generateLoad(mode, nAccounts, offset, nTxs, txRate,
-                                    batchSize, spikeInterval, spikeSize);
+    getLoadGenerator().generateLoad(cfg);
 }
 
 LoadGenerator&

src/main/ApplicationImpl.h

@@ -103,11 +103,7 @@ class ApplicationImpl : public Application
                 std::optional<TimePoint> const& manualCloseTime) override;
 
 #ifdef BUILD_TESTS
-    virtual void generateLoad(LoadGenMode mode, uint32_t nAccounts,
-                              uint32_t offset, uint32_t nTxs, uint32_t txRate,
-                              uint32_t batchSize,
-                              std::chrono::seconds spikeInterval,
-                              uint32_t spikeSize) override;
+    virtual void generateLoad(GeneratedLoadConfig cfg) override;
 
     virtual LoadGenerator& getLoadGenerator() override;
 #endif

src/main/CommandHandler.cpp

@@ -224,6 +224,20 @@ parseOptionalParamOrDefault(std::map<std::string, std::string> const& map,
     }
 }
 
+template <>
+bool
+parseOptionalParamOrDefault<bool>(std::map<std::string, std::string> const& map,
+                                  std::string const& key,
+                                  bool const& defaultValue)
+{
+    auto paramStr = parseOptionalParam<std::string>(map, key);
+    if (!paramStr)
+    {
+        return defaultValue;
+    }
+    return *paramStr == "true";
+}
+
 // Return a value only if the key exists and the value parses.
 // Otherwise, this throws an error.
 template <typename T>
@@ -946,41 +960,51 @@ CommandHandler::generateLoad(std::string const& params, std::string& retStr)
     {
         std::map<std::string, std::string> map;
         http::server::server::parseParams(params, map);
-
-        LoadGenMode mode = LoadGenerator::getMode(
+        GeneratedLoadConfig cfg;
+        cfg.mode = LoadGenerator::getMode(
             parseOptionalParamOrDefault<std::string>(map, "mode", "create"));
-        bool isCreate = mode == LoadGenMode::CREATE;
+        bool isCreate = cfg.mode == LoadGenMode::CREATE;
 
-        uint32_t nAccounts =
+        cfg.nAccounts =
             parseOptionalParamOrDefault<uint32_t>(map, "accounts", 1000);
-        uint32_t nTxs = parseOptionalParamOrDefault<uint32_t>(map, "txs", 0);
-        uint32_t txRate =
-            parseOptionalParamOrDefault<uint32_t>(map, "txrate", 10);
-        uint32_t batchSize = parseOptionalParamOrDefault<uint32_t>(
+        cfg.nTxs = parseOptionalParamOrDefault<uint32_t>(map, "txs", 0);
+        cfg.txRate = parseOptionalParamOrDefault<uint32_t>(map, "txrate", 10);
+        cfg.batchSize = parseOptionalParamOrDefault<uint32_t>(
             map, "batchsize", 100); // Only for account creations
-        uint32_t offset =
-            parseOptionalParamOrDefault<uint32_t>(map, "offset", 0);
+        cfg.offset = parseOptionalParamOrDefault<uint32_t>(map, "offset", 0);
         uint32_t spikeIntervalInt =
             parseOptionalParamOrDefault<uint32_t>(map, "spikeinterval", 0);
-        std::chrono::seconds spikeInterval(spikeIntervalInt);
-        uint32_t spikeSize =
+        cfg.spikeInterval = std::chrono::seconds(spikeIntervalInt);
+        cfg.spikeSize =
             parseOptionalParamOrDefault<uint32_t>(map, "spikesize", 0);
+        cfg.maxGeneratedFeeRate =
+            parseOptionalParam<uint32_t>(map, "maxfeerate");
+        cfg.skipLowFeeTxs =
+            parseOptionalParamOrDefault<bool>(map, "skiplowfeetxs", false);
 
-        uint32_t numItems = isCreate ? nAccounts : nTxs;
-        std::string itemType = isCreate ? "accounts" : "txs";
-
-        if (batchSize > 100)
+        if (cfg.batchSize > 100)
         {
-            batchSize = 100;
+            cfg.batchSize = 100;
             retStr = "Setting batch size to its limit of 100.";
         }
+        if (cfg.maxGeneratedFeeRate)
+        {
+            auto baseFee = mApp.getLedgerManager().getLastTxFee();
+            if (baseFee > *cfg.maxGeneratedFeeRate)
+            {
+                retStr = "maxfeerate is smaller than minimum base fee, load "
+                         "generation skipped.";
+                return;
+            }
+        }
 
-        mApp.generateLoad(mode, nAccounts, offset, nTxs, txRate, batchSize,
-                          spikeInterval, spikeSize);
+        uint32_t numItems = isCreate ? cfg.nAccounts : cfg.nTxs;
+        std::string itemType = isCreate ? "accounts" : "txs";
 
         retStr +=
             fmt::format(FMT_STRING(" Generating load: {:d} {:s}, {:d} tx/s"),
-                        numItems, itemType, txRate);
+                        numItems, itemType, cfg.txRate);
+        mApp.generateLoad(cfg);
     }
     else
     {

src/main/test/ApplicationUtilsTests.cpp

@@ -277,9 +277,10 @@ TEST_CASE("application setup", "[applicationutils]")
 
         // Generate a bit of load, and crank for some time
         auto& loadGen = validator->getLoadGenerator();
-        loadGen.generateLoad(LoadGenMode::CREATE, /* nAccounts */ 10, 0, 0,
-                             /*txRate*/ 1,
-                             /*batchSize*/ 1, std::chrono::seconds(0), 0);
+        loadGen.generateLoad(
+            GeneratedLoadConfig::createAccountsLoad(/* nAccounts */ 10,
+                                                    /* txRate */ 1,
+                                                    /* batchSize */ 1));
 
         auto& loadGenDone = validator->getMetrics().NewMeter(
             {"loadgen", "run", "complete"}, "run");
@@ -383,10 +384,10 @@ TEST_CASE("application setup", "[applicationutils]")
                                           .mCheckpointsDownloaded;
 
                     // Generate a bit of load, and crank for some time
-                    loadGen.generateLoad(
-                        LoadGenMode::PAY, /* nAccounts */ 10, 0, 10,
+                    loadGen.generateLoad(GeneratedLoadConfig::txLoad(
+                        LoadGenMode::PAY, /* nAccounts */ 10, /* nTxs */ 10,
                         /*txRate*/ 1,
-                        /*batchSize*/ 1, std::chrono::seconds(0), 0);
+                        /*batchSize*/ 1));
 
                     auto currLoadGenCount = loadGenDone.count();
 

src/overlay/test/OverlayTests.cpp

@@ -1590,9 +1590,10 @@ TEST_CASE("overlay flow control", "[overlay][flowcontrol]")
         // Generate a bit of load to flood transactions, make sure nodes can
         // close ledgers properly
         auto& loadGen = node->getLoadGenerator();
-        loadGen.generateLoad(LoadGenMode::CREATE, /* nAccounts */ 10, 0, 0,
-                             /*txRate*/ 1,
-                             /*batchSize*/ 1, std::chrono::seconds(0), 0);
+        loadGen.generateLoad(
+            GeneratedLoadConfig::createAccountsLoad(/* nAccounts */ 10,
+                                                    /* txRate */ 1,
+                                                    /* batchSize */ 1));
 
         auto& loadGenDone =
             node->getMetrics().NewMeter({"loadgen", "run", "complete"}, "run");
@@ -2204,9 +2205,10 @@ TEST_CASE("overlay pull mode loadgen", "[overlay][pullmode][acceptance]")
     // Create 5 txns each creating one new account.
     // Set a really high tx rate so we create the txns right away.
     auto const numAccounts = 5;
-    loadGen.generateLoad(LoadGenMode::CREATE, numAccounts, 0, 0,
-                         /*txRate*/ 1000,
-                         /*batchSize*/ 1, std::chrono::seconds(0), 0);
+    loadGen.generateLoad(GeneratedLoadConfig::createAccountsLoad(
+        /* nAccounts */ numAccounts,
+        /* txRate */ 1000,
+        /* batchSize */ 1));
 
     // Let the network close multiple ledgers.
     // If the logic to advertise or demand incorrectly sends more than

src/simulation/CoreTests.cpp

@@ -395,8 +395,10 @@ TEST_CASE(
     auto& app = *nodes[0]; // pick a node to generate load
 
     auto& loadGen = app.getLoadGenerator();
-    loadGen.generateLoad(LoadGenMode::CREATE, 3, 0, 0, 10, 100,
-                         std::chrono::seconds(0), 0);
+    loadGen.generateLoad(GeneratedLoadConfig::createAccountsLoad(
+        /* nAccounts */ 3,
+        /* txRate */ 10,
+        /* batchSize */ 100));
     try
     {
         simulation->crankUntil(
@@ -409,8 +411,8 @@ TEST_CASE(
             },
             3 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false);
 
-        loadGen.generateLoad(LoadGenMode::PAY, 3, 0, 10, 10, 100,
-                             std::chrono::seconds(0), 0);
+        loadGen.generateLoad(
+            GeneratedLoadConfig::txLoad(LoadGenMode::PAY, 3, 10, 10, 100));
         simulation->crankUntil(
             [&]() {
                 return simulation->haveAllExternalized(8, 2) &&
@@ -523,8 +525,10 @@ TEST_CASE("Accounts vs latency", "[scalability][!hide]")
     uint32_t numItems = 500000;
 
     // Create accounts
-    loadGen.generateLoad(LoadGenMode::CREATE, numItems, 0, 0, 10, 100,
-                         std::chrono::seconds(0), 0);
+    loadGen.generateLoad(GeneratedLoadConfig::createAccountsLoad(
+        /* nAccounts */ 10,
+        /* txRate */ 10,
+        /* batchSize */ 100));
 
     auto& complete =
         appPtr->getMetrics().NewMeter({"loadgen", "run", "complete"}, "run");
@@ -539,8 +543,8 @@ TEST_CASE("Accounts vs latency", "[scalability][!hide]")
     txtime.Clear();
 
     // Generate payment txs
-    loadGen.generateLoad(LoadGenMode::PAY, numItems, 0, numItems / 10, 10, 100,
-                         std::chrono::seconds(0), 0);
+    loadGen.generateLoad(GeneratedLoadConfig::txLoad(LoadGenMode::PAY, numItems,
+                                                     numItems / 10, 10, 100));
     while (!io.stopped() && complete.count() == 1)
     {
         clock.crank();
@@ -574,8 +578,10 @@ netTopologyTest(std::string const& name,
         auto& app = *nodes[0];
 
         auto& loadGen = app.getLoadGenerator();
-        loadGen.generateLoad(LoadGenMode::CREATE, 50, 0, 0, 10, 100,
-                             std::chrono::seconds(0), 0);
+        loadGen.generateLoad(GeneratedLoadConfig::createAccountsLoad(
+            /* nAccounts */ 50,
+            /* txRate */ 10,
+            /* batchSize */ 100));
         auto& complete =
             app.getMetrics().NewMeter({"loadgen", "run", "complete"}, "run");