Introduce parameters to load generator to skip low fee txs and genera…

…te diverse fees.

While it's currently possible to trigger surge pricing with load generator, the request need to fit exactly into 1x and 2x the tx set size, which is tricky if we want to also randomize operation counts.

This mode also allows to exercise the tx queue limiter, which is useful too.

docs/software/commands.md

@@ -320,24 +320,33 @@ format.
   is started
 
 ### The following HTTP commands are exposed on test instances
-* **generateload**
-  `generateload[?mode=(create|pay|pretend)&accounts=N&offset=K&txs=M&txrate=R&batchsize=L&spikesize=S&spikeinterval=I]`<br>
-  Artificially generate load for testing; must be used with
-  `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING` set to true.
-  * `create` mode creates new accounts.
-    Additionally, allows batching up to 100 account creations per transaction via 'batchsize'.
+* **generateload** `generateload[?mode=
+    (create|pay|pretend)&accounts=N&offset=K&txs=M&txrate=R&batchsize=L&spikesize=S&spikeinterval=I&maxfeerate=F&skiplowfeetxs=(0|1)]`
+
+    Artificially generate load for testing; must be used with
+    `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING` set to true.
+  * `create` mode creates new accounts. Additionally, allows batching up to 100
+    account creations per transaction via `batchsize`.
   * `pay` mode generates `PaymentOp` transactions on accounts specified
     (where the number of accounts can be offset).
-  * `pretend` mode generates transactions on accounts specified
-    (where the number of accounts can be offset). Operations in `pretend` mode are
-    designed to have a realistic size to help users "pretend" that they have real traffic.
+  * `pretend` mode generates transactions on accounts specified(where the number
+    of accounts can be offset). Operations in `pretend` mode are designed to
+    have a realistic size to help users "pretend" that they have real traffic.
     You can add optional configs `LOADGEN_OP_COUNT_FOR_TESTING` and
     `LOADGEN_OP_COUNT_DISTRIBUTION_FOR_TESTING` in the config file to specify
-    the # of ops / tx and how often they appear. More specifically, the probability
-    that a transaction contains `COUNT[i]` ops is
-    `DISTRIBUTION[i] / (DISTRIBUTION[0] + DISTRIBUTION[1] + ...)`.
-
-  For `pay` and `pretend`, when a nonzero I is given, a spike will occur every I seconds injecting S transactions on top of `txrate`.
+    the # of ops / tx and how often they appear. More specifically, the
+    probability that a transaction contains `COUNT[i]` ops is `DISTRIBUTION
+    [i] / (DISTRIBUTION[0] + DISTRIBUTION[1] + ...)`.
+
+  Non-`create` load generation makes use of the additional parameters:
+  * when a nonzero `spikeinterval` is given, a spike will occur every
+    `spikeinterval` seconds injecting `spikesize` transactions on top of
+    `txrate`
+  * `maxfeerate` defines the maximum per-operation fee for generated
+    transactions (when not specified only minimum base fee is used)
+  * when `skiplowfeetxs` is set to `true` the transactions that are not accepted by
+    the node due to having too low fee to pass the rate limiting are silently
+    skipped. Otherwise (by default), such transactions would cause load generation to fail.
 
 * **manualclose**
   If MANUAL_CLOSE is set to true in the .cfg file, this will cause the current

src/herder/Herder.h

@@ -141,6 +141,8 @@ class Herder
                      std::optional<SecretKey> skToSignValue = std::nullopt) = 0;
 
     virtual VirtualTimer const& getTriggerTimer() const = 0;
+
+    virtual TransactionQueue& getTransactionQueue() = 0;
 #endif
     // a peer needs our SCP state
     virtual void sendSCPStateToPeer(uint32 ledgerSeq, Peer::pointer peer) = 0;

src/herder/HerderImpl.h

@@ -168,7 +168,7 @@ class HerderImpl : public Herder
     // used for testing
     PendingEnvelopes& getPendingEnvelopes();
 
-    TransactionQueue& getTransactionQueue();
+    TransactionQueue& getTransactionQueue() override;
 #endif
 
     // helper function to verify envelopes are signed

src/herder/TransactionQueue.cpp

@@ -1272,6 +1272,18 @@ TransactionQueue::getQueueSizeOps() const
 {
     return mTxQueueLimiter->size();
 }
+
+std::optional<int64_t>
+TransactionQueue::getInQueueSeqNum(AccountID const& account) const
+{
+    auto stateIter = mAccountStates.find(account);
+    if (stateIter == mAccountStates.end())
+    {
+        return std::nullopt;
+    }
+    auto& transactions = stateIter->second.mTransactions;
+    return transactions.back().mTx->getSeqNum();
+}
 #endif
 
 size_t

src/herder/TransactionQueue.h

@@ -237,6 +237,7 @@ class TransactionQueue
 #ifdef BUILD_TESTS
   public:
     size_t getQueueSizeOps() const;
+    std::optional<int64_t> getInQueueSeqNum(AccountID const& account) const;
     std::function<void(TransactionFrameBasePtr&)> mTxBroadcastedEvent;
 #endif
 };

src/main/CommandHandler.cpp

@@ -224,6 +224,20 @@ parseOptionalParamOrDefault(std::map<std::string, std::string> const& map,
     }
 }
 
+template <>
+bool
+parseOptionalParamOrDefault<bool>(std::map<std::string, std::string> const& map,
+                                  std::string const& key,
+                                  bool const& defaultValue)
+{
+    auto paramStr = parseOptionalParam<std::string>(map, key);
+    if (!paramStr)
+    {
+        return defaultValue;
+    }
+    return *paramStr == "true";
+}
+
 // Return a value only if the key exists and the value parses.
 // Otherwise, this throws an error.
 template <typename T>
@@ -963,12 +977,26 @@ CommandHandler::generateLoad(std::string const& params, std::string& retStr)
         cfg.spikeInterval = std::chrono::seconds(spikeIntervalInt);
         cfg.spikeSize =
             parseOptionalParamOrDefault<uint32_t>(map, "spikesize", 0);
+        cfg.maxGeneratedFeeRate =
+            parseOptionalParam<uint32_t>(map, "maxfeerate");
+        cfg.skipLowFeeTxs =
+            parseOptionalParamOrDefault<bool>(map, "skiplowfeetxs", false);
 
         if (cfg.batchSize > 100)
         {
             cfg.batchSize = 100;
             retStr = "Setting batch size to its limit of 100.";
         }
+        if (cfg.maxGeneratedFeeRate)
+        {
+            auto baseFee = mApp.getLedgerManager().getLastTxFee();
+            if (baseFee > *cfg.maxGeneratedFeeRate)
+            {
+                retStr = "maxfeerate is smaller than minimum base fee, load "
+                         "generation skipped.";
+                return;
+            }
+        }
 
         uint32_t numItems = isCreate ? cfg.nAccounts : cfg.nTxs;
         std::string itemType = isCreate ? "accounts" : "txs";