Merge #17775: DecodeHexTx: Try case where txn has inputs first

27fc6a3 DecodeHexTx: Break out transaction decoding logic into own function (Gregory Sanders)
6020ce3 DecodeHexTx: Try case where txn has inputs first (Gregory Sanders)

Pull request description:

  Alternative/complementary to bitcoin/bitcoin#17773 to avoid random `decoderawtransaction` failures. Most cases this is used now is on complete transactions, especially with the uptake of PSBT.

ACKs for top commit:
  ajtowns:
    ACK 27fc6a3
  achow101:
    ACK 27fc6a3

Tree-SHA512: 0a836d7c9951bf7d2764507788dbcc871d520f1ea9b77d6b22f051f4d6224ed779aba0e4f28c5c165040095ee0c70b67080c39164d82de61b19158f7ae6fddb2

src/core_read.cpp

@@ -117,28 +117,23 @@ static bool CheckTxScriptsSanity(const CMutableTransaction& tx)
     return true;
 }
 
-bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness, bool try_witness)
+static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& tx_data, bool try_no_witness, bool try_witness)
 {
-    if (!IsHex(hex_tx)) {
-        return false;
-    }
-
-    std::vector<unsigned char> txData(ParseHex(hex_tx));
-
-    if (try_no_witness) {
-        CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS);
+    if (try_witness) {
+        CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION);
         try {
             ssData >> tx;
-            if (ssData.eof() && (!try_witness || CheckTxScriptsSanity(tx))) {
+            // If transaction looks sane, we don't try other mode even if requested
+            if (ssData.empty() && (!try_no_witness || CheckTxScriptsSanity(tx))) {
                 return true;
             }
         } catch (const std::exception&) {
             // Fall through.
         }
     }
 
-    if (try_witness) {
-        CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION);
+    if (try_no_witness) {
+        CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS);
         try {
             ssData >> tx;
             if (ssData.empty()) {
@@ -152,6 +147,16 @@ bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no
     return false;
 }
 
+bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness, bool try_witness)
+{
+    if (!IsHex(hex_tx)) {
+        return false;
+    }
+
+    std::vector<unsigned char> txData(ParseHex(hex_tx));
+    return DecodeTx(tx, txData, try_no_witness, try_witness);
+}
+
 bool DecodeHexBlockHeader(CBlockHeader& header, const std::string& hex_header)
 {
     if (!IsHex(hex_header)) return false;