Merge pull request #2838 from CounterpartyXCP/checkpoints

Add checkpoints

apiary.apib

@@ -1456,7 +1456,7 @@ Returns server information and the list of documented routes in JSON format.
         "result": {
             "server_ready": true,
             "network": "mainnet",
-            "version": "10.8.0-rc.1",
+            "version": "10.8.0-rc.2",
             "backend_height": 850214,
             "counterparty_height": 850214,
             "documentation": "https://counterpartycore.docs.apiary.io/",

counterparty-core/counterpartycore/lib/backend/bitcoind.py

@@ -22,6 +22,7 @@ def rpc_call(payload, retry=0):
     """Calls to bitcoin core and returns the response"""
     url = config.BACKEND_URL
     response = None
+    start_time = time.time()
 
     tries = 0
     broken_error = None
@@ -78,26 +79,31 @@ def rpc_call(payload, retry=0):
 
     # Batch query returns a list
     if isinstance(response_json, list):
-        return response_json
-    if "error" not in response_json.keys() or response_json["error"] is None:  # noqa: E711
-        return response_json["result"]
-    if response_json["error"]["code"] == -5:  # RPC_INVALID_ADDRESS_OR_KEY
+        result = response_json
+    elif "error" not in response_json.keys() or response_json["error"] is None:  # noqa: E711
+        result = response_json["result"]
+    elif response_json["error"]["code"] == -5:  # RPC_INVALID_ADDRESS_OR_KEY
         raise exceptions.BitcoindRPCError(
             f"{response_json['error']} Is `txindex` enabled in {config.BTC_NAME} Core?"
         )
-    if response_json["error"]["code"] in [-28, -8, -2]:
-        # “Verifying blocks...” or “Block height out of range” or “The network does not appear to fully agree!“
+    elif response_json["error"]["code"] in [-28, -8, -2]:
+        # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"
         logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})")
         logger.debug(f"Payload: {payload}")
         if retry >= 10:
             raise exceptions.BitcoindRPCError(
                 f"Backend not ready after {retry} retries. ({response_json['error']})"
             )
         # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970`
-        # seconds to start, this’ll hit the maximum recursion depth limit.
+        # seconds to start, this'll hit the maximum recursion depth limit.
         time.sleep(10)
         return rpc_call(payload, retry=retry + 1)
-    raise exceptions.BitcoindRPCError(response_json["error"]["message"])
+    else:
+        raise exceptions.BitcoindRPCError(response_json["error"]["message"])
+
+    elapsed = time.time() - start_time
+    logger.trace(f"Bitcoin Core RPC call {payload['method']} took {elapsed:.3f}s")
+    return result
 
 
 def rpc(method, params):

counterparty-core/counterpartycore/lib/check.py

@@ -675,6 +675,22 @@
         "ledger_hash": "f214d9ed443e0034e1d3508f016086a841fb89036baae1aae857d273ef2a0b76",
         "txlist_hash": "f378bfec19139f0210808ef8d6c548452cca9e631e5742786d2af7595534106c",
     },
+    872500: {
+        "ledger_hash": "33e10cb33cdc65e8fc2e048410e093f452af6a38c58d1e1dc813ae1c7f4e266c",
+        "txlist_hash": "2f4cf3574a61a7575445dc9ad6f8650750bc0f93a03e586d65ec1c69322009a3",
+    },
+    873000: {
+        "ledger_hash": "73afdb48583a746c524e6459df8e2f208c15487055cc72a8186415ca7a65b077",
+        "txlist_hash": "a52eee6dc316f3fac4c19c48b6b52c3018ba1ba44a6dcf34c52303e9031ec300",
+    },
+    873500: {
+        "ledger_hash": "94b33eaca19f90e99a5a624dacc774d88ed91def33be25526cdfab705cae3b89",
+        "txlist_hash": "e085948ac68caf6b0718373ca80e8fbee5436e88c3c95c3a8fec39d08db8e646",
+    },
+    874548: {
+        "ledger_hash": "8e55b64d0dfd85a58e3a9dd27ce49efd98559d96f16f53beb66a10b7671ea857",
+        "txlist_hash": "b3f549168f56702287c7b06c0348c4ac0adffcd219bab386d2f19326c0cd491c",
+    },
 }
 
 CONSENSUS_HASH_VERSION_TESTNET = 7
@@ -1044,6 +1060,24 @@ def check_need_reparse(version_minor, message):
                 )
 
 
+def check_need_rollback(version_minor, message):
+    if config.FORCE:
+        return
+    need_rollback_from = (
+        config.NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET
+        if config.TESTNET
+        else config.NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN
+    )
+    if need_rollback_from is not None:
+        for min_version_minor, min_version_block_index in need_rollback_from:
+            if version_minor < min_version_minor:
+                raise DatabaseVersionError(
+                    message=message,
+                    required_action="rollback",
+                    from_block_index=min_version_block_index,
+                )
+
+
 def database_version(db):
     if config.FORCE:
         return
@@ -1062,7 +1096,8 @@ def database_version(db):
         message = (
             f"Client minor version number mismatch: {version_minor} ≠ {config.VERSION_MINOR}. "
         )
-        message += "Checking if a reparse is needed..."
+        message += "Checking if a rollback or a reparse is needed..."
+        check_need_rollback(version_minor, message)
         check_need_reparse(version_minor, message)
         raise DatabaseVersionError(message=message, required_action=None)
     else:

counterparty-core/counterpartycore/lib/config.py

@@ -5,7 +5,7 @@
 
 
 # Semantic Version
-__version__ = "10.8.0-rc.1"  # for hatch
+__version__ = "10.8.0-rc.2"  # for hatch
 VERSION_STRING = __version__
 version = VERSION_STRING.split("-")[0].split(".")
 VERSION_MAJOR = int(version[0])
@@ -20,14 +20,16 @@
 # Fo example:
 # NEED_REPARSE_IF_MINOR_IS_LESS_THAN = (1, 800000)
 # means that we need to reparse from block 800000 if database minor version is less than 1
-NEED_REPARSE_IF_MINOR_IS_LESS_THAN = [(3, 0), (5, 865999), (6, 867000), (7, 869900), (8, 871780)]
+NEED_REPARSE_IF_MINOR_IS_LESS_THAN = [(3, 0), (5, 865999), (6, 867000), (7, 869900)]
 NEED_REPARSE_IF_MINOR_IS_LESS_THAN_TESTNET = [
     (3, 0),
     (5, 2925799),
     (6, 2925799),
     (7, 2925799),
-    (8, 3522632),
 ]
+NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN = [(8, 871780)]
+NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET = [(8, 3522632)]
+
 # Counterparty protocol
 TXTYPE_FORMAT = ">I"
 SHORT_TXTYPE_FORMAT = "B"

counterparty-core/counterpartycore/lib/mempool.py

@@ -78,7 +78,7 @@ def parse_mempool_transactions(db, raw_tx_list, timestamps=None):
             # save the events in memory
             transaction_events = cursor.fetchall()
             # we raise an exception to rollback the transaction
-            raise exceptions.MempoolError("Mempool transaction parsed successfully")
+            raise exceptions.MempoolError("Mempool transaction parsed successfully.")
     except exceptions.MempoolError:
         # save events in the mempool table
         for event in transaction_events:
@@ -103,7 +103,7 @@ def parse_mempool_transactions(db, raw_tx_list, timestamps=None):
                 )""",
                 event,
             )
-    logger.trace("Mempool transaction parsed successfully")
+    logger.trace("Mempool transaction parsed successfully.")
     util.PARSING_MEMPOOL = False
 
 
@@ -130,13 +130,15 @@ def parse_raw_mempool(db):
     raw_tx_list = []
     timestamps = {}
     cursor = db.cursor()
+    logger.debug(f"Found {len(raw_mempool)} transaction(s) in the mempool...")
     for txid, tx_info in raw_mempool.items():
         existing_tx_in_mempool = cursor.execute(
             "SELECT * FROM mempool WHERE tx_hash = ? LIMIT 1", (txid,)
         ).fetchone()
         if existing_tx_in_mempool:
             continue
         try:
+            logger.trace(f"Getting raw transaction `{txid}` from the mempool...")
             raw_tx = backend.bitcoind.getrawtransaction(txid)
             raw_tx_list.append(raw_tx)
             timestamps[txid] = tx_info["time"]
@@ -145,4 +147,6 @@ def parse_raw_mempool(db):
                 pass
             else:
                 raise e
+    logger.debug(f"Parsing {len(raw_tx_list)} transaction(s) from the mempool...")
     parse_mempool_transactions(db, raw_tx_list, timestamps)
+    logger.debug("Raw mempool parsed successfully.")

counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md

@@ -165,7 +165,7 @@ Returns server information and the list of documented routes in JSON format.
         "result": {
             "server_ready": true,
             "network": "mainnet",
-            "version": "10.8.0-rc.1",
+            "version": "10.8.0-rc.2",
             "backend_height": 850214,
             "counterparty_height": 850214,
             "documentation": "https://counterpartycore.docs.apiary.io/",

counterparty-core/requirements.txt

@@ -36,4 +36,4 @@ gunicorn==23.0.0
 waitress==3.0.1
 hypothesis==6.116.0
 bitcoin-utils==0.7.1
-counterparty-rs==10.8.0-rc.1
+counterparty-rs==10.8.0-rc.2

counterparty-rs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "counterparty-rs"
-version = "10.8.0-rc.1"
+version = "10.8.0-rc.2"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

counterparty-wallet/requirements.txt

@@ -5,4 +5,4 @@ colorlog==6.8.0
 python-dateutil==2.8.2
 requests==2.32.0
 termcolor==2.4.0
-counterparty-core==10.8.0-rc.1
+counterparty-core==10.8.0-rc.2

docker-compose.yml

@@ -15,7 +15,7 @@ x-addrindexrs-common: &addrindexrs-common
   restart: unless-stopped
 
 x-counterparty-common: &counterparty-common
-  image: counterparty/counterparty:v10.8.0-rc.1
+  image: counterparty/counterparty:v10.8.0-rc.2
   stop_grace_period: 1m
   volumes:
     - data:/root/.bitcoin

release-notes/release-notes-v10.8.0.md

@@ -44,6 +44,9 @@ This upgrade requires a mandatory, automatic reparse from block 871780.
 - Use `multiprocessing.Event` to stop API process when the Ledger process dies
 - Catch up with RPC when ZMQ is late
 - Restart RSFetcher when it returns `None` too many times
+- Exclude transactions by `SIGHASH`
+- Be able to trigger a rollback on a minor version change
+- Add several new checkpoints
 
 ## API