Merge pull request #133 from CounterpartyXCP/develop

1.3.0

ChangeLog.md

@@ -0,0 +1,41 @@
+## Client Versions ##
+* v1.3.0 (2015-10-31)
+    * Fixes periodic `blockfeed` hanging issue (where `counterblock` would still run, but not process new blocks from `counterparty-server`)
+    * Block processing is much more robust now if an exception is encountered (e.g. counterparty-server goes down). Should prevent additional hanging-type issues
+    * Tweaked `blockfeed` "caught up" checking logic. Should be more reliable
+    * Simplified `blockchain` module -- we call API methods on `counterparty-server` now, whereever possible, instead of reimplementing them on `counterblock`
+    * Enhance the information returned with `GET /_api`. Several new parameters added, including `ERROR` for easier diagnosing of most common error conditions.
+    * `GET /_api` now returns CORS headers, allowing it to be used with cross domain requests
+    * Added this `ChangeLog.md` file
+* v1.2.0 (2015-09-15)
+    * Move most counterblock functionality into plug-in modules.
+    * Pegs the pymongo library version, to avoid incompatibility issues with pymongo 3.0
+    * Improves exception logging for exceptions that happen within a greenlet thread context
+    * Fixes an issue with an exception on reorg processing (from counterparty-server's message feed).
+    * Modifies the state flow with rollbacks to streamline and simplify things a bit
+* 1.1.1 (2015-03-23)
+    * Fix some (uncaught) runtime errors that can cause significant stability problems in Counterblock in certain cases.
+    * Properly implements logging of uncaught errors to the counterblock log files.
+* 1.1.0 (2015-02-06)
+    * Updated to work with the new architecture of counterparty 9.49.4 (the configuration options have been adjusted to be like counterparty, e.g. no more --data-dir)
+    * new configuration file locations by default
+    * Added setup.py for setuptools / pip
+* 1.0.1 (2015-01-23)
+    * block state variable naming change
+    * get_jsonrpc_api() fix with abort_on_error=False
+    * bug fix with URL fetching, abort_on_error setting
+    * Fix for ambivalent variable name
+    * fix division per zero
+* 1.0.0 (2015-01-05)
+    * MAJOR: Added plugin (modular) functionality. counterblockd can now be easily extended to support custom functionality
+    * Increased JSON API request timeout to 120 seconds
+    * Implemented support for new order_match id format
+    * Implemented always trying/retrying for RPC calls
+    * Removed Callback and RPS
+    * Modularized Counterblockd functionality & plugin interface for third-party modules
+    * Optimized blockfeeds.py
+    * Fixed the difference of one satoshi between BTC balances returned by counterpartyd and counterblockd
+    * Implemented an alternative for counterpartyd api get_asset_info() method to speed up the login in counterwallet for wallet with a lot of assets
+    * Updated versions of deps (fixes issue with fetching SSL urls)
+    * Fixed the issue with passing JSON data in POST requests
+    * Added rollback command line argument and RollbackProcessor

README.md

@@ -1,3 +1,5 @@
+[![Slack Status](http://slack.counterparty.io/badge.svg)](http://slack.counterparty.io)
+
 counterblockd
 ==============
 

counterblock/lib/blockchain.py

@@ -8,8 +8,6 @@
 import decimal
 
 from repoze.lru import lru_cache
-import bitcoin as bitcoinlib
-import bitcoin.rpc as bitcoin_rpc
 from pycoin import encoding
 
 from counterblock.lib import config, util
@@ -74,9 +72,6 @@ def get_btc_balance(address, confirmed=True):
     unspent = confirmed_unspent if confirmed else all_unspent
     return sum(out['amount'] for out in unspent)
 
-def check():
-    pass
-
 def listunspent(address):
     outputs = get_unspent_txouts(address)
     utxo = []
@@ -119,27 +114,30 @@ def getaddressinfo(address):
         'unconfirmedBalanceSat': str(unconfirmed_balance * config.UNIT),
         'transactions': transactions
     }
-
-    return None
 
+def gettransaction_batch(txhash_list):
+    raw_txes = util.call_jsonrpc_api("getrawtransaction_batch", {'txhash_list': txhash_list, 'verbose': True}, abort_on_error=True)['result']
+    txes = {}
+    for tx_hash, tx in raw_txes.iteritems():
+        valueOut = 0
+        for vout in tx['vout']:
+            valueOut += vout['value']
+        txes[tx_hash] = {
+            'txid': tx_hash,
+            'version': tx['version'],
+            'locktime': tx['locktime'],
+            'confirmations': tx['confirmations'] if 'confirmations' in tx else 0,
+            'blocktime': tx['blocktime'] if 'blocktime' in tx else 0,
+            'blockhash': tx['blockhash'] if 'blockhash' in tx else 0,
+            'time': tx['time'] if 'time' in tx else 0,
+            'valueOut': valueOut,
+            'vin': tx['vin'],
+            'vout': tx['vout']
+        }
+    return txes
+
 def gettransaction(tx_hash):
-    tx = get_cached_raw_transaction(tx_hash)
-    valueOut = 0
-    for vout in tx['vout']:
-        valueOut += vout['value']
-    return {
-        'txid': tx_hash,
-        'version': tx['version'],
-        'locktime': tx['locktime'],
-        'confirmations': tx['confirmations'] if 'confirmations' in tx else 0,
-        'blocktime': tx['blocktime'] if 'blocktime' in tx else 0,
-        'blockhash': tx['blockhash'] if 'blockhash' in tx else 0,
-        'time': tx['time'] if 'time' in tx else 0,
-        'valueOut': valueOut,
-        'vin': tx['vin'],
-        'vout': tx['vout']
-    }
-    return None
+    return gettransaction_batch([tx_hash,])[tx_hash]
 
 def get_pubkey_from_transactions(address, raw_transactions):
     #for each transaction we got back, extract the vin, pubkey, go through, convert it to binary, and see if it reduces down to the given address
@@ -172,250 +170,18 @@ def get_pubkey_for_address(address):
 
     return pubkeys
 
-
-### Unconfirmed Transactions ###
-
-# cache
-UNCONFIRMED_ADDRINDEX = {}
-OLD_MEMPOOL = []
-
-@lru_cache(maxsize=4096)
-def get_cached_raw_transaction(tx_hash):
-    return bitcoind_rpc('getrawtransaction', [tx_hash, 1])
-
-@lru_cache(maxsize=8192)
-def get_cached_batch_raw_transactions(tx_hashes):
-    tx_hashes = json.loads(tx_hashes) # for lru_cache
-    call_id = 0
-    call_list = []
-    for tx_hash in tx_hashes:
-        call_list.append({
-            "method": 'getrawtransaction',
-            "params": [tx_hash, 1],
-            "jsonrpc": "2.0",
-            "id": call_id
-        })
-        call_id += 1
-
-    if config.TESTNET:
-        bitcoinlib.SelectParams('testnet')
-    proxy = bitcoin_rpc.Proxy(service_url=config.BACKEND_URL)
-    return proxy._batch(call_list)
-
-# TODO: use scriptpubkey_to_address()
-@lru_cache(maxsize=4096)
-def extract_addresses(tx):
-    tx = json.loads(tx) # for lru_cache
-    addresses = []
-
-    for vout in tx['vout']:
-        if 'addresses' in vout['scriptPubKey']:
-            addresses += vout['scriptPubKey']['addresses']
-
-    vin_hashes = [vin['txid'] for vin in tx['vin']]        
-    batch_responses = get_cached_batch_raw_transactions(json.dumps(vin_hashes))
-    tx_dict = {}
-    for response in batch_responses:
-        if 'error' not in response or response['error'] is None:
-            if 'result' in response and response['result'] is not None:
-                vin_tx = response['result']
-                tx_dict[vin_tx['txid']] = vin_tx
-
-    for vin in tx['vin']:
-        vin_tx = tx_dict[vin['txid']]
-        vout = vin_tx['vout'][vin['vout']]
-        if 'addresses' in vout['scriptPubKey']:
-            addresses += vout['scriptPubKey']['addresses']
-
-    return addresses
-
-class DecimalEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, D):
-            return format(obj, '.8f')
-        # Let the base class default method raise the TypeError
-        return json.JSONEncoder.default(self, obj)
-
-def add_tx_to_addrindex(tx):
-    global UNCONFIRMED_ADDRINDEX
-
-    addresses = extract_addresses(json.dumps(tx, cls=DecimalEncoder))
-    for address in addresses:
-        if address not in UNCONFIRMED_ADDRINDEX:
-            UNCONFIRMED_ADDRINDEX[address] = {}
-        UNCONFIRMED_ADDRINDEX[address][tx['txid']] = tx
-
-def remove_tx_from_addrindex(tx_hash):
-    global UNCONFIRMED_ADDRINDEX
-
-    for address in list(UNCONFIRMED_ADDRINDEX.keys()):
-        if tx_hash in UNCONFIRMED_ADDRINDEX[address]:
-            UNCONFIRMED_ADDRINDEX[address].pop(tx_hash)
-            if len(UNCONFIRMED_ADDRINDEX[address]) == 0:
-                UNCONFIRMED_ADDRINDEX.pop(address)
-
-def unconfirmed_transactions(address):
-    global UNCONFIRMED_ADDRINDEX
-
-    if address in UNCONFIRMED_ADDRINDEX:
-        return list(UNCONFIRMED_ADDRINDEX[address].values())
-    else:
-        return []
-
-def update_unconfirmed_addrindex():
-    global OLD_MEMPOOL
-
-    new_mempool = bitcoind_rpc('getrawmempool', [])
-
-    # remove confirmed txs
-    for tx_hash in OLD_MEMPOOL:
-        if tx_hash not in new_mempool:
-            remove_tx_from_addrindex(tx_hash)
-
-    # add new txs
-    new_tx_hashes = []
-    for tx_hash in new_mempool:
-        if tx_hash not in OLD_MEMPOOL:
-            new_tx_hashes.append(tx_hash)
-
-    if len(new_tx_hashes) > 0:
-        batch_responses = get_cached_batch_raw_transactions(json.dumps(new_tx_hashes))
-        for response in batch_responses:
-            if 'error' not in response or response['error'] is None:
-                if 'result' in response and response['result'] is not None:
-                    tx = response['result']
-                    add_tx_to_addrindex(tx)
-
-    OLD_MEMPOOL = list(new_mempool)
-
-def search_raw_transactions(address):
-    unconfirmed = unconfirmed_transactions(address)
-    try:
-        rawtransactions = bitcoind_rpc('searchrawtransactions', [address, 1, 0, 9999999])
-    except Exception, e:
-        if str(e).find('404') != -1:
-            raise Exception('Unknown RPC command: searchrawtransactions. Switch to jmcorgan.')
-        else:
-            raise Exception(str(e))
-    confirmed = [tx for tx in rawtransactions if tx['confirmations'] > 0]
-    return unconfirmed + confirmed
-
-### Multi-signature Addresses ###
-# NOTE: a pub is either a pubkey or a pubkeyhash
-
-class MultiSigAddressError (Exception):
-    pass
-
-def is_multisig(address):
-    array = address.split('_')
-    return (len(array) > 1)
-
-def canonical_address(address):
-    if is_multisig(address):
-        signatures_required, pubkeyhashes, signatures_possible = extract_array(address)
-        return construct_array(signatures_required, pubkeyhashes, signatures_possible)
-    else:
-        return address
-
-def test_array(signatures_required, pubs, signatures_possible):
-    try:
-        signatures_required, signatures_possible = int(signatures_required), int(signatures_possible)
-    except ValueError:
-        raise MultiSigAddressError('Signature values not integers.')
-    if signatures_required < 1 or signatures_required > 3:
-        raise MultiSigAddressError('Invalid signatures_required.')
-    if signatures_possible < 2 or signatures_possible > 3:
-        raise MultiSigAddressError('Invalid signatures_possible.')
-    if signatures_possible != len(pubs):
-        raise MultiSigAddressError('Incorrect number of pubkeys/pubkeyhashes in multi-signature address.')
-
-def construct_array(signatures_required, pubs, signatures_possible):
-    test_array(signatures_required, pubs, signatures_possible)
-    address = '_'.join([str(signatures_required)] + sorted(pubs) + [str(signatures_possible)])
-    return address
-
-def extract_array(address):
-    assert is_multisig(address)
-    array = address.split('_')
-    signatures_required, pubs, signatures_possible = array[0], sorted(array[1:-1]), array[-1]
-    test_array(signatures_required, pubs, signatures_possible)
-    return int(signatures_required), pubs, int(signatures_possible)
-
-def pubkeyhash_array(address):
-    signatures_required, pubkeyhashes, signatures_possible = extract_array(address)
-    return pubkeyhashes
-
-### Multi-signature Addresses ###
-
-def scriptpubkey_to_canonical_address(scriptpubkey):
-    if scriptpubkey['type'] == 'multisig':
-        asm = scriptpubkey['asm'].split(' ')
-        signatures_required = asm[0]
-        signatures_possible = len(asm) - 3
-        return "_".join([str(signatures_required)] + sorted(scriptpubkey['addresses']) + [str(signatures_possible)])
-    elif 'addresses' in scriptpubkey:
-        return scriptpubkey['addresses'][0]
-    else:
-        return None
-
+def search_raw_transactions(address, unconfirmed=True):
+    return util.call_jsonrpc_api("search_raw_transactions", {'address': address, 'unconfirmed': unconfirmed}, abort_on_error=True)['result']
 
 def get_unspent_txouts(source, return_confirmed=False):
     """returns a list of unspent outputs for a specific address
     @return: A list of dicts, with each entry in the dict having the following keys:
     """
-    # Get all coins.
-    outputs = {}
-    if is_multisig(source):
-        pubkeyhashes = pubkeyhash_array(source)
-        raw_transactions = search_raw_transactions(pubkeyhashes[1])
-    else:
-        pubkeyhashes = [source]
-        raw_transactions = search_raw_transactions(source)
-
-    canonical_source = canonical_address(source)
-
-    for tx in raw_transactions:
-        for vout in tx['vout']:
-            scriptpubkey = vout['scriptPubKey']
-            if scriptpubkey_to_canonical_address(scriptpubkey) == canonical_source:
-                txid = tx['txid']
-                confirmations = tx['confirmations'] if 'confirmations' in tx else 0
-                outkey = '{}{}'.format(txid, vout['n'])
-                if outkey not in outputs or outputs[outkey]['confirmations'] < confirmations:
-                    coin = {'amount': float(vout['value']),
-                            'confirmations': confirmations,
-                            'scriptPubKey': scriptpubkey['hex'],
-                            'txid': txid,
-                            'vout': vout['n']
-                           }
-                    outputs[outkey] = coin
-    outputs = outputs.values()
-
-    # Prune away spent coins.
-    unspent = []
-    confirmed_unspent = []
-    for output in outputs:
-        spent = False
-        confirmed_spent = False
-        for tx in raw_transactions:
-            for vin in tx['vin']:
-                if 'coinbase' in vin: continue
-                if (vin['txid'], vin['vout']) == (output['txid'], output['vout']):
-                    spent = True
-                    if 'confirmations' in tx and tx['confirmations'] > 0:
-                        confirmed_spent = True
-        if not spent:
-            unspent.append(output)
-        if not confirmed_spent and output['confirmations'] > 0:
-            confirmed_unspent.append(output)
-
-    unspent = sorted(unspent, key=lambda x: x['txid'])
-    confirmed_unspent = sorted(confirmed_unspent, key=lambda x: x['txid'])
-
+    txouts = util.call_jsonrpc_api("get_unspent_txouts", {'address': source, 'unconfirmed': True}, abort_on_error=True)['result']
     if return_confirmed:
-        return unspent, confirmed_unspent
+        return txouts, [output for output in txouts if output['confirmations'] > 0]
     else:
-        return unspent
+        return txouts
 
 def broadcast_tx(signed_tx_hex):
     return bitcoind_rpc('sendrawtransaction', [signed_tx_hex])