From 093cbc825d8945bdf76e94eaff48bd9b9a3eabf7 Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 5 Mar 2019 17:39:17 -0500 Subject: [PATCH 1/6] beginning of unit test --- tests/tests/history_api_tests.cpp | 151 ++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/tests/tests/history_api_tests.cpp b/tests/tests/history_api_tests.cpp index b5b8ef8199..b18daeb9ea 100644 --- a/tests/tests/history_api_tests.cpp +++ b/tests/tests/history_api_tests.cpp @@ -25,6 +25,7 @@ #include #include +#include #include @@ -604,4 +605,154 @@ BOOST_AUTO_TEST_CASE(get_account_history_operations) { } } +BOOST_AUTO_TEST_CASE( market_history ) +{ + class simple_tx + { + public: + simple_tx() {} + simple_tx( + const graphene::chain::account_object& from, + const graphene::chain::account_object& to, + const graphene::chain::price& amount) + : from(from), to(to), amount(amount) {} + const graphene::chain::account_object to; + const graphene::chain::account_object from; + const graphene::chain::price amount; + }; + class time_series + { + public: + time_series() {} + time_series(uint32_t secs, std::vector transactions) : secs(secs), transactions(transactions) {} + uint32_t secs; + std::vector transactions; + }; + try { + app.enable_plugin("market_history"); + graphene::app::history_api hist_api( app ); + + // create the needed things on the chain + ACTORS( (bob) (alice) ); + transfer( committee_account, alice.id, asset(1000000000) ); + transfer( committee_account, bob.id , asset(1000000000) ); + const graphene::chain::asset_object& usd = create_bitasset( "USD", account_id_type() ); + // set up feed producers + { + asset_update_feed_producers_operation op; + op.asset_to_update = usd.id; + op.issuer = committee_account; + op.new_feed_producers = {committee_account, alice.id, bob.id}; + trx.operations.push_back(op); + sign( trx, private_key ); + PUSH_TX( db, trx, ~0 ); + generate_block(); + trx.clear(); + publish_feed( committee_account, asset_id_type(), 2, usd.id, 1, asset_id_type() ); + publish_feed( alice.id, asset_id_type(), 1, usd.id, 2, asset_id_type() ); + publish_feed( bob.id, asset_id_type(), 2, usd.id, 1, asset_id_type() ); + } + const graphene::chain::asset_object& cny = create_bitasset( "CNY", account_id_type() ); + // set up feed producers + { + asset_update_feed_producers_operation op; + op.asset_to_update = cny.id; + op.issuer = committee_account; + op.new_feed_producers = {committee_account, alice.id, bob.id}; + trx.operations.push_back(op); + sign( trx, private_key ); + PUSH_TX( db, trx, ~0 ); + generate_block(); + trx.clear(); + publish_feed( committee_account, asset_id_type(), 1, cny.id, 2, asset_id_type() ); + publish_feed( alice.id, asset_id_type(), 1, cny.id, 2, asset_id_type() ); + publish_feed( bob.id, asset_id_type(), 2, cny.id, 1, asset_id_type() ); + } + generate_blocks( db.get_dynamic_global_properties().next_maintenance_time ); + borrow( alice, asset(100000, usd.id), asset(1000000) ); + borrow( bob , asset(100000, usd.id), asset(1000000) ); + borrow( alice, asset(100000, cny.id), asset(1000000) ); + borrow( bob , asset(100000, cny.id), asset(1000000) ); + + // get bucket size + boost::container::flat_set bucket_sizes = hist_api.get_market_history_buckets(); + uint32_t bucket_size = *bucket_sizes.begin(); // 15 secs when I checked. + + // make some transaction data + uint32_t current_price = 5000; + + // a function to transmit a transaction + auto transmit_tx = [this, alice, bob, alice_private_key, bob_private_key](const simple_tx& tx) { + const auto seller_private_key = ( tx.from.id == alice.id ? alice_private_key : bob_private_key ); + const auto buyer_private_key = ( tx.from.id == alice.id ? bob_private_key : alice_private_key ); + // create an order + graphene::chain::limit_order_create_operation limit; + limit.amount_to_sell = tx.amount.base; + limit.min_to_receive = tx.amount.quote; + limit.fill_or_kill = false; + limit.seller = tx.from.id; + limit.expiration = fc::time_point_sec( fc::time_point::now().sec_since_epoch() + 60 ); + // send the order + set_expiration( db, trx ); + trx.operations.push_back( limit ); + sign( trx, seller_private_key ); + PUSH_TX( db, trx, ~0 ); + trx.clear(); + // send a second order that will fill the first + graphene::chain::limit_order_create_operation buy; + buy.amount_to_sell = tx.amount.quote; + buy.min_to_receive = tx.amount.base; + buy.fill_or_kill = false; + buy.seller = tx.to.id; + buy.expiration = limit.expiration; + set_expiration( db, trx ); + trx.operations.push_back( buy ); + sign( trx, buyer_private_key ); + PUSH_TX( db, trx, ~0 ); + trx.clear(); + }; + + uint32_t num_bars = 10; + std::vector bars(num_bars); + + generate_blocks( HARDFORK_CORE_625_TIME ); + + for( uint32_t i = 0; i < num_bars; ++i ) // each bucket + { + uint32_t num_transactions = std::rand() % 100; + std::vector trxs(num_transactions); + for( int tx_num = 0; tx_num < num_transactions; ++tx_num ) + { + bool alice_buys = std::rand() % 2; + bool buy_usd = std::rand() % 2; + current_price = ( current_price + ( std::rand() % 2 ? 1 : -1) ); + simple_tx stx( + ( alice_buys ? alice : bob ), + ( alice_buys ? bob : alice ), + graphene::chain::price( + graphene::chain::asset(1, (buy_usd ? cny.id : usd.id ) ), + graphene::chain::asset(current_price, (buy_usd ? usd.id : cny.id ) ) + ) + ); + trxs.push_back(stx); + // send matching buy and sell to the server + transmit_tx(stx); + } + generate_block(); + bars.push_back( time_series(i, trxs) ); + } + // grab history + auto server_history = hist_api.get_market_history( + static_cast(usd.id), // asset_id_type + static_cast(cny.id), // asset_id_type + bucket_size, // bucket_seconds + fc::time_point_sec(0), // start + fc::time_point_sec(db.head_block_time()) ); // end + // compare what comes back with what we think we should have + BOOST_CHECK_EQUAL( server_history.size(), 0 ); + } catch ( fc::exception &e ) { + BOOST_FAIL( e.to_detail_string() ); + } +} + BOOST_AUTO_TEST_SUITE_END() From 72062a170b2b33a55978e89af7a8557735ea3dce Mon Sep 17 00:00:00 2001 From: John Jones Date: Thu, 7 Mar 2019 20:11:24 -0500 Subject: [PATCH 2/6] more information in test --- tests/tests/history_api_tests.cpp | 43 +++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/tests/tests/history_api_tests.cpp b/tests/tests/history_api_tests.cpp index b18daeb9ea..79ee556be1 100644 --- a/tests/tests/history_api_tests.cpp +++ b/tests/tests/history_api_tests.cpp @@ -605,6 +605,11 @@ BOOST_AUTO_TEST_CASE(get_account_history_operations) { } } +/*** + * This was an attempt to create market history. But due + * to the way it is done, it will take to long to generate + * the data needed. + */ BOOST_AUTO_TEST_CASE( market_history ) { class simple_tx @@ -634,8 +639,8 @@ BOOST_AUTO_TEST_CASE( market_history ) // create the needed things on the chain ACTORS( (bob) (alice) ); - transfer( committee_account, alice.id, asset(1000000000) ); - transfer( committee_account, bob.id , asset(1000000000) ); + transfer( committee_account, alice.id, asset(10000000000) ); + transfer( committee_account, bob.id , asset(10000000000) ); const graphene::chain::asset_object& usd = create_bitasset( "USD", account_id_type() ); // set up feed producers { @@ -669,17 +674,18 @@ BOOST_AUTO_TEST_CASE( market_history ) publish_feed( bob.id, asset_id_type(), 2, cny.id, 1, asset_id_type() ); } generate_blocks( db.get_dynamic_global_properties().next_maintenance_time ); - borrow( alice, asset(100000, usd.id), asset(1000000) ); - borrow( bob , asset(100000, usd.id), asset(1000000) ); - borrow( alice, asset(100000, cny.id), asset(1000000) ); - borrow( bob , asset(100000, cny.id), asset(1000000) ); + borrow( alice, asset(1000000, usd.id), asset(10000000) ); + borrow( bob , asset(1000000, usd.id), asset(10000000) ); + borrow( alice, asset(1000000, cny.id), asset(10000000) ); + borrow( bob , asset(1000000, cny.id), asset(10000000) ); // get bucket size boost::container::flat_set bucket_sizes = hist_api.get_market_history_buckets(); uint32_t bucket_size = *bucket_sizes.begin(); // 15 secs when I checked. + BOOST_TEST_MESSAGE( "Bucket size: " + std::to_string(bucket_size) ); // make some transaction data - uint32_t current_price = 5000; + uint32_t current_price = 500; // a function to transmit a transaction auto transmit_tx = [this, alice, bob, alice_private_key, bob_private_key](const simple_tx& tx) { @@ -712,7 +718,7 @@ BOOST_AUTO_TEST_CASE( market_history ) trx.clear(); }; - uint32_t num_bars = 10; + uint32_t num_bars = 800; std::vector bars(num_bars); generate_blocks( HARDFORK_CORE_625_TIME ); @@ -738,7 +744,13 @@ BOOST_AUTO_TEST_CASE( market_history ) // send matching buy and sell to the server transmit_tx(stx); } - generate_block(); + graphene::chain::signed_block blk = generate_block(); + if ( (i+1) % 10 == 0) + { + BOOST_TEST_MESSAGE( "Block " + std::to_string( (i+1)) + + " of " + std::to_string(num_bars) + + " created with " + std::to_string(blk.transactions.size()) + " transactions." ); + } bars.push_back( time_series(i, trxs) ); } // grab history @@ -748,8 +760,17 @@ BOOST_AUTO_TEST_CASE( market_history ) bucket_size, // bucket_seconds fc::time_point_sec(0), // start fc::time_point_sec(db.head_block_time()) ); // end - // compare what comes back with what we think we should have - BOOST_CHECK_EQUAL( server_history.size(), 0 ); + BOOST_CHECK_EQUAL( server_history.size(), 200 ); + // grab the last traunch of history + auto last_date = (*(--server_history.end())).key.open; + auto server_history2 = hist_api.get_market_history( + static_cast(usd.id), // asset_id_type + static_cast(cny.id), // asset_id_type + bucket_size, // bucket_seconds + fc::time_point_sec(last_date + bucket_size), // start + fc::time_point_sec(db.head_block_time()) ); // end + BOOST_CHECK_EQUAL( server_history2.size(), 67 ); + // now match up the data to assure it is correct. } catch ( fc::exception &e ) { BOOST_FAIL( e.to_detail_string() ); } From 79531f1a233d664a61ab606d3ae2f654ef54a66a Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 11 Mar 2019 09:35:42 -0500 Subject: [PATCH 3/6] Test working --- tests/tests/history_api_tests.cpp | 138 ++++++++++++++++++------------ 1 file changed, 85 insertions(+), 53 deletions(-) diff --git a/tests/tests/history_api_tests.cpp b/tests/tests/history_api_tests.cpp index 79ee556be1..b57ccd2e43 100644 --- a/tests/tests/history_api_tests.cpp +++ b/tests/tests/history_api_tests.cpp @@ -606,9 +606,10 @@ BOOST_AUTO_TEST_CASE(get_account_history_operations) { } /*** - * This was an attempt to create market history. But due - * to the way it is done, it will take to long to generate - * the data needed. + * This will create market history. But due to the way + * the server handles it, it will take to long to generate + * the data needed. This test should normally be commented + * out and used when needed. */ BOOST_AUTO_TEST_CASE( market_history ) { @@ -621,18 +622,10 @@ BOOST_AUTO_TEST_CASE( market_history ) const graphene::chain::account_object& to, const graphene::chain::price& amount) : from(from), to(to), amount(amount) {} - const graphene::chain::account_object to; const graphene::chain::account_object from; + const graphene::chain::account_object to; const graphene::chain::price amount; }; - class time_series - { - public: - time_series() {} - time_series(uint32_t secs, std::vector transactions) : secs(secs), transactions(transactions) {} - uint32_t secs; - std::vector transactions; - }; try { app.enable_plugin("market_history"); graphene::app::history_api hist_api( app ); @@ -647,8 +640,8 @@ BOOST_AUTO_TEST_CASE( market_history ) asset_update_feed_producers_operation op; op.asset_to_update = usd.id; op.issuer = committee_account; - op.new_feed_producers = {committee_account, alice.id, bob.id}; - trx.operations.push_back(op); + op.new_feed_producers = { committee_account, alice.id, bob.id }; + trx.operations.push_back( op ); sign( trx, private_key ); PUSH_TX( db, trx, ~0 ); generate_block(); @@ -663,8 +656,8 @@ BOOST_AUTO_TEST_CASE( market_history ) asset_update_feed_producers_operation op; op.asset_to_update = cny.id; op.issuer = committee_account; - op.new_feed_producers = {committee_account, alice.id, bob.id}; - trx.operations.push_back(op); + op.new_feed_producers = { committee_account, alice.id, bob.id }; + trx.operations.push_back( op ); sign( trx, private_key ); PUSH_TX( db, trx, ~0 ); generate_block(); @@ -682,10 +675,7 @@ BOOST_AUTO_TEST_CASE( market_history ) // get bucket size boost::container::flat_set bucket_sizes = hist_api.get_market_history_buckets(); uint32_t bucket_size = *bucket_sizes.begin(); // 15 secs when I checked. - BOOST_TEST_MESSAGE( "Bucket size: " + std::to_string(bucket_size) ); - - // make some transaction data - uint32_t current_price = 500; + BOOST_TEST_MESSAGE( "Creating transaction data. This will take some time. Bucket size: " + std::to_string(bucket_size) ); // a function to transmit a transaction auto transmit_tx = [this, alice, bob, alice_private_key, bob_private_key](const simple_tx& tx) { @@ -718,59 +708,101 @@ BOOST_AUTO_TEST_CASE( market_history ) trx.clear(); }; - uint32_t num_bars = 800; - std::vector bars(num_bars); - generate_blocks( HARDFORK_CORE_625_TIME ); - for( uint32_t i = 0; i < num_bars; ++i ) // each bucket + // make some transaction data + uint32_t current_price = 500; + uint32_t num_estimated_bars = 800; + + std::vector trxs; + uint64_t total_transactions_sent = 0; + // make a dead spot 15 blocks in for 4 blocks + for( uint32_t i = 0; i < num_estimated_bars; ++i ) // each (hopeful) bar { - uint32_t num_transactions = std::rand() % 100; - std::vector trxs(num_transactions); - for( int tx_num = 0; tx_num < num_transactions; ++tx_num ) + if ( i >= 15 && i < 19 ) { - bool alice_buys = std::rand() % 2; - bool buy_usd = std::rand() % 2; - current_price = ( current_price + ( std::rand() % 2 ? 1 : -1) ); - simple_tx stx( - ( alice_buys ? alice : bob ), - ( alice_buys ? bob : alice ), - graphene::chain::price( - graphene::chain::asset(1, (buy_usd ? cny.id : usd.id ) ), - graphene::chain::asset(current_price, (buy_usd ? usd.id : cny.id ) ) - ) - ); - trxs.push_back(stx); - // send matching buy and sell to the server - transmit_tx(stx); + // put something in the block so it is not empty + transfer( committee_account, bob.id , asset(1 * GRAPHENE_BLOCKCHAIN_PRECISION) ); } + else + { + uint32_t num_transactions = std::rand() % 100; + for( uint32_t tx_num = 0; tx_num < num_transactions; ++tx_num ) + { + bool alice_buys = std::rand() % 2; + bool buy_usd = std::rand() % 2; + current_price = ( current_price + ( std::rand() % 2 ? 1 : -1) ); + simple_tx stx( + ( alice_buys ? alice : bob ), + ( alice_buys ? bob : alice ), + graphene::chain::price( + graphene::chain::asset( 1, (buy_usd ? cny.id : usd.id ) ), + graphene::chain::asset( current_price, (buy_usd ? usd.id : cny.id ) ) + ) + ); + trxs.push_back( stx ); + // send matching buy and sell to the server + transmit_tx( stx ); + } + } + graphene::chain::signed_block blk = generate_block(); if ( (i+1) % 10 == 0) { + uint64_t transactions_sent_this_block = trxs.size() - total_transactions_sent; BOOST_TEST_MESSAGE( "Block " + std::to_string( (i+1)) - + " of " + std::to_string(num_bars) - + " created with " + std::to_string(blk.transactions.size()) + " transactions." ); + + " of " + std::to_string(num_estimated_bars) + + " created with " + std::to_string(transactions_sent_this_block) + " transactions." ); + total_transactions_sent = trxs.size(); } - bars.push_back( time_series(i, trxs) ); } // grab history auto server_history = hist_api.get_market_history( - static_cast(usd.id), // asset_id_type - static_cast(cny.id), // asset_id_type + static_cast( usd.id ), // asset_id_type + static_cast( cny.id ), // asset_id_type bucket_size, // bucket_seconds - fc::time_point_sec(0), // start - fc::time_point_sec(db.head_block_time()) ); // end + fc::time_point_sec( 0 ), // start + fc::time_point_sec( db.head_block_time() ) ); // end BOOST_CHECK_EQUAL( server_history.size(), 200 ); // grab the last traunch of history auto last_date = (*(--server_history.end())).key.open; auto server_history2 = hist_api.get_market_history( - static_cast(usd.id), // asset_id_type - static_cast(cny.id), // asset_id_type + static_cast(usd.id ), // asset_id_type + static_cast(cny.id ), // asset_id_type bucket_size, // bucket_seconds - fc::time_point_sec(last_date + bucket_size), // start - fc::time_point_sec(db.head_block_time()) ); // end + fc::time_point_sec( last_date + bucket_size ), // start + fc::time_point_sec( db.head_block_time() ) ); // end BOOST_CHECK_EQUAL( server_history2.size(), 67 ); - // now match up the data to assure it is correct. + // combine the two buckets + server_history.insert( server_history.end(), server_history2.begin(), server_history2.end() ); + // now match up the data to ensure it is correct. + size_t starting_trx_number = 0; + for(size_t bucket_number = 0; bucket_number < server_history.size(); ++bucket_number) + { + bucket_object current_bucket = server_history[bucket_number]; + int64_t cny_volume_total = 0; + for( size_t trx_number = starting_trx_number; trx_number < trxs.size(); ++trx_number ) + { + starting_trx_number = trx_number + 1; + auto tx = trxs[trx_number]; + if ( tx.amount.base.asset_id == usd.id ) + cny_volume_total += tx.amount.quote.amount.value; + else + cny_volume_total += tx.amount.base.amount.value; + if ( cny_volume_total > current_bucket.quote_volume.value ) + { + BOOST_FAIL( "Volume mismatch. Series volume: " + std::to_string(cny_volume_total) + + " Transaction number: " + std::to_string( trx_number ) + + " Bucket quote volume: " + std::to_string( current_bucket.quote_volume.value) + + " Bucket base volume: " + std::to_string( current_bucket.base_volume.value) + + " Bucket number: " + std::to_string( bucket_number ) ); + } + if ( cny_volume_total == current_bucket.quote_volume.value ) + { + break; + } + } // transaction loop + } // server_history loop } catch ( fc::exception &e ) { BOOST_FAIL( e.to_detail_string() ); } From 84e7636b4b51a47f1ed8afc230db963e780126ce Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 11 Mar 2019 10:41:02 -0500 Subject: [PATCH 4/6] Make output less verbose --- tests/tests/history_api_tests.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tests/history_api_tests.cpp b/tests/tests/history_api_tests.cpp index b57ccd2e43..76094effd1 100644 --- a/tests/tests/history_api_tests.cpp +++ b/tests/tests/history_api_tests.cpp @@ -715,7 +715,7 @@ BOOST_AUTO_TEST_CASE( market_history ) uint32_t num_estimated_bars = 800; std::vector trxs; - uint64_t total_transactions_sent = 0; + //uint64_t total_transactions_sent = 0; // make a dead spot 15 blocks in for 4 blocks for( uint32_t i = 0; i < num_estimated_bars; ++i ) // each (hopeful) bar { @@ -747,6 +747,7 @@ BOOST_AUTO_TEST_CASE( market_history ) } graphene::chain::signed_block blk = generate_block(); + /* if ( (i+1) % 10 == 0) { uint64_t transactions_sent_this_block = trxs.size() - total_transactions_sent; @@ -755,6 +756,7 @@ BOOST_AUTO_TEST_CASE( market_history ) + " created with " + std::to_string(transactions_sent_this_block) + " transactions." ); total_transactions_sent = trxs.size(); } + */ } // grab history auto server_history = hist_api.get_market_history( From 7c3101283db89f42677debab18909aa00d8dd796 Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 25 Mar 2019 12:23:59 -0500 Subject: [PATCH 5/6] Remove unnecessary trx signing --- tests/tests/history_api_tests.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/tests/history_api_tests.cpp b/tests/tests/history_api_tests.cpp index 76094effd1..9420d09677 100644 --- a/tests/tests/history_api_tests.cpp +++ b/tests/tests/history_api_tests.cpp @@ -642,7 +642,6 @@ BOOST_AUTO_TEST_CASE( market_history ) op.issuer = committee_account; op.new_feed_producers = { committee_account, alice.id, bob.id }; trx.operations.push_back( op ); - sign( trx, private_key ); PUSH_TX( db, trx, ~0 ); generate_block(); trx.clear(); @@ -658,7 +657,6 @@ BOOST_AUTO_TEST_CASE( market_history ) op.issuer = committee_account; op.new_feed_producers = { committee_account, alice.id, bob.id }; trx.operations.push_back( op ); - sign( trx, private_key ); PUSH_TX( db, trx, ~0 ); generate_block(); trx.clear(); @@ -691,7 +689,6 @@ BOOST_AUTO_TEST_CASE( market_history ) // send the order set_expiration( db, trx ); trx.operations.push_back( limit ); - sign( trx, seller_private_key ); PUSH_TX( db, trx, ~0 ); trx.clear(); // send a second order that will fill the first @@ -703,7 +700,6 @@ BOOST_AUTO_TEST_CASE( market_history ) buy.expiration = limit.expiration; set_expiration( db, trx ); trx.operations.push_back( buy ); - sign( trx, buyer_private_key ); PUSH_TX( db, trx, ~0 ); trx.clear(); }; From 21a53c74ac743103045a86e952cb8859a9834ce6 Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 25 Mar 2019 12:26:32 -0500 Subject: [PATCH 6/6] Remove unused code --- tests/tests/history_api_tests.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/tests/history_api_tests.cpp b/tests/tests/history_api_tests.cpp index 9420d09677..24d97153e4 100644 --- a/tests/tests/history_api_tests.cpp +++ b/tests/tests/history_api_tests.cpp @@ -711,7 +711,6 @@ BOOST_AUTO_TEST_CASE( market_history ) uint32_t num_estimated_bars = 800; std::vector trxs; - //uint64_t total_transactions_sent = 0; // make a dead spot 15 blocks in for 4 blocks for( uint32_t i = 0; i < num_estimated_bars; ++i ) // each (hopeful) bar { @@ -743,16 +742,6 @@ BOOST_AUTO_TEST_CASE( market_history ) } graphene::chain::signed_block blk = generate_block(); - /* - if ( (i+1) % 10 == 0) - { - uint64_t transactions_sent_this_block = trxs.size() - total_transactions_sent; - BOOST_TEST_MESSAGE( "Block " + std::to_string( (i+1)) - + " of " + std::to_string(num_estimated_bars) - + " created with " + std::to_string(transactions_sent_this_block) + " transactions." ); - total_transactions_sent = trxs.size(); - } - */ } // grab history auto server_history = hist_api.get_market_history(