From 0d2d66eff2c1b34aede30ab48b7380d83bfe59b1 Mon Sep 17 00:00:00 2001 From: chefsale Date: Tue, 29 Jun 2021 13:11:44 +0200 Subject: [PATCH 1/5] feat(cli): Implemented the download config method to the init command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I have tested three scenarios: 1) default config which generates the random chain id 2) betanet config which is generated 3) betanet config which is downloaded and should have tracked shards = [0] and boot_nodes set. Tested for the test chain command: ``` sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> ./target/debug/neard init Jun 29 13:12:05.580 INFO neard: Version: 1.2.0, Build: 7e0fe60a-modified, Latest Protocol: 45 Jun 29 13:12:05.593 INFO near: Generated node key, validator key, genesis file in /home/sandi/.near sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> cat /home/sandi/.near/config.json { "genesis_file": "genesis.json", "genesis_records_file": null, "validator_key_file": "validator_key.json", "node_key_file": "node_key.json", "rpc": { "addr": "0.0.0.0:3030", "cors_allowed_origins": [ "*" ], "polling_config": { "polling_interval": { "secs": 0, "nanos": 500000000 }, "polling_timeout": { "secs": 10, "nanos": 0 } }, "limits_config": { "json_payload_max_size": 10485760 } }, "telemetry": { "endpoints": [] }, "network": { "addr": "0.0.0.0:24567", "external_address": "", "boot_nodes": "", "max_num_peers": 40, "minimum_outbound_peers": 5, "ideal_connections_lo": 30, "ideal_connections_hi": 35, "peer_recent_time_window": { "secs": 600, "nanos": 0 }, "safe_set_size": 20, "archival_peer_connections_lower_bound": 10, "handshake_timeout": { "secs": 20, "nanos": 0 }, "reconnect_delay": { "secs": 60, "nanos": 0 }, "skip_sync_wait": true, "ban_window": { "secs": 10800, "nanos": 0 }, "blacklist": [], "ttl_account_id_router": { "secs": 3600, "nanos": 0 }, "peer_stats_period": { "secs": 5, "nanos": 0 } }, "consensus": { "min_num_peers": 3, "block_production_tracking_delay": { "secs": 0, "nanos": 100000000 }, "min_block_production_delay": { "secs": 0, "nanos": 600000000 }, "max_block_production_delay": { "secs": 2, "nanos": 0 }, "max_block_wait_delay": { "secs": 6, "nanos": 0 }, "reduce_wait_for_missing_block": { "secs": 0, "nanos": 100000000 }, "produce_empty_blocks": true, "block_fetch_horizon": 50, "state_fetch_horizon": 5, "block_header_fetch_horizon": 50, "catchup_step_period": { "secs": 0, "nanos": 100000000 }, "chunk_request_retry_period": { "secs": 0, "nanos": 400000000 }, "header_sync_initial_timeout": { "secs": 10, "nanos": 0 }, "header_sync_progress_timeout": { "secs": 2, "nanos": 0 }, "header_sync_stall_ban_timeout": { "secs": 120, "nanos": 0 }, "state_sync_timeout": { "secs": 60, "nanos": 0 }, "header_sync_expected_height_per_second": 10, "sync_check_period": { "secs": 10, "nanos": 0 }, "sync_step_period": { "secs": 0, "nanos": 10000000 }, "doomslug_step_period": { "secs": 0, "nanos": 100000000 } }, "tracked_accounts": [], "tracked_shards": [], "archive": false, "log_summary_style": "colored", "gc_blocks_limit": 2, "view_client_threads": 4, "epoch_sync_enabled": true, "view_client_throttle_period": { "secs": 30, "nanos": 0 }, "trie_viewer_state_size_limit": 50000 }⏎ sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> cat /home/sandi/.near/genesis.json { "protocol_version": 45, "genesis_time": "2021-06-29T11:12:05.592321543Z", "chain_id": "test-chain-t2VXm", "genesis_height": 0, "num_block_producer_seats": 50, "num_block_producer_seats_per_shard": [ 50 ], "avg_hidden_validator_seats_per_shard": [ 0 ], "dynamic_resharding": false, "protocol_upgrade_stake_threshold": [ 4, 5 ], "protocol_upgrade_num_epochs": 2, "epoch_length": 500, "gas_limit": 1000000000000000, "min_gas_price": "1000000000", "max_gas_price": "10000000000000000000000", "block_producer_kickout_threshold": 90, "chunk_producer_kickout_threshold": 90, "online_min_threshold": [ 9, 10 ], "online_max_threshold": [ 99, 100 ], "gas_price_adjustment_rate": [ 1, 100 ], "runtime_config": { "storage_amount_per_byte": "90900000000000000000", "transaction_costs": { "action_receipt_creation_config": { "send_sir": 108059500000, "send_not_sir": 108059500000, "execution": 108059500000 }, "data_receipt_creation_config": { "base_cost": { "send_sir": 4697339419375, "send_not_sir": 4697339419375, "execution": 4697339419375 }, "cost_per_byte": { "send_sir": 59357464, "send_not_sir": 59357464, "execution": 59357464 } }, "action_creation_config": { "create_account_cost": { "send_sir": 99607375000, "send_not_sir": 99607375000, "execution": 99607375000 }, "deploy_contract_cost": { "send_sir": 184765750000, "send_not_sir": 184765750000, "execution": 184765750000 }, "deploy_contract_cost_per_byte": { "send_sir": 6812999, "send_not_sir": 6812999, "execution": 6812999 }, "function_call_cost": { "send_sir": 2319861500000, "send_not_sir": 2319861500000, "execution": 2319861500000 }, "function_call_cost_per_byte": { "send_sir": 2235934, "send_not_sir": 2235934, "execution": 2235934 }, "transfer_cost": { "send_sir": 115123062500, "send_not_sir": 115123062500, "execution": 115123062500 }, "stake_cost": { "send_sir": 141715687500, "send_not_sir": 141715687500, "execution": 102217625000 }, "add_key_cost": { "full_access_cost": { "send_sir": 101765125000, "send_not_sir": 101765125000, "execution": 101765125000 }, "function_call_cost": { "send_sir": 102217625000, "send_not_sir": 102217625000, "execution": 102217625000 }, "function_call_cost_per_byte": { "send_sir": 1925331, "send_not_sir": 1925331, "execution": 1925331 } }, "delete_key_cost": { "send_sir": 94946625000, "send_not_sir": 94946625000, "execution": 94946625000 }, "delete_account_cost": { "send_sir": 147489000000, "send_not_sir": 147489000000, "execution": 147489000000 } }, "storage_usage_config": { "num_bytes_account": 100, "num_extra_bytes_record": 40 }, "burnt_gas_reward": [ 3, 10 ], "pessimistic_gas_price_inflation_ratio": [ 103, 100 ] }, "wasm_config": { "ext_costs": { "base": 264768111, "contract_compile_base": 35445963, "contract_compile_bytes": 216750, "read_memory_base": 2609863200, "read_memory_byte": 3801333, "write_memory_base": 2803794861, "write_memory_byte": 2723772, "read_register_base": 2517165186, "read_register_byte": 98562, "write_register_base": 2865522486, "write_register_byte": 3801564, "utf8_decoding_base": 3111779061, "utf8_decoding_byte": 291580479, "utf16_decoding_base": 3543313050, "utf16_decoding_byte": 163577493, "sha256_base": 4540970250, "sha256_byte": 24117351, "keccak256_base": 5879491275, "keccak256_byte": 21471105, "keccak512_base": 5811388236, "keccak512_byte": 36649701, "log_base": 3543313050, "log_byte": 13198791, "storage_write_base": 64196736000, "storage_write_key_byte": 70482867, "storage_write_value_byte": 31018539, "storage_write_evicted_byte": 32117307, "storage_read_base": 56356845750, "storage_read_key_byte": 30952533, "storage_read_value_byte": 5611005, "storage_remove_base": 53473030500, "storage_remove_key_byte": 38220384, "storage_remove_ret_value_byte": 11531556, "storage_has_key_base": 54039896625, "storage_has_key_byte": 30790845, "storage_iter_create_prefix_base": 0, "storage_iter_create_prefix_byte": 0, "storage_iter_create_range_base": 0, "storage_iter_create_from_byte": 0, "storage_iter_create_to_byte": 0, "storage_iter_next_base": 0, "storage_iter_next_key_byte": 0, "storage_iter_next_value_byte": 0, "touching_trie_node": 16101955926, "promise_and_base": 1465013400, "promise_and_per_promise": 5452176, "promise_return": 560152386, "validator_stake_base": 911834726400, "validator_total_stake_base": 911834726400 }, "grow_mem_cost": 1, "regular_op_cost": 3856371, "limit_config": { "max_gas_burnt": 200000000000000, "max_gas_burnt_view": 200000000000000, "max_stack_height": 16384, "initial_memory_pages": 1024, "max_memory_pages": 2048, "registers_memory_limit": 1073741824, "max_register_size": 104857600, "max_number_registers": 100, "max_number_logs": 100, "max_total_log_length": 16384, "max_total_prepaid_gas": 300000000000000, "max_actions_per_receipt": 100, "max_number_bytes_method_names": 2000, "max_length_method_name": 256, "max_arguments_length": 4194304, "max_length_returned_data": 4194304, "max_contract_size": 4194304, "max_length_storage_key": 4194304, "max_length_storage_value": 4194304, "max_promises_per_function_call_action": 1024, "max_number_input_data_dependencies": 128 } }, "account_creation_config": { "min_allowed_top_level_account_length": 0, "registrar_account_id": "registrar" } }, "validators": [ { "account_id": "test.near", "public_key": "ed25519:98jzeQmcPrVUjwrAawRsFotA5V4FyouDNsLWLM7W8koe", "amount": "50000000000000000000000000000000" } ], "transaction_validity_period": 100, "protocol_reward_rate": [ 1, 10 ], "max_inflation_rate": [ 1, 20 ], "total_supply": "2050000000000000000000000000000000", "num_blocks_per_year": 31536000, "protocol_treasury_account": "test.near", "fishermen_threshold": "10000000000000000000000000", "minimum_stake_divisor": 10, "records": [ { "Account": { "account_id": "test.near", "account": { "amount": "1000000000000000000000000000000000", "locked": "50000000000000000000000000000000", "code_hash": "11111111111111111111111111111111", "storage_usage": 0 } } }, { "AccessKey": { "account_id": "test.near", "public_key": "ed25519:98jzeQmcPrVUjwrAawRsFotA5V4FyouDNsLWLM7W8koe", "access_key": { "nonce": 0, "permission": "FullAccess" } } }, { "Account": { "account_id": "near", "account": { "amount": "1000000000000000000000000000000000", "locked": "0", "code_hash": "11111111111111111111111111111111", "storage_usage": 0 } } }, { "AccessKey": { "account_id": "near", "public_key": "ed25519:546XB2oHhj7PzUKHiH9Xve3Ze5q1JiW2WTh6abXFED3c", "access_key": { "nonce": 0, "permission": "FullAccess" } } } ] } ``` Tested for default betanet init: ``` sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> rm -rf ~/.near/ sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> ./target/debug/neard init --chain-id betanet --download-genesis Jun 29 13:14:38.613 INFO neard: Version: 1.2.0, Build: 7e0fe60a-modified, Latest Protocol: 45 Jun 29 13:14:38.614 INFO near: Downloading genesis file from: https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/betanet/genesis.json ... Jun 29 13:15:06.188 INFO near: Saved the genesis file to: /home/sandi/.near/genesis.json ... Jun 29 13:15:13.541 INFO near: Generated for betanet network node key and genesis file in /home/sandi/.near sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> cat /home/sandi/.near/config.json { "genesis_file": "genesis.json", "genesis_records_file": null, "validator_key_file": "validator_key.json", "node_key_file": "node_key.json", "rpc": { "addr": "0.0.0.0:3030", "cors_allowed_origins": [ "*" ], "polling_config": { "polling_interval": { "secs": 0, "nanos": 500000000 }, "polling_timeout": { "secs": 10, "nanos": 0 } }, "limits_config": { "json_payload_max_size": 10485760 } }, "telemetry": { "endpoints": [ "https://explorer.betanet.near.org/api/nodes" ] }, "network": { "addr": "0.0.0.0:24567", "external_address": "", "boot_nodes": "", "max_num_peers": 40, "minimum_outbound_peers": 5, "ideal_connections_lo": 30, "ideal_connections_hi": 35, "peer_recent_time_window": { "secs": 600, "nanos": 0 }, "safe_set_size": 20, "archival_peer_connections_lower_bound": 10, "handshake_timeout": { "secs": 20, "nanos": 0 }, "reconnect_delay": { "secs": 60, "nanos": 0 }, "skip_sync_wait": false, "ban_window": { "secs": 10800, "nanos": 0 }, "blacklist": [], "ttl_account_id_router": { "secs": 3600, "nanos": 0 }, "peer_stats_period": { "secs": 5, "nanos": 0 } }, "consensus": { "min_num_peers": 3, "block_production_tracking_delay": { "secs": 0, "nanos": 100000000 }, "min_block_production_delay": { "secs": 0, "nanos": 600000000 }, "max_block_production_delay": { "secs": 2, "nanos": 0 }, "max_block_wait_delay": { "secs": 6, "nanos": 0 }, "reduce_wait_for_missing_block": { "secs": 0, "nanos": 100000000 }, "produce_empty_blocks": true, "block_fetch_horizon": 50, "state_fetch_horizon": 5, "block_header_fetch_horizon": 50, "catchup_step_period": { "secs": 0, "nanos": 100000000 }, "chunk_request_retry_period": { "secs": 0, "nanos": 400000000 }, "header_sync_initial_timeout": { "secs": 10, "nanos": 0 }, "header_sync_progress_timeout": { "secs": 2, "nanos": 0 }, "header_sync_stall_ban_timeout": { "secs": 120, "nanos": 0 }, "state_sync_timeout": { "secs": 60, "nanos": 0 }, "header_sync_expected_height_per_second": 10, "sync_check_period": { "secs": 10, "nanos": 0 }, "sync_step_period": { "secs": 0, "nanos": 10000000 }, "doomslug_step_period": { "secs": 0, "nanos": 100000000 } }, "tracked_accounts": [], "tracked_shards": [], "archive": false, "log_summary_style": "colored", "gc_blocks_limit": 2, "view_client_threads": 4, "epoch_sync_enabled": true, "view_client_throttle_period": { "secs": 30, "nanos": 0 }, "trie_viewer_state_size_limit": 50000 }⏎ ``` Tested on betanet with the download config flag set: ``` sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> rm -rf ~/.near/ sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> ./target/debug/neard init --chain-id betanet --download-genesis --download-config Jun 29 13:19:12.545 INFO neard: Version: 1.2.0, Build: 3f3986d4-modified, Latest Protocol: 45 Jun 29 13:19:12.545 INFO near: Downloading config file from: https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/betanet/config.json ... Jun 29 13:19:13.476 INFO near: Saved the config file to: /home/sandi/.near/config.json ... Jun 29 13:19:13.479 INFO near: Downloading genesis file from: https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/betanet/genesis.json ... Jun 29 13:19:41.038 INFO near: Saved the genesis file to: /home/sandi/.near/genesis.json ... Jun 29 13:19:48.309 INFO near: Generated for betanet network node key and genesis file in /home/sandi/.near sandi@sandi-ThinkPad-X1-Carbon-7th ~/w/n/nearcore (download-config)> cat /home/sandi/.near/config.json { "genesis_file": "genesis.json", "genesis_records_file": null, "validator_key_file": "validator_key.json", "node_key_file": "node_key.json", "rpc": { "addr": "0.0.0.0:3030", "cors_allowed_origins": [ "*" ], "polling_config": { "polling_interval": { "secs": 0, "nanos": 500000000 }, "polling_timeout": { "secs": 10, "nanos": 0 } }, "limits_config": { "json_payload_max_size": 10485760 } }, "telemetry": { "endpoints": [ "https://explorer.betanet.near.org/api/nodes" ] }, "network": { "addr": "0.0.0.0:24567", "external_address": "", "boot_nodes": "ed25519:7PGseFbWxvYVgZ89K1uTJKYoKetWs7BJtbyXDzfbAcqX@34.94.17.55:24567,ed25519:6DSjZ8mvsRZDvFqFxo8tCKePG96omXW7eVYVSySmDk8e@35.196.178.178:24567", "max_num_peers": 40, "minimum_outbound_peers": 5, "ideal_connections_lo": 30, "ideal_connections_hi": 35, "peer_recent_time_window": { "secs": 600, "nanos": 0 }, "safe_set_size": 20, "archival_peer_connections_lower_bound": 10, "handshake_timeout": { "secs": 20, "nanos": 0 }, "reconnect_delay": { "secs": 60, "nanos": 0 }, "skip_sync_wait": false, "ban_window": { "secs": 10800, "nanos": 0 }, "blacklist": [], "ttl_account_id_router": { "secs": 3600, "nanos": 0 }, "peer_stats_period": { "secs": 5, "nanos": 0 } }, "consensus": { "min_num_peers": 3, "block_production_tracking_delay": { "secs": 0, "nanos": 100000000 }, "min_block_production_delay": { "secs": 0, "nanos": 600000000 }, "max_block_production_delay": { "secs": 2, "nanos": 0 }, "max_block_wait_delay": { "secs": 6, "nanos": 0 }, "reduce_wait_for_missing_block": { "secs": 0, "nanos": 100000000 }, "produce_empty_blocks": true, "block_fetch_horizon": 50, "state_fetch_horizon": 5, "block_header_fetch_horizon": 50, "catchup_step_period": { "secs": 0, "nanos": 100000000 }, "chunk_request_retry_period": { "secs": 0, "nanos": 400000000 }, "header_sync_initial_timeout": { "secs": 10, "nanos": 0 }, "header_sync_progress_timeout": { "secs": 2, "nanos": 0 }, "header_sync_stall_ban_timeout": { "secs": 120, "nanos": 0 }, "state_sync_timeout": { "secs": 60, "nanos": 0 }, "header_sync_expected_height_per_second": 10, "sync_check_period": { "secs": 10, "nanos": 0 }, "sync_step_period": { "secs": 0, "nanos": 10000000 }, "doomslug_step_period": { "secs": 0, "nanos": 100000000 } }, "tracked_accounts": [], "tracked_shards": [ 0 ], "archive": false, "log_summary_style": "colored", "gc_blocks_limit": 2, "view_client_threads": 4, "epoch_sync_enabled": true, "view_client_throttle_period": { "secs": 30, "nanos": 0 }, "trie_viewer_state_size_limit": 50000 }⏎ ``` --- chain/indexer/src/lib.rs | 12 +++++-- nearcore/src/config.rs | 53 +++++++++++++++++++++++++--- neard/src/cli.rs | 10 +++++- tools/indexer/example/src/configs.rs | 12 +++++-- 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/chain/indexer/src/lib.rs b/chain/indexer/src/lib.rs index db1f5170989..23c0c5732c7 100644 --- a/chain/indexer/src/lib.rs +++ b/chain/indexer/src/lib.rs @@ -37,9 +37,13 @@ pub struct InitConfigArgs { /// Genesis file to use when initializing testnet (including downloading) pub genesis: Option, /// Download the verified NEAR genesis file automatically. - pub download: bool, - /// Specify a custom download URL for the genesis-file. + pub should_download_genesis: bool, + /// Specify a custom download URL for the genesisfile. pub download_genesis_url: Option, + /// Download the verified NEAR config file automtically. + pub should_download_config: bool, + /// Specify a custom download URL for the config file. + pub download_config_url: Option, /// Specify a custom max_gas_burnt_view limit. pub max_gas_burnt_view: Option, } @@ -136,8 +140,10 @@ pub fn indexer_init_configs(dir: &std::path::PathBuf, params: InitConfigArgs) { params.num_shards, params.fast, params.genesis.as_deref(), - params.download, + params.should_download_genesis, params.download_genesis_url.as_deref(), + params.should_download_config, + params.download_config_url.as_deref(), params.max_gas_burnt_view, ) } diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index 9e0d509cd58..3f0efe362b5 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -790,8 +790,10 @@ pub fn init_configs( num_shards: NumShards, fast: bool, genesis: Option<&str>, - download: bool, + should_download_genesis: bool, download_genesis_url: Option<&str>, + should_download_config: bool, + download_config_url: Option<&str>, max_gas_burnt_view: Option, ) { fs::create_dir_all(dir).expect("Failed to create directory"); @@ -801,13 +803,25 @@ pub fn init_configs( let genesis_config = GenesisConfig::from_file(&dir.join(config.genesis_file)); panic!("Found existing config in {} with chain-id = {}. Use unsafe_reset_all to clear the folder.", dir.to_str().unwrap(), genesis_config.chain_id); } + + let mut config = Config::default(); let chain_id = chain_id .and_then(|c| if c.is_empty() { None } else { Some(c.to_string()) }) .unwrap_or_else(random_chain_id); - let mut config = Config::default(); + + if let Some(url) = download_config_url { + download_config(&url.to_string(), &dir.join(CONFIG_FILENAME)); + config = Config::from_file(&dir.join(CONFIG_FILENAME)); + } else if should_download_config { + let url = get_config_url(&chain_id); + download_config(&url, &dir.join(CONFIG_FILENAME)); + config = Config::from_file(&dir.join(CONFIG_FILENAME)); + } + if max_gas_burnt_view.is_some() { config.max_gas_burnt_view = max_gas_burnt_view; } + match chain_id.as_ref() { "mainnet" => { if test_seed.is_some() { @@ -816,7 +830,6 @@ pub fn init_configs( config.telemetry.endpoints.push(MAINNET_TELEMETRY_URL.to_string()); config.write_to_file(&dir.join(CONFIG_FILENAME)); - // TODO: add download genesis for mainnet let genesis: Genesis = serde_json::from_slice(*MAINNET_GENESIS_JSON) .expect("Failed to deserialize MainNet genesis"); if let Some(account_id) = account_id { @@ -850,7 +863,7 @@ pub fn init_configs( if let Some(url) = download_genesis_url { download_genesis(&url.to_string(), &genesis_path); - } else if download { + } else if should_download_genesis { let url = get_genesis_url(&chain_id); download_genesis(&url, &genesis_path); } else { @@ -1058,9 +1071,15 @@ pub fn get_genesis_url(chain_id: &String) -> String { ) } +pub fn get_config_url(chain_id: &String) -> String { + format!( + "https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/{}/config.json", + chain_id, + ) +} + pub fn download_genesis(url: &String, path: &PathBuf) { info!(target: "near", "Downloading genesis file from: {} ...", url); - let url = url.clone(); let path = path.clone(); @@ -1077,6 +1096,30 @@ pub fn download_genesis(url: &String, path: &PathBuf) { .await .expect("Genesis file is bigger than 10GB. Please make the limit higher."); + std::fs::write(&path, &body).expect("Failed to create / write a genesis file."); + + info!(target: "near", "Saved the genesis file to: {} ...", path.as_path().display()); + }); +} + +pub fn download_config(url: &String, path: &PathBuf) { + info!(target: "near", "Downloading config file from: {} ...", url); + let url = url.clone(); + let path = path.clone(); + + actix::System::new().block_on(async move { + let client = awc::Client::new(); + let mut response = + client.get(url).send().await.expect("Unable to download the config file"); + + // IMPORTANT: limit specifies the maximum size of the genesis + // In case where the genesis is bigger than the specified limit Overflow Error is thrown + let body = response + .body() + .limit(10_000) + .await + .expect("Genesis file is bigger than 10MB. Please make the limit higher."); + std::fs::write(&path, &body).expect("Failed to create / write a config file."); info!(target: "near", "Saved the config file to: {} ...", path.as_path().display()); diff --git a/neard/src/cli.rs b/neard/src/cli.rs index 54717efbd23..c8f0e70ca4b 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -102,6 +102,9 @@ pub(super) struct InitCmd { /// Download the verified NEAR genesis file automatically. #[clap(long)] download_genesis: bool, + /// Download the verified NEAR config file automatically. + #[clap(long)] + download_config: bool, /// Makes block production fast (TESTING ONLY). #[clap(long)] fast: bool, @@ -111,9 +114,12 @@ pub(super) struct InitCmd { /// Chain ID, by default creates new random. #[clap(long)] chain_id: Option, - /// Specify a custom download URL for the genesis-file. + /// Specify a custom download URL for the genesis file. #[clap(long)] download_genesis_url: Option, + /// Specify a custom download URL for the config file. + #[clap(long)] + download_config_url: Option, /// Genesis file to use when initializing testnet (including downloading). #[clap(long)] genesis: Option, @@ -149,6 +155,8 @@ impl InitCmd { self.genesis.as_deref(), self.download_genesis, self.download_genesis_url.as_deref(), + self.download_config, + self.download_config_url.as_deref(), self.max_gas_burnt_view, ); } diff --git a/tools/indexer/example/src/configs.rs b/tools/indexer/example/src/configs.rs index 93743c98e3d..ea0078e6733 100644 --- a/tools/indexer/example/src/configs.rs +++ b/tools/indexer/example/src/configs.rs @@ -46,10 +46,16 @@ pub(crate) struct InitConfigArgs { pub genesis: Option, #[clap(short, long)] /// Download the verified NEAR genesis file automatically. - pub download: bool, + pub should_download_genesis: bool, /// Specify a custom download URL for the genesis-file. #[clap(long)] pub download_genesis_url: Option, + #[clap(short, long)] + /// Download the verified NEAR config file automatically. + pub should_download_config: bool, + /// Specify a custom download URL for the config file. + #[clap(long)] + pub download_config_url: Option, /// Specify a custom max_gas_burnt_view limit. #[clap(long)] pub max_gas_burnt_view: Option, @@ -74,8 +80,10 @@ impl From for near_indexer::InitConfigArgs { num_shards: config_args.num_shards, fast: config_args.fast, genesis: config_args.genesis, - download: config_args.download, + should_download_genesis: config_args.should_download_genesis, download_genesis_url: config_args.download_genesis_url, + should_download_config: config_args.should_download_config, + download_config_url: config_args.download_config_url, max_gas_burnt_view: config_args.max_gas_burnt_view, } } From f32d7094cb368ea0b2b9a489bc92a388adebb27d Mon Sep 17 00:00:00 2001 From: Sandi Fatic Date: Tue, 29 Jun 2021 20:24:23 +0200 Subject: [PATCH 2/5] Update nearcore/src/config.rs Co-authored-by: Bohdan Khorolets --- nearcore/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index 3f0efe362b5..58280ee202b 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -1118,7 +1118,7 @@ pub fn download_config(url: &String, path: &PathBuf) { .body() .limit(10_000) .await - .expect("Genesis file is bigger than 10MB. Please make the limit higher."); + .expect("Config file is bigger than 10MB. Please make the limit higher."); std::fs::write(&path, &body).expect("Failed to create / write a config file."); From 3bb941f6b30c4c5e9d521a73f00f4cf92f182519 Mon Sep 17 00:00:00 2001 From: Sandi Fatic Date: Tue, 29 Jun 2021 20:24:30 +0200 Subject: [PATCH 3/5] Update nearcore/src/config.rs Co-authored-by: Bohdan Khorolets --- nearcore/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index 58280ee202b..f9b4663f7e5 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -1113,7 +1113,7 @@ pub fn download_config(url: &String, path: &PathBuf) { client.get(url).send().await.expect("Unable to download the config file"); // IMPORTANT: limit specifies the maximum size of the genesis - // In case where the genesis is bigger than the specified limit Overflow Error is thrown + // In case where the config is bigger than the specified limit Overflow Error is thrown let body = response .body() .limit(10_000) From c32b568546f9f5757f838f9443ed69b449264701 Mon Sep 17 00:00:00 2001 From: chefsale Date: Thu, 1 Jul 2021 15:08:33 +0200 Subject: [PATCH 4/5] fix typo --- chain/indexer/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/indexer/src/lib.rs b/chain/indexer/src/lib.rs index 23c0c5732c7..b83ab1b3425 100644 --- a/chain/indexer/src/lib.rs +++ b/chain/indexer/src/lib.rs @@ -38,7 +38,7 @@ pub struct InitConfigArgs { pub genesis: Option, /// Download the verified NEAR genesis file automatically. pub should_download_genesis: bool, - /// Specify a custom download URL for the genesisfile. + /// Specify a custom download URL for the genesis file. pub download_genesis_url: Option, /// Download the verified NEAR config file automtically. pub should_download_config: bool, From 7c3da719483991927cedfefc80735b2ac74399ac Mon Sep 17 00:00:00 2001 From: chefsale Date: Thu, 1 Jul 2021 23:42:35 +0200 Subject: [PATCH 5/5] applied frols comments --- chain/indexer/src/lib.rs | 8 ++--- nearcore/src/config.rs | 51 +++++++++------------------- tools/indexer/example/src/configs.rs | 12 +++---- 3 files changed, 26 insertions(+), 45 deletions(-) diff --git a/chain/indexer/src/lib.rs b/chain/indexer/src/lib.rs index b83ab1b3425..ad7c08704cd 100644 --- a/chain/indexer/src/lib.rs +++ b/chain/indexer/src/lib.rs @@ -37,11 +37,11 @@ pub struct InitConfigArgs { /// Genesis file to use when initializing testnet (including downloading) pub genesis: Option, /// Download the verified NEAR genesis file automatically. - pub should_download_genesis: bool, + pub download_genesis: bool, /// Specify a custom download URL for the genesis file. pub download_genesis_url: Option, /// Download the verified NEAR config file automtically. - pub should_download_config: bool, + pub download_config: bool, /// Specify a custom download URL for the config file. pub download_config_url: Option, /// Specify a custom max_gas_burnt_view limit. @@ -140,9 +140,9 @@ pub fn indexer_init_configs(dir: &std::path::PathBuf, params: InitConfigArgs) { params.num_shards, params.fast, params.genesis.as_deref(), - params.should_download_genesis, + params.download_genesis, params.download_genesis_url.as_deref(), - params.should_download_config, + params.download_config, params.download_config_url.as_deref(), params.max_gas_burnt_view, ) diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index f9b4663f7e5..ede11ecadc6 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -1078,52 +1078,33 @@ pub fn get_config_url(chain_id: &String) -> String { ) } -pub fn download_genesis(url: &String, path: &PathBuf) { - info!(target: "near", "Downloading genesis file from: {} ...", url); - let url = url.clone(); - let path = path.clone(); - +pub fn download_file(url: &String, path: &PathBuf, limit: usize) { actix::System::new().block_on(async move { let client = awc::Client::new(); - let mut response = - client.get(url).send().await.expect("Unable to download the genesis file"); - - // IMPORTANT: limit specifies the maximum size of the genesis - // In case where the genesis is bigger than the specified limit Overflow Error is thrown + let mut response = client.get(url).send().await.expect("Unable to download the file"); + // IMPORTANT: limit specifies the maximum size of the genesis or config file + // In case where the genesis or config file is bigger than the specified + // limit Overflow Error is thrown let body = response .body() - .limit(10_000_000_000) + .limit(limit) .await - .expect("Genesis file is bigger than 10GB. Please make the limit higher."); - - std::fs::write(&path, &body).expect("Failed to create / write a genesis file."); + .expect("File is bigger than specified limit. Please make the limit higher."); - info!(target: "near", "Saved the genesis file to: {} ...", path.as_path().display()); + std::fs::write(&path, &body).expect("Failed to create / write a file."); }); } +pub fn download_genesis(url: &String, path: &PathBuf) { + info!(target: "near", "Downloading genesis file from: {} ...", url); + download_file(&url, &path, 10_000_000_000); + info!(target: "near", "Saved the genesis file to: {} ...", path.as_path().display()); +} + pub fn download_config(url: &String, path: &PathBuf) { info!(target: "near", "Downloading config file from: {} ...", url); - let url = url.clone(); - let path = path.clone(); - - actix::System::new().block_on(async move { - let client = awc::Client::new(); - let mut response = - client.get(url).send().await.expect("Unable to download the config file"); - - // IMPORTANT: limit specifies the maximum size of the genesis - // In case where the config is bigger than the specified limit Overflow Error is thrown - let body = response - .body() - .limit(10_000) - .await - .expect("Config file is bigger than 10MB. Please make the limit higher."); - - std::fs::write(&path, &body).expect("Failed to create / write a config file."); - - info!(target: "near", "Saved the config file to: {} ...", path.as_path().display()); - }); + download_file(&url, &path, 10_000); + info!(target: "near", "Saved the config file to: {} ...", path.as_path().display()); } pub fn load_config_without_genesis_records(dir: &Path) -> NearConfig { diff --git a/tools/indexer/example/src/configs.rs b/tools/indexer/example/src/configs.rs index ea0078e6733..3c232fba73c 100644 --- a/tools/indexer/example/src/configs.rs +++ b/tools/indexer/example/src/configs.rs @@ -44,15 +44,15 @@ pub(crate) struct InitConfigArgs { /// Genesis file to use when initialize testnet (including downloading) #[clap(short, long)] pub genesis: Option, - #[clap(short, long)] + #[clap(long)] /// Download the verified NEAR genesis file automatically. - pub should_download_genesis: bool, + pub download_genesis: bool, /// Specify a custom download URL for the genesis-file. #[clap(long)] pub download_genesis_url: Option, - #[clap(short, long)] + #[clap(long)] /// Download the verified NEAR config file automatically. - pub should_download_config: bool, + pub download_config: bool, /// Specify a custom download URL for the config file. #[clap(long)] pub download_config_url: Option, @@ -80,9 +80,9 @@ impl From for near_indexer::InitConfigArgs { num_shards: config_args.num_shards, fast: config_args.fast, genesis: config_args.genesis, - should_download_genesis: config_args.should_download_genesis, + download_genesis: config_args.download_genesis, download_genesis_url: config_args.download_genesis_url, - should_download_config: config_args.should_download_config, + download_config: config_args.download_config, download_config_url: config_args.download_config_url, max_gas_burnt_view: config_args.max_gas_burnt_view, }