From 07aa92b60cc9dd732b817105ffe42f1f77611bc4 Mon Sep 17 00:00:00 2001 From: yourmoonlight Date: Mon, 25 Jul 2022 17:04:59 +0800 Subject: [PATCH 1/4] format code --- scripts/sync_block.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/sync_block.py b/scripts/sync_block.py index d7c925fefe..6dd2cef67e 100644 --- a/scripts/sync_block.py +++ b/scripts/sync_block.py @@ -7,7 +7,6 @@ import argparse - def get_height(method, url, post_data, headers): parsed_url = urllib.parse.urlparse(url) @@ -55,7 +54,6 @@ def check_or_do(network): print(export_cmd) os.system(export_cmd) - # tar block csv file filename = "block_%s_%s.csv" % (start, end) file_compress_name = "%s.tar.gz" % filename @@ -66,7 +64,8 @@ def check_or_do(network): # append and tar the block list csv file block_list_file_name = "block_list.csv" - append_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- bash -c 'echo %s >> /sc-data/%s' " % (network, filename, block_list_file_name) + append_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- bash -c 'echo %s >> /sc-data/%s' " % ( + network, filename, block_list_file_name) block_list_file_tar_name = "%s.tar.gz" % block_list_file_name tar_block_list_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- tar -czvf /sc-data/%s -C /sc-data/ %s " % ( network, block_list_file_tar_name, block_list_file_name) @@ -84,7 +83,8 @@ def check_or_do(network): os.system(cp_blocklist_tar_cmd) # export snapshot - export_snapshot_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- /starcoin/starcoin_db_exporter export-snapshot --db-path /sc-data/%s -n %s -o /sc-data/snapshot -t true" % (network, network, network) + export_snapshot_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- /starcoin/starcoin_db_exporter export-snapshot --db-path /sc-data/%s -n %s -o /sc-data/snapshot -t true" % ( + network, network, network) os.system(export_snapshot_cmd) # tar snapshot @@ -103,7 +103,8 @@ def check_or_do(network): if __name__ == "__main__": - parser = argparse.ArgumentParser(description='sync blocks for main, proxima, barnard') + parser = argparse.ArgumentParser( + description='sync blocks for main, proxima, barnard') parser.add_argument( '--net', metavar='net', From 11509879afec826a0180638715ca82d8c570866c Mon Sep 17 00:00:00 2001 From: yourmoonlight Date: Mon, 25 Jul 2022 19:53:19 +0800 Subject: [PATCH 2/4] check export snapshot status with the manifest file --- scripts/sync_block.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/sync_block.py b/scripts/sync_block.py index 6dd2cef67e..2f250838f4 100644 --- a/scripts/sync_block.py +++ b/scripts/sync_block.py @@ -82,10 +82,23 @@ def check_or_do(network): network, block_list_file_tar_name, network, block_list_file_tar_name) os.system(cp_blocklist_tar_cmd) + # back up last snapshot cp to backdir + # do the increment snapshot export + # check the snapshot is ok or not with the manifest file + # if not, recover the backdir, then exit. + # if ok, rm backup dir, and do the next step + os.system("cp -r /sc-data/snapshot /sc-data/snapshotbak") # export snapshot export_snapshot_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- /starcoin/starcoin_db_exporter export-snapshot --db-path /sc-data/%s -n %s -o /sc-data/snapshot -t true" % ( network, network, network) os.system(export_snapshot_cmd) + export_status = os.system( + "bash -c \"if [ $(less /sc-data/snapshot/manifest.csv| grep state_node | awk -F ' ' '{print$2}') -eq $(less /sc-data/snapshot/state_node | wc -l) ]; then exit 0; else exit 1;fi\"") + if export_status != 0: + os.system("rm -rf /sc-data/snapshot") + os.system("mv /sc-data/snapshotbak /sc-data/snapshot") + sys.exit(1) + os.system("rm -rf /sc-data/snapshotbak") # tar snapshot tar_snapshot_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- tar -czvf /sc-data/%s -C /sc-data/ %s " % ( From 86b6075957188cabdb9f62a76dc05bceebb28c6b Mon Sep 17 00:00:00 2001 From: yourmoonlight Date: Mon, 25 Jul 2022 20:04:13 +0800 Subject: [PATCH 3/4] export snapshot failed should not effect the export block logic --- scripts/sync_block.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/sync_block.py b/scripts/sync_block.py index 2f250838f4..f8ec72a63d 100644 --- a/scripts/sync_block.py +++ b/scripts/sync_block.py @@ -82,6 +82,10 @@ def check_or_do(network): network, block_list_file_tar_name, network, block_list_file_tar_name) os.system(cp_blocklist_tar_cmd) + # update the last_export_height + os.system("echo %s > ./last_export_height.txt" % end) + os.system("aws s3api put-object --bucket main.starcoin.org --key %s/last_export_height.txt --body ./last_export_height.txt" % network) + # back up last snapshot cp to backdir # do the increment snapshot export # check the snapshot is ok or not with the manifest file @@ -110,10 +114,6 @@ def check_or_do(network): network, "snapshot.tar.gz", network, "snapshot.tar.gz") os.system(cp_snapshot_tar_cmd) - # update the last_export_height - os.system("echo %s > ./last_export_height.txt" % end) - os.system("aws s3api put-object --bucket main.starcoin.org --key %s/last_export_height.txt --body ./last_export_height.txt" % network) - if __name__ == "__main__": parser = argparse.ArgumentParser( From 74720f61a9e7ea4d7854de6a85f92dba2170468b Mon Sep 17 00:00:00 2001 From: yourmoonlight Date: Mon, 25 Jul 2022 22:00:40 +0800 Subject: [PATCH 4/4] if anyone of the snapshot files not passed the check, the increment snapshot export failed --- scripts/sync_block.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/sync_block.py b/scripts/sync_block.py index f8ec72a63d..40430b9121 100644 --- a/scripts/sync_block.py +++ b/scripts/sync_block.py @@ -96,9 +96,18 @@ def check_or_do(network): export_snapshot_cmd = "kubectl exec -it -n starcoin-%s starcoin-1 -- /starcoin/starcoin_db_exporter export-snapshot --db-path /sc-data/%s -n %s -o /sc-data/snapshot -t true" % ( network, network, network) os.system(export_snapshot_cmd) - export_status = os.system( + export_state_node_status = os.system( "bash -c \"if [ $(less /sc-data/snapshot/manifest.csv| grep state_node | awk -F ' ' '{print$2}') -eq $(less /sc-data/snapshot/state_node | wc -l) ]; then exit 0; else exit 1;fi\"") - if export_status != 0: + export_acc_node_transaction_status = os.system( + "bash -c \"if [ $(less /sc-data/snapshot/manifest.csv| grep acc_node_transaction | awk -F ' ' '{print$2}') -eq $(less /sc-data/snapshot/acc_node_transaction | wc -l) ]; then exit 0; else exit 1;fi\"") + export_acc_node_block_status = os.system( + "bash -c \"if [ $(less /sc-data/snapshot/manifest.csv| grep acc_node_block | awk -F ' ' '{print$2}') -eq $(less /sc-data/snapshot/acc_node_block | wc -l) ]; then exit 0; else exit 1;fi\"") + export_block_status = os.system( + "bash -c \"if [ $(less /sc-data/snapshot/manifest.csv| grep -w block | awk -F ' ' '{print$2}') -eq $(less /sc-data/snapshot/block | wc -l) ]; then exit 0; else exit 1;fi\"") + export_block_info_status = os.system( + "bash -c \"if [ $(less /sc-data/snapshot/manifest.csv| grep block_info | awk -F ' ' '{print$2}') -eq $(less /sc-data/snapshot/block_info | wc -l) ]; then exit 0; else exit 1;fi\"") + + if export_state_node_status != 0 or export_acc_node_transaction_status != 0 or export_acc_node_block_status != 0 or export_block_status != 0 or export_block_info_status != 0: os.system("rm -rf /sc-data/snapshot") os.system("mv /sc-data/snapshotbak /sc-data/snapshot") sys.exit(1)