cleanup

ChainSafe · Jun 6, 2024 · b732021 · b732021
1 parent 9bea38f
commit b732021
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 26 deletions.
diff --git a/ansible/archival-snapshots/resources/init.sh b/ansible/archival-snapshots/resources/init.sh
@@ -3,10 +3,10 @@
 ## Enable strict error handling, command tracing, and pipefail
 set -eux
 
-echo "Current user: $(whoami)"
-
+# Initialize snapshots directory
 LITE_SNAPSHOT_DIR="/mnt/md0/exported/archival/lite_snapshots"
 DIFF_SNAPSHOT_DIR="/mnt/md0/exported/archival/diff_snapshots"
+FULL_SNAPSHOTS_DIR=/mnt/md0/exported/archival/snapshots
 
 if [ ! -d "$LITE_SNAPSHOT_DIR" ]; then
     mkdir -p "$LITE_SNAPSHOT_DIR"
@@ -22,4 +22,12 @@ else
     echo "$DIFF_SNAPSHOT_DIR exists"
 fi
 
+if [ ! -d "$FULL_SNAPSHOTS_DIR" ]; then
+    mkdir -p "$FULL_SNAPSHOTS_DIR"
+    echo "Created $FULL_SNAPSHOTS_DIR"
+else
+    echo "$FULL_SNAPSHOTS_DIR exists"
+fi
+
+# Trigger main script
 ./main.sh
diff --git a/ansible/archival-snapshots/resources/main.sh b/ansible/archival-snapshots/resources/main.sh
@@ -3,34 +3,33 @@
 ## Enable strict error handling, command tracing, and pipefail
 set -euxo pipefail
 
+## Set constants
 GENESIS_TIMESTAMP=1598306400
 SECONDS_PER_EPOCH=30
 
+## Set forest artifacts path
 FOREST="/mnt/md0/exported/archival/forest/forest"
 FOREST_CLI="/mnt/md0/exported/archival/forest/forest-cli"
 FOREST_TOOL="/mnt/md0/exported/archival/forest/forest-tool"
 
-CURRENT_SNAPSHOT=$(aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/lite/" | sort | tail -n 1 | awk '{print $NF}')
-CURRENT_EPOCH=$(echo "$CURRENT_SNAPSHOT" | awk -F'_' '{gsub(/[^0-9]/, "", $6); print $6}')
-FULL_SNAPSHOTS_DIR=/mnt/md0/exported/archival/snapshots
-CURRENT_FULL_SNAPSHOT_PATH="$FULL_SNAPSHOTS_DIR/$CURRENT_SNAPSHOT"
 LITE_SNAPSHOT_DIR=/mnt/md0/exported/archival/lite_snapshots
+FULL_SNAPSHOTS_DIR=/mnt/md0/exported/archival/snapshots
 
-# Check if full snapshots directory exist, if not create
-if [ ! -d "$FULL_SNAPSHOTS_DIR" ]; then
-    mkdir -p "$FULL_SNAPSHOTS_DIR"
-fi
+## Fetch last snapshot details
+LAST_SNAPSHOT=$(aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/lite/" | sort | tail -n 1 | awk '{print $NF}')
+LAST_EPOCH=$(echo "$LAST_SNAPSHOT" | awk -F'_' '{gsub(/[^0-9]/, "", $6); print $6}')
+LAST_FULL_SNAPSHOT_PATH="$FULL_SNAPSHOTS_DIR/$LAST_SNAPSHOT"
 
-if [ ! -f "$CURRENT_FULL_SNAPSHOT_PATH" ]; then
-    echo "Downloading last snapshot: $CURRENT_FULL_SNAPSHOT_PATH"
-    aws --profile prod --endpoint "$ENDPOINT" s3 cp "s3://forest-archive/mainnet/lite/$CURRENT_SNAPSHOT" "$CURRENT_FULL_SNAPSHOT_PATH"
-    echo "Last snapshot download: $CURRENT_FULL_SNAPSHOT_PATH"
+if [ ! -f "$LAST_FULL_SNAPSHOT_PATH" ]; then
+    echo "Downloading last snapshot: $LAST_FULL_SNAPSHOT_PATH"
+    aws --profile prod --endpoint "$ENDPOINT" s3 cp "s3://forest-archive/mainnet/lite/$LAST_SNAPSHOT" "$LAST_FULL_SNAPSHOT_PATH"
+    echo "Last snapshot download: $LAST_FULL_SNAPSHOT_PATH"
 else
-    echo "$CURRENT_FULL_SNAPSHOT_PATH snapshot already exists."
+    echo "$LAST_FULL_SNAPSHOT_PATH snapshot exists."
 fi
 
 echo "Starting forest daemon"
-nohup $FOREST --no-gc --config ./config.toml --save-token ./admin_token --rpc-address 127.0.0.1:3456 --metrics-address 127.0.0.1:5000 > forest.log 2>&1 &
+nohup $FOREST --no-gc --config ./config.toml --save-token ./admin_token --rpc-address 127.0.0.1:3456 --metrics-address 127.0.0.1:5000 --import-snapshot "$LAST_FULL_SNAPSHOT_PATH" > forest.log 2>&1 &
 FOREST_NODE_PID=$!
 
 sleep 30
@@ -44,24 +43,24 @@ function set_fullnode_api_info {
 }
 set_fullnode_api_info
 
+# Wait for network to sync
 echo "Waiting for forest to sync to latest network head"
 $FOREST_CLI sync wait
 
 # Get latest epoch using sync status
-echo "Current Height: $CURRENT_EPOCH"
+echo "Current Height: $LAST_EPOCH"
 LATEST_EPOCH=$($FOREST_CLI sync status | grep "Height:" | awk '{print $2}')
 echo "Latest Height: $LATEST_EPOCH"
 
-while ((LATEST_EPOCH - CURRENT_EPOCH > 30000)); do
+while ((LATEST_EPOCH - LAST_EPOCH > 30000)); do
    set_fullnode_api_info
-   NEW_EPOCH=$((CURRENT_EPOCH + 30000))
+   NEW_EPOCH=$((LAST_EPOCH + 30000))
    echo "Next Height: $NEW_EPOCH"
 
    # Export full snapshot to generate lite and diff snapshots
    EPOCH_TIMESTAMP=$((GENESIS_TIMESTAMP + NEW_EPOCH*SECONDS_PER_EPOCH))
    DATE=$(date --date=@"$EPOCH_TIMESTAMP" -u -I)
    NEW_SNAPSHOT="forest_snapshot_mainnet_${DATE}_height_${NEW_EPOCH}.forest.car.zst"
-
    if [ ! -f "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" ]; then
         echo "Exporting snapshot: $FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT"
         echo "USING FULLNODE API: $FULLNODE_API_INFO"
@@ -84,20 +83,20 @@ while ((LATEST_EPOCH - CURRENT_EPOCH > 30000)); do
    echo "Lite snapshot uploaded: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT"
 
    # Generate and upload diff snapshots
-   if [ ! -f "$CURRENT_FULL_SNAPSHOT_PATH" ]; then
+   if [ ! -f "$LAST_FULL_SNAPSHOT_PATH" ]; then
        echo "File does not exist. Exporting..."
-       $FOREST_CLI snapshot export --tipset "$CURRENT_EPOCH" --depth 30000 -o "$CURRENT_FULL_SNAPSHOT_PATH"
+       $FOREST_CLI snapshot export --tipset "$LAST_EPOCH" --depth 30000 -o "$LAST_FULL_SNAPSHOT_PATH"
    else
-       echo "$CURRENT_FULL_SNAPSHOT_PATH file exists."
+       echo "$LAST_FULL_SNAPSHOT_PATH file exists."
    fi
-   echo "Generating Diff snapshots: $CURRENT_EPOCH - $NEW_EPOCH"
-   ./diff_script.sh "$CURRENT_EPOCH" "$CURRENT_FULL_SNAPSHOT_PATH" "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT"
+   echo "Generating Diff snapshots: $LAST_EPOCH - $NEW_EPOCH"
+   ./diff_script.sh "$LAST_EPOCH" "$LAST_FULL_SNAPSHOT_PATH" "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT"
    echo "Diff snapshots generated successfully"
    echo "Uploading Diff snapshots"
    ./upload_diff.sh "$ENDPOINT"
    echo "Diff snapshots uploaded successfully"
 
-   CURRENT_EPOCH=$NEW_EPOCH
+   LAST_EPOCH=$NEW_EPOCH
 done
 
 kill -KILL $FOREST_NODE_PID