Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Separate test and summary step in validator script #2091

Merged
merged 1 commit into from
Aug 10, 2023
Merged

Separate test and summary step in validator script #2091

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 73 additions & 89 deletions .github/workflows/validate-1m-linux.sh → .github/workflows/validate-linux.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,35 @@
# This test script joins Earth and pokes some stuff

TEST_NETWORK=8056c2e21c000001
RUN_LENGTH=60
RUN_LENGTH=20
TEST_FINISHED=false
ZTO_VER=$(git describe --tags $(git rev-list --tags --max-count=1))
ZTO_COMMIT=$(git rev-parse HEAD)
ZTO_COMMIT_SHORT=$(git rev-parse --short HEAD)
TEST_DIR_PREFIX="$ZTO_VER-$ZTO_COMMIT_SHORT-test-results"
EXIT_TEST_FAILED=0

TEST_OK=0
TEST_FAIL=1

echo "Performing test on: $ZTO_VER-$ZTO_COMMIT_SHORT"
TEST_FILEPATH_PREFIX="$TEST_DIR_PREFIX/$ZTO_COMMIT_SHORT"
mkdir $TEST_DIR_PREFIX

# How long we will wait for ZT to come online before considering it a failure
MAX_WAIT_SECS=60

################################################################################
# Multi-node connectivity and performance test #
################################################################################
main() {
echo -e "\nRunning test for $RUN_LENGTH seconds"

test() {

echo -e "\nPerforming pre-flight checks"

check_exit_on_invalid_identity

echo -e "\nRunning test for $RUN_LENGTH seconds"

NS1="ip netns exec ns1"
NS2="ip netns exec ns2"

Expand Down Expand Up @@ -75,11 +84,11 @@ main() {
# Memory Leak Check #
################################################################################

FILENAME_MEMORY_LOG="$TEST_FILEPATH_PREFIX-memory.log"
export FILENAME_MEMORY_LOG="$TEST_FILEPATH_PREFIX-memory.log"

echo -e "\nStarting a ZeroTier instance in each namespace..."

time_test_start=$(date +%s)
export time_test_start=$(date +%s)

# Spam the CLI as ZeroTier is starting
spam_cli 100
Expand Down Expand Up @@ -127,7 +136,6 @@ main() {
################################################################################

echo "Waiting for ZeroTier to come online before attempting test..."
MAX_WAIT_SECS="${MAX_WAIT_SECS:-120}"
node1_online=false
node2_online=false
both_instances_online=false
Expand All @@ -139,13 +147,13 @@ main() {
node2_online="$($ZT2 -j info | jq '.online' 2>/dev/null)"
echo "Checking for online status: try #$s, node1:$node1_online, node2:$node2_online"
if [[ "$node1_online" == "true" ]]; then
time_zt_node1_online=$(date +%s)
export time_zt_node1_online=$(date +%s)
fi
if [[ "$node2_online" == "true" ]]; then
time_zt_node2_online=$(date +%s)
export time_zt_node2_online=$(date +%s)
fi
if [[ "$node2_online" == "true" && "$node1_online" == "true" ]]; then
both_instances_online=true
export both_instances_online=true
break
fi
sleep 1
Expand All @@ -172,10 +180,7 @@ main() {
$ZT2 status

if [[ "$both_instances_online" != "true" ]]; then
echo "One or more instances of ZeroTier failed to come online. Aborting test."
collect_zt_dump_files
collect_zt_log_files
exit 0
exit_test_and_generate_report $TEST_FAIL "one or more nodes failed to come online"
fi

echo -e "\nJoining networks"
Expand All @@ -199,18 +204,14 @@ main() {
$NS1 ping -c 16 $node2_ip4 >$PING12_FILENAME
$NS2 ping -c 16 $node1_ip4 >$PING21_FILENAME

# Parse ping statistics
ping_loss_percent_1_to_2="${ping_loss_percent_1_to_2:-100.0}"
ping_loss_percent_2_to_1="${ping_loss_percent_2_to_1:-100.0}"

ping_loss_percent_1_to_2=$(cat $PING12_FILENAME |
grep "packet loss" | awk '{print $6}' | sed 's/%//')
ping_loss_percent_2_to_1=$(cat $PING21_FILENAME |
grep "packet loss" | awk '{print $6}' | sed 's/%//')

# Normalize loss value
ping_loss_percent_1_to_2=$(echo "scale=2; $ping_loss_percent_1_to_2/100.0" | bc)
ping_loss_percent_2_to_1=$(echo "scale=2; $ping_loss_percent_2_to_1/100.0" | bc)
export ping_loss_percent_1_to_2=$(echo "scale=2; $ping_loss_percent_1_to_2/100.0" | bc)
export ping_loss_percent_2_to_1=$(echo "scale=2; $ping_loss_percent_2_to_1/100.0" | bc)

################################################################################
# CLI Check #
Expand Down Expand Up @@ -261,11 +262,9 @@ main() {

# TODO: Validate JSON

################################################################################
# Performance Test #
################################################################################
# Performance Test

FILENAME_PERF_JSON="$TEST_FILEPATH_PREFIX-iperf.json"
export FILENAME_PERF_JSON="$TEST_FILEPATH_PREFIX-iperf.json"

echo -e "\nBeginning performance test:"

Expand All @@ -281,15 +280,7 @@ main() {

cat $FILENAME_PERF_JSON

################################################################################
# Collect ZeroTier dump files #
################################################################################

collect_zt_dump_files

################################################################################
# Let ZeroTier idle long enough for various timers #
################################################################################
# Let ZeroTier idle long enough for various timers

echo -e "\nIdling ZeroTier for $RUN_LENGTH seconds..."
sleep $RUN_LENGTH
Expand All @@ -301,25 +292,44 @@ main() {

sleep 5

################################################################################
# Stop test #
################################################################################
# Stop test

echo -e "\nStopping memory check..."
sudo pkill -15 -f valgrind
sleep 10

time_test_end=$(date +%s)
export time_test_end=$(date +%s)

################################################################################
# Copy ZeroTier stdout/stderr logs #
################################################################################
exit_test_and_generate_report $TEST_OK "completed test"
}

collect_zt_log_files
################################################################################
# Generate report #
################################################################################

################################################################################
# Generate report #
################################################################################
exit_test_and_generate_report() {

echo "Exiting test with reason: $2 ($1)"

# Collect ZeroTier dump files

echo -e "\nCollecting ZeroTier dump files"

node1_id=$($ZT1 -j status | jq -r .address)
node2_id=$($ZT2 -j status | jq -r .address)

$ZT1 dump
mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node1_id.txt"

$ZT2 dump
mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node2_id.txt"

# Copy ZeroTier stdout/stderr logs

cp node_1.log "$TEST_FILEPATH_PREFIX-node-log-$node1_id.txt"
cp node_2.log "$TEST_FILEPATH_PREFIX-node-log-$node2_id.txt"

# Generate report

cat $FILENAME_MEMORY_LOG

Expand All @@ -328,9 +338,7 @@ main() {
POSSIBLY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
$FILENAME_MEMORY_LOG | grep "possibly" | awk '{print $1;}')

################################################################################
# Generate coverage report artifact and summary #
################################################################################
# Generate coverage report artifact and summary

FILENAME_COVERAGE_JSON="$TEST_FILEPATH_PREFIX-coverage.json"
FILENAME_COVERAGE_HTML="$TEST_FILEPATH_PREFIX-coverage.html"
Expand All @@ -350,22 +358,20 @@ main() {
COVERAGE_LINE_TOTAL="${COVERAGE_LINE_TOTAL:-0}"
COVERAGE_LINE_PERCENT="${COVERAGE_LINE_PERCENT:-0}"

################################################################################
# Default values #
################################################################################
# Default values

DEFINITELY_LOST="${DEFINITELY_LOST:-0}"
POSSIBLY_LOST="${POSSIBLY_LOST:-0}"
ping_loss_percent_1_to_2="${ping_loss_percent_1_to_2:-100.0}"
ping_loss_percent_2_to_1="${ping_loss_percent_2_to_1:-100.0}"

################################################################################
# Summarize and emit json for trend reporting #
################################################################################
# Summarize and emit json for trend reporting

FILENAME_SUMMARY="$TEST_FILEPATH_PREFIX-summary.json"

time_length_test=$((time_test_end - time_test_start))
time_length_zt_node1_online=$((time_zt_node1_online - time_zt_start))
time_length_zt_node2_online=$((time_zt_node2_online - time_zt_start))
time_to_node1_online=$((time_zt_node1_online - time_zt_start))
time_to_node2_online=$((time_zt_node2_online - time_zt_start))
#time_length_zt_join=$((time_zt_join_end-time_zt_join_start))
#time_length_zt_leave=$((time_zt_leave_end-time_zt_leave_start))
#time_length_zt_can_still_ping=$((time_zt_can_still_ping-time_zt_leave_start))
Expand All @@ -377,25 +383,20 @@ main() {
"commit":"$ZTO_COMMIT",
"arch_m":"$(uname -m)",
"arch_a":"$(uname -a)",
"binary_size":"$(stat -c %s zerotier-one)"
"time_length_test":$time_length_test,
"time_length_zt_node1_online":$time_length_zt_node1_online,
"time_length_zt_node2_online":$time_length_zt_node2_online,
"time_to_node1_online":$time_to_node1_online,
"time_to_node2_online":$time_to_node2_online,
"num_possible_bytes_lost": $POSSIBLY_LOST,
"num_definite_bytes_lost": $DEFINITELY_LOST,
"num_incorrect_settings": $POSSIBLY_LOST,
"num_bad_formattings": $POSSIBLY_LOST,
"percent_coverage_branches": $POSSIBLY_LOST,
"coverage_lines_covered": $COVERAGE_LINE_COVERED,
"coverage_lines_total": $COVERAGE_LINE_TOTAL,
"coverage_lines_percent": $COVERAGE_LINE_PERCENT,
"ping_loss_percent_1_to_2": $ping_loss_percent_1_to_2,
"ping_loss_percent_2_to_1": $ping_loss_percent_2_to_1,
"mean_latency_ping_random": $POSSIBLY_LOST,
"mean_latency_ping_netns": $POSSIBLY_LOST,
"mean_pdv_random": $POSSIBLY_LOST,
"mean_pdv_netns": $POSSIBLY_LOST,
"mean_perf_netns": $POSSIBLY_LOST,
"exit_test_failed": $EXIT_TEST_FAILED
"test_exit_code": $1,
"test_exit_reason":"$2"
}
EOF
)
Expand Down Expand Up @@ -436,6 +437,10 @@ spam_cli() {
done
}

################################################################################
# Check for proper exit on load of invalid identity #
################################################################################

check_exit_on_invalid_identity() {
echo "Checking ZeroTier exits on invalid identity..."
mkdir -p $(pwd)/exit_test
Expand All @@ -447,35 +452,14 @@ check_exit_on_invalid_identity() {
$ZT1 &
my_pid=$!

echo "Waiting 5 secons"
echo "Waiting 5 seconds"
sleep 5

# check if process is running
kill -0 $my_pid
if [ $? -eq 0 ]; then
EXIT_TEST_FAILED=1
echo "Exit test FAILED: Process still running after being fed an invalid identity"
else
echo "Exit test PASSED"
exit_test_and_generate_report $TEST_FAIL "Exit test FAILED: Process still running after being fed an invalid identity"
fi
}

collect_zt_dump_files() {
echo -e "\nCollecting ZeroTier dump files"

node1_id=$($ZT1 -j status | jq -r .address)
node2_id=$($ZT2 -j status | jq -r .address)

$ZT1 dump
mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node1_id.txt"

$ZT2 dump
mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node2_id.txt"
}

collect_zt_log_files() {
cp node_1.log "$TEST_FILEPATH_PREFIX-node-log-$node1_id.txt"
cp node_2.log "$TEST_FILEPATH_PREFIX-node-log-$node2_id.txt"
}

main "$@"
test "$@"
9 changes: 6 additions & 3 deletions .github/workflows/report.sh → .github/workflows/validate-report.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
################################################################################

DEFINITELY_LOST=$(cat *test-results/*summary.json | jq .num_definite_bytes_lost)
EXIT_CODE=$(cat *test-results/*summary.json | jq .exit_code)
EXIT_REASON=$(cat *test-results/*summary.json | jq .exit_reason)

cat *test-results/*summary.json

Expand All @@ -14,8 +16,9 @@ if [[ "$DEFINITELY_LOST" -gt 0 ]]; then
exit 1
fi

EXIT_TEST_FAILED=$(cat *test-results/*summary.json | jq .exit_test_failed)
# Catch-all for other non-zero exit codes

if [[ "$EXIT_TEST_FAILED" -gt 0 ]]; then
if [[ "$EXIT_CODE" -gt 0 ]]; then
echo "Test failed: $EXIT_REASON"
exit 1
fi
fi
8 changes: 4 additions & 4 deletions .github/workflows/validate.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ jobs:
run: |
sudo apt install -y valgrind xmlstarlet gcovr iperf3 tree
make one ZT_COVERAGE=1 ZT_TRACE=1
sudo chmod +x ./.github/workflows/validate-1m-linux.sh
sudo ./.github/workflows/validate-1m-linux.sh
sudo chmod +x ./.github/workflows/validate-linux.sh
sudo ./.github/workflows/validate-linux.sh

- name: Archive test results
uses: actions/upload-artifact@v3
Expand All @@ -51,6 +51,6 @@ jobs:

- name: final-report
run: |
sudo chmod +x ./.github/workflows/report.sh
sudo ./.github/workflows/report.sh
sudo chmod +x ./.github/workflows/validate-report.sh
sudo ./.github/workflows/validate-report.sh