Skip to content

Commit

Permalink
Merge pull request #625 from versity/test_cmdline_static_fix
Browse files Browse the repository at this point in the history
test: static bucket config debugging, add to github-actions
  • Loading branch information
benmcclelland authored Jun 11, 2024
2 parents d98ca9b + fdbb2d8 commit c6e1757
Show file tree
Hide file tree
Showing 9 changed files with 148 additions and 72 deletions.
27 changes: 22 additions & 5 deletions .github/workflows/system.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ jobs:
USERS_FOLDER: /tmp/iam1
AWS_ENDPOINT_URL: https://127.0.0.1:7070
RUN_SET: "s3cmd"
RECREATE_BUCKETS: "true"
PORT: 7070
- set: 2
LOCAL_FOLDER: /tmp/gw2
Expand All @@ -25,6 +26,7 @@ jobs:
USERS_FOLDER: /tmp/iam2
AWS_ENDPOINT_URL: https://127.0.0.1:7071
RUN_SET: "s3"
RECREATE_BUCKETS: "true"
PORT: 7071
- set: 3
LOCAL_FOLDER: /tmp/gw3
Expand All @@ -34,6 +36,7 @@ jobs:
USERS_FOLDER: /tmp/iam3
AWS_ENDPOINT_URL: https://127.0.0.1:7072
RUN_SET: "s3api"
RECREATE_BUCKETS: "true"
PORT: 7072
- set: 4
LOCAL_FOLDER: /tmp/gw4
Expand All @@ -43,17 +46,28 @@ jobs:
USERS_FOLDER: /tmp/iam4
AWS_ENDPOINT_URL: https://127.0.0.1:7073
RUN_SET: "mc"
RECREATE_BUCKETS: "true"
PORT: 7073
- set: 5
LOCAL_FOLDER: /tmp/gw4
BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
LOCAL_FOLDER: /tmp/gw5
BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
IAM_TYPE: s3
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7074
RUN_SET: "aws-user"
RECREATE_BUCKETS: "true"
PORT: 7074

- set: 6
LOCAL_FOLDER: /tmp/gw6
BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam6
AWS_ENDPOINT_URL: https://127.0.0.1:7075
RUN_SET: "aws"
RECREATE_BUCKETS: "false"
PORT: 7075
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
Expand Down Expand Up @@ -97,7 +111,7 @@ jobs:
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: posix
RECREATE_BUCKETS: true
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
CERT: ${{ github.workspace }}/cert.pem
KEY: ${{ github.workspace }}/versitygw.pem
S3CMD_CONFIG: tests/s3cfg.local.default
Expand All @@ -117,6 +131,9 @@ jobs:
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
mkdir $GOCOVERDIR $USERS_FOLDER
if [[ $RECREATE_BUCKETS == "false" ]]; then
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
fi
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
#- name: Build and run, s3 backend
Expand Down
16 changes: 9 additions & 7 deletions tests/env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,17 @@ check_env_vars() {
}

check_universal_vars() {
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
if [[ $BYPASS_ENV_FILE != "true" ]]; then
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
else
log 3 "Warning: no .env file found in tests folder"
fi
else
log 3 "Warning: no .env file found in tests folder"
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
fi
elif [[ $BYPASS_ENV_FILE != "true" ]]; then
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
fi
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets
Expand Down
40 changes: 40 additions & 0 deletions tests/remove_static.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/bash

source ./tests/setup.sh
source ./tests/util.sh
source ./tests/commands/delete_bucket.sh

delete_bucket_if_exists() {
if [[ $# -ne 2 ]]; then
log 2 "delete_bucket_if_exists command missing command type, name"
return 1
fi
bucket_exists "$1" "$2" || local exists_result=$?
if [[ $exists_result -eq 2 ]]; then
log 2 "error checking if bucket exists"
return 1
fi
if [[ $exists_result -eq 1 ]]; then
log 5 "bucket '$2' doesn't exist, skipping"
return 0
fi
if ! delete_bucket "$1" "$2"; then
log 2 "error deleting bucket"
return 1
fi
log 5 "bucket '$2' successfully deleted"
return 0
}

if ! setup; then
log 2 "error starting versity to set up static buckets"
exit 1
fi
if ! delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then
log 2 "error deleting static bucket one"
elif ! delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then
log 2 "error deleting static bucket two"
fi
if ! teardown; then
log 2 "error stopping versity"
fi
18 changes: 0 additions & 18 deletions tests/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@ handle_param() {
show_help
exit 0
;;
-s|--static)
export RECREATE_BUCKETS=false
;;
s3|s3api|aws|s3cmd|mc|aws-user)
set_command_type "$1"
;;
Expand All @@ -41,14 +38,6 @@ set_command_type() {
export command_type
}

if [[ -z $RECREATE_BUCKETS ]]; then
export RECREATE_BUCKETS=true
elif [[ $RECREATE_BUCKETS != true ]] && [[ $RECREATE_BUCKETS != false ]]; then
echo "Invalid RECREATE_BUCKETS value: $RECREATE_BUCKETS"
exit 1
else
export RECREATE_BUCKETS=$RECREATE_BUCKETS
fi
while [[ "$#" -gt 0 ]]; do
handle_param "$1"
shift # past argument or value
Expand All @@ -59,13 +48,6 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
exit 1
fi

if [[ $RECREATE_BUCKETS == false ]]; then
./tests/setup_static.sh || exit_code=$?
if [[ exit_code -ne 0 ]]; then
exit 1
fi
fi

case $command_type in
s3api|aws)
echo "Running aws tests ..."
Expand Down
4 changes: 0 additions & 4 deletions tests/run_all.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,6 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
exit 1
fi

# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
export RECREATE_BUCKETS

if ! ./tests/run.sh aws; then
exit 1
fi
Expand Down
41 changes: 37 additions & 4 deletions tests/setup_static.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,40 @@
#!/bin/bash

source ./tests/setup.sh
setup
aws --no-verify-ssl s3 mb s3://"$BUCKET_ONE_NAME"
aws --no-verify-ssl s3 mb s3://"$BUCKET_TWO_NAME"
teardown
source ./tests/util.sh
source ./tests/commands/create_bucket.sh

create_bucket_if_not_exists() {
if [[ $# -ne 2 ]]; then
log 2 "create_bucket_if_not_exists command missing command type, name"
return 1
fi
bucket_exists "$1" "$2" || local exists_result=$?
if [[ $exists_result -eq 2 ]]; then
log 2 "error checking if bucket exists"
return 1
fi
if [[ $exists_result -eq 0 ]]; then
log 5 "bucket '$2' already exists, skipping"
return 0
fi
if ! create_bucket_object_lock_enabled "$2"; then
log 2 "error creating bucket"
return 1
fi
log 5 "bucket '$2' successfully created"
return 0
}

if ! setup; then
log 2 "error starting versity to set up static buckets"
exit 1
fi
if ! create_bucket_if_not_exists "s3api" "$BUCKET_ONE_NAME"; then
log 2 "error creating static bucket one"
elif ! create_bucket_if_not_exists "s3api" "$BUCKET_TWO_NAME"; then
log 2 "error creating static bucket two"
fi
if ! teardown; then
log 2 "error stopping versity"
fi
51 changes: 26 additions & 25 deletions tests/test_aws.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,16 @@ export RUN_USERS=true
local bucket_file="bucket-file"
bucket_file_data="test file\n"

create_test_files "$bucket_file" || local created=$?
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
create_test_files "$bucket_file" || fail "error creating test files"
printf "%s" "$bucket_file_data" > "$test_file_folder/$bucket_file"

setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'"

run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
[[ $abort_result -eq 0 ]] || fail "Abort failed"
run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "abort failed"

object_exists "aws" "$BUCKET_ONE_NAME" "$bucket_file" || exists=$?
[[ $exists -eq 1 ]] || fail "Upload file exists after abort"
if object_exists "aws" "$BUCKET_ONE_NAME" "$bucket_file"; then
fail "Upload file exists after abort"
fi

delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
Expand All @@ -56,18 +55,15 @@ export RUN_USERS=true
local bucket_file="bucket-file"
bucket_file_data="test file\n"

create_test_files "$bucket_file" || local created=$?
create_test_files "$bucket_file" || fail "error creating test files"
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"

multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
[[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'"

multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "error performing multipart upload"

copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || compare_result=$?
[[ $compare_result -eq 0 ]] || fail "Files do not match"
copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "error copying file"
compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || fail "files do not match"

delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
Expand Down Expand Up @@ -105,25 +101,30 @@ export RUN_USERS=true
os_name="$(uname)"
if [[ "$os_name" == "Darwin" ]]; then
now=$(date -u +"%Y-%m-%dT%H:%M:%S")
five_seconds_later=$(date -j -v +10S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
later=$(date -j -v +15S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
else
now=$(date +"%Y-%m-%dT%H:%M:%S")
five_seconds_later=$(date -d "$now 10 seconds" +"%Y-%m-%dT%H:%M:%S")
later=$(date -d "$now 15 seconds" +"%Y-%m-%dT%H:%M:%S")
fi

create_test_files "$bucket_file" || fail "error creating test file"
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file

delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME" || fail "error deleting bucket, or checking for existence"
create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" || fail "error creating bucket"
# in static bucket config, bucket will still exist
bucket_exists "s3api" "$BUCKET_ONE_NAME" || local exists_result=$?
[[ $exists_result -ne 2 ]] || fail "error checking for bucket existence"
if [[ $exists_result -eq 1 ]]; then
create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" || fail "error creating bucket"
fi

log 5 "$five_seconds_later"
log 5 "LATER: $later"
multipart_upload_with_params "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 \
"$expected_content_type" \
"{\"$expected_meta_key\": \"$expected_meta_val\"}" \
"$expected_hold_status" \
"$expected_retention_mode" \
"$five_seconds_later" \
"$later" \
"$expected_tag_key=$expected_tag_val" || fail "error performing multipart upload"

head_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error getting metadata"
Expand All @@ -139,7 +140,7 @@ export RUN_USERS=true
retention_mode=$(echo "$raw_metadata" | jq -r ".ObjectLockMode")
[[ $retention_mode == "$expected_retention_mode" ]] || fail "retention mode mismatch ($retention_mode, $expected_retention_mode)"
retain_until_date=$(echo "$raw_metadata" | jq -r ".ObjectLockRetainUntilDate")
[[ $retain_until_date == "$five_seconds_later"* ]] || fail "retention date mismatch ($retain_until_date, $five_seconds_later)"
[[ $retain_until_date == "$later"* ]] || fail "retention date mismatch ($retain_until_date, $five_seconds_later)"

get_object_tagging "aws" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error getting tagging"
log 5 "tags: $tags"
Expand All @@ -154,7 +155,7 @@ export RUN_USERS=true
get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "error getting object"
compare_files "$test_file_folder/$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "files not equal"

sleep 10
sleep 15

delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files $bucket_file
Expand Down
17 changes: 10 additions & 7 deletions tests/util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -153,13 +153,15 @@ delete_bucket_or_contents() {
echo "error deleting bucket contents"
return 1
fi
log 5 "bucket contents deletion success"
return 0
fi
delete_bucket_recursive "$1" "$2" || local delete_result=$?
if [[ $delete_result -ne 0 ]]; then
echo "Bucket deletion error"
return 1
fi
log 5 "bucket deletion success"
return 0
}

Expand All @@ -180,10 +182,8 @@ delete_bucket_if_exists() {
log 2 "error deleting bucket or contents"
return 1
fi
#if [[ $RECREATE_BUCKETS == "false" ]]; then
log 5 "bucket and/or bucket data deletion success"
return 0
#fi
fi
if [[ $RECREATE_BUCKETS == "false" ]]; then
log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
Expand All @@ -208,12 +208,15 @@ setup_bucket() {
fi
local create_result
log 5 "util.setup_bucket: command type: $1, bucket name: $2"
create_bucket "$1" "$2" || create_result=$?
if [[ $create_result -ne 0 ]]; then
log 2 "Error creating bucket"
return 1
if [[ $RECREATE_BUCKETS == "true" ]]; then
if ! create_bucket "$1" "$2"; then
log 2 "Error creating bucket"
return 1
fi
log 5 "bucket creation success"
else
log 5 "skipping bucket re-creation"
fi
log 5 "Bucket creation success"
return 0
}

Expand Down
Loading

0 comments on commit c6e1757

Please sign in to comment.