Dose-701 Schedule zloop run with object store (openzfs#67)

Added a workflow to configure minio as s3 backend and run the zloop test with object storage.
The test with object store is controlled by the environment variable `ZTS_OBJECT_STORE`. 
Zloop continues to run using block based devices if this variable remains unset.

.github/workflows/zloop-object-store.yml

@@ -0,0 +1,88 @@
+name: zloop-object-store
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  zloop:
+    runs-on: ubuntu-latest
+    env:
+      TEST_DIR: /var/tmp/zloop
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install --yes -qq build-essential autoconf libtool gdb \
+          git alien fakeroot \
+          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
+          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
+          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
+          libpam0g-dev \
+          python-dev python-setuptools python-cffi python-packaging \
+          python3 python3-dev python3-setuptools python3-cffi python3-packaging
+    - name: Autogen.sh
+      run: |
+        sh autogen.sh
+    - name: Configure
+      run: |
+        ./configure --enable-debug --enable-debuginfo
+    - name: Make
+      run: |
+        make --no-print-directory -s pkg-utils pkg-kmod
+    - name: Install
+      run: |
+        sudo dpkg -i *.deb
+        # Update order of directories to search for modules, otherwise
+        #   Ubuntu will load kernel-shipped ones.
+        sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
+        sudo depmod
+        sudo modprobe zfs
+    - name: Setup minio
+      run: |
+        docker run -d -p 9000:9000 --name minio --network=host \
+                   -e "MINIO_ACCESS_KEY=minioadmin" \
+                   -e "MINIO_SECRET_KEY=minioadmin" \
+                   -v /tmp/data:/data \
+                   -v /tmp/config:/root/.minio \
+                   minio/minio server /data
+
+        export AWS_EC2_METADATA_DISABLED=true
+        aws configure set aws_access_key_id minioadmin && aws configure set aws_secret_access_key minioadmin
+        aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket
+        sudo cp -r $HOME/.aws /root
+    - name: Tests
+      run: |
+        sudo mkdir -p $TEST_DIR
+        # run for 20 minutes to have a total runner time of 30 minutes
+        sudo -E /usr/share/zfs/zloop.sh -t 1200 -l -m1 -- -T 120 -P 60
+      env:
+        LD_LIBRARY_PATH: /lib/x86_64-linux-gnu
+        ZTS_OBJECT_STORE: true
+        ZTS_BUCKET_NAME: testbucket
+        ZTS_OBJECT_ENDPOINT: http://127.0.0.1:9000
+        ZTS_REGION: us-west-2
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+    - name: Prepare artifacts
+      if: failure()
+      run: |
+        sudo chmod +r -R $TEST_DIR/
+    - uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: Logs
+        path: |
+          /var/tmp/zloop/*/
+          !/var/tmp/zloop/*/vdev/
+        if-no-files-found: ignore
+    - uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: Pool files
+        path: |
+          /var/tmp/zloop/*/vdev/
+        if-no-files-found: ignore

scripts/zloop.sh

@@ -181,6 +181,46 @@ function store_core
 	fi
 }
 
+#
+# Returns if S3 credentials are available for
+# configuring the zloop test
+#
+function are_s3_credentials_available
+{
+	[ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY" ] && \
+		return 0 || return 1
+}
+
+#
+# Configures and sets the S3 credentials using the aws cli tool
+#
+function configure_and_set_s3_credentials() {
+	# Check and comment out the AWS_ environment variables
+	# from the /etc/environment file
+	if grep -q "^AWS" /etc/environment 2>/dev/null; then
+		sudo sed -i "s/^AWS/# AWS/g" /etc/environment
+	fi
+	# If aws cli is installed and is in path
+	if command -v aws >/dev/null 2>&1; then
+		aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
+		aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
+		sudo mkdir -p /root/.aws && \
+			sudo cp ~/.aws/credentials /root/.aws/credentials
+	else
+		echo "The aws cli tool is missing or not available in the" \
+			" path. Aborting zloop test with object store."
+		exit 1
+	fi
+}
+
+#
+# Returns if object store is being used
+#
+function use_object_store
+{
+	[ -n "$ZTS_OBJECT_STORE" ] && return 0 || return 1
+}
+
 # parse arguments
 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
 coredir=$DEFAULTCOREDIR
@@ -250,64 +290,82 @@ while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do
 
 	zopt="-G -VVVVV"
 
-	# start each run with an empty directory
-	workdir="$basedir/$rundir"
-	or_die rm -rf "$workdir"
-	or_die mkdir "$workdir"
-
-	# switch between three types of configs
-	# 1/3 basic, 1/3 raidz mix, and 1/3 draid mix
-	choice=$((RANDOM % 3))
-
-	# ashift range 9 - 15
-	align=$(((RANDOM % 2) * 3 + 9))
-
-	# randomly use special classes
-	class="special=random"
-
-	if [[ $choice -eq 0 ]]; then
-		# basic mirror only
-		parity=1
-		mirrors=2
-		draid_data=0
-		draid_spares=0
-		raid_children=0
-		vdevs=2
-		raid_type="raidz"
-	elif [[ $choice -eq 1 ]]; then
-		# fully randomized mirror/raidz (sans dRAID)
-		parity=$(((RANDOM % 3) + 1))
-		mirrors=$(((RANDOM % 3) * 1))
-		draid_data=0
-		draid_spares=0
-		raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
-		vdevs=$(((RANDOM % 3) + 3))
-		raid_type="raidz"
+	if use_object_store; then
+		# If S3 credentials are provided configure and
+		# save it.
+		if are_s3_credentials_available; then
+			configure_and_set_s3_credentials
+		else
+			# For running test using instance profile
+			# we need to remove the underlying credentials
+			# stored in the disk
+			rm -f ~/.aws/credentials
+			sudo rm -f /root/.aws/credentials
+		fi
+		zopt="$zopt -O $ZTS_OBJECT_ENDPOINT"
+		zopt="$zopt -A $ZTS_REGION"
+		zopt="$zopt -b $ZTS_BUCKET_NAME"
+		[ -z "$ZTS_CREDS_PROFILE" ] && ZTS_CREDS_PROFILE="default"
+		zopt="$zopt -z $ZTS_CREDS_PROFILE"
 	else
-		# fully randomized dRAID (sans mirror/raidz)
-		parity=$(((RANDOM % 3) + 1))
-		mirrors=0
-		draid_data=$(((RANDOM % 8) + 3))
-		draid_spares=$(((RANDOM % 2) + parity))
-		stripe=$((draid_data + parity))
-		extra=$((draid_spares + (RANDOM % 4)))
-		raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra))
-		vdevs=$((RANDOM % 3))
-		raid_type="draid"
-	fi
-
-	zopt="$zopt -K $raid_type"
-	zopt="$zopt -m $mirrors"
-	zopt="$zopt -r $raid_children"
-	zopt="$zopt -D $draid_data"
-	zopt="$zopt -S $draid_spares"
-	zopt="$zopt -R $parity"
-	zopt="$zopt -v $vdevs"
-	zopt="$zopt -a $align"
-	zopt="$zopt -C $class"
-	zopt="$zopt -s $size"
-	zopt="$zopt -f $workdir"
+		# start each run with an empty directory
+		workdir="$basedir/$rundir"
+		or_die rm -rf "$workdir"
+		or_die mkdir "$workdir"
+
+		# switch between three types of configs
+		# 1/3 basic, 1/3 raidz mix, and 1/3 draid mix
+		choice=$((RANDOM % 3))
+
+		# ashift range 9 - 15
+		align=$(((RANDOM % 2) * 3 + 9))
+
+		# randomly use special classes
+		class="special=random"
+
+		if [[ $choice -eq 0 ]]; then
+			# basic mirror only
+			parity=1
+			mirrors=2
+			draid_data=0
+			draid_spares=0
+			raid_children=0
+			vdevs=2
+			raid_type="raidz"
+		elif [[ $choice -eq 1 ]]; then
+			# fully randomized mirror/raidz (sans dRAID)
+			parity=$(((RANDOM % 3) + 1))
+			mirrors=$(((RANDOM % 3) * 1))
+			draid_data=0
+			draid_spares=0
+			raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
+			vdevs=$(((RANDOM % 3) + 3))
+			raid_type="raidz"
+		else
+			# fully randomized dRAID (sans mirror/raidz)
+			parity=$(((RANDOM % 3) + 1))
+			mirrors=0
+			draid_data=$(((RANDOM % 8) + 3))
+			draid_spares=$(((RANDOM % 2) + parity))
+			stripe=$((draid_data + parity))
+			extra=$((draid_spares + (RANDOM % 4)))
+			raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra))
+			vdevs=$((RANDOM % 3))
+			raid_type="draid"
+		fi
 
+		zopt="$zopt -K $raid_type"
+		zopt="$zopt -m $mirrors"
+		zopt="$zopt -r $raid_children"
+		zopt="$zopt -D $draid_data"
+		zopt="$zopt -S $draid_spares"
+		zopt="$zopt -R $parity"
+		zopt="$zopt -v $vdevs"
+		zopt="$zopt -a $align"
+		zopt="$zopt -C $class"
+		zopt="$zopt -s $size"
+		zopt="$zopt -f $workdir"
+	fi
 	cmd="$ZTEST $zopt $*"
 	desc="$(date '+%m/%d %T') $cmd"
 	echo "$desc" | tee -a ztest.history