Skip to content

Latest commit

 

History

History
213 lines (188 loc) · 11 KB

m5.12xlarge-io1-ebs-benchmark.md

File metadata and controls

213 lines (188 loc) · 11 KB

m5.12xlarge 48vCPU 192GiB RAM io1 2048GiB provision IOPS 40000

Prepare

# Mount your EBS volume
##Use the lsblk command to view your available disk devices and their mount points
[ec2-user@ip-10-0-0-224 ~]$ sudo lsblk
NAME          MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
nvme1n1       259:0    0   2T  0 disk
nvme0n1       259:1    0  20G  0 disk
├─nvme0n1p1   259:2    0  20G  0 part /
└─nvme0n1p128 259:3    0   1M  0 part

##Determine whether there is a file system on the volume
[ec2-user@ip-10-0-0-166 ~]$ sudo file -s /dev/nvme1n1
/dev/nvme1n1: SGI XFS filesystem data (blksz 4096, inosz 512, v2 dirs)

redo:
[ec2-user@ip-10-0-0-224 ~]$ sudo file -s /dev/nvme1n1
/dev/nvme1n1: data

##use the mkfs -t command to create a file system on the volume
[ec2-user@ip-10-0-0-224 ~]$ sudo mkfs -t xfs /dev/nvme1n1
meta-data=/dev/nvme1n1           isize=512    agcount=4, agsize=134217728 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=1        finobt=1, sparse=0
data     =                       bsize=4096   blocks=536870912, imaxpct=5
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal log           bsize=4096   blocks=262144, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

##mount the volume
[ec2-user@ip-10-0-0-224 ~]$ sudo mkdir -p /mnt/p_iops_vol0
[ec2-user@ip-10-0-0-224 ~]$ sudo mount /dev/nvme1n1 /mnt/p_iops_vol0
[ec2-user@ip-10-0-0-224 ~]$ sudo lsblk
NAME          MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
nvme1n1       259:0    0   2T  0 disk /mnt/p_iops_vol0
nvme0n1       259:1    0  20G  0 disk
├─nvme0n1p1   259:2    0  20G  0 part /
└─nvme0n1p128 259:3    0   1M  0 part
[ec2-user@ip-10-0-0-224 ~]$ df -h
Filesystem      Size  Used Avail Use% Mounted on
devtmpfs         93G     0   93G   0% /dev
tmpfs            93G     0   93G   0% /dev/shm
tmpfs            93G  556K   93G   1% /run
tmpfs            93G     0   93G   0% /sys/fs/cgroup
/dev/nvme0n1p1   20G  1.4G   19G   7% /
tmpfs            19G     0   19G   0% /run/user/1000
/dev/nvme1n1    2.0T  2.1G  2.0T   1% /mnt/p_iops_vol0

##Automatically Mount an Attached Volume After Reboot
##Create a backup of your /etc/fstab file
sudo cp /etc/fstab /etc/fstab.orig

##Use the blkid command to find the UUID of the device.
[ec2-user@ip-10-0-0-224 ~]$ sudo blkid
/dev/nvme0n1p1: LABEL="/" UUID="a1e1011e-e38f-408e-878b-fed395b47ad6" TYPE="xfs" PARTLABEL="Linux" PARTUUID="48273af3-b295-415e-8978-b786bf246692"
/dev/nvme1n1: UUID="c5cb98cf-8612-4491-9b49-73d47500fa26" TYPE="xfs"
/dev/nvme0n1: PTUUID="f502fa32-96bf-48e8-bdf0-166c1e74f8fa" PTTYPE="gpt"
/dev/nvme0n1p128: PARTLABEL="BIOS Boot Partition" PARTUUID="48755cf9-8654-40ce-b91b-848334934b6c"

##Add the following entry to /etc/fstab to mount the device at the specified mount point.
sudo vim /etc/fstab
UUID=c5cb98cf-8612-4491-9b49-73d47500fa26 /mnt/p_iops_vol0 xfs defaults,nofail 0 2


##To verify that your entry works
[ec2-user@ip-10-0-0-166 ~]$ sudo umount /mnt/p_iops_vol0
[ec2-user@ip-10-0-0-166 ~]$ sudo mount -a
[ec2-user@ip-10-0-0-166 ~]$ df -h

Run the testing

## install XFS file system support
[ec2-user@ip-10-0-0-224 ~]$ sudo yum install -y xfsprogs
Loaded plugins: extras_suggestions, langpacks, priorities, update-motd
amzn2-core                                                                                            | 2.4 kB  00:00:00
Package xfsprogs-4.5.0-18.amzn2.0.1.x86_64 already installed and latest version
Nothing to do

## Install Benchmark Tools
[ec2-user ~]$ sudo yum install -y fio

## Disable C-States
[ec2-user@ip-10-0-0-166 ~]$ sudo cpupower idle-info | grep "Number of idle states:"
Number of idle states: 4
[ec2-user@ip-10-0-0-69 ~]$ N=4
[ec2-user@ip-10-0-0-69 ~]$ for i in `seq 1 $((N-1))`; do sudo cpupower idle-set -d $i; done
[ec2-user@ip-10-0-0-69 ~]$ sudo cpupower idle-info | grep "Number of idle states:"

performs 16 KB random write operations on new EBS volume

sudo fio --ioengine=libaio --iodepth=32 --refill_buffers --randrepeat=0 --directory=/mnt/p_iops_vol0 --name fio_test_file --direct=1 --rw=randwrite --bs=16k --size=1G --numjobs=16 --time_based --runtime=180 --group_reporting --norandommap
fio_test_file: (groupid=0, jobs=16): err= 0: pid=13513: Thu Aug  8 09:55:05 2019
  write: io=113112MB, bw=643465KB/s, iops=40216, runt=180004msec
    slat (usec): min=3, max=237609, avg=393.67, stdev=836.40
    clat (usec): min=861, max=250009, avg=12333.84, stdev=5065.85
     lat (usec): min=869, max=250032, avg=12727.51, stdev=5197.89
    clat percentiles (usec):
     |  1.00th=[ 3824],  5.00th=[ 5792], 10.00th=[ 6432], 20.00th=[ 7584],
     | 30.00th=[ 9280], 40.00th=[10048], 50.00th=[11968], 60.00th=[12992],
     | 70.00th=[14912], 80.00th=[16512], 90.00th=[19072], 95.00th=[21632],
     | 99.00th=[25472], 99.50th=[27008], 99.90th=[29824], 99.95th=[31616],
     | 99.99th=[36608]
    lat (usec) : 1000=0.01%
    lat (msec) : 2=0.01%, 4=1.33%, 10=37.84%, 20=53.48%, 50=7.33%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%
  cpu          : usr=1.05%, sys=1.55%, ctx=3183026, majf=0, minf=165
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=7239145/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=32

Run status group 0 (all jobs):
  WRITE: io=113112MB, aggrb=643465KB/s, minb=643465KB/s, maxb=643465KB/s, mint=180004msec, maxt=180004msec

Disk stats (read/write):
  nvme1n1: ios=4/7238362, merge=0/3, ticks=4/21976492, in_queue=22511988, util=100.00%

# without --ioengine=libaio --iodepth=32
sudo fio --directory=/mnt/p_iops_vol0 --name fio_test_file --direct=1 --rw=randwrite --bs=16k --size=1G --numjobs=16 --time_based --runtime=180 --group_reporting --norandommap
fio_test_file: (groupid=0, jobs=16): err= 0: pid=13581: Thu Aug  8 01:29:54 2019
  write: io=79524MB, bw=452401KB/s, iops=28275, runt=180001msec
    clat (usec): min=355, max=86699, avg=565.24, stdev=206.07
     lat (usec): min=355, max=86699, avg=565.44, stdev=206.07
    clat percentiles (usec):
     |  1.00th=[  450],  5.00th=[  474], 10.00th=[  486], 20.00th=[  502],
     | 30.00th=[  516], 40.00th=[  532], 50.00th=[  548], 60.00th=[  564],
     | 70.00th=[  588], 80.00th=[  612], 90.00th=[  660], 95.00th=[  700],
     | 99.00th=[  836], 99.50th=[  956], 99.90th=[ 1592], 99.95th=[ 3152],
     | 99.99th=[ 6240]
    lat (usec) : 500=18.10%, 750=79.67%, 1000=1.81%
    lat (msec) : 2=0.34%, 4=0.04%, 10=0.03%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.24%, sys=1.71%, ctx=5092987, majf=0, minf=150
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=5089540/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=79524MB, aggrb=452401KB/s, minb=452401KB/s, maxb=452401KB/s, mint=180001msec, maxt=180001msec

Disk stats (read/write):
  nvme1n1: ios=4/5090462, merge=0/2, ticks=0/2826948, in_queue=2696448, util=100.00%

redo performs 16 KB random read operations on new EBS volume

sudo fio --ioengine=libaio --iodepth=32 --refill_buffers --randrepeat=0 --directory=/mnt/p_iops_vol0 --name fio_test_file --direct=1 --rw=randread --bs=16k --size=1G --numjobs=16 --time_based --runtime=180 --group_reporting --norandommap 
fio_test_file: (groupid=0, jobs=16): err= 0: pid=13699: Thu Aug  8 10:06:23 2019
  read : io=113370MB, bw=644937KB/s, iops=40308, runt=180004msec
    slat (usec): min=2, max=8938, avg=395.24, stdev=801.56
    clat (usec): min=0, max=214214, avg=12305.99, stdev=5708.39
     lat (usec): min=3, max=214216, avg=12701.23, stdev=5873.50
    clat percentiles (usec):
     |  1.00th=[ 3472],  5.00th=[ 5152], 10.00th=[ 6112], 20.00th=[ 6816],
     | 30.00th=[ 8032], 40.00th=[ 9280], 50.00th=[10688], 60.00th=[13376],
     | 70.00th=[15936], 80.00th=[18048], 90.00th=[20096], 95.00th=[22144],
     | 99.00th=[25472], 99.50th=[26752], 99.90th=[29312], 99.95th=[30848],
     | 99.99th=[36608]
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.11%, 4=2.15%, 10=44.62%, 20=42.73%, 50=10.38%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.44%, sys=1.47%, ctx=2144521, majf=0, minf=2192
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=7255707/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=32

Run status group 0 (all jobs):
   READ: io=113370MB, aggrb=644937KB/s, minb=644937KB/s, maxb=644937KB/s, mint=180004msec, maxt=180004msec

Disk stats (read/write):
  nvme1n1: ios=7238486/6, merge=0/0, ticks=21942248/12, in_queue=22087228, util=100.00%

# without --ioengine=libaio --iodepth=32  
sudo fio --directory=/mnt/p_iops_vol0 --name fio_test_file --direct=1 --rw=randread --bs=16k --size=1G --numjobs=16 --time_based --runtime=180 --group_reporting --norandommap 
fio_test_file: (groupid=0, jobs=16): err= 0: pid=13787: Thu Aug  8 01:39:46 2019
  read : io=113093MB, bw=643372KB/s, iops=40210, runt=180001msec
    clat (usec): min=2, max=167226, avg=397.51, stdev=398.71
     lat (usec): min=2, max=167226, avg=397.55, stdev=398.71
    clat percentiles (usec):
     |  1.00th=[  211],  5.00th=[  266], 10.00th=[  302], 20.00th=[  334],
     | 30.00th=[  354], 40.00th=[  366], 50.00th=[  382], 60.00th=[  394],
     | 70.00th=[  410], 80.00th=[  434], 90.00th=[  482], 95.00th=[  532],
     | 99.00th=[  668], 99.50th=[  796], 99.90th=[ 3024], 99.95th=[ 4128],
     | 99.99th=[10944]
    lat (usec) : 4=0.01%, 10=0.01%, 250=3.65%, 500=88.55%, 750=7.23%
    lat (usec) : 1000=0.15%
    lat (msec) : 2=0.21%, 4=0.15%, 10=0.04%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.25%, sys=1.71%, ctx=7237350, majf=0, minf=207
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=7237977/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
   READ: io=113093MB, aggrb=643372KB/s, minb=643372KB/s, maxb=643372KB/s, mint=180001msec, maxt=180001msec

Disk stats (read/write):
  nvme1n1: ios=7228433/9, merge=0/13, ticks=2819176/0, in_queue=2698284, util=100.00%