diff --git a/README.md b/README.md index e4a1e8d..0919504 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # volbench -Standardized benchmarking for volumes +Standardized benchmarking for volumes using ```fio``` + +There are two flavors: +- ```cli```; benchmarking disks on a linux (virtual) machine +- ```k8s```; benchmarking persistent volumes on kubernetes cluster Example run: diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..e4a1e8d --- /dev/null +++ b/cli/README.md @@ -0,0 +1,6 @@ +# volbench +Standardized benchmarking for volumes + +Example run: + +![volbench sample run](https://github.com/chira001/volbench/blob/main/images/samplerun.png) diff --git a/cli/volbench.sh b/cli/volbench.sh new file mode 100755 index 0000000..6df0737 --- /dev/null +++ b/cli/volbench.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# published at https://github.com/chira001/volbench - PR and comments welcome +# originally inspired by https://github.com/leeliu/dbench + +## checking if fio is installed +if ! command -v fio &> /dev/null +then + echo "fio could not be found, please install fio." + command fio + exit +fi + +# specify a space seperated set of files to use as tests. tests are run in paralled across all files +FIO_files="/tmp/volbenchtest1 /tmp/volbenchtest2" +# note: the test files are not deleted at the end, to make it easy to run multiple tests +# please remember to delete the test files +# # specify the size of the test files +FIO_size=10MB +# specify a ramp time before recording values - this should be around 10 seconds +FIO_ramptime=10 +# specify a runtime for each test - should be 30s minimum, but 120 is preferred +FIO_runtime=120 +# # specify the percentage of read requests in mixed tests +FIO_rwmixread=75 +# specify how many write i/os before an fdatasync - 0 disables +FIO_fdatasync=0 + +# specify default number of jobs per file - default to 1 (don't change this) +FIO_numjobs=1 +#specify default offset_increment - default to 0 (don't change this) +FIO_offset_increment=0 + +while getopts d:s:r:t:w:f:hq option +do + case "${option}" in + d) FIO_files=${OPTARG};; + s) FIO_size=${OPTARG};; + r) FIO_ramptime=${OPTARG};; + t) FIO_runtime=${OPTARG};; + w) FIO_rwmixread=${OPTARG};; + f) FIO_fdatasync=${OPTARG};; + q) + FIO_size=10MB + FIO_ramptime=1 + FIO_runtime=2;; + h) + echo "Usage: $0 [OPTION]..." + echo "Standardized benchmarking for volumes." + echo + echo " -h show the usage" + echo " -q run $0 with reduce ramp and run time to do a real dry-run" + echo + echo "Mandatory arguments for short options." + echo " -d space separated set of files to use as tests in parallel." + echo " e.g. & default; $FIO_files " + echo " -s size of the test files with a measurement unit" + echo " e.g. & default; 10MB" + echo " -r ramp time in seconds before recording values" + echo " e.g. & default; 10" + echo " -t run time in for each test" + echo " e.g. & default; 120" + echo " -w workload ratio of read requests in percentage" + echo " e.g. & default; 75" + echo " -f how many write I/Os before a fdatasync happnes" + echo " e.g. & default; 0 (disable)" + echo + echo "Example: $0 -d \"/mnt/data/file1 /mnt/data/file2\" -s 100MB -r 5 -t 5 -w 50 -f 50" + echo "PR and Comments: " + exit ;; + esac +done + +if [[ $# -eq 0 ]] +then + echo "$0 will run with the following recommended parameters:" + echo " FIO_files=$FIO_files" + echo " FIO_size=$FIO_size" + echo " FIO_ramptime=$FIO_ramptime" + echo " FIO_runtime=$FIO_runtime" + echo " FIO_rwmixread=$FIO_rwmixread" + echo " FIO_fdatasync=$FIO_fdatasync" + echo "The total running time will be around 15 minutes." + echo + echo "For rapid dry-run, run \"$0 -q\" (total run time is around 30 seconds)." + echo "For details on the usage, run \"$0 -h\"." + echo + + # below read and case should be remove when containerized + #read -p "Continue with the above values? " answer + #case $answer in + # [yY] ) echo "Starting $0 with default parameters...";; + # [nN] ) exit;; + #esac +fi + + + +#define some colour escape codes +YELLOW='\033[1;33m' +GREEN='\033[1;32m' +CYAN='\033[1;34m' +NC='\033[0m' + + +# global setttings for all fio jobs +function fio-global { + echo "[global] +bs=${FIO_blocksize}k +ioengine=libaio +iodepth=$FIO_queuedepth +thread +direct=1 +fdatasync=$FIO_fdatasync +random_generator=tausworthe64 +random_distribution=random +rw=$FIO_readwrite +rwmixread=$FIO_rwmixread +percentage_random=$FIO_percentage_random +group_reporting=0 +time_based +ramp_time=$FIO_ramptime +runtime=$FIO_runtime" + echo +} + +# setup a job per test file +function fio-job { + for inp in $* + do + echo "[$inp]" + echo "numjobs=$FIO_numjobs" + echo "filename=$inp" + echo "size=$FIO_size" + echo "offset=0" + echo "offset_increment=$FIO_offset_increment" + done +} + +# parse the output of fio minimal data output +function fio-parse { + awk -F';' 'BEGIN {printf "%30s %8s %9s %8s %8s %9s %8s\n", "Test file", "R iops", "R lat ms", "R MB/s", "W iops", "W lat ms", "W MB/s"} {records +=1} {readsum += $8} {writesum += $49} {readmb += $7} {writemb += $48} {readlats += $40 } {writelats += $81} {printf "%30s '$YELLOW'%8d'$NC' '$GREEN'%9.3f'$NC' %8.1f '$YELLOW'%8d'$NC' '$GREEN'%9.3f'$NC' %8.1f \n", $3, $8, $40/1000, $7/1024, $49, $81/1000, $48/1024} END {printf "'$CYAN'%30s %8d %9.3f %8.1f %8d %9.3f %8.1f'$NC'\n\n", "TOTAL/Average", readsum, (readlats/records)/1000, readmb/1024, writesum, (writelats/records)/1000, writemb/1024}' +} + +#return a field from the minimal fio +function fio-getfield { + awk -F';' '{records +=1} {readsum += $8} {writesum += $49} {readmb += $7} {writemb += $48} {readlats += $40 } {writelats += $81} END {printf "%d %.3f %.1f %d %.3f %.1f\n", readsum, (readlats/records)/1000, readmb/1024, writesum, (writelats/records)/1000, writemb/1024}' | awk '{print $'$1'}' +} + +# sync and clear the caches, then run the fio job +function fio-run { + sync + + #if we are root, then drop the kernel caches + if [ $UID -eq 0 ] + then echo 1 > /proc/sys/vm/drop_caches + fi + + #combine the global params and the jobspec and run fio + ( fio-global ; fio-job $FIO_files ) | fio --minimal - +} + + +#start the suite of tests +echo +echo "Starting VolBench tests ..." +echo +echo "Test parameters:" +echo "FIO_files=$FIO_files" +echo "FIO_size=$FIO_size FIO_ramptime=$FIO_ramptime FIO_runtime=$FIO_runtime" +echo ; echo + + +# test concurrent random read iops - e.g. db queries/message bus +echo Testing read iops ... +FIO_blocksize=4k FIO_queuedepth=16 FIO_readwrite=randread FIO_percentage_random=100 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +READ_IOPS=`echo "$FIO_output" | fio-getfield 1` + +# test concurrent randdom write iops - e.g. db commits +echo Testing write iops ... +FIO_blocksize=4k FIO_queuedepth=16 FIO_readwrite=randwrite FIO_percentage_random=100 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +WRITE_IOPS=`echo "$FIO_output" | fio-getfield 4` + +# test read throughput +echo Testing read throughput ... +FIO_blocksize=128k FIO_queuedepth=16 FIO_readwrite=randread FIO_percentage_random=100 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +READ_MB=`echo "$FIO_output" | fio-getfield 3` + +# test write throughput +echo Testing write throughput ... +FIO_blocksize=128k FIO_queuedepth=16 FIO_readwrite=randwrite FIO_percentage_random=100 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +WRITE_MB=`echo "$FIO_output" | fio-getfield 6` + +# test read latency, low concurrency +echo Testing read latency ... +FIO_blocksize=4k FIO_queuedepth=4 FIO_readwrite=randread FIO_percentage_random=100 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +READ_LAT=`echo "$FIO_output" | fio-getfield 2` + +# test write latency, low concurrency +echo Testing write latency ... +FIO_blocksize=4k FIO_queuedepth=4 FIO_readwrite=randwrite FIO_percentage_random=100 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +WRITE_LAT=`echo "$FIO_output" | fio-getfield 5` + +# test concurrent read and write iops +echo Testing mixed iops ... +FIO_blocksize=4k FIO_queuedepth=16 FIO_readwrite=randrw FIO_percentage_random=100 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +READ_MIXED=`echo "$FIO_output" | fio-getfield 1` +WRITE_MIXED=`echo "$FIO_output" | fio-getfield 4` + +# update FIO_size and set increment to be able to split across 4 jobs +FIO_unit=`echo $FIO_size | sed 's/[0-9]//g'` +FIO_sizenumber=`echo $FIO_size | sed 's/[a-z]//ig'` +FIO_offset_increment=`expr $FIO_sizenumber / 4`$FIO_unit +FIO_oldsize=$FIO_size +FIO_size=$FIO_offset_increment + +# test read sequental throughput +echo Testing read seqential ... +FIO_blocksize=1M FIO_queuedepth=4 FIO_readwrite=read FIO_percentage_random=0 FIO_numjobs=4 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +READ_SEQ=`echo "$FIO_output" | fio-getfield 3` + +# test write sequention throughput +echo Testing write seqential ... +FIO_blocksize=1M FIO_queuedepth=4 FIO_readwrite=write FIO_percentage_random=0 FIO_numjobs=4 +FIO_output=$(fio-run) +echo "$FIO_output" | fio-parse +WRITE_SEQ=`echo "$FIO_output" | fio-getfield 6` + + +#output final report +echo +printf "%22s %10s %10s \n" "VolBench Summary" "Reads" "Writes" +echo "---------------------------------------------------------" +printf "%22s: %10s iops %10s iops\n" "I/O operations" ${READ_IOPS} ${WRITE_IOPS} +printf "%22s: %10s iops %10s iops\n" "Mixed I/O" ${READ_MIXED} ${WRITE_MIXED} +printf "%22s: %10s ms %10s ms \n" "Latency" ${READ_LAT} ${WRITE_LAT} +printf "%22s: %10s MB/s %10s MB/s\n" "Random throughput" ${READ_MB} ${WRITE_MB} +printf "%22s: %10s MB/s %10s MB/s\n" "Sequential throughput" ${READ_SEQ} ${WRITE_SEQ} +echo ; echo + +exit diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..164a5c2 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,97 @@ +## volbench on k8s +Standardized benchmarking for volumes hosted on kubernetes + +### how does it look like +To guarantee the simplest usage possible, the k8s version is composed of two files: +- ```volbench.sh``` which is the actual bash script calling ```fio``` with the relevant benchmarking profile +- ```volbench_storageos.yaml``` which is a standard yaml configuration file using the StorageOS ```fast``` default StorageClass +- ```volbench_hostpath.yaml```which is a standard yaml configuration file using the HostPath (local filesystem) as StorageClass + +Note: ```volbench.sh``` has very few changes from the CLI version to run smoothly as a pod on k8s. + +### what does ```volbench.yaml``` do +When applying the file towards a k8s cluster, it will create: +- a namespace called ```volbench``` +- a persistent volume claim called ```volbenchtemp1``` linked to the created namespace +- a pod called ```volbench-runner``` within the created namespace consuming the persistent volume + +Here is the full yaml file: +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: volbench +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: volbenchtemp1 + namespace: volbench +spec: + storageClassName: "fast" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: volbench-runner + namespace: volbench +spec: + containers: + - name: ubuntu + image: ubuntu:latest + command: ["/bin/sh"] + args: ["-c", 'apt update; apt install -y fio git; git clone --single-branch --branch containerized http://github.com/rovandep/volbench.git; /volbench/k8s/volbench.sh; sleep 36000'] + volumeMounts: + - mountPath: /mnt + name: tmp1 + env: + - name: FIO_files + value: "/mnt/volbenchtest1 /mnt/volbenchtest2" + - name: FIO_size + value: "1MB" + - name: FIO_ramptime + value: "1" + - name: FIO_runtime + value: "5" + - name: FIO_rwmixread + value: "75" + - name: FIO_fdatasync + value: "0" + volumes: + - name: tmp1 + persistentVolumeClaim: + claimName: volbenchtemp1 +``` + +Note: there are environment variables defined under ```env:``` within the YAML configuration file defining the ```fio``` benchmarking profile. +The following extract from ```volbench.sh``` provides a details about each variables: + +```bash +# specify a space seperated set of files to use as tests. tests are run in paralled across all files +FIO_files="/mnt/volbenchtest1 /mnt/volbenchtest2" +# note: the test files are not deleted at the end, to make it easy to run multiple tests +# please remember to delete the test files +# # specify the size of the test files +FIO_size=10MB +# specify a ramp time before recording values - this should be around 10 seconds +FIO_ramptime=10 +# specify a runtime for each test - should be 30s minimum, but 120 is preferred +FIO_runtime=10 +# # specify the percentage of read requests in mixed tests +FIO_rwmixread=75 +# specify how many write i/os before an fdatasync - 0 disables +FIO_fdatasync=0 +``` + +### how to use it +In a nutshell, from the above YAML output: +- the ```storageClassName``` might need to be change to match the existing ```storageClassName``` on the targeted k8s. +- the ```env``` field might need to be change to match the desired ```fio``` benchmarking profile. + +Then: +[![asciicast](https://asciinema.org/a/407266.svg)](https://asciinema.org/a/407266) diff --git a/volbench.sh b/k8s/volbench.sh similarity index 90% rename from volbench.sh rename to k8s/volbench.sh index 06a229c..c2f82b2 100755 --- a/volbench.sh +++ b/k8s/volbench.sh @@ -1,56 +1,27 @@ #!/usr/bin/env bash - # published at https://github.com/chira001/volbench - PR and comments welcome - # originally inspired by https://github.com/leeliu/dbench -## checking if fio is installed -if ! command -v fio &> /dev/null -then - echo "fio could not be found, please install fio." - exit -fi - # specify a space seperated set of files to use as tests. tests are run in paralled across all files -if [ -z "${FIO_files}" ] -then FIO_files="/tmp/volbenchtest1 /tmp/volbenchtest2" -fi - +#FIO_files="/tmp/volbenchtest1 /tmp/volbenchtest2" # note: the test files are not deleted at the end, to make it easy to run multiple tests # please remember to delete the test files - -# specify the size of the test files -if [ -z "${FIO_size}" ] -then FIO_size=100MB -fi - +# # specify the size of the test files +#FIO_size=10MB # specify a ramp time before recording values - this should be around 10 seconds -if [ -z "${FIO_ramptime}" ] -then FIO_ramptime=1s -fi - +#FIO_ramptime=10 # specify a runtime for each test - should be 30s minimum, but 120 is preferred -if [ -z "${FIO_runtime}" ] -then FIO_runtime=2s -fi - -# specify the percentage of read requests in mixed tests -if [ -z "${FIO_rwmixread}" ] -then FIO_rwmixread=75 -fi - +#FIO_runtime=10 +# # specify the percentage of read requests in mixed tests +#FIO_rwmixread=75 # specify how many write i/os before an fdatasync - 0 disables -if [ -z "${FIO_datasync}" ] -then FIO_fdatasync=0 -fi - +#FIO_fdatasync=0 # specify default number of jobs per file - default to 1 (don't change this) FIO_numjobs=1 #specify default offset_increment - default to 0 (don't change this) FIO_offset_increment=0 - #define some colour escape codes YELLOW='\033[1;33m' GREEN='\033[1;32m' @@ -58,7 +29,7 @@ CYAN='\033[1;34m' NC='\033[0m' -# global setttins for all fio jobs +# global setttings for all fio jobs function fio-global { echo "[global] bs=${FIO_blocksize}k @@ -107,9 +78,9 @@ function fio-run { sync #if we are root, then drop the kernel caches - if [ $UID -eq 0 ] - then echo 1 > /proc/sys/vm/drop_caches - fi + # if [ $UID -eq 0 ] + # then echo 1 > /proc/sys/vm/drop_caches + # fi #combine the global params and the jobspec and run fio ( fio-global ; fio-job $FIO_files ) | fio --minimal - diff --git a/k8s/volbench_hostpath.yaml b/k8s/volbench_hostpath.yaml new file mode 100644 index 0000000..c4aff13 --- /dev/null +++ b/k8s/volbench_hostpath.yaml @@ -0,0 +1,70 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: volbench-local +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: volbench-pv-hp +spec: + storageClassName: volbench-local + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + local: + path: "/tmp/volbench" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: volbench-hp +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: volbench-pvc-claim + namespace: volbench-hp +spec: + storageClassName: volbench-local + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: volbench-runner + namespace: volbench-hp +spec: + containers: + - name: ubuntu + image: ubuntu:latest + command: ["/bin/sh"] + args: ["-c", 'apt update; apt install -y fio git; git clone --single-branch --branch containerized http://github.com/rovandep/volbench.git; /volbench/k8s/volbench.sh; sleep 36000'] + volumeMounts: + - mountPath: /mnt + name: tmp1 + env: + - name: FIO_files + value: "/mnt/volbenchtest1 /mnt/volbenchtest2" + - name: FIO_size + value: "10MB" + - name: FIO_ramptime + value: "10" + - name: FIO_runtime + value: "60" + - name: FIO_rwmixread + value: "75" + - name: FIO_fdatasync + value: "0" + volumes: + - name: tmp1 + persistentVolumeClaim: + claimName: volbench-pvc-claim diff --git a/k8s/volbench_storageos.yaml b/k8s/volbench_storageos.yaml new file mode 100644 index 0000000..1c2528e --- /dev/null +++ b/k8s/volbench_storageos.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: volbench +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: volbenchtemp1 + namespace: volbench +spec: + storageClassName: "fast" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: volbench-runner + namespace: volbench +spec: + containers: + - name: ubuntu + image: ubuntu:latest + command: ["/bin/sh"] + args: ["-c", 'apt update; apt install -y fio git; git clone --single-branch --branch containerized http://github.com/rovandep/volbench.git; /volbench/k8s/volbench.sh; sleep 36000'] + volumeMounts: + - mountPath: /mnt + name: tmp1 + env: + - name: FIO_files + value: "/mnt/volbenchtest1 /mnt/volbenchtest2" + - name: FIO_size + value: "10MB" + - name: FIO_ramptime + value: "10" + - name: FIO_runtime + value: "60" + - name: FIO_rwmixread + value: "75" + - name: FIO_fdatasync + value: "0" + volumes: + - name: tmp1 + persistentVolumeClaim: + claimName: volbenchtemp1