IO alignment benchmark on a 4-disk RAID10

This page contains scripts and plain results of the IO alignment benchmarks run on this a 4-disk RAID10 array. For more details see these two blog posts:

• Aligning IO on a hard disk RAID – the Theory
• Aligning IO on a hard disk RAID – the Benchmarks

In the results:

1. Baseline - misalignment on the partition table, no LVM and no alignment settings in the file system.
2. Misalignment on the partition table, but proper alignment options on the file system.
3. 1M alignment in partition table but no options on the file system.
4. Partition table and file system properly aligned; sunit/swidth set during mkfs. No LVM at this point.
5. Partition table aligned properly; sunit/swidth set during mounting but not during mkfs.
6. Added LVM on top of aligned partition table, used proper file system alignment.

a and b stand for Write-Through and Write-Back benchmarks respectively.

Sequential reads

Sequential writes

Random reads

Random writes

OLTP results

#!/bin/sh
set -u
set -x
set -e
 
bn=$1-file
 
if [ -d /root/benchmarks/$bn ]; then
  echo "ERROR: directory exists, not overwriting results"
  exit 1
else
  mkdir -p /root/benchmarks/$bn
fi
 
mdf=/root/benchmarks/$bn/metadata
echo ===== MegaRAID info >> $mdf
/opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -aALL >> $mdf
echo ===== fdisk info >> $mdf
fdisk -lu >> $mdf
echo ===== xfs info >> $mdf
xfs_info /data >> $mdf
echo ===== mount >> $mdf
mount | grep /data >> $mdf
echo ===== my.cnf >> $mdf
cat /etc/mysql/my.cnf >> $mdf
bash /root/bin/summary >> $mdf
echo ===== extra lvm info >> $mdf
pvs -o +pe_start >> $mdf
 
cd /data
 
for size in 1G 4G 16G; do
   for mode in seqwr seqrd rndrd rndwr rndrw; do
      sysbench --test=fileio --file-num=64 --file-total-size=$size --init-rng=on --file-extra-flags=direct prepare
      # make sure files are not in cache
      for f in /data/test_file.*; do
         /root/bin/uncache $f
      done
      for threads in 1 4 8; do
         echo "====== testing $threads threads"
         echo PARAMS $size $mode $threads > /root/benchmarks/$bn/sysbench-size-$size-mode-$mode-threads-$threads
         sysbench --test=fileio --file-total-size=$size --file-test-mode=$mode \
            --max-time=60 --max-requests=100000000 --num-threads=$threads --init-rng=on \
            --file-num=64 --file-extra-flags=direct --file-fsync-freq=0 --file-block-size=16384 run \
            | tee -a /root/benchmarks/$bn/sysbench-size-$size-mode-$mode-threads-$threads 2>&1
      done
      sysbench --test=fileio --file-total-size=$size cleanup
   done
done

#!/bin/sh
set -u
set -x
set -e
 
bn=$1-oltp
if [ -d /root/benchmarks/$bn ]; then
  echo "ERROR: directory exists, not overwriting results"
  exit 1
else
  mkdir -p /root/benchmarks/$bn
fi
 
mdf=/root/benchmarks/$bn/metadata
echo ===== MegaRAID info >> $mdf
/opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -aALL >> $mdf
echo ===== fdisk info >> $mdf
fdisk -lu >> $mdf
echo ===== xfs info >> $mdf
xfs_info /data >> $mdf
echo ===== mount >> $mdf
mount | grep /data >> $mdf
echo ===== my.cnf >> $mdf
cat /etc/mysql/my.cnf >> $mdf
bash /root/bin/summary >> $mdf
echo ===== extra lvm info >> $mdf
pvs -o +pe_start >> $mdf
 
mysql -e "create database sbtest"
sysbench --test=oltp --oltp-table-size=20000000 --oltp-read-only=off --oltp-skip-trx=on --init-rng=on --max-requests=0 --oltp-dist-type=uniform --mysql-user=root prepare
 
for threads in 1 4 8 16 32; do
   vmstat 5 > /root/benchmarks/$bn/sysbench-oltp-threads-$threads-vmstat 2>&1 & vpid=$!
   while sleep 5; do cat /proc/diskstats >> /root/benchmarks/$bn/sysbench-oltp-threads-$threads-diskstats; date +'TS %s.%N %F %T' >> /root/benchmarks/$bn/sysbench-oltp-threads-$threads-diskstats; done  & dpid=$!
   mpstat -P ALL 5 >> /root/benchmarks/$bn/sysbench-oltp-threads-$threads-mpstat 2>&1 & mpid=$!
   sysbench --test=oltp --oltp-table-size=20000000 --oltp-read-only=off --oltp-skip-trx=on --init-rng=on --num-threads=$threads --max-requests=0 --oltp-dist-type=uniform --max-time=300 --mysql-         user=root run | tee -a /root/benchmarks/$bn/sysbench-oltp-threads-$threads 2>&1
   kill $vpid $dpid $mpid
done
sysbench --test=oltp --oltp-table-size=20000000 --oltp-read-only=off --oltp-skip-trx=on --init-rng=on --max-requests=0 --oltp-dist-type=uniform --mysql-user=root cleanup

#!/bin/bash
 
TESTN=7
 
# fdisk /dev/sda
# partprobe
# make sure changes are saved and:
# screen
# /root/benchmarks/script.sh
mkfs="mkfs.xfs -d su=64k,sw=2 -f /dev/mapper/Volume00-LogVol00"
mount="mount -o nobarrier,inode64 /dev/mapper/Volume00-LogVol00 /data"
 
/opt/MegaRAID/MegaCli/MegaCli64 -LDSetProp WT -Immediate -Lall -aALL
if $(grep /data /etc/mtab >/dev/null); then
  umount /data
fi
 
TEST="$TESTN"a
 
$mkfs
$mount
/root/benchmarks/script-file.sh $TEST
umount /data
sleep 10
 
$mkfs
$mount
cp -dpR /var/lib/mysql /data/mysql
/etc/init.d/mysql start
/root/benchmarks/script-oltp.sh $TEST
/etc/init.d/mysql stop
umount /data
sleep 10
 
/opt/MegaRAID/MegaCli/MegaCli64 -LDSetProp WB -Immediate -Lall -aALL
 
TEST="$TESTN"b
 
$mkfs
$mount
/root/benchmarks/script-file.sh $TEST
umount /data
sleep 10
 
$mkfs
$mount
cp -dpR /var/lib/mysql /data/mysql
/etc/init.d/mysql start
/root/benchmarks/script-oltp.sh $TEST
/etc/init.d/mysql stop
umount /data
sleep 10

Plain results:

#! /bin/bash
 
root=.
 
if [[ $# -lt 2 ]]; then
   echo "Usage: $0 (1G|4G|16G) (seqwr|seqrd|rndrd|rndwr|rndrw)"
   exit 1
fi
 
size=$1
mode=$2
 
t=file
   echo ===== $t benchmark: $size total size, $mode mode
   printf "test       1th       4th       8th\n"
   for b in 1a 2a 3a 4a 5a 6a 1b 2b 3b 4b 5b 6b; do
   printf "%4s" $b
      for th in 1 4 8; do
         bf=$root/$b-$t/sysbench-size-$size-mode-$mode-threads-$th
         if [ -f $bf ]; then
            res=$(awk '/Requests\/sec executed/ {print $1}' $bf)
         else
            res=-1
         fi
         printf "%10.2f" $res
      done
      echo
   done

Results for csv import (used for google docs):

#! /bin/bash
 
root=.
 
if [[ $# -lt 2 ]]; then
   echo "Usage: $0 (1G|4G|16G) (seqwr|seqrd|rndrd|rndwr|rndrw)"
   exit 1
fi
 
size=$1
mode=$2
 
t=file
   printf "%s %s WT,       1th,       4th,       8th\n" $size $mode
   for b in 1a 2a 3a 4a 5a 6a; do
   printf "%4s," $b
      for th in 1 4 8; do
         bf=$root/$b-$t/sysbench-size-$size-mode-$mode-threads-$th
         if [ -f $bf ]; then
            res=$(awk '/Requests\/sec executed/ {print $1}' $bf)
         else
            res=-1
         fi
         printf "%10.2f," $res
      done
      echo
   done
   printf "%s %s WB,       1th,       4th,       8th\n" $size $mode
   for b in 1b 2b 3b 4b 5b 6b; do
   printf "%4s," $b
      for th in 1 4 8; do
         bf=$root/$b-$t/sysbench-size-$size-mode-$mode-threads-$th
         if [ -f $bf ]; then
            res=$(awk '/Requests\/sec executed/ {print $1}' $bf)
         else
            res=-1
         fi
         printf "%10.2f," $res
      done
      echo
   done

Oltp plain results:

#! /bin/bash
 
root=.
 
t=oltp
   echo ===== $t benchmark
   printf "test       1th       4th       8th    16th    32th\n"
   for b in 1a 2a 3a 4a 5a 6a 1b 2b 3b 4b 5b 6b; do
   printf "%4s" $b
      for th in 1 4 8 16 32; do
         bf=$root/$b-$t/sysbench-oltp-threads-$th
         if [ -f $bf ]; then
            res=$(awk '/read\/write requests/{print $3/300}' $bf)
         else
            res=-1
         fi
         printf "%10.2f" $res
      done
      echo
   done

Oltp results for csv import

#! /bin/bash
 
root=.
 
t=oltp
   printf "test,       1th,       4th,       8th,    16th,    32th\n"
   for b in 1a 2a 3a 4a 5a 6a 1b 2b 3b 4b 5b 6b; do
   printf "%4s," $b
      for th in 1 4 8 16 32; do
         bf=$root/$b-$t/sysbench-oltp-threads-$th
         if [ -f $bf ]; then
            res=$(awk '/read\/write requests/{print $3/300}' $bf)
         else
            res=-1
         fi
         printf "%10.2f," $res
      done
      echo
   done