Chronopolis:app16app17

Hardware List

• Servers (2): Dell r410, 2xIntel E5620, 8Gb RAM
• Raid card (2): LSI 9280
• Hard Drives (50): Western Digital 2TB WDCWD2003FYYS0
• JBOD Chassis (4): AIC XJ-SA13-212R-B, 12-bay, sas

Raid Configuration

• One drive group per chassis
• 1 spare per chassis
• Two virtual drives per drive group (8.18T & 8.19T)
• stripe size: 512k
• write policy: write back w/ BBU
• read policy: Always read ahead

OS/LVM and Formatting

• OS: RHEL5, x86_64
• LVM is block aligned on 512k blocks
• xfs filesystem
• 4 volume groups, one for each virtual drive

[root@naraapp17 export]# pvcreate --dataalignment 512k /dev/sdb
  Physical volume "/dev/sdb" successfully created
[root@naraapp17 export]# pvcreate --dataalignment 512k /dev/sdc
  Physical volume "/dev/sdc" successfully created
[root@naraapp17 export]# pvcreate --dataalignment 512k /dev/sdd
  Physical volume "/dev/sdd" successfully created
[root@naraapp17 export]# pvcreate --dataalignment 512k /dev/sde
  Physical volume "/dev/sde" successfully created
[root@naraapp17 export]# pvs -o+pe_start
  PV         VG    Fmt  Attr PSize   PFree 1st PE 
  /dev/sda2  vol00 lvm2 a-   136.62G    0  192.00K
  /dev/sdb         lvm2 --     8.18T 8.18T 512.00K
  /dev/sdc         lvm2 --     8.19T 8.19T 512.00K
  /dev/sdd         lvm2 --     8.18T 8.18T 512.00K
  /dev/sde         lvm2 --     8.19T 8.19T 512.00K
[root@naraapp17 ~]# vgcreate group0-1 /dev/sdb
  Volume group "group0-1" successfully created
[root@naraapp17 ~]# vgcreate group0-2 /dev/sdc
  Volume group "group0-2" successfully created
[root@naraapp17 ~]# vgcreate group1-1 /dev/sdd
  Volume group "group1-1" successfully created
[root@naraapp17 ~]# vgcreate group1-2 /dev/sde
  Volume group "group1-2" successfully created
[root@naraapp17 ~]# lvcreate -n vault0-1 -l100%VG group0-1
  Logical volume "vault0-1" created
[root@naraapp17 ~]# lvcreate -n vault1-1 -l100%VG group1-1
  Logical volume "vault1-1" created
[root@naraapp17 ~]# lvcreate -n vault0-2 -l100%VG group0-2
  Logical volume "vault0-2" created
[root@naraapp17 ~]# lvcreate -n vault1-2 -l100%VG group1-2
  Logical volume "vault1-2" created
[root@naraapp17 ~]# lvs
  LV       VG       Attr   LSize  Origin Snap%  Move Log Copy%  Convert
  vault0-1 group0-1 -wi-a-  8.18T                                      
  vault0-2 group0-2 -wi-a-  8.19T                                      
  vault1-1 group1-1 -wi-a-  8.18T                                      
  vault1-2 group1-2 -wi-a-  8.19T                                      
  cache    vol00    -wi-ao  1.00G                                      
  rootfs   vol00    -wi-ao 19.53G                                      
  scratch0 vol00    -wi-ao 99.44G                                      
  swap     vol00    -wi-ao  2.00G                                      
  tmp      vol00    -wi-ao 14.66G             
[root@naraapp17 ~]# mkfs.xfs -d su=512k,sw=9  /dev/group0-1/vault0-1 
meta-data=/dev/group0-1/vault0-1 isize=256    agcount=32, agsize=68618752 blks
         =                       sectsz=512   attr=0
data     =                       bsize=4096   blocks=2195800064, imaxpct=25
         =                       sunit=128    swidth=1152 blks, unwritten=1
naming   =version 2              bsize=4096  
log      =internal log           bsize=4096   blocks=32768, version=1
         =                       sectsz=512   sunit=0 blks, lazy-count=0
realtime =none                   extsz=4096   blocks=0, rtextents=0
[root@naraapp17 ~]# mkfs.xfs -d su=512k,sw=9  /dev/group0-2/vault0-2 
meta-data=/dev/group0-2/vault0-2 isize=256    agcount=32, agsize=68699648 blks
         =                       sectsz=512   attr=0
data     =                       bsize=4096   blocks=2198386688, imaxpct=25
         =                       sunit=128    swidth=1152 blks, unwritten=1
naming   =version 2              bsize=4096  
log      =internal log           bsize=4096   blocks=32768, version=1
         =                       sectsz=512   sunit=0 blks, lazy-count=0
realtime =none                   extsz=4096   blocks=0, rtextents=0
[root@naraapp17 ~]# mkfs.xfs -d su=512k,sw=9  /dev/group1-2/vault1-2 
meta-data=/dev/group1-2/vault1-2 isize=256    agcount=32, agsize=68699648 blks
         =                       sectsz=512   attr=0
data     =                       bsize=4096   blocks=2198386688, imaxpct=25
         =                       sunit=128    swidth=1152 blks, unwritten=1
naming   =version 2              bsize=4096  
log      =internal log           bsize=4096   blocks=32768, version=1
         =                       sectsz=512   sunit=0 blks, lazy-count=0
realtime =none                   extsz=4096   blocks=0, rtextents=0
[root@naraapp17 ~]# mkfs.xfs -d su=512k,sw=9  /dev/group1-1/vault1-1 
meta-data=/dev/group1-1/vault1-1 isize=256    agcount=32, agsize=68618752 blks
         =                       sectsz=512   attr=0
data     =                       bsize=4096   blocks=2195800064, imaxpct=25
         =                       sunit=128    swidth=1152 blks, unwritten=1
naming   =version 2              bsize=4096  
log      =internal log           bsize=4096   blocks=32768, version=1
         =                       sectsz=512   sunit=0 blks, lazy-count=0
realtime =none                   extsz=4096   blocks=0, rtextents=0
## In /etc/fstab
/dev/group0-1/vault0-1  /export/vault0-1        xfs     defaults        1       2
/dev/group0-2/vault0-2  /export/vault0-2        xfs     defaults        1       2
/dev/group1-1/vault1-1  /export/vault1-1        xfs     defaults        1       2
/dev/group1-2/vault1-2  /export/vault1-2        xfs     defaults        1       2
##
[root@naraapp17 ~]# df -h /export/*
Filesystem            Size  Used Avail Use% Mounted on
/dev/mapper/group0--1-vault0--1
                      8.2T  5.1M  8.2T   1% /export/vault0-1
/dev/mapper/group0--2-vault0--2
                      8.2T  5.1M  8.2T   1% /export/vault0-2
/dev/mapper/group1--1-vault1--1
                      8.2T  5.1M  8.2T   1% /export/vault1-1
/dev/mapper/group1--2-vault1--2
                      8.2T  5.1M  8.2T   1% /export/vault1-2