Setting up LAMP on Centos 6.3 with SuExec and FastCGI for performance

Use this script to help create a LAMP server with FastCGI, SuExec and a simple script to setup WordPress Virtual hosts.

# TNWebServer Updated for CentOS 6.3

# SETUP
# ~~~~~
echo $'#\x21/bin/sh' > /etc/profile.d/x.sh
echo "PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" >> /etc/profile.d/x.sh
chmod 755 /etc/profile.d/x.sh

# Install EPEL
rpm -Uvh http://mirror.pnl.gov/epel/6/i386/epel-release-6-7.noarch.rpm

# Install prereqs
yum install httpd httpd-devel php-mysql mysql-server php gcc php-mbstring 

cd
mkdir build
cd build

# FastCGI
# ~~~~~~~
wget http://www.fastcgi.com/dist/mod_fastcgi-current.tar.gz
tar -zxvf mod_fastcgi-current.tar.gz
cd mod_fastcgi*

cp Makefile.AP2 Makefile

make top_dir=/usr/lib64/httpd
make install top_dir=/usr/lib64/httpd

echo "LoadModule fastcgi_module modules/mod_fastcgi.so" > /etc/httpd/conf.d/00-FastCGI.conf
echo "DirectoryIndex index.php default.php" >> /etc/httpd/conf.d/00-FastCGI.conf
echo "" >> /etc/httpd/conf.d/00-FastCGI.conf
echo "" >> /etc/httpd/conf.d/00-FastCGI.conf
echo "  AddHandler fastcgi-script .fcgi" >> /etc/httpd/conf.d/00-FastCGI.conf
echo "  FastCgiWrapper /usr/sbin/suexec" >> /etc/httpd/conf.d/00-FastCGI.conf
echo "  FastCgiIpcDir fcgi/" >> /etc/httpd/conf.d/00-FastCGI.conf
echo "  FastCgiConfig -singleThreshold 1 -autoUpdate -idle-timeout 240 -pass-header HTTP_AUTHORIZATION" >> /etc/httpd/conf.d/00-FastCGI.conf
echo "" >> /etc/httpd/conf.d/00-FastCGI.conf

echo "#Disabled" > /etc/httpd/conf.d/php.conf

# turn on vhosting
echo "NameVirtualHost *:80" > /etc/httpd/conf.d/01-EnableVirtualHost.conf
echo "Include conf.d/hosts/*" >> /etc/httpd/conf.d/01-EnableVirtualHost.conf

chmod 755 /usr/sbin/suexec
chmod +s /usr/sbin/suexec

mkdir /etc/httpd/fcgi
mkdir /etc/httpd/fcgi/dynamic
mkdir /etc/httpd/conf.d/hosts
chmod 777 /etc/httpd/fcgi -R

# Switch to worker mode
echo "HTTPD=/usr/sbin/httpd.worker" >> /etc/sysconfig/httpd


# Create the skeleton
cd /opt
mkdir skel
mkdir skel/logs
mkdir skel/homepage
mkdir skel/cgi-bin
echo $'#\x21/bin/sh' > skel/cgi-bin/php.fcgi
echo "PHP_CGI=/usr/bin/php-cgi" >> skel/cgi-bin/php.fcgi
echo "PHP_FCGI_CHILDREN=2" >> skel/cgi-bin/php.fcgi
echo "PHP_FCGI_MAX_REQUESTS=1000" >> skel/cgi-bin/php.fcgi
echo "### no editing below ###" >> skel/cgi-bin/php.fcgi
echo "export PHP_FCGI_CHILDREN" >> skel/cgi-bin/php.fcgi
echo "export PHP_FCGI_MAX_REQUESTS" >> skel/cgi-bin/php.fcgi
echo "exec \$PHP_CGI" >> skel/cgi-bin/php.fcgi

chmod 755 skel/cgi-bin/php.fcgi


# Vhost template
echo "" > /etc/httpd/conf.d/template
echo "        ServerAdmin webmaster@DOMAIN" >> /etc/httpd/conf.d/template
echo "        ServerName DOMAIN" >> /etc/httpd/conf.d/template
echo "        ServerAlias *.DOMAIN" >> /etc/httpd/conf.d/template
echo "        DocumentRoot /var/www/USER/homepage" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        SuexecUserGroup USER USER" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        " >> /etc/httpd/conf.d/template
echo "                Options FollowSymLinks" >> /etc/httpd/conf.d/template
echo "                AllowOverride None" >> /etc/httpd/conf.d/template
echo "        " >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        " >> /etc/httpd/conf.d/template
echo "                Options -Indexes FollowSymLinks -MultiViews" >> /etc/httpd/conf.d/template
echo "                AllowOverride all" >> /etc/httpd/conf.d/template
echo "                Order allow,deny" >> /etc/httpd/conf.d/template
echo "                Allow from all" >> /etc/httpd/conf.d/template
echo "        " >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        ScriptAlias /cgi-bin/ /var/www/USER/cgi-bin/" >> /etc/httpd/conf.d/template
echo "        " >> /etc/httpd/conf.d/template
echo "                AllowOverride None" >> /etc/httpd/conf.d/template
echo "                Options ExecCGI -MultiViews +SymLinksIfOwnerMatch" >> /etc/httpd/conf.d/template
echo "                Order allow,deny" >> /etc/httpd/conf.d/template
echo "                Allow from all" >> /etc/httpd/conf.d/template
echo "        " >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        AddHandler php-fastcgi .php" >> /etc/httpd/conf.d/template
echo "        AddType application/x-httpd-php .php" >> /etc/httpd/conf.d/template
echo "        DirectoryIndex index.html index.php" >> /etc/httpd/conf.d/template
echo "        Action php-fastcgi /cgi-bin/php.fcgi" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        ServerSignature On" >> /etc/httpd/conf.d/template
echo "        ErrorLog logs/USER/error.log" >> /etc/httpd/conf.d/template
echo "        CustomLog logs/USER/access.log combined" >> /etc/httpd/conf.d/template
echo "        # Uncomment for Dedicated server" >> /etc/httpd/conf.d/template
echo "        # FastCgiServer /var/www/USER/cgi-bin/php.fcgi -user USER -group USER" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template

# create a shortcut script to create webs
# usage: createweb username domainname.tld
echo $'#\x21/bin/sh' > /usr/sbin/createweb
echo "if [ \$# -ne 3 ]" >> /usr/sbin/createweb
echo "then" >> /usr/sbin/createweb
echo "  echo Usage: createweb {username} {websitedomainname} {password}" >> /usr/sbin/createweb
echo "  echo Example: createweb somedomainuser somedomain.com password" >> /usr/sbin/createweb
echo "  exit 1" >> /usr/sbin/createweb
echo "fi" >> /usr/sbin/createweb
echo "useradd -b /var/www -d /var/www/\$1 -m -k /opt/skel -s /bin/false \$1" >> /usr/sbin/createweb
echo "chmod 755 /var/www/\$1" >> /usr/sbin/createweb
echo "chmod 755 /var/www/\$1/cgi-bin/php.fcgi" >> /usr/sbin/createweb
echo "ln -s /var/www/\$1/logs /var/log/httpd/\$1" >> /usr/sbin/createweb
echo "cp /etc/httpd/conf.d/template /etc/httpd/conf.d/hosts/\$2" >> /usr/sbin/createweb
echo "replace DOMAIN \$2 -- /etc/httpd/conf.d/hosts/\$2" >> /usr/sbin/createweb
echo "replace USER \$1 -- /etc/httpd/conf.d/hosts/\$2" >> /usr/sbin/createweb
echo "echo \"\$3\" | passwd --stdin \$1" >> /usr/sbin/createweb
echo "chown root.root /var/www/\$1" >> /usr/sbin/createweb
echo "echo \"CREATE DATABASE \$1;\" | mysql" >> /usr/sbin/createweb
echo "echo \"GRANT ALL PRIVILEGES ON \$1.* to \$1@localhost identified by '\$3';\" | mysql" >> /usr/sbin/createweb
chmod 700 /usr/sbin/createweb

# chroot jail ssh
cd /etc/ssh

echo "Port 24" > sshd_config24
echo "Protocol 2" >> sshd_config24
echo "SyslogFacility AUTHPRIV" >> sshd_config24
echo "PermitRootLogin no" >> sshd_config24
echo "PasswordAuthentication yes" >> sshd_config24
echo "GSSAPIAuthentication yes" >> sshd_config24
echo "GSSAPICleanupCredentials yes" >> sshd_config24
echo "UsePAM yes" >> sshd_config24
echo "AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES" >> sshd_config24
echo "AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT" >> sshd_config24
echo "AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE" >> sshd_config24
echo "AcceptEnv XMODIFIERS" >> sshd_config24
echo "X11Forwarding yes" >> sshd_config24
echo "ChrootDirectory /var/www/%u" >> sshd_config24
echo "Subsystem       sftp    internal-sftp" >> sshd_config24

# Startup on reboot
echo "/usr/sbin/sshd -f /etc/ssh/sshd_config24" >> /etc/rc.d/rc.local

# Start Now
/usr/sbin/sshd -f /etc/ssh/sshd_config24

chkconfig httpd on

# MYSQL
# ~~~~~
chkconfig mysqld on
/etc/init.d/mysqld start
/usr/bin/mysql_secure_installation

# store the root password into a my.cnf file
echo "[client]" > ~/.my.cnf
echo "password=MYSQLPASSWORD" > ~/.my.cnf
chmod 600 ~/.my.cnf

# Script to auto install Word Press
# ~~~~~
echo $'#\x21/bin/sh' > /usr/sbin/installwp
echo "if [ \$# -ne 2 ]" >> /usr/sbin/installwp
echo "then" >> /usr/sbin/installwp
echo "  echo Usage: installwp {username} {version}" >> /usr/sbin/installwp
echo "  echo Example: installwp test 3.4.2" >> /usr/sbin/installwp
echo "  exit 1" >> /usr/sbin/installwp
echo "fi" >> /usr/sbin/installwp
echo "" >> /usr/sbin/installwp
echo "cd /var/www/\$1/homepage" >> /usr/sbin/installwp
echo "svn co http://core.svn.wordpress.org/tags/\$2 ." >> /usr/sbin/installwp
echo "chown \$1.\$1 /var/www/\$1/homepage -R" >> /usr/sbin/installwp

chmod 700 /usr/sbin/installwp

Lync 2010 with WildCard Certificates

Lync 2010, is a terrific piece of software that allows for simple, effective collaboration for all sizes of businesses. Unfortunately,the architecture of the application is kludged together in a half baked form. What this also means that is that the Lync team decided to support “SAN” or “UCC” certificates so they can secure multiple domains with one certificate. However, they neglected to support wildcard certificates completely. This has been somewhat resolved with the latest Cumulative Update 6 (CU6), but still a nightmare to get going.

Even with this blog post, you may find it a nightmare to get it to work. Here are the steps that worked for me. This is for a 3 server setup, two front ends, and one edge.

1) Get a good wildcard cert. The EssentialSSL Wildcard from Comodo worked for me. The PositiveSSL did NOT.

2) Get the thumbprint of the SSL.

3) Install the certificate on all the machines. Make sure its in the My Computer sectionof the Certificate Manager

4) Open the Lync Powershell.

5) For the front end pools, enter the following command
Set-CSCertificate -Type Default,WebServicesInternal,WebServicesExternal -Thumbprint <thumbprint> -Verbose -Confirm:$false -Report c:\temp\test.html

6) For the edge pools, enter the following command
Set-CSCertificate -Type AccessEdgeExternal,DataEdgeExternal,AudioVideoAuthentication -Thumbprint <thumbprint> -Verbose -Confirm:$false -Report "c:\temp\test.html"

7) On all machines, create the reg dword key
[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL]
"Sendtrustedissuerlist"=dword:00000000

Reboot. Check your event logs to see if it took. When I used the PositiveSSL Wildcard, it did not work -the system simply didn’t trust it. The EssentialSSL worked fine.

Lync 2010 Hanging up after call connected

Hello, a quick post. Over the past few weeks, I noticed that Lync calls were hanging up on me for no apparent reason. After it hung up on me with a client that needed help right away, I got frustrated and tried to figure out what the problem was.

After looking around, I found the issue to be multiple ip’s on my workstation that were un-routable. I had added static IP’s to my workstation to configure some routers and switches, and didn’t remove them after I was done.

Apparently, Lync tries the other IP’s if they think its going to work better – in the process, it hangs up the call when it tries to connect to one of the non routable ip’s.

So, if you have having an issue with Lync which causes a call to connect, and then get disconnected after 5-10 seconds – look at your ip configuration.

Possibly related to Call Admission Control and Media Bypass.

Enable MPIO on Hyper-V R2

Hello, this is more a post to remind myself on how to install MPIO on Hyper-V R2. I’ve been experimenting with MPIO and my WSS server’s to make a more robust cluster, and need to use MPIO to have proper failover when maintaining the iSCSI /SAN network.

By default Hyper-V R2 does not habe the MPIO feature installed, so this needs to be enabled by issuing the command

Dism /online /enable-feature:MultipathIo

Copy and paste the above, as it is case sensitive. It doesn’t end here however, we still need to register the iSCSI initiator to use MPIO. Start up the MPIO control panel

mpiocpl

Check the box “Add support for iSCSI devices”

image

Then click “Add”

image

Reboot when ready. Once rebooted, simply add a new session to your iSCSI target, and you can then use the MCS/MPIO to verify and set the MPIO configuration (round robin or failover only). Using this, I have verified I can get double the throughput when using two interfaces on an WSS target.

Centos 6.1 LAMP Server with Fast CGI and SuExec

Here are my notes on creating a LAMP server with FastCGI/SuExec on the latest CentOS 6.1. I prefer to use CentOS since it is fully Hyper-V happy, and quiet well supported.

This assumes the “Basic Server” installation on CentOS 6.1. MySQL is installed, but not configured – but that’s an easy one to do. This also includes a bit at the end to run SSH for SFTP on port 24 so that you can allow external users in without them messing around with other users data.

# Centos 6.1 LAMP Server with Fast CGI and SuExec

# Install prereqs
yum install httpd httpd-devel php-mysql mysql-server php gcc

# Fast CGI
cd /opt
wget http://www.fastcgi.com/dist/mod_fastcgi-current.tar.gz
tar -zxvf mod_fastcgi-current.tar.gz
cd mod_fastcgi*

cp Makefile.AP2 Makefile

make top_dir=/usr/lib64/httpd
make install top_dir=/usr/lib64/httpd

echo "LoadModule fastcgi_module modules/mod_fastcgi.so" > /etc/httpd/conf.d/mod_fastcgi.conf
echo "DirectoryIndex index.php default.php" >> /etc/httpd/conf.d/mod_fastcgi.conf
echo "" >> /etc/httpd/conf.d/mod_fastcgi.conf
echo "<IfModule mod_fastcgi.c>" >> /etc/httpd/conf.d/mod_fastcgi.conf
echo "  AddHandler fastcgi-script .fcgi" >> /etc/httpd/conf.d/mod_fastcgi.conf
echo "  FastCgiWrapper /usr/sbin/suexec" >> /etc/httpd/conf.d/mod_fastcgi.conf
echo "  FastCgiIpcDir fcgi/" >> /etc/httpd/conf.d/mod_fastcgi.conf
echo "  FastCgiConfig -singleThreshold 1 -autoUpdate -idle-timeout 240 -pass-header HTTP_AUTHORIZATION" >> /etc/httpd/conf.d/mod_fastcgi.conf
echo "</IfModule>" >> /etc/httpd/conf.d/mod_fastcgi.conf

chmod 755 /usr/sbin/suexec
chmod +s /usr/sbin/suexec

mkdir /etc/httpd/fcgi
mkdir /etc/httpd/fcgi/dynamic
chmod 777 /etc/httpd/fcgi -R

# Switch to worker mode
echo "HTTPD=/usr/sbin/httpd.worker" >> /etc/sysconfig/httpd

# Create the skeleton
cd /opt
mkdir skel
mkdir skel/logs
mkdir skel/homepage
mkdir skel/cgi-bin
echo $’#\x21/bin/sh’ > skel/cgi-bin/php.fcgi
echo "PHP_CGI=/usr/bin/php-cgi" >> skel/cgi-bin/php.fcgi
echo "PHP_FCGI_CHILDREN=2" >> skel/cgi-bin/php.fcgi
echo "PHP_FCGI_MAX_REQUESTS=1000" >> skel/cgi-bin/php.fcgi
echo "### no editing below ###" >> skel/cgi-bin/php.fcgi
echo "export PHP_FCGI_CHILDREN" >> skel/cgi-bin/php.fcgi
echo "export PHP_FCGI_MAX_REQUESTS" >> skel/cgi-bin/php.fcgi
echo "exec \$PHP_CGI" >> skel/cgi-bin/php.fcgi

chmod 755 skel/cgi-bin/php.fcgi

# Vhost template
mkdir /etc/httpd/conf.d/hosts
echo "<VirtualHost *:80>" > /etc/httpd/conf.d/template
echo "        ServerAdmin webmaster@DOMAIN" >> /etc/httpd/conf.d/template
echo "        ServerName DOMAIN" >> /etc/httpd/conf.d/template
echo "        ServerAlias *.DOMAIN" >> /etc/httpd/conf.d/template
echo "        DocumentRoot /var/www/USER/homepage" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        SuexecUserGroup USER USER" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        <Directory />" >> /etc/httpd/conf.d/template
echo "                Options FollowSymLinks" >> /etc/httpd/conf.d/template
echo "                AllowOverride None" >> /etc/httpd/conf.d/template
echo "        </Directory>" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        <Directory /var/www/USER/homepage/>" >> /etc/httpd/conf.d/template
echo "                Options -Indexes FollowSymLinks -MultiViews" >> /etc/httpd/conf.d/template
echo "                AllowOverride all" >> /etc/httpd/conf.d/template
echo "                Order allow,deny" >> /etc/httpd/conf.d/template
echo "                Allow from all" >> /etc/httpd/conf.d/template
echo "        </Directory>" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        ScriptAlias /cgi-bin/ /var/www/USER/cgi-bin/" >> /etc/httpd/conf.d/template
echo "        <Directory "/var/www/USER/cgi-bin/">" >> /etc/httpd/conf.d/template
echo "                AllowOverride None" >> /etc/httpd/conf.d/template
echo "                Options ExecCGI -MultiViews +SymLinksIfOwnerMatch" >> /etc/httpd/conf.d/template
echo "                Order allow,deny" >> /etc/httpd/conf.d/template
echo "                Allow from all" >> /etc/httpd/conf.d/template
echo "        </Directory>" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        AddHandler php-fastcgi .php" >> /etc/httpd/conf.d/template
echo "        AddType application/x-httpd-php .php" >> /etc/httpd/conf.d/template
echo "        DirectoryIndex index.html index.php" >> /etc/httpd/conf.d/template
echo "        Action php-fastcgi /cgi-bin/php.fcgi" >> /etc/httpd/conf.d/template
echo "" >> /etc/httpd/conf.d/template
echo "        ServerSignature On" >> /etc/httpd/conf.d/template
echo "        ErrorLog logs/USER/error.log" >> /etc/httpd/conf.d/template
echo "        CustomLog logs/USER/access.log combined" >> /etc/httpd/conf.d/template
echo "</VirtualHost>" >> /etc/httpd/conf.d/template

# turn on vhosting
echo "NameVirtualHost *:80" > /etc/httpd/conf.d/00-EnableVirtualHost.conf
echo "Include conf.d/hosts/*" >> /etc/httpd/conf.d/00-EnableVirtualHost.conf

# create a shortcut script to create webs
# usage: createweb username domainname.tld
echo $’#\x21/bin/sh’ > /usr/sbin/createweb
echo "useradd -b /var/www -d /var/www/\$1 -m -k /opt/skel -s /bin/false \$1" >> /usr/sbin/createweb
echo "chmod 755 /var/www/\$1" >> /usr/sbin/createweb
echo "chmod 755 /var/www/\$1/cgi-bin/php.fcgi" >> /usr/sbin/createweb
echo "ln -s /var/www/\$1/logs /var/log/httpd/\$1" >> /usr/sbin/createweb
echo "cp /etc/httpd/conf.d/template /etc/httpd/conf.d/hosts/\$2" >> /usr/sbin/createweb
echo "replace DOMAIN \$2 — /etc/httpd/conf.d/hosts/\$2" >> /usr/sbin/createweb
echo "replace USER \$1 — /etc/httpd/conf.d/hosts/\$2" >> /usr/sbin/createweb
echo "passwd \$1" >> /usr/sbin/createweb
echo "chown root.root /var/www/$1" >> /usr/sbin/createweb
chmod 700 /usr/sbin/createweb

# chroot jail ssh.
cd /etc/ssh

cp sshd_config sshd_config24
nano sshd_config24

# Change the following
Port 24
PermitRootLogin no
ChrootDirectory /var/www/%u
Subsystem       sftp    internal-sftp

# Startup on reboot
echo "/usr/sbin/sshd -f /etc/ssh/sshd_config24" >> /etc/rc.d/rc.local

#Start Now
/usr/sbin/sshd -f /etc/ssh/sshd_config24

Quick Guide to installing Asterisk 1.8 on Centos 6.0

Quick notes on installing asterisk 1.8 on Centos 6.0 x64. I used this to connect Microsoft Lync to my SIP providers.

yum install wget
yum install kernel-devel gcc make gcc-c++ libxml2-devel
yum install perl ncurses-devel

wget http://downloads.asterisk.org/pub/telephony/asterisk/releases/asterisk-1.8.5.0.tar.gz
wget http://downloads.asterisk.org/pub/telephony/dahdi-linux-complete/releases/dahdi-linux-complete-2.5.0+2.5.0.tar.gz
tar -xvzf asterisk-1.8*
tar -xvzf dahdi-linux-complete-*
cd  dahdi-linux-complete*
make all

make install
make config
chkconfig dahdi on
service dahdi start
cd ..
cd asterisk-*
make clean
./configure
make menuselect
make
make install
make samples
make config
chkconfig asterisk on

Lync Application Sharing and Software Firewalls

Quick post on a discovery I made. I setup a complete infrastructure on an Hyper-V host for a customer (co-located at a datacenter), and one of the components of this infrastructure was Lync 2010. Lync is a phenomenal piece of software that allows for full collaboration for an enterprise without using third party services.

After the complete setup, we noticed that external users were not getting a great experience using desktop sharing / application sharing features from within Lync. Upon further investigation, it turned out the firewall was getting slammed (a virtual instance of pfSense on the hyper-v host). It appears that Lync floods the servers with as much udp traffic as configured to allow, and this causes a problem for the pfSense install which uses emulated network cards. My theory is that the software interrupts that are getting triggered for the emulated network card eats up all the cpu time on the guest OS giving it little time to do the routing/natting it needs to accomplish.

We solved this by using a very simple Linux firewall, although I could have easily just moved the pfSense install to a real machine and would have accomplished the same result. I also wonder if using a CentOS based firewall on the Hyper-V host would have solved the problem since it is supported using Hyper-V’s synthetic nics.

How to rebuild Intel Raid (isw) on Linux

For years, I’ve ran many small servers running the popular ICH/ISW Intel Storage Matrix RAID in Raid-1 configuration. For many years this has worked absolutely perfectly with no issues on both Windows and Linux. But something has always really bugged me. What do i do when (and they will) a drive fails? How does ISW handle it?

On Windows, this is simple, you launch the Storage Matrix software and click rebuild (if it isn’t rebuilding automagically). But how do you do this on a Linux server which has no Storage Matrix software? After hours of Googling, i came across the command “dmraid -R”. But that didn’t work in my test environments.

So i spent a whole afternoon figuring this out. This is what i found.

DMRaid Works. Sort of

DMRaid is the linux implementation of popular onboard RAID setups. Your raid can be from Intel, Nvidia, Promise and a few others who do implement it. Intel is the most common one, and that’s the one i generally have on all my Intel servers. What *you* may find is that your implementation is different, but this posting should help you.

My test setup was a simple ICH6R machine with two 160gb Seagate hard drives. I booted up the machine, went into the Intel raid setup, and created a 20gb mirror partition called “System”. I then installed CentOS 5.5 32bit on this machine, and went to work.

Initial results

The first thing i did, was find out what i’ve got. Running “dmraid -s” gave me

[root@nasri ~]# dmraid -s
*** Group superset isw_djhffiddde
–> Active Subset
name   : isw_djhffiddde_System
size   : 41942528
stride : 256
type   : mirror
status : ok
subsets: 0
devs   : 2
spares : 0

Then running “dmraid -r” gave me

[root@nasri ~]# dmraid -r
/dev/sda: isw, "isw_djhffiddde", GROUP, ok, 312581806 sectors, data@ 0
/dev/sdb: isw, "isw_djhffiddde", GROUP, ok, 312581806 sectors, data@ 0

This tells me, my mirror set is running, and has two drives attached and all is happy.

Broken results

I then, turned the machine off, and yanked a drive, inserted a different drive, and turned it back on. After fiddling with the bios for a few minutes (my machine wanted to boot form the newly installed drive, not the raid) i got back in, and this is what i saw

[root@nasri ~]# dmraid -s
ERROR: isw: wrong number of devices in RAID set "isw_djhffiddde_System" [1/2] on /dev/sda
*** Group superset isw_djhffiddde
–> *Inconsistent* Active Subset
name   : isw_djhffiddde_System
size   : 41942528
stride : 256
type   : mirror
status : inconsistent
subsets: 0
devs   : 1
spares : 0

and

[root@nasri ~]# dmraid -r
/dev/sda: isw, "isw_djhffiddde", GROUP, ok, 312581806 sectors, data@ 0

So, dmraid tells me that the raid is broken and inconsistent. Great. That’s what i want to see when a disk fails in my raid sets. According to the man pages, and the Google, to repair it you use “dmraid -R <raid id> /dev/<device>”

So, here goes.

[root@nasri ~]# dmraid -R isw_djhffiddde_System /dev/sdb
ERROR: isw: wrong number of devices in RAID set "isw_djhffiddde_System" [1/2] on /dev/sda
isw: drive to rebuild: /dev/sdb

RAID set "isw_djhffiddde_System" already active
device "isw_djhffiddde_System" is now registered with dmeventd for monitoring
Error: Unable to write to descriptor!
Error: Unable to execute set command!
Error: Unable to write to descriptor!
Error: Unable to execute set command!

Hrm. Error’s. I don’t like errors. What’s happened? To be honest, I’ll never know – but it seems like it was not working. dmraid thinks its working, but i cant see it. I cant really hear any grumblings from the drive, nor can i see the LED’s flash. dmraid tells me the following:

[root@nasri ~]# dmraid -s
*** Group superset isw_djhffiddde
–> Active Subset
name   : isw_djhffiddde_System
size   : 41942528
stride : 256
type   : mirror
status : nosync
subsets: 0
devs   : 2
spares : 0

Ok, so its not inconsistent now, but it is “nosync”, which i cannot figure out what it means. I should look at the source code, but i cant be bothered.

Alright, so it appears that its not working.

Plan B

To figure out if its doing something, i turned the machine off and removed the new drive, and put in a Western Digital Raptor. Something that makes sounds. Booted up, and dmraid still showed the same stuff, inconsistent raid set. Now, i added the new WDRaptor to this set.

[root@nasri ~]# dmraid -R isw_djhffiddde_System /dev/sdb
ERROR: isw: wrong number of devices in RAID set "isw_djhffiddde_System" [1/2] on /dev/sda
isw: drive to rebuild: /dev/sdb

RAID set "isw_djhffiddde_System" already active
device "isw_djhffiddde_System" is now registered with dmeventd for monitoring

Oh wow, much better. On top of that, i could hear the grumblings of the WD, and i could see LED activity. So, it works!

I also found a command to monitor this progress. Its called “dmsetup status”

[root@nasri ~]# dmsetup status
isw_djhffiddde_Systemp2: 0 41720805 linear
isw_djhffiddde_Systemp1: 0 208782 linear
isw_djhffiddde_System: 0 41942776 mirror 2 8:16 8:0 928/1280 1 AA 1 core
VolGroup00-LogVol01: 0 4128768 linear
VolGroup00-LogVol00: 0 37552128 linear

[root@nasri ~]# dmsetup status
isw_djhffiddde_Systemp2: 0 41720805 linear
isw_djhffiddde_Systemp1: 0 208782 linear
isw_djhffiddde_System: 0 41942776 mirror 2 8:16 8:0 936/1280 1 AA 1 core
VolGroup00-LogVol01: 0 4128768 linear
VolGroup00-LogVol00: 0 37552128 linear

[root@nasri ~]# dmsetup status
isw_djhffiddde_Systemp2: 0 41720805 linear
isw_djhffiddde_Systemp1: 0 208782 linear
isw_djhffiddde_System: 0 41942776 mirror 2 8:16 8:0 1280/1280 1 AA 1 core
VolGroup00-LogVol01: 0 4128768 linear
VolGroup00-LogVol00: 0 37552128 linear

And finally

[root@nasri ~]# dmraid -r
/dev/sdb: isw, "isw_djhffiddde", GROUP, ok, 312581806 sectors, data@ 0
/dev/sda: isw, "isw_djhffiddde", GROUP, ok, 72303838 sectors, data@ 0
[root@nasri ~]# dmraid -s
*** Group superset isw_djhffiddde
–> Active Subset
name   : isw_djhffiddde_System
size   : 41942528
stride : 256
type   : mirror
status : ok
subsets: 0
devs   : 2
spares : 0

So. This is why it “sort of” works. It didn’t work with another Seagate drive, but it worked with a different drive. Consequently, i yanked the good 80gb drive from this set, and plugged in a 750gb Seagate, and was able to mirror back to that without a problem. Maybe initially it was my drives.

Conclusion

To fix your broken Raid1’s on your Intel raid’s, use “dmraid -R <raidid> <dev>” and watch “dmsetup status” and wait for the ratio to be 1.

How to install the SNMP service on Microsoft Hyper-V R2

Another quick post/reminder to myself. I’ve been experimenting with the idea of using Cacti to monitor the performance of my Hyper-V servers, so i needed SNMP on my HyperV machines. However there is no UI to add that feature into the core installs. So, to install SNMP on HyperV R2, use the following command line

start /w ocsetup SNMP-SC

That’s it!