TrixBox High Availability cluster using drbd

Use this at your own risk. The script on this page is only intended to be run in a test environment, it will format partitions on your hard disk and is only intended to be run on a fresh install of Trixbox 2.6! For a more complete clustering solution check out Asterisk High Availability Solutions

I am not in anyway an expert in setting up clusters on linux, but thought I would share my little script in an attempt to get some feedback for improvements and to share with other users wanting to try setting up a similar system.

I am testing this on two identical Dell Poweredge R200 servers installing Trixbox using the advanced option so I could specify the disk partitions I wanted to create.

/dev/sda1 /boot
/dev/sda2 /
/dev/sda3 /share
/dev/sda4 swap

The server has two network interfaces
eth0 is used to communication to the LAN
eth1 is used for DRBD disk replication and is connected via a crossover cable

Enjoy! – Royce

References Used:
Elastix Forum Post

#!/bin/bash

# Make sure you edit this section to your requirements! Start
cluster_ip='192.168.16.30'
domain_name="domain.local"
gateway='192.168.16.254'
primary_dns='192.168.16.2'
secondary_dns='192.168.16.2'
master_hostname='asterisk-master'
master_ip_address_eth0='192.168.16.31'
master_ip_address_eth1='10.10.10.1'
slave_hostname='asterisk-slave'
slave_ip_address_eth0='192.168.16.32'
slave_ip_address_eth1='10.10.10.2'
subnet_mask_eth0='255.255.255.0'
subnet_mask_eth1='255.255.255.0'
drbd_disk='/dev/sda3'
drbd_device='/dev/drbd0'
# Make sure you edit this section to your requirements! End

clear
echo "TrixBox High Availability Installation Script"
echo
echo "1. Master node"
echo "2. Master node (pause after each section)"
echo "3. Slave node"
echo "4. Slave node (pause after each section)"
echo "5. Exit"
echo
echo -n "Please select the installation type. "

keypress="0"
until [ $keypress = "1" ] || [ $keypress = "2" ] || [ $keypress = "3" ] || [ $keypress = "4" ] || [ $keypress = "5" ]; do
  read -s -n 1 keypress
done

case $keypress in
  1)
    node='master'
    debug=0
    server_hostname=$master_hostname
    server_ip_address_eth0=$master_ip_address_eth0
    server_ip_address_eth1=$master_ip_address_eth1
    ;;
  2) 
    node='master'
    debug=1
    server_hostname=$master_hostname
    server_ip_address_eth0=$master_ip_address_eth0
    server_ip_address_eth1=$master_ip_address_eth1
    ;;
  3)
    node='slave'
    debug=0
    server_hostname=$slave_hostname
    server_ip_address_eth0=$slave_ip_address_eth0
    server_ip_address_eth1=$slave_ip_address_eth1
    ;;
  4)
    node='slave'
    debug=1
    server_hostname=$slave_hostname
    server_ip_address_eth0=$slave_ip_address_eth0
    server_ip_address_eth1=$slave_ip_address_eth1
    ;;
  5)
    echo
    exit
    ;;
esac

clear
echo "Installation will continue with the following settings:"
echo
echo "Cluster IP address -" $cluster_ip
echo
echo "Node name -" $server_hostname.$domain_name
echo "Node IP address (eth0) -" $server_ip_address_eth0
echo "Node IP address (eth1) -" $server_ip_address_eth1
echo "Node Subnet mask (eth0) -" $subnet_mask_eth0
echo "Node Subnet mask (eth1) -" $subnet_mask_eth1
echo "Node Gateway -" $gateway
echo "Node Primary DNS -" $primary_dns
echo "Node Secondary DNS -" $secondary_dns
echo "Master node name -" $master_hostname.$domain_name
echo "Master node IP address (eth0) -" $master_ip_address_eth0
echo "Master node IP address (eth1) -" $master_ip_address_eth1
echo "Slave node name -" $slave_hostname.$domain_name
echo "Slave node IP address (eth0) -" $slave_ip_address_eth0
echo "Slave node IP address (eth1) -" $slave_ip_address_eth1
echo "DRBD disk partition -" $drbd_disk
echo "DRBD device -" $drbd_device
if [ $debug == 1 ]
then
  echo
  echo "Script will pause after each section has finished."
fi
echo
echo
echo -n "Press 'Y' to proceed, 'N' to exit. "

keypress="a"
until [ $keypress = "y" ] || [ $keypress = "Y" ] || [ $keypress = "n" ] || [ $keypress = "N" ]; do
  read -s -n 1 keypress
done

echo

if [ $keypress = "n" ] || [ $keypress = "N" ]
then
  echo "Installation aborted."
  exit;
fi

echo "Starting trixbox $node node cluster installation."

# Update all installed packages
yum -y update

echo
echo "Phase 'Update all installed packages' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Download the required files that are not in the current trixbox repository
wget -nv -P /tmp http://vault.centos.org/5.1/extras/i386/RPMS/drbd82-8.2.6-1.el5.centos.i386.rpm
wget -nv -P /tmp http://vault.centos.org/5.1/extras/i386/RPMS/kmod-drbd82-8.2.6-1.2.6.18_53.1.4.el5.i686.rpm
wget -nv -P /tmp ftp://ftp.tummy.com/pub/tummy/drbdlinks/old-releases/drbdlinks-1.11-1.noarch.rpm

echo
echo "Phase 'Download required packages' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Install the downloaded RPMs above
rpm -ih /tmp/drbd82-8.2.6-1.el5.centos.i386.rpm
rpm -ih /tmp/kmod-drbd82-8.2.6-1.2.6.18_53.1.4.el5.i686.rpm
rpm -ih /tmp/drbdlinks-1.11-1.noarch.rpm

echo
echo "Phase 'Installing downloaded packages' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Install the redhat-lsb scripts so that drbdlinks runs correctly
yum -y install redhat-lsb

echo
echo "Phase 'Install lsb scripts' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

#install heartbeat ( needed to do this twice!! for some reason heartbeat doesn't install the first time? )
yum -y install heartbeat
yum -y install heartbeat

echo
echo "Phase 'Install heartbeat' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Change the hostname
echo $server_hostname.$domain_name >> /etc/hostname 
mv /etc/sysconfig/network /etc/sysconfig/network.backup 
sed "s/HOSTNAME=trixbox1.localdomain/HOSTNAME=$server_hostname.$domain_name/g" /etc/sysconfig/network.backup > /etc/sysconfig/network
mv /etc/hosts /etc/hosts.backup 
sed "s/trixbox1.localdomain/$server_hostname.$domain_name/g" /etc/hosts.backup > /etc/hosts
mv /etc/hosts /etc/hosts.backup2
sed "s/trixbox1/$server_hostname/g" /etc/hosts.backup2 > /etc/hosts
sudo /bin/hostname -F /etc/hostname 

echo
echo "Phase 'Change hostname' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Configure eth0
mv /etc/sysconfig/network-scripts/ifcfg-eth0 /etc/sysconfig/network-scripts/ifcfg-eth0.backup
cat << EOF >> /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
ONBOOT=yes
BOOTPROTO=static
IPADDR=$server_ip_address_eth0
NETMASK=$subnet_mask_eth0
EOF

echo
echo "Phase 'Change eth0 settings' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Configure eth1
mv /etc/sysconfig/network-scripts/ifcfg-eth1 /etc/sysconfig/network-scripts/ifcfg-eth1.backup
cat << EOF >> /etc/sysconfig/network-scripts/ifcfg-eth1
DEVICE=eth1
ONBOOT=yes
BOOTPROTO=static
IPADDR=$server_ip_address_eth1
NETMASK=$subnet_mask_eth1
EOF

echo
echo "Phase 'Change eth1 settings' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Update hosts file adding an entries so each node can find each other
cat << EOF >> /etc/hosts
$master_ip_address_eth0		$master_hostname.$domain_name	$master_hostname
$slave_ip_address_eth0		$slave_hostname.$domain_name	$slave_hostname
EOF

mv /etc/resolv.conf /etc/resolv.conf.backup
cat << EOF >> /etc/resolv.conf
nameserver $primary_dns
nameserver $secondary_dns
EOF

echo "GATEWAY=$gateway" >> /etc/sysconfig/network

echo
echo "Phase 'Update /etc/hosts, /etc/resolv.conf and adding default gateway' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Restart the network
service network restart

echo
echo "Phase 'Network restart' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Configure /etc/drbd.conf
mv /etc/drbd.conf /etc/drbd.conf.backup
cat << EOF >> /etc/drbd.conf
global
{
usage-count no;
}
common
{
syncer
{
rate 100M;
}
}    
resource "r0"
{ 
protocol C;
on $master_hostname.$domain_name
{ 
device $drbd_device;
disk $drbd_disk;
address $master_ip_address_eth1:7789;
meta-disk internal;
} 
on $slave_hostname.$domain_name
{ 
device $drbd_device;
disk $drbd_disk;
address $slave_ip_address_eth1:7789; 
meta-disk internal;
}
disk
{
on-io-error detach;
}
net
{
max-buffers 2048;
ko-count 4;
} 
syncer
{ 
rate 100M;
al-extents 257;
}
startup
{ 
wfc-timeout 0;
degr-wfc-timeout 120;
} 
}
EOF

echo
echo "Phase 'Create drbd.conf' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Update drbd0/share partition from /etc/fstab

umount /share
mv /etc/fstab /etc/fstab.backup 
sed '/share/d' /etc/fstab.backup > /etc/fstab

if [ ! -e /share ]
then
  mkdir /share  
fi
cat << EOF >>  /etc/fstab 
$drbd_device              /share                ext3    defaults,noauto   0 0 
EOF

echo
echo "Phase 'Update drbd0/share partition from /etc/fstab' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Create drbd node 
dd if=/dev/zero bs\=1M count\=1 of\=$drbd_disk; sync
drbdadm create-md r0
service drbd start  
 
echo
echo "Phase 'Create drbd0 node' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Make primary drbd node
if [ $node == "master" ]
then
  drbdadm -- --overwrite-data-of-peer primary r0
  mke2fs -j $drbd_device
  mount $drbd_device

  echo
  echo "Phase 'Make primary node' complete."
  if [ $debug == 1 ]
  then
    echo "Press any key to continue."
    read -n 1 keypress
  fi
  echo
fi

# Stop the services whos data and/or configurations we are moving to the drbd shared disk
service mysqld stop
service postfix stop
service asterisk stop
service httpd stop
service ircd stop
service xinetd stop

echo
echo "Phase 'Stop running services' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Create /etc/drbdlinks.conf
mv /etc/drbdlinks.conf /etc/drbdlinks.conf.backup 
cat << EOF >> /etc/drbdlinks.conf
mountpoint('/share')
link('/var/ftp')
link('/var/spool/asterisk')
link('/var/spool/mail')
link('/var/spool/vbox')
link('/var/trixbox_load')
link('/var/www')
link('/var/lib/asterisk')
link('/var/lib/dav')
link('/var/lib/ircd')
link('/var/lib/mysql')
link('/var/lib/php')
link('/etc/asterisk')
link('/etc/httpd')
link('/etc/ircd')
link('/etc/php.d')
link('/etc/postfix')
link('/tftpboot')
EOF

echo 
echo "Phase 'Create drbdlinks.conf' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

if [ $node == "master" ]
then
  # Backup and move the data creating the entries in /etc/drbdlinks.conf as we go
  cd /share
  # /var/ftp
  echo "Moving /var/ftp started."
  tar -zcf var-ftp.tar.gz /var/ftp
  tar -zxf var-ftp.tar.gz
  echo "Moving /var/ftp completed."
  echo
  # /var/spool/asterisk
  echo "Moving /var/spool/asterisk started."
  tar -zcf var-spool-asterisk.tar.gz /var/spool/asterisk
  tar -zxf var-spool-asterisk.tar.gz
  echo "Moving /var/spool/asterisk completed."
  echo
  # /var/spool/mail
  echo "Moving /var/spool/mail started."
  tar -zcf var-spool-mail.tar.gz /var/spool/mail
  tar -zxf var-spool-mail.tar.gz
  echo "Moving /var/spool/mail completed."
  echo
  # /var/spool/vbox
  echo "Moving /var/spool/vbox started."
  tar -zcf var-spool-vbox.tar.gz /var/spool/vbox
  tar -zxf var-spool-vbox.tar.gz
  echo "Moving /var/spool/vbox completed."
  echo
  # /var/trixbox_load
  echo "Moving /var/trixbox_load started."
  tar -zcf var-trixbox_load.tar.gz /var/trixbox_load
  tar -zxf var-trixbox_load.tar.gz
  echo "Moving /var/trixbox_load completed."
  echo
  # /var/www
  echo "Moving /var/www started."
  tar -zcf var-www.tar.gz /var/www
  tar -zxf var-www.tar.gz
  echo "Moving /var/www completed."
  echo
  # /var/lib/asterisk
  echo "Moving /var/lib/asterisk started."
  tar -zcf var-lib-asterisk.tar.gz /var/lib/asterisk
  tar -zxf var-lib-asterisk.tar.gz
  echo "Moving /var/lib/asterisk completed."
  echo
  # /var/lib/dav
  echo "Moving /var/lib/dav started."
  tar -zcf var-lib-dav.tar.gz /var/lib/dav
  tar -zxf var-lib-dav.tar.gz
  echo "Moving /var/lib/dav completed."
  echo
  # /var/lib/ircd
  echo "Moving /var/lib/ircd started."
  tar -zcf var-lib-ircd.tar.gz /var/lib/ircd
  tar -zxf var-lib-ircd.tar.gz
  echo "Moving /var/lib/ircd completed."
  echo
  # /var/lib/mysql
  echo "Moving /var/lib/mysql started."
  tar -zcf var-lib-mysql.tar.gz /var/lib/mysql
  tar -zxf var-lib-mysql.tar.gz
  echo "Moving /var/lib/mysql completed."
  echo
  # /var/lib/php
  echo "Moving /var/lib/php started."
  tar -zcf var-lib-php.tar.gz /var/lib/php
  tar -zxf var-lib-php.tar.gz
  echo "Moving /var/lib/php completed."
  echo
  # /etc/asterisk


  echo "Moving /etc/asterisk started."
  tar -zcf etc-asterisk.tar.gz /etc/asterisk
  tar -zxf etc-asterisk.tar.gz
  echo "Moving /etc/asterisk completed."
  echo
  # /etc/httpd
  echo "Moving /etc/httpd started."
  tar -zcf etc-httpd.tar.gz /etc/httpd
  tar -zxf etc-httpd.tar.gz
  echo "Moving /etc/httpd completed."
  echo
  # /etc/ircd
  echo "Moving /etc/ircd started."
  tar -zcf etc-ircd.tar.gz /etc/ircd
  tar -zxf etc-ircd.tar.gz
  echo "Moving /etc/ircd completed."
  echo
  # /etc/php.d
  echo "Moving /etc/php.d started."
  tar -zcf etc-php.d.tar.gz /etc/php.d
  tar -zxf etc-php.d.tar.gz
  echo "Moving /etc/php.d completed."
  echo
  # /etc/postfix
  echo "Moving /etc/postfix started."
  tar -zcf etc-postfix.tar.gz /etc/postfix
  tar -zxf etc-postfix.tar.gz
  echo "Moving /etc/postfix completed."
  echo
  # /tftpboot
  echo "Moving /tftpboot started."
  tar -zcf tftpboot.tar.gz /tftpboot
  tar -zxf tftpboot.tar.gz
  echo "Moving /tftpboot completed."
  echo

  # Remove the tar.gz files used to move our data
  rm -fr /share/*.tar.gz

  # Fix some symlinks for /etc/httpd
  rm -fr /share/etc/httpd/logs
  rm -fr /share/etc/httpd/modules
  rm -fr /share/etc/httpd/run

  ln -s /var/log/httpd /share/etc/httpd/logs
  ln -s /usr/lib/httpd/modules /share/etc/httpd/modules
  ln -s /var/run /share/etc/httpd/run

  echo
  echo "Phase 'Data move' complete."
  if [ $debug == 1 ]
  then
    echo "Press any key to continue."
    read -n 1 keypress
  fi
  echo
fi

# Tidy up installation by deleting data that is on our drbd disk
rm -rf /tftpboot/*  
rm -rf /etc/postfix/*  
rm -rf /etc/php.d/*  
rm -rf /etc/ircd/*  
rm -rf /etc/httpd/*  
rm -rf /etc/asterisk/*  
rm -rf /var/lib/php/*  
rm -rf /var/lib/mysql/*  
rm -rf /var/lib/ircd/*  
rm -rf /var/lib/dav/*  
rm -rf /var/lib/asterisk/*  
rm -rf /var/www/*  
rm -rf /var/trixbox_load/*  
rm -rf /var/spool/vbox/*  
rm -rf /var/spool/mail/*
rm -rf /var/spool/asterisk/*  
rm -rf /var/ftp/*

echo
echo "Phase 'Deleting replicated data from original location' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Configure /etc/ha.d/ha.cf
rm -rf /etc/ha.d/ha.cf 
cat << EOF >> /etc/ha.d/ha.cf 
debugfile /var/log/ha-debug  
logfile /var/log/ha-log  
logfacility local0            # This is deprecated 
keepalive 1                   # Interval between heartbeat (HB « ») packets. 
deadtime 10                   # How quickly HB determines a dead node. 
warntime 5                    # Time HB will issue a late HB. 
initdead 120                  # Time delay needed by HB to report a dead node. 
udpport 694                   # UDP port HB uses to communicate between nodes. 
#ping 192.168.160.103          # Ping virtual host to simulate network resource. 
bcast eth0                    # Which interface to use for HB packets. 
auto_failback on              # Auto promotion of primary node upon return to cluster. 
node $master_hostname.$domain_name
node $slave_hostname.$domain_name
#respawn hacluster /usr/lib/heartbeat/ipfail 
EOF

echo
echo "Phase 'Create ha.cf' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Configure /etc/ha.d/haresources 
rm -rf /etc/ha.d/haresources 
cat << EOF >> /etc/ha.d/haresources 
$master_hostname.$domain_name $cluster_ip/$subnet_mask_eth0/eth0 drbddisk::r0 Filesystem::$drbd_device::/share::ext3 drbdlinks mysqld postfix asterisk httpd ircd amportal xinetd 
EOF

echo
echo "Phase 'Create haresources' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi

echo

# Configure  /etc/ha.d/authkeys 
rm -rf /etc/ha.d/authkeys 
cat << EOF >> /etc/ha.d/authkeys 
auth 1  
1 crc
EOF
chmod 600 /etc/ha.d/authkeys 

echo
echo "Phase 'Create authkeys' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Change service startup and shutdown in init scripts
chkconfig --levels 345 mysqld off
chkconfig --levels 345 postfix off
chkconfig --levels 345 asterisk off
chkconfig --levels 345 httpd off
chkconfig --levels 345 ircd off
chkconfig --levels 345 xinetd off
chkconfig --levels 345 heartbeat on

echo
echo "Phase 'Change service runlevels' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Create a script for ha-log quick access 
echo "tail -f /var/log/ha-log" >> /usr/bin/ha 
chmod a+x /usr/bin/ha 

echo
echo "Phase 'Create ha-log quick access' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Configure /etc/init.d/amportal
rm -rf /etc/init.d/amportal
cat << EOF >> /etc/init.d/amportal
#! /bin/sh
#
# Source function library.
. /etc/rc.d/init.d/functions
RETVAL=0
PROCNAME=portal
# See how we were called.
case "\$1" in
start)
/usr/sbin/amportal start
RETVAL=0
echo
;;
stop)
/usr/sbin/amportal stop
RETVAL=0
echo
;;
status)
status \$PROCNAME
RETVAL=\$?
;;
restart|reload)
\$0 stop
\$0 start
RETVAL=\$?
;;
*)
echo "Usage: amportal {start|stop|status|restart}"
exit 1
esac
exit \$RETVAL
EOF
chmod 755 /etc/init.d/amportal

echo
echo "Phase 'Create amportal init script' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

# Remove amportal startup configuration from /etc/rc.local
mv /etc/rc.local /etc/rc.local.backup 
sed '/amportal/d' /etc/rc.local.backup > /etc/rc.local

echo
echo "Phase 'Remove amportal startup configuration from rc.local' complete."
if [ $debug == 1 ]
then
  echo "Press any key to continue."
  read -n 1 keypress
fi
echo

if [ $node == "slave" ]
then
  # Test connectivity between the nodes
  ping -c 5 $master_hostname.$domain_name
  ping -c 5 $slave_hostname.$domain_name

  echo
  echo "Phase 'Testing connectivity' complete."
  if [ $debug == 1 ]
  then
    echo "Press any key to continue."
    read -n 1 keypress
  fi
  echo
fi

# Start the heartbeat service 
service heartbeat start

clear
echo "trixbox $node node cluster installation finished."

echo
echo "Showing /etc/log/ha-log (Press Ctrl-C to quit)"
echo
/usr/bin/ha