Heartbeat nfs DRBD實驗
一、DRBD試驗簡介
本實驗部署DRBD HEARDBEAT NFS 環境,建立一個高可用(HA)的文件伺服器集群.在方案中,通
過DRBD保證了伺服器數據的完整性和一致性.DRBD類似於一個網路RAID-1功能.當你將數據寫入本
地文件系統時,數據還將會被發送到網路中另一台主機上,以相同的形式記錄在一個另文件系統中.主節
點與備節點的數據可以保證實時相互同步.當本地主伺服器出現故障時,備份伺服器上還會保留有一份相
同的數據,可以繼續使用.在高可用(HA)中使用DRBD功能,可以代替使用一個共享盤陣.因為數據同時
存在於本地主伺服器和備份伺服器上.切換時,遠程主機只要使用它上面的那份備份數據,就可以繼續提
供主伺服器上相同的服務,並且client用戶對主伺服器的故障無感知.
實驗環境
虛擬機操作系統:Centos5.4 32bit
兩台伺服器:node1.a.com ip 192.168.101.250
node2.1.com ip 192.168.101.249
VIP:192.168.101.200
兩台伺服器將/dev/sda4互為鏡像
兩台伺服器/etc/export配置相同
Node1配置:
給介面配置ip
修改主機名
[root@jun ~]# vim /etc/sysconfig/network
NETWORKING=yes
NETWORKING_IPV6=yes
HOSTNAME=node1.a.com
[root@jun ~]# init 6 重啟使主機名生效
[root@node1 ~]# hwclock –s 同步時間,有
修改hosts文件:
[root@node1 ~]# vim /etc/hosts 修改dns緩存
127.0.0.1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
192.168.101.250 node1.a.com
192.168.101.249 node2.a.com
配置yum:
[root@node1 ~]# vim /etc/yum.repos.d/server.repo
[rhel-server]
name=Red Hat Enterprise Linux server
baseurl=file:///mnt/cdrom/Server
enabled=1
gpgcheck=1
gpgkey=file:///mnt/cdrom/RPM-GPG-KEY-redhat-release
同時要掛載光碟
[root@node1 ~]# yum -y localinstall *.rpm –nogpgcheck 安裝
drbd包[root@node1 ~]# modprobe drbd 載入drbd模塊
[root@node1 ~]# lsmod |grep drbd 查看載入
drbd 228528 0
創建新分區:
[root@node1 ~]# fdisk /dev/sda
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Selected partition 4
First cylinder (1416-5221, default 1416): 1416
Last cylinder or size or sizeM or sizeK (1416-5221, default 5221): 1G
Command (m for help): w
[root@node1 ~]# partprobe /dev/sda
[root@node1 ~]# cat /proc/partitions
major minor #blocks name
8 0 41943040 sda
8 1 104391 sda1
8 2 10241437 sda2
8 3 1020127 sda3
8
4 9775552 sda4複製相關文件到node2上:
[root@node1 ~]# scp *.rpm node2.a.com:/root
修改配置文件:
[root@node1 ~]# cd /etc/
[root@node1 etc]# cp /usr/share/doc/drbd83-8.3.8/drbd.conf ./
[root@node1 ~]# cd /etc/drbd.d/
[root@node1 drbd.d]# ll
total 4
-rwxr-xr-x 1 root root 1418 Jun 4 2010 global_common.conf
[root@node1 drbd.d]# cp global_common.conf global_common.conf.bak 備份文件
[root@node1 drbd.d]# vim global_common.conf
1 global {
2 usage-count no;
3 # minor-count dialog-refresh disable-ip-verification
4 }
5
6 common {
7 protocol C;
8
9 startup {
10 wfc-timeout 120;
11 degr-wfc-timeout 120;
12 }
13
14 disk {
15 on-io-error detach;
16 # fencing resource-only;
17 }
18 net {
19 cram-hmac-alg "sha1";
20 shared-secret "mydrbdlab";
21
22 }
23
24 syncer {
25 rate 100m;
26 }
27 }
4、檢測配置文件
[root@node1 drbd.d]# drbdadm adjust r0
no resources defined
[root@node1 drbd.d]# vim web.res 創建資源文件
resource
web {on node1.a.com {
device /dev/drbd0;
disk /dev/sda4;
address 192.168.101.250:7789;
meta-disk internal;
}
on node2.a.com {
device /dev/drbd0;
disk /dev/sda4;
address 192.168.101.249:7789;
meta-disk internal;
[root@node1 drbd.d]# scp /etc/drbd.conf node2.a.com:/etc/
[root@node1 drbd.d]# scp /etc/drbd.d/* node2.a.com:/etc/drbd.d/
[root@node1 ~]# drbdadm create-md web 創建ro資源
[root@node1 drbd.d]# service drbd start 兩邊需要都啟動,才可以啟動成功
[root@node1 drbd.d]# drbdadm -- --overwrite-data-of-peer primary web 將其設為主設備
[root@node1 drbd.d]# watch -n 1 'cat /proc/drbd'
[root@node1 ~]# cat /proc/drbd 查看drbd狀態
version: 8.3.8 (api:88/proto:86-94)
GIT-hash: d78846e52224fd00562f7c225bcc25b2d422321d build by mockbuild@builder10.centos.org, 2010-06-04 08:04:16
0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r----
ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
[root@node1 drbd.d]# mkfs -t ext3 -L drbdweb /dev/drbd0 格式化分區
[root@node1 ~]# mkdir /data
[root@node1 ~]# mount /dev/drbd0 /data 掛載分區
[root@node1 ~]# service drbd status
drbd driver loaded OK; device status:
version: 8.3.8 (api:88/proto:86-94)
GIT-hash: d78846e52224fd00562f7c225bcc25b2d422321d build by mockbuild@builder10.centos.org, 2010-06-04 08:04:16
m:res cs ro ds p mounted fstype
0:web Connected Primary/Secondary UpToDate/UpToDate C /mnt/1 ext3
[root@node1 ~]# vim /etc/exports 兩台伺服器一致
/data *(rw,sync,insecure,no_root_squash,no_wdelay)
[root@node1 ~]# service portmap start
Starting portmap: [ OK ]
[root@node1 ~]# service nfs start
Starting NFS services: [ OK ]
Starting NFS quotas: [ OK ]
Starting NFS daemon: [ OK ]
Starting NFS mountd: [ OK ]
[root@node1 ~]# chkconfig portmap on
[root@node1 ~]# chkconfig nfs on
[root@node1 ~]# vim /etc/init.d/nfs
116 stop)
117 # Stop daemons.
118 echo -n $"Shutting down NFS mountd: "
119 killproc rpc.mountd
120 echo
121 echo -n $"Shutting down NFS daemon: "
122 killproc nfsd -9 兩台伺服器都修改nfs 啟動腳本.將/etc/init.d/nfs 腳本中的stop 部分中的killproc ,nfsd -2 修改為 -9
[root@node1 ~]# scp /etc/exports node2.a.com:/etc/
安裝相關包:
[root@node1 ~]# yum localinstall heartbeat-2.1.4-9.el5.i386.rpm heartbeat-pils-2.1.4-10.el5.i386.rpm libnet-1.1.4-3.el5.i386.rpm heartbeat-stonith-2.1.4-10.el5.i386.rpm perl-MailTools-1.77-1.el5.noarch.rpm
拷貝配置文檔:
[root@node1 ~]# cd /usr/share/doc/heartbeat-2.1.4/
[root@node1 heartbeat-2.1.4]# cp authkeys ha.cf haresources /etc/ha.d/
安裝配置文檔:
[root@node1 heartbeat-2.1.4]# cd /etc/ha.d/
[root@node1 ha.d]# vim ha.cf
24 debugfile /var/log/ha-debug
29 logfile /var/log/ha-log
48 keepalive 2
49 #
56 deadtime 30
76 udpport 692
121 ucast eth0 192.168.101.249
157 auto_failback off
211 node node1.a.com
212 node node2.a.com
220 ping 192.168.101.1
指明主伺服器
[root@node1 ha.d]# echo "node1.a.com IPaddr::192.168.101.200/24/eth0 drbddisk::web Filesystem::/dev/drbd0::/data::ext3 killnfsd" >> /etc/ha.d/haresources
[root@node1 ha.d]# vim authkeys
23 auth 1
24 1 crc
[root@node1 ha.d]# echo "killall -9 nfsd; /etc/init.d/nfs restart; exit 0" >> ./ resource.d/killnfsd
[root@node1 ha.d]# chmod 600 ./authkeys
[root@node1 ha.d]# chmod 755 ./resource.d/killnfsd
[root@node1 ha.d]# service heartbeat start
[root@node1 ha.d]# service heartbeat restart
[root@node1 ~]# chkconfig heartbeat on
查看node1 是否出現
vip:
Node1上面出現vip 192.168.101.200,node2上面沒有:
[root@jun ~]# vim /etc/sysconfig/network
NETWORKING=yes
NETWORKING_IPV6=yes
HOSTNAME=node2.a.com
[root@jun ~]# init 6
[root@node2 ~]# hwclock -s
[root@node2 ~]# vim /etc/hosts
127.0.0.1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
192.168.101.250 node1.a.com
192.168.101.249 node2.a.com
[root@node2 ~]# vim /etc/yum.repos.d/server.repo
[rhel-server]
name=Red Hat Enterprise Linux server
baseurl=file:///mnt/cdrom/Server
enabled=1
gpgcheck=1
gpgkey=file:///mnt/cdrom/RPM-GPG-KEY-redhat-release
同時要掛載光碟
Drbd配置:
[root@node2 ~]# yum -y localinstall *.rpm –nogpgcheck
[root@node2 ~]# fdisk /dev/sda
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Selected partition 4
First cylinder (1416-5221, default 1416): 1416
Last cylinder or size or sizeM or sizeK (1416-5221, default 5221): 10000m
Command (m for help): w
[root@node2 ~]# partprobe /dev/sda
[root@node2 ~]# cat /proc/partitions
major minor #blocks name
8 0 41943040 sda
8 1 104391 sda1
8 2 10241437 sda2
8 3 1020127 sda3
8 4 9775552 sda4
[root@node2 ~]# drbdadm create-md web
[root@node1 drbd.d]# service drbd start
[root@node2 drbd.d]# cat /proc/drbd
version: 8.3.8 (api:88/proto:86-94)
GIT-hash: d78846e52224fd00562f7c225bcc25b2d422321d build by mockbuild@builder10.centos.org, 2010-06-04 08:04:16
0: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r----
ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:9775216
[root@node2 ~]# service drbd status
drbd driver loaded OK; device status:
version: 8.3.8 (api:88/proto:86-94)
GIT-hash: d78846e52224fd00562f7c225bcc25b2d422321d build by mockbuild@builder10.centos.org, 2010-06-04 08:04:16
m:res cs ro ds p mounted fstype
0:web Connected Secondary/Primary UpToDate/UpToDate C
Nfs配置:
[root@node2 ~]# service portmap start
Starting portmap: [ OK ]
[root@node2 ~]# service nfs start
Starting NFS services: [ OK ]
Starting NFS quotas: [ OK ]
Starting NFS daemon: [ OK ]
Starting NFS mountd: [ OK ]
[root@node2 ~]# chkconfig portmap on
[root@node2 ~]# chkconfig nfs on
配置啟動腳本
[root@node2 ~]# vim /etc/init.d/nfs
116 stop)
117 # Stop daemons.
118 echo -n $"Shutting down NFS mountd: "
119 killproc rpc.mountd
120 echo
121 echo -n $"Shutting down NFS daemon: "
122 killproc nfsd -9 兩台伺服器都修改nfs 啟動腳本.將/etc/init.d/nfs 腳本中的stop 部分中的killproc ,nfsd -2 修改為 -9
[root@node1 ha.d]# service heartbeat start
Starting High-Availability services:
2012/05/08_00:52:46 INFO: Resource is stopped
[ OK ]
Heartbeat配置:
安裝heartbeat相關包:
[root@node2 ~]# yum localinstall heartbeat-2.1.4-9.el5.i386.rpm heartbeat-pils-2.1.4-10.el5.i386.rpm heartbeat-stonith-2.1.4-10.el5.i386.rpm libnet-1.1.4-3.el5.i386.rpm perl-MailTools-1.77-1.el5.noarch.rpm –nogpgcheck
拷貝配置文檔並配置配置文檔
[root@node2 ~]# cd /usr/share/doc/heartbeat-2.1.4/
[root@node2 heartbeat-2.1.4]# cp authkeys ha.cf haresources /etc/ha.d/
[root@node2 heartbeat-2.1.4]# cd /etc/ha.d/
[root@node2 ha.d]# ls
README.config authkeys ha.cf harc haresources rc.d resource.d shellfuncs
[root@node2 ha.d]# vim ha.cf
24 debugfile /var/log/ha-debug
29 logfile /var/log/ha-log
48 keepalive 2
49 #
56 deadtime 30
76 udpport 692
121 ucast eth0 192.168.101.249
157 auto_failback off
211 node node1.a.com
212 node node2.a.com
220 ping 192.168.101.1
指明主伺服器:
[root@node2 ha.d]# echo "node1.a.com IPaddr::192.168.101.200/24/eth0 drbddisk::web Filesystem::/dev/drbd0::/data::ext3 killnfsd" >> /etc/ha.d/haresources
[root@node2 ha.d]# vim authkeys
23 auth 1
24 1 crc
[root@node2 ha.d]# echo "killall -9 nfsd; /etc/init.d/nfs restart; exit 0" >> ./ resource.d/killnfsd
修改文件許可權
[root@node2 ha.d]# chmod 600 ./authkeys
[root@node2 ha.d]# chmod 755 ./resource.d/killnfsd
[root@node2 ha.d]# service heartbeat start
Starting High-Availability services:
2012/05/08_00:52:46 INFO: Resource is stopped
[ OK ]
[root@node2 ha.d]# service heartbeat restart
[root@node2 ~]# chkconfig heartbeat on
測試機配置:
1、在測試機上將192.168.10.188:/data掛到本地/mnt/nfs
[root@server5 ~]# mkdir /mnt/nfs
[root@server5 ~]# mount 192.168.101.200:/data /mnt/nfs
2、在測試機上創建測試shell,二秒一個
[root@server5 ~]# vim /mnt/test.sh
while true
do
echo ---\> trying touch x : `date`
touch x
echo \<----- done touch x : `date`
echo
sleep 2
done
cd /mnt/nfs
[root@server5 nfs]# bash /mnt/test.sh
3、將主節點server1的heartbeat服務停止,則備節點server2接管服務
[root@server5 nfs]# bash /mnt/test.sh
---> trying touch x : Sun May 6 20:03:12 CST 2012
<----- done touch x : Sun May 6 20:03:12 CST 2012
---> trying touch x : Sun May 6 20:03:14 CST 2012
<----- done touch x : Sun May 6 20:03:14 CST 2012
---> trying touch x : Sun May 6 20:03:16 CST 2012
<----- done touch x : Sun May 6 20:03:16 CST 2012
---> trying touch x : Sun May 6 20:03:18 CST 2012
<----- done touch x : Sun May 6 20:03:18 CST 2012
---> trying touch x : Sun May 6 20:03:20 CST 2012
<----- done touch x : Sun May 6 20:03:20 CST 2012
---> trying touch x : Sun May 6 20:03:22 CST 2012
<----- done touch x : Sun May 6 20:03:22 CST 2012
---> trying touch x : Sun May 6 20:03:24 CST 2012
<----- done touch x : Sun May 6 20:03:24 CST 2012
---> trying touch x : Sun May 6 20:03:26 CST 2012
<----- done touch x : Sun May 6 20:03:26 CST 2012
---> trying touch x : Sun May 6 20:03:28 CST 2012
touch: cannot touch `x': Stale NFS file handle
<----- done touch x : Sun May 6 20:03:28 CST 2012 可以看到發生切換.
---> trying touch x : Sun May 6 20:03:33 CST 2012
<----- done touch x : Sun May 6 20:03:33 CST 2012
---> trying touch x : Sun May 6 20:03:35 CST 2012
<----- done touch x : Sun May 6 20:03:35 CST 2012
本文出自 「網路技術」 博客,請務必保留此出處http://xiaogang6.blog.51cto.com/3046480/857714
[火星人 ] DRBD nfs heartbeat--linux的高可用性和存儲已經有966次圍觀