利用LVS+Keepalived+MFS搭建高性能、大容量FTP
最近做的项目需要存大量数据，并且下载并发很高，现有的FTP满足不了需求，而且存在单点问题，厂内的某基础平台又不稳定，因此有了本文的工作
MFS为开源的分布式数据存储平台，按照HDFS的架构编写的C++版本，对于小文件有很好的性能，在国内也有不错的应用(包括豆瓣)
原生MFS使用的网络连接模型为select，存在性能问题，我采用的是OP修改后的版本
架构图：
效果图：
MFS Master安装：
可以参考：http://www.cnblogs.com/oubo/archive/2012/05/04/2482893.html
http://www.keyofnet.com/?p=457
tar zxvf mfs-1.6.17-1.tar.gz
./configure --enable-shared=no --enable-static=yes --prefix=/home/work/local/mfs-1.6.17
make
make install
cd /home/work/local/mfs-1.6.17/etc
cp mfsmaster.cfg.dist mfsmaster.cfg
cp mfsexports.cfg.dist mfsexports.cfg
cp mfsmetalogger.cfg.dist mfsmetalogger.cfg
修改mfsmaster.cfg(chunk停机后将文件复制到其他机器的时间)
REPLICATIONS_DELAY_DISCONNECT = 5
修改mfsexports.cfg中的密码
# Allow everything but "meta".
* / rw,alldirs,maproot=0,password=123
# Allow "meta".
* . rw,password=abc
cd /home/work/local/mfs-1.6.17/var/mfs
cp metadata.mfs.empty metadata.mfs
./sbin/mfsmaster start
./sbin/mfscgiserv -P 8425
MFS Backup安装与Master类似
MFS Chunk安装：
MASTER=10.x.x.x
tar zxvf fuse-2.8.3.tar.gz
cd fuse-2.8.3
./configure
make
make install
mkdir -p /mnt/mfschunks1
mount /dev/sdb1 /mnt/mfschunks1
chown -R work:work /mnt/mfschunks1
# Mount Disk（文件虚拟盘符方法）
#mkdir -p /home/work/opdir/mfs/storage
#rm -f /home/work/opdir/mfs/storage/mfschunks1
#dd if=/dev/zero of=/home/work/opdir/mfs/storage/mfschunks1 bs=2G count=250
#mkfs -t ext3 -F /home/work/opdir/mfs/storage/mfschunks1
#mkdir -p /mnt/mfschunks1
#mount -t ext3 -o loop /home/work/opdir/mfs/storage/mfschunks1 /mnt/mfschunks1
#chown -R work:work /mnt/mfschunks1
#chown -R work:work /home/work/opdir/mfs/
# HOSTS
echo -e "$MASTER\t\tmfsmaster" >> /etc/hosts
# Install Chunk
tar zxvf mfs-1.6.17-1.tar.gz
cd mfs-1.6.17
export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH
./configure --enable-shared=no --enable-static=yes --prefix=/home/work/local/mfs-1.6.17
make
make install
cp /home/work/local/mfs-1.6.17/etc/mfschunkserver.cfg.dist /home/work/local/mfs-1.6.17/etc/mfschunkserver.cfg
cp /home/work/local/mfs-1.6.17/etc/mfshdd.cfg.dist /home/work/local/mfs-1.6.17/etc/mfshdd.cfg
echo "/mnt/mfschunks1" >> /home/work/local/mfs-1.6.17/etc/mfshdd.cfg
chown -R work:work /home/work/local/mfs-1.6.17
# Start Chunk
su - work -c 'cd /home/work/local/mfs-1.6.17/sbin/ && ./mfschunkserver start'
MFS Mount机器：
# Mount MFS
mkdir -p /mnt/mfs && /home/work/local/mfs-1.6.17/bin/mfsmount /mnt/mfs -H mfsmaster
mkdir -p /mnt/mfsmeta && /home/work/local/mfs-1.6.17/bin/mfsmount -m /mnt/mfsmeta/ -H mfsmaster
#umount /mnt/mfs
#如果有密码的话，需要加参数：-o mfspassword=abc
# Copy Num(3) & Trash(24Hour)
/home/work/local/mfs-1.6.17/bin/mfssetgoal -r 3 /mnt/mfs
/home/work/local/mfs-1.6.17/bin/mfssettrashtime -r 86400 /mnt/mfs/
#如果无法安装fuse，则需要处理：
ls /lib/modules/$(uname -r)/kernel/fs/fuse
获得linux源代码
cp $(cat build.sh | grep ^_CONFIG_FILE | awk -F "=" '{print $2}' | tr -d "\"") ./.config
make -j 8
编译fuse:
./configure --with-kernel=<linux源代码路径>
make
ls kernel/fuse.ko
insmod kernel/fuse.ko
lsmod | grep fuse
# 生成随机密码命令
< /dev/urandom tr -dc A-Za-z0-9 | head -c32; echo
LVS+Keepalived安装：
分为MFS和FTP两个LVS的安装，这里只讲FTP
虚拟IP解释：http://zhangguoli1997.iteye.com/blog/656792
可以参考：http://wangyu.iteye.com/blog/424057
http://blog.sina.com.cn/s/blog_6c2e6f1f0100sgqv.html
http://xlogin.blog.51cto.com/3473583/827725
http://www.xmydlinux.org/201102/331.html
http://zh.linuxvirtualserver.org/node/605
LVS Master：
cd /usr/src/kernels/
tar zxvf linux2-6-32_1-3-0-0_PD_BL.tar.gz
ln -s /usr/src/kernels/linux2-6-32_1-3-0-0_PD_BL/ /usr/src/linux
tar zxvf ipvsadm-1.24.tar.gz
cd ipvsadm-1.24
make
make install
tar zxvf keepalived-1.1.17.tar.gz
cd keepalived-1.1.17
echo '#define UTS_RELEASE "2.6.32"' >> /usr/src/linux/include/linux/version.h
./configure
make
make install
cp /usr/local/etc/rc.d/init.d/keepalived /etc/rc.d/init.d/
cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/
mkdir /etc/keepalived
cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/
cp /usr/local/sbin/keepalived /usr/sbin/
echo 1 > /proc/sys/net/ipv4/ip_forward
Keepalived.conf:
! Configuration File for keepalived
global_defs {
notification_email {
abc@abc.com
}
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_MASTER // 备机这里改成LVS_BACKUP
}
vrrp_instance VI_1 {
state MASTER // 备机这里改成BACKUP
interface eth1
virtual_router_id 51
priority 200
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
x.x.x.x
}
}
virtual_server x.x.x.x 21 {
delay_loop 3
lb_algo wlc
lb_kind TUN
persistence_timeout 5
protocol TCP
real_server 10.65.x.x {
weight 3
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 21
}
}
real_server 10.26.x.x {
weight 3
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 21
}
}
}
}
启动：
service keepalived start
LVS RealSvr:
#!/bin/bash
#description: start realserver with tun mode
VIP=x.x.x.x
/etc/rc.d/init.d/functions
case "$1" in
start)
echo "start LVS of REALServer"
/sbin/modprobe ipip
/sbin/ifconfig tunl0 $VIP broadcast $VIP netmask 255.255.255.255 up
/sbin/route add -host $VIP dev tunl0
echo "1">/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2">/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1">/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2">/proc/sys/net/ipv4/conf/all/arp_announce
echo "0">/proc/sys/net/ipv4/conf/eth1/rp_filter
echo "0">/proc/sys/net/ipv4/conf/tunl0/rp_filter
echo "0">/proc/sys/net/ipv4/conf/all/rp_filter
;;
stop)
/sbin/ifconfig tunl0 down
echo "close LVS Director server"
echo "0">/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0">/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0">/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0">/proc/sys/net/ipv4/conf/all/arp_announce
echo "1">/proc/sys/net/ipv4/conf/eth1/rp_filter
echo "0">/proc/sys/net/ipv4/conf/all/rp_filter
echo "1">/proc/sys/net/ipv4/conf/tunl0/rp_filter
/sbin/modprobe -r ipip
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
LVS查看：
ipvsadm -l
LVS监控：
使用lvs-rrd，可以参考：http://salogs.com/2009/09/475/
在安装rrdtool时遇到libart编译的问题时，可以参考：http://bbs.chinaunix.net/thread-958359-1-1.html
遇到tcl问题，可以参考：http://gzmaster.blog.51cto.com/299556/65372
效果图：
