目前市面上跨主机通信第三方主流的解决方法有flannel, weave, Pipework, OpenvSwitch等
这里使用OpenvSwitch，它实现比较简单，成熟且功能强大。
架构图如下：
实验环境：
一、基础环境设置
docker1
1. 配置固定IP
[root@localhost ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens32
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME=ens32
UUID=75963e3f-b289-4bbd-8489-44f6f2b8c7f0
DEVICE=ens32
ONBOOT=yes
IPADDR=192.168.0.10
PREFIX=24
GATEWAY=192.168.0.1
DNS1=114.114.114.114
[root@localhost ~]# systemctl restart network
2. 更改主机名
[root@localhost ~]# hostnamectl set-hostname docker1
[root@localhost ~]# exit //重新登陆即可
[root@docker1 ~]#
3.关闭防火墙
[root@docker1 ~]# systemctl stop firewalld
[root@docker1 ~]# systemctl disable firewalld
4. 同步系统时间
[root@docker1 ~]# yum -y install ntp
[root@docker1 ~]# systemctl enable ntpd.service
[root@docker1 ~]# ntpdate cn.pool.ntp.org
[root@docker1 ~]# hwclock -w
[root@docker1 ~]# crontab -e
0 2 * * * ntpdate ntpdate cn.pool.ntp.org && hwclock -w
5. 安装docker
[root@docker1 ~]# yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
[root@docker1 ~]# yum install -y docker-ce
[root@docker1 ~]# systemctl start docker
[root@docker1 ~]# systemctl enable docker
6. 安装网桥bridge-utils
[root@docker1 ~]# yum -y install bridge-utils
7.关闭selinux
[root@docker1 ~]# vim /etc/sysconfig/selinux
SELINUX=disabled
[root@docker1 ~]# reboot
docker2
1. 配置固定IP
[root@localhost ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens32
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens32
UUID=75963e3f-b289-4bbd-8489-44f6f2b8c7f0
DEVICE=ens32
ONBOOT=yes
IPADDR=192.168.0.10
PREFIX=24
GATEWAY=192.168.0.1
DNS1=114.114.114.114
[root@localhost ~]# systemctl restart network
2. 更改主机名
[root@localhost ~]# hostnamectl set-hostname docker2
[root@localhost ~]# exit //重新登陆即可
[root@docker2~]#
3.关闭防火墙
[root@docker2~]# systemctl stop firewalld
[root@docker2~]# systemctl disable firewalld
4. 同步系统时间
[root@docker2~]# yum -y install ntp
[root@docker2~]# systemctl enable ntpd.service
[root@docker2~]# ntpdate cn.pool.ntp.org
[root@docker2~]# hwclock -w
[root@docker2~]# crontab -e
0 2 * * * ntpdate ntpdate cn.pool.ntp.org && hwclock -w
5. 安装docker
[root@docker2~]# yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
[root@docker2~]# yum install -y docker-ce
[root@docker2~]# systemctl start docker
[root@docker2~]# systemctl enable docker
6. 安装网桥bridge-utils
[root@docker2~]# yum -y install bridge-utils
7.关闭selinux
[root@docker2~]# vim /etc/sysconfig/selinux
SELINUX=disabled
[root@docker2~]# reboot
二、openvswitch安装配置
docker1配置
1.安装依赖
[root@docker1 ~]# yum -y install make gcc openssl-devel autoconf automake rpm-build redhat-rpm-config python-devel openssl-devel kernel-devel kernel-debug-devel libtool wget
2. 预处理
[root@docker1 ~]# cd /usr/local/src/
[root@docker1 src]# mkdir -p ~/rpmbuild/SOURCES
[root@docker1 src]# wget http://openvswitch.org/releases/openvswitch-2.9.0.tar.gz
[root@docker1 src]# cp openvswitch-2.9.0.tar.gz ~/rpmbuild/SOURCES/
[root@docker1 src]# tar xf openvswitch-2.9.0.tar.gz
[root@docker1 src]# sed 's/openvswitch-kmod, //g' openvswitch-2.9.0/rhel/openvswitch.spec > openvswitch-2.9.0/rhel/openvswitch_no_kmod.spec
3. 构建rpm包
[root@docker1 src]# rpmbuild -bb --nocheck ./openvswitch-2.9.0/rhel/openvswitch_no_kmod.spec
//系统有报错
error: Failed build dependencies:
python-six is needed by openvswitch-2.9.0-1.x86_64
selinux-policy-devel is needed by openvswitch-2.9.0-1.x86_64
python-sphinx is needed by openvswitch-2.9.0-1.x86_64
原因：还需要安装以下依赖
[root@docker1 src]# yum -y install python-six selinux-policy-devel python-sphinx
4. 安装
[root@docker1 src]# cd ~
[root@docker1 ~]# yum -y install ~/rpmbuild/RPMS/x86_64/openvswitch-devel-2.9.0-1.x86_64.rpm
//将生成的RPM包openvswitch-devel-2.9.0-1.x86_64.rpm拷贝到docker2上，到时后docker2可以直接安装使用
5. 启动openvswitch服务
[root@docker1 ~]# systemctl start openvswitch.service
[root@docker1 ~]# systemctl enable openvswitch.service
[root@docker1 ~]# systemctl status openvswitch.service
docker2配置
1. 将docker1生成的openvswitch-2.9.0-1.x86_64.rpm拷贝到docker2后直接yum安装
[root@docker2 ~]# scp -r root@192.168.0.10:/root/rpmbuild/SOURCES//root/rpmbuild/RPMS/x86_64/openvswitch-2.9.0-1.x86_64.rpm .
[root@docker2 ~]# yum -y install openvswitch-2.9.0-1.x86_64.rpm
2. 启动openvswitch服务
[root@docker2 ~]# systemctl start openvswitch.service
[root@docker2 ~]# systemctl enable openvswitch.service
[root@docker2 ~]# systemctl status openvswitch.service
三、跨主机通信
docker1
1. 开启路由转发功能
[root@docker1 ~]# vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
[root@docker1 ~]# sysctl -p
net.ipv4.ip_forward = 1
2. 修改docker默认网段（即docker0网段，默认是172.17.0.0/24）
[root@docker1 ~]# vim /etc/docker/daemon.json
{
"registry-mirrors": ["https://4zy0t91p.mirror.aliyuncs.com"], //阿里docker加速器配置
"bip": "10.0.0.1/24" //docker0网段修改
}
3. 建立OVS Bridge
1.> 在两个主机上创建隧道网桥br0，并通过gre0协议创建隧道
[root@docker1 ~]# ovs-vsctl add-br br0
[root@docker1 ~]# ovs-vsctl add-port br0 gre0 -- set Interface gre0 type=gre options:remote_ip=192.168.0.10
2.> 将br0作为接口并纳入docker0网桥
[root@docker1 ~]# brctl addif docker0 br0
4. 配置路由
作用：用于转发到其它宿主机上docker容器，其中ens32为真实的网卡设备名称，需要根据实际网卡设备名称配置
在docker1上添加路由表，使docker1能到达docker2定义的10.0.1.0/24网段，意思是告诉docker1主机 10.0.1.0/24网段的ip在192.168.0.20的docker2上。
[root@docker1 ~]# vim /etc/sysconfig/network-scripts/route-ens32
10.0.1.0/24 via 192.168.0.20 dev ens32
5. 重启docker服务并查看
[root@docker1 ~]# systemctl daemon-reload
[root@docker1 ~]# systemctl restart docker
[root@docker1 ~]# systemctl restart network //这三步很关键，要依次重启
[root@docker1 ~]# brctl show
bridge name bridge id STP enabled interfaces
docker0 8000.0242d78c3863 no br0
[root@docker1 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.0.1 0.0.0.0 UG 100 0 0 ens32
10.0.0.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
10.0.1.0 192.168.0.20 255.255.255.0 UG 100 0 0 ens32
192.168.0.0 0.0.0.0 255.255.255.0 U 100 0 0 ens32
[root@docker1 ~]# ifconfig
docker0: flags=4099 mtu 1500
inet 10.0.0.1 netmask 255.255.255.0 broadcast 10.0.10.255
inet6 fe80::42:d7ff:fe8c:3863 prefixlen 64 scopeid 0x20
ether 02:42:d7:8c:38:63 txqueuelen 0 (Ethernet)
RX packets 6 bytes 392 (392.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 14 bytes 1124 (1.0 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
6. 测试网络
[root@docker1 ~]# docker pull cirros
[root@docker1 ~]# docker run -it cirros /bin/sh
1.> 查看网络
[root@c778cae8a2e5 /]# ifconfig
eth0 Link encap:Ethernet HWaddr 02:42:0A:00:0A:02
inet addr:10.0.0.2 Bcast:10.0.10.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:3 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:258 (258.0 B) TX bytes:0 (0.0 B)
2.> ping外网
[root@c778cae8a2e5 /]# ping www.baidu.com
PING www.baidu.com (61.135.169.121): 56 data bytes
64 bytes from 61.135.169.121: seq=0 ttl=56 time=4.502 ms
64 bytes from 61.135.169.121: seq=1 ttl=56 time=8.104 ms
...
3.> ping docker1宿主机
[root@c778cae8a2e5 /]# ping 192.168.0.10
PING 192.168.0.10 (192.168.0.10): 56 data bytes
64 bytes from 192.168.0.10: seq=0 ttl=63 time=0.414 ms
64 bytes from 192.168.0.10: seq=1 ttl=63 time=0.692 ms
...
4.> ping docker2宿主机
[root@c778cae8a2e5 /]# ping 192.168.0.20
PING 192.168.0.20 (192.168.0.20): 56 data bytes
64 bytes from 192.168.0.20: seq=0 ttl=63 time=0.414 ms
64 bytes from 192.168.0.20: seq=1 ttl=63 time=0.692 ms
...
docker2
1. 开启路由转发功能
[root@docker2 ~]# vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
[root@docker2 ~]# sysctl -p
net.ipv4.ip_forward = 1
2. 修改docker默认网段（即docker0网段，默认是172.17.0.0/24）
[root@docker2 ~]# vim /etc/docker/daemon.json
{
"registry-mirrors": ["https://4zy0t91p.mirror.aliyuncs.com"], //阿里docker加速器配置
"bip": "10.0.1.1/24" //docker0网段修改
}
3. 建立OVS Bridge
1.> 在两个主机上创建隧道网桥br0，并通过gre0协议创建隧道
[root@docker2 ~]# ovs-vsctl add-br br0
[root@docker2 ~]# ovs-vsctl add-port br0 gre0 -- set Interface gre0 type=gre options:remote_ip=192.168.0.20
2.> 将br0作为接口并纳入docker0网桥
[root@docker2 ~]# brctl addif docker0 br0
4. 配置路由
作用：用于转发到其它宿主机上docker容器，其中ens32为真实的网卡设备名称，需要根据实际网卡设备名称配置
在docker2上添加路由表，使docker2能到达docker1定义的10.0.0.0/24网段，意思是告诉docker2主机 10.0.0.0/24网段的ip在192.168.0.10的docker1上。
[root@docker2 ~]# vim /etc/sysconfig/network-scripts/route-ens32
10.0.0.0/24 via 192.168.0.10 dev ens32
5. 重启docker服务并查看
[root@docker2 ~]# systemctl daemon-reload
[root@docker2 ~]# systemctl restart docker
[root@docker2 ~]# systemctl restart network //这三步很关键，要依次重启
[root@docker2 ~]# brctl show
bridge name bridge id STP enabled interfaces
docker0 8000.0242d78c3863 no br0
[root@docker2 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.0.1 0.0.0.0 UG 100 0 0 ens32
10.0.1.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
10.0.0.0 192.168.0.10 255.255.255.0 UG 100 0 0 ens32
192.168.0.0 0.0.0.0 255.255.255.0 U 100 0 0 ens32
[root@docker2 ~]# ifconfig
docker0: flags=4099 mtu 1500
inet 10.0.1.1 netmask 255.255.255.0 broadcast 10.0.10.255
inet6 fe80::42:d7ff:fe8c:3863 prefixlen 64 scopeid 0x20
ether 02:42:d7:8c:38:63 txqueuelen 0 (Ethernet)
RX packets 6 bytes 392 (392.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 14 bytes 1124 (1.0 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
6. 测试网络
[root@docker2 ~]# docker pull cirros
[root@docker2 ~]# docker run -it cirros /bin/sh
1.> 查看网络
[root@d2b2a56abf59 /]# ifconfig
eth0 Link encap:Ethernet HWaddr 02:42:0A:00:0A:02
inet addr:10.0.1.2 Bcast:10.0.10.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:3 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:258 (258.0 B) TX bytes:0 (0.0 B)
2.> ping外网
[root@d2b2a56abf59 /]# ping www.baidu.com
PING www.baidu.com (61.135.169.121): 56 data bytes
64 bytes from 61.135.169.121: seq=0 ttl=56 time=4.502 ms
64 bytes from 61.135.169.121: seq=1 ttl=56 time=8.104 ms
...
3.> ping docker2宿主机
[root@d2b2a56abf59 /]# ping 192.168.0.20
PING 192.168.0.20 (192.168.0.20): 56 data bytes
64 bytes from 192.168.0.20: seq=0 ttl=63 time=0.414 ms
64 bytes from 192.168.0.20: seq=1 ttl=63 time=0.692 ms
...
4.> ping docker1宿主机
[root@d2b2a56abf59 /]# ping 192.168.0.10
PING 192.168.0.10 (192.168.0.10): 56 data bytes
64 bytes from 192.168.0.10: seq=0 ttl=63 time=0.414 ms
64 bytes from 192.168.0.10: seq=1 ttl=63 time=0.692 ms
...
5.> 容器互ping
在docker1的容器中ping docker2中的容器
[root@c778cae8a2e5 /]# ping 10.0.1.2
PING 10.0.1.2 (10.0.1.2): 56 data bytes
64 bytes from 10.0.1.2: seq=0 ttl=62 time=0.316 ms
64 bytes from 10.0.1.2: seq=1 ttl=62 time=0.813 ms
...
在docker2的容器中ping docker1中的容器
[root@d2b2a56abf59 /]# ping 10.0.0.2
PING 10.0.0.2 (10.0.0.2): 56 data bytes
64 bytes from 10.0.0.2: seq=0 ttl=63 time=0.414 ms
64 bytes from 10.0.0.2: seq=1 ttl=63 time=0.692 ms
...
上述证明：容器与容器之间、容器与宿主机、容器与外网都是通的
原创作品，允许转载，转载时请务必以超链接形式标明文章 原始出处 、作者信息和本声明。
