實驗環境如下圖所示：
設定如下：
#!/bin/bash sudo ip netns add ns1 sudo ip link add ns1veth1 type veth peer name eth0 netns ns1 sudo ip netns add ns2 sudo ip link add ns2veth1 type veth peer name eth0 netns ns2 sudo ip link set ns1veth1 master vrftest sudo ip link set ns2veth1 master vrftest sudo ip link set ns2veth1 up sudo ip link set ns1veth1 up sudo ip addr add 1.1.1.254/24 dev ns1veth1 sudo ip addr add 2.2.2.254/24 dev ns2veth1 sudo ip netns exec ns2 ip addr add 2.2.2.1/24 dev eth0 sudo ip netns exec ns1 ip addr add 1.1.1.1/24 dev eth0 sudo ip netns exec ns1 ip link set eth0 up sudo ip netns exec ns1 ip link set lo up sudo ip netns exec ns1 ip route add default via 1.1.1.254 dev eth0 sudo ip netns exec ns2 ip link set eth0 up sudo ip netns exec ns2 ip link set lo up sudo ip netns exec ns2 ip route add default via 2.2.2.254 dev eth0
實驗使用c語言寫了兩個通訊端互動程式：
- 伺服器:vrfs
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<errno.h>
#include<sys/types.h>
#include<sys/socket.h>
#include<netinet/in.h>
#include <unistd.h>
#define MAXLINE 4096
int main(int argc, char** argv)
{
int listenfd, connfd;
struct sockaddr_in servaddr;
char buff[4096];
int n;
int on = 1;
if( (listenfd = socket(AF_INET, SOCK_STREAM, 0)) == -1 ){
printf("create socket error: %s(errno: %d)\n",strerror(errno),errno);
exit(0);
}
setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, (void *)&on,
sizeof(on));
setsockopt(listenfd, SOL_SOCKET, SO_REUSEPORT, (void *)&on,
sizeof(on));
memset(&servaddr, 0, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
servaddr.sin_port = htons(6666);
if(argc == 2){
printf("vrf device name: %s\r\n", argv[1]);
if(0 > setsockopt(listenfd, SOL_SOCKET, SO_BINDTODEVICE, argv[1], strlen(argv[1])+1)){
printf("bind socket master dev error: %s(errno: %d)\n",strerror(errno),errno);
exit(0);
}
}
if( bind(listenfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) == -1){
printf("bind socket error: %s(errno: %d)\n",strerror(errno),errno);
exit(0);
}
if( listen(listenfd, 10) == -1){
printf("listen socket error: %s(errno: %d)\n",strerror(errno),errno);
exit(0);
}
printf("======waiting for client's request======\n");
while(1){
if((connfd = accept(listenfd, (struct sockaddr*)NULL, NULL)) == -1){
printf("accept socket error: %s(errno: %d)",strerror(errno),errno);
continue;
}
n = recv(connfd, buff, MAXLINE, 0);
buff[n] = '\0';
printf("recv msg from client: %s\n", buff);
close(connfd);
}
close(listenfd);
}- 使用者端程式：vrfc
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<errno.h>
#include<sys/types.h>
#include<sys/socket.h>
#include<netinet/in.h>
#include<unistd.h>
#define MAXLINE 4096
#include <arpa/inet.h>
int main(int argc, char** argv)
{
int sockfd, n;
char *sendline = "hello vrf";
struct sockaddr_in servaddr;
if( argc != 2){
printf("usage: ./client <ipaddress> [master device]\n");
exit(0);
}
if( (sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0){
printf("create socket error: %s(errno: %d)\n", strerror(errno),errno);
exit(0);
}
memset(&servaddr, 0, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_port = htons(6666);
if( inet_pton(AF_INET, argv[1], &servaddr.sin_addr) <= 0){
printf("inet_pton error for %s\n",argv[1]);
exit(0);
}
if(argc == 3){
printf("vrf device name: %s\r\n", argv[2]);
if(0 > setsockopt(sockfd, SOL_SOCKET, SO_BINDTODEVICE, argv[2], strlen(argv[2])+1)){
printf("bind socket master dev error: %s(errno: %d)\n",strerror(errno),errno);
exit(0);
}
}
if( connect(sockfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) < 0){
printf("connect error: %s(errno: %d)\n",strerror(errno),errno);
exit(0);
}
printf("send msg to server: hello vrf\n");
if( send(sockfd, sendline, strlen(sendline), 0) < 0)
{
printf("send msg error: %s(errno: %d)\n", strerror(errno), errno);
exit(0);
}
close(sockfd);
exit(0);
}實驗一：驚群效應
在預設VRF環境下，啟動兩個程序，監聽相同的埠和地址：程式中套介面使用了SO_REUSEADDR和SO_REUSEPORT。檢視核心如何處理驚群效應。
console1:
admin@ubuntu:~/vrfsocket$ for i in {0..9}; do ./vrfc 127.0.0.1; done
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$console2:
admin@ubuntu:~/vrfsocket$ ./vrfs ======waiting for client's request====== recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf
console3:
admin@ubuntu:~/vrfsocket$ ./vrfs ======waiting for client's request====== recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf
結論
新核心似乎已經能夠處理驚群效應了，收到請求時不再通知所有監聽該埠的伺服器程式，而是會進行一定的負載均衡排程處理。
實驗二：啟動兩個伺服器，一個繫結VRF，一個不繫結，使用者端不繫結VRF
console1:
admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 127.0.0.1; done
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$console2:
root@ubuntu:/home/admin/vrfsocket# ./vrfs ======waiting for client's request====== recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf
console3:
root@ubuntu:/home/admin/vrfsocket# ./vrfs vrftest vrf device name: vrftest ======waiting for client's request======
結論：伺服器監聽通訊端繫結VRF後，不再處理預設VRF中的請求
實驗三：啟動兩個伺服器，一個繫結VRF，一個不繫結，使用者端繫結VRF
console1：
admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 1.1.1.254 vrftest; done
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$console2：在root使用者下執行
root@ubuntu:/home/admin/vrfsocket# ./vrfs ======waiting for client's request======
console3：在root使用者下執行。
root@ubuntu:/home/admin/vrfsocket# ./vrfs vrftest vrf device name: vrftest ======waiting for client's request====== recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf
結論：伺服器監聽通訊端不繫結VRF，不能處理非預設VRF中的請求
實驗四：設定sudo sysctl -w net.ipv4.tcp_l3mdev_accept=1
啟動兩個伺服器，一個繫結VRF，一個不繫結，使用者端不繫結VRF
console1：
admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 127.0.0.1; done
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$console2:
root@ubuntu:/home/admin/vrfsocket# ./vrfs ======waiting for client's request====== recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf
console3:
root@ubuntu:/home/admin/vrfsocket# ./vrfs vrftest vrf device name: vrftest ======waiting for client's request======
啟動一個伺服器，繫結VRF，使用者端不繫結VRF
console1：
admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 127.0.0.1; done
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
connect error: Connection refused(errno: 111)
admin@ubuntu:~/vrfsocket$console3:
root@ubuntu:/home/admin/vrfsocket# ./vrfs vrftest vrf device name: vrftest ======waiting for client's request======
啟動一個伺服器，繫結VRF，使用者端繫結VRF
console1：
admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 1.1.1.254 vrftest; done
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$console3:
root@ubuntu:/home/admin/vrfsocket# ./vrfs vrftest vrf device name: vrftest ======waiting for client's request====== recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf
啟動兩個伺服器，一個繫結VRF，一個不繫結，使用者端繫結VRF
console1：
admin@ubuntu:~/vrfsocket$ for i in {0..9}; do sudo ./vrfc 1.1.1.254 vrftest; done
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
vrf device name: vrftest
send msg to server: hello vrf
admin@ubuntu:~/vrfsocket$console2:
root@ubuntu:/home/admin/vrfsocket# ./vrfs ======waiting for client's request====== recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf recv msg from client: hello vrf
console3:
root@ubuntu:/home/admin/vrfsocket# ./vrfs vrftest vrf device name: vrftest ======waiting for client's request======
在開啟sudo sysctl -w net.ipv4.tcp_l3mdev_accept=1後，預設VRF中的監聽通訊端能夠處理所有VRF中的請求，且優先順序高於其它的VRF的監聽通訊端。
總結
| 序號 | 結論 |
|---|---|
| 1 | 多個伺服器器監聽同一地址和埠，核心會進行負載均衡，選擇喚醒其中一個程序處理請求。 |
| 2 | 預設VRF中的伺服器程序不能處理非預設VRF中的請求，非預設VRF中的伺服器程序不能處理其它VRF中的請求 |
| 3 | 開啟net.ipv4.tcp_l3mdev_accept=1後，預設VRF中的伺服器程序可以處理任意VRF中的請求，且優先順序最高 |
| 4 | 開啟net.ipv4.tcp_l3mdev_accept=1後，非預設VRF中的伺服器程序不能處理其它VRF中的請求，在處理本VRF中的流量時，優先順序低於預設VRF中的程序。 |