### 参考https://www.ctyun.cn/developer/article/433031470325829
# https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/
#可能需要安装对应依赖包，按提示安装 
yum install -y createrepo perl lsof kernel-rpm-macros python-devel  tk gcc-gfortran
./mlnx_add_kernel_support.sh -m ./
./mlnxofedinstall
/etc/init.d/openibd restart
/etc/init.d/opensmd restart
# https://network.nvidia.com/products/adapter-software/firmware-tools/
sudo mst start
sudo mst restart
sudo mst status -v
lspci -tvv | grep Mellanox
lspci -s cc:00.0 -vv | egrep -i 'Lnkcap Lnksta'
#查看信息
ibv_devices
#ibv_devinfo

#查看隐射关系
ibdev2netdev
#显示并检查汇总信息
mlnx_tune
#查看工作状态
ibstatus
#重启 卡在初始化状态
sudo systemctl restart opensmd
/etc/init.d/openibd restart
/etc/init.d/opensmd restart
sudo systemctl start opensmd
sudo systemctl enable opensmd
#测试速率
rm -rf ib.out
ib_write_bw -d mlx5_0 & ib_write_bw -d mlx5_1 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_0 & ib_write_bw -d mlx5_4 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_0 & ib_write_bw -d mlx5_5 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_1 & ib_write_bw -d mlx5_0 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_1 & ib_write_bw -d mlx5_4 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_1 & ib_write_bw -d mlx5_5 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_4 & ib_write_bw -d mlx5_0 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_4 & ib_write_bw -d mlx5_1 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_4 & ib_write_bw -d mlx5_5 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_5 & ib_write_bw -d mlx5_0 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_5 & ib_write_bw -d mlx5_1 127.0.0.1 --report_gbits >> ib.out
sleep 5
ib_write_bw -d mlx5_5 & ib_write_bw -d mlx5_4 127.0.0.1 --report_gbits >> ib.out
sleep 5
grep 65536 ib.out
#绑核 nvidia-smi topo -m 
taskset -c 0-15 ib_write_bw -d mlx5_0 & taskset -c 32-47 ib_write_bw -d mlx5_1 127.0.0.1 --report_gbits
taskset -c 0-15 ib_write_bw -d mlx5_0 & taskset -c 64-79 ib_write_bw -d mlx5_4 127.0.0.1 --report_gbits
taskset -c 0-15 ib_write_bw -d mlx5_0 & taskset -c 96-111 ib_write_bw -d mlx5_5 127.0.0.1 --report_gbits

taskset -c 64-79 ib_write_bw -d mlx5_4 & taskset -c 96-111 ib_write_bw -d mlx5_5 127.0.0.1 --report_gbits

#查看模块状态
mlxlink -d 0000:17:00.1
mlxconfig -d /dev/mst/mt41682_pciconf0 q | grep -i model 

mlxconfig -d /dev/mst/mt4123_pciconf0 query
#设置成普通光卡
mlxconfig -d /dev/mst/mt41682_pciconf0 s INTERNAL_CPU_MODEL=0
#切换IB卡模式为 InfiniBand 2代表eth模式；1代表IB模式
mlxconfig -y -d /dev/mst/mt4119_pciconf0 set LINK_TYPE_P1=1 
mlxconfig -d /dev/mst/mt4123_pciconf0 set LINK_TYPE_P1=2

##配置    https://access.redhat.com/documentation/zh-cn/red_hat_enterprise_linux/7/html/networking_guide/sec-configure_ipoib_using_the_networkmanager_command_line_tool_nmcli
#在两个单独的命令中创建和修改 IPoIB。
~]$ nmcli con add type infiniband con-name mlx4_ib0 ifname mlx4_ib0 transport-mode connected mtu 65520
Connection 'mlx4_ib0' (8029a0d7-8b05-49ff-a826-2a6d722025cc) successfully added.
~]$ nmcli con edit mlx4_ib0
===| nmcli interactive connection editor |===

Editing existing 'infiniband' connection: 'mlx4_ib0'

Type 'help' or '?' for available commands.
Type 'describe [>setting<.>prop<]' for detailed property description.

You may edit the following settings: connection, infiniband, ipv4, ipv6
nmcli> set infiniband.mac-address 80:00:02:00:fe:80:00:00:00:00:00:00:f4:52:14:03:00:7b:cb:a3
nmcli> save
Connection 'mlx4_ib3' (8029a0d7-8b05-49ff-a826-2a6d722025cc) successfully updated.
nmcli> quit
#用一个命令
nmcli con add type infiniband con-name mlx4_ib0 ifname mlx4_ib0 transport-mode connected mtu 65520  infiniband.mac-address 80:00:02:00:fe:80:00:00:00:00:00:00:f4:52:14:03:00:7b:cb:a3
nmcli con edit mlx4_ib0
#如果需要 P_Key 接口，请使用 nmcli 创建一个，如下所示：
nmcli con add type infiniband con-name mlx4_ib0.8002 ifname mlx4_ib0.8002 parent mlx4_ib0 p-key 0x8002
nmcli con modify mlx4_ib0.8002 infiniband.mtu 65520 infiniband.transport-mode connected ipv4.ignore-auto-dns yes ipv4.ignore-auto-routes yes ipv4.never-default true ipv6.ignore-auto-dns yes ipv6.ignore-auto-routes yes ipv6.never-default true

#修改后重启ib内核生效
~]$ rmmod ib_ipoib
~]$ modprobe ib_ipoib
#常见的配置文件
~]$ more ifcfg-mlx4_ib0
DEVICE=mlx4_ib0
TYPE=InfiniBand
ONBOOT=yes
HWADDR=80:00:00:4c:fe:80:00:00:00:00:00:00:f4:52:14:03:00:7b:cb:a1
BOOTPROTO=none
IPADDR=172.31.0.254
PREFIX=24
NETWORK=172.31.0.0
BROADCAST=172.31.0.255
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
MTU=65520
CONNECTED_MODE=yes
NAME=mlx4_ib0


#连通性测试
#RDMA 连通性测试
#在 RDMA 协议栈下使用 rping 工具测试网络连通性，该工具等价于 TCP/IP 的 ping 。与 ping 不同，rping 工作时需首先启动一个 server side 进程，然后从 client side 试图向 server side 发起连接（而 ping 不需要 server side ）。
#RDMA Server 端示例：

rping -s -a 172.16.34.1 -p 9401 -v
# -s: 启动 server 端进程
# -a: 绑定 IP 地址 (某个 IPoIB，这里选取 memserver34 第一个 IB 网卡)
# -p: 监听端口号 (缺省为 9400)
# -v: 打印输出信息
# server side 启动后首先阻塞等待 client 连接
# 当有 client 完成连接并断开后 server side 才停止
#RDMA Client 端示例：

rping -c -I 172.16.16.1 -a 172.16.34.1 -p 9401 -v
# -c: 启动 client 端进程
# -I: 指定自身使用的 IPoIB 地址 (可缺省，如缺省则从路由表查找)
# -a: 服务端的 IPoIB 地址
# -p: 服务端监听的端口号
# -v: 打印输出信息
# client side 连接成功后将不停发送测试数据，按 Ctrl+C 停止
# 可选 -C 选项限定发送测试数据次数，例如 -C 10
#如果 RDMA 连接正常，server 和 client 的终端均会有数据输出，例如：

ping data: rdma-ping-0: ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqr
ping data: rdma-ping-1: BCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrs