#nvidia cuda
#安装依赖 gcc make kernel
yum install -y make gcc kernel-devel-$(uname -r) kernel-headers-$(uname -r) which cmake
sudo apt-get install -y make gcc linux-headers-$(uname -r)
#缺少libelf-dev相关依赖，可以尝试以下方法 
sudo apt-get install build-essential
sudo dnf install elfutils-libelf-devel
#验证   删除多余header内核 apt-get remove 
ls -l /usr/src/
ls -l /usr/src/kernels/
make -v
gcc -v
#屏蔽第三方开源nvida驱动 nouveau    #新的cuda安装驱动时会自带参数屏蔽
echo -e "blacklist nouveau\noptions nouveau modeset=0" > /etc/modprobe.d/blacklist.conf
#centos
sudo dracut -force
#debian
sudo update-initramfs -u
#验证 没有显示即为成功
lsmod | grep nouveau

##查看pci信息   yum install pciutils -y
lspci -nn | grep -i nvidia
lspci -d 10de: -vvv | grep --color Width
lspci -d 19e5: -vvv | grep --color Width
#dcu k100
lspci -d 1d94:6210
#确认 GPU 是否被内核识别
lspci -nnk -d 10de:
##卸载驱动
sudo apt-get --purge remove nvidia*
sudo apt autoremove
####安装驱动	https://www.nvidia.cn/Download/Find.aspx?lang=cn -no-opengl-files不安装OpenGL文件（最重要）；-no-x-check安装驱动时不检查X服务；-no-nouveau-check安装驱动时不检查nouveau
sudo sh NVIDIA-Linux-*.run --ui=none --no-questions --accept-license --disable-nouveau --no-cc-version-check --install-libglvnd  2>&1
#卸载
sudo nvidia-uninstall
ps -ef | grep nvidia | awk '{print $2}' | xargs kill -9
sudo rmmod nvidia
sudo rmmod nvidia uvn
sudo rmmod nvidia modeset
#关闭桌面
sudo systemctl stop display-manager

###常驻内存
cd /usr/share/doc/NVIDIA_GLX-1.0/samples
tar xvf nvidia-persistenced-init.tar.bz2
cd nvidia-persistenced-init
./install.sh
sudo systemctl enable --now nvidia-persistenced
#持久模式
nvidia-smi -pm 1

#查看显卡信息
nvidia-smi
#查看拓扑结构
nvidia-smi topo -m
nvidia-smi topo -p2p rw
#整机拓扑   查看NUMA
yum install hwloc-libs hwloc-gui
lstopo topo.png
###定位坏卡 
#nvidia-smi -q 查看BUS-ID和显卡的序列号之间的对应关系
nvidia-smi -q
#BUS-ID （BUS Address）和卡槽slot的对应关系
sudo dmidecode -t slot | egrep -i "slot|add"
#sn和busid
nvidia-smi -q | grep -E "Serial Number|Bus Id"

##ecc 查看 关闭
nvidia-smi -q -d ECC
nvidia-smi -e 0
#bug收集
nvidia-bug-report.sh
#监控温度和功耗等参数
nvidia-smi dmon -s uctpm
p = Power + Temperature
u = Utilization (sm/mem/enc/dec/jpeg/ofa)
c = Clocks (mclk/pclk)
m = Memory usage (FB、BAR1、CC)
t = PCIe Rx/Tx 吞吐量 (MB/s)
e = ECC 单/双 bit 错误 + PCIe Replay 错误
v = Power/Thermal 违规次数 
nvidia-smi dmon -s tp
#设置最大时钟 图形时钟，内存时钟  nvidia-smi -i 0 -q -d CLOCK | grep -A 2 Max
nvidia-smi -lgc 2520,2520
#解除锁频
nvidia-smi -rgc   


###安装CUDA       https://developer.nvidia.com/cuda-toolkit-archive
sudo sh cuda_*_linux.run

#加入全局环境变量
vi /etc/profile
export CUDA_HOME=/usr/local/cuda
export PATH=$PATH:$CUDA_HOME/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_HOME/lib64
:wq
source /etc/profile
nvcc -V

###测试包 https://github.com/nvidia/cuda-samples
git clone https://github.com/NVIDIA/cuda-samples.git
#编译
cd cuda-samples
yum install -y cmake
mkdir build && cd build
cmake ..
make -j$(nproc)
#指定测试卡
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
#查看显卡信息
nvidia-smi > cuda.out
nvidia-smi topo -m >> cuda.out
##p2p测试
#cd Samples/5_Domain_Specific/p2pBandwidthLatencyTest
./Samples/5_Domain_Specific/p2pBandwidthLatencyTest/p2pBandwidthLatencyTest >> cuda.out
##带宽测试  #绑核   taskset -c 32-47 ./Samples/1_Utilities/bandwidthTest/bandwidthTest --device=0
#cd Samples/1_Utilities/bandwidthTest
for i in {0..7} ; do ./Samples/1_Utilities/bandwidthTest/bandwidthTest --device=$i >> cuda.out ; done
grep "   32000000" cuda.out  >> cuda.out



##GPUburn
#https://github.com/wilicc/gpu-burn
#最新版地址 http://wili.cc/blog/entries/gpu-burn/gpu_burn-1.1.tar.gz
wget https://github.com/wilicc/gpu-burn/archive/refs/heads/master.zip
unzip gpu-burn-master.zip
cd gpu-burn-master
make
#将nvidia-smi输出成csv表格，五分钟记录一次
rm -rf *.csv
nvidia-smi \
  --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,\
pcie.link.gen.max,pcie.link.gen.current,\
pcie.link.width.max,pcie.link.width.current,\
temperature.gpu,power.draw,fan.speed,\
utilization.gpu,utilization.memory,\
memory.total,memory.free,memory.used \
  --format=csv -l 300 \
  > nvidia-smi_$(date +%Y%m%d_%H%M%S).csv & ./gpu_burn 3600

#终止nvidia-smi进程，并提取温度数据到8cards_temps.csv
ps -ef | grep nvidia-smi | awk '{print $2}' | xargs kill -9
mapfile -t temps < <(awk -F, 'NR>1{print $10}' nvidia-smi_*.csv)
printf '%s\n' "${temps[@]}" | paste -d, - - - - - - - - >8cards_temps.csv

#记录一次ipmitool传感器数据 包含温度和功耗
date +%Y%m%d_%H%M%S && ipmitool sdr | grep -Ei 'degrees C | Watts'
#记录ipmitool sdr信息
mapfile -t r1 < <(
  ipmitool sdr |
  grep -Ei 'RPM|degrees C|Watts' |
  awk '{printf "%-25s | %-15s\n", $1, $3}'
)
sleep 100
mapfile -t r2 < <(
  ipmitool sdr |
  grep -Ei 'RPM|degrees C|Watts' |
  awk '{printf "| %-15s\n", $3}'
)
sleep 300
mapfile -t r3 < <(
  ipmitool sdr |
  grep -Ei 'RPM|degrees C|Watts' |
  awk '{printf "| %-15s\n", $3}'
)
sleep 600
mapfile -t r4 < <(
  ipmitool sdr |
  grep -Ei 'RPM|degrees C|Watts' |
  awk '{printf "| %-15s\n", $3}'
)
printf "%-25s   %-15s   %-15s   %-15s   %-15s\n" "" "0s" "100s" "300s" "600s" 
paste -d '' <(printf '%s\n' "${r1[@]}") <(printf '%s\n' "${r2[@]}") <(printf '%s\n' "${r3[@]}") <(printf '%s\n' "${r4[@]}") 


##NCCL性能测试  https://developer.nvidia.com/nccl/nccl-download
#在线安装
sudo yum install -y yum-utils
sudo yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
sudo dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
sudo dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel10/x86_64/cuda-rhel10.repo

wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb

sudo dpkg -i cuda-keyring_1.0-1_all.deb
sudo apt-get update
#cuda12.4
sudo apt install -y libnccl2=2.20.5-1+cuda12.4 libnccl-dev=2.20.5-1+cuda12.4
sudo yum install -y libnccl-2.20.5-1+cuda12.4 libnccl-devel-2.20.5-1+cuda12.4 libnccl-static-2.20.5-1+cuda12.4
#12.8
sudo apt install libnccl2=2.26.2-1+cuda12.8 libnccl-dev=2.26.2-1+cuda12.8
sudo yum install libnccl-2.26.2-1+cuda12.8 libnccl-devel-2.26.2-1+cuda12.8 libnccl-static-2.26.2-1+cuda12.8
#12.9
sudo apt install libnccl2=2.27.6-1+cuda12.9 libnccl-dev=2.27.6-1+cuda12.9
sudo yum install libnccl-2.27.5-1+cuda12.9 libnccl-devel-2.27.5-1+cuda12.9 libnccl-static-2.27.5-1+cuda12.9
#13.0
sudo yum install libnccl-2.28.9-1+cuda13.0 libnccl-devel-2.28.9-1+cuda13.0 libnccl-static-2.28.9-1+cuda13.0 -y
# 本地安装模式只需要配置CUDA_HOME   https://github.com/NVIDIA/nccl-tests/
make CUDA_HOME=/usr/local/cuda
#缺少g++ sudo yum install devtoolset-7-gcc-c++
./build/all_reduce_perf -b 8 -e 128M -f 2 -g 8
NCCL_NTHREADS=$(nproc) NCCL_MIN_NCHANNELS=8  ./build/all_reduce_perf -b 1g -e 8g -f 2 -g 8
#不支持p2p
NCCL_P2P_DISABLE=1 ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 8

GPU_COUNT=$(nvidia-smi --list-gpus | wc -l)
NCCL_NTHREADS=$(nproc) NCCL_MIN_NCHANNELS=$GPU_COUNT 
echo '##########all_reduce_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' > cuda.nccl.out
./all_reduce_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########all_gather_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./all_gather_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########all_to_all_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./alltoall_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########broadcast_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./broadcast_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########gather_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./gather_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########hypercube_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./hypercube_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########reduce_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./reduce_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########reduce_scatter_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./reduce_scatter_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########scatter_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./scatter_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
echo '##########sendrecv_perf -b 1m -e 1g -f 2 -g $GPU_COUNT' >> cuda.nccl.out
./sendrecv_perf -b 1m -e 1g -f 2 -g $GPU_COUNT >> cuda.nccl.out
sleep 10
#查看是否开启iommu
dmesg | grep -i iommu

##shoc浮点测试 Scalable HeterOgeneous Computing	https://github.com/vetter/shoc
#下载地址 https://codeload.github.com/vetter/shoc/tar.gz/refs/tags/v1.1.4
#编译
./configure CUDA_CPPFLAGS="-gencode=arch=compute_37,code=sm_37"
make install -j 4
#运行 s参数为节点数 d参数为节点的设备数
./shocdriver -s 4 -cuda -d 0


#arm麒麟
#需要gcc make kernel	https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=arm64-sbsa&Compilation=Native&Distribution=RHEL&target_version=8&target_type=runfile_local
#关闭桌面模式
service lightdm stop
#安装cuda
wget https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda_11.7.1_515.65.01_linux_sbsa.run
sudo sh cuda_11.7.1_515.65.01_linux_sbsa.run






###datacenter-gpu-manager https://docs.nvidia.com/datacenter/dcgm/latest/user-guide/getting-started.html
# https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/
#关闭服务
sudo systemctl list-unit-files nvidia-dcgm.service > /dev/null && \
sudo systemctl stop nvidia-dcgm
#卸载
sudo dpkg --list datacenter-gpu-manager &> /dev/null && \
sudo apt purge --yes datacenter-gpu-manager
sudo dpkg --list datacenter-gpu-manager-config &> /dev/null && \
sudo apt purge --yes datacenter-gpu-manager-config
sudo dnf list --installed datacenter-gpu-manager &> /dev/null && \
sudo dnf remove --assumeyes datacenter-gpu-manager
sudo dnf list --installed datacenter-gpu-manager-config &> /dev/null && \
sudo dnf remove --assumeyes datacenter-gpu-manager-config
#更新源
sudo apt-get update
sudo dnf clean expire-cache
#安装
CUDA_VERSION=$(nvidia-smi | sed -E -n 's/.*CUDA Version: ([0-9]+)[.].*/\1/p')
sudo apt-get install --yes \
                       --install-recommends \
                       datacenter-gpu-manager-4-cuda${CUDA_VERSION}
sudo apt install --yes datacenter-gpu-manager-4-dev
                       
CUDA_VERSION=$(nvidia-smi | sed -E -n 's/.*CUDA Version: ([0-9]+)[.].*/\1/p')
sudo dnf install -y datacenter-gpu-manager-4-cuda${CUDA_VERSION}*
#常用命令
sudo systemctl --now enable nvidia-dcgm
dcgmi discovery -l
dcgmi diag -r 4
dcgmproftester13 -t 1001,1004 -d 300
#实时监控GPU状态
watch -n 1 'nvidia-smi --query-gpu=index,clocks.gr,clocks.mem,power.draw --format=csv'
###field-diag
#https://docs.nvidia.com/deploy/hw-field-diag/index.html#topic_2_2_1_1
fieldiag skip_nvlink p0only
fieldiag skip_nvlink p0only pciid=0:4b:00.0
fieldiag skip_nvlink p0only gen3

#nvlink 管理工具 https://developer.download.nvidia.cn/compute/cuda/repos/
version=575.64.03
yum -y install yum-utils
yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
yum install -y nvidia-fabric-manager-${version}-1
sudo apt-get -y install nvidia-fabricmanager-545_545.23.08-1
sudo apt-get install -y ./nvidia-fabricmanager-535_535.104.05-1_amd64.deb
sudo apt-get remove -y ./nvidia-fabricmanager-535_535.104.05-1_amd64.deb
systemctl enable nvidia-fabricmanager
systemctl start nvidia-fabricmanager
systemctl status nvidia-fabricmanager


##matrixMul
cd cuda-samples/Samples/0_Introduction/matrixMul
make
./matrixMul -device=0 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &
sleep 5 &
./matrixMul -device=1 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &
./matrixMul -device=2 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &
./matrixMul -device=3 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &
./matrixMul -device=4 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &
./matrixMul -device=5 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &
./matrixMul -device=6 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &
./matrixMul -device=7 -wA=20480 -hA=20480 -wB=20480 -hB=20480 &


##OpenCL-Benchmark
git clone https://github.com/ProjectPhysX/OpenCL-Benchmark.git
cd OpenCL-Benchmark
chmod +x make.sh
./make.sh

#docker toolkit
wget https://developer.download.nvidia.com/compute/cuda/repos/debian13/x86_64/cuda-keyring_1.1-1_all.deb
dpkg -i cuda-keyring_1.1-1_all.deb
apt update
apt install cuda-toolkit