OpenStack GPU passthrough

NVIDIA_PCI_ID=$(lspci -nn | grep NVIDIA | cut -d "[" -f3 | cut -d "]" -f1)
 
echo "blacklist nouveau" >> /etc/modprobe.d/blacklist-nvidia.conf
echo "blacklist nvidiafb" >> /etc/modprobe.d/blacklist-nvidia.conf
echo vfio-pci >> /etc/modules-load.d/vfio-pci.conf
echo options vfio-pci ids=${NVIDIA_PCI_ID} >> /etc/modprobe.d/gpu-vfio.conf
 
sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT="amd_iommu=on"/g' /etc/default/grub
 
update-grub
 
reboot

#/etc/modprobe.d/kvm.conf
#options kvm ignore_msrs=Y
 
#echo "options vfio-pci ids=${NVIDIA_PCI_ID}" >> /etc/modprobe.d/vfio.conf
#echo "softdep drm pre: vfio-pci" >> /etc/modprobe.d/vfio.conf
#echo "options kvm ignore_msrs=1" >> /etc/modprobe.d/kvm.conf
#echo "vfio vfio_iommu_type1 vfio_virqfd vfio_pci ids=${NVIDIA_PCI_ID}" >> /etc/initramfs-tools/modules
#echo GRUB_CMDLINE_LINUX_DEFAULT=\"intel_iommu=on vfio-pci.ids=${NVIDIA_PCI_ID} vfio_iommu_type1.allow_unsafe_interrupts=1 modprobe.blacklist=nvidiafb,nouveau\" >> /etc/default/grub
#echo GRUB_CMDLINE_LINUX_DEFAULT=\"amd_iommu=on vfio-pci.ids=${NVIDIA_PCI_ID} vfio_iommu_type1.allow_unsafe_interrupts=1 modprobe.blacklist=nvidiafb,nouveau\" >> /etc/default/grub
#echo 'GRUB_CMDLINE_LINUX_DEFAULT="amd_iommu=on kvm.ignore_msrs=1 vfio-pci.ids=10de:27b8"' >> /etc/default/grub
#echo 'GRUB_CMDLINE_LINUX_DEFAULT="amd_iommu=on kvm.ignore_msrs=1"' >> /etc/default/grub
 
# iommu=pt

# sudo lspci -s 02:00.0 -k
02:00.0 3D controller: NVIDIA Corporation Device 27b8 (rev a1)
	Subsystem: NVIDIA Corporation Device 16ca
	Kernel driver in use: vfio-pci
	Kernel modules: nvidiafb, nouveau

# /etc/kolla/config/ew/nova/nova-compute.conf
...
[pci]
device_spec = { "vendor_id": "10de", "product_id": "27b8" }
 
 
# /etc/kolla/config/ew/nova/nova-api.conf
...
[pci]
alias = { "vendor_id":"10de", "product_id":"27b8", "device_type":"type-PF", "name":"gpu" }
 
 
# /etc/kolla/config/ew/nova/nova-scheduler.conf
[filter_scheduler]
...
available_filters = nova.scheduler.filters.all_filters
enabled_filters = ...,PciPassthroughFilter

openstack flavor create gpu-8c64g \
    --vcpus 8 \
    --ram 65536 \
    --disk 100 \
    --property "pci_passthrough:alias"="gpu:1"

openstack server create --flavor gpu-16c64g --image "Ubuntu 22.04" --network test-network --security-group test-secgroup --key-name test-keypair gpu-test-$$

cat <<EOF> gpu.cloud-config
#cloud-config
package_upgrade: true
packages:
- ubuntu-drivers-common
#- nvidia-driver-550-server
runcmd:
- ubuntu-drivers install
- echo nvidia-smi > /etc/update-motd.d/99-gpu
- chmod 755 /etc/update-motd.d/99-gpu
EOF
 
# create VM
openstack server create --flavor gpu-32c96g --image "Ubuntu 22.04" --network test-network --security-group test-secgroup --key-name test-keypair gpu-test-$$ --user-data gpu.cloud-config 

sudo apt update && sudo apt install -y ubuntu-drivers-common
#sudo apt install -y nvidia-driver-550-server
 
# apt install ubuntu-drivers-common
# ubuntu-drivers devices
sudo ubuntu-drivers install
 
# show GPU usage on login
echo -e '#!/bin/bash\n\nnvidia-smi' | sudo tee /etc/update-motd.d/99-gpu
sudo chmod 755 /etc/update-motd.d/99-gpu
 
reboot

wget https://cdn.geekbench.com/Geekbench-5.4.1-Linux.tar.gz
tar -xzvf Geekbench-5.4.1-Linux.tar.gz
cd Geekbench-5.4.1-Linux
./geekbench5 --compute CUDA