FIX: While synchronizing instance power states, found XX instances in the database and XX instances on the hypervisor

Check kibana logs for
"While synchronizing instance power states, found"

Check libvirt VMs vs. nova DB

COMPUTE_NODE=com1-dev
 
# get shutoff VMs on compute node
VMS_COMPUTE=$(ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh list --state-shutoff --uuid | sed '/^$/d' | sort)
# echo "${VMS_COMPUTE}"
 
# get shutoff VMs from nova DB
VMS_NOVA=$(ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server list --all --host ${COMPUTE_NODE} -c ID -f value --status SHUTOFF" | sort)
# echo "${VMS_NOVA}"
 
# diff shutoff VMs
comm -3 <(echo "${VMS_COMPUTE}") <(echo "${VMS_NOVA}")

Remove shutdown VM from libvirt

VM_ID=c3fbcc6b-2dfe-4aa1-82bd-522b161a37a9
 
# OPTINAL: backup VM xml config
# ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dumpxml ${VM_ID} > /tmp/${COMPUTE_NODE}.${VM_ID}.xml
 
# show VM details (verfiy that OS-EXT-SRV-ATTR:host is not ${VMS_COMPUTE})
ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server show -c OS-EXT-SRV-ATTR:host -c name -c status ${VM_ID}"
 
# show VM details on compute node (verify that State: is "shut off")
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dominfo ${VM_ID}
 
# remove unnecessary VM
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh undefine ${VM_ID}