Check kibana logs for
"While synchronizing instance power states, found"
Check libvirt VMs vs. nova DB
COMPUTE_NODE=com1-dev # get shutoff VMs on compute node VMS_COMPUTE=$(ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh list --state-shutoff --uuid | sed '/^$/d' | sort) # echo "${VMS_COMPUTE}" # get shutoff VMs from nova DB VMS_NOVA=$(ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server list --all --host ${COMPUTE_NODE} -c ID -f value --status SHUTOFF" | sort) # echo "${VMS_NOVA}" # diff shutoff VMs comm -3 <(echo "${VMS_COMPUTE}") <(echo "${VMS_NOVA}")
Remove shutdown VM from libvirt
VM_ID=c3fbcc6b-2dfe-4aa1-82bd-522b161a37a9 # OPTINAL: backup VM xml config # ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dumpxml ${VM_ID} > /tmp/${COMPUTE_NODE}.${VM_ID}.xml # show VM details (verfiy that OS-EXT-SRV-ATTR:host is not ${VMS_COMPUTE}) ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server show -c OS-EXT-SRV-ATTR:host -c name -c status ${VM_ID}" # show VM details on compute node (verify that State: is "shut off") ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dominfo ${VM_ID} # remove unnecessary VM ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh undefine ${VM_ID}