Check kibana logs for
"While synchronizing instance power states, found"

Check libvirt VMs vs. nova DB

COMPUTE_NODE=com1-dev

# get shutoff VMs on compute node
VMS_COMPUTE=$(ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh list --state-shutoff --uuid | sed '/^$/d' | sort)
# echo "${VMS_COMPUTE}"

# get shutoff VMs from nova DB
VMS_NOVA=$(ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server list --all --host ${COMPUTE_NODE} -c ID -f value --status SHUTOFF" | sort)
# echo "${VMS_NOVA}"

# diff shutoff VMs
comm -3 <(echo "${VMS_COMPUTE}") <(echo "${VMS_NOVA}")

Remove shutdown VM from libvirt

VM_ID=c3fbcc6b-2dfe-4aa1-82bd-522b161a37a9

# OPTINAL: backup VM xml config
# ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dumpxml ${VM_ID} > /tmp/${COMPUTE_NODE}.${VM_ID}.xml

# show VM details (verfiy that OS-EXT-SRV-ATTR:host is not ${VMS_COMPUTE})
ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server show -c OS-EXT-SRV-ATTR:host -c name -c status ${VM_ID}"

# show VM details on compute node (verify that State: is "shut off")
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dominfo ${VM_ID}

# remove unnecessary VM
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh undefine ${VM_ID}