OpenStack: compute (nova)

openstack

CLI
https://docs.openstack.org/nova/latest/admin/services.html

# list hypervisor details
openstack hypervisor list --long

# list VMs with availability zone
openstack server list --long -c ID -c Name -c Status -c Networks -c "Image Name" -c "Flavor Name" -c "Availability Zone"

# list VMs on all hypervisor
openstack server list --all --long  -c ID -c Name -c Host

# list VMs on specific hypervisor
openstack server list --all-projects --host ${COMPUTE_NODE}

# get VM count by hypervisor
openstack server list --all --long  -c Host -f value | sort | uniq -c

# list compute nodes
openstack compute service list --service nova-compute

# list compute service
openstack compute service list --host ${OS_NODE}

# add / enable compute service
openstack compute service set --enable com1-dev nova-compute

# disable compute service
for OS_SERVICE in $(openstack compute service list --host ${OS_NODE} -c Binary -f value); do
    openstack compute service set --disable --disable-reason "Maintenance" ${OS_NODE} ${OS_SERVICE}
done

# Search for server witch status error
openstack server list --all --status ERROR

# Search for server with status resizing
openstack server list --all --status=VERIFY_RESIZE

# List instances / VMs
openstack server list
openstack server list -c ID -c Name -c Status -c Networks -c Host --long

# Show VM diagnostics / statistics
nova diagnostics ${SERVER_ID}
openstack server show --diagnostics ${SERVER_ID}

# show hypervisor usage
openstack usage list

Disable compute node

openstack compute service set --disable os-com2-dev nova-compute
openstack hypervisor list 
openstack compute service list --service nova-compute
openstack aggregate show

Debug

# Search for server processes on wrong compute node
for COMPUTE_NODE in $(openstack compute service list --service nova-compute -c Host -f value); do
    for UUID in $(ssh ${COMPUTE_NODE} pgrep qemu -a | grep -o -P '(?<=-uuid ).*(?= -smbios)'); do
        VM_HOST=$(openstack server show -c "OS-EXT-SRV-ATTR:host" -f value ${UUID})
        if [ -z "${VM_HOST}" ]; then
            echo "Server process ${UUID} on ${COMPUTE_NODE} not available in OpenStack"
        else
            if [ "${VM_HOST}" != "${COMPUTE_NODE}" ]; then
                echo "VM ${UUID} on wrong compute node ${COMPUTE_NODE}"
            fi
        fi
    done
done

Remove compute service / server

openstack server list --all-projects --host ${NODE_ID}
openstack compute service list --host ${NODE_ID}
openstack compute service delete ${NODE_ID}

Manually rebalance VMs

# show hypervisor usage
openstack hypervisor list --long

# get processes with uses swap
grep VmSwap /proc/*/status | grep -v " 0 kB"

# get VMs with high CPU usage
ssh compute-node-2

# VMs by CPU usage
ssh ${COMPUTE_NODE} ps -eo pid,%cpu,cmd --sort="-%cpu" --no-headers | head -5 | grep -o -P '^[0-9]?.*(?<=-uuid ).*(?= -smbios)\b' | awk '{ print $1,$2,$NF }'

# VMs by RAM usage
ssh ${COMPUTE_NODE} ps -eo pid,size,cmd --sort="-size" --no-headers | head -5 | grep -o -P '^[0-9]?.*(?<=-uuid ).*(?= -smbios)\b' | awk '{ print $1,$2,$NF }'

openstack server show ${SEVER_ID}

# live migrate VM to specific hypervisor
openstack server list --all --status ACTIVE --host comX-stage | grep large
openstack server migrate --os-compute-api-version 2.30 --live-migration --wait --host comX-stage ${SEVER_ID}

evacuate
https://docs.openstack.org/nova/latest/admin/evacuate.html

openstack server list --all-projects --host com3-dev
openstack server set --state error 8041442a-9775-47c8-91be-e27286e731bd
nova evacuate 8041442a-9775-47c8-91be-e27286e731bd

aggregate

openstack aggregate list
openstack aggregate show 9
openstack aggregate add host 9 com10-stage

Add compute node

openstack compute service list

vi /etc/kolla/inventory
...
[external-compute]
new_compute_node_2
...

cd /etc/kolla/config/foo
kolla-ansible -i inventory deploy --limit comX-dev -e 'ansible_python_interpreter=/usr/bin/python3'

Remove compute node

COMPUTE_HOST=com1-dev

# ensure all VMs are migrated out from the compute node
openstack server list --all-projects --host ${COMPUTE_HOST}

# remove compute service
COMPUTE_SERVICE_ID=$(openstack compute service list --service nova-compute --host ${COMPUTE_HOST} -c ID -f value)
echo ${COMPUTE_SERVICE_ID}
openstack compute service delete ${COMPUTE_SERVICE_ID}

# remove network service
NETWORK_AGENT_ID=$(openstack network agent list --host ${HYPERVISOR_ID} -c ID -f value)
echo ${NETWORK_AGENT_ID}
openstack network agent delete ${NETWORK_AGENT_ID}

# OPTIONAL: check no remaining resource_providers_allocations
http://www.panticz.de/openstack/resource-provider

# OPTIONAL: delete resource provider
openstack catalog list | grep placement
PLACEMENT_ENDPOINT=http://nova-placement.service.dev.i.example.com:8780

TOKEN=$(openstack token issue -f value -c id)
curl ${PLACEMENT_ENDPOINT}/resource_providers -H "x-auth-token: ${TOKEN}" | python -m json.tool

# delete resource provider
UUID=bf003af0-3541-4220-a5d5-c7c2e57abf22
curl ${PLACEMENT_ENDPOINT}/resource_providers/${UUID} -H "x-auth-token: $TOKEN" -X DELETE

Get CPU flags

cat /proc/cpuinfo | grep flags | head -1  | cut -d ":" -f2 | tr " " "\n" | sort

Get VM initially availability zones

mysql -u root -p"$DB_PASS" -h ctl1-stage.stage.i.example.com -D nova_api -e 'select json_extract(spec,"$.\"nova_object.data\".availability_zone") as availability_zone from request_specs where instance_uuid="78fe0882-c809-4ab7-99d4-920731fed315"\G'

Get qemu version

docker exec -t nova_libvirt qemu-x86_64 -version

Remove deleted VM in nova DB

mysql
use nova;
select uuid, hostname, vm_state, task_state, power_state, deleted from instances where uuid="43699fce-3b64-41e5-bb82-9e8bc4f5eeda";
update instances set deleted=1, vm_state="deleted" where uuid="5345281e-eafb-4de7-9c36-9ff00fe8635d";

Show VM deletion time

SELECT id, uuid, deleted, deleted_at, vm_state FROM nova.instances WHERE uuid = "${SERVER_ID}";

Get CPU flags

NODE_NAME=com1.dev
openstack hypervisor show ${NODE_NAME} -c cpu_info -f json | jq  '.cpu_info.features | sort'

Fix No server with a name or ID of 'ca931c85-864a-4c2f-9677-5d9602cdc70b' exists. / Cleanup deleted instance
https://bugs.launchpad.net/nova/+bug/1871925

openstack server list --all-projects --status ERROR

openstack server delete ca931c85-864a-4c2f-9677-5d9602cdc70b
No server with a name or ID of 'ca931c85-864a-4c2f-9677-5d9602cdc70b' exists.

get cell_id

select id,name from nova_api.cell_mappings where name is not NULL;

INSERT INTO nova_api.instance_mappings (created_at,updated_at,instance_uuid,cell_id,project_id) VALUES (NOW(),NOW(),'ca931c85-864a-4c2f-9677-5d9602cdc70b',1,'0789a3937d624510ac058a86e72f54a9');

openstack server delete ca931c85-864a-4c2f-9677-5d9602cdc70b

Delete affinity

openstack server group list --all-projects --long

# connect to DB
use nova_api;

SELECT id FROM nova_api.instance_group_member WHERE instance_uuid = 'e86f0316-1f9f-4f1e-af22-3d4b924611fa';
DELETE FROM nova_api.instance_group_member WHERE id = 1234;