openstack

FIX: While synchronizing instance power states, found XX instances in the database and XX instances on the hypervisor

Check kibana logs for
"While synchronizing instance power states, found"

Check libvirt VMs vs. nova DB

COMPUTE_NODE=com1-dev
 
# get shutoff VMs on compute node
VMS_COMPUTE=$(ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh list --state-shutoff --uuid | sed '/^$/d' | sort)
# echo "${VMS_COMPUTE}"
 
# get shutoff VMs from nova DB
VMS_NOVA=$(ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server list --all --host ${COMPUTE_NODE} -c ID -f value --status SHUTOFF" | sort)
# echo "${VMS_NOVA}"
 
# diff shutoff VMs
comm -3 <(echo "${VMS_COMPUTE}") <(echo "${VMS_NOVA}")

Remove shutdown VM from libvirt

OpenVswitch: cleanup interfaces without tap device

Show bridges without tab device

brctl show | egrep "qvb|tap" | sed '$!N;/\n.*tap/d;P;D' | awk '{print substr($1,4,8)}'

Get interface details

TOKEN=cac559da
 
# show port details
docker exec openvswitch_vswitchd ovsdb-client dump | grep ${TOKEN}
 
# get OVS port
docker exec openvswitch_vswitchd ovs-vsctl list-ports br-int | grep ${TOKEN}
 
# get OVS interface
docker exec openvswitch_vswitchd ovs-vsctl list-ifaces br-int | grep ${TOKEN}
 
# show host bridges
brctl show | grep ${TOKEN}
 
# show host interfaces
ip a | grep ${TOKEN}

Delete interface

OpenStack: project

Create project

openstack project create foo-project --domain bar
 
ROLES="
creator
heat_stack_owner
load-balancer_member
member
"
 
for ROLE in ${ROLES}; do
    openstack role add --user ${USER_ID} --project ${PROJECT_ID} ${ROLE}
done
 
openstack role assignment list --names --user ${USER_ID}

OpenStack: Horizon

List templates

docker exec -it horizon bash
ls -l /var/lib/kolla/venv/lib/python2.7/site-packages/openstack_dashboard/themes

Configure theme
/etc/kolla/horizon/custom_local_settings

AVAILABLE_THEMES = [
    ('default', 'Default', 'themes/default'),
    ('foo_bar', 'Foo bar', 'themes/foo_bar')
]
 
SELECTABLE_THEMES = [
    ('foo_bar', 'Foo bar', 'themes/foo_bar')
]
 
DEFAULT_THEME = 'foo_bar'

Floating IPs

https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html

openstack floating ip list
 
openstack floating ip create public-net
FLOATING_IP=$(openstack floating ip create public -c floating_ip_address -f value --floating-ip-address 10.11.12.13)
 
openstack floating ip show 1.2.3.4
openstack floating ip list --project ${PROJECT_ID} --long
 
openstack floating ip create --floating-ip-address 1.2.3.4 public
 
# request specific floating IP for a project
openstack floating ip create public --floating-ip-address 1.2.3.4 --project foo-project
 
# show used floating IPs
NETWORK_ID=$(openstack subnet list --network public -c Network -f value)
openstack ip availability show -c total_ips -c used_ips ${NETWORK_ID}
 
# add ip to server
openstack floating ip set --port ${PORT_ID} ${FLOATING_IP}

OpenStack: compute

CLI
https://docs.openstack.org/nova/latest/admin/services.html

# list VMs on all hypervisor
openstack server list --all --long  -c Name -c Host
 
# list VMs on specific hypervisor
openstack server list --all -c Name -f value --host ${COMPUTE_NODE}
 
# get VM count by hypervisor
openstack server list --all --long  -c Host -f value | sort | uniq -c
 
# list compute nodes
openstack compute service list
 
# list compute service
openstack compute service list --host ${OS_NODE}
 
# disable compute service
for OS_SERVICE in $(openstack compute service list --host ${OS_NODE} -c Binary -f value); do
    openstack compute service set --disable --disable-reason "Maintenance" ${OS_NODE} ${OS_SERVICE}
done
 
# Search for server witch status error
openstack server list --all --status ERROR
 
# Search for server with status resizing
openstack server list --all --status=VERIFY_RESIZE
 
# List instances / VMs
openstack server list
openstack server list -c ID -c Name -c Status -c Networks -c Host --long

Debug

OpenStack: Floating IP port forward (in development / experimental)

List floatin IPs

openstack floating ip list
+--------------------------------------+---------------------+------------------+--------------------------------------+--------------------------------------+----------------------------------+
| ID                                   | Floating IP Address | Fixed IP Address | Port                                 | Floating Network                     | Project                          |
+--------------------------------------+---------------------+------------------+--------------------------------------+--------------------------------------+----------------------------------+
| dc049c28-6562-4c37-834b-d3a612d4b580 | 1.2.3.4        | None             | None                                 | 39583230-154f-4b56-a56e-2fd83c9986ce | 1eede1bdc28344f3acf6b48b232e406f |
+--------------------------------------+---------------------+------------------+--------------------------------------+--------------------------------------+----------------------------------+

List VMs

OpenStack: Debug DHCP

Get HIGH lease rate not found

for NODE in $(openstack compute service list --service nova-scheduler -c Host -f value); do
    echo ${NODE}
    ssh ${NODE} cat /var/lib/docker/volumes/kolla_logs/_data/neutron/dnsmasq.log | grep "lease not found" | grep Sep | awk '{print $1, $2}' | uniq -c | awk '$1 > 100 {print}'
    echo
done

Many dhcp requests

Subscribe to openstack