openstack | panticz.de

OpenStack: Recreate neutron port

Get broken / INACTIVE ports

DB_NAME=neutron
DB_USER=neutron
DB_PASS=$(grep neutron_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
mysql --host=${DB_HOST} --password=${DB_PASS} --port=6033 --user=${DB_USER} --database=${DB_NAME}

select port_id, host from ml2_port_bindings where status = 'INACTIVE';
select port_id, host from ml2_port_bindings where status = 'INACTIVE' and host = 'com3-dev';
select * from ml2_port_bindings where port_id = "65db0c73-1533-4c36-9fd4-79b5e4e9704c";
#select * from ml2_port_binding_levels where port_id = "65db0c73-1533-4c36-9fd4-79b5e4e9704c" \G

# debug
PORT_IDS=$(mysql --host=${DB_HOST} --password=${DB_PASS} --port=6033 --user=${DB_USER} --database=${DB_NAME} \
  -se "select port_id from ml2_port_bindings where status = 'INACTIVE'")
echo ${PORT_IDS}

# get VMs
for PORT_ID in ${PORT_IDS}; do
    echo ${PORT_ID}
    openstack server show -c name -c "OS-EXT-SRV-ATTR:host" $(openstack port show -c device_id -f value ${PORT_ID})
done

Recreate port

SERVER_ID=f4b8828d-a717-49f6-8ad3-7b0bb1921123

nova interface-list ${SERVER_ID}

PORT_ID=57944199-014c-4907-9d6e-1fb233151234

# debug
openstack port show ${PORT_ID} -c name -c network_id -c mac_address -c fixed_ips -c security_group_ids -c project_id -c device_id

get port data

FIX: While synchronizing instance power states, found XX instances in the database and XX instances on the hypervisor

Check kibana logs for
"While synchronizing instance power states, found"

Check libvirt VMs vs. nova DB

COMPUTE_NODE=com1-dev

# get shutoff VMs on compute node
VMS_COMPUTE=$(ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh list --state-shutoff --uuid | sed '/^$/d' | sort)
# echo "${VMS_COMPUTE}"

# get shutoff VMs from nova DB
VMS_NOVA=$(ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server list --all --host ${COMPUTE_NODE} -c ID -f value --status SHUTOFF" | sort)
# echo "${VMS_NOVA}"

# diff shutoff VMs
comm -3 <(echo "${VMS_COMPUTE}") <(echo "${VMS_NOVA}")

Remove shutdown VM from libvirt

VM_ID=c3fbcc6b-2dfe-4aa1-82bd-522b161a37a9

# OPTINAL: backup VM xml config
# ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dumpxml ${VM_ID} > /tmp/${COMPUTE_NODE}.${VM_ID}.xml

# show VM details (verfiy that OS-EXT-SRV-ATTR:host is not ${VMS_COMPUTE})
ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server show -c OS-EXT-SRV-ATTR:host -c name -c status ${VM_ID}"

# show VM details on compute node (verify that State: is "shut off")
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dominfo ${VM_ID}

# remove unnecessary VM
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh undefine ${VM_ID}

OpenVswitch: cleanup interfaces without tap device

Show bridges without tab device

brctl show | egrep "qvb|tap" | sed '$!N;/\n.*tap/d;P;D' | awk '{print substr($1,4,8)}'

Get interface details

TOKEN=cac559da

# show port details
docker exec openvswitch_vswitchd ovsdb-client dump | grep ${TOKEN}

# get OVS port
docker exec openvswitch_vswitchd ovs-vsctl list-ports br-int | grep ${TOKEN}

# get OVS interface
docker exec openvswitch_vswitchd ovs-vsctl list-ifaces br-int | grep ${TOKEN}

# show host bridges
brctl show | grep ${TOKEN}

# show host interfaces
ip a | grep ${TOKEN}

Delete interface

PORT_TOKEN=a101dd83

# get port ID
PORT_ID=$(docker exec openvswitch_vswitchd ovs-vsctl list-ports br-int | grep ${PORT_TOKEN})
echo ${PORT_ID}

# remove port
docker exec openvswitch_vswitchd ovs-vsctl del-port br-int ${PORT_ID}

# remove bridge
BRIDGE_ID=${PORT_ID/qvo/qbr}
echo ${BRIDGE_ID}
ip link set ${BRIDGE_ID} down
brctl delbr ${BRIDGE_ID}

# remove interface
ip link delete ${PORT_ID}

OpenStack: project

Create project

USER_ID=xxxxxxxxxx
PROJECT_ID=xxxxxxxxx

openstack project create foo-project --domain bar

ROLES="
creator
heat_stack_owner
load-balancer_member
member
_member_
"

for ROLE in ${ROLES}; do
    openstack role add --user ${USER_ID} --project ${PROJECT_ID} ${ROLE}
done

openstack role assignment list --names --user ${USER_ID}

OpenStack: Horizon

List templates

docker exec -it horizon bash
ls -l /var/lib/kolla/venv/lib/python2.7/site-packages/openstack_dashboard/themes

Configure theme
/etc/kolla/horizon/custom_local_settings

AVAILABLE_THEMES = [
    ('default', 'Default', 'themes/default'),
    ('foo_bar', 'Foo bar', 'themes/foo_bar')
]

SELECTABLE_THEMES = [
    ('foo_bar', 'Foo bar', 'themes/foo_bar')
]

DEFAULT_THEME = 'foo_bar'

Floating IPs

# list all floating IPs
openstack floating ip list
openstack floating ip show 1.2.3.4
openstack floating ip list --project ${PROJECT_ID} --long

# Allocate floating IP
openstack floating ip create public 

# Allocate specific floating IP for a project (as cloud admin)
openstack floating ip create public --floating-ip-address 10.11.12.13 --project foo-project
FLOATING_IP=$(openstack floating ip create public -c floating_ip_address -f value --floating-ip-address 10.11.12.13)

# Associate floating IP to instance
openstack server add floating ip ${INSTANCE_NAME_OR_ID} ${FLOATING_IP}

# Disassociate floating IP
openstack server remove floating ip ${INSTANCE_NAME_OR_ID} ${FLOATING_IP}

# Release floating IP
openstack floating ip delete ${FLOATING_IP}

# show used floating IPs
NETWORK_ID=$(openstack subnet list --network public -c Network -f value)
openstack ip availability show -c total_ips -c used_ips ${NETWORK_ID}

# Add floating IP to port
openstack floating ip set --port ${PORT_ID} ${FLOATING_IP}

List reserved floating IPs per project

PROJECT_IDS=$(openstack floating ip list --status DOWN  -f value -c Project | sort -u)
for PROJECT_ID in ${PROJECT_IDS}; do
    PROJECT_NAME="$(openstack project show ${PROJECT_ID} -c name -f value)"
    DOMAIN_ID="$(openstack project show ${PROJECT_ID} -c domain_id -f value)"
    DOMAIN_NAME="$(openstack domain show ${DOMAIN_ID} -c name -f value)"

    echo "# ${DOMAIN_NAME} > ${PROJECT_NAME} (${PROJECT_ID})"

    openstack floating ip list --status DOWN  --project ${PROJECT_ID} -c "Floating IP Address" -f value
    echo
done

List available / used floating IP

OpenStack: compute (nova)

CLI
https://docs.openstack.org/nova/latest/admin/services.html

# list hypervisor details
openstack hypervisor list --long

# list VMs with availability zone
openstack server list --long -c ID -c Name -c Status -c Networks -c "Image Name" -c "Flavor Name" -c "Availability Zone"

# list VMs on all hypervisor
openstack server list --all --long  -c ID -c Name -c Host

# list VMs on specific hypervisor
openstack server list --all-projects --host ${COMPUTE_NODE}

# get VM count by hypervisor
openstack server list --all --long  -c Host -f value | sort | uniq -c

# list compute nodes
openstack compute service list --service nova-compute

# list compute service
openstack compute service list --host ${OS_NODE}

# add / enable compute service
openstack compute service set --enable com1-dev nova-compute

# disable compute service
for OS_SERVICE in $(openstack compute service list --host ${OS_NODE} -c Binary -f value); do
    openstack compute service set --disable --disable-reason "Maintenance" ${OS_NODE} ${OS_SERVICE}
done

# Search for server witch status error
openstack server list --all --status ERROR

# Search for server with status resizing
openstack server list --all --status=VERIFY_RESIZE

# List instances / VMs
openstack server list
openstack server list -c ID -c Name -c Status -c Networks -c Host --long

# Show VM diagnostics / statistics
nova diagnostics ${SERVER_ID}
openstack server show --diagnostics ${SERVER_ID}

# show hypervisor usage
openstack usage list

Disable compute node

OpenStack: Floating IP port forward (in development / experimental)

List floatin IPs

openstack floating ip list
+--------------------------------------+---------------------+------------------+--------------------------------------+--------------------------------------+----------------------------------+
| ID                                   | Floating IP Address | Fixed IP Address | Port                                 | Floating Network                     | Project                          |
+--------------------------------------+---------------------+------------------+--------------------------------------+--------------------------------------+----------------------------------+
| dc049c28-6562-4c37-834b-d3a612d4b580 | 1.2.3.4        | None             | None                                 | 39583230-154f-4b56-a56e-2fd83c9986ce | 1eede1bdc28344f3acf6b48b232e406f |
+--------------------------------------+---------------------+------------------+--------------------------------------+--------------------------------------+----------------------------------+

List VMs

openstack server list
+--------------------------------------+---------------------------------------------------+--------+---------------------------------+-----------------------------------+-----------+
| ID                                   | Name                                              | Status | Networks                        | Image                             | Flavor    |
+--------------------------------------+---------------------------------------------------+--------+---------------------------------+-----------------------------------+-----------+
| d278cb52-d258-491b-8787-25aaab75f84c | dev-u1910                                         | ACTIVE | dev-net=10.0.1.9                | Ubuntu 19.10 minimal              | m1.micro  |
| 1da2e275-e680-4c69-be00-93871274ceb3 | dev-u1804                                         | ACTIVE | dev-net=10.0.1.12               | Ubuntu 18.04                      | m1.micro  |
+--------------------------------------+---------------------------------------------------+--------+---------------------------------+-----------------------------------+-----------+

Get port IDs for server

openstack port list --server ${SERVER_ID} -c id -f value
4f20d5a9-bdc6-4046-b1bc-291c4d4bf86d

openstack port list --server ${SERVER_ID} -c id -f value
9e583adb-618f-42ba-96ee-1a7b89a377b3

Create port forward to HTTP

Octavia: Allow SSH login to Amphora VM

Allow SSH access

LB_ID=foo-lb01-prod

AMPHORA_ID=$(openstack loadbalancer amphora list --loadbalancer ${LB_ID} --role MASTER -c id -f value)
AMPHORA_COMPUTE_ID=$(openstack loadbalancer amphora show ${AMPHORA_ID} -c compute_id -f value)
LB_NETWORK_IP=$(openstack loadbalancer amphora show ${AMPHORA_ID} -c lb_network_ip -f value)
SECURITY_GROUP_ID=$(openstack port list --server ${AMPHORA_COMPUTE_ID} --fixed-ip "ip-address=${LB_NETWORK_IP}" -c security_group_ids -f value)

# DEBUG: show ingress tcp rules
openstack security group rule list --ingress --protocol tcp ${SECURITY_GROUP_ID}
openstack security group rule create --protocol tcp --dst-port 22:22 --remote-ip 172.16.0.0/12  ${SECURITY_GROUP_ID}
openstack loadbalancer amphora list --loadbalancer ${LB_ID} -c  lb_network_ip -c role -f value
openstack loadbalancer amphora list --loadbalancer ${LB_ID} -c  lb_network_ip --role MASTER -f value

# login to amphora VM from OpenStack control node
ssh local@ctl1-dev.dev.i.example.com
ssh -i ~/.ssh/id_rsa_octavia ubuntu@${AMPHORA_VM_IP}

Manuall SSH access

Openstack: Debug networking

Video: OpenStack Neutron Packet Walkthrough (DVR) by David Mahler
https://www.youtube.com/watch?v=7IXEtUEZslg

SERVER_ID=11111111-2222-3333-4444-555555555555

nova interface-list ${SERVER_ID}

PORT_TOKEN=$(nova interface-list ${SERVER_ID} | grep ACTIVE | cut -d "|" -f3 | cut -d"-" -f1)
echo PORT_TOKEN=${PORT_TOKEN}

COMPUTE_NODE=$(openstack server show -c "OS-EXT-SRV-ATTR:hypervisor_hostname" -f value ${SERVER_ID})
echo ${COMPUTE_NODE}

ssh ${COMPUTE_NODE} brctl show | grep ${PORT_TOKEN}
TAP_DEVICE=$(ssh ${COMPUTE_NODE} brctl show | grep ${PORT_TOKEN} | grep tap)

TAP_DEVICE=$(ssh ${COMPUTE_NODE} brctl show | grep ${PORT_TOKEN} | grep tap)
echo ${TAP_DEVICE}

# second console
echo tcpdump -i ${TAP_DEVICE} -n -e icmp
ssh ${COMPUTE_NODE}



# 3th console
ssh ${COMPUTE_NODE}
iptables -S | grep o${PORT_TOKEN} | grep ${PORT_TOKEN}
# todo:
iptables -S neutron-openvswi-s${PORT_TOKEN}-f

iptables -S neutron-openvswi-sg-chain | grep #${PORT_TOKEN}

docker exec -it openvswitch_vswitchd bash
ovs-vsctl show

ovs-vsctl show | grep -U1 ${PORT_TOKEN}

# NORMAL missiong
watch -n .5 "ovs-ofctl dump-flows br-int table=0 | grep -v n_packets=0"


ovs-appctl -t /var/run/openvswitch/ovs-vswitchd.*.ctl fdb/show br-int | egrep "MAC|fa:11:22:33:44:55"

ovs-ofctl show br-int | grep 293 -A5

brctl show | grep qvo${PORT_TOKEN}-fb

Notes

openstack