OpenStack: OpenvSwitch

Get VM data

# OPTINAL: find VM name
# openstack server list --all | grep XXX

# get FROM VM ID
SERVER_STRING=www-dev
SERVER_ID=$(openstack server list --all --name ${SERVER_STRING} -c ID -f value)
echo ${SERVER_ID}

SERVER_IP_ADDRESS=10.0.0

PORT_TOKEN=$(nova interface-list ${SERVER_ID} | grep ${SERVER_IP_ADDRESS} | awk -F'|' '/ACTIVE/ {gsub(/ /,"",$3); print substr($3,0,9)}')
NETWORK_TOKEN=$(nova interface-list ${SERVER_ID} | grep ${SERVER_IP_ADDRESS} | awk -F'|' '/ACTIVE/ {gsub(/ /,"",$4); print substr($4,0,9)}')
SERVER_NAME=$(openstack server show ${SERVER_ID} -c name -f value)
SERVER_IP=$(nova interface-list ${SERVER_ID} | grep ${SERVER_IP_ADDRESS} | awk -F'|' '/ACTIVE/ {gsub(/ /,"",$5); print $5}')
SERVER_MAC=$(nova interface-list ${SERVER_ID} | grep ${SERVER_IP_ADDRESS} | awk -F'|' '/ACTIVE/ {gsub(/ /,"",$6); print $6}')
COMPUTE_NODE=$(openstack server show -c "OS-EXT-SRV-ATTR:hypervisor_hostname" -f value ${SERVER_ID})
SERVER_DEV=$(ssh ${COMPUTE_NODE} brctl show | grep ${PORT_TOKEN} | paste - - | sed -e 's/[\t]/ /g;s/  */ /g' | awk '{print $5" "$4}')

# show server data
echo "
SERVER_NAME: ${SERVER_NAME}
SERVER_ID: ${SERVER_ID}
SERVER_IP: ${SERVER_IP}
SERVER_MAC: ${SERVER_MAC}
SERVER_DEV: ${SERVER_DEV}
PORT_TOKEN: ${PORT_TOKEN}
NETWORK_TOKEN: ${NETWORK_TOKEN}
COMPUTE_NODE: ${COMPUTE_NODE}
"

Search for broken openvswitch devices

# show interaces
docker exec -it openvswitch_vswitchd ovs-vsctl show

# delete interface
docker exec openvswitch_vswitchd ovs-vsctl -- del-port br-int my-interface1

# get "No such device" devices
for i in $(docker exec -it openvswitch_vswitchd ovs-vsctl show | grep "No such device" | awk '{print $7}'); do
    docker exec -it openvswitch_vswitchd ovs-vsctl list Interface | grep -C 3 $i | grep vm-uuid | awk -F\" '{ print $6}'
done

# ssh kolla.example.com
UUIDS="
aaaaaaaa-11111-2222-3333-111111
aaaaaaaa-11111-2222-3333-222222
aaaaaaaa-11111-2222-3333-333333
"

# get VM ID for ovs device
for VM_ID in ${UUIDS}; do
    openstack server show -f table -c OS-EXT-SRV-ATTR:host -c OS-EXT-SRV-ATTR:hostname -c id -f value ${VM_ID} | paste - - -
done

# search for unnecessary Open vSwitch interfaces
for COMPUTE_NODE in $(openstack compute service list --service nova-compute -c Host -f value); do
    for PORT_TOKEN in $(ssh ${COMPUTE_NODE} brctl show | egrep "qvb|tap" | sed '$!N;/\n.*tap/d;P;D' | awk '{print substr($1,4,8)}'); do
        PORT_ID=$(openstack port list -c id -f value | grep ${PORT_TOKEN})
        if [ -n "${PORT_ID}" ]; then
            SERVER_ID=$(openstack port show -c device_id -f value ${PORT_ID})
            if [ "$(openstack server show -c status -f value ${SERVER_ID})" != "SHUTOFF" ]; then
                echo "COMPUTE_NODE: ${COMPUTE_NODE}"
                echo "PORT_TOKEN: ${PORT_TOKEN}"
                openstack server show -c name -c id -c project_id -c status ${SERVER_ID}

                EXIT_CODE=1
            fi
        else
            echo "Port not found for PORT_TOKEN ${PORT_TOKEN} on COMPUTE_NODE ${COMPUTE_NODE}"

            EXIT_CODE=1
        fi
    done
done

# workaround for: ovs-appctl: cannot read pidfile "/var/run/openvswitch/ovs-vswitchd.pid" (No such file or directory)
docker exec -it openvswitch_vswitchd ovs-appctl -t /var/run/openvswitch/ovs-vswitchd.*.ctl fdb/show br-int

Recreate ARP entry in OpenVswitch

ssh compute-node1.example.com
docker exec -it openvswitch_vswitchd bash

# source and target host on same compute node
ovs-ofctl dump-flows br-int | grep 10.0.0.11 | grep arp

# source and target host on different compute nodes
ovs-ofctl dump-flows br-tun | grep 10.0.0.11 | grep arp

# workaround: recreate ARP entry
SERVER_STRING=www1-foo-dev
SERVER_ID=$(openstack server list --all --name ${SERVER_STRING} -c ID -f value)
PORT_ID=$(nova interface-list ${SERVER_ID} | awk -F'|' '/10.0.0/ {print $3}')
openstack port set --disable ${PORT_ID}
openstack port set --enable ${PORT_ID}

Check ARP entry

(openvswitch-vswitchd)[root@com4-dev /]# ovs-ofctl dump-flows br-tun | grep arp | grep "10.0.0.34 "

Debug OVS target compute route

ssh com4-dev
docker exec -it openvswitch_vswitchd bash

# get "output:xx"
ovs-ofctl dump-flows br-tun table=20 | grep fa:16:3e:42:af:1d
 cookie=0xd5f791238cfcad5a, duration=80675.893s, table=20, n_packets=1489118, n_bytes=119328628, idle_age=0, hard_age=65534, priority=2,dl_vlan=110,dl_dst=fa:16:3e:42:af:1d actions=strip_vlan,load:0x276b->NXM_NX_TUN_ID[],output:25

# get "vxlan-*"
ovs-ofctl show br-tun | grep " 25("
 25(vxlan-0a224192): addr:d6:49:2e:b1:5c:0e

# get "remote_ip=xxx"
ovs-vsctl show | grep vxlan-0a224192 -A2
        Port "vxlan-0a224192"
            Interface "vxlan-0a224192"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="10.0.67.144", out_key=flow, remote_ip="10.0.65.146", tos=inherit}

# reverse lookup compute node
dig -x 10.0.65.146

tcpdump traffic on source compute

ssh com4
docker exec -it openvswitch_vswitchd bash

# run tcpdump on tap device
tcpdump -i tape7ae657e-9a -n -e icmp and host 10.0.0.34
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on tape7ae657e-9a, link-type EN10MB (Ethernet), capture size 262144 bytes
16:48:38.258747 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98: 10.0.0.15 > 10.0.0.34: ICMP echo request, id 18918, seq 231, length 64
16:48:39.282825 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98: 10.0.0.15 > 10.0.0.34: ICMP echo request, id 18918, seq 232, length 64
16:48:40.306772 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98: 10.0.0.15 > 10.0.0.34: ICMP echo request, id 18918, seq 233, length 64

# define target
TARGET_IP=10.0.0.34
TARGET_MAC=fa:16:3e:42:af:1d


# OPTIONAL: verify target going over patch-tun
OPEN_FLOW_PORT=$(ovs-appctl -t /var/run/openvswitch/ovs-vswitchd.*.ctl fdb/show br-int | egrep ${TARGET_MAC} | awk '{print $1}')
echo ${OPEN_FLOW_PORT}
2

OPEN_FLOW_INTERFACE=$(ovs-ofctl show br-int | grep " ${OPEN_FLOW_PORT}(" | cut -d "(" -f2 | cut -d ")" -f1)
echo ${OPEN_FLOW_INTERFACE}
patch-tun


# get vxlan interace
TARGET_OPENFLOW_PORT=$(ovs-ofctl dump-flows br-tun | egrep "table=20.*${TARGET_MAC}.*strip_vlan" | cut -d "," -f12 | cut -d":" -f2)
echo ${TARGET_OPENFLOW_PORT}
25

ovs-ofctl show br-tun | grep " ${TARGET_OPENFLOW_PORT}("
 25(vxlan-0a224192): addr:d6:49:2e:b1:5c:0e

VXLAN_INTERFACE=$(ovs-ofctl show br-tun | grep " ${TARGET_OPENFLOW_PORT}(" | cut -d"(" -f 2 | cut -d")" -f1)
echo ${VXLAN_INTERFACE}
vxlan-0a224192

# run tcpdump on vxlan interface
ovs-tcpdump -i ${VXLAN_INTERFACE} -n -e icmp and host ${TARGET_IP}
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ovsmi978355, link-type EN10MB (Ethernet), capture size 262144 bytes
16:56:40.565094 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98: 10.0.0.15 > 10.0.0.34: ICMP echo request, id 18918, seq 702, length 64
16:56:41.589084 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98: 10.0.0.15 > 10.0.0.34: ICMP echo request, id 18918, seq 703, length 64
16:56:42.613105 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98: 10.0.0.15 > 10.0.0.34: ICMP echo request, id 18918, seq 704, length 64

Dump traffic on target compute node

ssh com2-dev
docker exec -it openvswitch_vswitchd bash
VXLAN_FROM_IP=10.34.67.144
TARGET_IP=10.20.0.34

VXLAN_INTERFACE_INCOMMING=$(ovs-vsctl show | grep -B3 "remote_ip=\"${VXLAN_FROM_IP}\"" | grep "Port \"vxlan-" | cut -d'"' -f2)
echo ${VXLAN_INTERFACE_INCOMMING}
vxlan-0a224390

# dump traffic on vxlan
ovs-tcpdump -i ${VXLAN_INTERFACE_INCOMMING} -n -e icmp and host ${TARGET_IP}
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ovsmi138605, link-type EN10MB (Ethernet), capture size 262144 bytes
....
17:07:24.310548 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98:10.0.0.15 >10.0.0.34: ICMP echo request, id 22172, seq 200, length 64
17:07:25.334366 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98:10.0.0.15 >10.0.0.34: ICMP echo request, id 22172, seq 201, length 64
17:07:26.358355 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98:10.0.0.15 >10.0.0.34: ICMP echo request, id 22172, seq 202, length 64

exit

# todo: debug chain from vxlan to tap device

# dump traffic on tap device
tcpdump -i ${TAP_DEVICE} -n -e icmp and host10.0.0.34
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on tapd9cdff27-9c, link-type EN10MB (Ethernet), capture size 262144 bytes
17:23:00.292321 fa:16:3e:b0:62:d3 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98:10.0.0.26 >10.0.0.34: ICMP echo request, id 16324, seq 300, length 64
17:23:00.694029 fa:16:3e:bd:f0:f0 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98:10.0.0.15 >10.0.0.34: ICMP echo request, id 24555, seq 298, length 64
17:23:01.316292 fa:16:3e:b0:62:d3 > fa:16:3e:42:af:1d, ethertype IPv4 (0x0800), length 98:10.0.0.26 >10.0.0.34: ICMP echo request, id 16324, seq 301, length 64

Clenaup

# cleanup all
docker exec -it neutron_openvswitch_agent neutron-ovs-cleanup

# cleanup tcp-dumpport
docker exec -i openvswitch_vswitchd ovs-vsctl show | egrep "ovsmi|mibr-int"
docker exec -i openvswitch_vswitchd ovs-vsctl del-port br-tun ovsmi954582
docker exec -i openvswitch_vswitchd ovs-vsctl del-port br-int mibr-int

Links
https://aptira.com/openstack-rules-how-openvswitch-works-inside-openstack/
http://www.tuxfixer.com/how-to-find-qbr-qvo-interfaces-of-the-particular-instance-in-openstack/