Fix Octavia / Amphora VMs

openstack

List Amphora VMs

openstack server list --project service

Fix broken/out Amphora VMs / Ports
http://eavesdrop.openstack.org/irclogs/%23openstack-lbaas/%23openstack-lbaas.2019-05-17.log.html

Fix broken amphora port
DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
 
# check VRRP ports
for i in $(openstack port list -f value | grep octavia-lb-vrrp| awk '{ print $1}'); do
	echo $i; openstack port show $i -c allowed_address_pairs -f value
done > /tmp/vrrp.txt
 
 
# List broken loadbalancer
openstack loadbalancer list | grep -v ACTIVE
openstack loadbalancer amphora list | grep -v ALLOCATED
 
 
# Set LB ID
LB_ID=b98ea4bb-0631-4be9-a9e8-8841bf56bf2f
# openstack loadbalancer show ${LB_ID}
 
# Show amphora status
openstack loadbalancer amphora list --loadbalancer ${LB_ID}
 
# ensure LB provisioning_status is ERROR (deprected?)
# mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
#	--execute="update load_balancer set provisioning_status = 'ERROR' where id = '${LB_ID}'"
 
# OPTIONAL: delete standalone amphora entries
# mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="delete from amphora where role = 'STANDALONE' and load_balancer_id='${LB_ID}'"
 
LB_HA_IP=$(mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} --skip-column-names -s \
	--execute="select distinct ha_ip from amphora where load_balancer_id='${LB_ID}'")
# grep -B1 ${LB_HA_IP} /tmp/vrrp.txt
 
# Check / show existing LB ports
grep -B1 ${LB_HA_IP} /tmp/vrrp.txt | egrep '^[a-Z0-9]+' | \
	xargs -L1 openstack port show -c id -f value
 
# Show loadbalancer status in DB
mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \
	--execute="select * from amphora where load_balancer_id='${LB_ID}'"
 
# check if both VRRP ports exist and got a MASTER and a BACKUP entry in the DB
# "recover" a DELETED master or backup amphorae and ensure to use a unallocated fake IP for lb_network_ip
LB_AMPHORA_ID=3db411b1-4b20-429f-80a5-3cb7c451876b
mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
	--execute="update amphora set status='ALLOCATED', lb_network_ip='172.16.200.123' where id='${LB_AMPHORA_ID}'";
 
# OPTIONAL: delete broken amphora entry
# mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="delete from amphora where id = '${LB_AMPHORA_ID}' and load_balancer_id='${LB_ID}'"
 
# ensure LB provisioning_status is ACTIVE
mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
  --execute="update load_balancer set provisioning_status = 'ACTIVE' where id = '${LB_ID}'"
 
# failover amphora
openstack loadbalancer amphora list --loadbalancer ${LB_ID}
openstack loadbalancer amphora failover ${LB_AMPHORA_ID}
 
# DEBUG
watch openstack loadbalancer amphora list --loadbalancer ${LB_ID}

Recreate VRRP port

# Show loadbalancer status in DB
mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \
	--execute="select * from amphora where load_balancer_id='${LB_ID}'"
 
openstack port show 7ba30f9b-6506-4976-8f6f-9548fee42c6e
 
openstack port create octavia-lb-vrrp-$(uuidgen) \
  --network ${LB_NETWORK} \
  --fixed-ip ip-address='${AMPHORA_IP}',subnet='${LB_SUBNET}' \
  --security-group ${LB_SECURITY_GROUP} \
  --allowed-address ip-address=${LB_HA_IP}
 
mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \
  --execute="update amphora set status='ALLOCATED', role='MASTER', vrrp_interface='eth1', vrrp_priority=100, vrrp_port_id='34aef610-d38f-4eca-8c7d-aad19e6e6f5f', vrrp_ip='192.168.248.32' where id='ecdcceeb-39a0-4a51-a872-6af81edc9ab2'";

Failover all Amphora VMs in status ERROR

LB_IDS=$(openstack loadbalancer amphora list --status ERROR -c loadbalancer_id -f value | sort -u)
for LB_ID in ${LB_IDS}; do
    openstack loadbalancer failover ${LB_ID}
    sleep 300
    openstack loadbalancer amphora list --loadbalancer ${LB_ID}
done

Set LoadBalancer state to ACTIVE

# get DB credentials (kolla-ansible)
DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
 
# Update loadbalancer state to ACTIVE
LB_ID=xxxxxxxxxxxxxx
 
mysql --host=db.service.i.ewcs.ch --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="update load_balancer set provisioning_status = 'ACTIVE' where id = '${LB_ID}'"
 
# Debug
openstack loadbalancer show ${LB_ID}
 
 
# Update LB pool state to ACTIVE
POOL_ID=xxxxxxxxxxxxxx
 
mysql --host=db.service.i.ewcs.ch --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="update pool set provisioning_status = 'ACTIVE' where id = '${POOL_ID}'"
 
# Debug
openstack loadbalancer pool list --loadbalancer ${LB_ID}
 
 
# Update LB pool member state to ACTIVE
MEMBER_ID=xxxxxxxxxxxxxx
 
mysql --host=db.service.i.ewcs.ch --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="update member set provisioning_status = 'ACTIVE' where id = '${MEMBER_ID}'"
 
# Debug
openstack loadbalancer member list ${POOL_ID}

Find broken Octavia Loadbalancer / Amphora VMs
The "loadbalancer amphora failover" command are avaiable since OSC Stein release: https://docs.openstack.org/releasenotes/python-octaviaclient/stein.html but the python-octaviaclient 1.7 OSC also works with OS Rocky realase. (pip install -U "python-octaviaclient==1.7.0")

DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
 
LB_IDS=$(openstack loadbalancer amphora list --status ERROR -c loadbalancer_id -f value | sort | uniq)
for LB_ID in ${LB_IDS}; do
    # update LB status to ACTIVE
    mysql --host=db.service.stage.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \
        --execute="update load_balancer set provisioning_status='ACTIVE' where id='${LB_ID}';"
 
 
    # get amphora ID
    AMPHORA_IDS=$(openstack loadbalancer amphora list --status ERROR --loadbalancer ${LB_ID} -c id -f value)
 
	for AMPHORA_ID in ${AMPHORA_IDS}; do
    	# failover broken amphora VMs
	    openstack loadbalancer amphora failover ${AMPHORA_ID}
 
	    sleep 60
	done
done

Manual debug Amphora VMs

# List broken LoadBalancer instances
openstack loadbalancer list --provisioning-status ERROR
openstack loadbalancer list --provisioning-status PENDING_UPDATE
 
# List broken Loadbalancer VMs
openstack loadbalancer amphora list --provisioning-status ERROR
openstack loadbalancer amphora list --role STANDALONE

Show Loadbalancer state

LB_ID=0ce30f0e-1d75-486c-a09f-79125abf44b8
 
# List LoadBalancer details
openstack loadbalancer show ${LB_ID}
 
# List LoadBalancer VMs details
openstack loadbalancer amphora list --loadbalancer ${LB_ID}
 
# show project
PROJECT_ID=$(openstack loadbalancer show -c project_id -f value ${LB_ID})
openstack project show -c name -f value ${PROJECT_ID}
 
# show domain
DOMAIN_ID=$(openstack project show -c domain_id -f value ${PROJECT_ID})
openstack domain show -c name -f value ${DOMAIN_ID}

Manual update provisioning_status from PENDING_UPDATE / ERROR state to ACTIVE in Octavia Database
Currently there is no OpenStack CLI command to change the provisioning_status with the CLI

DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
 
mysql --host=db.service.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="update load_balancer set provisioning_status = 'ACTIVE' where id = '${LB_ID}'"

Trigger manual LoadBalancer failover
This will cause a short interruption of the LB service since both VMs will by recreated

# manually failover loadbalancer
openstack loadbalancer failover ${LB_ID}
 
# manually failover amphora vm
AMPHORA_ID=$(openstack loadbalancer amphora list --loadbalancer ${LB_ID} --status ERROR -c id -f value)
openstack loadbalancer amphora failover ${AMPHORA_ID}

Check LoadBalancer state

# check LoadBalancer state
openstack loadbalancer show ${LB_ID}
 
# check LoadBalancer VMs state
openstack loadbalancer amphora list --loadbalancer ${LB_ID}

Debug

# list all loadbalancer instances
openstack loadbalancer list
 
# list all Amphora instances
openstack loadbalancer amphora list
 
# List all Amphora VMs
openstack server list --all --long --name amphora
 
# get LoadBalancer count
openstack loadbalancer list -c id -f value | wc -l
 
# get Amphora VMs count
openstack loadbalancer amphora list -c loadbalancer_id -f value | wc -l
openstack server list --all --long --name amphora -c Name -f value | wc -l
 
# restart LB VM
openstack server reboot --soft amphora-${AMPHORA_ID}
 
# show LB VM console
openstack console log show amphora-${AMPHORA_ID}
 
# check amphora vs. server VM
openstack server list --all-projects --name amphora -c Name -f value | cut -d "-" -f2- | sort > /tmp/amphora_vms
openstack loadbalancer amphora list --status ALLOCATED -c id -f value | sort > /tmp/amphora_lb
diff /tmp/amphora_vms /tmp/amphora_lb
 
openstack loadbalancer amphora list | grep 558196ab-6215-4f4d-a4e8-9eee2a300d33
openstack server show amphora-558196ab-6215-4f4d-a4e8-9eee2a300d33
 
openstack loadbalancer amphora failover 558196ab-6215-4f4d-a4e8-9eee2a300d33

Debug DB

# show amphora VMs
openstack loadbalancer amphora list --loadbalancer  ${LB_ID}
 
# show defined LB MAC / IP
LB_VIP_NETWORK_ID=$(openstack loadbalancer show ${LB_ID} -c vip_network_id -f value)
openstack port list --network ${LB_VIP_NETWORK_ID} | grep vrrp
LB_PORT_IDS=$(openstack port list --network ${LB_VIP_NETWORK_ID} | grep vrrp | cut -d" " -f2)
for LB_PORT_ID in ${LB_PORT_IDS}; do
     LB_PORT_DEVICE_ID=$(openstack port show ${LB_PORT_ID} -c device_id -f value)
     openstack server show ${LB_PORT_DEVICE_ID} -c name -f value
done
 
# show used VM MAC / IP
LB_AMPHORA_VMS=$(openstack loadbalancer amphora list --loadbalancer  ${LB_ID} -c id -f value)
for LB_AMPHORA_VM in ${LB_AMPHORA_VMS}; do
    openstack port list --server amphora-${LB_AMPHORA_VM}
done
 
# show definded MAC / IP in DB
mysql --host=db.service.i.example.net --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="select * from amphora where load_balancer_id='${LB_ID}' order by updated_at desc"
 
 
LB_AMPHORA_ID=606cf400-b626-4473-8343-5d25b1c04a0d
 
# update amphora status ERROR to DELETED
mysql --host=db.service.i.example.net --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \
    --execute="update amphora set status='DELETED' where id='${LB_AMPHORA_ID}'";
 
 
 
openstack server delete amphora-${LB_AMPHORA_ID}
 
 
 
mysql --host=db.service.i.example.net --port=6033 --database=octavia --user=octavia --password=${DB_PASS} 
--execute="update amphora set status='ALLOCATED', role='BACKUP', vrrp_priority=90, vrrp_port_id='f53eb837-cf88-4008-b019-5a6637e33d48', vrrp_ip='10.15.10.19' where id='45d90b49-45af-4c87-9f72-33929178e929'";    
 
 
 
openstack loadbalancer list | grep -v ACTIVE
openstack loadbalancer amphora list | grep -Ev "ALLOCATED|READY"
openstack loadbalancer amphora list | grep -Ev "MASTER|BACKUP|READY"
 
 
 
# check VRRP ports
for i in $(openstack port list -f value | grep octavia-lb-vrrp| awk '{ print $1}'); do
	echo $i; openstack port show $i -c allowed_address_pairs -f value
done > /tmp/vrrp.txt
 
LB_HA_IP=$(openstack loadbalancer show ${LB_ID} -c vip_address -f value)
grep -B1 ${LB_HA_IP} /tmp/vrrp.txt