OpenStack: Octavia LoadBalancer (LBaaS)

CLI

openstack loadbalancer list --project ${PROJECT_ID}
openstack floating ip list --project ${PROJECT_ID} --fixed-ip-address 10.1.2.13

Create Amphora image

sudo apt install -y python-pip git qemu qemu-utils debootstrap kpartx
sudo pip install diskimage-builder
git clone https://review.openstack.org/p/openstack/octavia
cd octavia
sudo ./diskimage-create/diskimage-create.sh -d bionic -t raw
chmod a+r ./amphora-x64-haproxy.raw

Upload Amphora image

#openstack image create --container-format bare --disk-format qcow2 --private --file amphora-x64-haproxy.qcow2 --tag amphora amphora
openstack image create --container-format bare --disk-format raw --public --file amphora-x64-haproxy.raw --tag amphora amphora --os-cloud=dev-admin
openstack image set --protected amphora --os-cloud=dev-admin
openstack image list --long

Configure OpenStack

OCTAVIA_KEYSTONE_PASSWORD=$(cat /etc/kolla/passwords.yml | grep octavia_keystone_password | cut -d":" -f2)
echo ${OCTAVIA_KEYSTONE_PASSWORD}
#openstack flavor create --id octavia --disk 20 --private --ram 512 --vcpus 1 octavia --os-cloud=dev-admin
openstack flavor create --disk 20 --public --ram 512 --vcpus 1 octavia --os-cloud=dev-admin
openstack --os-username octavia --os-password ${OCTAVIA_KEYSTONE_PASSWORD} keypair create --public-key ~/.ssh/id_rsa.pub octavia_ssh_key --os-cloud=dev-admin
openstack security group create --description 'used by Octavia amphora instance' octavia --os-cloud=dev-admin
openstack security group rule create --protocol icmp octavia --os-cloud=dev-admin
openstack security group rule create --protocol tcp --dst-port 5555 --egress octavia --os-cloud=dev-admin
openstack security group rule create --protocol tcp --dst-port 9443 --ingress octavia --os-cloud=dev-admin

Deploy Octavia (with kolla-ansible

openstack image create amphora   --file /root/amphora-x64-haproxy.raw   --disk-format raw   --min-disk 0   --min-ram 0   --private   --protected   --tag amphora   --property os_distro=ubuntu   --property os_admin_user=ubuntu   --property os_version="18.04"
octavia_amp_boot_network_list="$(openstack network create lb-mgmt-net -f value -c id)"
octavia_subnet="$(openstack subnet create lb-mgmt-subnet1 --network lb-mgmt-net --subnet-range 172.16.0.0/12 --allocation-pool start=172.16.100.1,end=172.31.200.200 -f value -c id)"
octavia_amp_flavor_id="$(openstack flavor create --disk 20 --private --ram 4096 --vcpus 2 octavia -f value -c id)"
octavia_amp_secgroup_list="$(openstack security group create octavia -f value -c id)"
openstack security group rule create --protocol icmp $octavia_amp_secgroup_list
openstack security group rule create --protocol tcp --dst-port 5555 --egress $octavia_amp_secgroup_list
openstack security group rule create --protocol tcp --dst-port 9443 --ingress $octavia_amp_secgroup_list
openstack role add --user admin --domain Default admin
nova keypair-add --pub-key=/root/.ssh/id_rsa.pub --user $(openstack user show octavia --domain Default -f value -c id) octavia_ssh_key
echo "octavia_amp_boot_network_list: ${octavia_amp_boot_network_list}" >> /etc/kolla/globals.yml
echo "octavia_amp_secgroup_list: ${octavia_amp_secgroup_list}" >> /etc/kolla/globals.yml
echo "octavia_amp_flavor_id: ${octavia_amp_flavor_id}" >> /etc/kolla/globals.yml
# sed -i "s|^octavia_amp_boot_network_list: .*|octavia_amp_boot_network_list: ${octavia_amp_boot_network_list}|g" /etc/kolla/globals.yml
# sed -i "s|^octavia_amp_secgroup_list: .*|octavia_amp_secgroup_list: ${octavia_amp_secgroup_list}|g" /etc/kolla/globals.yml
# sed -i "s|^octavia_amp_flavor_id: .*|octavia_amp_flavor_id: ${octavia_amp_flavor_id}|g" /etc/kolla/globals.yml
cd /opt/kolla-ansible/tools
./kolla-ansible -i ../ansible/inventory/all-in-one deploy -t octavia
 
# debug
docker restart octavia_api             octavia_health_manager  octavia_housekeeping    octavia_worker

Create LoadBalancer

OS_USER=foo
LB_NAME=lb1
LB_SUBNET=${OS_USER}-subnet
LB_BACKEND_IPS="10.0.1.11 10.0.1.12"
 
 
#
# Convert certificates
# (EW wildcard certificates are available under: ewos1-prov2-prod.i.ewcs.ch:/root/certificates/)
#
# Convert first Let's Encrypt certificate to p12
openssl pkcs12 -export -out cert1.p12 -inkey privkey1.pem -in cert1.pem -certfile chain1.pem -passout "pass:"
openstack secret store --name=cert1 -t "application/octet-stream" -e base64 --payload="$(base64 < cert1.p12)"
 
# Convert second Let's Encrypt certificate to p12
openssl pkcs12 -export -out cert2.p12 -inkey privkey1.pem -in cert1.pem -certfile chain1.pem -passout "pass:"
openstack secret store --name=cert2 -t 'application/octet-stream' -e base64 --payload="$(base64 < cert2.p12)"
 
# DEBUG: show available certificates
openstack secret list
 
 
#
# Configure LB
#
# Create LoadBalancer
LB_PORT_ID=$(openstack loadbalancer create --name ${OS_USER}-${LB_NAME} --vip-subnet-id ${LB_SUBNET} -c vip_port_id -f value)
while ! [ "$(openstack loadbalancer show ${OS_USER}-${LB_NAME} -c provisioning_status -f value)" == "ACTIVE" ]; do sleep 1; done
 
# DEBUG: list available LB
openstack loadbalancer list
 
# Add floating IP
# (217.20.192.8 is already configured as https://lb1.dev.ewcs.ch / https://lb1.stage.ewcs.ch in DNS)
FLOATING_IP=$(openstack floating ip create public -c floating_ip_address -f value --floating-ip-address 217.71.95.8)
openstack floating ip set --port ${LB_PORT_ID} ${FLOATING_IP}
 
# DEBUG: show floating ip
echo ${FLOATING_IP}
 
 
#
# Configure HTTP
#
# Create HTTP listener
openstack loadbalancer listener create ${OS_USER}-${LB_NAME} \
  --name ${OS_USER}-${LB_NAME}-http-listener \
  --insert-headers X-Forwarded-For=true,X-Forwarded-Proto=true \
  --protocol HTTP \
  --protocol-port 80
 
# DEBUG: list LB listener
openstack loadbalancer listener list
 
# Create pool and add member
openstack loadbalancer pool create --name ${OS_USER}-${LB_NAME}-http-pool --lb-algorithm ROUND_ROBIN --listener ${OS_USER}-${LB_NAME}-http-listener --protocol HTTP
for LB_BACKEND_IP in ${LB_BACKEND_IPS}; do
    openstack loadbalancer member create ${OS_USER}-${LB_NAME}-http-pool --address ${LB_BACKEND_IP} --protocol-port 80
done
 
# DEBUG: list LB member
openstack loadbalancer member list ${OS_USER}-${LB_NAME}-http-pool
 
# DEBUG: test http LB
curl http://${FLOATING_IP} -I
 
 
#
# Configure HTTPS
#
# Create HTTPS listener
CERTIFICATE_1=$(openstack secret list --name cert1 -c "Secret href" -f value)
CERTIFICATE_2=$(openstack secret list --name cert2 -c "Secret href" -f value)
openstack loadbalancer listener create ${LB_NAME} \
  --name ${OS_USER}-${LB_NAME}-https-listener \
  --protocol-port 443 \
  --protocol TERMINATED_HTTPS  \
  --insert-headers X-Forwarded-For=true,X-Forwarded-Proto=true \
  --default-tls-container=${CERTIFICATE_1} \
  --sni-container-refs ${CERTIFICATE_1} ${CERTIFICATE_2}
while ! [ "$(openstack loadbalancer listener show ${OS_USER}-https-listener -c provisioning_status -f value)" == "ACTIVE" ]; do sleep 1; done
 
# Create pool and add member
openstack loadbalancer pool create --name ${OS_USER}-${LB_NAME}-https-pool --lb-algorithm ROUND_ROBIN --listener ${OS_USER}-${LB_NAME}-https-listener --protocol HTTP
for LB_BACKEND_IP in ${LB_BACKEND_IPS}; do
    openstack loadbalancer member create ${OS_USER}-${LB_NAME}-https-pool --address ${LB_BACKEND_IP} --protocol-port 80
done

Delete LoadBalancer (recursive)

openstack loadbalancer delete --cascade LB_NAME

Statistics

openstack loadbalancer stats show 900d0e4a-7cfd-4dc0-8c72-2a4116917502
 
#
# Debug
#
openstack secret list
openstack loadbalancer list
openstack loadbalancer listener list --loadbalancer lb1
openstack loadbalancer member list http-pool
openstack loadbalancer amphora list
 
openstack loadbalancer amphora list --loadbalancer 0ce30f0e-1d75-486c-a09f-79125abf44b8
+--------------------------------------+--------------------------------------+-----------+--------+---------------+-------------+
| id                                   | loadbalancer_id                      | status    | role   | lb_network_ip | ha_ip       |
+--------------------------------------+--------------------------------------+-----------+--------+---------------+-------------+
| daee2f88-01fd-4ffa-b80d-15c63771d99d | 0ce30f0e-1d75-486c-a09f-79125abf44b8 | ERROR     | BACKUP | 172.16.100.30 | 172.28.0.26 |
| f22186b1-2865-4f4a-aae2-7f869b7aae12 | 0ce30f0e-1d75-486c-a09f-79125abf44b8 | ALLOCATED | MASTER | 172.16.100.5  | 172.28.0.26 |
+--------------------------------------+--------------------------------------+-----------+--------+---------------+-------------+
openstack server show amphora-${AMPHORA_ID} -c id -f value
ssh compute1 ps axf | grep dce0e225-9f23-46a2-b2e0-6e027ddfd6d0

Enable / Disable LB

openstack loadbalancer set --disable  2fc1a9be-a9e4-4288-9464-cf486d266483
openstack loadbalancer set --enable  2fc1a9be-a9e4-4288-9464-cf486d266483

Find broken LB

# List broken LoadBalancer instances
openstack loadbalancer list --provisioning-status ERROR
openstack loadbalancer list --provisioning-status PENDING_UPDATE
 
# List broken Loadbalancer VMs
openstack loadbalancer amphora list --provisioning-status ERROR
openstack loadbalancer amphora list --role STANDALONE

Debug

# List LB VMs (admin only)
openstack loadbalancer amphora list --os-cloud=dev-admin --loadbalancer foo-lb1
 
# Show LB VM details (admin only)
openstack server show amphora-<AMPHORA_ID> --os-cloud=dev-admin
 
# Show LB port IPs
LB_PORT_IDS=$(openstack loadbalancer amphora list --loadbalancer pko-lb4 -c id -f value)
for LB_PORT_ID in ${LB_PORT_IDS}; do
    openstack port list | grep octavia-lb-vrrp-${LB_PORT_ID}
done
 
# get broken / single VM LB
# find single ampora VM
for LOADBALANCER_ID in $(openstack loadbalancer list -c id -f value); do
    if [ $(openstack loadbalancer amphora list --loadbalancer ${LOADBALANCER_ID} -f value | wc -l) -lt 2 ]; then
        openstack loadbalancer show ${LOADBALANCER_ID}
    fi
done

API

curl -s -H "X-Auth-Token: $(openstack token issue -c id -f value)" -X GET http://octavia.service.i.example.com:9876/v2/lbaas/loadbalancers | jq

Test: create certificate
https://docs.openstack.org/de/api-quick-start/api-quick-start.html

CERT_1=$(openstack secret store -c "Secret href" -f value --name=cert_1 -t "application/octet-stream" -e base64 --payload="$(base64 < cert.pem)")
PRIV_1=$(openstack secret store -c "Secret href" -f value --name=privkey_1 -t "application/octet-stream" -e base64 --payload="$(base64 < privkey.pem)")
 
curl -X POST -H "X-Auth-Token: ${TOKEN}" -H "Content-Type:application/json" -d "{
    \"type\": \"certificate\",
    \"name\": \"certificate container 1\",
    \"secret_refs\": [
        {
            \"name\": \"certificate\",
            \"secret_ref\": \"${CERT_1}\"
        },
        {
            \"name\": \"private_key\",
            \"secret_ref\": \"${PRIV_1}\"
        }
    ]
}" https://barbican.service.stage.example.com/v1/containers

Log
https://docs.openstack.org/octavia/latest/admin/log-offloading.html

Recreate OVS octavia port

PORT_ID=1313ff21-3333-1223-3334-4daf969c1234
CTL_HOST_MAC=00:11:22:00:00:04
 
docker exec openvswitch_vswitchd ovs-vsctl -- --may-exist add-port br-int o-hm0 -- \
    set Interface o-hm0 type=internal -- \
    set Interface o-hm0 mac=\"${CTL_HOST_MAC}\" -- \
    set Interface o-hm0 external-ids:iface-status=active -- \
    set Interface o-hm0 external-ids:iface-id=${PORT_ID} -- \
    set Interface o-hm0 external-ids:skip_cleanup=true -- \
    set Interface o-hm0 external-ids:attached-mac=\"${CTL_HOST_MAC}\"

Links
https://releases.openstack.org/teams/octavia.html#team-rocky-octavia
https://docs.openstack.org/python-octaviaclient/latest/cli/index.html
https://docs.openstack.org/mitaka/networking-guide/config-lbaas.html
https://docs.openstack.org/octavia/latest/user/guides/basic-cookbook.html
https://shreddedbacon.com/post/openstack-kolla/
https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/13/html/networking_guide/sec-octavia
https://docs.openstack.org/octavia/latest/user/feature-classification/index.html#operation_insert_headers_X_Forwarded_For
https://github.com/openstack/octavia/blob/master/api-ref/source/v2/listener.inc
http://www.loadbalancer.org/blog/nginx-and-x-forwarded-for-header/
https://dataops.ga/2018/12/octavia/
https://docs.openstack.org/octavia/latest/admin/guides/operator-maintenance.html - rotate Amphora images
https://docs.openstack.org/releasenotes/octavia/rocky.html - available versions