OpenStack: Octavia LoadBalancer (LBaaS)

openstack

Releasenotes
https://docs.openstack.org/releasenotes/octavia/

CLI
https://docs.openstack.org/octavia/latest/user/guides/basic-cookbook.html
https://clouddocs.web.cern.ch/networking/load_balancing.html

openstack loadbalancer list --project ${PROJECT_ID}
openstack floating ip list --project ${PROJECT_ID} --fixed-ip-address 10.1.2.13
 
# show loadbalancer listener
openstack loadbalancer listener list --loadbalancer ${LB_NAME} -c id -f value | \
    xargs -L1 openstack loadbalancer listener show -c name -c id -c protocol -c default_tls_container_ref -c sni_container_refs
 
# get for HTTPS listener ID
openstack loadbalancer listener list --loadbalancer ${LB_NAME} -f json | jq -r '.[] | select(.protocol_port == 443) | .id'

Create Amphora image

sudo apt install -y python3-pip git qemu qemu-utils debootstrap kpartx
sudo pip3 install diskimage-builder
 
git clone https://github.com/openstack/octavia.git -b stable/stein
cd octavia
sudo ./diskimage-create/diskimage-create.sh -d bionic -t raw
chmod a+r ./amphora-x64-haproxy.raw

Upload Amphora image

#openstack image create --container-format bare --disk-format qcow2 --private --file amphora-x64-haproxy.qcow2 --tag amphora amphora
openstack image create --container-format bare --disk-format raw --public --file amphora-x64-haproxy.raw --tag amphora amphora --os-cloud=dev-admin
openstack image set --protected amphora --os-cloud=dev-admin
openstack image list --long

Configure OpenStack

OCTAVIA_KEYSTONE_PASSWORD=$(cat /etc/kolla/passwords.yml | grep octavia_keystone_password | cut -d":" -f2)
echo ${OCTAVIA_KEYSTONE_PASSWORD}
#openstack flavor create --id octavia --disk 20 --private --ram 512 --vcpus 1 octavia
openstack flavor create --disk 20 --public --ram 512 --vcpus 1 octavia
openstack --os-username octavia --os-password ${OCTAVIA_KEYSTONE_PASSWORD} keypair create --public-key ~/.ssh/id_rsa.pub octavia_ssh_key
openstack security group create --description 'used by Octavia amphora instance' octavia
openstack security group rule create --protocol icmp octavia
openstack security group rule create --protocol tcp --dst-port 5555 --egress octavia
openstack security group rule create --protocol tcp --dst-port 9443 --ingress octavia

Deploy Octavia (with kolla-ansible
https://docs.openstack.org/octavia/queens/configuration/configref.html

openstack image create amphora   --file /root/amphora-x64-haproxy.raw   --disk-format raw   --min-disk 0   --min-ram 0   --private   --protected   --tag amphora   --property os_distro=ubuntu   --property os_admin_user=ubuntu   --property os_version="18.04"
octavia_amp_boot_network_list="$(openstack network create lb-mgmt-net -f value -c id)"
octavia_subnet="$(openstack subnet create lb-mgmt-subnet1 --network lb-mgmt-net --subnet-range 172.16.0.0/12 --allocation-pool start=172.16.100.1,end=172.31.200.200 -f value -c id)"
octavia_amp_flavor_id="$(openstack flavor create --disk 20 --private --ram 4096 --vcpus 2 octavia -f value -c id)"
octavia_amp_secgroup_list="$(openstack security group create octavia -f value -c id)"
openstack security group rule create --protocol icmp $octavia_amp_secgroup_list
openstack security group rule create --protocol tcp --dst-port 5555 --egress $octavia_amp_secgroup_list
openstack security group rule create --protocol tcp --dst-port 9443 --ingress $octavia_amp_secgroup_list
openstack role add --user admin --domain Default admin
nova keypair-add --pub-key=/root/.ssh/id_rsa.pub --user $(openstack user show octavia --domain Default -f value -c id) octavia_ssh_key
echo "octavia_amp_boot_network_list: ${octavia_amp_boot_network_list}" >> /etc/kolla/globals.yml
echo "octavia_amp_secgroup_list: ${octavia_amp_secgroup_list}" >> /etc/kolla/globals.yml
echo "octavia_amp_flavor_id: ${octavia_amp_flavor_id}" >> /etc/kolla/globals.yml
# sed -i "s|^octavia_amp_boot_network_list: .*|octavia_amp_boot_network_list: ${octavia_amp_boot_network_list}|g" /etc/kolla/globals.yml
# sed -i "s|^octavia_amp_secgroup_list: .*|octavia_amp_secgroup_list: ${octavia_amp_secgroup_list}|g" /etc/kolla/globals.yml
# sed -i "s|^octavia_amp_flavor_id: .*|octavia_amp_flavor_id: ${octavia_amp_flavor_id}|g" /etc/kolla/globals.yml
cd /opt/kolla-ansible/tools
./kolla-ansible -i ../ansible/inventory/all-in-one deploy -t octavia
 
# debug
docker restart octavia_api             octavia_health_manager  octavia_housekeeping    octavia_worker

Create LoadBalancer

OS_USER=foo
LB_NAME=lb1
LB_SUBNET=${OS_USER}-subnet
LB_BACKEND_IPS="10.0.1.11 10.0.1.12"
 
 
#
# Convert certificates
#
# Convert first Let's Encrypt certificate to p12
openssl pkcs12 -export -out cert1.p12 -inkey privkey1.pem -in cert1.pem -certfile chain1.pem -passout "pass:"
openstack secret store --name=cert1 -t "application/octet-stream" -e base64 --payload="$(base64 < cert1.p12)"
 
# Convert second Let's Encrypt certificate to p12
openssl pkcs12 -export -out cert2.p12 -inkey privkey1.pem -in cert1.pem -certfile chain1.pem -passout "pass:"
openstack secret store --name=cert2 -t 'application/octet-stream' -e base64 --payload="$(base64 < cert2.p12)"
 
# DEBUG: show available certificates
openstack secret list
 
 
#
# Configure LB
#
# Create LoadBalancer
LB_PORT_ID=$(openstack loadbalancer create --name ${OS_USER}-${LB_NAME} --vip-subnet-id ${LB_SUBNET} -c vip_port_id -f value)
while ! [ "$(openstack loadbalancer show ${OS_USER}-${LB_NAME} -c provisioning_status -f value)" == "ACTIVE" ]; do sleep 1; done
 
# DEBUG: list available LB
openstack loadbalancer list
 
# Add floating IP
FLOATING_IP=$(openstack floating ip create public -c floating_ip_address -f value --floating-ip-address 10.71.95.8)
openstack floating ip set --port ${LB_PORT_ID} ${FLOATING_IP}
 
# DEBUG: show floating ip
echo ${FLOATING_IP}
 
 
#
# Configure HTTP
#
# Create HTTP listener
openstack loadbalancer listener create ${OS_USER}-${LB_NAME} \
  --name ${OS_USER}-${LB_NAME}-http-listener \
  --insert-headers X-Forwarded-For=true,X-Forwarded-Proto=true \
  --protocol HTTP \
  --protocol-port 80
 
# DEBUG: list LB listener
openstack loadbalancer listener list
 
# Create pool and add member
openstack loadbalancer pool create --name ${OS_USER}-${LB_NAME}-http-pool --lb-algorithm ROUND_ROBIN --listener ${OS_USER}-${LB_NAME}-http-listener --protocol HTTP
for LB_BACKEND_IP in ${LB_BACKEND_IPS}; do
    openstack loadbalancer member create ${OS_USER}-${LB_NAME}-http-pool --address ${LB_BACKEND_IP} --protocol-port 80
done
 
# DEBUG: list LB member
openstack loadbalancer member list ${OS_USER}-${LB_NAME}-http-pool
 
# DEBUG: test http LB
curl http://${FLOATING_IP} -I
 
 
#
# Configure HTTPS
#
# Create HTTPS listener
# CERTIFICATE_1=https://barbican.service.example.com/v1/secrets/c223df80-84fb-45de-9d47-c0a02a0c5b75
 
CERTIFICATE_1=$(openstack secret list --name cert1 -c "Secret href" -f value)
CERTIFICATE_2=$(openstack secret list --name cert2 -c "Secret href" -f value)
 
openstack loadbalancer listener create ${LB_NAME} \
  --name ${OS_USER}-${LB_NAME}-https-listener \
  --protocol-port 443 \
  --protocol TERMINATED_HTTPS  \
  --insert-headers X-Forwarded-For=true,X-Forwarded-Port=true \
  --default-tls-container=${CERTIFICATE_1} \
  --sni-container-refs ${CERTIFICATE_1} ${CERTIFICATE_2} \
  --default-pool ${LB_POOL_ID} \
 
# X-Forwarded-Proto=true
 
while ! [ "$(openstack loadbalancer listener show ${OS_USER}-https-listener -c provisioning_status -f value)" == "ACTIVE" ]; do sleep 1; done
 
# Create pool and add member
openstack loadbalancer pool create --name ${OS_USER}-${LB_NAME}-https-pool --lb-algorithm ROUND_ROBIN --listener ${OS_USER}-${LB_NAME}-https-listener --protocol HTTP
for LB_BACKEND_IP in ${LB_BACKEND_IPS}; do
    openstack loadbalancer member create ${OS_USER}-${LB_NAME}-https-pool --address ${LB_BACKEND_IP} --protocol-port 80
done

Delete LoadBalancer (recursive)

openstack loadbalancer delete --cascade LB_NAME

Statistics

openstack loadbalancer stats show ${LB_ID}
openstack loadbalancer listener stats show ${LB_LISTENER_ID}
 
#
# Debug
#
<code>
openstack secret list
openstack loadbalancer list
openstack loadbalancer listener list --loadbalancer lb1
openstack loadbalancer member list http-pool
openstack loadbalancer amphora list
 
openstack loadbalancer amphora list --loadbalancer 0ce30f0e-1d75-486c-a09f-79125abf44b8
+--------------------------------------+--------------------------------------+-----------+--------+---------------+-------------+
| id                                   | loadbalancer_id                      | status    | role   | lb_network_ip | ha_ip       |
+--------------------------------------+--------------------------------------+-----------+--------+---------------+-------------+
| daee2f88-01fd-4ffa-b80d-15c63771d99d | 0ce30f0e-1d75-486c-a09f-79125abf44b8 | ERROR     | BACKUP | 172.16.100.30 | 172.28.0.26 |
| f22186b1-2865-4f4a-aae2-7f869b7aae12 | 0ce30f0e-1d75-486c-a09f-79125abf44b8 | ALLOCATED | MASTER | 172.16.100.5  | 172.28.0.26 |
+--------------------------------------+--------------------------------------+-----------+--------+---------------+-------------+
openstack server show amphora-${AMPHORA_ID} -c id -f value
ssh compute1 ps axf | grep dce0e225-9f23-46a2-b2e0-6e027ddfd6d0
 
# show updated LB
openstack loadbalancer list --provisioning-status PENDING_UPDATE -c id -f value | xargs -L1 openstack loadbalancer show -c updated_at -c provisioning_status -c id -c name

Enable / Disable LB

openstack loadbalancer set --disable  2fc1a9be-a9e4-4288-9464-cf486d266483
openstack loadbalancer set --enable  2fc1a9be-a9e4-4288-9464-cf486d266483

Find broken LB

# List broken LoadBalancer instances
openstack loadbalancer list --provisioning-status ERROR
openstack loadbalancer list --provisioning-status PENDING_UPDATE
 
# List broken Loadbalancer VMs
openstack loadbalancer amphora list --provisioning-status ERROR
openstack loadbalancer amphora list --role STANDALONE

Debug

# List LB VMs (admin only)
openstack loadbalancer amphora list --os-cloud=dev-admin --loadbalancer foo-lb1
 
# Show LB VM details (admin only)
openstack server show amphora-<AMPHORA_ID> --os-cloud=dev-admin
 
# Show LB port IPs
LB_PORT_IDS=$(openstack loadbalancer amphora list --loadbalancer pko-lb4 -c id -f value)
for LB_PORT_ID in ${LB_PORT_IDS}; do
    openstack port list | grep octavia-lb-vrrp-${LB_PORT_ID}
done
 
# get broken / single VM LB
# find single ampora VM
for LOADBALANCER_ID in $(openstack loadbalancer list -c id -f value); do
    if [ $(openstack loadbalancer amphora list --loadbalancer ${LOADBALANCER_ID} -f value | wc -l) -lt 2 ]; then
        openstack loadbalancer show ${LOADBALANCER_ID}
    fi
done

API

curl -s -H "X-Auth-Token: $(openstack token issue -c id -f value)" -X GET http://octavia.service.i.example.com:9876/v2/lbaas/loadbalancers | jq

Generate certificate
https://github.com/openstack/octavia.git

git clone https://github.com/openstack/octavia.git -b stable/stein
sed -i "s#foobar#\$(grep octavia_ca_password /etc/kolla/passwords.yml | awk '{print \$2}')#g" octavia/bin/create_certificates.sh
 
/opt/octavia/bin/create_certificates.sh /etc/kolla/config/foo/octavia /etc/kolla/config/foo/octavia/openssl.cnf

Test: create certificate
https://docs.openstack.org/de/api-quick-start/api-quick-start.html

CERT_1=$(openstack secret store -c "Secret href" -f value --name=cert_1 -t "application/octet-stream" -e base64 --payload="$(base64 < cert.pem)")
PRIV_1=$(openstack secret store -c "Secret href" -f value --name=privkey_1 -t "application/octet-stream" -e base64 --payload="$(base64 < privkey.pem)")
 
curl -X POST -H "X-Auth-Token: ${TOKEN}" -H "Content-Type:application/json" -d "{
    \"type\": \"certificate\",
    \"name\": \"certificate container 1\",
    \"secret_refs\": [
        {
            \"name\": \"certificate\",
            \"secret_ref\": \"${CERT_1}\"
        },
        {
            \"name\": \"private_key\",
            \"secret_ref\": \"${PRIV_1}\"
        }
    ]
}" https://barbican.service.stage.example.com/v1/containers

Log
https://docs.openstack.org/octavia/latest/admin/log-offloading.html
http://www.panticz.de/haproxy

# HAProxy configuration files
sed -i 's/notice/info/g' /var/lib/octavia/*/haproxy.cfg
 
vi /var/lib/octavia/*/haproxy.cfg
...
defaults
   log global
   mode tcp
   option tcplog
   option logasap
...
 
# check HAproxy configuration
ls /var/lib/octavia/*/haproxy.cfg | xargs -L1 haproxy -c -f
 
# restart HAProxy
#pgrep -a haproxy
#/usr/sbin/haproxy -Ws -f /var/lib/octavia/12d13d89-d8e5-4c1e-bb9f-f7b8bb700a67/haproxy.cfg -f /var/lib/octavia/haproxy-default-user-group.conf -p /var/lib/octavia/12d13d89-d8e5-4c1e-bb9f-f7b8bb700a67/12d13d89-d8e5-4c1e-bb9f-f7b8bb700a67.pid -L nPBdf6NS4c5Tq3WsOcB8Lc9aFjA -sf $(cat /var/lib/octavia/12d13d89-d8e5-4c1e-bb9f-f7b8bb700a67/12d13d89-d8e5-4c1e-bb9f-f7b8bb700a67.pid)
killall haproxy
 
# watch logfile
tail -f /var/log/haproxy.log

Allow user / octavia access to certificate
https://docs.mirantis.com/mcp/q4-18/mcp-deployment-guide/advanced-config/configure-octavia/example-lb-topology-tls.html

openstack acl user add -u ${USER_ID} https://barbican.service.example.com/v1/secrets/1111111-2222-3333-4444-5555555555555

Update secred / certificate

# set certificate (as admin)
openstack loadbalancer listener set --default-tls-container-ref=https://barbican.service.example.com/v1/secrets/11111111-2222-3333-4444-5555555555 
 ${LOADBALANCER_LISTENER_ID}
 
openstack loadbalancer listener show ${LOADBALANCER_LISTENER_ID}

Update certificate

openstack loadbalancer listener list --loadbalancer ${LB_NAME} -f json | jq -r '.[] | select(.protocol_port == 443) | .id'
 
# broken?
https://storyboard.openstack.org/#!/story/2001971
openstack loadbalancer listener set --sni-container-refs https://barbican.service.example.com/v1/secrets/999a68f0-c55e-48bc-9707-520e89a22a66 https://barbican.service.example.com/v1/secrets/7837480b-0f7e-4750-8331-2f5809ccd90b -- 53613fb4-d048-49e2-8d86-4d4f057ad911

Fake LB ping (ping only floating IP in the namespace)

LB_ID=foo-lb01-prod
 
VIP_ADDRESS=$(openstack loadbalancer show ${LB_ID} -c vip_address -f value)
FLOATING_IP_PORT=$(openstack floating ip list --fixed-ip-address ${VIP_ADDRESS} -c Port -f value)
SECURITY_GROUP_ID=$(openstack port show ${FLOATING_IP_PORT} -c security_group_ids -f value)
 
# DEBUG: show existing ingress ICMP rules
openstack security group rule list --ingress --protocol icmp ${SECURITY_GROUP_ID}
 
# allow incomming ICMP traffic 
openstack security group rule create --remote-ip 0.0.0.0/0 --protocol icmp --ingress ${SECURITY_GROUP_ID}

Fix deletet certificate / secret
https://bugs.launchpad.net/octavia/+bug/1852599

DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS}
 
select * from listener where load_balancer_id  = 'b41c7bfb-0411-48c9-b133-6ee95c4dc20e' \G
update listener set tls_certificate_id='' where load_balancer_id = 'b41c7bfb-0411-48c9-b133-6ee95c4dc20e';
delete from sni where listener_id = '894abf55-2257-469f-b102-987a90342abe';
update listener set tls_certificate_id='https://barbican.service.example.com/v1/secrets/68c2c456-eb17-43f5-a99e-57dc099046c5' where load_balancer_id = 'b41c7bfb-0411-48c9-b133-6ee95c4dc20e';

TCP listener

# tcp 80 listener
openstack loadbalancer listener create --name foo-lb1-tcp-80 --protocol TCP --protocol-port 80 foo-lb1
openstack loadbalancer pool create --name foo-lb1-tcp-80-pool --lb-algorithm ROUND_ROBIN --listener foo-lb1-tcp-80 --protocol TCP
#openstack loadbalancer healthmonitor create --delay 5 --max-retries 4 --timeout 10 --type TCP foo-lb1-tcp-80-pool
openstack loadbalancer member create --subnet-id foo-subnet --address 10.0.1.13 --protocol-port 80 foo-lb1-tcp-80-pool
 
# tcp 443 listener
openstack loadbalancer listener create --name foo-lb1-tcp-443 --protocol TCP --protocol-port 443 foo-lb1
openstack loadbalancer pool create --name foo-lb1-tcp-443-pool --lb-algorithm ROUND_ROBIN --listener foo-lb1-tcp-443 --protocol TCP
#openstack loadbalancer healthmonitor create --delay 5 --max-retries 4 --timeout 10 --type TCP foo-lb1-tcp-443-pool
openstack loadbalancer member create --subnet-id foo-subnet --address 10.0.1.13 --protocol-port 443 foo-lb1-tcp-443-pool

Assign flating IP

LB_PORT_ID=$(openstack loadbalancer show pako-lb1 -c vip_port_id -f value)
openstack floating ip set --port ${LB_PORT_ID} 8.9.10.11

Upgrade
https://docs.openstack.org/octavia/latest/admin/guides/upgrade.html

Octavia Policies
https://docs.openstack.org/octavia/latest/configuration/policy.html

Migrate all Octavia Amphora VM out from specific compute node

# openstack server list --all-projects --host az1-com4-prod
 
AMPHORA_ID=amphora-xxxxx-xxxx-xxxx-xxxx-xxxxx
 
AMPHORA_ID=${AMPHORA_ID#*-}
openstack loadbalancer amphora list | grep ${AMPHORA_ID}
 
LB_ID=$(openstack loadbalancer amphora list | grep ${AMPHORA_ID} | cut -d "|" -f3)
openstack loadbalancer show ${LB_ID}
openstack loadbalancer amphora list --loadbalancer ${LB_ID}
 
echo "Failover Amphora ${AMPHORA_ID}"
openstack loadbalancer amphora failover ${AMPHORA_ID}
 
sleep 60
openstack loadbalancer amphora list --loadbalancer ${LB_ID}
openstack loadbalancer show ${LB_ID}

Failover all / ERROR Octavia LoadBalancer (Amphora image update)

LB_IDS=$(openstack loadbalancer amphora list --status ERROR -c loadbalancer_id -f value | sort -u)
for LB_ID in ${LB_IDS}; do
    echo "Failover LoadBalancer: ${LB_ID}"
    openstack loadbalancer failover ${LB_ID}
    while [[ $(openstack loadbalancer amphora list --loadbalancer ${LB_ID} -f value -c status | grep ALLOCATED | wc -l) < 2 ]]; do sleep 1; done
    sleep 10
    openstack loadbalancer amphora list --loadbalancer ${LB_ID}    
done
 
# List amphora VMs
openstack server list --project service

Recreate Octavia port

openstack port create \
  --network 45bb34d3-70e2-43ea-aff7-dfd7750ce68a \
  --fixed-ip 'ip-address=192.168.38.221,subnet=68f2f3ca-9fab-4ac1-9b46-e8c9a078ac66' \
  --device lb-84cd79db-9bd1-4490-9d3f-acdf668f00ef \
  --device-owner Octavia \
  --security-group lb-1dbacdba-ccf3-4319-b15a-296fcdaadce3 \
  --project a772e4ab888e4f039b3430d688f4559d \
  --description my-lb01-dev \
  octavia-lb-1dbacdba-ccf3-4319-b15a-296fcdaadce3

Links
https://releases.openstack.org/teams/octavia.html#team-rocky-octavia
https://docs.openstack.org/python-octaviaclient/latest/cli/index.html
https://docs.openstack.org/mitaka/networking-guide/config-lbaas.html
https://docs.openstack.org/octavia/latest/user/guides/basic-cookbook.html
https://shreddedbacon.com/post/openstack-kolla/
https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/13/html/networking_guide/sec-octavia
https://docs.openstack.org/octavia/latest/user/feature-classification/index.html#operation_insert_headers_X_Forwarded_For
https://github.com/openstack/octavia/blob/master/api-ref/source/v2/listener.inc
http://www.loadbalancer.org/blog/nginx-and-x-forwarded-for-header/
https://dataops.ga/2018/12/octavia/
https://docs.openstack.org/octavia/latest/admin/guides/operator-maintenance.html - rotate Amphora images
https://docs.openstack.org/releasenotes/octavia/rocky.html - available versions