OpenStack: Debug / cleanup DHCP

openstack subnet set --no-dhcp ${SUBNET_ID}
openstack subnet set --dhcp ${SUBNET_ID}

MAX_DHCP_NS=3
SUBNET_IDS=$(openstack subnet list --dhcp -c ID -f value)
for SUBNET_ID in ${SUBNET_IDS}; do
    NETWORK_ID=$(openstack subnet show ${SUBNET_ID} -c network_id -f value)
    DHCP_PORTS="$(openstack port list --device-owner network:dhcp --network ${NETWORK_ID} -c ID -c binding_host_id -c fixed_ips -c status -f value)"
 
    if [ $(echo "${DHCP_PORTS}" | wc -l) -ne ${MAX_DHCP_NS} ]; then
        echo "NETWORK_ID: ${NETWORK_ID}"
        echo "${DHCP_PORTS}"
 
        echo
    fi
done

NETWORK_ID=xxxxxx-xxxxxx-xxxxxx-xxxxxx-xxxxxx
CONTROL_NODE=az3-ctl3-prod
 
DHCP_AGENT_ID=$(openstack network agent list --host ${CONTROL_NODE} --agent-type dhcp -f value -c ID)
echo "DHCP_AGENT_ID: ${DHCP_AGENT_ID}"
 
# Add DHCP namespace
openstack network agent add network ${DHCP_AGENT_ID} ${NETWORK_ID} --dhcp
 
# Remove DHCP namespace
openstack network agent remove network ${DHCP_AGENT_ID} ${NETWORK_ID} --dhcp
 
# List DHCP namespaces for network
openstack port list --device-owner network:dhcp --network ${NETWORK_ID} -c id -c mac_address -c fixed_ips -c status -c binding_host_id

MAX_DHCP_NS=3
NETWORK_ID=8f397a1d-ee0e-4203-b8d9-803a244c429c
 
# DEBUG: openstack port list --network ${NETWORK_ID} --device-owner network:dhcp -c ID -c binding_host_id -c fixed_ips -c status --sort-column binding_host_id
DHCP_PORT_COUNT=$(openstack port list --network ${NETWORK_ID} --device-owner network:dhcp -f value | wc -l)
if [ ${DHCP_PORT_COUNT} -gt ${MAX_DHCP_NS} ]; then
    DHCP_AGENT_HOST=$(openstack port list --network ${NETWORK_ID} --device-owner network:dhcp -c binding_host_id -f value | sort -u | tail -1)
    echo "DHCP_AGENT_HOST: ${DHCP_AGENT_HOST}"
    # DEBUG: openstack network agent list --agent-type dhcp
    DHCP_AGENT_ID=$(openstack network agent list --agent-type dhcp --host ${DHCP_AGENT_HOST} -c ID -f value)
    echo "DHCP_AGENT_ID: ${DHCP_AGENT_ID}"
 
    # delete namespace
    openstack network agent remove network --dhcp ${DHCP_AGENT_ID} ${NETWORK_ID}
    sleep 10
 
    # delete obsolete DHCP port
    DHCP_PORT_ID=$(openstack port list --network ${NETWORK_ID} --device-owner network:dhcp -c ID -c binding_host_id -c fixed_ips -c status -f json | jq -r '.[] | select(.Status == "DOWN") .ID')
    echo "DHCP_PORT_ID: ${DHCP_PORT_ID}"
 
    openstack port delete ${DHCP_PORT_ID}
 
    # list remaining router namespaces
    openstack port list --network ${NETWORK_ID} --device-owner network:dhcp -c ID -c binding_host_id -c fixed_ips -c status
fi
 
# Debug on control node
tail -f /var/lib/docker/volumes/kolla_logs/_data/neutron/neutron-dhcp-agent.log
ip netns | grep dhcp | grep ${NETWORK_ID}

for NODE in $(openstack compute service list --service nova-scheduler -c Host -f value); do
    echo ${NODE}
    ssh ${NODE} cat /var/lib/docker/volumes/kolla_logs/_data/neutron/dnsmasq.log | grep "lease not found" | grep Sep | awk '{print $1, $2}' | uniq -c | awk '$1 > 100 {print}'
    echo
done

for NODE in $(openstack compute service list --service nova-scheduler -c Host -f value); do
    echo "NODE: ${NODE}"
 
    OUTPUT=$(ssh ${NODE} cat /var/lib/docker/volumes/kolla_logs/_data/neutron/dnsmasq.log | grep Sep | awk '/DHCPREQUEST/ {print $7}' | sort | uniq -c | awk '$1 > 100 {print $2}')
    for MAC in ${OUTPUT}; do
        echo "MAC: ${MAC}"
 
        PORT_ID=$(openstack port list --mac-address ${MAC} -c id -f value)
        echo "PORT: ${PORT_ID}"
 
        SERVER_ID=$(openstack port show ${PORT_ID} -c device_id -f value)
        echo "SERVER: ${SERVER_ID}"
 
        echo $(openstack server show ${SERVER_ID} -c addresses -c name -c id -c status -f value | paste - - - -)
 
        echo
    done
    echo
done

NETWORK_ID=xxxx-xxxx-xxxx-xxxx
 
SUBNET_ID=$(openstack port list --device-owner network:dhcp --network ${NETWORK_ID} -c fixed_ips -f json | jq -r  '. | first | ."Fixed IP Addresses" | first | .subnet_id')
echo "SUBNET_ID=${SUBNET_ID}"
 
openstack subnet set --no-dhcp ${SUBNET_ID}
sleep 10
openstack subnet set --dhcp ${SUBNET_ID}
sleep 10
openstack port list --device-owner network:dhcp --network ${NETWORK_ID} -c id -c mac_address -c fixed_ips -c status -c binding_host_id