Links
https://github.com/kelseyhightower/kubernetes-the-hard-way

Configure OpenStack application credentials

mkdir -p ~/.config/openstack

cat < ~/.config/openstack/clouds.yaml
clouds:
  dev-foo:
    auth_type: "v3applicationcredential"
    auth:
      auth_url: https://keystone.service.dev.example.com/v3
      application_credential_id: "YOUR_CREDENTIAL_ID"
      application_credential_secret: "YOUR_CREDENTIAL_PASS"
EOF

Install Terraform

cat < /tmp/install-terraform.yml 
---
- hosts: localhost
  tasks:
    - name: Get latest Terraform version
      uri:
        url: https://checkpoint-api.hashicorp.com/v1/check/terraform
      register: response

    - set_fact:
        terraform_download_url: "{{ response.json.current_download_url }}"
        terraform_version: "{{ response.json.current_version }}"

    - name: Download Terraform {{ terraform_version }}
      unarchive:
        src: "{{ terraform_download_url }}terraform_{{ terraform_version }}_{{ ansible_system | lower }}_amd64.zip"
        remote_src: yes
        dest: ~/bin
        creates: ~/bin/terraform
        mode: 0550
EOF

ansible-playbook /tmp/install-terraform.yml

Create test env on OpenStack

#!/bin/bash

source /etc/kolla/admin-openrc.sh


function show_lb_owner() {
    LB_ID=$1

    # show project
    PROJECT_ID=$(openstack loadbalancer show -c project_id -f value ${LB_ID})
    PROJECT_NAME=$(openstack project show -c name -f value ${PROJECT_ID})

    # show domain
    DOMAIN_ID=$(openstack project show -c domain_id -f value ${PROJECT_ID})
    DOMAIN_NAME=$(openstack domain show -c name -f value ${DOMAIN_ID})

    echo "Domain: ${DOMAIN_NAME}"
    echo "Project: ${PROJECT_NAME}"
}


EXIT_CODE=0


# list broken LB
OUTPUT="$(openstack loadbalancer amphora list --provisioning-status ERROR)"
if [ -n "${OUTPUT}" ]; then
    echo "${OUTPUT}"

    EXIT_CODE=1
fi

# search for broken LB
for LB_ID in $(openstack loadbalancer amphora list --provisioning-status ERROR -c loadbalancer_id -f value | uniq | grep -v None); do
    # show LB details
    CMD="openstack loadbalancer show ${LB_ID}"
    echo "${CMD}"
    ${CMD}

    # show LB VMs details
    CMD="openstack loadbalancer amphora list --loadbalancer ${LB_ID}"
    echo "${CMD}"
    ${CMD}

    # show LB owner
    show_lb_owner ${LB_ID}

    echo

    EXIT_CODE=1
done

# search for single amphora VMs
for LB_ID in $(openstack loadbalancer list -c id -f value); do
    if [ $(openstack loadbalancer amphora list --loadbalancer ${LB_ID} -f value | wc -l) -ne 2 ]; then
        # openstack loadbalancer amphora list --provisioning-status ERROR
        # openstack loadbalancer amphora list --role STANDALONE

        echo "Single amphora VM:"
        CMD="openstack loadbalancer amphora list --loadbalancer ${LB_ID}"
        echo "${CMD}"
        ${CMD}

        # show LB details
        CMD="openstack loadbalancer show ${LB_ID}"
        echo "${CMD}"
        ${CMD}

        # show LB owner
        show_lb_owner ${LB_ID}

        echo

# create container
lxc launch ubuntu:20.04 osc
lxc shell osc

# install OpenStack CLI
apt install -y python3-openstackclient python3-neutron-vpnaas python3-octaviaclient python3-barbicanclient
openstack complete | sudo tee /etc/bash_completion.d/openstack
source /etc/bash_completion

# configure connection
mkdir -p ~/.config/openstack
cat < ~/.config/openstack/clouds.yaml
clouds:
  dev-foo-app:
    auth:
      auth_url: https://keystone.service.example.com/v3
      application_credential_id: "xxxxxxxx"
      application_credential_secret: "xxxxxxxx"
    region_name: "eu-fra1"
    interface: "public"
    identity_api_version: 3
    auth_type: "v3applicationcredential"
EOF

echo export OS_CLOUD=dev-foo-app >> .bashrc

# test
export OS_CLOUD=dev-foo-app
openstack image list

List

openstack vpn ipsec site connection list
openstack vpn endpoint group list
openstack vpn service list
openstack vpn ipsec policy list
openstack vpn ike policy list

Setup

# install
sudo apt-get install -y strongswan

# Left (Peer client, behind NAT)
Ubuntu Client IP: 212.8.9.10
Ubuntu net: 192.168.178.0/24

OpenStack VPN IP: 217.50.60.70
OpenStack Net: 10.0.1.0/24

Create OpenStack VPN endpoint
http://www.panticz.de/openstack/vpn-fritzbox

/etc/ipsec.conf

# Peer, e.g. FritzBox
VPN_LEFT_IP=$(curl -s ipinfo.io/ip)
VPN_LEFT_NET=$(ip -o -4 a | grep -v ": lo" | cut -d " " -f7)   # e.g 10.0.100.0/24 

# Right (OpenStack VPNaaS)
# OpenStack VPN Service IP:
# VPN_SERVICE_ID=$(openstack vpn service list -c ID -f value)
# openstack vpn service show ${VPN_SERVICE_ID} -c external_v4_ip -f value
VPN_RIGHT_IP=1.2.3.4

# OpenStack subnet netmask
# for eatch subnet
# openstack vpn ipsec site connection list -f json --long | jq -r ".[] | select(.\"VPN Service\" == \"${VPN_SERVICE_ID}\") .\"Local Endpoint Group ID\""
# openstack subnet show ${SUBNET_ID} -c cidr -f value
VPN_RIGHT_NET=10.0.1.0/24 

mv /etc/ipsec.conf /etc/ipsec.conf.org
cat < /etc/ipsec.conf
config setup

conn vpn1
 keyexchange=ikev1
 left=%defaultroute
 leftid=${VPN_LEFT_IP}
 leftsubnet=${VPN_LEFT_NET}
 leftauth=psk
 leftfirewall=yes
 authby=psk
 auto=start
 ike=aes256-sha512-modp1024
 esp=aes256-sha512
 right=${VPN_RIGHT_IP}
 rightsubnet=${VPN_RIGHT_NET}
 rightauth=psk
 ikelifetime=3600s
 keylife=3600s
 type=tunnel
EOF

/etc/ipsec.secrets

IFS=$(echo -en "\n\b")

PROJECTS_JSON=$(openstack project list --long -f json)
for PROJECT_JSON in $(echo "${PROJECTS_JSON}" | jq -c '.[]'); do
    PROJECT_ID=$(echo ${PROJECT_JSON} | jq -r .ID)
    PROJECT_NAME=$(echo ${PROJECT_JSON} | jq -r .Name)
    DOMAIN_ID=$(echo ${PROJECT_JSON} | jq -r '."Domain ID"')
    DOMAIN_JSON=$(openstack domain show  ${DOMAIN_ID} -f json)
    DOMAIN_NAME=$(echo ${DOMAIN_JSON} | jq -r .name)

    openstack server list --all-projects --long --project ${PROJECT_ID} --sort-column Name -f json | jq .[] | \
        jq --arg project_name ${PROJECT_NAME} --arg domain_name ${DOMAIN_NAME} '. + {"Project": $project_name, "Domain": $domain_name}'
done | jq --slurp .

source /etc/kolla/admin-openrc.sh

IFS=$(echo -en "\n\b")

function get_vm_details() {
    LINE=$1

    SERVER_ID=$(echo ${LINE} | cut -d" " -f3)
    SERVER_JSON=$(openstack server show ${SERVER_ID} -f json)
    SERVER_NAME=$(echo ${SERVER_JSON} | jq -r .name)
    SERVER_PROJECT_ID=$(echo ${SERVER_JSON} | jq -r .project_id)
    SERVER_PROJECT_JSON=$(openstack project show ${SERVER_PROJECT_ID} -f json)
    SERVER_PROJECT_NAME=$(echo ${SERVER_PROJECT_JSON} | jq -r .name)

    echo "${LINE} ${SERVER_NAME} ${SERVER_PROJECT_NAME}"
}

for COMPUTE_NODE in $(openstack compute service list --service nova-compute -c Host -f value | sort); do
    echo "# ${COMPUTE_NODE}"

    echo "VMs by CPU usage"
    OUTPUT="$(ssh ${COMPUTE_NODE} ps -eo pid,%cpu,cmd --sort="-%cpu" --no-headers | head -5 | grep -o -P '^[0-9]?.*(?<=-uuid ).*(?= -smbios)\b' | awk '{ print $1,$2,$NF }')"
    for LINE in ${OUTPUT}; do
        get_vm_details "${LINE}"
    done

    echo "VMs by RAM usage"
    OUTPUT="$(ssh ${COMPUTE_NODE} ps -eo pid,size,cmd --sort="-size" --no-headers | head -5 | grep -o -P '^[0-9]?.*(?<=-uuid ).*(?= -smbios)\b' | awk '{ print $1,$2,$NF }')"
    for LINE in ${OUTPUT}; do
        get_vm_details "${LINE}"
    done

    echo
done

Ping VM from IP namespace

#!/bin/bash

export OS_ENV="@globals.environment@"


if [ "${OS_ENV}" == "dev" ]; then
    export PYENV_ROOT="$HOME/.pyenv"
    export PATH="$PYENV_ROOT/bin:$PATH"
    eval "$(pyenv init -)"
fi

source /etc/kolla/admin-openrc.sh

EXIT_CODE=0

# search for broken ovs entry in DB
for NODE in $(openstack compute service list -c Host -f value | sort -u); do
    OUTPUT=$(ssh ${NODE} docker exec openvswitch_vswitchd ovsdb-client dump | grep qvo | egrep -v "tag|mac" | cut -d "\"" -f2)
    for PORT in ${OUTPUT}; do
        printf "%-20s %s\n" "${NODE}" "${PORT}"

        EXIT_CODE=1
    done
done

# Search for "No such device" OpenvSwitch ports
for NODE in $(openstack compute service list -c Host -f value | sort -u); do
    OUTPUT="$(ssh ${NODE} docker exec -i openvswitch_vswitchd ovs-vsctl show | grep "No such device")"
    if [ -n "${OUTPUT}" ]; then
        echo "${NODE}"
        echo "${OUTPUT}"

        EXIT_CODE=1
    fi
done

# search for unnecessary Open vSwitch interfaces on physical hosts
for NODE in $(openstack compute service list -c Host -f value | sort -u); do
    for PORT_TOKEN in $(ssh ${NODE} brctl show | egrep "qvb|tap" | sed '$!N;/\n.*tap/d;P;D' | awk '{print substr($1,4,8)}'); do
        PORT_ID=$(openstack port list -c id -f value | grep ${PORT_TOKEN})
        if [ -n "${PORT_ID}" ]; then
            SERVER_ID=$(openstack port show -c device_id -f value ${PORT_ID})
            SERVER_STATUS=$(openstack server show -c status -f value ${SERVER_ID})
            if [ "${SERVER_STATUS}" != "SHUTOFF" ] && [ "${SERVER_STATUS}" != "VERIFY_RESIZE" ]; then
                echo "NODE: ${NODE}"
                echo "PORT_TOKEN: ${PORT_TOKEN}"
                openstack server show -c name -c id -c project_id -c status ${SERVER_ID}

                EXIT_CODE=1
            fi
        else

#!/bin/bash

source /etc/kolla/admin-openrc.sh

EXIT_CODE=0

# search for server with status ERROR
OUTPUT="$(openstack server list --all --status=ERROR -c ID -c Name -c Status -f value)"
#openstack server show ${SERVER_ID} -c fault -f value
if [ -n "${OUTPUT}" ]; then
    echo "${OUTPUT}"

    EXIT_CODE=1
fi

# search for server with status VERIFY_RESIZE
OUTPUT="$(openstack server list --all --status=VERIFY_RESIZE -c ID -c Name -c Status -f value)"
if [ -n "${OUTPUT}" ]; then
    echo "${OUTPUT}"

    EXIT_CODE=1
fi

# search for server processes on wrong compute node
for COMPUTE_NODE in $(openstack compute service list --service nova-compute -c Host -f value); do
    for SERVER_ID in $(ssh ${COMPUTE_NODE} pgrep qemu -a | grep -o -P '(?<=-uuid ).*(?= -smbios)'); do
        VM_HOST=$(openstack server show -c "OS-EXT-SRV-ATTR:host" -f value ${SERVER_ID})
        if [ -n "${VM_HOST}" ]; then
            if [ "${VM_HOST}" != "${COMPUTE_NODE}" ]; then
                echo "VM ${SERVER_ID} on wrong compute node ${COMPUTE_NODE}"
                openstack server show ${SERVER_ID} -c OS-EXT-SRV-ATTR:hostname -c name -c id -c status -c OS-EXT-SRV-ATTR:instance_name

                EXIT_CODE=1
            fi
        else
            echo "Server process ${SERVER_ID} on ${COMPUTE_NODE} not available in OpenStack"

            EXIT_CODE=1
        fi
    done
done

# send notification
echo
exit ${EXIT_CODE}