Upload new image(s)

IMGDIR=/var/www/images

# Windows Server 2016 Standard
openstack image create "Windows Server 2016 Standard (NEW)" \
    --file ${IMGDIR}/windows-server-2016-standard.raw \
    --disk-format raw \
    --min-disk 20 \
    --min-ram 2048 \
    --community \
    --property hw_scsi_model=virtio-scsi \
    --property hw_disk_bus=scsi \
    --property hw_qemu_guest_agent=yes \
    --property os_distro=windows \
    --property os_admin_user=Administrator \
    --property os_version="2016" \
    --property os_build="14393.3384"
#    --protected \

openstack image set --protected "Windows Server 2016 Standard"


# Windows Server 2019 Standard
openstack image create "Windows Server 2019 Standard (NEW)" \
    --file $IMGDIR/windows-server-2019-standard.raw \
    --disk-format raw \
    --min-disk 20 \
    --min-ram 2048 \
    --community \
    --property hw_scsi_model=virtio-scsi \
    --property hw_disk_bus=scsi \
    --property hw_qemu_guest_agent=yes \
    --property os_distro=windows \
    --property os_admin_user=Administrator \
    --property os_version="2019" \
    --property os_build="17763.914"
#    --protected \

openstack image set --protected "Windows Server 2019 Standard"

Show community images (unprivileged user

Install
https://www.elastic.co/guide/en/elasticsearch/client/curator/5.x/apt-repository.html

wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb [arch=amd64] https://packages.elastic.co/curator/5/debian stable main" > /etc/apt/sources.list.d/curator.list
sudo apt-get update && sudo apt-get install elasticsearch-curator

# sudo apt install -y elasticsearch-curator
# pip install elasticsearch-curator

Show usage

curl elasticsearch.example.com:9200/_cat/indices

curator.yml

---
client:
  hosts:
    - elasticsearch.example.com
  port: 9200
  url_prefix:
  use_ssl: False
  certificate:
  client_cert:
  client_key:
  ssl_no_validate: False
  http_auth:
  timeout: 30
  master_only: False

logging:
  loglevel: INFO
  logfile:
  logformat: default
  blacklist: ['elasticsearch', 'urllib3']

delete_indices.yml

---
actions:
  1:
    action: delete_indices
    description: >-
      Delete indices older than 14 days
    options:
      ignore_empty_list: True
      timeout_override:
      continue_if_exception: False
      disable_action: False
    filters:
#    - filtertype: pattern
#      kind: prefix
#      value: logstash-
#      #value: flog-
#      exclude:
#    - filtertype: pattern
#      kind: prefix
#      value: flog-
#      exclude:
    - filtertype: age
      source: name
      direction: older
      timestring: '%Y.%m.%d'
      unit: days
      unit_count: 7
      exclude:

Delete old data

curator --config ./curator.yml ./delete_indices.yml #--dry-run

Links
http://www.madhur.co.in/blog/2017/04/09/usingcuratordeleteelasticindex.html
https://cezarypiatek.github.io/post/scheduled-elasticsearch-cleanup/

Configure kolla-ansible

#echo 'designate_tag: "7.0.1.2"' >> /etc/kolla/globals.yml
sed -i 's/[#]dns_interface:/dns_interface:/g' /etc/kolla/globals.yml
sed -i 's/[#]enable_designate: .*/enable_designate: "yes"/g' /etc/kolla/globals.yml
sed -i 's/[#]enable_horizon_designate:/enable_horizon_designate:/g' /etc/kolla/globals.yml
sed -i 's/[#]designate_ns_record: .*/designate_ns_record: "pool.{{ os_environment }}.example.com"/g' /etc/kolla/globals.yml

Configure pool

mkdir -p /etc/kolla/config/foo/designate

/etc/kolla/config/foo/designate/pools.yaml
- name: default-bind
  description: Default BIND9 Pool
  attributes: {}
  ns_records:
    - hostname: ns1.dev.i.example.com.
      priority: 1
  nameservers:
    - host: 10.0.4.45
      port: 53
  targets:
    - type: bind9
      description: BIND9 Server 1
      masters:
        - host: 10.0.4.135
          port: 5354
      options:
        host: 10.0.4.45
        port: 53
        rndc_host: 10.0.4.45
        rndc_port: 953
        rndc_key_file: /etc/designate/rndc.key

Deploy Designate container

kolla-ansible -i /etc/kolla/inventory deploy -t designate

# Test endpoint
curl designate.service.${ENV}.i.example.com:9001

Configure neutron

grep designate_keystone_password /etc/kolla/passwords.yml 

vi /etc/kolla/config/ew/neutron.conf
[DEFAULT]
...
external_dns_driver = designate
dns_domain = pool.stage.example.com.

[designate]
url = http://designate.service.stage.i.example.com:9001/v2
auth_type = password
auth_url = http://keystone-admin.service.stage.i.example.com:35357
username = designate
password = xxxxxxxxxxxxxxxxxxx
project_name = service
project_domain_name = default
user_domain_name = default
allow_reverse_dns_lookup = True
ipv4_ptr_zone_prefix_size = 24
ipv6_ptr_zone_prefix_size = 116
ptr_zone_email = foo@example.com

kolla-ansible -i /etc/kolla/inventory reconfigure -t neutron

Install designate client

Get LEGO binary
https://github.com/go-acme/lego/releases/

LEGO_VERSION=$(curl -s https://api.github.com/repos/go-acme/lego/releases/latest | grep '"tag_name"' | cut -d'"' -f4)
curl -L "https://github.com/go-acme/lego/releases/download/${LEGO_VERSION}/lego_${LEGO_VERSION}_linux_amd64.tar.gz" | tar -xz
sudo mv lego /usr/local/sbin
sudo chown root:root /usr/local/sbin
sudo chmod 644 /usr/local/sbin

Show certificates

ll ~/.lego/certificates/

OpenStack Designate

# test with designage
/tmp/lego --accept-tos --email="foo@bar.com" --domains="*.bar.com" --dns designate run
... designate: some credentials information are missing: OS_AUTH_URL,OS_USERNAME,OS_PASSWORD,OS_TENANT_NAME,OS_REGION_NAME

# stop service on port 80 / 443
service nginx stop
service apache2 stop

# Request certificate
/tmp/lego --accept-tos --email="foo@bar.com" --domains="bar.com" --http run

# Request wildcard certificate
/tmp/lego --accept-tos --email="foo@bar.com" --domains="bar.com" --domains="*.bar.com" --dns manual run

# restart service on port 80 / 443
service nginx start
service apache2 start

Request wildcart certificate

./lego \
    --server https://acme-v02.api.letsencrypt.org/directory \
    --accept-tos=true \
    --dns.resolvers="ns1.example.com:53" \
    --dns.resolvers="ns2.example.com:53" \
    --email="info@example.com" \
    --dns=nictool \
    --key-type=rsa2048 \
    --pem \
    --domains="*.${DOMAIN}" \
    run

Renew certificates
https://go-acme.github.io/lego/usage/cli/examples/

Configure default storage

lxc storage create zfs zfs source=rpool/lxd
lxc profile device add default root disk path=/ pool=zfs

lxc storage list
lxc storage delete default

# zfs
lxc storage create zfs zfs source=tank/lxd
lxc storage list

# delete default storage
lxc storage volume list default
lxc storage volume delete default image/7d788819a5a97433db8470ee68370ec69e829b429800fa28b5524f0411490ce9
lxc storage delete default

# move container to another storage
CONTAINER=www1
lxc move ${CONTAINER} ${CONTAINER}-tmp -s nvme
lxc move ${CONTAINER}-tmp ${CONTAINER}
lxc start ${CONTAINER}

Configure ZFS storage

lxc profile device del dev-zfs root
lxc profile device add dev-zfs root disk path=/ pool=zfs

Storage

# change container storage quota
lxc config device set  root size 100GB

# lvm thin pool
lvcreate -L 250G --thinpool kvm system
lxc storage create kvm lvm source=system lvm.thinpool_name=kvm

# unix-block
lxc config device add c1 xvdb1 unix-block source=/dev/xvdb1 required=false
lxc config device remove gitlab-runner3-dev xvdb2

Container root

/var/lib/lxd/storage-pools/default/containers/*/rootfs/

Links
https://github.com/lxc/lxd/blob/master/doc/storage.md
https://linuxcontainers.org/lxd/docs/master/storage

Optinal: Create nested LXD container
http://www.panticz.de/lxd/nesting

CONTAINER_NAME=gitlab-runner1-dev
lxc launch ubuntu:18.04 ${CONTAINER_NAME} -p disk-zfs -p nic-dev-mgmt -c boot.autostart=true -c security.nesting=true -c security.privileged=true
#-c volatile.dev-mgmt.hwaddr=00:11:22:33:44:55

lxc exec ${CONTAINER_NAME} -- apt update
lxc exec ${CONTAINER_NAME} -- apt dist-upgrade
lxc exec ${CONTAINER_NAME} -- apt purge -y lxd lxd-client snapd unattended-upgrades
lxc exec ${CONTAINER_NAME} -- apt autoremove

lxc file push /root/.ssh/authorized_keys ${CONTAINER_NAME}/root/.ssh/authorized_keys
lxc exec ${CONTAINER_NAME} -- bash -c "sed -i 's/eth0:/dev-mgmt:/g' /etc/netplan/50-cloud-init.yaml"
lxc exec ${CONTAINER_NAME} -- netplan apply

printf 'lxc.apparmor.profile = unconfined\nlxc.cgroup.devices.allow = a\nlxc.mount.auto=proc:rw sys:rw\nlxc.cap.drop=' | lxc config set ${CONTAINER_NAME} raw.lxc -
lxc restart ${CONTAINER_NAME}

Install Docker inside LXD container
# http://www.panticz.de/install-docker

apt update
sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
sudo apt-get install -y docker-ce

Install GitLab-runner (as Docker container)
https://docs.gitlab.com/runner/install/docker.html

docker run -d \
  --name gitlab-runner \
  --restart always \
  -v /srv/gitlab-runner/config:/etc/gitlab-runner \
  -v /var/run/docker.sock:/var/run/docker.sock \
  gitlab/gitlab-runner:latest

Configue GitLab-runner

Install
https://desktop.jitsi.org/Main/DebianRepository.html

wget -qO - https://download.jitsi.org/jitsi-key.gpg.key | sudo apt-key add -
sudo sh -c "echo 'deb https://download.jitsi.org stable/' > /etc/apt/sources.list.d/jitsi-stable.list"
sudo apt-get -y update
sudo apt-get -y install jitsi

Get broken / INACTIVE ports

DB_NAME=neutron
DB_USER=neutron
DB_PASS=$(grep neutron_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
mysql --host=${DB_HOST} --password=${DB_PASS} --port=6033 --user=${DB_USER} --database=${DB_NAME}

select port_id, host from ml2_port_bindings where status = 'INACTIVE';
select port_id, host from ml2_port_bindings where status = 'INACTIVE' and host = 'com3-dev';
select * from ml2_port_bindings where port_id = "65db0c73-1533-4c36-9fd4-79b5e4e9704c";
#select * from ml2_port_binding_levels where port_id = "65db0c73-1533-4c36-9fd4-79b5e4e9704c" \G

# debug
PORT_IDS=$(mysql --host=${DB_HOST} --password=${DB_PASS} --port=6033 --user=${DB_USER} --database=${DB_NAME} \
  -se "select port_id from ml2_port_bindings where status = 'INACTIVE'")
echo ${PORT_IDS}

# get VMs
for PORT_ID in ${PORT_IDS}; do
    echo ${PORT_ID}
    openstack server show -c name -c "OS-EXT-SRV-ATTR:host" $(openstack port show -c device_id -f value ${PORT_ID})
done

Recreate port

SERVER_ID=f4b8828d-a717-49f6-8ad3-7b0bb1921123

nova interface-list ${SERVER_ID}

PORT_ID=57944199-014c-4907-9d6e-1fb233151234

# debug
openstack port show ${PORT_ID} -c name -c network_id -c mac_address -c fixed_ips -c security_group_ids -c project_id -c device_id

get port data

Check kibana logs for
"While synchronizing instance power states, found"

Check libvirt VMs vs. nova DB

COMPUTE_NODE=com1-dev

# get shutoff VMs on compute node
VMS_COMPUTE=$(ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh list --state-shutoff --uuid | sed '/^$/d' | sort)
# echo "${VMS_COMPUTE}"

# get shutoff VMs from nova DB
VMS_NOVA=$(ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server list --all --host ${COMPUTE_NODE} -c ID -f value --status SHUTOFF" | sort)
# echo "${VMS_NOVA}"

# diff shutoff VMs
comm -3 <(echo "${VMS_COMPUTE}") <(echo "${VMS_NOVA}")

Remove shutdown VM from libvirt

VM_ID=c3fbcc6b-2dfe-4aa1-82bd-522b161a37a9

# OPTINAL: backup VM xml config
# ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dumpxml ${VM_ID} > /tmp/${COMPUTE_NODE}.${VM_ID}.xml

# show VM details (verfiy that OS-EXT-SRV-ATTR:host is not ${VMS_COMPUTE})
ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server show -c OS-EXT-SRV-ATTR:host -c name -c status ${VM_ID}"

# show VM details on compute node (verify that State: is "shut off")
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dominfo ${VM_ID}

# remove unnecessary VM
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh undefine ${VM_ID}

Show bridges without tab device

brctl show | egrep "qvb|tap" | sed '$!N;/\n.*tap/d;P;D' | awk '{print substr($1,4,8)}'

Get interface details

TOKEN=cac559da

# show port details
docker exec openvswitch_vswitchd ovsdb-client dump | grep ${TOKEN}

# get OVS port
docker exec openvswitch_vswitchd ovs-vsctl list-ports br-int | grep ${TOKEN}

# get OVS interface
docker exec openvswitch_vswitchd ovs-vsctl list-ifaces br-int | grep ${TOKEN}

# show host bridges
brctl show | grep ${TOKEN}

# show host interfaces
ip a | grep ${TOKEN}

Delete interface

PORT_TOKEN=a101dd83

# get port ID
PORT_ID=$(docker exec openvswitch_vswitchd ovs-vsctl list-ports br-int | grep ${PORT_TOKEN})
echo ${PORT_ID}

# remove port
docker exec openvswitch_vswitchd ovs-vsctl del-port br-int ${PORT_ID}

# remove bridge
BRIDGE_ID=${PORT_ID/qvo/qbr}
echo ${BRIDGE_ID}
ip link set ${BRIDGE_ID} down
brctl delbr ${BRIDGE_ID}

# remove interface
ip link delete ${PORT_ID}