Install LXC (Linux Containers) under Ubuntu

Installation
http://www.panticz.de/install_lxc

Create container
export LANG=en_US.UTF-8
export CONTAINER=wheezy

sudo sudo lxc-destroy -n ${CONTAINER}
sudo lxc-create -t debian -n ${CONTAINER}
sudo lxc-start -d -n ${CONTAINER}

echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' | sudo tee /var/lib/lxc/${CONTAINER}/rootfs/etc/apt/apt.conf

sudo lxc-attach -n ${CONTAINER} -- apt-get clean
sudo lxc-attach -n ${CONTAINER} -- apt-get update
sudo lxc-attach -n ${CONTAINER} -- apt-get dist-upgrade -y
sudo lxc-attach -n ${CONTAINER} -- apt-get install -y wget vim

# start container in forderground
lxc-start -n vm1 -F

# optional
echo "lxc.start.auto = 1" | tee -a /var/lib/lxc/${CONTAINER}/config

# connect to container
ssh root@$(sudo lxc-info -i -H -n ${CONTAINER})

Install LXC from testing on Debian
echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list.d/testing.list
apt-get update
apt-get -t testing install -y lxc
sed -i 's|lxc.network.type = empty|lxc.network.type = veth|' /etc/lxc/default.conf
echo "lxc.network.link = lxcbr0" >> /etc/lxc/default.conf

Update LXC container templates
wget -q --no-check-certificate https://raw.githubusercontent.com/panticz/lxc/master/scripts/update_templates.sh -O - | sudo bash -

# required for fedora container
sudo apt-get install -y yum curl

OPTIONAL: install wget and vim by default under debian
sed -i '/iproute/a wget,\\\nvim,\\' /usr/share/lxc/templates/lxc-debian

OPTIONAL: enable apt-cache
sed -i 's|#MIRROR="http://:3142/archive.ubuntu.com/ubuntu"|MIRROR="http://apt-cacher:3142/archive.ubuntu.com/ubuntu"|g' /etc/default/lxc
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /etc/apt/apt.conf
apt-get update

OPTIONAL: create volume for lxc containers
lvcreate --name lxc-data --size 128G vg0
mkfs.ext4 /dev/vg0/lxc-data
echo "/dev/vg0/lxc-data /var/lib/lxc/ ext4 defaults 0 0" >> /etc/fstab

create vm
# install debian wheezy
LANG=en_US.UTF-8
lxc-create -n vm1 -t debian

# create debian squeeze
export MIRROR="http://apt-cacher:3142/ftp.debian.org/debian"
lxc-create -n squeeze1 -t debian -- template-options -r squeeze

# create debian jessie
sudo lxc-create -n jessie -t debian -- template-options -r jessie
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/jessie/config
sudo lxc-start -n jessie

# create debian sid
lxc-create -n sid -t debian -- template-options -r sid
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/sid/config

# create lvm container
lxc-create -n wheezy -t debian -B lvm --vgname vg0

# create ubuntu precise 32 bit
export MIRROR="http://apt-cacher:3142/archive.ubuntu.com/ubuntu"
sudo lxc-create -t ubuntu -n lpdev4 -- -r precise -a i386

List LXC container sort by memory usage
lxc-top -s m

lxc-attach, run command in a container
lxc-attach -n wheezy -- ls -l

lxc-create -n vm3 -t ubuntu
# lxc-create -n foo -f lxc.conf

# OPTIONAL: enable apt-cache in VM
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /var/lib/lxc/vm3/rootfs/etc/apt/apt.conf

configure autostart
echo "lxc.start.auto = 1" >> /var/lib/lxc/vm3/config

# start all autostart container on boot
sed -i 's|OPTIONS=|OPTIONS="-a"|' /etc/default/lxc

# Container configuration
/var/lib/lxc/vm1/config
lxc.group = dev

# start vm
lxc-start -n vm1

# bypass apparmor (LTSP server test)
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/vm3/config

templates
/usr/share/lxc/templates/

# dep
# apt-get install -y bridge-utils debootstrap
# libvirt-bin

# configure fs
echo "cgroup /sys/fs/cgroup cgroup defaults 0 0" >> /etc/fstab
mount -a

Snapshots
lxc-stop -n squeeze
echo "before installing squid3" > /tmp/comment
sudo lxc-snapshot -n squeeze -c /tmp/comment
lxc-snapshot -L -C -n squeeze

# restore
sudo lxc-snapshot -n squeeze -r snap0

# list snapshots
sudo lxc-snapshot -n squeeze -L

# destry snapshot
sudo lxc-snapshot -n squeeze -d snap0

Templates
/usr/share/lxc/templates/lxc-ubuntu -h
/usr/share/lxc/templates/lxc-debian -h

Nested LXC container

https://www.stgraber.org/2013/12/21/lxc-1-0-advanced-container-usage/
http://serverfault.com/questions/366575/is-it-possible-to-start-lxc-container-inside-lxc-container

sudo apt-get install lxc
sudo lxc-create -t ubuntu -n host-container -t ubuntu
#E# sudo wget https://www.stgraber.org/download/lxc-with-nesting -O /etc/apparmor.d/lxc/lxc-with-nesting
sudo /etc/init.d/apparmor reload

# lxc.aa_profile = lxc-container-with-nesting
echo "lxc.aa_profile = unconfined" | sudo tee -a /var/lib/lxc/host-container/config
sudo lxc-start -n host-container

# user: ubuntu
# pass: ubuntu

# in host-container
sudo apt-get install lxc
sudo lxc-create -n sub-container -t ubuntu
sudo lxc-start -n sub-container

# on HOST, list nested container
sudo lxc-ls --fancy --nesting

configure static ip
cat <> /var/lib/lxc/ubuntu/config
lxc.network.ipv4 = 10.0.0.14
lxc.network.ipv4.gateway = auto
EOF

Limit resources
http://serverfault.com/questions/444232/limit-memory-and-cpu-with-lxc-execute
https://www.thomas-krenn.com/de/wiki/Linux_Containers_LXC
https://github.com/lxc/lxd/blob/master/doc/configuration.md
https://www.stgraber.org/2016/03/26/lxd-2-0-resource-control-412/

lxc.cgroup.cpuset.cpus = 16-23
lxc.cgroup.memory.limit_in_bytes = 30720M
lxc.cgroup.memory.memsw.limit_in_bytes = 32768M

??
lxc.cgroup.cpu.cfs_period_us = 100000
lxc.cgroup.cpu.cfs_quota_us = 200000

Perfomance / Limits
# list limits
for CONTAINER in $(lxc-ls --running); do
printf "%-40s %s\n" ${CONTAINER} $(lxc-cgroup -n ${CONTAINER} cpu.shares)
done

# get limits from configuration files
grep cpu /var/lib/lxc/*/config | grep -v '#lxc.cgroup.cpu.shares'

# configure cpu.shares on the fly
lxc-cgroup -n www.example.com cpu.shares 256

lxc-cgroup -n www.example.com cpu.shares 256
lxc-cgroup -n www.example.com cpuset.cpus "0,3"
lxc-cgroup -n www.example.com cpuset.cpus "0-3"
lxc-cgroup -n www.example.com cpu.shares 512
echo "lxc.cgroup.cpu.shares = 512" >> /var/lib/lxc/www.example.com cpu.shares/config

# change memory on the fly
lxc-cgroup -n cacti memory.limit_in_bytes 1G

on runnting container
lxc-cgroup -n $CONTAINER cpuset.cpus 0

get container IP
sudo lxc-ls -f -F ipv4 jessie | tail -1

lxc.cgroup.cpu.shares=256
lxc.cgroup.blkio.weight=500

lxc-cgroup -n ol6ctr1 cpuset.cpus 0-7
lxc-cgroup -n ol6ctr1 cpuset.cpus 0,1
lxc-cgroup -n ol6ctr2 cpu.shares 256
lxc-cgroup -n ol6ctr2 blkio.weight 500
lxc-cgroup -n ol6ctr2 memory.soft_limit_in_bytes 268435456
lxc-cgroup -n ol6ctr2 memory.limit_in_bytes 53687091

Move container to other machine
sudo rsync -ae "ssh -i ./.ssh/id_rsa" /var/lib/lxc/apt-cacher root@lxc2:/var/lib/lxc/

# check kernel config
lxc-checkconfig

# create ssh container
lxc-create -n ssh -t sshd

# list avaiable templates
ls -l /usr/share/lxc/templates/

lxc-create --template download --name gentoo

# debug
lxc-start --logfile /tmp/lxc-vm1.log --logpriority DEBUG -n vm1

#
# LXC-Web-Panel
# http://claudyus.github.io/LXC-Web-Panel/
# http://claudyus.github.io/LXC-Web-Panel/download.html
#
wget -O - http://claudyus.github.io/LXC-Web-Panel/claudyus.gpg.key | apt-key add -
echo "deb http://claudyus.github.io/LXC-Web-Panel/ debian/" | tee /etc/apt/sources.list.d/lwp.list
apt-get update
apt-get install lwp

cp /etc/lwp/lwp.example.conf /etc/lwp/lwp.conf

http://192.168.1.176:5000/
user: admin
pass: admin

rename container
FROM=jessie
TO=letsencrypt
lxc-stop -n ${FROM}
mv /var/lib/lxc/${FROM} /var/lib/lxc/${TO}
sed -i "s|${FROM}|${TO}|g" /var/lib/lxc/${TO}/config
echo ${TO%%.*} > /var/lib/lxc/${TO}/rootfs/etc/hostname
sed -i "s|${FROM}|${TO%%.*}|g" /var/lib/lxc/${TO}/rootfs/etc/hosts

Create VM on ramdisc
# cat /var/lib/lxc//config
lxc.mount.entry = /dev/shm var/lib/lxc none bind 0 0

# manual
#mount --bind /dev/shm /var/lib/lxc
# cat /etc/fstab
#/dev/shm /var/lib/lxc none bind 0 0

# test
LANG=C SUITE=jessie MIRROR=http://httpredir.debian.org/debian lxc-create -n debian8 -t debian
lxc-create -n debian8 -t debian -- -r jessie

LXC Web Panel
https://lxc-webpanel.github.io/

# set MAC for a container
sed -i 's|lxc.network.hwaddr = .*|lxc.network.hwaddr = 00:11:22:33:44:55|' /var/lib/lxc/${CONTAINER}/config

# passthrough NIC to container
lxc.network.type = phys
lxc.network.link = eth5
lxc.network.name = eth1

# dnsmasq
/etc/lxc/dnsmasq.conf
/var/lib/misc/dnsmasq.lxcbr0.leases
service lxc-net restart

# remove unused interfaces
for i in $(brctl show | grep veth | sed "s/

[ ]*/ /g" ); do
    brctl delif lxcbr0 $i
done
for i in $(ifconfig | grep veth | cut -d" " -f1); do
    ip link delete $i
done
 
# mounts
lxc.mount.entry=/media/www var/www none bind,create=dir,rw  0 0
 
<strong>Console</strong>
lxc-console -n xenial
# exit console with ctrl + a, q
 
<strong>SSH in LXC container</strong>
cat /var/lib/lxc/<container>/config 
#lxc.cgroup.devices.allow = c 10:229 rwm
lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0
# run in container
# sudo mknod /dev/fuse c 10 229
https://wiki.kubuntu.org/FuseUserns
 
<strong>Create device inside container</strong>
cat /var/lib/lxc/<container>/config 
#lxc.hook.autodev=/var/lib/lxc/vpn.example.com/autodev
#lxc.cgroup.devices.allow = c 10:200 rwm
 
cat /var/lib/lxc/<container>/autodev 
#!/bin/bash
cd ${LXC_ROOTFS_MOUNT}/dev
mkdir net
mknod net/tun c 10 200
chmod 0666 net/tun
 
 
<strong>Mount devices from host into cointainer</strong>
lxc.mount = /var/lib/lxc/www.example.com/fstab
cat /var/lib/lxc/www.example.com/fstab
/mount/ftp media/ftp none bind,create=dir,rw
 
<strong>Forward traffic to container from LXC host</strong>
ifconfig br0:1 10.0.3.1 up
echo 1 > /proc/sys/net/ipv4/ip_forward
up iptables -t nat -A POSTROUTING -o br0 -j MASQUERADE
iptables -A FORWARD -i eth0 -p tcp --dport 10022 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10022 -j DNAT --to-destination 10.0.3.10:22
iptables -A FORWARD -i eth0 -p tcp --dport 10080 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10080 -j DNAT --to-destination 10.0.3.10:80
 
<strong>ZFS</strong>
zfs create -o canmount=off -o mountpoint=none rpool/lxc
lxc-create -n test1 -t ubuntu -B zfs --zfsroot=rpool/lxc
 
 
<strong>Troubleshooting</strong>
# Replace upstart with sysvinit to make possible start older Ubuntu releases when stock on init start)
chroot /mnt
apt-get install sysvinit
# uncomment all entry in /etc/fstab
chmod 766 /dev/null
 
# fix console
# cat /etc/inittab
1:2345:respawn:/sbin/getty 38400 console
 
 
 
<strong>Links</strong>
http://www.kobashicomputing.com/a-lxc-container-tutorial
https://linuxcontainers.org/lxc/manpages/man5/lxc.container.conf.5.html
https://github.com/lxc/lxd-pkg-ubuntu
https://github.com/lxc/lxc
http://www.funtoo.org/Linux_Containers
https://www.stgraber.org/2013/12/21/lxc-1-0-your-second-container/
http://askubuntu.com/questions/256304/public-ip-address-for-lxc-container/311003#311003
http://wiki.gentoo.org/wiki/LXC
http://www.linuxcertif.com/man/5/lxc.conf/
https://www.docker.io/
http://wiki.debian.org/LXC
http://blog.bodhizazen.net/linux/lxc-configure-ubuntu-lucid-containers/
http://blog.foaa.de/2010/05/lxc-on-debian-squeeze/
http://www.thomas-krenn.com/de/wiki/Perl_warning_Setting_locale_failed_unter_Debian
https://packages.debian.org/de/jessie/lxc - Debian jessie LXC package