Install LXC (Linux Containers) under Ubuntu

Installation

apt install -y bridge-utils debootstrap lxc-templates lxc

http://www.panticz.de/install_lxc

Create container

export LANG=en_US.UTF-8
export CONTAINER=wheezy
 
sudo sudo lxc-destroy -n ${CONTAINER}
sudo lxc-create -t debian -n ${CONTAINER}
sudo lxc-start -d -n ${CONTAINER}
 
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' | sudo tee /var/lib/lxc/${CONTAINER}/rootfs/etc/apt/apt.conf
 
sudo lxc-attach -n ${CONTAINER} -- apt-get clean
sudo lxc-attach -n ${CONTAINER} -- apt-get update
sudo lxc-attach -n ${CONTAINER} -- apt-get dist-upgrade -y
sudo lxc-attach -n ${CONTAINER} -- apt-get install -y wget vim
 
# start container in forderground
lxc-start -n vm1  -F
 
# optional
echo "lxc.start.auto = 1" | tee -a /var/lib/lxc/${CONTAINER}/config
 
# connect to container
ssh root@$(sudo lxc-info -i -H -n ${CONTAINER})

Install LXC from testing on Debian

echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list.d/testing.list
apt-get update
apt-get -t testing install -y lxc
sed -i 's|lxc.network.type = empty|lxc.network.type = veth|' /etc/lxc/default.conf
echo "lxc.network.link = lxcbr0" >> /etc/lxc/default.conf

Update LXC container templates

wget -q --no-check-certificate https://raw.githubusercontent.com/panticz/lxc/master/scripts/lxc-update-templates.sh -O - | sudo bash -
 
# required for fedora container
sudo apt-get install -y yum  curl

OPTIONAL: install wget and vim by default under debian

sed -i '/iproute/a wget,\\\nvim,\\' /usr/share/lxc/templates/lxc-debian

OPTIONAL: enable apt-cache

sed -i 's|#MIRROR="http://<host-ip-addr>:3142/archive.ubuntu.com/ubuntu"|MIRROR="http://apt-cacher:3142/archive.ubuntu.com/ubuntu"|g' /etc/default/lxc
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /etc/apt/apt.conf 
apt-get update

OPTIONAL: create volume for lxc containers

lvcreate --name lxc-data --size 128G vg0
mkfs.ext4 /dev/vg0/lxc-data
echo "/dev/vg0/lxc-data    /var/lib/lxc/           ext4    defaults        0       0" >> /etc/fstab

create vm

# install debian wheezy
LANG=en_US.UTF-8
lxc-create -n vm1 -t debian
 
# create debian squeeze
export MIRROR="http://apt-cacher:3142/ftp.debian.org/debian"
lxc-create -n squeeze1 -t debian -- template-options -r squeeze
 
# create debian jessie
sudo lxc-create -n jessie -t debian -- template-options -r jessie
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/jessie/config
sudo lxc-start -n jessie
 
# create debian sid
lxc-create -n sid -t debian -- template-options -r sid
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/sid/config
 
# create lvm container
lxc-create -n wheezy -t debian -B lvm --vgname vg0
 
# create ubuntu precise 32 bit
export MIRROR="http://apt-cacher:3142/archive.ubuntu.com/ubuntu"
sudo lxc-create -t ubuntu -n lpdev4 -- -r precise -a i386

List LXC container sort by memory usage

lxc-top -s m

lxc-attach, run command in a container

lxc-attach -n wheezy -- ls -l
 
lxc-create -n vm3 -t ubuntu
# lxc-create -n foo -f lxc.conf
 
# OPTIONAL: enable apt-cache in VM
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /var/lib/lxc/vm3/rootfs/etc/apt/apt.conf

configure autostart

echo "lxc.start.auto = 1" >> /var/lib/lxc/vm3/config
 
# start all autostart container on boot
sed -i 's|OPTIONS=|OPTIONS="-a"|' /etc/default/lxc
 
# Container configuration
/var/lib/lxc/vm1/config
lxc.group = dev
 
# start vm
lxc-start -n vm1
 
# bypass apparmor (LTSP server test)
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/vm3/config

templates

/usr/share/lxc/templates/
 
# dep
# apt-get install -y bridge-utils debootstrap
# libvirt-bin
 
# configure fs
echo "cgroup          /sys/fs/cgroup         cgroup  defaults        0       0" >> /etc/fstab
mount -a

Snapshots

lxc-stop -n squeeze
echo "before installing squid3" > /tmp/comment
sudo lxc-snapshot -n squeeze -c /tmp/comment
lxc-snapshot -L -C -n squeeze
 
# restore
sudo lxc-snapshot -n oracle -r snap0
 
# list snapshots
sudo lxc-snapshot -n squeeze -L
 
# destry snapshot
sudo lxc-snapshot -n squeeze -d snap0

Templates

/usr/share/lxc/templates/lxc-ubuntu -h
/usr/share/lxc/templates/lxc-debian -h

Nested LXC container
https://www.stgraber.org/2013/12/21/lxc-1-0-advanced-container-usage/
http://serverfault.com/questions/366575/is-it-possible-to-start-lxc-container-inside-lxc-container

sudo apt-get install lxc
sudo lxc-create -t ubuntu -n host-container -t ubuntu
#E# sudo wget https://www.stgraber.org/download/lxc-with-nesting -O /etc/apparmor.d/lxc/lxc-with-nesting
sudo /etc/init.d/apparmor reload
 
# lxc.aa_profile = lxc-container-with-nesting
echo "lxc.aa_profile = unconfined" | sudo tee -a /var/lib/lxc/host-container/config
sudo lxc-start -n host-container
 
# user: ubuntu
# pass: ubuntu
 
# in host-container
sudo apt-get install lxc
sudo lxc-create -n sub-container -t ubuntu
sudo lxc-start -n sub-container
 
# on HOST, list nested container
sudo lxc-ls --fancy --nesting

configure static ip

cat <<EOF>> /var/lib/lxc/ubuntu/config
lxc.network.ipv4 = 10.0.0.14
lxc.network.ipv4.gateway = auto
EOF

Limit resources
http://serverfault.com/questions/444232/limit-memory-and-cpu-with-lxc-execute
https://www.thomas-krenn.com/de/wiki/Linux_Containers_LXC
https://github.com/lxc/lxd/blob/master/doc/configuration.md
https://www.stgraber.org/2016/03/26/lxd-2-0-resource-control-412/

lxc.cgroup.cpuset.cpus                 = 16-23
lxc.cgroup.memory.limit_in_bytes       = 30720M
lxc.cgroup.memory.memsw.limit_in_bytes = 32768M
 
??
lxc.cgroup.cpu.cfs_period_us = 100000
lxc.cgroup.cpu.cfs_quota_us = 200000

Perfomance / Limits

# list limits
for CONTAINER in $(lxc-ls --running); do
    printf "%-40s %s\n" ${CONTAINER} $(lxc-cgroup -n ${CONTAINER} cpu.shares)
done
 
# get limits from configuration files
grep cpu /var/lib/lxc/*/config | grep -v '#lxc.cgroup.cpu.shares'
 
# configure cpu.shares on the fly
lxc-cgroup -n www.example.com cpu.shares 256
 
lxc-cgroup -n www.example.com cpu.shares 256
lxc-cgroup -n www.example.com cpuset.cpus "0,3"
lxc-cgroup -n www.example.com cpuset.cpus "0-3"
lxc-cgroup -n www.example.com cpu.shares 512
echo "lxc.cgroup.cpu.shares = 512" >> /var/lib/lxc/www.example.com cpu.shares/config
 
# change memory on the fly
lxc-cgroup -n cacti memory.limit_in_bytes 1G
 
on runnting container
lxc-cgroup -n $CONTAINER cpuset.cpus 0

get container IP

sudo lxc-ls -f -F ipv4 jessie | tail -1
 
lxc.cgroup.cpu.shares=256
lxc.cgroup.blkio.weight=500
 
lxc-cgroup -n ol6ctr1 cpuset.cpus 0-7
lxc-cgroup -n ol6ctr1 cpuset.cpus 0,1
lxc-cgroup -n ol6ctr2 cpu.shares 256
lxc-cgroup -n ol6ctr2 blkio.weight 500
lxc-cgroup -n ol6ctr2 memory.soft_limit_in_bytes 268435456
lxc-cgroup -n ol6ctr2 memory.limit_in_bytes 53687091

Live migrate / copy container to localhost

LXC_HOST=lxc1.example.com
CONTAINER=www.example.com
sudo rsync -ae "ssh -i ./.ssh/id_rsa" --numeric-ids --exclude=proc/* --exclude=sys/* --exclude=tmp/* root@${LXC_HOST}:/var/lib/lxc/${CONTAINER} /var/lib/lxc/

Restore container

sudo rsync --numeric-ids -ae "ssh -i ./.ssh/id_rsa" root@bkp1.exampe.com:/vo1/backup/rsnapshot/daily.0/lxc1.example.com/var/lib/lxc/foo.vm /var/lib/lxc/
 
# check kernel config
lxc-checkconfig
 
# create ssh container
lxc-create -n ssh -t sshd
 
# list avaiable templates
ls -l /usr/share/lxc/templates/
 
lxc-create --template download --name gentoo
 
# debug
lxc-start --logfile /tmp/lxc-vm1.log --logpriority DEBUG -n vm1

LXC-Web-Panel
http://claudyus.github.io/LXC-Web-Panel/
http://claudyus.github.io/LXC-Web-Panel/download.html

wget -O - http://claudyus.github.io/LXC-Web-Panel/claudyus.gpg.key | apt-key add -
echo "deb http://claudyus.github.io/LXC-Web-Panel/ debian/" | tee /etc/apt/sources.list.d/lwp.list
apt-get update
apt-get install lwp
 
cp /etc/lwp/lwp.example.conf /etc/lwp/lwp.conf
 
http://192.168.1.176:5000/
user: admin
pass: admin

rename container

FROM=jessie
TO=letsencrypt
lxc-stop -n ${FROM}
mv /var/lib/lxc/${FROM} /var/lib/lxc/${TO}
sed -i "s|${FROM}|${TO}|g" /var/lib/lxc/${TO}/config
echo ${TO%%.*} > /var/lib/lxc/${TO}/rootfs/etc/hostname
sed -i "s|${FROM}|${TO%%.*}|g" /var/lib/lxc/${TO}/rootfs/etc/hosts

Create VM on ramdisc
# cat /var/lib/lxc//config
lxc.mount.entry = /dev/shm var/lib/lxc none bind 0 0

# manual
#mount --bind /dev/shm /var/lib/lxc
# cat /etc/fstab
#/dev/shm /var/lib/lxc none bind 0 0

# test
LANG=C SUITE=jessie MIRROR=http://httpredir.debian.org/debian lxc-create -n debian8 -t debian
lxc-create -n debian8 -t debian -- -r jessie

LXC Web Panel
https://lxc-webpanel.github.io/

# set MAC for a container
sed -i 's|lxc.network.hwaddr = .*|lxc.network.hwaddr = 00:11:22:33:44:55|' /var/lib/lxc/${CONTAINER}/config

# passthrough NIC to container
lxc.network.type = phys
lxc.network.link = eth5
lxc.network.name = eth1

lxc.network.type = veth
lxc.network.link = fai
lxc.network.flags = up

# dnsmasq
echo "dhcp-host=fai.dev,10.0.3.100" >> /etc/lxc/dnsmasq.conf
echo "dhcp-host=00:00:00:00:01:23,www.example.com,10.0.3.123,infinite" >> /etc/lxc/dnsmasq.conf
sed -i 's|#LXC_DHCP_CONFILE|LXC_DHCP_CONFILE|g' /etc/default/lxc-net
rm /var/lib/misc/dnsmasq.lxcbr0.leases
service lxc-net restart
service lxc restart

# remove unused interfaces
for i in $(brctl show | grep veth | sed "s/[ \t][ ]*/ /g" ); do
brctl delif lxcbr0 $i
done
for i in $(ifconfig | grep veth | cut -d" " -f1); do
ip link delete $i
done

# mounts
lxc.mount.entry=/media/www var/www none bind,create=dir,rw 0 0

Console
lxc-console -n xenial
# exit console with ctrl + a, q

SSH in LXC container
cat /var/lib/lxc//config
#lxc.cgroup.devices.allow = c 10:229 rwm
lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0
# run in container
# sudo mknod /dev/fuse c 10 229
https://wiki.kubuntu.org/FuseUserns

Create device inside container
cat /var/lib/lxc//config
#lxc.hook.autodev=/var/lib/lxc/vpn.example.com/autodev
#lxc.cgroup.devices.allow = c 10:200 rwm

cat /var/lib/lxc//autodev
#!/bin/bash
cd ${LXC_ROOTFS_MOUNT}/dev
mkdir net
mknod net/tun c 10 200
chmod 0666 net/tun

Mount devices from host into cointainer
lxc.mount = /var/lib/lxc/www.example.com/fstab
cat /var/lib/lxc/www.example.com/fstab
/mount/ftp media/ftp none bind,create=dir,rw

Forward traffic to container from LXC host
ifconfig br0:1 10.0.3.1 up
echo 1 > /proc/sys/net/ipv4/ip_forward
up iptables -t nat -A POSTROUTING -o br0 -j MASQUERADE
iptables -A FORWARD -i eth0 -p tcp --dport 10022 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10022 -j DNAT --to-destination 10.0.3.10:22
iptables -A FORWARD -i eth0 -p tcp --dport 10080 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10080 -j DNAT --to-destination 10.0.3.10:80

ZFS
zfs create -o canmount=off -o mountpoint=none rpool/lxc
lxc-create -n test1 -t ubuntu -B zfs --zfsroot=rpool/lxc

Troubleshooting
# Replace upstart with sysvinit to make possible start older Ubuntu releases when stock on init start)
chroot /mnt
apt-get install sysvinit
# uncomment all entry in /etc/fstab
chmod 766 /dev/null
mkdir /var/run/network

# fix console
# cat /etc/inittab
#1:2345:respawn:/sbin/getty 38400 tty1
1:2345:respawn:/sbin/getty 38400 console

Access block device (broken?)
# cat /var/lib/lxc/trusty/config
lxc.aa_profile = lxc-container-default-with-mounting
lxc.cgroup.devices.allow = b 8:16 rwm
#lxc.cgroup.devices.allow = b 8:17 rwm
lxc.autodev = 1
lxc.hook.autodev = /var/lib/lxc/trusty/mount-hook.sh

# cat /var/lib/lxc/trusty/mount-hook.sh
#!/bin/sh
mknod -m 777 ${LXC_ROOTFS_MOUNT}/dev/sdb b 8 16
#mknod -m 777 ${LXC_ROOTFS_MOUNT}/dev/sdb1 b 8 17

Distribution scripts
/usr/share/debootstrap/scripts

Update to LXC 2.1
https://github.com/lxc/lxc/issues/1651
lxc-update-config -c /var/lib/lxc/*/config
#sed -i 's|lxc.network|lxc.net.0|g' /var/lib/lxc/*/config
#sed -i 's|lxc.utsname|lxc.uts.name|g' /var/lib/lxc/*/config
#sed -i 's|lxc.rootfs =|lxc.rootfs.path =|g' /var/lib/lxc/*/config
##sed -i 's|lxc.rootfs.backend|#lxc.rootfs.backend|g' /var/lib/lxc/*/config

# strong screen inside lxc container
sh -c "exec >/dev/tty 2>/dev/tty Resize LXC LVM on the fly
TARGET=conainer01; PARTITION=_usr_local_atmail_users; lvdisplay -C | grep ${TARGET}${PARTITION}
mount -v /dev/lxc/${TARGET}${PARTITION} /mnt && lvextend -r -L+10g lxc/${TARGET}${PARTITION} && umount -v /mnt

Links
http://www.kobashicomputing.com/a-lxc-container-tutorial
https://linuxcontainers.org/lxc/manpages/man5/lxc.container.conf.5.html
https://github.com/lxc/lxd-pkg-ubuntu
https://github.com/lxc/lxc
http://www.funtoo.org/Linux_Containers
https://www.stgraber.org/2013/12/21/lxc-1-0-your-second-container/
http://askubuntu.com/questions/256304/public-ip-address-for-lxc-container/311003#311003
http://wiki.gentoo.org/wiki/LXC
http://www.linuxcertif.com/man/5/lxc.conf/
https://www.docker.io/
http://wiki.debian.org/LXC
http://blog.bodhizazen.net/linux/lxc-configure-ubuntu-lucid-containers/
http://blog.foaa.de/2010/05/lxc-on-debian-squeeze/
http://www.thomas-krenn.com/de/wiki/Perl_warning_Setting_locale_failed_unter_Debian
https://packages.debian.org/de/jessie/lxc - Debian jessie LXC package