Install LXC (Linux Containers) under Ubuntu


Create container
export LANG=en_US.UTF-8
export CONTAINER=wheezy

sudo sudo lxc-destroy -n ${CONTAINER}
sudo lxc-create -t debian -n ${CONTAINER}
sudo lxc-start -d -n ${CONTAINER}

echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' | sudo tee /var/lib/lxc/${CONTAINER}/rootfs/etc/apt/apt.conf

sudo lxc-attach -n ${CONTAINER} -- apt-get clean
sudo lxc-attach -n ${CONTAINER} -- apt-get update
sudo lxc-attach -n ${CONTAINER} -- apt-get dist-upgrade -y
sudo lxc-attach -n ${CONTAINER} -- apt-get install -y wget vim

# start container in forderground
lxc-start -n vm1 -F

# optional
echo " = 1" | tee -a /var/lib/lxc/${CONTAINER}/config

# connect to container
ssh root@$(sudo lxc-info -i -H -n ${CONTAINER})

Install LXC from testing on Debian
echo "deb testing main" >> /etc/apt/sources.list.d/testing.list
apt-get update
apt-get -t testing install -y lxc
sed -i 's| = empty| = veth|' /etc/lxc/default.conf
echo " = lxcbr0" >> /etc/lxc/default.conf

Update LXC container templates
wget -q --no-check-certificate -O - | sudo bash -

# required for fedora container
sudo apt-get install -y yum curl

OPTIONAL: install wget and vim by default under debian
sed -i '/iproute/a wget,\\\nvim,\\' /usr/share/lxc/templates/lxc-debian

OPTIONAL: enable apt-cache
sed -i 's|#MIRROR="http://:3142/"|MIRROR="http://apt-cacher:3142/"|g' /etc/default/lxc
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /etc/apt/apt.conf
apt-get update

OPTIONAL: create volume for lxc containers
lvcreate --name lxc-data --size 128G vg0
mkfs.ext4 /dev/vg0/lxc-data
echo "/dev/vg0/lxc-data /var/lib/lxc/ ext4 defaults 0 0" >> /etc/fstab

create vm
# install debian wheezy
lxc-create -n vm1 -t debian

# create debian squeeze
export MIRROR="http://apt-cacher:3142/"
lxc-create -n squeeze1 -t debian -- template-options -r squeeze

# create debian jessie
sudo lxc-create -n jessie -t debian -- template-options -r jessie
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/jessie/config
sudo lxc-start -n jessie

# create debian sid
lxc-create -n sid -t debian -- template-options -r sid
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/sid/config

# create lvm container
lxc-create -n wheezy -t debian -B lvm --vgname vg0

# create ubuntu precise 32 bit
export MIRROR="http://apt-cacher:3142/"
sudo lxc-create -t ubuntu -n lpdev4 -- -r precise -a i386

List LXC container sort by memory usage
lxc-top -s m

lxc-attach, run command in a container
lxc-attach -n wheezy -- ls -l

lxc-create -n vm3 -t ubuntu
# lxc-create -n foo -f lxc.conf

# OPTIONAL: enable apt-cache in VM
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /var/lib/lxc/vm3/rootfs/etc/apt/apt.conf

configure autostart
echo " = 1" >> /var/lib/lxc/vm3/config

# start all autostart container on boot
sed -i 's|OPTIONS=|OPTIONS="-a"|' /etc/default/lxc

# Container configuration
/var/lib/lxc/vm1/config = dev

# start vm
lxc-start -n vm1

# bypass apparmor (LTSP server test)
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/vm3/config


# dep
# apt-get install -y bridge-utils debootstrap
# libvirt-bin

# configure fs
echo "cgroup /sys/fs/cgroup cgroup defaults 0 0" >> /etc/fstab
mount -a

lxc-stop -n squeeze
echo "before installing squid3" > /tmp/comment
sudo lxc-snapshot -n squeeze -c /tmp/comment
lxc-snapshot -L -C -n squeeze

# restore
sudo lxc-snapshot -n squeeze -r snap0

# list snapshots
sudo lxc-snapshot -n squeeze -L

# destry snapshot
sudo lxc-snapshot -n squeeze -d snap0

/usr/share/lxc/templates/lxc-ubuntu -h
/usr/share/lxc/templates/lxc-debian -h

Nested LXC container

sudo apt-get install lxc
sudo lxc-create -t ubuntu -n host-container -t ubuntu
#E# sudo wget -O /etc/apparmor.d/lxc/lxc-with-nesting
sudo /etc/init.d/apparmor reload

# lxc.aa_profile = lxc-container-with-nesting
echo "lxc.aa_profile = unconfined" | sudo tee -a /var/lib/lxc/host-container/config
sudo lxc-start -n host-container

# user: ubuntu
# pass: ubuntu

# in host-container
sudo apt-get install lxc
sudo lxc-create -n sub-container -t ubuntu
sudo lxc-start -n sub-container

# on HOST, list nested container
sudo lxc-ls --fancy --nesting

configure static ip
cat <> /var/lib/lxc/ubuntu/config = = auto

Limit resources

lxc.cgroup.cpuset.cpus = 16-23
lxc.cgroup.memory.limit_in_bytes = 30720M
lxc.cgroup.memory.memsw.limit_in_bytes = 32768M

lxc.cgroup.cpu.cfs_period_us = 100000
lxc.cgroup.cpu.cfs_quota_us = 200000

Perfomance / Limits
# list limits
for CONTAINER in $(lxc-ls --running); do
printf "%-40s %s\n" ${CONTAINER} $(lxc-cgroup -n ${CONTAINER} cpu.shares)

# get limits from configuration files
grep cpu /var/lib/lxc/*/config | grep -v '#lxc.cgroup.cpu.shares'

# configure cpu.shares on the fly
lxc-cgroup -n cpu.shares 256

lxc-cgroup -n cpu.shares 256
lxc-cgroup -n cpuset.cpus "0,3"
lxc-cgroup -n cpuset.cpus "0-3"
lxc-cgroup -n cpu.shares 512
echo "lxc.cgroup.cpu.shares = 512" >> /var/lib/lxc/ cpu.shares/config

# change memory on the fly
lxc-cgroup -n cacti memory.limit_in_bytes 1G

on runnting container
lxc-cgroup -n $CONTAINER cpuset.cpus 0

get container IP
sudo lxc-ls -f -F ipv4 jessie | tail -1


lxc-cgroup -n ol6ctr1 cpuset.cpus 0-7
lxc-cgroup -n ol6ctr1 cpuset.cpus 0,1
lxc-cgroup -n ol6ctr2 cpu.shares 256
lxc-cgroup -n ol6ctr2 blkio.weight 500
lxc-cgroup -n ol6ctr2 memory.soft_limit_in_bytes 268435456
lxc-cgroup -n ol6ctr2 memory.limit_in_bytes 53687091

Move container to other machine
sudo rsync -ae "ssh -i ./.ssh/id_rsa" /var/lib/lxc/apt-cacher root@lxc2:/var/lib/lxc/

# check kernel config

# create ssh container
lxc-create -n ssh -t sshd

# list avaiable templates
ls -l /usr/share/lxc/templates/

lxc-create --template download --name gentoo

# debug
lxc-start --logfile /tmp/lxc-vm1.log --logpriority DEBUG -n vm1

# LXC-Web-Panel
wget -O - | apt-key add -
echo "deb debian/" | tee /etc/apt/sources.list.d/lwp.list
apt-get update
apt-get install lwp

cp /etc/lwp/lwp.example.conf /etc/lwp/lwp.conf
user: admin
pass: admin

rename container
lxc-stop -n ${FROM}
mv /var/lib/lxc/${FROM} /var/lib/lxc/${TO}
sed -i "s|${FROM}|${TO}|g" /var/lib/lxc/${TO}/config
echo ${TO%%.*} > /var/lib/lxc/${TO}/rootfs/etc/hostname
sed -i "s|${FROM}|${TO%%.*}|g" /var/lib/lxc/${TO}/rootfs/etc/hosts

Create VM on ramdisc
# cat /var/lib/lxc//config
lxc.mount.entry = /dev/shm var/lib/lxc none bind 0 0

# manual
#mount --bind /dev/shm /var/lib/lxc
# cat /etc/fstab
#/dev/shm /var/lib/lxc none bind 0 0

# test
LANG=C SUITE=jessie MIRROR= lxc-create -n debian8 -t debian
lxc-create -n debian8 -t debian -- -r jessie

LXC Web Panel

# set MAC for a container
sed -i 's| = .*| = 00:11:22:33:44:55|' /var/lib/lxc/${CONTAINER}/config

# passthrough NIC to container = phys = eth5 = eth1 = veth = fai = up

# dnsmasq
echo "," >> /etc/lxc/dnsmasq.conf
sed -i 's|#LXC_DHCP_CONFILE|LXC_DHCP_CONFILE|g' /etc/default/lxc-net
rm /var/lib/misc/dnsmasq.lxcbr0.leases
service lxc-net restart
service lxc restart

# remove unused interfaces
for i in $(brctl show | grep veth | sed "s/

[ ]*/ /g" ); do
    brctl delif lxcbr0 $i
for i in $(ifconfig | grep veth | cut -d" " -f1); do
    ip link delete $i
# mounts
lxc.mount.entry=/media/www var/www none bind,create=dir,rw  0 0
lxc-console -n xenial
# exit console with ctrl + a, q
<strong>SSH in LXC container</strong>
cat /var/lib/lxc/<container>/config 
#lxc.cgroup.devices.allow = c 10:229 rwm
lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0
# run in container
# sudo mknod /dev/fuse c 10 229
<strong>Create device inside container</strong>
cat /var/lib/lxc/<container>/config 
#lxc.cgroup.devices.allow = c 10:200 rwm
cat /var/lib/lxc/<container>/autodev 
mkdir net
mknod net/tun c 10 200
chmod 0666 net/tun
<strong>Mount devices from host into cointainer</strong>
lxc.mount = /var/lib/lxc/
cat /var/lib/lxc/
/mount/ftp media/ftp none bind,create=dir,rw
<strong>Forward traffic to container from LXC host</strong>
ifconfig br0:1 up
echo 1 > /proc/sys/net/ipv4/ip_forward
up iptables -t nat -A POSTROUTING -o br0 -j MASQUERADE
iptables -A FORWARD -i eth0 -p tcp --dport 10022 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10022 -j DNAT --to-destination
iptables -A FORWARD -i eth0 -p tcp --dport 10080 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10080 -j DNAT --to-destination
zfs create -o canmount=off -o mountpoint=none rpool/lxc
lxc-create -n test1 -t ubuntu -B zfs --zfsroot=rpool/lxc
# Replace upstart with sysvinit to make possible start older Ubuntu releases when stock on init start)
chroot /mnt
apt-get install sysvinit
# uncomment all entry in /etc/fstab
chmod 766 /dev/null
mkdir /var/run/network
# fix console
# cat /etc/inittab
#1:2345:respawn:/sbin/getty 38400 tty1
1:2345:respawn:/sbin/getty 38400 console
<strong>Access block device (broken?)</strong>
# cat /var/lib/lxc/trusty/config
lxc.aa_profile = lxc-container-default-with-mounting
lxc.cgroup.devices.allow = b 8:16 rwm
#lxc.cgroup.devices.allow = b 8:17 rwm
lxc.autodev = 1
lxc.hook.autodev = /var/lib/lxc/trusty/
# cat /var/lib/lxc/trusty/
mknod -m 777 ${LXC_ROOTFS_MOUNT}/dev/sdb b 8 16
#mknod -m 777 ${LXC_ROOTFS_MOUNT}/dev/sdb1 b 8 17
<strong>Distribution scripts</strong>
<strong>Update to LXC 2.1</strong>
lxc-update-config -c /var/lib/lxc/*/config
#sed -i 's|||g' /var/lib/lxc/*/config
#sed -i 's|lxc.utsname||g' /var/lib/lxc/*/config
#sed -i 's|lxc.rootfs =|lxc.rootfs.path =|g' /var/lib/lxc/*/config
##sed -i 's|lxc.rootfs.backend|#lxc.rootfs.backend|g' /var/lib/lxc/*/config
# strong screen inside lxc container
sh -c "exec >/dev/tty 2>/dev/tty </dev/tty && /usr/bin/screen -x"
# convert include to include_tasks
find -name "*.yml" -exec sed -i 's|include|include_tasks|g' {} \;
# wait util all cointainer are shutdown
while [ $(lxc-ls -1 --running | wc -l) -gt 0  ]; do 
    sleep 1
    echo .
<strong>Links</strong> - Debian jessie LXC package