Install LXC (Linux Containers) under Ubuntu


apt install -y bridge-utils debootstrap lxc-templates lxc

Create container

export LANG=en_US.UTF-8
export CONTAINER=wheezy
sudo sudo lxc-destroy -n ${CONTAINER}
sudo lxc-create -t debian -n ${CONTAINER}
sudo lxc-start -d -n ${CONTAINER}
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' | sudo tee /var/lib/lxc/${CONTAINER}/rootfs/etc/apt/apt.conf
sudo lxc-attach -n ${CONTAINER} -- apt-get clean
sudo lxc-attach -n ${CONTAINER} -- apt-get update
sudo lxc-attach -n ${CONTAINER} -- apt-get dist-upgrade -y
sudo lxc-attach -n ${CONTAINER} -- apt-get install -y wget vim
# start container in forderground
lxc-start -n vm1  -F
# optional
echo " = 1" | tee -a /var/lib/lxc/${CONTAINER}/config
# connect to container
ssh root@$(sudo lxc-info -i -H -n ${CONTAINER})

Install LXC from testing on Debian

echo "deb testing main" >> /etc/apt/sources.list.d/testing.list
apt-get update
apt-get -t testing install -y lxc
sed -i 's| = empty| = veth|' /etc/lxc/default.conf
echo " = lxcbr0" >> /etc/lxc/default.conf

Update LXC container templates

wget -q --no-check-certificate -O - | sudo bash -
# required for fedora container
sudo apt-get install -y yum  curl

OPTIONAL: install wget and vim by default under debian

sed -i '/iproute/a wget,\\\nvim,\\' /usr/share/lxc/templates/lxc-debian

OPTIONAL: enable apt-cache

sed -i 's|#MIRROR="http://<host-ip-addr>:3142/"|MIRROR="http://apt-cacher:3142/"|g' /etc/default/lxc
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /etc/apt/apt.conf 
apt-get update

OPTIONAL: create volume for lxc containers

lvcreate --name lxc-data --size 128G vg0
mkfs.ext4 /dev/vg0/lxc-data
echo "/dev/vg0/lxc-data    /var/lib/lxc/           ext4    defaults        0       0" >> /etc/fstab

create vm

# install debian wheezy
lxc-create -n vm1 -t debian
# create debian squeeze
export MIRROR="http://apt-cacher:3142/"
lxc-create -n squeeze1 -t debian -- template-options -r squeeze
# create debian jessie
sudo lxc-create -n jessie -t debian -- template-options -r jessie
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/jessie/config
sudo lxc-start -n jessie
# create debian sid
lxc-create -n sid -t debian -- template-options -r sid
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/sid/config
# create lvm container
lxc-create -n wheezy -t debian -B lvm --vgname vg0
# create ubuntu precise 32 bit
export MIRROR="http://apt-cacher:3142/"
sudo lxc-create -t ubuntu -n lpdev4 -- -r precise -a i386

List LXC container sort by memory usage

lxc-top -s m

lxc-attach, run command in a container

lxc-attach -n wheezy -- ls -l
lxc-create -n vm3 -t ubuntu
# lxc-create -n foo -f lxc.conf
# OPTIONAL: enable apt-cache in VM
echo 'Acquire::http::Proxy "http://apt-cacher:3142/";' >> /var/lib/lxc/vm3/rootfs/etc/apt/apt.conf

configure autostart

echo " = 1" >> /var/lib/lxc/vm3/config
# start all autostart container on boot
sed -i 's|OPTIONS=|OPTIONS="-a"|' /etc/default/lxc
# Container configuration
/var/lib/lxc/vm1/config = dev
# start vm
lxc-start -n vm1
# bypass apparmor (LTSP server test)
echo "lxc.aa_profile = unconfined" >> /var/lib/lxc/vm3/config


# dep
# apt-get install -y bridge-utils debootstrap
# libvirt-bin
# configure fs
echo "cgroup          /sys/fs/cgroup         cgroup  defaults        0       0" >> /etc/fstab
mount -a


lxc-stop -n squeeze
echo "before installing squid3" > /tmp/comment
sudo lxc-snapshot -n squeeze -c /tmp/comment
lxc-snapshot -L -C -n squeeze
# restore
sudo lxc-snapshot -n oracle -r snap0
# list snapshots
sudo lxc-snapshot -n squeeze -L
# destry snapshot
sudo lxc-snapshot -n squeeze -d snap0


/usr/share/lxc/templates/lxc-ubuntu -h
/usr/share/lxc/templates/lxc-debian -h

Nested LXC container

sudo apt-get install lxc
sudo lxc-create -t ubuntu -n host-container -t ubuntu
#E# sudo wget -O /etc/apparmor.d/lxc/lxc-with-nesting
sudo /etc/init.d/apparmor reload
# lxc.aa_profile = lxc-container-with-nesting
echo "lxc.aa_profile = unconfined" | sudo tee -a /var/lib/lxc/host-container/config
sudo lxc-start -n host-container
# user: ubuntu
# pass: ubuntu
# in host-container
sudo apt-get install lxc
sudo lxc-create -n sub-container -t ubuntu
sudo lxc-start -n sub-container
# on HOST, list nested container
sudo lxc-ls --fancy --nesting

configure static ip

cat <<EOF>> /var/lib/lxc/ubuntu/config = = auto

Limit resources

lxc.cgroup.cpuset.cpus                 = 16-23
lxc.cgroup.memory.limit_in_bytes       = 30720M
lxc.cgroup.memory.memsw.limit_in_bytes = 32768M
lxc.cgroup.cpu.cfs_period_us = 100000
lxc.cgroup.cpu.cfs_quota_us = 200000

Perfomance / Limits

# list limits
for CONTAINER in $(lxc-ls --running); do
    printf "%-40s %s\n" ${CONTAINER} $(lxc-cgroup -n ${CONTAINER} cpu.shares)
# get limits from configuration files
grep cpu /var/lib/lxc/*/config | grep -v '#lxc.cgroup.cpu.shares'
# configure cpu.shares on the fly
lxc-cgroup -n cpu.shares 256
lxc-cgroup -n cpu.shares 256
lxc-cgroup -n cpuset.cpus "0,3"
lxc-cgroup -n cpuset.cpus "0-3"
lxc-cgroup -n cpu.shares 512
echo "lxc.cgroup.cpu.shares = 512" >> /var/lib/lxc/ cpu.shares/config
# change memory on the fly
lxc-cgroup -n cacti memory.limit_in_bytes 1G
on runnting container
lxc-cgroup -n $CONTAINER cpuset.cpus 0

get container IP

sudo lxc-ls -f -F ipv4 jessie | tail -1
lxc-cgroup -n ol6ctr1 cpuset.cpus 0-7
lxc-cgroup -n ol6ctr1 cpuset.cpus 0,1
lxc-cgroup -n ol6ctr2 cpu.shares 256
lxc-cgroup -n ol6ctr2 blkio.weight 500
lxc-cgroup -n ol6ctr2 memory.soft_limit_in_bytes 268435456
lxc-cgroup -n ol6ctr2 memory.limit_in_bytes 53687091

Live migrate / copy container to localhost
sudo rsync -ae "ssh -i ./.ssh/id_rsa" --numeric-ids --exclude=proc/* --exclude=sys/* --exclude=tmp/* root@${LXC_HOST}:/var/lib/lxc/${CONTAINER} /var/lib/lxc/

Restore container

sudo rsync --numeric-ids -ae "ssh -i ./.ssh/id_rsa" /var/lib/lxc/
# check kernel config
# create ssh container
lxc-create -n ssh -t sshd
# list avaiable templates
ls -l /usr/share/lxc/templates/
lxc-create --template download --name gentoo
# debug
lxc-start --logfile /tmp/lxc-vm1.log --logpriority DEBUG -n vm1


wget -O - | apt-key add -
echo "deb debian/" | tee /etc/apt/sources.list.d/lwp.list
apt-get update
apt-get install lwp
cp /etc/lwp/lwp.example.conf /etc/lwp/lwp.conf
user: admin
pass: admin

rename container

lxc-stop -n ${FROM}
mv /var/lib/lxc/${FROM} /var/lib/lxc/${TO}
sed -i "s|${FROM}|${TO}|g" /var/lib/lxc/${TO}/config
echo ${TO%%.*} > /var/lib/lxc/${TO}/rootfs/etc/hostname
sed -i "s|${FROM}|${TO%%.*}|g" /var/lib/lxc/${TO}/rootfs/etc/hosts

Create VM on ramdisc
# cat /var/lib/lxc//config
lxc.mount.entry = /dev/shm var/lib/lxc none bind 0 0

# manual
#mount --bind /dev/shm /var/lib/lxc
# cat /etc/fstab
#/dev/shm /var/lib/lxc none bind 0 0

# test
LANG=C SUITE=jessie MIRROR= lxc-create -n debian8 -t debian
lxc-create -n debian8 -t debian -- -r jessie

LXC Web Panel

# set MAC for a container
sed -i 's| = .*| = 00:11:22:33:44:55|' /var/lib/lxc/${CONTAINER}/config

# passthrough NIC to container = phys = eth5 = eth1 = veth = fai = up

# dnsmasq
echo "," >> /etc/lxc/dnsmasq.conf
echo "dhcp-host=00:00:00:00:01:23,,,infinite" >> /etc/lxc/dnsmasq.conf
sed -i 's|#LXC_DHCP_CONFILE|LXC_DHCP_CONFILE|g' /etc/default/lxc-net
rm /var/lib/misc/dnsmasq.lxcbr0.leases
service lxc-net restart
service lxc restart

# remove unused interfaces
for i in $(brctl show | grep veth | sed "s/[ \t][ ]*/ /g" ); do
brctl delif lxcbr0 $i
for i in $(ifconfig | grep veth | cut -d" " -f1); do
ip link delete $i

# mounts
lxc.mount.entry=/media/www var/www none bind,create=dir,rw 0 0

lxc-console -n xenial
# exit console with ctrl + a, q

SSH in LXC container
cat /var/lib/lxc//config
#lxc.cgroup.devices.allow = c 10:229 rwm
lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0
# run in container
# sudo mknod /dev/fuse c 10 229

Create device inside container
cat /var/lib/lxc//config
#lxc.cgroup.devices.allow = c 10:200 rwm

cat /var/lib/lxc//autodev
mkdir net
mknod net/tun c 10 200
chmod 0666 net/tun

Mount devices from host into cointainer
lxc.mount = /var/lib/lxc/
cat /var/lib/lxc/
/mount/ftp media/ftp none bind,create=dir,rw

Forward traffic to container from LXC host
ifconfig br0:1 up
echo 1 > /proc/sys/net/ipv4/ip_forward
up iptables -t nat -A POSTROUTING -o br0 -j MASQUERADE
iptables -A FORWARD -i eth0 -p tcp --dport 10022 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10022 -j DNAT --to-destination
iptables -A FORWARD -i eth0 -p tcp --dport 10080 -j ACCEPT
iptables -t nat -A PREROUTING -p tcp --dport 10080 -j DNAT --to-destination

zfs create -o canmount=off -o mountpoint=none rpool/lxc
lxc-create -n test1 -t ubuntu -B zfs --zfsroot=rpool/lxc

# Replace upstart with sysvinit to make possible start older Ubuntu releases when stock on init start)
chroot /mnt
apt-get install sysvinit
# uncomment all entry in /etc/fstab
chmod 766 /dev/null
mkdir /var/run/network

# fix console
# cat /etc/inittab
#1:2345:respawn:/sbin/getty 38400 tty1
1:2345:respawn:/sbin/getty 38400 console

Access block device (broken?)
# cat /var/lib/lxc/trusty/config
lxc.aa_profile = lxc-container-default-with-mounting
lxc.cgroup.devices.allow = b 8:16 rwm
#lxc.cgroup.devices.allow = b 8:17 rwm
lxc.autodev = 1
lxc.hook.autodev = /var/lib/lxc/trusty/

# cat /var/lib/lxc/trusty/
mknod -m 777 ${LXC_ROOTFS_MOUNT}/dev/sdb b 8 16
#mknod -m 777 ${LXC_ROOTFS_MOUNT}/dev/sdb1 b 8 17

Distribution scripts

Update to LXC 2.1
lxc-update-config -c /var/lib/lxc/*/config
#sed -i 's|||g' /var/lib/lxc/*/config
#sed -i 's|lxc.utsname||g' /var/lib/lxc/*/config
#sed -i 's|lxc.rootfs =|lxc.rootfs.path =|g' /var/lib/lxc/*/config
##sed -i 's|lxc.rootfs.backend|#lxc.rootfs.backend|g' /var/lib/lxc/*/config

# strong screen inside lxc container
sh -c "exec >/dev/tty 2>/dev/tty Resize LXC LVM on the fly
TARGET=conainer01; PARTITION=_usr_local_atmail_users; lvdisplay -C | grep ${TARGET}${PARTITION}
mount -v /dev/lxc/${TARGET}${PARTITION} /mnt && lvextend -r -L+10g lxc/${TARGET}${PARTITION} && umount -v /mnt

Links - Debian jessie LXC package