storage | panticz.de

NVMe

Install

sudo apt install -y nvme-cli

CLI

# list devices
nvme list

nvme smart-log /dev/nvme0n1

# rescan device
for nvme in /dev/nvme[0-9]; do
    sudo nvme ns-rescan $nvme
done

isdct show -d DeviceStatus,Index,Firmware,FirmwareUpdateAvailable -intelssd

# format
https://manpages.ubuntu.com/manpages/jammy/en/man1/nvme-format.1.html
nvme format --force /dev/nvmeXn1
blkdiscard --force /dev/nvmeXn1

Fix nvme1: ignoring ctrl due to duplicate subnqn
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1803692

# dmesg | grep nvme
[    2.546620] nvme nvme0: pci function 0000:5e:00.0
[    2.552447] nvme nvme1: pci function 0000:5f:00.0
[    2.768347] nvme nvme1: ignoring ctrl due to duplicate subnqn (nqn.2017-12.org.nvmexpress:uuid:11111111-2222-3333-4444-555555555555).
[    2.775422] nvme nvme1: Removing after probe failure status: -22
[    2.779813]  nvme0n1: p1 p2

Fix by upgrade NVMe firmware
http://www.panticz.de/intel/nvme

Delete

nvme format --ses=1 /dev/nvme1

Namespaces
https://www.linuxjournal.com/content/data-flash-part-ii-using-nvme-drives-and-creating-nvme-over-fabrics-network

s3cmd

Install s3cmd (s3 CLI client)

sudo apt install -y s3cmd

Configuration file

s3cmd --configure
${HOME}/.s3cfg

CLI

s3cmd mb s3://buk1
s3cmd put /tmp/1m.img s3://buk1
s3cmd ls s3://buk1
s3cmd du
s3cmd rm s3://buk1/1m.img
s3cmd mv s3://buk1/1m.img s3://buk2/1m.img
s3cmd rb s3://buk1
s3cmd rb s3://buk1 --recursive
s3cmd rm s3://lfs/ --recursive --force

# list all objects
s3cmd --host s3.exampl3.com \
  --access_key=${AWS_ACCESS_KEY_ID} \
  --secret_key=${AWS_SECRET_ACCESS_KEY} \
  --host-bucket X \
  la

s3cmd \
  --access_key xxxxxxxxxxx  \
  --secret_key xxxxxxxxxxxxxx \
  --host-bucket '%(bucket)s.s3.example.com' \
  --host s3.example.com \
  --signature-v2 \
  --no-preserve \
  ls s3://foo-duply

s3cmd \
  --host s3.example.com \
  --access_key=${S3_ACCESS_KEY} \
  --secret_key=${S3_SECRET_KEY} \
  --host-bucket X \
  --recursive \
  ls s3://images/ | grep ubuntu-22.04-amd64-disk.raw | tail -1 | cut -d"/" -f2-

ACL

s3cmd setacl s3://bucket/path/to/file --acl-public
s3cmd info s3://bucket/path/to/file
s3cmd setacl s3://bucket/path/to/file --acl-private

Links
https://lollyrock.com/articles/s3cmd-with-radosgw/
https://linuxconfig.org/getting-started-with-aws-s3cmd-command-by-examples

s3fs

Install

sudo apt install -y s3fs

Create credentials

echo ${BUCKET}:${ACCESS_KEY}:${SECRET_KEY} >> ~/.passwd-s3fs
chmod 600 ~/.passwd-s3fs

Mount S3 bucket

s3fs bucket1 /tmp/mnt -o url=https://s3.example.com

Parameter

-o passwd_file=~/.passwd-s3fs
-o url=https://s3.example.com,allow_other,umask=0000
-o use_cache=/tmp/cache

/etc/fstab

mybucket1.mydomain.org /mnt/mybucket1 fuse.s3fs _netdev,allow_other,passwd_file=/home/ftpuser/.passwd-aws-s3fs,default_acl=public-read,uid=1001,gid=65534 0 0

Links
https://github.com/s3fs-fuse/s3fs-fuse
https://gridscale.io/community/tutorials/s3-fuse-ubuntu/

CephFS

Install

sudo apt -y install ceph-fuse

Mount

mount -t ceph node01.srv.world:6789:/ /mnt -o name=admin,secretfile=admin.key

Links
https://www.server-world.info/en/note?os=Ubuntu_18.04&p=ceph&f=3

CEPH

Releases
https://docs.ceph.com/docs/master/releases/

CLI

ceph -v
ceph -s
ceph df

ceph status
ceph health detail
ceph df detail
ceph osd lspools
ceph osd pool ls detail

# replication size / count
ceph osd pool set os-dev_glance min_size 1
ceph osd pool set os-dev_glance size 2
ceph osd pool get os-dev_glance size

# disable rebuild
ceph osd set noout 

# enable rebuild
ceph osd unset noout

#default 1 3 3
ceph tell 'osd.*' injectargs '--osd_max_backfills 1 --osd_recovery_max_active 1 --osd_recovery_op_priority 3'

# health
ceph health detail 
ceph --version
rados list-inconsistent-obj 6.115 --format=json-pretty
ceph pg repair 6.115
ceph -w

# erasure
ceph osd erasure-code-profile ls
ceph osd pool create   erasure 

ceph health

# watch
ceph -w

ceph osd unset norecover
ceph osd unset nobackfill
ceph osd unset noout
ceph health

# get cluster’s FSID/UUID
ceph fsid

# additional commands
ceph tell mon.\* injectargs '--osd_pool_default_size=2'
ceph config set mgr target_max_misplaced_ratio .01
ceph osd set-require-min-compat-client luminous

monitoring

ceph mgr module enable prometheus

balancer

ceph mgr module enable balancer
ceph balancer mode upmap
ceph balancer on

rados / pool

rados df

Monitoring
https://github.com/ceph/ceph-nagios-plugins/releases

# Build nagios-plugins-ceph
sudo apt-get install -y devscripts fakeroot build-essential dh-python
git clone https://github.com/ceph/ceph-nagios-plugins.git
cd ceph-nagios-plugins#
make deb
sudo dpkg -i nagios-plugins-ceph_*_all.deb

# create wrapper for kolla-ansible installation
cat < /usr/bin/ceph
#!/bin/bash
docker exec -it ceph_mon ceph $@
EOF
chmod +x /usr/bin/ceph

/usr/lib/nagios/plugins/check_ceph_health

Replace OSD

DRBD

# cat /etc/drbd.d/global_common.conf 
global {
		usage-count	yes;
}

common {
	startup {
		degr-wfc-timeout	0;
	}

	net {
		cram-hmac-alg	sha1;
		shared-secret	****************;
	}

	disk {
		on-io-error	detach;
	}
}

# cat /etc/drbd.d/r0.res 
resource r0 {
	on scld.sedo.de.intern {
		volume 0 {
			device		/dev/drbd0;
			disk		/dev/vg0/lvol0;
			flexible-meta-disk	internal;
		}
		address		192.168.255.1:7788;
	}
	on ubuntu {
		volume 0 {
			device		/dev/drbd0;
			disk		/dev/sda3;
			flexible-meta-disk	internal;
		}
		address		192.168.255.2:7788;
	}
}

drbdadm disk-options --c-plan-ahead=0 --resync-rate=900M --c-fill-target=0 r0

# status
drbd-overview

# get resources
cat /etc/drbd.d/db.res | grep resource | cut -d" " -f2

# resolve split brain
dmesg | grep -i "split-brain" 

# on slave (split brain victim)
drbdadm secondary db1
drbdadm connect --discard-my-data  db

# on master
drbdadm connect db

Links
https://docs.linbit.com/doc/users-guide-84/s-resolve-split-brain/
https://wiki.ubuntuusers.de/DRBD/

storage