s3 | panticz.de

goofys

mkdir -p ~/.local/bin
wget https://github.com/kahing/goofys/releases/download/v0.24.0/goofys -qO ~/.local/bin/goofys
chmod +x ~/.local/bin/goofys

~/.local/bin/goofys --endpoint https://s3.example.com images /media/images

Links
https://github.com/kahing/goofys/releases
https://gist.github.com/arfon/d93c788b4dcfd12029c8fb0d2fdeeb4c#file-goofys-bash-L8

restic

Install

sudo apt install -y restic

Configure systemd

cat < ~/.config/systemd/user/restic-backup.service
[Unit]
Description=Restic backup service

[Service]
Type=oneshot
Environment=BACKUP_PATHS="vault.pass vault-private.pass .gtkrc-2.0 .bashrc .icons .themes .thunderbird .config .ssh/config"
Environment=BACKUP_EXCLUDES="--exclude=Code --exclude=google-chrome --exclude=VirtualBox --exclude=ImapMail"
Environment=RETENTION_HOURS=8
Environment=RETENTION_DAYS=5
Environment=RETENTION_WEEKS=0
Environment=RETENTION_MONTHS=0
Environment=RETENTION_YEARS=0
Environment=RESTIC_REPOSITORY=%h/Backup
Environment=RESTIC_PASSWORD=sup3r_s3cr3t_r3p0_p4ssw0rd
ExecStart=restic backup --tag systemd.timer $BACKUP_EXCLUDES $BACKUP_PATHS
ExecStartPost=restic forget --tag systemd.timer --group-by "paths,tags" --prune --keep-hourly $RETENTION_HOURS --keep-daily $RETENTION_DAYS 
#--keep-weekly $RETENTION_WEEKS --keep-monthly $RETENTION_MONTHS --keep-yearly $RETENTION_YEARS
ExecStartPost=rsync -aHAXx --numeric-ids --delete --stats --info=progress2 --out-format="[%%t]:%%o:%%f:Last Modified %%M" --verbose -e "ssh -o Compression=no -x" "%h/Backup/" restic@BACKUP_SERVER_FQDN:/home/restic/HOSTNAME/
EOF
<code>

<strong>Links</strong>
https://www.geekbundle.org/backups-mit-restic/

s3cmd

Install s3cmd (s3 CLI client)

sudo apt install -y s3cmd

Configuration file

s3cmd --configure
${HOME}/.s3cfg

CLI

s3cmd mb s3://buk1
s3cmd put /tmp/1m.img s3://buk1
s3cmd ls s3://buk1
s3cmd du
s3cmd rm s3://buk1/1m.img
s3cmd mv s3://buk1/1m.img s3://buk2/1m.img
s3cmd rb s3://buk1
s3cmd rb s3://buk1 --recursive
s3cmd rm s3://lfs/ --recursive --force

# list all objects
s3cmd --host s3.exampl3.com \
  --access_key=${AWS_ACCESS_KEY_ID} \
  --secret_key=${AWS_SECRET_ACCESS_KEY} \
  --host-bucket X \
  la

s3cmd \
  --access_key xxxxxxxxxxx  \
  --secret_key xxxxxxxxxxxxxx \
  --host-bucket '%(bucket)s.s3.example.com' \
  --host s3.example.com \
  --signature-v2 \
  --no-preserve \
  ls s3://foo-duply

s3cmd \
  --host s3.example.com \
  --access_key=${S3_ACCESS_KEY} \
  --secret_key=${S3_SECRET_KEY} \
  --host-bucket X \
  --recursive \
  ls s3://images/ | grep ubuntu-22.04-amd64-disk.raw | tail -1 | cut -d"/" -f2-

ACL

s3cmd setacl s3://bucket/path/to/file --acl-public
s3cmd info s3://bucket/path/to/file
s3cmd setacl s3://bucket/path/to/file --acl-private

Links
https://lollyrock.com/articles/s3cmd-with-radosgw/
https://linuxconfig.org/getting-started-with-aws-s3cmd-command-by-examples

s3fs

Install

sudo apt install -y s3fs

Create credentials

echo ${BUCKET}:${ACCESS_KEY}:${SECRET_KEY} >> ~/.passwd-s3fs
chmod 600 ~/.passwd-s3fs

Mount S3 bucket

s3fs bucket1 /tmp/mnt -o url=https://s3.example.com

Parameter

-o passwd_file=~/.passwd-s3fs
-o url=https://s3.example.com,allow_other,umask=0000
-o use_cache=/tmp/cache

/etc/fstab

mybucket1.mydomain.org /mnt/mybucket1 fuse.s3fs _netdev,allow_other,passwd_file=/home/ftpuser/.passwd-aws-s3fs,default_acl=public-read,uid=1001,gid=65534 0 0

Links
https://github.com/s3fs-fuse/s3fs-fuse
https://gridscale.io/community/tutorials/s3-fuse-ubuntu/

GitLab: Backup to S3

Configure Git

# /etc/gitlab/gitlab.rb
gitlab_rails['backup_upload_connection'] = {
  'provider' => 'AWS',
  'region' => 'foo-west-1',
  'aws_access_key_id' => 'KEY123',
  'aws_secret_access_key' => 'PASS124',
  'endpoint' => 'https://s3.example.com'
}
gitlab_rails['backup_upload_remote_directory'] = 'backups'
gitlab_rails['backup_keep_time'] = 604800

gitlab-ctl reconfigure

Test

gitlab-rake gitlab:backup:create

Configure periodic backup

# crontab -e
0 2 * * * /opt/gitlab/bin/gitlab-rake gitlab:backup:create CRON=1

Links
https://docs.gitlab.com/ce/raketasks/backup_restore.html#uploading-backups-to-a-remote-cloud-storage

GitLab: LFS on S3

Configure GitLab

...
gitlab_rails['lfs_enabled'] = true
gitlab_rails['lfs_object_store_enabled'] = true
gitlab_rails['lfs_object_store_proxy_download'] = true
gitlab_rails['lfs_object_store_remote_directory'] = "lfs"
gitlab_rails['lfs_object_store_connection'] = {
   'provider' => 'AWS',
   'aws_access_key_id' => 'KEY123',
   'aws_secret_access_key' => 'PASS1234',
   'endpoint' => 'https://s3.example.com',
}

gitlab-ctl reconfigure

Install (on client)

apt-get install -y git git-lfs

# create LFS testfile
head -c 1M /dev/urandom > file2.bin

Add LFS file to repository

git lfs install
git lfs track *.bin
# git lfs track "*.bin"
# git lfs track images/
git add .gitattributes *.bin
git commit -m "Commit LFS file"
git push

Fix me
LFS file download / checkout broken

Debug

git lfs ls-files
git lfs env

Links
https://about.gitlab.com/2017/01/30/getting-started-with-git-lfs-tutorial/
http://blog.testdouble.com/posts/2017-01-25-how-and-why-to-use-git-lfs

S3 storage under Linux / Ubuntu

CrossFTP
https://www.crossftp.com/download.htm

wget https://www.crossftp.com/crossftp_1.99.9.deb -qP /tmp/
sudo dpkg -i /tmp/crossftp_*.deb

DragonDisk (s3 GUI manager)
http://www.s3-client.com/

sudo add-apt-repository -y ppa:rock-core/qt4
sudo apt-get install -y libqt4-dbus libqt4-network libqt4-xml libqtcore4 libqtgui4

# installation
wget http://www.s3-client.com/download-amazon-s3-client/dragondisk_1.0.5-0ubuntu_amd64.deb -qO /tmp/dragondisk_1.0.5-0ubuntu_amd64.deb
sudo dpkg -i /tmp/dragondisk_1.0.5-0ubuntu_amd64.deb

Ansible s3 module
https://docs.ansible.com/ansible/latest/modules/aws_s3_module.html
https://linuxhint.com/installing_minio_ubuntu/

Retention
https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config
https://help.switch.ch/engines/documentation/s3-like-object-storage/s3-lifecycle/
https://jahed.dev/2018/05/20/enforcing-retention-policies-on-s3/

wget https://dl.minio.io/client/mc/release/linux-amd64/mc

mc ls mycloud/images
mc mb mycloud/images/2020-02-06/
mc cp amphora-x64-haproxy.raw.bz2 mycloud/images/2020-02-06/

Get credentials for bucket

# ssh cephadm.dev.i.example.com
radosgw-admin bucket list | jq -r '.[]' | egrep -i image | while read bucket; do
    radosgw-admin metadata get "bucket:${bucket}" | jq -c '.data | (.owner, (.bucket | {name, tenant, bucket_id}))'
done

# ssh kolla.dev.i.example.com
openstack project show ${PROJECT_ID}
openstack credential list --type ec2 | grep ${PROJECT_ID}
openstack user show ${USER_ID}

awscli

GitLab registry with S3 storage

Configure CEPH

touch /tmp/placeholder
s3cmd mb s3://gitlab
s3cmd put /tmp/placeholder s3://gitlab
s3cmd ls s3://gitlab
s3cmd du s3://gitlab

Configure GitLab

# /etc/gitlab/gitlab.rb
...
registry_external_url 'https://registry.example.com'
registry_nginx['enable'] = true
registry_nginx['ssl_certificate'] = "/etc/gitlab/ssl/registry.example.com.crt"
registry_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/registry.example.com.key"
#registry['log_level'] = "debug"
#registry['storage_delete_enabled'] = true
...
registry['storage'] = {
  's3' => {
     'accesskey' => 'YOUR_ACCESS_KEY',
     'secretkey' => 'YOUR_SECRET_KEY',
     'bucket'    => 'gitlab',
     'region'    => 'us-west-1',
     'regionendpoint' => 'https://s3.example.com'
  },
  'redirect' => {
    'disable' => true
  }
}

cleanup registry (test):
sudo gitlab-ctl registry-garbage-collect

Links
https://gitlab.com/gitlab-org/gitlab-ce/issues/19356
https://docs.gitlab.com/ce/administration/container_registry.html
https://docs.docker.com/registry/configuration/#storage
https://icicimov.github.io/blog/server/GitLab-server-with-LDAP-and-S3-backend/