↓Chase to main content
  1. 2021/
  2. November/

Recovering from a data loss hot mess

Creating the new pool #

Zpool metadata 
root@pine:~/# zpool add special Wolfspyre mirror /dev/nvme5n1p2 /dev/nvme4n1p2 mirror /dev/nvme3n1p2 /dev/nvme2n1p2 mirror /dev/nvme1m1p2 /dev/nvme0n1p2
root@pine:~/# zpool add Wolfspyre special  mirror /dev/nvme5n1p2 /dev/nvme4n1p2 mirror /dev/nvme3n1p2 /dev/nvme2n1p2 mirror /dev/nvme1n1p2 /dev/nvme0n1p2
Create a draid pool 
root@pine:~/ # zpool create Pine draid1:3d /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1 draid1:3d /dev/sde1 /dev/sdf1 /dev/sdg1 /dev/sdh1 draid1:3d /dev/sdk1 /dev/sdl1 /dev/sdm1 /dev/sdn1 -o failmode=continue -o listsnapshots=on -o autotrim=on -o altroot=/mnt

Initialize the NVME partitions 
for nvme in `seq 0 5`; do echo "nvme${nvme}"; cat /root/new/nvme/bare512log |  sed -e "s/FOO/${nvme}/" > /root/new/nvme/nvme${nvme}n1; sfdisk /dev/nvme${nvme}n1 < /root/new/nvme/nvme${nvme}n1; done
Add logvols 
zpool add Pine log  \
mirror /dev/nvme0n1p1 /dev/nvme1n1p1 \
mirror /dev/nvme2n1p1 /dev/nvme3n1p1 \
mirror /dev/nvme4n1p1 /dev/nvme5n1p1 \
mirror /dev/nvme5n1p2 /dev/nvme4n1p2 \
mirror /dev/nvme3n1p2 /dev/nvme2n1p2 \
mirror /dev/nvme1n1p2 /dev/nvme0n1p2
add cache partitions to the new pool 
zpool add Pine cache \
 /dev/nvme0n1p7 /dev/nvme1n1p7 /dev/nvme2n1p7 /dev/nvme3n1p7  /dev/nvme4n1p7 /dev/nvme5n1p7 \
 /dev/nvme0n1p8 /dev/nvme1n1p8 /dev/nvme2n1p8 /dev/nvme3n1p8  /dev/nvme4n1p8 /dev/nvme5n1p8 \
 /dev/nvme0n1p9 /dev/nvme1n1p9 /dev/nvme2n1p9 /dev/nvme3n1p9  /dev/nvme4n1p9 /dev/nvme5n1p9 \
 /dev/nvme5n1p10 /dev/nvme4n1p10 /dev/nvme3n1p10 /dev/nvme2n1p10 /dev/nvme1n1p10 /dev/nvme0n1p10

Replacing data #

Nfsmount copy targets 
root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/biggun /mnt/Oak/biggun
root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/Fu /mnt/Oak/Fu
root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/ck /mnt/Oak/ck
root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/me /mnt/Oak/me

tar copy data #

replacing data: tar 
cd /mnt/Oak/ck/Fuck/proxmoxstorage/.zfs/snapshot/migrate2;  tar cpf - . | pv | ( cd /mnt/Pine/proxmoxstorage/ && tar xpf - )
cd /mnt/Oak/me/oakendruidlikepine/tftproot ;  tar cpf - . | pv -pbarIT | ( cd /mnt/Pine/tftproot/ && tar xpf - )
cd /mnt/Oak/Fu/Repo/Repo ; tar cpf - .|pv|(cd /mnt/Pine/Repo/ && tar xpf -)
cd /mnt/Pine/Backups/Infrastructure/; pv /mnt/Oak/biggun/Shares/Infrastructure.tar | tar xpf -
cd /mnt/Pine/Shares/Share; pv /mnt/Oak/biggun/Shares/Share.tar | tar -xf - --strip-components=4 -C .
cd /mnt/Pine/OSImages; pv -pbretT /mnt/Oak/me/OSImages.tar | tar xpf -
cd /mnt/Pine/Art/; pv /mnt/Oak/biggun/Art.tar | tar xpf -
cd /mnt/Pine/homedirs; pv  /mnt/Oak/biggun/Homedirs.tar | tar -xf  - --strip-components=4  -C .
cd /mnt/Pine/Shares/Software; pv  /mnt/Oak/biggun/Shares/Software.tar | tar -xf  - --strip-components=4  -C .

Rsync #

replacing data: rsync 
rsync -ah '/mnt/Oak/me/Shares/Store/Store/.zfs/snapshot/auto-2021-11-10_00-00/' /mnt/Pine/Shares/Cooler/
rsync -ah /mnt/Oak/me/oakendruidlikepine/iTunes/Music/Music/ /mnt/Pine/iTunes/Music/
rsync -ah /mnt/Oak/Fu/iTunes/Movies/ /mnt/Pine/iTunes/Movies/
rsync -ah --info=progress2 /mnt/Oak/ck/iTunes/Movies/ /mnt/Pine/iTunes/Movies/
rsync -ah --info=progress2 /mnt/Oak/ck/Dell/Dell/ /mnt/Pine/OSImages/Dell

Rsyncing block dev images #

replacing data: rsync blockdevs 
pv -pbrITEE < /mnt/Oak/me/oakendruidlikepine/zd0.img > /dev/zvol/Pine/PineVMs/Nextcloud/BlueBoxBlockstorage
pv -pbrITEE < /mnt/Oak/me/oakendruidlikepine/zd16.img > /dev/zvol/Pine/PineVMs/Nextcloud/WolfspyreNextcloud-nxbd
pv -pbrITEE < /mnt/Oak/me/oakendruidlikepine/zd32.img > /dev/zvol/Pine/PineVMs/Nextcloud/NextCloudData

Evaluating blocksize #

in the hours of panic, I did find This guy’s super cool note-to-self[^supercooltuning] on tuning zfs

evaluating block size 
export _S=/mnt/Oak/me/oakendruidlikepine/Documentation/Documentation 
export _T=/mnt/Pine/Shares/Documentation
export _Tzf=`echo ${_T}|sed -e 's/^\/mnt\///'`
export _Sused=`du -sb ${_S}|awk '{print $1}'`
echo "Source: ${_S}: ${_Sused}"
echo "Dest: ${_T} (zfs fs: ${_Tzf} )"
cd ${_S} && time tar cpf - . | pv -bparIT -s ${_Sused} | (cd ${_T} && tar xpf -) && zfs get recordsize,compressratio,used ${_Tzf} && df --block-size=1KiB --output=used,iused,source ${_T}

evaluating block size 
export _S=/mnt/Oak/me/oakendruidlikepine/Drivers/Drivers/.zfs/snapshot/auto-2021-11-10_00-00
export _T=/mnt/Pine/Shares/Drivers

export _Tzf=`echo ${_T}|sed -e 's/^\/mnt\///'`
export _Sused=`du -sb ${_S}|awk '{print $1}'`
echo "Source: ${_S}: ${_Sused}"
echo "Dest: ${_T} (zfs fs: ${_Tzf} )"
cd ${_S} && time tar cpf - . | pv -bparIT -s ${_Sused} | (cd ${_T} && tar xpf -) && zfs get recordsize,compressratio,used ${_Tzf} && df --block-size=1KiB --output=used,iused,source ${_T}
evaluating block size 
export _S=/mnt/Oak/me/oakendruidlikepine/iTunes/Music/Music
export _T=/mnt/Pine/iTunes/Music
export _Tzf=`echo ${_T}|sed -e 's/^\/mnt\///'`
export _Sused=`du -sb ${_S}|awk '{print $1}'`
echo "Source: ${_S}: ${_Sused}"
echo "Dest: ${_T} (zfs fs: ${_Tzf} )"
wait what the fuck did I type? 
root@pine:/mnt/Pine/iTunes# history|grep cat
  295   pv -pbrITN tarball < homedirs_2014.tgz | zcat | pv -pbrITN Decompressed | tar tpf - .
  297   pv -pbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -pbrITN Decompressed | tar xpf - .
  299   pv -pbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -pbrN Decompressed | tar xpf - .
  300   pv -pbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -N Decompressed | tar xpf - .
  301   pv -pcbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cN Decompressed | tar xpf - .
  302   pv -pcbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - .
  303   pv -cpbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - .
  304   pv -cpbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed > tar xpf - .
  307   pv -cpbrITN '   Tarball' < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - .
  308   pv -cpbrITN '     Tarball' < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - .
  311   pv -cpbrITN '     Tarball' --size 9686970668 < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - .
  313   pv -cpbrITN '     Tarball' --size 9686970668 < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - .
  425  cat /sys/block/*/queue/nr_requests
  470  cat /root/find_subfile_distribution.sh
  474  cat /root/fix_nic_buffers.sh
  520  cat /sys/module/zfs/parameters/zfs_vdev_scheduler
  522  cat /sys/module/zfs/parameters/zfs_max_recordsize
  524  cat /sys/module/zfs/parameters/zfs_read_chunk_size
  525  cat /sys/module/zfs/parameters/zfs_read_history
  528  cat /sys/module/zfs/parameters/zfs_arc_max
  529  cat /sys/module/zfs/parameters/l2arc_write_max
  592  cat /sys/block/*/queue/scheduler
  593  cat /sys/block/*/queue/scheduler
  594  cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
  596  cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

wait what the fuck did I type? 
root@pine:/mnt/Pine/iTunes# history|grep echo
  201  if [[ "source `which build_util.sh`;echo $?" == 0 ]]; then info "build_util found: `which build_util.sh`"; fi
  203  which build_util.sh; echo $?
  204  which build_util.sh >/dev/null; echo $?
  205  _BU=`which build_util.sh` ; echo $?
  206  _BU=`which build_utila.sh` ; echo $?
  207  export _BU=`which build_util.sh` ; echo $?
  208  export _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then echo woot;fi
  211  export _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";else echo "ERRORexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi   build_util.sh not sourcableexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi  ";exit 1;fi
  212  export _BU=`which abuild_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then (source ${_BU}; info "Build Util found: ${_BU}";else echo "ERRORexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi   build_util.sh not sourcableexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi  ";exit 1;fi
  213  export _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}||exit 1; info "Build Util found: ${_BU}";else echo "ERRORexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi   build_util.sh not sourcableexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi  ";exit 1;fi
  426  for blockdev in `ls /sys/block|egrep -v zd`; do echo ${blockdev};done
  427  for blockdev in `ls /sys/block|egrep  (zd,dm-|(.*))`; do echo ${blockdev};done
  428  for blockdev in `ls /sys/block|egrep  -v (zd,dm-|(.*))`; do echo ${blockdev};done
  429  for blockdev in `ls /sys/block|egrep  -v '(zd,dm-|(.*))'`; do echo ${blockdev};done
  430  for blockdev in `ls /sys/block|egrep  '(zd,dm-|(.*))'`; do echo ${blockdev};done
  431  for blockdev in `ls /sys/block|egrep  '(zd,dm-,(.*))'`; do echo ${blockdev};done
  432  for blockdev in `ls /sys/block|egrep  -v '(zd,dm-,(.*))'`; do echo ${blockdev};done
  433  for blockdev in `ls /sys/block|egrep  -v 'zd,dm-,(.*)'`; do echo ${blockdev};done
  434  for blockdev in `ls /sys/block|egrep  -v 'zd,dm-'`; do echo ${blockdev};done
  435  for blockdev in `ls /sys/block|egrep  -v '(zd,dm-)'`; do echo ${blockdev};done
  436  for blockdev in `ls /sys/block|egrep  -v '(zd),(dm-)'`; do echo ${blockdev};done
  437  for blockdev in `ls /sys/block|egrep  -v "(zd),(dm-)"`; do echo ${blockdev};done
  438  for blockdev in `ls /sys/block|egrep  -v "(zd,dm-)"`; do echo ${blockdev};done
  439  for blockdev in `ls /sys/block|egrep  -v "zd,dm-"`; do echo ${blockdev};done
  440  for blockdev in `ls /sys/block|egrep  -v '(zd,dm)`; do echo ${blockdev};done
  441  for blockdev in `ls /sys/block|egrep  -v '(zd,dm)'`; do echo ${blockdev};done
  442  for blockdev in `ls /sys/block`; do echo ${blockdev};done
  443  for blockdev in `ls /sys/block|sed -e '/(zd,dm)/d'`; do echo ${blockdev};done
  444  for blockdev in `ls /sys/block|sed -e '/zd,dm/d'`; do echo ${blockdev};done
  445  for blockdev in `ls /sys/block|sed -ei '/zd,dm/d'`; do echo ${blockdev};done
  446  for blockdev in `ls /sys/block|sed -ie '/zd,dm/d'`; do echo ${blockdev};done
  447  for blockdev in `ls /sys/block|sed -e '/zd,dm/d'`; do echo ${blockdev};done
  448  for blockdev in `ls /sys/block|sed -e '/^zd/d'`; do echo ${blockdev};done
  449  for blockdev in `ls /sys/block|sed -e '/^(zd,dm)/d'`; do echo ${blockdev};done
  450  for blockdev in `ls /sys/block|sed -e '/^{zd}{dm}/d'`; do echo ${blockdev};done
  451  for blockdev in `ls /sys/block|sed -e '/^{zd}/d'`; do echo ${blockdev};done
  452  for blockdev in `ls /sys/block|sed -e '/^(zd)/d'`; do echo ${blockdev};done
  453  for blockdev in `ls /sys/block|sed -e '/^(z,d)/d'`; do echo ${blockdev};done
  454  for blockdev in `ls /sys/block|sed -e '/^z,d/d'`; do echo ${blockdev};done
  455  for blockdev in `ls /sys/block|sed -e '/^[z,d]/d'`; do echo ${blockdev};done
  456  for blockdev in `ls /sys/block|sed -e '/^[zd][dm]/d'`; do echo ${blockdev};done
  457  for blockdev in `ls /sys/block|sed -e '/^[zd][dm][sr0]/d'`; do echo ${blockdev};done
  458  for blockdev in `ls /sys/block|sed -e '/^([zd],[dm],[sr0])/d'`; do echo ${blockdev};done
  459  for blockdev in `ls /sys/block|sed -e '/^[zd][dm][sr0]/d'`; do echo ${blockdev};done
  460  for blockdev in `ls /sys/block|sed -e '/^[zd][dm]||[sr0]/d'`; do echo ${blockdev};done
  461  for blockdev in `ls /sys/block|sed -e '/^[zd][dm]/d'`; do echo ${blockdev};done
  462  for blockdev in `ls /sys/block|sed -e '/^[zd][m]/d'`; do echo ${blockdev};done
  463  for blockdev in `ls /sys/block|sed -e '/^zd/d'`; do echo ${blockdev};done
  464  for blockdev in `ls /sys/block|sed -e '/^(zd)/d'`; do echo ${blockdev};done
  465  for blockdev in `ls /sys/block|sed -e '/^"zd"/d'`; do echo ${blockdev};done
  466  for blockdev in `ls /sys/block|sed -e '/^zd/d' -e /^dm-/d -e/^sr0/d'`; do echo ${blockdev};done
  467  for blockdev in `ls /sys/block|sed -e '/^zd/d' -e /^dm\-/d -e/^sr0/d'`; do echo ${blockdev};done
  468  for blockdev in `ls /sys/block|sed -e '/^zd/d' -e '/^dm\-/d' -e'/^sr0/d'`; do echo ${blockdev};done
  469  for blockdev in `ls /sys/block|sed -e '/^zd/d' -e '/^dm-/d' -e'/^sr0/d'`; do echo ${blockdev};done
  499  export _BU=`which build_util.shhhh`; source ${_BU}>/dev/null 2>&1|| echo "Can't find build_util.sh! breaking!";exit -1
  500  export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| echo "Can't find build_util.sh! breaking!";exit -1
  501  export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| (echo "Can't find build_util.sh! breaking!";exit -1)
  502  export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| 'echo "build_util.sh not locatable";exit -1'
  503  export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| `echo "build_util.sh not locatable";exit -1`
  504  echo $PATH
  505  export _BU=`which build_util.sh||exit`; source ${_BU}>/dev/null 2>&1|| `echo "build_util.sh not locatable";exit -1`
  506  export _BU=`which build_util.sh||exit`&& source ${_BU}>/dev/null 2>&1|| `echo "build_util.sh not locatable";exit -1`
  508  echo $PATH
  510  echo $PATH
  519  echo 4096 > /sys/block/sdo/queue/nr_requests
  521  echo deadline > /sys/module/zfs/parameters/zfs_vdev_scheduler
  523  echo 8388608 > /sys/module/zfs/parameters/zfs_max_recordsize
  530  echo 134217728 >  /sys/module/zfs/parameters/l2arc_write_max
  553  echo  16777216 > /sys/module/zfs/parameters/zfs_max_recordsize
  595  for CPUFREQ in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do     [ -f $CPUFREQ ] || continue;     echo -n performance > $CPUFREQ; done