The WolfspyreLabs Blog/ November/ Recovering from a data loss hot mess/ Recovering from a data loss hot mess Creating the new pool # Zpool metadata root@pine:~/# zpool add special Wolfspyre mirror /dev/nvme5n1p2 /dev/nvme4n1p2 mirror /dev/nvme3n1p2 /dev/nvme2n1p2 mirror /dev/nvme1m1p2 /dev/nvme0n1p2 root@pine:~/# zpool add Wolfspyre special mirror /dev/nvme5n1p2 /dev/nvme4n1p2 mirror /dev/nvme3n1p2 /dev/nvme2n1p2 mirror /dev/nvme1n1p2 /dev/nvme0n1p2 Create a draid pool root@pine:~/ # zpool create Pine draid1:3d /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1 draid1:3d /dev/sde1 /dev/sdf1 /dev/sdg1 /dev/sdh1 draid1:3d /dev/sdk1 /dev/sdl1 /dev/sdm1 /dev/sdn1 -o failmode=continue -o listsnapshots=on -o autotrim=on -o altroot=/mnt Initialize the NVME partitions for nvme in `seq 0 5`; do echo "nvme${nvme}"; cat /root/new/nvme/bare512log | sed -e "s/FOO/${nvme}/" > /root/new/nvme/nvme${nvme}n1; sfdisk /dev/nvme${nvme}n1 < /root/new/nvme/nvme${nvme}n1; done Add logvols zpool add Pine log \ mirror /dev/nvme0n1p1 /dev/nvme1n1p1 \ mirror /dev/nvme2n1p1 /dev/nvme3n1p1 \ mirror /dev/nvme4n1p1 /dev/nvme5n1p1 \ mirror /dev/nvme5n1p2 /dev/nvme4n1p2 \ mirror /dev/nvme3n1p2 /dev/nvme2n1p2 \ mirror /dev/nvme1n1p2 /dev/nvme0n1p2 add cache partitions to the new pool zpool add Pine cache \ /dev/nvme0n1p7 /dev/nvme1n1p7 /dev/nvme2n1p7 /dev/nvme3n1p7 /dev/nvme4n1p7 /dev/nvme5n1p7 \ /dev/nvme0n1p8 /dev/nvme1n1p8 /dev/nvme2n1p8 /dev/nvme3n1p8 /dev/nvme4n1p8 /dev/nvme5n1p8 \ /dev/nvme0n1p9 /dev/nvme1n1p9 /dev/nvme2n1p9 /dev/nvme3n1p9 /dev/nvme4n1p9 /dev/nvme5n1p9 \ /dev/nvme5n1p10 /dev/nvme4n1p10 /dev/nvme3n1p10 /dev/nvme2n1p10 /dev/nvme1n1p10 /dev/nvme0n1p10 Replacing data # Nfsmount copy targets root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/biggun /mnt/Oak/biggun root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/Fu /mnt/Oak/Fu root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/ck /mnt/Oak/ck root@pine:~/new/nvme# mount -t nfs -o ro 198.18.198.22:/mnt/me /mnt/Oak/me tar copy data # replacing data: tar cd /mnt/Oak/ck/Fuck/proxmoxstorage/.zfs/snapshot/migrate2; tar cpf - . | pv | ( cd /mnt/Pine/proxmoxstorage/ && tar xpf - ) cd /mnt/Oak/me/oakendruidlikepine/tftproot ; tar cpf - . | pv -pbarIT | ( cd /mnt/Pine/tftproot/ && tar xpf - ) cd /mnt/Oak/Fu/Repo/Repo ; tar cpf - .|pv|(cd /mnt/Pine/Repo/ && tar xpf -) cd /mnt/Pine/Backups/Infrastructure/; pv /mnt/Oak/biggun/Shares/Infrastructure.tar | tar xpf - cd /mnt/Pine/Shares/Share; pv /mnt/Oak/biggun/Shares/Share.tar | tar -xf - --strip-components=4 -C . cd /mnt/Pine/OSImages; pv -pbretT /mnt/Oak/me/OSImages.tar | tar xpf - cd /mnt/Pine/Art/; pv /mnt/Oak/biggun/Art.tar | tar xpf - cd /mnt/Pine/homedirs; pv /mnt/Oak/biggun/Homedirs.tar | tar -xf - --strip-components=4 -C . cd /mnt/Pine/Shares/Software; pv /mnt/Oak/biggun/Shares/Software.tar | tar -xf - --strip-components=4 -C . Rsync # replacing data: rsync rsync -ah '/mnt/Oak/me/Shares/Store/Store/.zfs/snapshot/auto-2021-11-10_00-00/' /mnt/Pine/Shares/Cooler/ rsync -ah /mnt/Oak/me/oakendruidlikepine/iTunes/Music/Music/ /mnt/Pine/iTunes/Music/ rsync -ah /mnt/Oak/Fu/iTunes/Movies/ /mnt/Pine/iTunes/Movies/ rsync -ah --info=progress2 /mnt/Oak/ck/iTunes/Movies/ /mnt/Pine/iTunes/Movies/ rsync -ah --info=progress2 /mnt/Oak/ck/Dell/Dell/ /mnt/Pine/OSImages/Dell Rsyncing block dev images # replacing data: rsync blockdevs pv -pbrITEE < /mnt/Oak/me/oakendruidlikepine/zd0.img > /dev/zvol/Pine/PineVMs/Nextcloud/BlueBoxBlockstorage pv -pbrITEE < /mnt/Oak/me/oakendruidlikepine/zd16.img > /dev/zvol/Pine/PineVMs/Nextcloud/WolfspyreNextcloud-nxbd pv -pbrITEE < /mnt/Oak/me/oakendruidlikepine/zd32.img > /dev/zvol/Pine/PineVMs/Nextcloud/NextCloudData Evaluating blocksize #in the hours of panic, I did find This guy’s super cool note-to-self[^supercooltuning] on tuning zfs evaluating block size export _S=/mnt/Oak/me/oakendruidlikepine/Documentation/Documentation export _T=/mnt/Pine/Shares/Documentation export _Tzf=`echo ${_T}|sed -e 's/^\/mnt\///'` export _Sused=`du -sb ${_S}|awk '{print $1}'` echo "Source: ${_S}: ${_Sused}" echo "Dest: ${_T} (zfs fs: ${_Tzf} )" cd ${_S} && time tar cpf - . | pv -bparIT -s ${_Sused} | (cd ${_T} && tar xpf -) && zfs get recordsize,compressratio,used ${_Tzf} && df --block-size=1KiB --output=used,iused,source ${_T} evaluating block size export _S=/mnt/Oak/me/oakendruidlikepine/Drivers/Drivers/.zfs/snapshot/auto-2021-11-10_00-00 export _T=/mnt/Pine/Shares/Drivers export _Tzf=`echo ${_T}|sed -e 's/^\/mnt\///'` export _Sused=`du -sb ${_S}|awk '{print $1}'` echo "Source: ${_S}: ${_Sused}" echo "Dest: ${_T} (zfs fs: ${_Tzf} )" cd ${_S} && time tar cpf - . | pv -bparIT -s ${_Sused} | (cd ${_T} && tar xpf -) && zfs get recordsize,compressratio,used ${_Tzf} && df --block-size=1KiB --output=used,iused,source ${_T} evaluating block size export _S=/mnt/Oak/me/oakendruidlikepine/iTunes/Music/Music export _T=/mnt/Pine/iTunes/Music export _Tzf=`echo ${_T}|sed -e 's/^\/mnt\///'` export _Sused=`du -sb ${_S}|awk '{print $1}'` echo "Source: ${_S}: ${_Sused}" echo "Dest: ${_T} (zfs fs: ${_Tzf} )" wait what the fuck did I type? root@pine:/mnt/Pine/iTunes# history|grep cat 295 pv -pbrITN tarball < homedirs_2014.tgz | zcat | pv -pbrITN Decompressed | tar tpf - . 297 pv -pbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -pbrITN Decompressed | tar xpf - . 299 pv -pbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -pbrN Decompressed | tar xpf - . 300 pv -pbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -N Decompressed | tar xpf - . 301 pv -pcbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cN Decompressed | tar xpf - . 302 pv -pcbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - . 303 pv -cpbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - . 304 pv -cpbrITN Tarball < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed > tar xpf - . 307 pv -cpbrITN ' Tarball' < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - . 308 pv -cpbrITN ' Tarball' < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - . 311 pv -cpbrITN ' Tarball' --size 9686970668 < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - . 313 pv -cpbrITN ' Tarball' --size 9686970668 < homedirs_2014.tgz | zcat -cd | pv -cbrpITN Decompressed | tar xpf - . 425 cat /sys/block/*/queue/nr_requests 470 cat /root/find_subfile_distribution.sh 474 cat /root/fix_nic_buffers.sh 520 cat /sys/module/zfs/parameters/zfs_vdev_scheduler 522 cat /sys/module/zfs/parameters/zfs_max_recordsize 524 cat /sys/module/zfs/parameters/zfs_read_chunk_size 525 cat /sys/module/zfs/parameters/zfs_read_history 528 cat /sys/module/zfs/parameters/zfs_arc_max 529 cat /sys/module/zfs/parameters/l2arc_write_max 592 cat /sys/block/*/queue/scheduler 593 cat /sys/block/*/queue/scheduler 594 cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor 596 cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor wait what the fuck did I type? root@pine:/mnt/Pine/iTunes# history|grep echo 201 if [[ "source `which build_util.sh`;echo $?" == 0 ]]; then info "build_util found: `which build_util.sh`"; fi 203 which build_util.sh; echo $? 204 which build_util.sh >/dev/null; echo $? 205 _BU=`which build_util.sh` ; echo $? 206 _BU=`which build_utila.sh` ; echo $? 207 export _BU=`which build_util.sh` ; echo $? 208 export _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then echo woot;fi 211 export _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";else echo "ERRORexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi build_util.sh not sourcableexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi ";exit 1;fi 212 export _BU=`which abuild_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then (source ${_BU}; info "Build Util found: ${_BU}";else echo "ERRORexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi build_util.sh not sourcableexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi ";exit 1;fi 213 export _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}||exit 1; info "Build Util found: ${_BU}";else echo "ERRORexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi build_util.sh not sourcableexport _BU=`which build_util.sh` ; _RC=$?; if [[ $_RC == 0 ]]; then source ${_BU}; info "Build Util found: ${_BU}";fi ";exit 1;fi 426 for blockdev in `ls /sys/block|egrep -v zd`; do echo ${blockdev};done 427 for blockdev in `ls /sys/block|egrep (zd,dm-|(.*))`; do echo ${blockdev};done 428 for blockdev in `ls /sys/block|egrep -v (zd,dm-|(.*))`; do echo ${blockdev};done 429 for blockdev in `ls /sys/block|egrep -v '(zd,dm-|(.*))'`; do echo ${blockdev};done 430 for blockdev in `ls /sys/block|egrep '(zd,dm-|(.*))'`; do echo ${blockdev};done 431 for blockdev in `ls /sys/block|egrep '(zd,dm-,(.*))'`; do echo ${blockdev};done 432 for blockdev in `ls /sys/block|egrep -v '(zd,dm-,(.*))'`; do echo ${blockdev};done 433 for blockdev in `ls /sys/block|egrep -v 'zd,dm-,(.*)'`; do echo ${blockdev};done 434 for blockdev in `ls /sys/block|egrep -v 'zd,dm-'`; do echo ${blockdev};done 435 for blockdev in `ls /sys/block|egrep -v '(zd,dm-)'`; do echo ${blockdev};done 436 for blockdev in `ls /sys/block|egrep -v '(zd),(dm-)'`; do echo ${blockdev};done 437 for blockdev in `ls /sys/block|egrep -v "(zd),(dm-)"`; do echo ${blockdev};done 438 for blockdev in `ls /sys/block|egrep -v "(zd,dm-)"`; do echo ${blockdev};done 439 for blockdev in `ls /sys/block|egrep -v "zd,dm-"`; do echo ${blockdev};done 440 for blockdev in `ls /sys/block|egrep -v '(zd,dm)`; do echo ${blockdev};done 441 for blockdev in `ls /sys/block|egrep -v '(zd,dm)'`; do echo ${blockdev};done 442 for blockdev in `ls /sys/block`; do echo ${blockdev};done 443 for blockdev in `ls /sys/block|sed -e '/(zd,dm)/d'`; do echo ${blockdev};done 444 for blockdev in `ls /sys/block|sed -e '/zd,dm/d'`; do echo ${blockdev};done 445 for blockdev in `ls /sys/block|sed -ei '/zd,dm/d'`; do echo ${blockdev};done 446 for blockdev in `ls /sys/block|sed -ie '/zd,dm/d'`; do echo ${blockdev};done 447 for blockdev in `ls /sys/block|sed -e '/zd,dm/d'`; do echo ${blockdev};done 448 for blockdev in `ls /sys/block|sed -e '/^zd/d'`; do echo ${blockdev};done 449 for blockdev in `ls /sys/block|sed -e '/^(zd,dm)/d'`; do echo ${blockdev};done 450 for blockdev in `ls /sys/block|sed -e '/^{zd}{dm}/d'`; do echo ${blockdev};done 451 for blockdev in `ls /sys/block|sed -e '/^{zd}/d'`; do echo ${blockdev};done 452 for blockdev in `ls /sys/block|sed -e '/^(zd)/d'`; do echo ${blockdev};done 453 for blockdev in `ls /sys/block|sed -e '/^(z,d)/d'`; do echo ${blockdev};done 454 for blockdev in `ls /sys/block|sed -e '/^z,d/d'`; do echo ${blockdev};done 455 for blockdev in `ls /sys/block|sed -e '/^[z,d]/d'`; do echo ${blockdev};done 456 for blockdev in `ls /sys/block|sed -e '/^[zd][dm]/d'`; do echo ${blockdev};done 457 for blockdev in `ls /sys/block|sed -e '/^[zd][dm][sr0]/d'`; do echo ${blockdev};done 458 for blockdev in `ls /sys/block|sed -e '/^([zd],[dm],[sr0])/d'`; do echo ${blockdev};done 459 for blockdev in `ls /sys/block|sed -e '/^[zd][dm][sr0]/d'`; do echo ${blockdev};done 460 for blockdev in `ls /sys/block|sed -e '/^[zd][dm]||[sr0]/d'`; do echo ${blockdev};done 461 for blockdev in `ls /sys/block|sed -e '/^[zd][dm]/d'`; do echo ${blockdev};done 462 for blockdev in `ls /sys/block|sed -e '/^[zd][m]/d'`; do echo ${blockdev};done 463 for blockdev in `ls /sys/block|sed -e '/^zd/d'`; do echo ${blockdev};done 464 for blockdev in `ls /sys/block|sed -e '/^(zd)/d'`; do echo ${blockdev};done 465 for blockdev in `ls /sys/block|sed -e '/^"zd"/d'`; do echo ${blockdev};done 466 for blockdev in `ls /sys/block|sed -e '/^zd/d' -e /^dm-/d -e/^sr0/d'`; do echo ${blockdev};done 467 for blockdev in `ls /sys/block|sed -e '/^zd/d' -e /^dm\-/d -e/^sr0/d'`; do echo ${blockdev};done 468 for blockdev in `ls /sys/block|sed -e '/^zd/d' -e '/^dm\-/d' -e'/^sr0/d'`; do echo ${blockdev};done 469 for blockdev in `ls /sys/block|sed -e '/^zd/d' -e '/^dm-/d' -e'/^sr0/d'`; do echo ${blockdev};done 499 export _BU=`which build_util.shhhh`; source ${_BU}>/dev/null 2>&1|| echo "Can't find build_util.sh! breaking!";exit -1 500 export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| echo "Can't find build_util.sh! breaking!";exit -1 501 export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| (echo "Can't find build_util.sh! breaking!";exit -1) 502 export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| 'echo "build_util.sh not locatable";exit -1' 503 export _BU=`which build_util.sh`; source ${_BU}>/dev/null 2>&1|| `echo "build_util.sh not locatable";exit -1` 504 echo $PATH 505 export _BU=`which build_util.sh||exit`; source ${_BU}>/dev/null 2>&1|| `echo "build_util.sh not locatable";exit -1` 506 export _BU=`which build_util.sh||exit`&& source ${_BU}>/dev/null 2>&1|| `echo "build_util.sh not locatable";exit -1` 508 echo $PATH 510 echo $PATH 519 echo 4096 > /sys/block/sdo/queue/nr_requests 521 echo deadline > /sys/module/zfs/parameters/zfs_vdev_scheduler 523 echo 8388608 > /sys/module/zfs/parameters/zfs_max_recordsize 530 echo 134217728 > /sys/module/zfs/parameters/l2arc_write_max 553 echo 16777216 > /sys/module/zfs/parameters/zfs_max_recordsize 595 for CPUFREQ in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do [ -f $CPUFREQ ] || continue; echo -n performance > $CPUFREQ; done