From b368b27faa179841735dfcb136e95e11fef201b2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 28 Jun 2019 18:06:43 -0600 Subject: [PATCH 001/324] New project organization * Script work is brought into better focus * New layout will be used to better package the python scripts * No hidden folders (they will be hidden at build time) * All building should be done under setup/ * Avoids ambiguous .bin/.cbin folders at root of project --- .../include/syslinux/splash.png => docs/TODO | 0 {Images => images}/ConEmu.png | Bin {Images => images}/Linux.png | Bin {Images => images}/Pxelinux.png | Bin {Images => images}/Syslinux.png | Bin {Images => images}/WinPE.jpg | Bin {Images => images}/WizardHat.xcf | Bin {Images => images}/logo.svg | 0 {Images => images}/rEFInd.png | Bin {.bin/Scripts => scripts}/Copy WizardKit.cmd | 0 {.bin/Scripts => scripts}/Launch.cmd | 0 {.bin/Scripts => scripts}/Launcher_Template.cmd | 0 {.bin/Scripts => scripts}/add-known-networks | 0 {.bin/Scripts => scripts}/apple-fans | 0 {.bin/Scripts => scripts}/borrowed/acpi.py | 0 .../borrowed/knownpaths-LICENSE.txt | 0 {.bin/Scripts => scripts}/borrowed/knownpaths.py | 0 {.bin/Scripts => scripts}/borrowed/set-eol.ps1 | 0 {.bin/Scripts => scripts}/build_kit.ps1 | 0 {.bin/Scripts => scripts}/build_pe.ps1 | 0 {.bin/Scripts => scripts}/echo-and-hold | 0 {.bin/Scripts => scripts}/init_client_dir.cmd | 0 {.bin/Scripts => scripts}/launch-in-tmux | 0 {.bin/Scripts => scripts}/mount-raw-image | 0 {.bin/Scripts => scripts}/msword-search | 0 .../outer_scripts_to_review}/activate.py | 0 .../outer_scripts_to_review}/build-ufd | 0 .../outer_scripts_to_review}/cbs_fix.py | 0 .../outer_scripts_to_review}/check_disk.py | 0 .../outer_scripts_to_review}/ddrescue-tui | 0 .../outer_scripts_to_review}/ddrescue-tui-menu | 0 .../outer_scripts_to_review}/dism.py | 0 .../outer_scripts_to_review}/hw-diags | 0 .../outer_scripts_to_review}/hw-diags-audio | 0 .../outer_scripts_to_review}/hw-diags-iobenchmark | 0 .../outer_scripts_to_review}/hw-diags-menu | 0 .../outer_scripts_to_review}/hw-diags-network | 0 .../outer_scripts_to_review}/hw-diags-prime95 | 0 .../outer_scripts_to_review}/hw-drive-info | 0 .../outer_scripts_to_review}/hw-info | 0 .../outer_scripts_to_review}/hw-sensors | 0 .../outer_scripts_to_review}/hw-sensors-monitor | 0 .../outer_scripts_to_review}/install_sw_bundle.py | 0 .../outer_scripts_to_review}/install_vcredists.py | 0 .../outer_scripts_to_review}/mount-all-volumes | 0 .../outer_scripts_to_review}/mount-backup-shares | 0 .../outer_scripts_to_review}/safemode_enter.py | 0 .../outer_scripts_to_review}/safemode_exit.py | 0 .../outer_scripts_to_review}/sfc_scan.py | 0 .../outer_scripts_to_review}/system_diagnostics.py | 0 .../outer_scripts_to_review}/system_setup.py | 0 .../outer_scripts_to_review}/transferred_keys.py | 0 .../outer_scripts_to_review}/update_kit.py | 0 .../outer_scripts_to_review}/user_data_transfer.py | 0 .../outer_scripts_to_review}/windows_updates.py | 0 .../outer_scripts_to_review}/winpe_root_menu.py | 0 {.bin/Scripts => scripts}/pacinit | 0 {.bin/Scripts => scripts}/photorec-sort | 0 {.bin/Scripts => scripts}/remount-rw | 0 {.bin/Scripts => scripts}/wk-power-command | 0 {.bin/Scripts => scripts/wk.prev}/debug/hw_diags.py | 0 .../wk.prev}/functions/activation.py | 0 .../Scripts => scripts/wk.prev}/functions/backup.py | 0 .../wk.prev}/functions/browsers.py | 0 .../wk.prev}/functions/cleanup.py | 0 .../Scripts => scripts/wk.prev}/functions/common.py | 0 {.bin/Scripts => scripts/wk.prev}/functions/data.py | 0 .../wk.prev}/functions/ddrescue.py | 0 {.bin/Scripts => scripts/wk.prev}/functions/disk.py | 0 .../wk.prev}/functions/hw_diags.py | 0 {.bin/Scripts => scripts/wk.prev}/functions/info.py | 0 {.bin/Scripts => scripts/wk.prev}/functions/json.py | 0 .../wk.prev}/functions/network.py | 0 .../wk.prev}/functions/product_keys.py | 0 .../wk.prev}/functions/repairs.py | 0 .../wk.prev}/functions/safemode.py | 0 .../wk.prev}/functions/sensors.py | 0 .../Scripts => scripts/wk.prev}/functions/setup.py | 0 .../wk.prev}/functions/sw_diags.py | 0 .../wk.prev}/functions/threading.py | 0 {.bin/Scripts => scripts/wk.prev}/functions/tmux.py | 0 {.bin/Scripts => scripts/wk.prev}/functions/ufd.py | 0 .../Scripts => scripts/wk.prev}/functions/update.py | 0 .../wk.prev}/functions/windows_setup.py | 0 .../wk.prev}/functions/windows_updates.py | 0 .../wk.prev}/functions/winpe_menus.py | 0 .../wk.prev}/settings/browsers.py | 0 .../Scripts => scripts/wk.prev}/settings/cleanup.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/data.py | 0 .../wk.prev}/settings/ddrescue.py | 0 .../wk.prev}/settings/hw_diags.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/info.py | 0 .../wk.prev}/settings/launchers.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/main.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/music.py | 0 .../wk.prev}/settings/partition_uids.py | 0 .../Scripts => scripts/wk.prev}/settings/sensors.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/setup.py | 0 .../Scripts => scripts/wk.prev}/settings/sources.py | 0 .../wk.prev}/settings/sw_diags.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/tools.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/ufd.py | 0 .../wk.prev}/settings/windows_builds.py | 0 .../wk.prev}/settings/windows_setup.py | 0 {.bin/Scripts => scripts/wk.prev}/settings/winpe.py | 0 .../etc/skel/.Xauthority => scripts/wk/exe.py | 0 scripts/wk/io.py | 0 scripts/wk/net.py | 0 scripts/wk/std.py | 0 Build Linux => setup/build_linux | 0 Build PE.cmd => setup/build_pe.cmd | 0 Build Kit.cmd => setup/build_windows.cmd | 0 {.linux_items => setup/linux}/authorized_keys | 0 {.linux_items => setup/linux}/build_additions.txt | 0 .../linux}/include/EFI/boot/icons/dgpu.png | Bin .../linux}/include/EFI/boot/icons/wk_arch.png | Bin .../linux}/include/EFI/boot/icons/wk_memtest.png | Bin .../linux}/include/EFI/boot/icons/wk_win.png | Bin .../linux}/include/EFI/boot/refind.conf | 0 .../linux}/include/EFI/boot/selection_big.png | Bin .../linux}/include/EFI/boot/selection_small.png | Bin .../linux}/include/airootfs/etc/default/ufw | 0 .../linux}/include/airootfs/etc/hostname | 0 .../linux}/include/airootfs/etc/hosts | 0 .../linux}/include/airootfs/etc/locale.conf | 0 .../linux}/include/airootfs/etc/locale.gen | 0 .../linux}/include/airootfs/etc/motd | 0 .../etc/polkit-1/rules.d/49-nopasswd_global.rules | 0 .../linux}/include/airootfs/etc/skel/.aliases | 0 .../linux}/include/airootfs/etc/skel/.bashrc | 0 .../linux}/include/airootfs/etc/skel/.dircolors | 0 .../include/airootfs/etc/skel/.rsync_exclusions | 0 .../linux}/include/airootfs/etc/skel/.tmux.conf | 0 .../include/airootfs/etc/skel/.update_network | 0 .../linux}/include/airootfs/etc/skel/.vimrc | 0 .../linux}/include/airootfs/etc/skel/.zlogin | 0 .../linux}/include/airootfs/etc/skel/.zshrc | 0 .../multi-user.target.wants/NetworkManager.service | 0 .../system/multi-user.target.wants/rngd.service | 0 .../system/multi-user.target.wants/sshd.service | 0 .../systemd-timesyncd.service | 0 .../system/multi-user.target.wants/ufw.service | 0 .../airootfs/etc/udev/rules.d/99-udisks2.rules | 0 .../linux}/include/airootfs/etc/udevil/udevil.conf | 0 .../linux}/include/airootfs/etc/ufw/ufw.conf | 0 .../linux}/include/airootfs/etc/ufw/user.rules | 0 .../linux}/include/airootfs/etc/ufw/user6.rules | 0 .../linux}/include/airootfs/etc/vconsole.conf | 0 .../linux}/include/isolinux/isolinux.cfg | 0 setup/linux/include/syslinux/splash.png | 0 .../linux}/include/syslinux/syslinux.cfg | 0 .../linux}/include/syslinux/wk.cfg | 0 .../linux}/include/syslinux/wk_hdt.cfg | 0 .../linux}/include/syslinux/wk_head.cfg | 0 .../linux}/include/syslinux/wk_iso.cfg | 0 .../linux}/include/syslinux/wk_iso_linux.cfg | 0 .../linux}/include/syslinux/wk_pxe.cfg | 0 .../linux}/include/syslinux/wk_pxe_linux.cfg | 0 .../linux}/include/syslinux/wk_pxe_winpe.cfg | 0 .../linux}/include/syslinux/wk_sys.cfg | 0 .../linux}/include/syslinux/wk_sys_linux.cfg | 0 .../linux}/include/syslinux/wk_sys_winpe.cfg | 0 .../linux}/include/syslinux/wk_tail.cfg | 0 .../linux}/include_x/airootfs/etc/oblogout.conf | 0 setup/linux/include_x/airootfs/etc/skel/.Xauthority | 0 .../linux}/include_x/airootfs/etc/skel/.Xresources | 0 .../airootfs/etc/skel/.config/Thunar/accels.scm | 0 .../airootfs/etc/skel/.config/Thunar/uca.xml | 0 .../airootfs/etc/skel/.config/dunst/dunstrc | 0 .../airootfs/etc/skel/.config/gtk-3.0/settings.ini | 0 .../include_x/airootfs/etc/skel/.config/i3/config | 0 .../airootfs/etc/skel/.config/i3status/config | 0 .../airootfs/etc/skel/.config/mimeapps.list | 0 .../airootfs/etc/skel/.config/openbox/autostart | 0 .../airootfs/etc/skel/.config/openbox/environment | 0 .../airootfs/etc/skel/.config/openbox/menu.xml | 0 .../airootfs/etc/skel/.config/openbox/rc.xml | 0 .../include_x/airootfs/etc/skel/.config/rofi/config | 0 .../user/timers.target.wants/update-conky.timer | 0 .../skel/.config/systemd/user/update-conky.service | 0 .../skel/.config/systemd/user/update-conky.timer | 0 .../airootfs/etc/skel/.config/tint2/tint2rc | 0 .../airootfs/etc/skel/.config/volumeicon/volumeicon | 0 .../linux}/include_x/airootfs/etc/skel/.conky_start | 0 .../include_x/airootfs/etc/skel/.conkyrc_base | 0 .../linux}/include_x/airootfs/etc/skel/.gtkrc-2.0 | 0 .../include_x/airootfs/etc/skel/.start_desktop_apps | 0 .../include_x/airootfs/etc/skel/.update_conky | 0 .../linux}/include_x/airootfs/etc/skel/.update_x | 0 .../linux}/include_x/airootfs/etc/skel/.wallpaper | 0 .../linux}/include_x/airootfs/etc/skel/.xinitrc | 0 .../linux}/include_x/airootfs/etc/skel/.zlogin | 0 .../share/applications/Hardware Diagnostics.desktop | 0 .../share/applications/Hardware Information.desktop | 0 .../usr/share/applications/NetworkTest.desktop | 0 {.linux_items => setup/linux}/known_networks | 0 {.linux_items => setup/linux}/packages/aur | 0 {.linux_items => setup/linux}/packages/dependencies | 0 {.linux_items => setup/linux}/packages/live_add | 0 {.linux_items => setup/linux}/packages/live_add_min | 0 {.linux_items => setup/linux}/packages/live_add_x | 0 {.linux_items => setup/linux}/packages/live_remove | 0 {.pe_items => setup/pe}/System32/Winpeshl.ini | 0 {.pe_items => setup/pe}/System32/menu.cmd | 0 {.pe_items/_include => setup/pe/bin}/CPU-Z/cpuz.ini | 0 .../_include => setup/pe/bin}/ConEmu/ConEmu.xml | 0 .../_include => setup/pe/bin}/HWiNFO/HWiNFO.INI | 0 .../pe/bin}/NotepadPlusPlus/config.xml | 0 .../pe/bin}/NotepadPlusPlus/npp.cmd | 0 .../pe/bin}/NotepadPlusPlus/stylers.model.xml | 0 .../_include => setup/pe/bin}/Q-Dir/Q-Dir.ini | 0 .../windows}/Drivers/Extras/AMD.url | 0 .../Extras/Dell (FTP - Browse for Drivers).url | 0 .../Drivers/Extras/Dell (Simplified Interface).url | 0 .../windows}/Drivers/Extras/Dell (Support Site).url | 0 .../windows}/Drivers/Extras/Device Remover.url | 0 .../Drivers/Extras/Display Driver Uninstaller.url | 0 {.kit_items => setup/windows}/Drivers/Extras/HP.url | 0 .../Extras/Intel Driver & Support Assistant.url | 0 .../windows}/Drivers/Extras/NVIDIA.url | 0 .../Drivers/Extras/Samsung Tools & Software.url | 0 .../windows}/Installers/BackBlaze.url | 0 .../windows}/Misc/Fix Missing Optical Drive.reg | 0 .../windows}/Misc/Nirsoft Utilities - Outlook.url | 0 .../windows}/Misc/Nirsoft Utilities - Passwords.url | 0 .../windows}/Misc/Sysinternals Suite (Live).url | 0 .../AV Removal Tools/AV Removal Tools.url | 0 .../windows}/Uninstallers/AV Removal Tools/AVG.url | 0 .../Uninstallers/AV Removal Tools/Avast.url | 0 .../Uninstallers/AV Removal Tools/Avira.url | 0 .../windows}/Uninstallers/AV Removal Tools/ESET.url | 0 .../Uninstallers/AV Removal Tools/Kaspersky.url | 0 .../windows}/Uninstallers/AV Removal Tools/MBAM.url | 0 .../Uninstallers/AV Removal Tools/McAfee.url | 0 .../Uninstallers/AV Removal Tools/Norton.url | 0 {.bin => setup/windows/bin}/ConEmu/ConEmu.xml | 0 {.bin => setup/windows/bin}/HWiNFO/general.ini | 0 {.bin => setup/windows/bin}/_Drivers/SDIO/sdi.cfg | 0 .../windows/cbin}/_include/AIDA64/full.rpf | 0 .../cbin}/_include/AIDA64/installed_programs.rpf | 0 .../windows/cbin}/_include/AIDA64/licenses.rpf | 0 .../windows/cbin}/_include/BleachBit/BleachBit.ini | 0 .../cbin}/_include/NotepadPlusPlus/config.xml | 0 .../windows/cbin}/_include/XMPlay/xmplay.ini | 0 .../cbin}/_include/XYplorerFree/Data/XYplorer.ini | 0 .../_include/_Drivers/Intel RST/SetupRST_13.x.txt | 0 .../windows/cbin}/_include/_Office/2016_hb_32.xml | 0 .../windows/cbin}/_include/_Office/2016_hb_64.xml | 0 .../windows/cbin}/_include/_Office/2016_hs_32.xml | 0 .../windows/cbin}/_include/_Office/2016_hs_64.xml | 0 .../windows/cbin}/_include/_Office/2019_hb_32.xml | 0 .../windows/cbin}/_include/_Office/2019_hb_64.xml | 0 .../windows/cbin}/_include/_Office/2019_hs_32.xml | 0 .../windows/cbin}/_include/_Office/2019_hs_64.xml | 0 .../windows/cbin}/_include/_Office/365_32.xml | 0 .../windows/cbin}/_include/_Office/365_64.xml | 0 .../cbin}/_include/_vcredists/InstallAll.bat | 0 257 files changed, 0 insertions(+), 0 deletions(-) rename .linux_items/include/syslinux/splash.png => docs/TODO (100%) rename {Images => images}/ConEmu.png (100%) rename {Images => images}/Linux.png (100%) rename {Images => images}/Pxelinux.png (100%) rename {Images => images}/Syslinux.png (100%) rename {Images => images}/WinPE.jpg (100%) rename {Images => images}/WizardHat.xcf (100%) rename {Images => images}/logo.svg (100%) rename {Images => images}/rEFInd.png (100%) rename {.bin/Scripts => scripts}/Copy WizardKit.cmd (100%) rename {.bin/Scripts => scripts}/Launch.cmd (100%) rename {.bin/Scripts => scripts}/Launcher_Template.cmd (100%) rename {.bin/Scripts => scripts}/add-known-networks (100%) rename {.bin/Scripts => scripts}/apple-fans (100%) rename {.bin/Scripts => scripts}/borrowed/acpi.py (100%) rename {.bin/Scripts => scripts}/borrowed/knownpaths-LICENSE.txt (100%) rename {.bin/Scripts => scripts}/borrowed/knownpaths.py (100%) rename {.bin/Scripts => scripts}/borrowed/set-eol.ps1 (100%) rename {.bin/Scripts => scripts}/build_kit.ps1 (100%) rename {.bin/Scripts => scripts}/build_pe.ps1 (100%) rename {.bin/Scripts => scripts}/echo-and-hold (100%) rename {.bin/Scripts => scripts}/init_client_dir.cmd (100%) rename {.bin/Scripts => scripts}/launch-in-tmux (100%) rename {.bin/Scripts => scripts}/mount-raw-image (100%) rename {.bin/Scripts => scripts}/msword-search (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/activate.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/build-ufd (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/cbs_fix.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/check_disk.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/ddrescue-tui (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/ddrescue-tui-menu (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/dism.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-diags (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-diags-audio (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-diags-iobenchmark (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-diags-menu (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-diags-network (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-diags-prime95 (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-drive-info (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-info (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-sensors (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/hw-sensors-monitor (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/install_sw_bundle.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/install_vcredists.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/mount-all-volumes (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/mount-backup-shares (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/safemode_enter.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/safemode_exit.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/sfc_scan.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/system_diagnostics.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/system_setup.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/transferred_keys.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/update_kit.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/user_data_transfer.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/windows_updates.py (100%) rename {.bin/Scripts => scripts/outer_scripts_to_review}/winpe_root_menu.py (100%) rename {.bin/Scripts => scripts}/pacinit (100%) rename {.bin/Scripts => scripts}/photorec-sort (100%) rename {.bin/Scripts => scripts}/remount-rw (100%) rename {.bin/Scripts => scripts}/wk-power-command (100%) rename {.bin/Scripts => scripts/wk.prev}/debug/hw_diags.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/activation.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/backup.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/browsers.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/cleanup.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/common.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/data.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/ddrescue.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/disk.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/hw_diags.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/info.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/json.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/network.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/product_keys.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/repairs.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/safemode.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/sensors.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/setup.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/sw_diags.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/threading.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/tmux.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/ufd.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/update.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/windows_setup.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/windows_updates.py (100%) rename {.bin/Scripts => scripts/wk.prev}/functions/winpe_menus.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/browsers.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/cleanup.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/data.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/ddrescue.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/hw_diags.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/info.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/launchers.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/main.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/music.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/partition_uids.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/sensors.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/setup.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/sources.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/sw_diags.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/tools.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/ufd.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/windows_builds.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/windows_setup.py (100%) rename {.bin/Scripts => scripts/wk.prev}/settings/winpe.py (100%) rename .linux_items/include_x/airootfs/etc/skel/.Xauthority => scripts/wk/exe.py (100%) create mode 100644 scripts/wk/io.py create mode 100644 scripts/wk/net.py create mode 100644 scripts/wk/std.py rename Build Linux => setup/build_linux (100%) rename Build PE.cmd => setup/build_pe.cmd (100%) rename Build Kit.cmd => setup/build_windows.cmd (100%) rename {.linux_items => setup/linux}/authorized_keys (100%) rename {.linux_items => setup/linux}/build_additions.txt (100%) rename {.linux_items => setup/linux}/include/EFI/boot/icons/dgpu.png (100%) rename {.linux_items => setup/linux}/include/EFI/boot/icons/wk_arch.png (100%) rename {.linux_items => setup/linux}/include/EFI/boot/icons/wk_memtest.png (100%) rename {.linux_items => setup/linux}/include/EFI/boot/icons/wk_win.png (100%) rename {.linux_items => setup/linux}/include/EFI/boot/refind.conf (100%) rename {.linux_items => setup/linux}/include/EFI/boot/selection_big.png (100%) rename {.linux_items => setup/linux}/include/EFI/boot/selection_small.png (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/default/ufw (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/hostname (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/hosts (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/locale.conf (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/locale.gen (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/motd (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/polkit-1/rules.d/49-nopasswd_global.rules (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.aliases (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.bashrc (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.dircolors (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.rsync_exclusions (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.tmux.conf (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.update_network (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.vimrc (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.zlogin (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/skel/.zshrc (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/systemd/system/multi-user.target.wants/NetworkManager.service (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/systemd/system/multi-user.target.wants/rngd.service (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/systemd/system/multi-user.target.wants/sshd.service (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/systemd/system/multi-user.target.wants/systemd-timesyncd.service (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/systemd/system/multi-user.target.wants/ufw.service (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/udev/rules.d/99-udisks2.rules (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/udevil/udevil.conf (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/ufw/ufw.conf (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/ufw/user.rules (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/ufw/user6.rules (100%) rename {.linux_items => setup/linux}/include/airootfs/etc/vconsole.conf (100%) rename {.linux_items => setup/linux}/include/isolinux/isolinux.cfg (100%) create mode 100644 setup/linux/include/syslinux/splash.png rename {.linux_items => setup/linux}/include/syslinux/syslinux.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_hdt.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_head.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_iso.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_iso_linux.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_pxe.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_pxe_linux.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_pxe_winpe.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_sys.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_sys_linux.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_sys_winpe.cfg (100%) rename {.linux_items => setup/linux}/include/syslinux/wk_tail.cfg (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/oblogout.conf (100%) create mode 100644 setup/linux/include_x/airootfs/etc/skel/.Xauthority rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.Xresources (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/Thunar/accels.scm (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/Thunar/uca.xml (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/dunst/dunstrc (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/gtk-3.0/settings.ini (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/i3/config (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/i3status/config (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/mimeapps.list (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/openbox/autostart (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/openbox/environment (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/openbox/menu.xml (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/openbox/rc.xml (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/rofi/config (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/systemd/user/timers.target.wants/update-conky.timer (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.service (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.timer (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/tint2/tint2rc (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.config/volumeicon/volumeicon (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.conky_start (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.conkyrc_base (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.gtkrc-2.0 (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.start_desktop_apps (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.update_conky (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.update_x (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.wallpaper (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.xinitrc (100%) rename {.linux_items => setup/linux}/include_x/airootfs/etc/skel/.zlogin (100%) rename {.linux_items => setup/linux}/include_x/airootfs/usr/share/applications/Hardware Diagnostics.desktop (100%) rename {.linux_items => setup/linux}/include_x/airootfs/usr/share/applications/Hardware Information.desktop (100%) rename {.linux_items => setup/linux}/include_x/airootfs/usr/share/applications/NetworkTest.desktop (100%) rename {.linux_items => setup/linux}/known_networks (100%) rename {.linux_items => setup/linux}/packages/aur (100%) rename {.linux_items => setup/linux}/packages/dependencies (100%) rename {.linux_items => setup/linux}/packages/live_add (100%) rename {.linux_items => setup/linux}/packages/live_add_min (100%) rename {.linux_items => setup/linux}/packages/live_add_x (100%) rename {.linux_items => setup/linux}/packages/live_remove (100%) rename {.pe_items => setup/pe}/System32/Winpeshl.ini (100%) rename {.pe_items => setup/pe}/System32/menu.cmd (100%) rename {.pe_items/_include => setup/pe/bin}/CPU-Z/cpuz.ini (100%) rename {.pe_items/_include => setup/pe/bin}/ConEmu/ConEmu.xml (100%) rename {.pe_items/_include => setup/pe/bin}/HWiNFO/HWiNFO.INI (100%) rename {.pe_items/_include => setup/pe/bin}/NotepadPlusPlus/config.xml (100%) rename {.pe_items/_include => setup/pe/bin}/NotepadPlusPlus/npp.cmd (100%) rename {.pe_items/_include => setup/pe/bin}/NotepadPlusPlus/stylers.model.xml (100%) rename {.pe_items/_include => setup/pe/bin}/Q-Dir/Q-Dir.ini (100%) rename {.kit_items => setup/windows}/Drivers/Extras/AMD.url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/Dell (FTP - Browse for Drivers).url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/Dell (Simplified Interface).url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/Dell (Support Site).url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/Device Remover.url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/Display Driver Uninstaller.url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/HP.url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/Intel Driver & Support Assistant.url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/NVIDIA.url (100%) rename {.kit_items => setup/windows}/Drivers/Extras/Samsung Tools & Software.url (100%) rename {.kit_items => setup/windows}/Installers/BackBlaze.url (100%) rename {.kit_items => setup/windows}/Misc/Fix Missing Optical Drive.reg (100%) rename {.kit_items => setup/windows}/Misc/Nirsoft Utilities - Outlook.url (100%) rename {.kit_items => setup/windows}/Misc/Nirsoft Utilities - Passwords.url (100%) rename {.kit_items => setup/windows}/Misc/Sysinternals Suite (Live).url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/AV Removal Tools.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/AVG.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/Avast.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/Avira.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/ESET.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/Kaspersky.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/MBAM.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/McAfee.url (100%) rename {.kit_items => setup/windows}/Uninstallers/AV Removal Tools/Norton.url (100%) rename {.bin => setup/windows/bin}/ConEmu/ConEmu.xml (100%) rename {.bin => setup/windows/bin}/HWiNFO/general.ini (100%) rename {.bin => setup/windows/bin}/_Drivers/SDIO/sdi.cfg (100%) rename {.cbin => setup/windows/cbin}/_include/AIDA64/full.rpf (100%) rename {.cbin => setup/windows/cbin}/_include/AIDA64/installed_programs.rpf (100%) rename {.cbin => setup/windows/cbin}/_include/AIDA64/licenses.rpf (100%) rename {.cbin => setup/windows/cbin}/_include/BleachBit/BleachBit.ini (100%) rename {.cbin => setup/windows/cbin}/_include/NotepadPlusPlus/config.xml (100%) rename {.cbin => setup/windows/cbin}/_include/XMPlay/xmplay.ini (100%) rename {.cbin => setup/windows/cbin}/_include/XYplorerFree/Data/XYplorer.ini (100%) rename {.cbin => setup/windows/cbin}/_include/_Drivers/Intel RST/SetupRST_13.x.txt (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2016_hb_32.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2016_hb_64.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2016_hs_32.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2016_hs_64.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2019_hb_32.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2019_hb_64.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2019_hs_32.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/2019_hs_64.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/365_32.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_Office/365_64.xml (100%) rename {.cbin => setup/windows/cbin}/_include/_vcredists/InstallAll.bat (100%) diff --git a/.linux_items/include/syslinux/splash.png b/docs/TODO similarity index 100% rename from .linux_items/include/syslinux/splash.png rename to docs/TODO diff --git a/Images/ConEmu.png b/images/ConEmu.png similarity index 100% rename from Images/ConEmu.png rename to images/ConEmu.png diff --git a/Images/Linux.png b/images/Linux.png similarity index 100% rename from Images/Linux.png rename to images/Linux.png diff --git a/Images/Pxelinux.png b/images/Pxelinux.png similarity index 100% rename from Images/Pxelinux.png rename to images/Pxelinux.png diff --git a/Images/Syslinux.png b/images/Syslinux.png similarity index 100% rename from Images/Syslinux.png rename to images/Syslinux.png diff --git a/Images/WinPE.jpg b/images/WinPE.jpg similarity index 100% rename from Images/WinPE.jpg rename to images/WinPE.jpg diff --git a/Images/WizardHat.xcf b/images/WizardHat.xcf similarity index 100% rename from Images/WizardHat.xcf rename to images/WizardHat.xcf diff --git a/Images/logo.svg b/images/logo.svg similarity index 100% rename from Images/logo.svg rename to images/logo.svg diff --git a/Images/rEFInd.png b/images/rEFInd.png similarity index 100% rename from Images/rEFInd.png rename to images/rEFInd.png diff --git a/.bin/Scripts/Copy WizardKit.cmd b/scripts/Copy WizardKit.cmd similarity index 100% rename from .bin/Scripts/Copy WizardKit.cmd rename to scripts/Copy WizardKit.cmd diff --git a/.bin/Scripts/Launch.cmd b/scripts/Launch.cmd similarity index 100% rename from .bin/Scripts/Launch.cmd rename to scripts/Launch.cmd diff --git a/.bin/Scripts/Launcher_Template.cmd b/scripts/Launcher_Template.cmd similarity index 100% rename from .bin/Scripts/Launcher_Template.cmd rename to scripts/Launcher_Template.cmd diff --git a/.bin/Scripts/add-known-networks b/scripts/add-known-networks similarity index 100% rename from .bin/Scripts/add-known-networks rename to scripts/add-known-networks diff --git a/.bin/Scripts/apple-fans b/scripts/apple-fans similarity index 100% rename from .bin/Scripts/apple-fans rename to scripts/apple-fans diff --git a/.bin/Scripts/borrowed/acpi.py b/scripts/borrowed/acpi.py similarity index 100% rename from .bin/Scripts/borrowed/acpi.py rename to scripts/borrowed/acpi.py diff --git a/.bin/Scripts/borrowed/knownpaths-LICENSE.txt b/scripts/borrowed/knownpaths-LICENSE.txt similarity index 100% rename from .bin/Scripts/borrowed/knownpaths-LICENSE.txt rename to scripts/borrowed/knownpaths-LICENSE.txt diff --git a/.bin/Scripts/borrowed/knownpaths.py b/scripts/borrowed/knownpaths.py similarity index 100% rename from .bin/Scripts/borrowed/knownpaths.py rename to scripts/borrowed/knownpaths.py diff --git a/.bin/Scripts/borrowed/set-eol.ps1 b/scripts/borrowed/set-eol.ps1 similarity index 100% rename from .bin/Scripts/borrowed/set-eol.ps1 rename to scripts/borrowed/set-eol.ps1 diff --git a/.bin/Scripts/build_kit.ps1 b/scripts/build_kit.ps1 similarity index 100% rename from .bin/Scripts/build_kit.ps1 rename to scripts/build_kit.ps1 diff --git a/.bin/Scripts/build_pe.ps1 b/scripts/build_pe.ps1 similarity index 100% rename from .bin/Scripts/build_pe.ps1 rename to scripts/build_pe.ps1 diff --git a/.bin/Scripts/echo-and-hold b/scripts/echo-and-hold similarity index 100% rename from .bin/Scripts/echo-and-hold rename to scripts/echo-and-hold diff --git a/.bin/Scripts/init_client_dir.cmd b/scripts/init_client_dir.cmd similarity index 100% rename from .bin/Scripts/init_client_dir.cmd rename to scripts/init_client_dir.cmd diff --git a/.bin/Scripts/launch-in-tmux b/scripts/launch-in-tmux similarity index 100% rename from .bin/Scripts/launch-in-tmux rename to scripts/launch-in-tmux diff --git a/.bin/Scripts/mount-raw-image b/scripts/mount-raw-image similarity index 100% rename from .bin/Scripts/mount-raw-image rename to scripts/mount-raw-image diff --git a/.bin/Scripts/msword-search b/scripts/msword-search similarity index 100% rename from .bin/Scripts/msword-search rename to scripts/msword-search diff --git a/.bin/Scripts/activate.py b/scripts/outer_scripts_to_review/activate.py similarity index 100% rename from .bin/Scripts/activate.py rename to scripts/outer_scripts_to_review/activate.py diff --git a/.bin/Scripts/build-ufd b/scripts/outer_scripts_to_review/build-ufd similarity index 100% rename from .bin/Scripts/build-ufd rename to scripts/outer_scripts_to_review/build-ufd diff --git a/.bin/Scripts/cbs_fix.py b/scripts/outer_scripts_to_review/cbs_fix.py similarity index 100% rename from .bin/Scripts/cbs_fix.py rename to scripts/outer_scripts_to_review/cbs_fix.py diff --git a/.bin/Scripts/check_disk.py b/scripts/outer_scripts_to_review/check_disk.py similarity index 100% rename from .bin/Scripts/check_disk.py rename to scripts/outer_scripts_to_review/check_disk.py diff --git a/.bin/Scripts/ddrescue-tui b/scripts/outer_scripts_to_review/ddrescue-tui similarity index 100% rename from .bin/Scripts/ddrescue-tui rename to scripts/outer_scripts_to_review/ddrescue-tui diff --git a/.bin/Scripts/ddrescue-tui-menu b/scripts/outer_scripts_to_review/ddrescue-tui-menu similarity index 100% rename from .bin/Scripts/ddrescue-tui-menu rename to scripts/outer_scripts_to_review/ddrescue-tui-menu diff --git a/.bin/Scripts/dism.py b/scripts/outer_scripts_to_review/dism.py similarity index 100% rename from .bin/Scripts/dism.py rename to scripts/outer_scripts_to_review/dism.py diff --git a/.bin/Scripts/hw-diags b/scripts/outer_scripts_to_review/hw-diags similarity index 100% rename from .bin/Scripts/hw-diags rename to scripts/outer_scripts_to_review/hw-diags diff --git a/.bin/Scripts/hw-diags-audio b/scripts/outer_scripts_to_review/hw-diags-audio similarity index 100% rename from .bin/Scripts/hw-diags-audio rename to scripts/outer_scripts_to_review/hw-diags-audio diff --git a/.bin/Scripts/hw-diags-iobenchmark b/scripts/outer_scripts_to_review/hw-diags-iobenchmark similarity index 100% rename from .bin/Scripts/hw-diags-iobenchmark rename to scripts/outer_scripts_to_review/hw-diags-iobenchmark diff --git a/.bin/Scripts/hw-diags-menu b/scripts/outer_scripts_to_review/hw-diags-menu similarity index 100% rename from .bin/Scripts/hw-diags-menu rename to scripts/outer_scripts_to_review/hw-diags-menu diff --git a/.bin/Scripts/hw-diags-network b/scripts/outer_scripts_to_review/hw-diags-network similarity index 100% rename from .bin/Scripts/hw-diags-network rename to scripts/outer_scripts_to_review/hw-diags-network diff --git a/.bin/Scripts/hw-diags-prime95 b/scripts/outer_scripts_to_review/hw-diags-prime95 similarity index 100% rename from .bin/Scripts/hw-diags-prime95 rename to scripts/outer_scripts_to_review/hw-diags-prime95 diff --git a/.bin/Scripts/hw-drive-info b/scripts/outer_scripts_to_review/hw-drive-info similarity index 100% rename from .bin/Scripts/hw-drive-info rename to scripts/outer_scripts_to_review/hw-drive-info diff --git a/.bin/Scripts/hw-info b/scripts/outer_scripts_to_review/hw-info similarity index 100% rename from .bin/Scripts/hw-info rename to scripts/outer_scripts_to_review/hw-info diff --git a/.bin/Scripts/hw-sensors b/scripts/outer_scripts_to_review/hw-sensors similarity index 100% rename from .bin/Scripts/hw-sensors rename to scripts/outer_scripts_to_review/hw-sensors diff --git a/.bin/Scripts/hw-sensors-monitor b/scripts/outer_scripts_to_review/hw-sensors-monitor similarity index 100% rename from .bin/Scripts/hw-sensors-monitor rename to scripts/outer_scripts_to_review/hw-sensors-monitor diff --git a/.bin/Scripts/install_sw_bundle.py b/scripts/outer_scripts_to_review/install_sw_bundle.py similarity index 100% rename from .bin/Scripts/install_sw_bundle.py rename to scripts/outer_scripts_to_review/install_sw_bundle.py diff --git a/.bin/Scripts/install_vcredists.py b/scripts/outer_scripts_to_review/install_vcredists.py similarity index 100% rename from .bin/Scripts/install_vcredists.py rename to scripts/outer_scripts_to_review/install_vcredists.py diff --git a/.bin/Scripts/mount-all-volumes b/scripts/outer_scripts_to_review/mount-all-volumes similarity index 100% rename from .bin/Scripts/mount-all-volumes rename to scripts/outer_scripts_to_review/mount-all-volumes diff --git a/.bin/Scripts/mount-backup-shares b/scripts/outer_scripts_to_review/mount-backup-shares similarity index 100% rename from .bin/Scripts/mount-backup-shares rename to scripts/outer_scripts_to_review/mount-backup-shares diff --git a/.bin/Scripts/safemode_enter.py b/scripts/outer_scripts_to_review/safemode_enter.py similarity index 100% rename from .bin/Scripts/safemode_enter.py rename to scripts/outer_scripts_to_review/safemode_enter.py diff --git a/.bin/Scripts/safemode_exit.py b/scripts/outer_scripts_to_review/safemode_exit.py similarity index 100% rename from .bin/Scripts/safemode_exit.py rename to scripts/outer_scripts_to_review/safemode_exit.py diff --git a/.bin/Scripts/sfc_scan.py b/scripts/outer_scripts_to_review/sfc_scan.py similarity index 100% rename from .bin/Scripts/sfc_scan.py rename to scripts/outer_scripts_to_review/sfc_scan.py diff --git a/.bin/Scripts/system_diagnostics.py b/scripts/outer_scripts_to_review/system_diagnostics.py similarity index 100% rename from .bin/Scripts/system_diagnostics.py rename to scripts/outer_scripts_to_review/system_diagnostics.py diff --git a/.bin/Scripts/system_setup.py b/scripts/outer_scripts_to_review/system_setup.py similarity index 100% rename from .bin/Scripts/system_setup.py rename to scripts/outer_scripts_to_review/system_setup.py diff --git a/.bin/Scripts/transferred_keys.py b/scripts/outer_scripts_to_review/transferred_keys.py similarity index 100% rename from .bin/Scripts/transferred_keys.py rename to scripts/outer_scripts_to_review/transferred_keys.py diff --git a/.bin/Scripts/update_kit.py b/scripts/outer_scripts_to_review/update_kit.py similarity index 100% rename from .bin/Scripts/update_kit.py rename to scripts/outer_scripts_to_review/update_kit.py diff --git a/.bin/Scripts/user_data_transfer.py b/scripts/outer_scripts_to_review/user_data_transfer.py similarity index 100% rename from .bin/Scripts/user_data_transfer.py rename to scripts/outer_scripts_to_review/user_data_transfer.py diff --git a/.bin/Scripts/windows_updates.py b/scripts/outer_scripts_to_review/windows_updates.py similarity index 100% rename from .bin/Scripts/windows_updates.py rename to scripts/outer_scripts_to_review/windows_updates.py diff --git a/.bin/Scripts/winpe_root_menu.py b/scripts/outer_scripts_to_review/winpe_root_menu.py similarity index 100% rename from .bin/Scripts/winpe_root_menu.py rename to scripts/outer_scripts_to_review/winpe_root_menu.py diff --git a/.bin/Scripts/pacinit b/scripts/pacinit similarity index 100% rename from .bin/Scripts/pacinit rename to scripts/pacinit diff --git a/.bin/Scripts/photorec-sort b/scripts/photorec-sort similarity index 100% rename from .bin/Scripts/photorec-sort rename to scripts/photorec-sort diff --git a/.bin/Scripts/remount-rw b/scripts/remount-rw similarity index 100% rename from .bin/Scripts/remount-rw rename to scripts/remount-rw diff --git a/.bin/Scripts/wk-power-command b/scripts/wk-power-command similarity index 100% rename from .bin/Scripts/wk-power-command rename to scripts/wk-power-command diff --git a/.bin/Scripts/debug/hw_diags.py b/scripts/wk.prev/debug/hw_diags.py similarity index 100% rename from .bin/Scripts/debug/hw_diags.py rename to scripts/wk.prev/debug/hw_diags.py diff --git a/.bin/Scripts/functions/activation.py b/scripts/wk.prev/functions/activation.py similarity index 100% rename from .bin/Scripts/functions/activation.py rename to scripts/wk.prev/functions/activation.py diff --git a/.bin/Scripts/functions/backup.py b/scripts/wk.prev/functions/backup.py similarity index 100% rename from .bin/Scripts/functions/backup.py rename to scripts/wk.prev/functions/backup.py diff --git a/.bin/Scripts/functions/browsers.py b/scripts/wk.prev/functions/browsers.py similarity index 100% rename from .bin/Scripts/functions/browsers.py rename to scripts/wk.prev/functions/browsers.py diff --git a/.bin/Scripts/functions/cleanup.py b/scripts/wk.prev/functions/cleanup.py similarity index 100% rename from .bin/Scripts/functions/cleanup.py rename to scripts/wk.prev/functions/cleanup.py diff --git a/.bin/Scripts/functions/common.py b/scripts/wk.prev/functions/common.py similarity index 100% rename from .bin/Scripts/functions/common.py rename to scripts/wk.prev/functions/common.py diff --git a/.bin/Scripts/functions/data.py b/scripts/wk.prev/functions/data.py similarity index 100% rename from .bin/Scripts/functions/data.py rename to scripts/wk.prev/functions/data.py diff --git a/.bin/Scripts/functions/ddrescue.py b/scripts/wk.prev/functions/ddrescue.py similarity index 100% rename from .bin/Scripts/functions/ddrescue.py rename to scripts/wk.prev/functions/ddrescue.py diff --git a/.bin/Scripts/functions/disk.py b/scripts/wk.prev/functions/disk.py similarity index 100% rename from .bin/Scripts/functions/disk.py rename to scripts/wk.prev/functions/disk.py diff --git a/.bin/Scripts/functions/hw_diags.py b/scripts/wk.prev/functions/hw_diags.py similarity index 100% rename from .bin/Scripts/functions/hw_diags.py rename to scripts/wk.prev/functions/hw_diags.py diff --git a/.bin/Scripts/functions/info.py b/scripts/wk.prev/functions/info.py similarity index 100% rename from .bin/Scripts/functions/info.py rename to scripts/wk.prev/functions/info.py diff --git a/.bin/Scripts/functions/json.py b/scripts/wk.prev/functions/json.py similarity index 100% rename from .bin/Scripts/functions/json.py rename to scripts/wk.prev/functions/json.py diff --git a/.bin/Scripts/functions/network.py b/scripts/wk.prev/functions/network.py similarity index 100% rename from .bin/Scripts/functions/network.py rename to scripts/wk.prev/functions/network.py diff --git a/.bin/Scripts/functions/product_keys.py b/scripts/wk.prev/functions/product_keys.py similarity index 100% rename from .bin/Scripts/functions/product_keys.py rename to scripts/wk.prev/functions/product_keys.py diff --git a/.bin/Scripts/functions/repairs.py b/scripts/wk.prev/functions/repairs.py similarity index 100% rename from .bin/Scripts/functions/repairs.py rename to scripts/wk.prev/functions/repairs.py diff --git a/.bin/Scripts/functions/safemode.py b/scripts/wk.prev/functions/safemode.py similarity index 100% rename from .bin/Scripts/functions/safemode.py rename to scripts/wk.prev/functions/safemode.py diff --git a/.bin/Scripts/functions/sensors.py b/scripts/wk.prev/functions/sensors.py similarity index 100% rename from .bin/Scripts/functions/sensors.py rename to scripts/wk.prev/functions/sensors.py diff --git a/.bin/Scripts/functions/setup.py b/scripts/wk.prev/functions/setup.py similarity index 100% rename from .bin/Scripts/functions/setup.py rename to scripts/wk.prev/functions/setup.py diff --git a/.bin/Scripts/functions/sw_diags.py b/scripts/wk.prev/functions/sw_diags.py similarity index 100% rename from .bin/Scripts/functions/sw_diags.py rename to scripts/wk.prev/functions/sw_diags.py diff --git a/.bin/Scripts/functions/threading.py b/scripts/wk.prev/functions/threading.py similarity index 100% rename from .bin/Scripts/functions/threading.py rename to scripts/wk.prev/functions/threading.py diff --git a/.bin/Scripts/functions/tmux.py b/scripts/wk.prev/functions/tmux.py similarity index 100% rename from .bin/Scripts/functions/tmux.py rename to scripts/wk.prev/functions/tmux.py diff --git a/.bin/Scripts/functions/ufd.py b/scripts/wk.prev/functions/ufd.py similarity index 100% rename from .bin/Scripts/functions/ufd.py rename to scripts/wk.prev/functions/ufd.py diff --git a/.bin/Scripts/functions/update.py b/scripts/wk.prev/functions/update.py similarity index 100% rename from .bin/Scripts/functions/update.py rename to scripts/wk.prev/functions/update.py diff --git a/.bin/Scripts/functions/windows_setup.py b/scripts/wk.prev/functions/windows_setup.py similarity index 100% rename from .bin/Scripts/functions/windows_setup.py rename to scripts/wk.prev/functions/windows_setup.py diff --git a/.bin/Scripts/functions/windows_updates.py b/scripts/wk.prev/functions/windows_updates.py similarity index 100% rename from .bin/Scripts/functions/windows_updates.py rename to scripts/wk.prev/functions/windows_updates.py diff --git a/.bin/Scripts/functions/winpe_menus.py b/scripts/wk.prev/functions/winpe_menus.py similarity index 100% rename from .bin/Scripts/functions/winpe_menus.py rename to scripts/wk.prev/functions/winpe_menus.py diff --git a/.bin/Scripts/settings/browsers.py b/scripts/wk.prev/settings/browsers.py similarity index 100% rename from .bin/Scripts/settings/browsers.py rename to scripts/wk.prev/settings/browsers.py diff --git a/.bin/Scripts/settings/cleanup.py b/scripts/wk.prev/settings/cleanup.py similarity index 100% rename from .bin/Scripts/settings/cleanup.py rename to scripts/wk.prev/settings/cleanup.py diff --git a/.bin/Scripts/settings/data.py b/scripts/wk.prev/settings/data.py similarity index 100% rename from .bin/Scripts/settings/data.py rename to scripts/wk.prev/settings/data.py diff --git a/.bin/Scripts/settings/ddrescue.py b/scripts/wk.prev/settings/ddrescue.py similarity index 100% rename from .bin/Scripts/settings/ddrescue.py rename to scripts/wk.prev/settings/ddrescue.py diff --git a/.bin/Scripts/settings/hw_diags.py b/scripts/wk.prev/settings/hw_diags.py similarity index 100% rename from .bin/Scripts/settings/hw_diags.py rename to scripts/wk.prev/settings/hw_diags.py diff --git a/.bin/Scripts/settings/info.py b/scripts/wk.prev/settings/info.py similarity index 100% rename from .bin/Scripts/settings/info.py rename to scripts/wk.prev/settings/info.py diff --git a/.bin/Scripts/settings/launchers.py b/scripts/wk.prev/settings/launchers.py similarity index 100% rename from .bin/Scripts/settings/launchers.py rename to scripts/wk.prev/settings/launchers.py diff --git a/.bin/Scripts/settings/main.py b/scripts/wk.prev/settings/main.py similarity index 100% rename from .bin/Scripts/settings/main.py rename to scripts/wk.prev/settings/main.py diff --git a/.bin/Scripts/settings/music.py b/scripts/wk.prev/settings/music.py similarity index 100% rename from .bin/Scripts/settings/music.py rename to scripts/wk.prev/settings/music.py diff --git a/.bin/Scripts/settings/partition_uids.py b/scripts/wk.prev/settings/partition_uids.py similarity index 100% rename from .bin/Scripts/settings/partition_uids.py rename to scripts/wk.prev/settings/partition_uids.py diff --git a/.bin/Scripts/settings/sensors.py b/scripts/wk.prev/settings/sensors.py similarity index 100% rename from .bin/Scripts/settings/sensors.py rename to scripts/wk.prev/settings/sensors.py diff --git a/.bin/Scripts/settings/setup.py b/scripts/wk.prev/settings/setup.py similarity index 100% rename from .bin/Scripts/settings/setup.py rename to scripts/wk.prev/settings/setup.py diff --git a/.bin/Scripts/settings/sources.py b/scripts/wk.prev/settings/sources.py similarity index 100% rename from .bin/Scripts/settings/sources.py rename to scripts/wk.prev/settings/sources.py diff --git a/.bin/Scripts/settings/sw_diags.py b/scripts/wk.prev/settings/sw_diags.py similarity index 100% rename from .bin/Scripts/settings/sw_diags.py rename to scripts/wk.prev/settings/sw_diags.py diff --git a/.bin/Scripts/settings/tools.py b/scripts/wk.prev/settings/tools.py similarity index 100% rename from .bin/Scripts/settings/tools.py rename to scripts/wk.prev/settings/tools.py diff --git a/.bin/Scripts/settings/ufd.py b/scripts/wk.prev/settings/ufd.py similarity index 100% rename from .bin/Scripts/settings/ufd.py rename to scripts/wk.prev/settings/ufd.py diff --git a/.bin/Scripts/settings/windows_builds.py b/scripts/wk.prev/settings/windows_builds.py similarity index 100% rename from .bin/Scripts/settings/windows_builds.py rename to scripts/wk.prev/settings/windows_builds.py diff --git a/.bin/Scripts/settings/windows_setup.py b/scripts/wk.prev/settings/windows_setup.py similarity index 100% rename from .bin/Scripts/settings/windows_setup.py rename to scripts/wk.prev/settings/windows_setup.py diff --git a/.bin/Scripts/settings/winpe.py b/scripts/wk.prev/settings/winpe.py similarity index 100% rename from .bin/Scripts/settings/winpe.py rename to scripts/wk.prev/settings/winpe.py diff --git a/.linux_items/include_x/airootfs/etc/skel/.Xauthority b/scripts/wk/exe.py similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.Xauthority rename to scripts/wk/exe.py diff --git a/scripts/wk/io.py b/scripts/wk/io.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/wk/net.py b/scripts/wk/net.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/wk/std.py b/scripts/wk/std.py new file mode 100644 index 00000000..e69de29b diff --git a/Build Linux b/setup/build_linux similarity index 100% rename from Build Linux rename to setup/build_linux diff --git a/Build PE.cmd b/setup/build_pe.cmd similarity index 100% rename from Build PE.cmd rename to setup/build_pe.cmd diff --git a/Build Kit.cmd b/setup/build_windows.cmd similarity index 100% rename from Build Kit.cmd rename to setup/build_windows.cmd diff --git a/.linux_items/authorized_keys b/setup/linux/authorized_keys similarity index 100% rename from .linux_items/authorized_keys rename to setup/linux/authorized_keys diff --git a/.linux_items/build_additions.txt b/setup/linux/build_additions.txt similarity index 100% rename from .linux_items/build_additions.txt rename to setup/linux/build_additions.txt diff --git a/.linux_items/include/EFI/boot/icons/dgpu.png b/setup/linux/include/EFI/boot/icons/dgpu.png similarity index 100% rename from .linux_items/include/EFI/boot/icons/dgpu.png rename to setup/linux/include/EFI/boot/icons/dgpu.png diff --git a/.linux_items/include/EFI/boot/icons/wk_arch.png b/setup/linux/include/EFI/boot/icons/wk_arch.png similarity index 100% rename from .linux_items/include/EFI/boot/icons/wk_arch.png rename to setup/linux/include/EFI/boot/icons/wk_arch.png diff --git a/.linux_items/include/EFI/boot/icons/wk_memtest.png b/setup/linux/include/EFI/boot/icons/wk_memtest.png similarity index 100% rename from .linux_items/include/EFI/boot/icons/wk_memtest.png rename to setup/linux/include/EFI/boot/icons/wk_memtest.png diff --git a/.linux_items/include/EFI/boot/icons/wk_win.png b/setup/linux/include/EFI/boot/icons/wk_win.png similarity index 100% rename from .linux_items/include/EFI/boot/icons/wk_win.png rename to setup/linux/include/EFI/boot/icons/wk_win.png diff --git a/.linux_items/include/EFI/boot/refind.conf b/setup/linux/include/EFI/boot/refind.conf similarity index 100% rename from .linux_items/include/EFI/boot/refind.conf rename to setup/linux/include/EFI/boot/refind.conf diff --git a/.linux_items/include/EFI/boot/selection_big.png b/setup/linux/include/EFI/boot/selection_big.png similarity index 100% rename from .linux_items/include/EFI/boot/selection_big.png rename to setup/linux/include/EFI/boot/selection_big.png diff --git a/.linux_items/include/EFI/boot/selection_small.png b/setup/linux/include/EFI/boot/selection_small.png similarity index 100% rename from .linux_items/include/EFI/boot/selection_small.png rename to setup/linux/include/EFI/boot/selection_small.png diff --git a/.linux_items/include/airootfs/etc/default/ufw b/setup/linux/include/airootfs/etc/default/ufw similarity index 100% rename from .linux_items/include/airootfs/etc/default/ufw rename to setup/linux/include/airootfs/etc/default/ufw diff --git a/.linux_items/include/airootfs/etc/hostname b/setup/linux/include/airootfs/etc/hostname similarity index 100% rename from .linux_items/include/airootfs/etc/hostname rename to setup/linux/include/airootfs/etc/hostname diff --git a/.linux_items/include/airootfs/etc/hosts b/setup/linux/include/airootfs/etc/hosts similarity index 100% rename from .linux_items/include/airootfs/etc/hosts rename to setup/linux/include/airootfs/etc/hosts diff --git a/.linux_items/include/airootfs/etc/locale.conf b/setup/linux/include/airootfs/etc/locale.conf similarity index 100% rename from .linux_items/include/airootfs/etc/locale.conf rename to setup/linux/include/airootfs/etc/locale.conf diff --git a/.linux_items/include/airootfs/etc/locale.gen b/setup/linux/include/airootfs/etc/locale.gen similarity index 100% rename from .linux_items/include/airootfs/etc/locale.gen rename to setup/linux/include/airootfs/etc/locale.gen diff --git a/.linux_items/include/airootfs/etc/motd b/setup/linux/include/airootfs/etc/motd similarity index 100% rename from .linux_items/include/airootfs/etc/motd rename to setup/linux/include/airootfs/etc/motd diff --git a/.linux_items/include/airootfs/etc/polkit-1/rules.d/49-nopasswd_global.rules b/setup/linux/include/airootfs/etc/polkit-1/rules.d/49-nopasswd_global.rules similarity index 100% rename from .linux_items/include/airootfs/etc/polkit-1/rules.d/49-nopasswd_global.rules rename to setup/linux/include/airootfs/etc/polkit-1/rules.d/49-nopasswd_global.rules diff --git a/.linux_items/include/airootfs/etc/skel/.aliases b/setup/linux/include/airootfs/etc/skel/.aliases similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.aliases rename to setup/linux/include/airootfs/etc/skel/.aliases diff --git a/.linux_items/include/airootfs/etc/skel/.bashrc b/setup/linux/include/airootfs/etc/skel/.bashrc similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.bashrc rename to setup/linux/include/airootfs/etc/skel/.bashrc diff --git a/.linux_items/include/airootfs/etc/skel/.dircolors b/setup/linux/include/airootfs/etc/skel/.dircolors similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.dircolors rename to setup/linux/include/airootfs/etc/skel/.dircolors diff --git a/.linux_items/include/airootfs/etc/skel/.rsync_exclusions b/setup/linux/include/airootfs/etc/skel/.rsync_exclusions similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.rsync_exclusions rename to setup/linux/include/airootfs/etc/skel/.rsync_exclusions diff --git a/.linux_items/include/airootfs/etc/skel/.tmux.conf b/setup/linux/include/airootfs/etc/skel/.tmux.conf similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.tmux.conf rename to setup/linux/include/airootfs/etc/skel/.tmux.conf diff --git a/.linux_items/include/airootfs/etc/skel/.update_network b/setup/linux/include/airootfs/etc/skel/.update_network similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.update_network rename to setup/linux/include/airootfs/etc/skel/.update_network diff --git a/.linux_items/include/airootfs/etc/skel/.vimrc b/setup/linux/include/airootfs/etc/skel/.vimrc similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.vimrc rename to setup/linux/include/airootfs/etc/skel/.vimrc diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/setup/linux/include/airootfs/etc/skel/.zlogin similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.zlogin rename to setup/linux/include/airootfs/etc/skel/.zlogin diff --git a/.linux_items/include/airootfs/etc/skel/.zshrc b/setup/linux/include/airootfs/etc/skel/.zshrc similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.zshrc rename to setup/linux/include/airootfs/etc/skel/.zshrc diff --git a/.linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/NetworkManager.service b/setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/NetworkManager.service similarity index 100% rename from .linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/NetworkManager.service rename to setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/NetworkManager.service diff --git a/.linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/rngd.service b/setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/rngd.service similarity index 100% rename from .linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/rngd.service rename to setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/rngd.service diff --git a/.linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/sshd.service b/setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/sshd.service similarity index 100% rename from .linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/sshd.service rename to setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/sshd.service diff --git a/.linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/systemd-timesyncd.service b/setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/systemd-timesyncd.service similarity index 100% rename from .linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/systemd-timesyncd.service rename to setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/systemd-timesyncd.service diff --git a/.linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/ufw.service b/setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/ufw.service similarity index 100% rename from .linux_items/include/airootfs/etc/systemd/system/multi-user.target.wants/ufw.service rename to setup/linux/include/airootfs/etc/systemd/system/multi-user.target.wants/ufw.service diff --git a/.linux_items/include/airootfs/etc/udev/rules.d/99-udisks2.rules b/setup/linux/include/airootfs/etc/udev/rules.d/99-udisks2.rules similarity index 100% rename from .linux_items/include/airootfs/etc/udev/rules.d/99-udisks2.rules rename to setup/linux/include/airootfs/etc/udev/rules.d/99-udisks2.rules diff --git a/.linux_items/include/airootfs/etc/udevil/udevil.conf b/setup/linux/include/airootfs/etc/udevil/udevil.conf similarity index 100% rename from .linux_items/include/airootfs/etc/udevil/udevil.conf rename to setup/linux/include/airootfs/etc/udevil/udevil.conf diff --git a/.linux_items/include/airootfs/etc/ufw/ufw.conf b/setup/linux/include/airootfs/etc/ufw/ufw.conf similarity index 100% rename from .linux_items/include/airootfs/etc/ufw/ufw.conf rename to setup/linux/include/airootfs/etc/ufw/ufw.conf diff --git a/.linux_items/include/airootfs/etc/ufw/user.rules b/setup/linux/include/airootfs/etc/ufw/user.rules similarity index 100% rename from .linux_items/include/airootfs/etc/ufw/user.rules rename to setup/linux/include/airootfs/etc/ufw/user.rules diff --git a/.linux_items/include/airootfs/etc/ufw/user6.rules b/setup/linux/include/airootfs/etc/ufw/user6.rules similarity index 100% rename from .linux_items/include/airootfs/etc/ufw/user6.rules rename to setup/linux/include/airootfs/etc/ufw/user6.rules diff --git a/.linux_items/include/airootfs/etc/vconsole.conf b/setup/linux/include/airootfs/etc/vconsole.conf similarity index 100% rename from .linux_items/include/airootfs/etc/vconsole.conf rename to setup/linux/include/airootfs/etc/vconsole.conf diff --git a/.linux_items/include/isolinux/isolinux.cfg b/setup/linux/include/isolinux/isolinux.cfg similarity index 100% rename from .linux_items/include/isolinux/isolinux.cfg rename to setup/linux/include/isolinux/isolinux.cfg diff --git a/setup/linux/include/syslinux/splash.png b/setup/linux/include/syslinux/splash.png new file mode 100644 index 00000000..e69de29b diff --git a/.linux_items/include/syslinux/syslinux.cfg b/setup/linux/include/syslinux/syslinux.cfg similarity index 100% rename from .linux_items/include/syslinux/syslinux.cfg rename to setup/linux/include/syslinux/syslinux.cfg diff --git a/.linux_items/include/syslinux/wk.cfg b/setup/linux/include/syslinux/wk.cfg similarity index 100% rename from .linux_items/include/syslinux/wk.cfg rename to setup/linux/include/syslinux/wk.cfg diff --git a/.linux_items/include/syslinux/wk_hdt.cfg b/setup/linux/include/syslinux/wk_hdt.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_hdt.cfg rename to setup/linux/include/syslinux/wk_hdt.cfg diff --git a/.linux_items/include/syslinux/wk_head.cfg b/setup/linux/include/syslinux/wk_head.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_head.cfg rename to setup/linux/include/syslinux/wk_head.cfg diff --git a/.linux_items/include/syslinux/wk_iso.cfg b/setup/linux/include/syslinux/wk_iso.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_iso.cfg rename to setup/linux/include/syslinux/wk_iso.cfg diff --git a/.linux_items/include/syslinux/wk_iso_linux.cfg b/setup/linux/include/syslinux/wk_iso_linux.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_iso_linux.cfg rename to setup/linux/include/syslinux/wk_iso_linux.cfg diff --git a/.linux_items/include/syslinux/wk_pxe.cfg b/setup/linux/include/syslinux/wk_pxe.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_pxe.cfg rename to setup/linux/include/syslinux/wk_pxe.cfg diff --git a/.linux_items/include/syslinux/wk_pxe_linux.cfg b/setup/linux/include/syslinux/wk_pxe_linux.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_pxe_linux.cfg rename to setup/linux/include/syslinux/wk_pxe_linux.cfg diff --git a/.linux_items/include/syslinux/wk_pxe_winpe.cfg b/setup/linux/include/syslinux/wk_pxe_winpe.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_pxe_winpe.cfg rename to setup/linux/include/syslinux/wk_pxe_winpe.cfg diff --git a/.linux_items/include/syslinux/wk_sys.cfg b/setup/linux/include/syslinux/wk_sys.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_sys.cfg rename to setup/linux/include/syslinux/wk_sys.cfg diff --git a/.linux_items/include/syslinux/wk_sys_linux.cfg b/setup/linux/include/syslinux/wk_sys_linux.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_sys_linux.cfg rename to setup/linux/include/syslinux/wk_sys_linux.cfg diff --git a/.linux_items/include/syslinux/wk_sys_winpe.cfg b/setup/linux/include/syslinux/wk_sys_winpe.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_sys_winpe.cfg rename to setup/linux/include/syslinux/wk_sys_winpe.cfg diff --git a/.linux_items/include/syslinux/wk_tail.cfg b/setup/linux/include/syslinux/wk_tail.cfg similarity index 100% rename from .linux_items/include/syslinux/wk_tail.cfg rename to setup/linux/include/syslinux/wk_tail.cfg diff --git a/.linux_items/include_x/airootfs/etc/oblogout.conf b/setup/linux/include_x/airootfs/etc/oblogout.conf similarity index 100% rename from .linux_items/include_x/airootfs/etc/oblogout.conf rename to setup/linux/include_x/airootfs/etc/oblogout.conf diff --git a/setup/linux/include_x/airootfs/etc/skel/.Xauthority b/setup/linux/include_x/airootfs/etc/skel/.Xauthority new file mode 100644 index 00000000..e69de29b diff --git a/.linux_items/include_x/airootfs/etc/skel/.Xresources b/setup/linux/include_x/airootfs/etc/skel/.Xresources similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.Xresources rename to setup/linux/include_x/airootfs/etc/skel/.Xresources diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/Thunar/accels.scm b/setup/linux/include_x/airootfs/etc/skel/.config/Thunar/accels.scm similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/Thunar/accels.scm rename to setup/linux/include_x/airootfs/etc/skel/.config/Thunar/accels.scm diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/Thunar/uca.xml b/setup/linux/include_x/airootfs/etc/skel/.config/Thunar/uca.xml similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/Thunar/uca.xml rename to setup/linux/include_x/airootfs/etc/skel/.config/Thunar/uca.xml diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/dunst/dunstrc b/setup/linux/include_x/airootfs/etc/skel/.config/dunst/dunstrc similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/dunst/dunstrc rename to setup/linux/include_x/airootfs/etc/skel/.config/dunst/dunstrc diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/gtk-3.0/settings.ini b/setup/linux/include_x/airootfs/etc/skel/.config/gtk-3.0/settings.ini similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/gtk-3.0/settings.ini rename to setup/linux/include_x/airootfs/etc/skel/.config/gtk-3.0/settings.ini diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/i3/config b/setup/linux/include_x/airootfs/etc/skel/.config/i3/config similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/i3/config rename to setup/linux/include_x/airootfs/etc/skel/.config/i3/config diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/i3status/config b/setup/linux/include_x/airootfs/etc/skel/.config/i3status/config similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/i3status/config rename to setup/linux/include_x/airootfs/etc/skel/.config/i3status/config diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/mimeapps.list b/setup/linux/include_x/airootfs/etc/skel/.config/mimeapps.list similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/mimeapps.list rename to setup/linux/include_x/airootfs/etc/skel/.config/mimeapps.list diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/openbox/autostart b/setup/linux/include_x/airootfs/etc/skel/.config/openbox/autostart similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/openbox/autostart rename to setup/linux/include_x/airootfs/etc/skel/.config/openbox/autostart diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/openbox/environment b/setup/linux/include_x/airootfs/etc/skel/.config/openbox/environment similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/openbox/environment rename to setup/linux/include_x/airootfs/etc/skel/.config/openbox/environment diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/openbox/menu.xml b/setup/linux/include_x/airootfs/etc/skel/.config/openbox/menu.xml similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/openbox/menu.xml rename to setup/linux/include_x/airootfs/etc/skel/.config/openbox/menu.xml diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/openbox/rc.xml b/setup/linux/include_x/airootfs/etc/skel/.config/openbox/rc.xml similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/openbox/rc.xml rename to setup/linux/include_x/airootfs/etc/skel/.config/openbox/rc.xml diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/rofi/config b/setup/linux/include_x/airootfs/etc/skel/.config/rofi/config similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/rofi/config rename to setup/linux/include_x/airootfs/etc/skel/.config/rofi/config diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/systemd/user/timers.target.wants/update-conky.timer b/setup/linux/include_x/airootfs/etc/skel/.config/systemd/user/timers.target.wants/update-conky.timer similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/systemd/user/timers.target.wants/update-conky.timer rename to setup/linux/include_x/airootfs/etc/skel/.config/systemd/user/timers.target.wants/update-conky.timer diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.service b/setup/linux/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.service similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.service rename to setup/linux/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.service diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.timer b/setup/linux/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.timer similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.timer rename to setup/linux/include_x/airootfs/etc/skel/.config/systemd/user/update-conky.timer diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/tint2/tint2rc b/setup/linux/include_x/airootfs/etc/skel/.config/tint2/tint2rc similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/tint2/tint2rc rename to setup/linux/include_x/airootfs/etc/skel/.config/tint2/tint2rc diff --git a/.linux_items/include_x/airootfs/etc/skel/.config/volumeicon/volumeicon b/setup/linux/include_x/airootfs/etc/skel/.config/volumeicon/volumeicon similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.config/volumeicon/volumeicon rename to setup/linux/include_x/airootfs/etc/skel/.config/volumeicon/volumeicon diff --git a/.linux_items/include_x/airootfs/etc/skel/.conky_start b/setup/linux/include_x/airootfs/etc/skel/.conky_start similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.conky_start rename to setup/linux/include_x/airootfs/etc/skel/.conky_start diff --git a/.linux_items/include_x/airootfs/etc/skel/.conkyrc_base b/setup/linux/include_x/airootfs/etc/skel/.conkyrc_base similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.conkyrc_base rename to setup/linux/include_x/airootfs/etc/skel/.conkyrc_base diff --git a/.linux_items/include_x/airootfs/etc/skel/.gtkrc-2.0 b/setup/linux/include_x/airootfs/etc/skel/.gtkrc-2.0 similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.gtkrc-2.0 rename to setup/linux/include_x/airootfs/etc/skel/.gtkrc-2.0 diff --git a/.linux_items/include_x/airootfs/etc/skel/.start_desktop_apps b/setup/linux/include_x/airootfs/etc/skel/.start_desktop_apps similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.start_desktop_apps rename to setup/linux/include_x/airootfs/etc/skel/.start_desktop_apps diff --git a/.linux_items/include_x/airootfs/etc/skel/.update_conky b/setup/linux/include_x/airootfs/etc/skel/.update_conky similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.update_conky rename to setup/linux/include_x/airootfs/etc/skel/.update_conky diff --git a/.linux_items/include_x/airootfs/etc/skel/.update_x b/setup/linux/include_x/airootfs/etc/skel/.update_x similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.update_x rename to setup/linux/include_x/airootfs/etc/skel/.update_x diff --git a/.linux_items/include_x/airootfs/etc/skel/.wallpaper b/setup/linux/include_x/airootfs/etc/skel/.wallpaper similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.wallpaper rename to setup/linux/include_x/airootfs/etc/skel/.wallpaper diff --git a/.linux_items/include_x/airootfs/etc/skel/.xinitrc b/setup/linux/include_x/airootfs/etc/skel/.xinitrc similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.xinitrc rename to setup/linux/include_x/airootfs/etc/skel/.xinitrc diff --git a/.linux_items/include_x/airootfs/etc/skel/.zlogin b/setup/linux/include_x/airootfs/etc/skel/.zlogin similarity index 100% rename from .linux_items/include_x/airootfs/etc/skel/.zlogin rename to setup/linux/include_x/airootfs/etc/skel/.zlogin diff --git a/.linux_items/include_x/airootfs/usr/share/applications/Hardware Diagnostics.desktop b/setup/linux/include_x/airootfs/usr/share/applications/Hardware Diagnostics.desktop similarity index 100% rename from .linux_items/include_x/airootfs/usr/share/applications/Hardware Diagnostics.desktop rename to setup/linux/include_x/airootfs/usr/share/applications/Hardware Diagnostics.desktop diff --git a/.linux_items/include_x/airootfs/usr/share/applications/Hardware Information.desktop b/setup/linux/include_x/airootfs/usr/share/applications/Hardware Information.desktop similarity index 100% rename from .linux_items/include_x/airootfs/usr/share/applications/Hardware Information.desktop rename to setup/linux/include_x/airootfs/usr/share/applications/Hardware Information.desktop diff --git a/.linux_items/include_x/airootfs/usr/share/applications/NetworkTest.desktop b/setup/linux/include_x/airootfs/usr/share/applications/NetworkTest.desktop similarity index 100% rename from .linux_items/include_x/airootfs/usr/share/applications/NetworkTest.desktop rename to setup/linux/include_x/airootfs/usr/share/applications/NetworkTest.desktop diff --git a/.linux_items/known_networks b/setup/linux/known_networks similarity index 100% rename from .linux_items/known_networks rename to setup/linux/known_networks diff --git a/.linux_items/packages/aur b/setup/linux/packages/aur similarity index 100% rename from .linux_items/packages/aur rename to setup/linux/packages/aur diff --git a/.linux_items/packages/dependencies b/setup/linux/packages/dependencies similarity index 100% rename from .linux_items/packages/dependencies rename to setup/linux/packages/dependencies diff --git a/.linux_items/packages/live_add b/setup/linux/packages/live_add similarity index 100% rename from .linux_items/packages/live_add rename to setup/linux/packages/live_add diff --git a/.linux_items/packages/live_add_min b/setup/linux/packages/live_add_min similarity index 100% rename from .linux_items/packages/live_add_min rename to setup/linux/packages/live_add_min diff --git a/.linux_items/packages/live_add_x b/setup/linux/packages/live_add_x similarity index 100% rename from .linux_items/packages/live_add_x rename to setup/linux/packages/live_add_x diff --git a/.linux_items/packages/live_remove b/setup/linux/packages/live_remove similarity index 100% rename from .linux_items/packages/live_remove rename to setup/linux/packages/live_remove diff --git a/.pe_items/System32/Winpeshl.ini b/setup/pe/System32/Winpeshl.ini similarity index 100% rename from .pe_items/System32/Winpeshl.ini rename to setup/pe/System32/Winpeshl.ini diff --git a/.pe_items/System32/menu.cmd b/setup/pe/System32/menu.cmd similarity index 100% rename from .pe_items/System32/menu.cmd rename to setup/pe/System32/menu.cmd diff --git a/.pe_items/_include/CPU-Z/cpuz.ini b/setup/pe/bin/CPU-Z/cpuz.ini similarity index 100% rename from .pe_items/_include/CPU-Z/cpuz.ini rename to setup/pe/bin/CPU-Z/cpuz.ini diff --git a/.pe_items/_include/ConEmu/ConEmu.xml b/setup/pe/bin/ConEmu/ConEmu.xml similarity index 100% rename from .pe_items/_include/ConEmu/ConEmu.xml rename to setup/pe/bin/ConEmu/ConEmu.xml diff --git a/.pe_items/_include/HWiNFO/HWiNFO.INI b/setup/pe/bin/HWiNFO/HWiNFO.INI similarity index 100% rename from .pe_items/_include/HWiNFO/HWiNFO.INI rename to setup/pe/bin/HWiNFO/HWiNFO.INI diff --git a/.pe_items/_include/NotepadPlusPlus/config.xml b/setup/pe/bin/NotepadPlusPlus/config.xml similarity index 100% rename from .pe_items/_include/NotepadPlusPlus/config.xml rename to setup/pe/bin/NotepadPlusPlus/config.xml diff --git a/.pe_items/_include/NotepadPlusPlus/npp.cmd b/setup/pe/bin/NotepadPlusPlus/npp.cmd similarity index 100% rename from .pe_items/_include/NotepadPlusPlus/npp.cmd rename to setup/pe/bin/NotepadPlusPlus/npp.cmd diff --git a/.pe_items/_include/NotepadPlusPlus/stylers.model.xml b/setup/pe/bin/NotepadPlusPlus/stylers.model.xml similarity index 100% rename from .pe_items/_include/NotepadPlusPlus/stylers.model.xml rename to setup/pe/bin/NotepadPlusPlus/stylers.model.xml diff --git a/.pe_items/_include/Q-Dir/Q-Dir.ini b/setup/pe/bin/Q-Dir/Q-Dir.ini similarity index 100% rename from .pe_items/_include/Q-Dir/Q-Dir.ini rename to setup/pe/bin/Q-Dir/Q-Dir.ini diff --git a/.kit_items/Drivers/Extras/AMD.url b/setup/windows/Drivers/Extras/AMD.url similarity index 100% rename from .kit_items/Drivers/Extras/AMD.url rename to setup/windows/Drivers/Extras/AMD.url diff --git a/.kit_items/Drivers/Extras/Dell (FTP - Browse for Drivers).url b/setup/windows/Drivers/Extras/Dell (FTP - Browse for Drivers).url similarity index 100% rename from .kit_items/Drivers/Extras/Dell (FTP - Browse for Drivers).url rename to setup/windows/Drivers/Extras/Dell (FTP - Browse for Drivers).url diff --git a/.kit_items/Drivers/Extras/Dell (Simplified Interface).url b/setup/windows/Drivers/Extras/Dell (Simplified Interface).url similarity index 100% rename from .kit_items/Drivers/Extras/Dell (Simplified Interface).url rename to setup/windows/Drivers/Extras/Dell (Simplified Interface).url diff --git a/.kit_items/Drivers/Extras/Dell (Support Site).url b/setup/windows/Drivers/Extras/Dell (Support Site).url similarity index 100% rename from .kit_items/Drivers/Extras/Dell (Support Site).url rename to setup/windows/Drivers/Extras/Dell (Support Site).url diff --git a/.kit_items/Drivers/Extras/Device Remover.url b/setup/windows/Drivers/Extras/Device Remover.url similarity index 100% rename from .kit_items/Drivers/Extras/Device Remover.url rename to setup/windows/Drivers/Extras/Device Remover.url diff --git a/.kit_items/Drivers/Extras/Display Driver Uninstaller.url b/setup/windows/Drivers/Extras/Display Driver Uninstaller.url similarity index 100% rename from .kit_items/Drivers/Extras/Display Driver Uninstaller.url rename to setup/windows/Drivers/Extras/Display Driver Uninstaller.url diff --git a/.kit_items/Drivers/Extras/HP.url b/setup/windows/Drivers/Extras/HP.url similarity index 100% rename from .kit_items/Drivers/Extras/HP.url rename to setup/windows/Drivers/Extras/HP.url diff --git a/.kit_items/Drivers/Extras/Intel Driver & Support Assistant.url b/setup/windows/Drivers/Extras/Intel Driver & Support Assistant.url similarity index 100% rename from .kit_items/Drivers/Extras/Intel Driver & Support Assistant.url rename to setup/windows/Drivers/Extras/Intel Driver & Support Assistant.url diff --git a/.kit_items/Drivers/Extras/NVIDIA.url b/setup/windows/Drivers/Extras/NVIDIA.url similarity index 100% rename from .kit_items/Drivers/Extras/NVIDIA.url rename to setup/windows/Drivers/Extras/NVIDIA.url diff --git a/.kit_items/Drivers/Extras/Samsung Tools & Software.url b/setup/windows/Drivers/Extras/Samsung Tools & Software.url similarity index 100% rename from .kit_items/Drivers/Extras/Samsung Tools & Software.url rename to setup/windows/Drivers/Extras/Samsung Tools & Software.url diff --git a/.kit_items/Installers/BackBlaze.url b/setup/windows/Installers/BackBlaze.url similarity index 100% rename from .kit_items/Installers/BackBlaze.url rename to setup/windows/Installers/BackBlaze.url diff --git a/.kit_items/Misc/Fix Missing Optical Drive.reg b/setup/windows/Misc/Fix Missing Optical Drive.reg similarity index 100% rename from .kit_items/Misc/Fix Missing Optical Drive.reg rename to setup/windows/Misc/Fix Missing Optical Drive.reg diff --git a/.kit_items/Misc/Nirsoft Utilities - Outlook.url b/setup/windows/Misc/Nirsoft Utilities - Outlook.url similarity index 100% rename from .kit_items/Misc/Nirsoft Utilities - Outlook.url rename to setup/windows/Misc/Nirsoft Utilities - Outlook.url diff --git a/.kit_items/Misc/Nirsoft Utilities - Passwords.url b/setup/windows/Misc/Nirsoft Utilities - Passwords.url similarity index 100% rename from .kit_items/Misc/Nirsoft Utilities - Passwords.url rename to setup/windows/Misc/Nirsoft Utilities - Passwords.url diff --git a/.kit_items/Misc/Sysinternals Suite (Live).url b/setup/windows/Misc/Sysinternals Suite (Live).url similarity index 100% rename from .kit_items/Misc/Sysinternals Suite (Live).url rename to setup/windows/Misc/Sysinternals Suite (Live).url diff --git a/.kit_items/Uninstallers/AV Removal Tools/AV Removal Tools.url b/setup/windows/Uninstallers/AV Removal Tools/AV Removal Tools.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/AV Removal Tools.url rename to setup/windows/Uninstallers/AV Removal Tools/AV Removal Tools.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/AVG.url b/setup/windows/Uninstallers/AV Removal Tools/AVG.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/AVG.url rename to setup/windows/Uninstallers/AV Removal Tools/AVG.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/Avast.url b/setup/windows/Uninstallers/AV Removal Tools/Avast.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/Avast.url rename to setup/windows/Uninstallers/AV Removal Tools/Avast.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/Avira.url b/setup/windows/Uninstallers/AV Removal Tools/Avira.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/Avira.url rename to setup/windows/Uninstallers/AV Removal Tools/Avira.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/ESET.url b/setup/windows/Uninstallers/AV Removal Tools/ESET.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/ESET.url rename to setup/windows/Uninstallers/AV Removal Tools/ESET.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/Kaspersky.url b/setup/windows/Uninstallers/AV Removal Tools/Kaspersky.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/Kaspersky.url rename to setup/windows/Uninstallers/AV Removal Tools/Kaspersky.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/MBAM.url b/setup/windows/Uninstallers/AV Removal Tools/MBAM.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/MBAM.url rename to setup/windows/Uninstallers/AV Removal Tools/MBAM.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/McAfee.url b/setup/windows/Uninstallers/AV Removal Tools/McAfee.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/McAfee.url rename to setup/windows/Uninstallers/AV Removal Tools/McAfee.url diff --git a/.kit_items/Uninstallers/AV Removal Tools/Norton.url b/setup/windows/Uninstallers/AV Removal Tools/Norton.url similarity index 100% rename from .kit_items/Uninstallers/AV Removal Tools/Norton.url rename to setup/windows/Uninstallers/AV Removal Tools/Norton.url diff --git a/.bin/ConEmu/ConEmu.xml b/setup/windows/bin/ConEmu/ConEmu.xml similarity index 100% rename from .bin/ConEmu/ConEmu.xml rename to setup/windows/bin/ConEmu/ConEmu.xml diff --git a/.bin/HWiNFO/general.ini b/setup/windows/bin/HWiNFO/general.ini similarity index 100% rename from .bin/HWiNFO/general.ini rename to setup/windows/bin/HWiNFO/general.ini diff --git a/.bin/_Drivers/SDIO/sdi.cfg b/setup/windows/bin/_Drivers/SDIO/sdi.cfg similarity index 100% rename from .bin/_Drivers/SDIO/sdi.cfg rename to setup/windows/bin/_Drivers/SDIO/sdi.cfg diff --git a/.cbin/_include/AIDA64/full.rpf b/setup/windows/cbin/_include/AIDA64/full.rpf similarity index 100% rename from .cbin/_include/AIDA64/full.rpf rename to setup/windows/cbin/_include/AIDA64/full.rpf diff --git a/.cbin/_include/AIDA64/installed_programs.rpf b/setup/windows/cbin/_include/AIDA64/installed_programs.rpf similarity index 100% rename from .cbin/_include/AIDA64/installed_programs.rpf rename to setup/windows/cbin/_include/AIDA64/installed_programs.rpf diff --git a/.cbin/_include/AIDA64/licenses.rpf b/setup/windows/cbin/_include/AIDA64/licenses.rpf similarity index 100% rename from .cbin/_include/AIDA64/licenses.rpf rename to setup/windows/cbin/_include/AIDA64/licenses.rpf diff --git a/.cbin/_include/BleachBit/BleachBit.ini b/setup/windows/cbin/_include/BleachBit/BleachBit.ini similarity index 100% rename from .cbin/_include/BleachBit/BleachBit.ini rename to setup/windows/cbin/_include/BleachBit/BleachBit.ini diff --git a/.cbin/_include/NotepadPlusPlus/config.xml b/setup/windows/cbin/_include/NotepadPlusPlus/config.xml similarity index 100% rename from .cbin/_include/NotepadPlusPlus/config.xml rename to setup/windows/cbin/_include/NotepadPlusPlus/config.xml diff --git a/.cbin/_include/XMPlay/xmplay.ini b/setup/windows/cbin/_include/XMPlay/xmplay.ini similarity index 100% rename from .cbin/_include/XMPlay/xmplay.ini rename to setup/windows/cbin/_include/XMPlay/xmplay.ini diff --git a/.cbin/_include/XYplorerFree/Data/XYplorer.ini b/setup/windows/cbin/_include/XYplorerFree/Data/XYplorer.ini similarity index 100% rename from .cbin/_include/XYplorerFree/Data/XYplorer.ini rename to setup/windows/cbin/_include/XYplorerFree/Data/XYplorer.ini diff --git a/.cbin/_include/_Drivers/Intel RST/SetupRST_13.x.txt b/setup/windows/cbin/_include/_Drivers/Intel RST/SetupRST_13.x.txt similarity index 100% rename from .cbin/_include/_Drivers/Intel RST/SetupRST_13.x.txt rename to setup/windows/cbin/_include/_Drivers/Intel RST/SetupRST_13.x.txt diff --git a/.cbin/_include/_Office/2016_hb_32.xml b/setup/windows/cbin/_include/_Office/2016_hb_32.xml similarity index 100% rename from .cbin/_include/_Office/2016_hb_32.xml rename to setup/windows/cbin/_include/_Office/2016_hb_32.xml diff --git a/.cbin/_include/_Office/2016_hb_64.xml b/setup/windows/cbin/_include/_Office/2016_hb_64.xml similarity index 100% rename from .cbin/_include/_Office/2016_hb_64.xml rename to setup/windows/cbin/_include/_Office/2016_hb_64.xml diff --git a/.cbin/_include/_Office/2016_hs_32.xml b/setup/windows/cbin/_include/_Office/2016_hs_32.xml similarity index 100% rename from .cbin/_include/_Office/2016_hs_32.xml rename to setup/windows/cbin/_include/_Office/2016_hs_32.xml diff --git a/.cbin/_include/_Office/2016_hs_64.xml b/setup/windows/cbin/_include/_Office/2016_hs_64.xml similarity index 100% rename from .cbin/_include/_Office/2016_hs_64.xml rename to setup/windows/cbin/_include/_Office/2016_hs_64.xml diff --git a/.cbin/_include/_Office/2019_hb_32.xml b/setup/windows/cbin/_include/_Office/2019_hb_32.xml similarity index 100% rename from .cbin/_include/_Office/2019_hb_32.xml rename to setup/windows/cbin/_include/_Office/2019_hb_32.xml diff --git a/.cbin/_include/_Office/2019_hb_64.xml b/setup/windows/cbin/_include/_Office/2019_hb_64.xml similarity index 100% rename from .cbin/_include/_Office/2019_hb_64.xml rename to setup/windows/cbin/_include/_Office/2019_hb_64.xml diff --git a/.cbin/_include/_Office/2019_hs_32.xml b/setup/windows/cbin/_include/_Office/2019_hs_32.xml similarity index 100% rename from .cbin/_include/_Office/2019_hs_32.xml rename to setup/windows/cbin/_include/_Office/2019_hs_32.xml diff --git a/.cbin/_include/_Office/2019_hs_64.xml b/setup/windows/cbin/_include/_Office/2019_hs_64.xml similarity index 100% rename from .cbin/_include/_Office/2019_hs_64.xml rename to setup/windows/cbin/_include/_Office/2019_hs_64.xml diff --git a/.cbin/_include/_Office/365_32.xml b/setup/windows/cbin/_include/_Office/365_32.xml similarity index 100% rename from .cbin/_include/_Office/365_32.xml rename to setup/windows/cbin/_include/_Office/365_32.xml diff --git a/.cbin/_include/_Office/365_64.xml b/setup/windows/cbin/_include/_Office/365_64.xml similarity index 100% rename from .cbin/_include/_Office/365_64.xml rename to setup/windows/cbin/_include/_Office/365_64.xml diff --git a/.cbin/_include/_vcredists/InstallAll.bat b/setup/windows/cbin/_include/_vcredists/InstallAll.bat similarity index 100% rename from .cbin/_include/_vcredists/InstallAll.bat rename to setup/windows/cbin/_include/_vcredists/InstallAll.bat From bca9c19053352fe53c12e3328370c9af15782961 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 29 Jun 2019 19:54:02 -0600 Subject: [PATCH 002/324] New safer text input function * Avoids EOFError exceptions and just repeats the prompt --- scripts/wk/__init__.py | 11 ++++++++++ scripts/wk/io.py | 6 ++++++ scripts/wk/std.py | 49 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 scripts/wk/__init__.py diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py new file mode 100644 index 00000000..df1d92ac --- /dev/null +++ b/scripts/wk/__init__.py @@ -0,0 +1,11 @@ +'''WizardKit: wk module init''' + +import wk.cfg +import wk.exe +import wk.hw +import wk.io +import wk.kit +import wk.net +import wk.os +import wk.std +import wk.sw diff --git a/scripts/wk/io.py b/scripts/wk/io.py index e69de29b..469408c3 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -0,0 +1,6 @@ +'''WizardKit: I/O Functions''' +# vim: sts=2 sw=2 ts=2 + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/std.py b/scripts/wk/std.py index e69de29b..e71eaa27 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -0,0 +1,49 @@ +'''WizardKit: Standard Functions''' +# vim: sts=2 sw=2 ts=2 + +import os +import sys + +try: + from termios import tcflush, TCIOFLUSH +except ImportError: + if os.name == 'posix': + raise + + +# STATIC VARIABLES +COLORS = { + 'CLEAR': '\033[0m', + 'RED': '\033[31m', + 'ORANGE': '\033[31;1m', + 'GREEN': '\033[32m', + 'YELLOW': '\033[33m', + 'BLUE': '\033[34m', + 'PURPLE': '\033[35m', + 'CYAN': '\033[36m', + } + + +# Functions +def input_text(prompt='Enter text'): + """Get text from user, returns string.""" + prompt = str(prompt) + response = None + if prompt[-1:] != ' ': + prompt += ' ' + + while response is None: + if os.name == 'posix': + # Flush input to (hopefully) avoid EOFError + tcflush(sys.stdin, TCIOFLUSH) + try: + response = input(prompt) + except EOFError: + # Ignore and try again + print('') + + return response + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From 96837ff7749f9df7a9d0314649d159c7b6cdbe3a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 29 Jun 2019 19:58:44 -0600 Subject: [PATCH 003/324] Going to use the logging module for logging --- scripts/wk/std.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index e71eaa27..cd86d714 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,6 +1,7 @@ '''WizardKit: Standard Functions''' # vim: sts=2 sw=2 ts=2 +import logging import os import sys @@ -22,6 +23,7 @@ COLORS = { 'PURPLE': '\033[35m', 'CYAN': '\033[36m', } +LOG = logging.getLogger(__name__) # Functions @@ -38,9 +40,11 @@ def input_text(prompt='Enter text'): tcflush(sys.stdin, TCIOFLUSH) try: response = input(prompt) + LOG.debug('%s.input_text response: %s', __name__, response) except EOFError: # Ignore and try again - print('') + LOG.warning('Exception occured', exc_info=True) + print('', flush=True) return response From 0427d2586f342f387607532de4ac4d179602532e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 29 Jun 2019 20:52:38 -0600 Subject: [PATCH 004/324] Added ask and pause functions --- scripts/wk/std.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index cd86d714..b492fa9b 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -3,6 +3,7 @@ import logging import os +import re import sys try: @@ -27,6 +28,20 @@ LOG = logging.getLogger(__name__) # Functions +def ask(prompt='Kotaero!'): + """Prompt the user with a Y/N question, returns bool.""" + answer = None + prompt = '{} [Y/N]: '.format(prompt) + while answer is None: + tmp = input_text(prompt) + if re.search(r'^y(es|)$', tmp, re.IGNORECASE): + answer = True + elif re.search(r'^n(o|ope|)$', tmp, re.IGNORECASE): + answer = False + LOG.info('%s%s', prompt, 'Yes' if answer else 'No') + return answer + + def input_text(prompt='Enter text'): """Get text from user, returns string.""" prompt = str(prompt) @@ -40,7 +55,7 @@ def input_text(prompt='Enter text'): tcflush(sys.stdin, TCIOFLUSH) try: response = input(prompt) - LOG.debug('%s.input_text response: %s', __name__, response) + LOG.debug('%s%s', prompt, response) except EOFError: # Ignore and try again LOG.warning('Exception occured', exc_info=True) @@ -49,5 +64,10 @@ def input_text(prompt='Enter text'): return response +def pause(prompt='Press Enter to continue... '): + """Simple pause implementation.""" + input_text(prompt) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From f7a114ee44e493b9301c001563f01827bbc857fa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 29 Jun 2019 21:23:37 -0600 Subject: [PATCH 005/324] Added cfg/logging.py and cfg/main.py --- scripts/wk/cfg/__init__.py | 4 ++++ scripts/wk/cfg/logging.py | 20 ++++++++++++++++++++ scripts/wk/cfg/main.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 scripts/wk/cfg/__init__.py create mode 100644 scripts/wk/cfg/logging.py create mode 100644 scripts/wk/cfg/main.py diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py new file mode 100644 index 00000000..a33fe168 --- /dev/null +++ b/scripts/wk/cfg/__init__.py @@ -0,0 +1,4 @@ +'''WizardKit: cfg module init''' + +import wk.cfg.logging +import wk.cfg.main diff --git a/scripts/wk/cfg/logging.py b/scripts/wk/cfg/logging.py new file mode 100644 index 00000000..e3501f03 --- /dev/null +++ b/scripts/wk/cfg/logging.py @@ -0,0 +1,20 @@ +'''WizardKit: Config - Logging''' +# vim: sts=2 sw=2 ts=2 + +import logging + + +DEBUG = { + 'level': logging.DEBUG, + 'format': '[%(asctime)s %(levelname)s] [%(name)s.%(funcName)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S %z', + } +DEFAULT = { + 'level': logging.INFO, + 'format': '[%(asctime)s %(levelname)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S %z', + } + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/cfg/main.py b/scripts/wk/cfg/main.py new file mode 100644 index 00000000..60731500 --- /dev/null +++ b/scripts/wk/cfg/main.py @@ -0,0 +1,29 @@ +'''WizardKit: Config - Main + +NOTE: A non-standard format is used for BASH/BATCH/PYTHON compatibility''' +# pylint: disable=bad-whitespace +# vim: sts=2 sw=2 ts=2 + + +# Features +ENABLED_OPEN_LOGS = False +ENABLED_TICKET_NUMBERS = False +ENABLED_UPLOAD_DATA = False + +# Main Kit +ARCHIVE_PASSWORD='Abracadabra' +KIT_NAME_FULL='WizardKit' +KIT_NAME_SHORT='WK' +SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' + +# Live Linux +ROOT_PASSWORD='Abracadabra' +TECH_PASSWORD='Abracadabra' + +# Time Zones +LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values +WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From dfde06a2fd6a4271fc4ec3ebbefbca78dd27c465 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 29 Jun 2019 21:33:47 -0600 Subject: [PATCH 006/324] Updated __init__ files --- scripts/wk/__init__.py | 18 +++++++++--------- scripts/wk/cfg/__init__.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index df1d92ac..4946422a 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -1,11 +1,11 @@ '''WizardKit: wk module init''' -import wk.cfg -import wk.exe -import wk.hw -import wk.io -import wk.kit -import wk.net -import wk.os -import wk.std -import wk.sw +from wk import cfg +from wk import exe +from wk import hw +from wk import io +from wk import kit +from wk import net +from wk import os +from wk import std +from wk import sw diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py index a33fe168..f89a13bf 100644 --- a/scripts/wk/cfg/__init__.py +++ b/scripts/wk/cfg/__init__.py @@ -1,4 +1,4 @@ '''WizardKit: cfg module init''' -import wk.cfg.logging -import wk.cfg.main +from wk.cfg import logging +from wk.cfg import main From beabbd9c7bc30c597acf8ed3d8177a6666b2a0d6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 2 Jul 2019 20:52:28 -0600 Subject: [PATCH 007/324] Bugfix --- scripts/wk.prev/settings/hw_diags.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wk.prev/settings/hw_diags.py b/scripts/wk.prev/settings/hw_diags.py index 048f489b..7957fda2 100644 --- a/scripts/wk.prev/settings/hw_diags.py +++ b/scripts/wk.prev/settings/hw_diags.py @@ -94,9 +94,10 @@ ATTRIBUTES = { }, } ATTRIBUTE_COLORS = ( + # NOTE: The order here is important; least important to most important. + ('Warning', 'YELLOW'), ('Error', 'RED'), ('Maximum', 'PURPLE'), - ('Warning', 'YELLOW'), ) KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' From 4cc54011b74d65b0c518c9791f35bd086273efba Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Jul 2019 22:02:48 -0600 Subject: [PATCH 008/324] Added version check --- scripts/wk/__init__.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 4946422a..4a9a7586 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -1,4 +1,7 @@ '''WizardKit: wk module init''' +# vim: sts=2 sw=2 ts=2 + +import sys from wk import cfg from wk import exe @@ -9,3 +12,21 @@ from wk import net from wk import os from wk import std from wk import sw + +# Check env +if sys.version_info < (3, 5): + # Unsupported + raise RuntimeError( + 'This package is unsupported on Python {major}.{minor}'.format( + **sys.version_info, + )) +if sys.version_info < (3, 7): + # Untested + raise UserWarning( + 'Python {major}.{minor} is untested for this package'.format( + **sys.version_info, + )) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From acd92b3e50b6bd05dcce0419436b58076c14058c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Jul 2019 22:09:22 -0600 Subject: [PATCH 009/324] Added logging functions * Logging is always initialized if importing the whole package * Support switching to DEBUG mode * Support changing the log dir (and optionally log name) --- scripts/wk/__init__.py | 5 ++ scripts/wk/cfg/__init__.py | 2 +- scripts/wk/cfg/{logging.py => log.py} | 8 +-- scripts/wk/log.py | 88 +++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) rename scripts/wk/cfg/{logging.py => log.py} (77%) create mode 100644 scripts/wk/log.py diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 4a9a7586..5df657c4 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -8,11 +8,13 @@ from wk import exe from wk import hw from wk import io from wk import kit +from wk import log from wk import net from wk import os from wk import std from wk import sw + # Check env if sys.version_info < (3, 5): # Unsupported @@ -27,6 +29,9 @@ if sys.version_info < (3, 7): **sys.version_info, )) +# Init +log.start() + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py index f89a13bf..74adea8e 100644 --- a/scripts/wk/cfg/__init__.py +++ b/scripts/wk/cfg/__init__.py @@ -1,4 +1,4 @@ '''WizardKit: cfg module init''' -from wk.cfg import logging +from wk.cfg import log from wk.cfg import main diff --git a/scripts/wk/cfg/logging.py b/scripts/wk/cfg/log.py similarity index 77% rename from scripts/wk/cfg/logging.py rename to scripts/wk/cfg/log.py index e3501f03..925be4d3 100644 --- a/scripts/wk/cfg/logging.py +++ b/scripts/wk/cfg/log.py @@ -1,16 +1,14 @@ -'''WizardKit: Config - Logging''' +'''WizardKit: Config - Log''' # vim: sts=2 sw=2 ts=2 -import logging - DEBUG = { - 'level': logging.DEBUG, + 'level': 'DEBUG', 'format': '[%(asctime)s %(levelname)s] [%(name)s.%(funcName)s] %(message)s', 'datefmt': '%Y-%m-%d %H:%M:%S %z', } DEFAULT = { - 'level': logging.INFO, + 'level': 'INFO', 'format': '[%(asctime)s %(levelname)s] %(message)s', 'datefmt': '%Y-%m-%d %H:%M:%S %z', } diff --git a/scripts/wk/log.py b/scripts/wk/log.py new file mode 100644 index 00000000..d8bf82be --- /dev/null +++ b/scripts/wk/log.py @@ -0,0 +1,88 @@ +'''WizardKit: Log Functions''' +# vim: sts=2 sw=2 ts=2 + +import logging +import os +import pathlib +import shutil +import time + +from . import cfg + + +# STATIC VARIABLES +LOG = logging.getLogger(__name__) + + +# Functions +def enable_debug_mode(): + """Configures logging for better debugging.""" + root_logger = logging.getLogger() + for handler in root_logger.handlers: + formatter = logging.Formatter( + datefmt=cfg.log.DEBUG['datefmt'], + fmt=cfg.log.DEBUG['format'], + ) + handler.setFormatter(formatter) + root_logger.setLevel('DEBUG') + + +def update_log_path(dest_dir, dest_filename=''): + """Copies current log file to new dir and updates the root logger.""" + dest = pathlib.Path(dest_dir) + dest = dest.expanduser() + root_logger = logging.getLogger() + cur_handler = root_logger.handlers[0] + + # Safety checks + if len(root_logger.handlers) > 1: + raise NotImplementedError('update_log_path() only supports a single handler.') + if not isinstance(cur_handler, logging.FileHandler): + raise NotImplementedError('update_log_path() only supports FileHandlers.') + + # Set source + source = pathlib.Path(cur_handler.baseFilename) + source = source.resolve() + + # Copy original log to new location + if dest_filename: + dest = dest.joinpath(dest_filename) + else: + dest = dest.joinpath(source.name) + dest = dest.resolve() + os.makedirs(dest.parent, exist_ok=True) + shutil.copy(source, dest) + + # Create new cur_handler (preserving formatter settings) + new_handler = logging.FileHandler(dest, mode='a') + new_handler.setFormatter(cur_handler.formatter) + + # Replace current handler + root_logger.removeHandler(cur_handler) + root_logger.addHandler(new_handler) + + +def start(config=None): + """Configure and start logging using safe defaults.""" + log_dir = '{}/Logs/'.format(os.path.expanduser('~')) + log_path = '{}/{}_{}.log'.format( + log_dir, + cfg.main.KIT_NAME_FULL, + time.strftime('%Y-%m-%d_%H%M%z'), + ) + root_logger = logging.getLogger() + + # Safety checks + if not config: + config = cfg.log.DEFAULT + if root_logger.hasHandlers(): + raise UserWarning('Logging already started.') + + # Create log_dir + os.makedirs(log_dir, exist_ok=True) + + # Config logger + logging.basicConfig(filename=log_path, **config) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From 23eda17bd3f08e7c4ef846fdcb7a76326974f8dd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Jul 2019 16:55:39 -0600 Subject: [PATCH 010/324] Ensure logging is shutdown when exiting --- scripts/wk/log.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index d8bf82be..b10218fe 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -1,6 +1,7 @@ '''WizardKit: Log Functions''' # vim: sts=2 sw=2 ts=2 +import atexit import logging import os import pathlib @@ -84,5 +85,8 @@ def start(config=None): # Config logger logging.basicConfig(filename=log_path, **config) + # Register shutdown to run atexit + atexit.register(logging.shutdown) + if __name__ == '__main__': print("This file is not meant to be called directly.") From ca67ed392f7a6afadf98b9680fefcb4d59428ada Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Jul 2019 16:56:03 -0600 Subject: [PATCH 011/324] Avoid clobbering existing files --- scripts/wk/log.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index b10218fe..39330de7 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -11,10 +11,6 @@ import time from . import cfg -# STATIC VARIABLES -LOG = logging.getLogger(__name__) - - # Functions def enable_debug_mode(): """Configures logging for better debugging.""" @@ -30,10 +26,12 @@ def enable_debug_mode(): def update_log_path(dest_dir, dest_filename=''): """Copies current log file to new dir and updates the root logger.""" - dest = pathlib.Path(dest_dir) - dest = dest.expanduser() root_logger = logging.getLogger() cur_handler = root_logger.handlers[0] + dest = pathlib.Path(dest_dir) + dest = dest.expanduser() + source = pathlib.Path(cur_handler.baseFilename) + source = source.resolve() # Safety checks if len(root_logger.handlers) > 1: @@ -41,16 +39,14 @@ def update_log_path(dest_dir, dest_filename=''): if not isinstance(cur_handler, logging.FileHandler): raise NotImplementedError('update_log_path() only supports FileHandlers.') - # Set source - source = pathlib.Path(cur_handler.baseFilename) - source = source.resolve() - # Copy original log to new location if dest_filename: dest = dest.joinpath(dest_filename) else: dest = dest.joinpath(source.name) dest = dest.resolve() + if dest.exists(): + raise FileExistsError('Refusing to clobber: {}'.format(dest)) os.makedirs(dest.parent, exist_ok=True) shutil.copy(source, dest) From b98397d491d109eb85f22049cc5d537f6154de09 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Jul 2019 17:32:17 -0600 Subject: [PATCH 012/324] Include seconds in default log name --- scripts/wk/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 39330de7..571c9a77 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -65,7 +65,7 @@ def start(config=None): log_path = '{}/{}_{}.log'.format( log_dir, cfg.main.KIT_NAME_FULL, - time.strftime('%Y-%m-%d_%H%M%z'), + time.strftime('%Y-%m-%d_%H%M%S%z'), ) root_logger = logging.getLogger() From 8b4daa507bea50cf2415bda782881c937216975e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 13 Jul 2019 17:23:35 -0600 Subject: [PATCH 013/324] Use different default log_path under Windows --- scripts/wk/log.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 571c9a77..63947635 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -61,12 +61,21 @@ def update_log_path(dest_dir, dest_filename=''): def start(config=None): """Configure and start logging using safe defaults.""" - log_dir = '{}/Logs/'.format(os.path.expanduser('~')) - log_path = '{}/{}_{}.log'.format( - log_dir, - cfg.main.KIT_NAME_FULL, - time.strftime('%Y-%m-%d_%H%M%S%z'), - ) + if os.name == 'nt': + log_path = '{drive}/{short}/Logs/{date}/{full}/log_{datetime}.log'.format( + drive=os.environ.get('SYSTEMDRIVE', 'C:'), + short=cfg.main.KIT_NAME_SHORT, + date=time.strftime('%y-%m-%d'), + full=cfg.main.KIT_NAME_FULL, + datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), + ) + else: + log_path = '{home}/Logs/{full}_{datetime}.log'.format( + home=os.path.expanduser('~'), + full=cfg.main.KIT_NAME_FULL, + datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), + ) + log_path = pathlib.Path(log_path).resolve() root_logger = logging.getLogger() # Safety checks @@ -76,7 +85,7 @@ def start(config=None): raise UserWarning('Logging already started.') # Create log_dir - os.makedirs(log_dir, exist_ok=True) + os.makedirs(log_path.parent, exist_ok=True) # Config logger logging.basicConfig(filename=log_path, **config) From f30a6dd3db66426581c6f95e6853bf4d73749fba Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 13 Jul 2019 17:29:30 -0600 Subject: [PATCH 014/324] Typo fix --- scripts/wk/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 63947635..6c2eaa97 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -65,7 +65,7 @@ def start(config=None): log_path = '{drive}/{short}/Logs/{date}/{full}/log_{datetime}.log'.format( drive=os.environ.get('SYSTEMDRIVE', 'C:'), short=cfg.main.KIT_NAME_SHORT, - date=time.strftime('%y-%m-%d'), + date=time.strftime('%Y-%m-%d'), full=cfg.main.KIT_NAME_FULL, datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), ) From 36e70c48ba7f6152e8404f5bf1e145c7fdf223de Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 13 Jul 2019 17:42:44 -0600 Subject: [PATCH 015/324] Adjuested log_path under Windows --- scripts/wk/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 6c2eaa97..e6605e24 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -62,7 +62,7 @@ def update_log_path(dest_dir, dest_filename=''): def start(config=None): """Configure and start logging using safe defaults.""" if os.name == 'nt': - log_path = '{drive}/{short}/Logs/{date}/{full}/log_{datetime}.log'.format( + log_path = '{drive}/{short}/Logs/{date}/{full}/{datetime}.log'.format( drive=os.environ.get('SYSTEMDRIVE', 'C:'), short=cfg.main.KIT_NAME_SHORT, date=time.strftime('%Y-%m-%d'), From bdf53b435afb65bd8842dab485214319876ac91e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 13 Jul 2019 17:45:39 -0600 Subject: [PATCH 016/324] Added placeholder files for submodules --- scripts/wk/hw/__init__.py | 1 + scripts/wk/kit/__init__.py | 1 + scripts/wk/os/__init__.py | 1 + scripts/wk/sw/__init__.py | 1 + 4 files changed, 4 insertions(+) create mode 100644 scripts/wk/hw/__init__.py create mode 100644 scripts/wk/kit/__init__.py create mode 100644 scripts/wk/os/__init__.py create mode 100644 scripts/wk/sw/__init__.py diff --git a/scripts/wk/hw/__init__.py b/scripts/wk/hw/__init__.py new file mode 100644 index 00000000..6c19c9c7 --- /dev/null +++ b/scripts/wk/hw/__init__.py @@ -0,0 +1 @@ +'''WizardKit: hw module init''' diff --git a/scripts/wk/kit/__init__.py b/scripts/wk/kit/__init__.py new file mode 100644 index 00000000..61f06215 --- /dev/null +++ b/scripts/wk/kit/__init__.py @@ -0,0 +1 @@ +'''WizardKit: kit module init''' diff --git a/scripts/wk/os/__init__.py b/scripts/wk/os/__init__.py new file mode 100644 index 00000000..da1056d8 --- /dev/null +++ b/scripts/wk/os/__init__.py @@ -0,0 +1 @@ +'''WizardKit: os module init''' diff --git a/scripts/wk/sw/__init__.py b/scripts/wk/sw/__init__.py new file mode 100644 index 00000000..f64f5df6 --- /dev/null +++ b/scripts/wk/sw/__init__.py @@ -0,0 +1 @@ +'''WizardKit: sw module init''' From 69284859657228bc916f37d83678063dfd847b88 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 13 Jul 2019 19:01:57 -0600 Subject: [PATCH 017/324] Added print functions * Uses new method for printing in color (no global var usage) --- scripts/wk/std.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index b492fa9b..eaea806a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,6 +1,7 @@ '''WizardKit: Standard Functions''' # vim: sts=2 sw=2 ts=2 +import itertools import logging import os import re @@ -69,5 +70,54 @@ def pause(prompt='Press Enter to continue... '): input_text(prompt) +def print_colored(strings, colors, **kwargs): + """Prints strings in the colors specified and adds to log.""" + msg = '' + print_options = { + 'end': kwargs.get('end', '\n'), + 'file': kwargs.get('file', sys.stdout), + 'flush': kwargs.get('flush', False), + } + + # Build new string with color escapes added + for string, color in itertools.zip_longest(strings, colors): + msg += '{}{}{}'.format( + COLORS.get(color, COLORS['CLEAR']), + string, + COLORS['CLEAR'], + ) + + print(msg, **print_options) + LOG.log( + level=logging.getLevelName(kwargs.get('level', 'INFO')), + msg=''.join(strings), + ) + + +def print_error(msg, **kwargs): + """Prints message in RED and adds to log.""" + print_colored([mgs], ['RED'], level='ERROR', **kwargs) + + +def print_info(msg, **kwargs): + """Prints message in BLUE and adds to log.""" + print_colored([mgs], ['BLUE'], **kwargs) + + +def print_standard(msg, **kwargs): + """Prints message and adds to log.""" + print_colored([mgs], [None], **kwargs) + + +def print_success(msg, **kwargs): + """Prints message in GREEN and adds to log.""" + print_colored([mgs], ['GREEN'], **kwargs) + + +def print_warning(msg, **kwargs): + """Prints message in YELLOW and adds to log.""" + print_colored([mgs], ['YELLOW'], level='WARNING', **kwargs) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 9da283f7fc67a6a9ed369632770ce6f401bb0a76 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Jul 2019 16:42:28 -0600 Subject: [PATCH 018/324] Adjusted print functions * Logging is now always done at the DEBUG level --- scripts/wk/std.py | 53 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index eaea806a..0dac27de 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -33,12 +33,16 @@ def ask(prompt='Kotaero!'): """Prompt the user with a Y/N question, returns bool.""" answer = None prompt = '{} [Y/N]: '.format(prompt) + + # Loop until acceptable answer is given while answer is None: tmp = input_text(prompt) - if re.search(r'^y(es|)$', tmp, re.IGNORECASE): + if re.search(r'^y(es|up|)$', tmp, re.IGNORECASE): answer = True elif re.search(r'^n(o|ope|)$', tmp, re.IGNORECASE): answer = False + + # Done LOG.info('%s%s', prompt, 'Yes' if answer else 'No') return answer @@ -67,11 +71,13 @@ def input_text(prompt='Enter text'): def pause(prompt='Press Enter to continue... '): """Simple pause implementation.""" + LOG.debug('prompt: %s', prompt) input_text(prompt) def print_colored(strings, colors, **kwargs): - """Prints strings in the colors specified and adds to log.""" + """Prints strings in the colors specified.""" + LOG.debug('strings: %s, colors: %s, kwargs: %s', strings, colors, kwargs) msg = '' print_options = { 'end': kwargs.get('end', '\n'), @@ -88,35 +94,50 @@ def print_colored(strings, colors, **kwargs): ) print(msg, **print_options) - LOG.log( - level=logging.getLevelName(kwargs.get('level', 'INFO')), - msg=''.join(strings), - ) def print_error(msg, **kwargs): - """Prints message in RED and adds to log.""" - print_colored([mgs], ['RED'], level='ERROR', **kwargs) + """Prints message in RED.""" + LOG.debug('msg: %s, kwargs: %s', msg, kwargs) + if 'file' not in kwargs: + # Only set if not specified + kwargs['file'] = sys.stderr + print_colored([msg], ['RED'], **kwargs) def print_info(msg, **kwargs): - """Prints message in BLUE and adds to log.""" - print_colored([mgs], ['BLUE'], **kwargs) + """Prints message in BLUE.""" + LOG.debug('msg: %s, kwargs: %s', msg, kwargs) + print_colored([msg], ['BLUE'], **kwargs) def print_standard(msg, **kwargs): - """Prints message and adds to log.""" - print_colored([mgs], [None], **kwargs) + """Prints message.""" + LOG.debug('msg: %s, kwargs: %s', msg, kwargs) + print_colored([msg], [None], **kwargs) def print_success(msg, **kwargs): - """Prints message in GREEN and adds to log.""" - print_colored([mgs], ['GREEN'], **kwargs) + """Prints message in GREEN.""" + LOG.debug('msg: %s, kwargs: %s', msg, kwargs) + print_colored([msg], ['GREEN'], **kwargs) def print_warning(msg, **kwargs): - """Prints message in YELLOW and adds to log.""" - print_colored([mgs], ['YELLOW'], level='WARNING', **kwargs) + """Prints message in YELLOW.""" + LOG.debug('msg: %s, kwargs: %s', msg, kwargs) + if 'file' not in kwargs: + # Only set if not specified + kwargs['file'] = sys.stderr + print_colored([msg], ['YELLOW'], **kwargs) + + +def strip_colors(string): + """Strip known ANSI color escapes from string, returns str.""" + LOG.debug('string: %s', string) + for color in COLORS.values(): + string = string.replace(color, '') + return string if __name__ == '__main__': From ac7fcb2e0b4e229d36298b2cb4b324ba637681dc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Jul 2019 17:52:21 -0600 Subject: [PATCH 019/324] Added clear_screen() --- scripts/wk/std.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 0dac27de..ec829d63 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -47,6 +47,14 @@ def ask(prompt='Kotaero!'): return answer +def clear_screen(): + """Simple wrapper for clear/cls.""" + if os.name == 'nt': + os.system('cls') + else: + os.system('clear') + + def input_text(prompt='Enter text'): """Get text from user, returns string.""" prompt = str(prompt) From 86d9979a7f3d5f88d84e6a3b5bcb597db888d729 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Jul 2019 17:53:14 -0600 Subject: [PATCH 020/324] Added sleep() --- scripts/wk/std.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index ec829d63..37949af6 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -6,6 +6,7 @@ import logging import os import re import sys +import time try: from termios import tcflush, TCIOFLUSH @@ -148,5 +149,11 @@ def strip_colors(string): return string +def sleep(seconds=2): + """Simple wrapper for time.sleep.""" + LOG.debug('Sleeping for %s seconds', seconds) + time.sleep(seconds) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From b0aa7b0218479f0d5c22b247be77d17f3fd36737 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Jul 2019 17:53:20 -0600 Subject: [PATCH 021/324] Added beep() --- scripts/wk/std.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 37949af6..174c4e92 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -48,6 +48,16 @@ def ask(prompt='Kotaero!'): return answer +def beep(repeat=1): + """Play system bell with optional repeat.""" + # TODO: Verify Windows functionality + while repeat >= 1: + # Print bell char without a newline + print('\a', end='', flush=True) + sleep(0.5) + repeat -= 1 + + def clear_screen(): """Simple wrapper for clear/cls.""" if os.name == 'nt': From 6e43340acbd097f2127a49c4ae33955c5d09dbeb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 17:55:20 -0600 Subject: [PATCH 022/324] Adjusted debug logging --- scripts/wk/std.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 174c4e92..4c786149 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -90,7 +90,6 @@ def input_text(prompt='Enter text'): def pause(prompt='Press Enter to continue... '): """Simple pause implementation.""" - LOG.debug('prompt: %s', prompt) input_text(prompt) @@ -161,7 +160,6 @@ def strip_colors(string): def sleep(seconds=2): """Simple wrapper for time.sleep.""" - LOG.debug('Sleeping for %s seconds', seconds) time.sleep(seconds) From b60f3575274133ad52ed2db584c2d22a7cf51b29 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 17:55:38 -0600 Subject: [PATCH 023/324] Added abort() --- scripts/wk/std.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 4c786149..64a1c492 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -30,6 +30,16 @@ LOG = logging.getLogger(__name__) # Functions +def abort(prompt='Aborted.', show_prompt=True, return_code=1): + """Abort script.""" + print_warning(prompt) + LOG.warning(prompt) + if show_prompt: + sleep(1) + pause(prompt='Press Enter to exit... ') + sys.exit(return_code) + + def ask(prompt='Kotaero!'): """Prompt the user with a Y/N question, returns bool.""" answer = None From 1ffbd29ed503c87ad911a3a3c620ac3b8cb3cf85 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 17:56:23 -0600 Subject: [PATCH 024/324] Added choice() --- scripts/wk/std.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 64a1c492..79aaffdb 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -68,6 +68,28 @@ def beep(repeat=1): repeat -= 1 +def choice(choices, prompt='答えろ！'): + """Choose an option from a provided list, returns str. + + Choices provided will be converted to uppercase and returned as such. + Similar to the commands choice (Windows) and select (Linux).""" + LOG.debug('choices: %s, prompt: %s', choices, prompt) + answer = None + choices = [str(c).upper()[:1] for c in choices] + prompt = '{} [{}]: '.format(prompt, '/'.join(choices)) + regex = '^({})$'.format('|'.join(choices)) + + # Loop until acceptable answer is given + while answer is None: + tmp = input_text(prompt=prompt) + if re.search(regex, tmp, re.IGNORECASE): + answer = tmp.upper() + + # Done + LOG.info('%s %s', prompt, answer) + return answer + + def clear_screen(): """Simple wrapper for clear/cls.""" if os.name == 'nt': From 7a0a618b3f2633631b86aa58366f49d7f9b7c813 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 18:02:42 -0600 Subject: [PATCH 025/324] Reordered functions in wk.std --- scripts/wk/std.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 79aaffdb..0d0989a9 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -182,6 +182,11 @@ def print_warning(msg, **kwargs): print_colored([msg], ['YELLOW'], **kwargs) +def sleep(seconds=2): + """Simple wrapper for time.sleep.""" + time.sleep(seconds) + + def strip_colors(string): """Strip known ANSI color escapes from string, returns str.""" LOG.debug('string: %s', string) @@ -190,10 +195,5 @@ def strip_colors(string): return string -def sleep(seconds=2): - """Simple wrapper for time.sleep.""" - time.sleep(seconds) - - if __name__ == '__main__': print("This file is not meant to be called directly.") From f985c03490584cee5320e52bb6880ac007845aa3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 18:28:05 -0600 Subject: [PATCH 026/324] Added string_to_bytes() * Renamed from convert_to_bytes() for clarity * Now supports KB vs KiB --- scripts/wk/std.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 0d0989a9..3f866545 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -27,6 +27,9 @@ COLORS = { 'CYAN': '\033[36m', } LOG = logging.getLogger(__name__) +REGEX_SIZE_STRING = re.compile( + r'(?P\d+\.?\d*)\s+(?P[PTGMKB])(?PI?)B?' + ) # Functions @@ -187,6 +190,42 @@ def sleep(seconds=2): time.sleep(seconds) +def string_to_bytes(size, assume_binary=False): + """Convert human-readable size str to bytes and return an int.""" + LOG.debug('size: %s, assume_binary: %s', size, assume_binary) + scale = 1000 + size = str(size) + tmp = REGEX_SIZE_STRING.search(size.upper()) + + # Set scale + if tmp.group('binary') or assume_binary: + scale = 1024 + + # Convert to bytes + if tmp: + size = float(tmp.group('size')) + units = tmp.group('units') + if units == 'P': + size *= scale ** 5 + if units == 'T': + size *= scale ** 4 + elif units == 'G': + size *= scale ** 3 + elif units == 'M': + size *= scale ** 2 + elif units == 'K': + size *= scale ** 1 + elif units == 'B': + size *= scale ** 0 + size = int(size) + else: + return -1 + + # Done + LOG.debug('bytes: %s', size) + return size + + def strip_colors(string): """Strip known ANSI color escapes from string, returns str.""" LOG.debug('string: %s', string) From 2cbe99952f9eb0d25abc8ba596035d620c78a205 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 18:38:56 -0600 Subject: [PATCH 027/324] Updated string_to_binary() * Raise an exception if the string can't be parsed as a valid size * Handle strings without spaces (e.g. '1.44mb') * Handle negative values --- scripts/wk/std.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 3f866545..04ae898a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -28,7 +28,7 @@ COLORS = { } LOG = logging.getLogger(__name__) REGEX_SIZE_STRING = re.compile( - r'(?P\d+\.?\d*)\s+(?P[PTGMKB])(?PI?)B?' + r'(?P\-?\d+\.?\d*)\s*(?P[PTGMKB])(?PI?)B?' ) @@ -197,29 +197,30 @@ def string_to_bytes(size, assume_binary=False): size = str(size) tmp = REGEX_SIZE_STRING.search(size.upper()) + # Raise exception if string can't be parsed as a size + if not tmp: + raise ValueError('invalid size string: {}'.format(size)) + # Set scale if tmp.group('binary') or assume_binary: scale = 1024 # Convert to bytes - if tmp: - size = float(tmp.group('size')) - units = tmp.group('units') - if units == 'P': - size *= scale ** 5 - if units == 'T': - size *= scale ** 4 - elif units == 'G': - size *= scale ** 3 - elif units == 'M': - size *= scale ** 2 - elif units == 'K': - size *= scale ** 1 - elif units == 'B': - size *= scale ** 0 - size = int(size) - else: - return -1 + size = float(tmp.group('size')) + units = tmp.group('units') + if units == 'P': + size *= scale ** 5 + if units == 'T': + size *= scale ** 4 + elif units == 'G': + size *= scale ** 3 + elif units == 'M': + size *= scale ** 2 + elif units == 'K': + size *= scale ** 1 + elif units == 'B': + size *= scale ** 0 + size = int(size) # Done LOG.debug('bytes: %s', size) From 04ca9b9fff0c58a703c13b830c3a25377b39cd61 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 19:22:02 -0600 Subject: [PATCH 028/324] Added bytes_to_string() * Renamed from human_readable_size() for clarity * Now supports KB vs KiB * Now supports negative values * Removed width logic as that should be handled elsewhere * Removed auto conversion to bytes if passed a string * An exception will now be raised if an invalid size is given --- scripts/wk/std.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 04ae898a..f66b3f8d 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -71,6 +71,53 @@ def beep(repeat=1): repeat -= 1 +def bytes_to_string(size, decimals=0, use_binary=True): + """Convert size into a human-readable format, returns str.""" + LOG.debug( + 'size: %s, decimals: %s, use_binary: %s', + size, + decimals, + use_binary, + ) + size = float(size) + + # Set scale + scale = 1000 + suffix = 'B' + if use_binary: + scale = 1024 + suffix = 'iB' + + # Convert to sensible units + if abs(size) >= scale ** 5: + size /= scale ** 5 + units = 'P' + suffix + elif abs(size) >= scale ** 4: + size /= scale ** 4 + units = 'T' + suffix + elif abs(size) >= scale ** 3: + size /= scale ** 3 + units = 'G' + suffix + elif abs(size) >= scale ** 2: + size /= scale ** 2 + units = 'M' + suffix + elif abs(size) >= scale ** 1: + size /= scale ** 1 + units = 'K' + suffix + else: + size /= scale ** 0 + units = ' {}B'.format(' ' if use_binary else '') + size = '{size:0.{decimals}f} {units}'.format( + size=size, + decimals=decimals, + units=units, + ) + + # Done + LOG.debug('string: %s', size) + return size + + def choice(choices, prompt='答えろ！'): """Choose an option from a provided list, returns str. From 6e963fe5da96858ea263fc8e70e47676ed3ffcd5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 21:30:20 -0600 Subject: [PATCH 029/324] Added wk/cfg/net.py --- scripts/wk/cfg/__init__.py | 1 + scripts/wk/cfg/net.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 scripts/wk/cfg/net.py diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py index 74adea8e..ff52308f 100644 --- a/scripts/wk/cfg/__init__.py +++ b/scripts/wk/cfg/__init__.py @@ -2,3 +2,4 @@ from wk.cfg import log from wk.cfg import main +from wk.cfg import net diff --git a/scripts/wk/cfg/net.py b/scripts/wk/cfg/net.py new file mode 100644 index 00000000..6d165272 --- /dev/null +++ b/scripts/wk/cfg/net.py @@ -0,0 +1,20 @@ +'''WizardKit: Config - Net''' +# vim: sts=2 sw=2 ts=2 + + +# SERVER VARIABLES +CRASH_SERVER = { + #'Name': 'CrashServer', + #'Url': '', + #'User': '', + #'Pass': '', + } + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From c9d35a0e2fae584486a0058fbf8d5bcc5ce11b1b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 21:31:25 -0600 Subject: [PATCH 030/324] Adjusted update_log_path() * Should result in more uniform log names --- scripts/wk/log.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index e6605e24..6181c32c 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -24,14 +24,19 @@ def enable_debug_mode(): root_logger.setLevel('DEBUG') -def update_log_path(dest_dir, dest_filename=''): - """Copies current log file to new dir and updates the root logger.""" +def update_log_path(dest_dir, dest_name=''): + """Copies current log file to new dir and updates the root logger. + + NOTE: A timestamp and extension will be added to dest_name if provided.""" root_logger = logging.getLogger() cur_handler = root_logger.handlers[0] dest = pathlib.Path(dest_dir) dest = dest.expanduser() - source = pathlib.Path(cur_handler.baseFilename) - source = source.resolve() + if dest_name: + dest_name = '{name}_{datetime}.log'.format( + name=dest_name, + datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), + ) # Safety checks if len(root_logger.handlers) > 1: @@ -40,8 +45,10 @@ def update_log_path(dest_dir, dest_filename=''): raise NotImplementedError('update_log_path() only supports FileHandlers.') # Copy original log to new location - if dest_filename: - dest = dest.joinpath(dest_filename) + source = pathlib.Path(cur_handler.baseFilename) + source = source.resolve() + if dest_name: + dest = dest.joinpath(dest_name) else: dest = dest.joinpath(source.name) dest = dest.resolve() @@ -82,7 +89,7 @@ def start(config=None): if not config: config = cfg.log.DEFAULT if root_logger.hasHandlers(): - raise UserWarning('Logging already started.') + raise UserWarning('Logging already started, results may be unpredictable.') # Create log_dir os.makedirs(log_path.parent, exist_ok=True) From 77238ad41a6308a7c2dee0c5921855ac0193ab68 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 21:32:02 -0600 Subject: [PATCH 031/324] Catch log.start() UserWarning * May revert this down the road --- scripts/wk/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 5df657c4..c52ff157 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -30,7 +30,10 @@ if sys.version_info < (3, 7): )) # Init -log.start() +try: + log.start() +except UserWarning as err: + std.print_warning(err) if __name__ == '__main__': From 5c817717e78f7705e781ac069fc0df163e539aca Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 21:34:14 -0600 Subject: [PATCH 032/324] Added upload_debug_report() * Partial replacement for major_exception() * Splitting that function into smaller parts --- scripts/wk/std.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index f66b3f8d..8c8ee0e6 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -282,5 +282,52 @@ def strip_colors(string): return string +def upload_debug_report(report, reason='DEBUG'): + """Upload debug report to CRASH_SERVER as specified in wk.cfg.net.""" + import pathlib + import requests + from wk.cfg.net import CRASH_SERVER as server + LOG.info('Uploading debug report to %s', server.get('Name', '?')) + + # Check if the required server details are available + if not all(server.get(key, False) for key in ('Name', 'Url', 'User')): + msg = 'Server details missing, aborting upload.' + LOG.error(msg) + print_error(msg) + raise UserWarning(msg) + + # Set filename (based on the logging config if possible) + ## NOTE: This is using a gross assumption on the logging setup + ## That's why there's such a broad exception if something goes wrong + ## TODO: Test under all platforms + root_logger = logging.getLogger() + try: + filename = root_logger.handlers[0].baseFilename + filename = pathlib.Path(filename).name + filename = re.sub(r'^(.*)_(\d{4}-\d{2}-\d{2}.*)', r'\1', filename) + except Exception: #pylint: disable=broad-except + filename = 'Unknown' + filename = '{base}_{reason}_{datetime}.log'.format( + base=filename, + reason=reason, + datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), + ) + LOG.debug('filename: %s', filename) + + # Upload report + url = '{}/{}'.format(server['Url'], filename) + response = requests.put( + url, + data=report, + headers=server.get('Headers', {'X-Requested-With': 'XMLHttpRequest'}), + auth=(server['User'], server.get('Pass', '')), + ) + + # Check response + if not response.ok: + # Using generic exception since we don't care why this failed + raise Exception('Failed to upload report') + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 4cb52a28a6852184a4926ea37309a27fc1e661e8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 21:35:27 -0600 Subject: [PATCH 033/324] Removing useless print_standard() * No longer needed with the separation of printing and logging --- scripts/wk/std.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 8c8ee0e6..a1b193f4 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -211,12 +211,6 @@ def print_info(msg, **kwargs): print_colored([msg], ['BLUE'], **kwargs) -def print_standard(msg, **kwargs): - """Prints message.""" - LOG.debug('msg: %s, kwargs: %s', msg, kwargs) - print_colored([msg], [None], **kwargs) - - def print_success(msg, **kwargs): """Prints message in GREEN.""" LOG.debug('msg: %s, kwargs: %s', msg, kwargs) From 1997cdcefdc03ae8b628966bf7f614c0a49149c5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 17 Jul 2019 21:37:03 -0600 Subject: [PATCH 034/324] Reordered log.py functions --- scripts/wk/log.py | 69 ++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 6181c32c..61cfbdc2 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -24,6 +24,41 @@ def enable_debug_mode(): root_logger.setLevel('DEBUG') +def start(config=None): + """Configure and start logging using safe defaults.""" + if os.name == 'nt': + log_path = '{drive}/{short}/Logs/{date}/{full}/{datetime}.log'.format( + drive=os.environ.get('SYSTEMDRIVE', 'C:'), + short=cfg.main.KIT_NAME_SHORT, + date=time.strftime('%Y-%m-%d'), + full=cfg.main.KIT_NAME_FULL, + datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), + ) + else: + log_path = '{home}/Logs/{full}_{datetime}.log'.format( + home=os.path.expanduser('~'), + full=cfg.main.KIT_NAME_FULL, + datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), + ) + log_path = pathlib.Path(log_path).resolve() + root_logger = logging.getLogger() + + # Safety checks + if not config: + config = cfg.log.DEFAULT + if root_logger.hasHandlers(): + raise UserWarning('Logging already started, results may be unpredictable.') + + # Create log_dir + os.makedirs(log_path.parent, exist_ok=True) + + # Config logger + logging.basicConfig(filename=log_path, **config) + + # Register shutdown to run atexit + atexit.register(logging.shutdown) + + def update_log_path(dest_dir, dest_name=''): """Copies current log file to new dir and updates the root logger. @@ -66,39 +101,5 @@ def update_log_path(dest_dir, dest_name=''): root_logger.addHandler(new_handler) -def start(config=None): - """Configure and start logging using safe defaults.""" - if os.name == 'nt': - log_path = '{drive}/{short}/Logs/{date}/{full}/{datetime}.log'.format( - drive=os.environ.get('SYSTEMDRIVE', 'C:'), - short=cfg.main.KIT_NAME_SHORT, - date=time.strftime('%Y-%m-%d'), - full=cfg.main.KIT_NAME_FULL, - datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), - ) - else: - log_path = '{home}/Logs/{full}_{datetime}.log'.format( - home=os.path.expanduser('~'), - full=cfg.main.KIT_NAME_FULL, - datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), - ) - log_path = pathlib.Path(log_path).resolve() - root_logger = logging.getLogger() - - # Safety checks - if not config: - config = cfg.log.DEFAULT - if root_logger.hasHandlers(): - raise UserWarning('Logging already started, results may be unpredictable.') - - # Create log_dir - os.makedirs(log_path.parent, exist_ok=True) - - # Config logger - logging.basicConfig(filename=log_path, **config) - - # Register shutdown to run atexit - atexit.register(logging.shutdown) - if __name__ == '__main__': print("This file is not meant to be called directly.") From 1829c3b2f3efc1ac4ed745b078872dbeb96fd436 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 25 Jul 2019 21:33:19 -0600 Subject: [PATCH 035/324] Moved CRASH_SERVER to wk.cfg.main --- scripts/wk/cfg/main.py | 7 +++++++ scripts/wk/cfg/net.py | 14 -------------- scripts/wk/std.py | 18 +++++++++++------- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/scripts/wk/cfg/main.py b/scripts/wk/cfg/main.py index 60731500..48ab0088 100644 --- a/scripts/wk/cfg/main.py +++ b/scripts/wk/cfg/main.py @@ -24,6 +24,13 @@ TECH_PASSWORD='Abracadabra' LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values +# Misc +CRASH_SERVER = { + #'Name': 'CrashServer', + #'Url': '', + #'User': '', + #'Pass': '', + } if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/cfg/net.py b/scripts/wk/cfg/net.py index 6d165272..8a84620c 100644 --- a/scripts/wk/cfg/net.py +++ b/scripts/wk/cfg/net.py @@ -2,19 +2,5 @@ # vim: sts=2 sw=2 ts=2 -# SERVER VARIABLES -CRASH_SERVER = { - #'Name': 'CrashServer', - #'Url': '', - #'User': '', - #'Pass': '', - } - - -if __name__ == '__main__': - print("This file is not meant to be called directly.") - -# vim: sts=2 sw=2 ts=2 - if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/std.py b/scripts/wk/std.py index a1b193f4..d23ae35f 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -8,6 +8,8 @@ import re import sys import time +from wk.cfg.main import CRASH_SERVER + try: from termios import tcflush, TCIOFLUSH except ImportError: @@ -277,14 +279,13 @@ def strip_colors(string): def upload_debug_report(report, reason='DEBUG'): - """Upload debug report to CRASH_SERVER as specified in wk.cfg.net.""" + """Upload debug report to CRASH_SERVER as specified in wk.cfg.main.""" import pathlib import requests - from wk.cfg.net import CRASH_SERVER as server - LOG.info('Uploading debug report to %s', server.get('Name', '?')) + LOG.info('Uploading debug report to %s', CRASH_SERVER.get('Name', '?')) # Check if the required server details are available - if not all(server.get(key, False) for key in ('Name', 'Url', 'User')): + if not all(CRASH_SERVER.get(key, False) for key in ('Name', 'Url', 'User')): msg = 'Server details missing, aborting upload.' LOG.error(msg) print_error(msg) @@ -309,12 +310,15 @@ def upload_debug_report(report, reason='DEBUG'): LOG.debug('filename: %s', filename) # Upload report - url = '{}/{}'.format(server['Url'], filename) + url = '{}/{}'.format(CRASH_SERVER['Url'], filename) response = requests.put( url, data=report, - headers=server.get('Headers', {'X-Requested-With': 'XMLHttpRequest'}), - auth=(server['User'], server.get('Pass', '')), + headers=CRASH_SERVER.get( + 'Headers', + {'X-Requested-With': 'XMLHttpRequest'}, + ), + auth=(CRASH_SERVER['User'], CRASH_SERVER.get('Pass', '')), ) # Check response From f1d53e698b9881e753b3225c2127442e7e6263b1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 25 Jul 2019 21:34:22 -0600 Subject: [PATCH 036/324] Added major_exception() --- scripts/wk/std.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index d23ae35f..abf9eba4 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -7,8 +7,9 @@ import os import re import sys import time +import traceback -from wk.cfg.main import CRASH_SERVER +from wk.cfg.main import CRASH_SERVER, ENABLED_UPLOAD_DATA, SUPPORT_MESSAGE try: from termios import tcflush, TCIOFLUSH @@ -172,6 +173,37 @@ def input_text(prompt='Enter text'): return response +def major_exception(): + """Display traceback, optionally upload detailes, and exit.""" + LOG.critical('Major exception encountered', exc_info=True) + print_error('Major exception') + print_warning(SUPPORT_MESSAGE) + print(traceback.format_exc()) + + # Build report + ## TODO + report = 'TODO\n' + + # Upload details + prompt = 'Upload details to {}?'.format( + CRASH_SERVER.get('Name', '?'), + ) + if ENABLED_UPLOAD_DATA and ask(prompt): + print('Uploading... ', end='', flush=True) + try: + upload_debug_report(report, reason='CRASH') + except Exception: #pylint: disable=broad-except + print_colored(['FAILED'], ['RED']) + LOG.error('Upload failed') + else: + print_success('SUCCESS') + LOG.info('Upload successful') + + # Done + pause('Press Enter to exit... ') + raise SystemExit(1) + + def pause(prompt='Press Enter to continue... '): """Simple pause implementation.""" input_text(prompt) From 0757b9fe552bfc2faebe77e624d2f20e757b8aed Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 25 Jul 2019 22:03:15 -0600 Subject: [PATCH 037/324] Added get_log_filepath() * Much safer method than what was in upload_debug_report() --- scripts/wk/std.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index abf9eba4..64efc650 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -4,6 +4,7 @@ import itertools import logging import os +import pathlib import re import sys import time @@ -151,6 +152,24 @@ def clear_screen(): os.system('clear') +def get_log_filepath(): + """Get the log filepath from the root logger, returns pathlib.Path obj. + + NOTE: This will use the first handler baseFilename it finds (if any).""" + # TODO: Test under all platforms + log_filepath = None + root_logger = logging.getLogger() + + # Check handlers + for handler in root_logger.handlers: + if hasattr(handler, 'baseFilename'): + log_filepath = pathlib.Path(handler.baseFilename).resolve() + break + + # Done + return log_filepath + + def input_text(prompt='Enter text'): """Get text from user, returns string.""" prompt = str(prompt) @@ -312,7 +331,6 @@ def strip_colors(string): def upload_debug_report(report, reason='DEBUG'): """Upload debug report to CRASH_SERVER as specified in wk.cfg.main.""" - import pathlib import requests LOG.info('Uploading debug report to %s', CRASH_SERVER.get('Name', '?')) @@ -324,18 +342,13 @@ def upload_debug_report(report, reason='DEBUG'): raise UserWarning(msg) # Set filename (based on the logging config if possible) - ## NOTE: This is using a gross assumption on the logging setup - ## That's why there's such a broad exception if something goes wrong - ## TODO: Test under all platforms - root_logger = logging.getLogger() - try: - filename = root_logger.handlers[0].baseFilename - filename = pathlib.Path(filename).name - filename = re.sub(r'^(.*)_(\d{4}-\d{2}-\d{2}.*)', r'\1', filename) - except Exception: #pylint: disable=broad-except - filename = 'Unknown' - filename = '{base}_{reason}_{datetime}.log'.format( - base=filename, + filename = 'Unknown' + log_path = get_log_filepath() + if log_path: + # Strip everything but the prefix + filename = re.sub(r'^(.*)_(\d{4}-\d{2}-\d{2}.*)', r'\1', log_path.name) + filename = '{prefix}_{reason}_{datetime}.log'.format( + prefix=filename, reason=reason, datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), ) From a60e298f02ca96c053fcc7195993a4531d2ddec2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 6 Aug 2019 19:29:40 -0600 Subject: [PATCH 038/324] Updated bytes_to_string() --- scripts/wk/std.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 64efc650..e7b4aa1e 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -84,6 +84,7 @@ def bytes_to_string(size, decimals=0, use_binary=True): use_binary, ) size = float(size) + abs_size = abs(size) # Set scale scale = 1000 @@ -93,19 +94,19 @@ def bytes_to_string(size, decimals=0, use_binary=True): suffix = 'iB' # Convert to sensible units - if abs(size) >= scale ** 5: + if abs_size >= scale ** 5: size /= scale ** 5 units = 'P' + suffix - elif abs(size) >= scale ** 4: + elif abs_size >= scale ** 4: size /= scale ** 4 units = 'T' + suffix - elif abs(size) >= scale ** 3: + elif abs_size >= scale ** 3: size /= scale ** 3 units = 'G' + suffix - elif abs(size) >= scale ** 2: + elif abs_size >= scale ** 2: size /= scale ** 2 units = 'M' + suffix - elif abs(size) >= scale ** 1: + elif abs_size >= scale ** 1: size /= scale ** 1 units = 'K' + suffix else: From a0027122c961be299e2cbc6490e7a075a53a81b7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 6 Aug 2019 19:49:30 -0600 Subject: [PATCH 039/324] Added set_title() * Only Windows is supported ATM --- scripts/wk/std.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index e7b4aa1e..867b5a0c 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -280,6 +280,14 @@ def print_warning(msg, **kwargs): print_colored([msg], ['YELLOW'], **kwargs) +def set_title(title): + """Set window title.""" + if os.name == 'nt': + os.system('title {}'.format(title)) + else: + raise NotImplementedError + + def sleep(seconds=2): """Simple wrapper for time.sleep.""" time.sleep(seconds) From 2f720210e9fd67fe11cf2919561f58e42042e3e1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 6 Aug 2019 20:56:03 -0600 Subject: [PATCH 040/324] Added generate_debug_report() --- scripts/wk/std.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 867b5a0c..7e67cdd4 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -5,6 +5,7 @@ import itertools import logging import os import pathlib +import platform import re import sys import time @@ -171,6 +172,40 @@ def get_log_filepath(): return log_filepath +def generate_debug_report(): + """Generate debug report with various runtime details, returns str.""" + import socket + report = [] + func_list = ( + 'architecture', + 'machine', + 'platform', + 'python_version', + ) + + # Platform + report.append('[Platform]') + report.append(' {:<24} {}'.format( + 'FQDN', + socket.getfqdn(), + )) + for func in func_list: + report.append(' {:<24} {}'.format( + func.replace('_', ' ').title(), + getattr(platform, func)(), + )) + report.append('') + + # Environment + report.append('[Environment Variables]') + for key, value in sorted(os.environ.items()): + report.append(' {:<24} {}'.format(key, value)) + report.append('') + + # Done + return '\n'.join(report) + + def input_text(prompt='Enter text'): """Get text from user, returns string.""" prompt = str(prompt) From 5d40d74c4620fcd39cf6de1d6685da4ac01ae207 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 6 Aug 2019 21:10:13 -0600 Subject: [PATCH 041/324] Include sys.argv in debug report --- scripts/wk/std.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 7e67cdd4..a7755bc5 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -175,25 +175,22 @@ def get_log_filepath(): def generate_debug_report(): """Generate debug report with various runtime details, returns str.""" import socket - report = [] - func_list = ( + platform_function_list = ( 'architecture', 'machine', 'platform', 'python_version', ) + report = [] - # Platform - report.append('[Platform]') - report.append(' {:<24} {}'.format( - 'FQDN', - socket.getfqdn(), - )) - for func in func_list: - report.append(' {:<24} {}'.format( - func.replace('_', ' ').title(), - getattr(platform, func)(), - )) + # System + report.append('[System]') + report.append(' {:<24} {}'.format('FQDN', socket.getfqdn())) + for func in platform_function_list: + func_name = func.replace('_', ' ').capitalize() + func_result = getattr(platform, func)() + report.append(' {:<24} {}'.format(func_name, func_result)) + report.append(' {:<24} {}'.format('Python sys.argv', sys.argv)) report.append('') # Environment From bde8d33f20baacde528de1935f582bad8596accd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 6 Aug 2019 21:51:43 -0600 Subject: [PATCH 042/324] Include logging data in debug report if available --- scripts/wk/std.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index a7755bc5..4f49f924 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -173,7 +173,7 @@ def get_log_filepath(): def generate_debug_report(): - """Generate debug report with various runtime details, returns str.""" + """Generate debug report, returns str.""" import socket platform_function_list = ( 'architecture', @@ -183,7 +183,19 @@ def generate_debug_report(): ) report = [] + # Logging data + log_path = get_log_filepath() + if log_path: + report.append('------ Start Log -------') + report.append('') + with open(log_path, 'r') as log_file: + report.extend(log_file.read().splitlines()) + report.append('') + report.append('------- End Log --------') + # System + report.append('--- Start debug info ---') + report.append('') report.append('[System]') report.append(' {:<24} {}'.format('FQDN', socket.getfqdn())) for func in platform_function_list: @@ -200,6 +212,7 @@ def generate_debug_report(): report.append('') # Done + report.append('---- End debug info ----') return '\n'.join(report) From f75feca345de9c022f974f6ecd2f08ba4d3885e0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 6 Aug 2019 21:52:11 -0600 Subject: [PATCH 043/324] get_log_filepath() is working under all platforms --- scripts/wk/std.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 4f49f924..023f656f 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -158,7 +158,6 @@ def get_log_filepath(): """Get the log filepath from the root logger, returns pathlib.Path obj. NOTE: This will use the first handler baseFilename it finds (if any).""" - # TODO: Test under all platforms log_filepath = None root_logger = logging.getLogger() From 2270236e4228a34549000ae6a8137603ca96e2c0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 7 Aug 2019 19:25:02 -0600 Subject: [PATCH 044/324] major_exception() working --- scripts/wk/std.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 023f656f..33737fca 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -245,8 +245,7 @@ def major_exception(): print(traceback.format_exc()) # Build report - ## TODO - report = 'TODO\n' + report = generate_debug_report() # Upload details prompt = 'Upload details to {}?'.format( @@ -258,7 +257,7 @@ def major_exception(): upload_debug_report(report, reason='CRASH') except Exception: #pylint: disable=broad-except print_colored(['FAILED'], ['RED']) - LOG.error('Upload failed') + LOG.error('Upload failed', exc_info=True) else: print_success('SUCCESS') LOG.info('Upload successful') @@ -384,8 +383,8 @@ def strip_colors(string): def upload_debug_report(report, reason='DEBUG'): """Upload debug report to CRASH_SERVER as specified in wk.cfg.main.""" - import requests LOG.info('Uploading debug report to %s', CRASH_SERVER.get('Name', '?')) + import requests # Check if the required server details are available if not all(CRASH_SERVER.get(key, False) for key in ('Name', 'Url', 'User')): From cf5b546183c560c3ec3a25c38d47bd47ec3cdaad Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 7 Aug 2019 19:27:49 -0600 Subject: [PATCH 045/324] Compress report before uploading * It's the new default but it can be disabled --- scripts/wk/cfg/main.py | 9 +++++---- scripts/wk/std.py | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/wk/cfg/main.py b/scripts/wk/cfg/main.py index 48ab0088..e73b5758 100644 --- a/scripts/wk/cfg/main.py +++ b/scripts/wk/cfg/main.py @@ -26,10 +26,11 @@ WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values # Misc CRASH_SERVER = { - #'Name': 'CrashServer', - #'Url': '', - #'User': '', - #'Pass': '', + 'Name': 'CrashServer', + 'Url': '', + 'User': '', + 'Pass': '', + 'Headers': {'X-Requested-With': 'XMLHttpRequest'}, } if __name__ == '__main__': diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 33737fca..17a24690 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -3,6 +3,7 @@ import itertools import logging +import lzma import os import pathlib import platform @@ -381,10 +382,13 @@ def strip_colors(string): return string -def upload_debug_report(report, reason='DEBUG'): +def upload_debug_report(report, compress=True, reason='DEBUG'): """Upload debug report to CRASH_SERVER as specified in wk.cfg.main.""" LOG.info('Uploading debug report to %s', CRASH_SERVER.get('Name', '?')) import requests + headers = CRASH_SERVER.get('Headers', {'X-Requested-With': 'XMLHttpRequest'}) + if compress: + headers['Content-Type'] = 'application/octet-stream' # Check if the required server details are available if not all(CRASH_SERVER.get(key, False) for key in ('Name', 'Url', 'User')): @@ -406,15 +410,17 @@ def upload_debug_report(report, reason='DEBUG'): ) LOG.debug('filename: %s', filename) + # Compress report + if compress: + filename += '.xz' + xz_report = lzma.compress(report.encode('utf8')) + # Upload report url = '{}/{}'.format(CRASH_SERVER['Url'], filename) response = requests.put( url, - data=report, - headers=CRASH_SERVER.get( - 'Headers', - {'X-Requested-With': 'XMLHttpRequest'}, - ), + data=xz_report if compress else report, + headers=headers, auth=(CRASH_SERVER['User'], CRASH_SERVER.get('Pass', '')), ) From 11d9a5203ef632a86f0236f0d02a899507f89490 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 16:42:00 -0600 Subject: [PATCH 046/324] Adjusted docstrings --- scripts/wk/log.py | 3 ++- scripts/wk/std.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 61cfbdc2..91fa12b0 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -62,7 +62,8 @@ def start(config=None): def update_log_path(dest_dir, dest_name=''): """Copies current log file to new dir and updates the root logger. - NOTE: A timestamp and extension will be added to dest_name if provided.""" + NOTE: A timestamp and extension will be added to dest_name if provided. + """ root_logger = logging.getLogger() cur_handler = root_logger.handlers[0] dest = pathlib.Path(dest_dir) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 17a24690..b9f24c1c 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -129,7 +129,8 @@ def choice(choices, prompt='答えろ！'): """Choose an option from a provided list, returns str. Choices provided will be converted to uppercase and returned as such. - Similar to the commands choice (Windows) and select (Linux).""" + Similar to the commands choice (Windows) and select (Linux). + """ LOG.debug('choices: %s, prompt: %s', choices, prompt) answer = None choices = [str(c).upper()[:1] for c in choices] @@ -158,7 +159,8 @@ def clear_screen(): def get_log_filepath(): """Get the log filepath from the root logger, returns pathlib.Path obj. - NOTE: This will use the first handler baseFilename it finds (if any).""" + NOTE: This will use the first handler baseFilename it finds (if any). + """ log_filepath = None root_logger = logging.getLogger() @@ -346,7 +348,7 @@ def string_to_bytes(size, assume_binary=False): # Raise exception if string can't be parsed as a size if not tmp: - raise ValueError('invalid size string: {}'.format(size)) + raise ValueError('Invalid size string: {}'.format(size)) # Set scale if tmp.group('binary') or assume_binary: From 36e2ad8522dfad7a9b5eab385db73d016d75c80a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 17:17:46 -0600 Subject: [PATCH 047/324] Updated cfg/main * Added INDENT and WIDTH --- scripts/wk/cfg/main.py | 16 +++++++++++----- scripts/wk/std.py | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/wk/cfg/main.py b/scripts/wk/cfg/main.py index e73b5758..6f5ac11f 100644 --- a/scripts/wk/cfg/main.py +++ b/scripts/wk/cfg/main.py @@ -6,9 +6,9 @@ NOTE: A non-standard format is used for BASH/BATCH/PYTHON compatibility''' # Features -ENABLED_OPEN_LOGS = False -ENABLED_TICKET_NUMBERS = False -ENABLED_UPLOAD_DATA = False +ENABLED_OPEN_LOGS=False +ENABLED_TICKET_NUMBERS=False +ENABLED_UPLOAD_DATA=False # Main Kit ARCHIVE_PASSWORD='Abracadabra' @@ -16,13 +16,19 @@ KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' +# Text Formatting +INDENT=4 +WIDTH=32 + # Live Linux ROOT_PASSWORD='Abracadabra' TECH_PASSWORD='Abracadabra' # Time Zones -LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values -WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values +## See 'timedatectl list-timezones' for valid Linux values +## See 'tzutil /l' for valid Windows values +LINUX_TIME_ZONE='America/Denver' +WINDOWS_TIME_ZONE='Mountain Standard Time' # Misc CRASH_SERVER = { diff --git a/scripts/wk/std.py b/scripts/wk/std.py index b9f24c1c..41c592ed 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -13,6 +13,7 @@ import time import traceback from wk.cfg.main import CRASH_SERVER, ENABLED_UPLOAD_DATA, SUPPORT_MESSAGE +from wk.cfg.main import INDENT, WIDTH try: from termios import tcflush, TCIOFLUSH From ff4e371b320ff0be668eb6d6d9c1e222093a7bc4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 19:00:01 -0600 Subject: [PATCH 048/324] Added get_exception() and try_and_print() * try_and_print needs the format_..() functions finished before it can be used * Raised minimum Python version to 3.7 * Probably could go with 3.6 but meh --- scripts/wk/__init__.py | 8 +-- scripts/wk/std.py | 107 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 10 deletions(-) diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index c52ff157..efd0af93 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -16,18 +16,12 @@ from wk import sw # Check env -if sys.version_info < (3, 5): +if sys.version_info < (3, 7): # Unsupported raise RuntimeError( 'This package is unsupported on Python {major}.{minor}'.format( **sys.version_info, )) -if sys.version_info < (3, 7): - # Untested - raise UserWarning( - 'Python {major}.{minor} is untested for this package'.format( - **sys.version_info, - )) # Init try: diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 41c592ed..5e274073 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,5 +1,6 @@ '''WizardKit: Standard Functions''' # vim: sts=2 sw=2 ts=2 +#TODO Replace .format()s with f-strings import itertools import logging @@ -12,15 +13,15 @@ import sys import time import traceback -from wk.cfg.main import CRASH_SERVER, ENABLED_UPLOAD_DATA, SUPPORT_MESSAGE -from wk.cfg.main import INDENT, WIDTH - try: from termios import tcflush, TCIOFLUSH except ImportError: if os.name == 'posix': raise +from wk.cfg.main import CRASH_SERVER, ENABLED_UPLOAD_DATA, SUPPORT_MESSAGE +from wk.cfg.main import INDENT, WIDTH + # STATIC VARIABLES COLORS = { @@ -157,6 +158,27 @@ def clear_screen(): os.system('clear') +def format_exception_message(_exception, indent=INDENT, width=WIDTH): + """TODO""" + return 'TODO' + + +def format_function_output(output, indent=INDENT, width=WIDTH): + """TODO""" + return 'TODO' + + +def get_exception(name): + """Get exception by name, returns exception object.""" + LOG.debug('Getting exception: %s', name) + try: + obj = getattr(sys.modules[__name__], name) + except AttributeError: + # Try builtin classes + obj = getattr(sys.modules['builtins'], name) + return obj + + def get_log_filepath(): """Get the log filepath from the root logger, returns pathlib.Path obj. @@ -385,6 +407,85 @@ def strip_colors(string): return string +def try_and_print( + message, function, *args, + msg_good='CS', msg_bad='NS', indent=INDENT, width=WIDTH, + w_exceptions=None, e_exceptions=None, + catch_all=True, print_return=False, verbose=False, + **kwargs): + # pylint: disable=catching-non-exception,unused-argument,too-many-locals + """Run a function and print the results, returns results as dict. + + If catch_all is True then (nearly) all exceptions will be caught. + Otherwise if an exception occurs that wasn't specified it will be + re-raised. + + If print_return is True then the output from the function will be used + instead of msg_good, msg_bad, or exception text. The output should be + a list or a subprocess.CompletedProcess object. + + If verbose is True then exception names or messages will be used for + the result message. Otherwise it will simply be set to result_bad. + + If specified w_exceptions and e_exceptions should be lists of + exception class names. Details from the excceptions will be used to + format more clear result messages. + """ + LOG.debug('function: %s.%s', function.__module__, function.__name__) + LOG.debug('args: %s', args) + LOG.debug('kwargs: %s', kwargs) + LOG.debug('w_exceptions: %s', w_exceptions) + LOG.debug('e_exceptions: %s', e_exceptions) + LOG.debug( + 'catch_all: %s, print_return: %s, verbose: %s', + catch_all, + print_return, + verbose, + ) + f_exception = None + output = None + result_msg = 'UNKNOWN' + w_exceptions = tuple(get_exception(e) for e in w_exceptions) + e_exceptions = tuple(get_exception(e) for e in e_exceptions) + + # Run function and catch exceptions + print(f'{" "*indent}{message:<{width}}', end='', flush=True) + LOG.info('Running function: %s.%s', function.__module__, function.__name__) + try: + output = function(*args, **kwargs) + if print_return: + result_msg = format_function_output(output, indent, width) + else: + result_msg = msg_good + print_success(result_msg) + except w_exceptions as _exception: + result_msg = format_exception_message(_exception, indent, width) + print_warning(result_msg) + f_exception = _exception + except e_exceptions as _exception: + result_msg = format_exception_message(_exception, indent, width) + print_error(result_msg) + f_exception = _exception + except Exception as _exception: # pylint: disable=broad-except + if verbose: + result_msg = format_exception_message(_exception, indent, width) + else: + result_msg = msg_bad + print_error(result_msg) + f_exception = _exception + + # Re-raise error if necessary + if f_exception and not catch_all: + raise #pylint: disable=misplaced-bare-raise + + # Done + return { + 'Failed': bool(f_exception), + 'Exception': f_exception, + 'Output': output, + } + + def upload_debug_report(report, compress=True, reason='DEBUG'): """Upload debug report to CRASH_SERVER as specified in wk.cfg.main.""" LOG.info('Uploading debug report to %s', CRASH_SERVER.get('Name', '?')) From 4e5bef23daba4349b28ac2c8b57951239c1ce544 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 19:06:06 -0600 Subject: [PATCH 049/324] Added format_function_output() --- scripts/wk/std.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 5e274073..efcac245 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -13,6 +13,7 @@ import sys import time import traceback +from subprocess import CompletedProcess try: from termios import tcflush, TCIOFLUSH except ImportError: @@ -164,8 +165,32 @@ def format_exception_message(_exception, indent=INDENT, width=WIDTH): def format_function_output(output, indent=INDENT, width=WIDTH): - """TODO""" - return 'TODO' + """Format function output for use in try_and_print(), returns str.""" + LOG.debug('formatting output: %s', output) + + # Ensure we're working with a list + if isinstance(output, CompletedProcess): + stdout = output.stdout + if not isinstance(stdout, str): + stdout = stdout.decode('utf8') + output = stdout.strip().splitlines() + else: + output = list(output) + + # Safety check + if not output: + # Going to ignore empty function output for now + LOG.error('Output is empty') + return 'UNKNOWN' + + # Build result_msg + result_msg = f'{output.pop(0)}' + if output: + output = [f'{" "*(indent+width)}{line}' for line in output] + result_msg += '\n' + '\n'.join(output) + + # Done + return result_msg def get_exception(name): From 147b9d2035907a2cd17cca9e11c44f8c539ab369 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 19:50:45 -0600 Subject: [PATCH 050/324] Added format_exception_message() --- scripts/wk/std.py | 50 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index efcac245..af3b0828 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -13,7 +13,7 @@ import sys import time import traceback -from subprocess import CompletedProcess +from subprocess import CalledProcessError, CompletedProcess try: from termios import tcflush, TCIOFLUSH except ImportError: @@ -160,13 +160,55 @@ def clear_screen(): def format_exception_message(_exception, indent=INDENT, width=WIDTH): - """TODO""" - return 'TODO' + """Format using the exception's args or name, returns str.""" + # pylint: disable=broad-except + LOG.debug('Formatting exception: %s', _exception) + message = None + + # Use known argument index or first string found + try: + if isinstance(_exception, CalledProcessError): + message = _exception.stderr + if not isinstance(message, str): + message = message.decode('utf-8') + message = message.strip() + elif isinstance(_exception, FileNotFoundError): + message = _exception.args[1] + else: + for arg in _exception.args: + if isinstance(arg, str): + message = arg + break + except Exception: + # Just use the exception name instead + pass + + # Safety check + if not message: + try: + message = _exception.__class__.__name__ + except Exception: + message = 'UNKNOWN ERROR' + + # Fix multi-line messages + if '\n' in message: + try: + lines = [ + f'{" "*(indent+width)}{line.strip()}' + for line in message.splitlines() if line.strip() + ] + lines[0] = lines[0].strip() + message = '\n'.join(lines) + except Exception: + pass + + # Done + return message def format_function_output(output, indent=INDENT, width=WIDTH): """Format function output for use in try_and_print(), returns str.""" - LOG.debug('formatting output: %s', output) + LOG.debug('Formatting output: %s', output) # Ensure we're working with a list if isinstance(output, CompletedProcess): From d722754f122d0987913e8e3af0629f0eba183453 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 20:10:13 -0600 Subject: [PATCH 051/324] Replaced UserWarnings with proper exceptions --- scripts/wk/std.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index af3b0828..f9dc4a45 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -349,7 +349,7 @@ def major_exception(): try: upload_debug_report(report, reason='CRASH') except Exception: #pylint: disable=broad-except - print_colored(['FAILED'], ['RED']) + print_error('FAILED') LOG.error('Upload failed', exc_info=True) else: print_success('SUCCESS') @@ -566,7 +566,7 @@ def upload_debug_report(report, compress=True, reason='DEBUG'): msg = 'Server details missing, aborting upload.' LOG.error(msg) print_error(msg) - raise UserWarning(msg) + raise RuntimeError(msg) # Set filename (based on the logging config if possible) filename = 'Unknown' @@ -597,8 +597,7 @@ def upload_debug_report(report, compress=True, reason='DEBUG'): # Check response if not response.ok: - # Using generic exception since we don't care why this failed - raise Exception('Failed to upload report') + raise RuntimeError('Failed to upload report') if __name__ == '__main__': From 4100c382807f4522b8984de446457c75f2e0498f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 20:34:05 -0600 Subject: [PATCH 052/324] Added generic exception classes --- scripts/wk/std.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index f9dc4a45..ca1bd360 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -20,8 +20,13 @@ except ImportError: if os.name == 'posix': raise -from wk.cfg.main import CRASH_SERVER, ENABLED_UPLOAD_DATA, SUPPORT_MESSAGE -from wk.cfg.main import INDENT, WIDTH +from wk.cfg.main import ( + CRASH_SERVER, + ENABLED_UPLOAD_DATA, + INDENT, + SUPPORT_MESSAGE, + WIDTH, + ) # STATIC VARIABLES @@ -41,6 +46,20 @@ REGEX_SIZE_STRING = re.compile( ) +# Exception Classes +class GenericAbort(Exception): + """Exception used for aborts selected by the user at runtime.""" + +class GenericError(Exception): + """Exception used when the built-in exceptions don't fit.""" + +class GenericWarning(Exception): + """Exception used to highlight non-critical events. + + NOTE: Avoiding built-in warning exceptions in case the + warnings filter has been changed from the default. + """ + # Functions def abort(prompt='Aborted.', show_prompt=True, return_code=1): """Abort script.""" From b314a9f1e2b69c7e418776317aa09187e26c4dc9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 9 Aug 2019 21:57:41 -0600 Subject: [PATCH 053/324] Bugfix updates for try_and_print() --- scripts/wk/std.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index ca1bd360..1615e958 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -181,7 +181,10 @@ def clear_screen(): def format_exception_message(_exception, indent=INDENT, width=WIDTH): """Format using the exception's args or name, returns str.""" # pylint: disable=broad-except - LOG.debug('Formatting exception: %s', _exception) + LOG.debug( + 'Formatting exception: %s', + _exception.__class__.__name__, + ) message = None # Use known argument index or first string found @@ -193,6 +196,8 @@ def format_exception_message(_exception, indent=INDENT, width=WIDTH): message = message.strip() elif isinstance(_exception, FileNotFoundError): message = _exception.args[1] + elif isinstance(_exception, ZeroDivisionError): + message = 'ZeroDivisionError' else: for arg in _exception.args: if isinstance(arg, str): @@ -229,6 +234,9 @@ def format_function_output(output, indent=INDENT, width=WIDTH): """Format function output for use in try_and_print(), returns str.""" LOG.debug('Formatting output: %s', output) + if not output: + raise GenericWarning('No output') + # Ensure we're working with a list if isinstance(output, CompletedProcess): stdout = output.stdout @@ -531,6 +539,12 @@ def try_and_print( f_exception = None output = None result_msg = 'UNKNOWN' + + # Build tuples of exceptions + if not w_exceptions: + w_exceptions = ('GenericWarning',) + if not e_exceptions: + e_exceptions = ('GenericError',) w_exceptions = tuple(get_exception(e) for e in w_exceptions) e_exceptions = tuple(get_exception(e) for e in e_exceptions) @@ -541,9 +555,9 @@ def try_and_print( output = function(*args, **kwargs) if print_return: result_msg = format_function_output(output, indent, width) + print(result_msg) else: - result_msg = msg_good - print_success(result_msg) + print_success(msg_good) except w_exceptions as _exception: result_msg = format_exception_message(_exception, indent, width) print_warning(result_msg) @@ -559,10 +573,9 @@ def try_and_print( result_msg = msg_bad print_error(result_msg) f_exception = _exception - - # Re-raise error if necessary - if f_exception and not catch_all: - raise #pylint: disable=misplaced-bare-raise + if not catch_all: + # Re-raise error as necessary + raise # Done return { From c43539a92dcc8e15ee4c12b60a4967da03277046 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 16:00:08 -0600 Subject: [PATCH 054/324] Switching to f-strings where appropriate --- scripts/wk/__init__.py | 9 ++++---- scripts/wk/log.py | 11 ++++------ scripts/wk/std.py | 47 +++++++++++++++--------------------------- 3 files changed, 25 insertions(+), 42 deletions(-) diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index efd0af93..670e78d7 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -1,7 +1,7 @@ '''WizardKit: wk module init''' # vim: sts=2 sw=2 ts=2 -import sys +from sys import version_info as version from wk import cfg from wk import exe @@ -16,12 +16,11 @@ from wk import sw # Check env -if sys.version_info < (3, 7): +if version < (3, 7): # Unsupported raise RuntimeError( - 'This package is unsupported on Python {major}.{minor}'.format( - **sys.version_info, - )) + f'This package is unsupported on Python {version.major}.{version.minor}' + ) # Init try: diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 91fa12b0..ec6d76c8 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -69,16 +69,13 @@ def update_log_path(dest_dir, dest_name=''): dest = pathlib.Path(dest_dir) dest = dest.expanduser() if dest_name: - dest_name = '{name}_{datetime}.log'.format( - name=dest_name, - datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), - ) + dest_name = f'{dest_name}_{time.strftime("%Y-%m-%d_%H%M%S%z")}.log' # Safety checks if len(root_logger.handlers) > 1: - raise NotImplementedError('update_log_path() only supports a single handler.') + raise NotImplementedError('Multiple handlers not supported') if not isinstance(cur_handler, logging.FileHandler): - raise NotImplementedError('update_log_path() only supports FileHandlers.') + raise NotImplementedError('Only FileHandlers are supported') # Copy original log to new location source = pathlib.Path(cur_handler.baseFilename) @@ -89,7 +86,7 @@ def update_log_path(dest_dir, dest_name=''): dest = dest.joinpath(source.name) dest = dest.resolve() if dest.exists(): - raise FileExistsError('Refusing to clobber: {}'.format(dest)) + raise FileExistsError(f'Refusing to clobber: {dest}') os.makedirs(dest.parent, exist_ok=True) shutil.copy(source, dest) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 1615e958..6e7c26d4 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,6 +1,5 @@ '''WizardKit: Standard Functions''' # vim: sts=2 sw=2 ts=2 -#TODO Replace .format()s with f-strings import itertools import logging @@ -74,7 +73,7 @@ def abort(prompt='Aborted.', show_prompt=True, return_code=1): def ask(prompt='Kotaero!'): """Prompt the user with a Y/N question, returns bool.""" answer = None - prompt = '{} [Y/N]: '.format(prompt) + prompt = f'{prompt} [Y/N]: ' # Loop until acceptable answer is given while answer is None: @@ -135,12 +134,8 @@ def bytes_to_string(size, decimals=0, use_binary=True): units = 'K' + suffix else: size /= scale ** 0 - units = ' {}B'.format(' ' if use_binary else '') - size = '{size:0.{decimals}f} {units}'.format( - size=size, - decimals=decimals, - units=units, - ) + units = f' {" " if use_binary else ""}B' + size = f'{size:0.{decimals}f} {units}' # Done LOG.debug('string: %s', size) @@ -156,8 +151,8 @@ def choice(choices, prompt='答えろ！'): LOG.debug('choices: %s, prompt: %s', choices, prompt) answer = None choices = [str(c).upper()[:1] for c in choices] - prompt = '{} [{}]: '.format(prompt, '/'.join(choices)) - regex = '^({})$'.format('|'.join(choices)) + prompt = f'{prompt} [{"/".join(choices)}]' + regex = f'^({"|".join(choices)})$' # Loop until acceptable answer is given while answer is None: @@ -316,18 +311,18 @@ def generate_debug_report(): report.append('--- Start debug info ---') report.append('') report.append('[System]') - report.append(' {:<24} {}'.format('FQDN', socket.getfqdn())) + report.append(f' {"FQDN":<24} {socket.getfqdn()}') for func in platform_function_list: func_name = func.replace('_', ' ').capitalize() func_result = getattr(platform, func)() - report.append(' {:<24} {}'.format(func_name, func_result)) - report.append(' {:<24} {}'.format('Python sys.argv', sys.argv)) + report.append(f' {func_name:<24} {func_result}') + report.append(f' {"Python sys.argv":<24} {sys.argv}') report.append('') # Environment report.append('[Environment Variables]') for key, value in sorted(os.environ.items()): - report.append(' {:<24} {}'.format(key, value)) + report.append(f' {key:<24} {value}') report.append('') # Done @@ -368,9 +363,7 @@ def major_exception(): report = generate_debug_report() # Upload details - prompt = 'Upload details to {}?'.format( - CRASH_SERVER.get('Name', '?'), - ) + prompt = f'Upload details to {CRASH_SERVER.get("Name", "?")}?' if ENABLED_UPLOAD_DATA and ask(prompt): print('Uploading... ', end='', flush=True) try: @@ -395,6 +388,7 @@ def pause(prompt='Press Enter to continue... '): def print_colored(strings, colors, **kwargs): """Prints strings in the colors specified.""" LOG.debug('strings: %s, colors: %s, kwargs: %s', strings, colors, kwargs) + clear_code = COLORS['CLEAR'] msg = '' print_options = { 'end': kwargs.get('end', '\n'), @@ -404,11 +398,8 @@ def print_colored(strings, colors, **kwargs): # Build new string with color escapes added for string, color in itertools.zip_longest(strings, colors): - msg += '{}{}{}'.format( - COLORS.get(color, COLORS['CLEAR']), - string, - COLORS['CLEAR'], - ) + color_code = COLORS.get(color, clear_code) + msg += f'{color_code}{string}{clear_code}' print(msg, **print_options) @@ -446,7 +437,7 @@ def print_warning(msg, **kwargs): def set_title(title): """Set window title.""" if os.name == 'nt': - os.system('title {}'.format(title)) + os.system(f'title {title}') else: raise NotImplementedError @@ -465,7 +456,7 @@ def string_to_bytes(size, assume_binary=False): # Raise exception if string can't be parsed as a size if not tmp: - raise ValueError('Invalid size string: {}'.format(size)) + raise ValueError(f'Invalid size string: {size}') # Set scale if tmp.group('binary') or assume_binary: @@ -606,11 +597,7 @@ def upload_debug_report(report, compress=True, reason='DEBUG'): if log_path: # Strip everything but the prefix filename = re.sub(r'^(.*)_(\d{4}-\d{2}-\d{2}.*)', r'\1', log_path.name) - filename = '{prefix}_{reason}_{datetime}.log'.format( - prefix=filename, - reason=reason, - datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), - ) + filename = f'{filename}_{reason}_{time.strftime("%Y-%m-%d_%H%M%S%z")}.log' LOG.debug('filename: %s', filename) # Compress report @@ -619,7 +606,7 @@ def upload_debug_report(report, compress=True, reason='DEBUG'): xz_report = lzma.compress(report.encode('utf8')) # Upload report - url = '{}/{}'.format(CRASH_SERVER['Url'], filename) + url = f'{CRASH_SERVER["Url"]}/{filename}' response = requests.put( url, data=xz_report if compress else report, From 516dc88d44c8f2e3c3a1b3c2dc57978ee8a35caf Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 19:21:44 -0600 Subject: [PATCH 055/324] Removed improper NotImplementedError usage --- scripts/wk/log.py | 4 ++-- scripts/wk/std.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index ec6d76c8..13934b0d 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -73,9 +73,9 @@ def update_log_path(dest_dir, dest_name=''): # Safety checks if len(root_logger.handlers) > 1: - raise NotImplementedError('Multiple handlers not supported') + raise RuntimeError('Multiple handlers not supported') if not isinstance(cur_handler, logging.FileHandler): - raise NotImplementedError('Only FileHandlers are supported') + raise RuntimeError('Only FileHandlers are supported') # Copy original log to new location source = pathlib.Path(cur_handler.baseFilename) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 6e7c26d4..93813d23 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -439,7 +439,7 @@ def set_title(title): if os.name == 'nt': os.system(f'title {title}') else: - raise NotImplementedError + print_error('Setting the title is only supported under Windows.') def sleep(seconds=2): From 0707d650f646c5f7229ab03331ab606a275a1df7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 20:32:37 -0600 Subject: [PATCH 056/324] Started work on the new Menu() object * This will replace the old menu_select() function * This will contain all toggle/set/selection logic * Which would allow for simpler usage in other sections/scripts/etc --- scripts/wk/std.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 93813d23..9cb57b68 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -12,11 +12,13 @@ import sys import time import traceback +from collections import OrderedDict from subprocess import CalledProcessError, CompletedProcess try: from termios import tcflush, TCIOFLUSH except ImportError: if os.name == 'posix': + # Not worried about this under Windows raise from wk.cfg.main import ( @@ -59,6 +61,94 @@ class GenericWarning(Exception): warnings filter has been changed from the default. """ + +# Classes +class Menu(): + """Object for tracking menu specific data and methods. + + Menu items are added to an OrderedDict so the order is preserved.""" + def __init__(self, title='[Untitled Menu]'): + self.actions = OrderedDict() + self.options = OrderedDict() + self.sets = OrderedDict() + self.toggles = OrderedDict() + self.disabled_str = 'DISABLED' + self.separator = '─' + self.title = title + + def add_action(self, name, details=None): + """Add action to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', True) + self.actions[name] = details + + def add_option(self, name, details=None): + """Add option to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', True) + self.options[name] = details + + def add_set(self, name, details=None): + """Add set to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', True) + + # Safety check + if 'Targets' not in details: + raise KeyError('Menu set has no targets') + + # Add set + self.sets[name] = details + + def add_toggle(self, name, details=None): + """Add toggle to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', True) + self.toggles[name] = details + + def get_separator_string(self): + """Format separator length based on name lengths, returns str.""" + separator_length = 0 + + # Loop over all item names + for section in (self.actions, self.options, self.sets, self.toggles): + for name in section.keys(): + separator_length = max(separator_length, len(name)) + separator_length += 1 + + # Done + return self.separator * separator_length + + def show(self): + """Print menu to screen.""" + separator_string = self.get_separator_string() + menu_lines = [self.title, separator_string] + + # Sets & toggles + if self.sets: + for items in self.sets.items(): + menu_lines.append(items) + if self.toggles: + for items in self.toggles.items(): + menu_lines.append(items) + if self.sets or self.toggles: + menu_lines.append(separator_string) + + # Options + if self.options: + for items in self.options.items(): + menu_lines.append(items) + menu_lines.append(separator_string) + + # Actions + for items in self.actions.items(): + menu_lines.append(items) + + # Show menu + menu_lines = [str(line) for line in menu_lines] + print('\n'.join(menu_lines)) + + # Functions def abort(prompt='Aborted.', show_prompt=True, return_code=1): """Abort script.""" From 7a9c569251ba2a027c25458645f8ee4596547a46 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 21:15:59 -0600 Subject: [PATCH 057/324] Separating public and private methods --- scripts/wk/std.py | 62 +++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 9cb57b68..fddd1c0c 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -76,37 +76,11 @@ class Menu(): self.separator = '─' self.title = title - def add_action(self, name, details=None): - """Add action to menu.""" - details = details if details else {} - details['Enabled'] = details.get('Enabled', True) - self.actions[name] = details - def add_option(self, name, details=None): - """Add option to menu.""" - details = details if details else {} - details['Enabled'] = details.get('Enabled', True) - self.options[name] = details - def add_set(self, name, details=None): - """Add set to menu.""" - details = details if details else {} - details['Enabled'] = details.get('Enabled', True) - # Safety check - if 'Targets' not in details: - raise KeyError('Menu set has no targets') - # Add set - self.sets[name] = details - - def add_toggle(self, name, details=None): - """Add toggle to menu.""" - details = details if details else {} - details['Enabled'] = details.get('Enabled', True) - self.toggles[name] = details - - def get_separator_string(self): + def _get_separator_string(self): """Format separator length based on name lengths, returns str.""" separator_length = 0 @@ -119,9 +93,9 @@ class Menu(): # Done return self.separator * separator_length - def show(self): + def _show(self): """Print menu to screen.""" - separator_string = self.get_separator_string() + separator_string = self._get_separator_string() menu_lines = [self.title, separator_string] # Sets & toggles @@ -148,6 +122,36 @@ class Menu(): menu_lines = [str(line) for line in menu_lines] print('\n'.join(menu_lines)) + def add_action(self, name, details=None): + """Add action to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', False) + self.actions[name] = details + + def add_option(self, name, details=None): + """Add option to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', False) + self.options[name] = details + + def add_set(self, name, details=None): + """Add set to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', False) + + # Safety check + if 'Targets' not in details: + raise KeyError('Menu set has no targets') + + # Add set + self.sets[name] = details + + def add_toggle(self, name, details=None): + """Add toggle to menu.""" + details = details if details else {} + details['Enabled'] = details.get('Enabled', False) + self.toggles[name] = details + # Functions def abort(prompt='Aborted.', show_prompt=True, return_code=1): From 8cedac738e03fc0a964e512c04dee42aa2cb74f9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 21:18:47 -0600 Subject: [PATCH 058/324] Added _get_display_name() and _update() to Menu() * _update() * Calls _get_display_name() * Used to update the state of the menu * Will add set logic to this method later --- scripts/wk/std.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index fddd1c0c..b155413b 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -76,9 +76,21 @@ class Menu(): self.separator = '─' self.title = title + def _get_display_name(self, name, details, index=None, no_checkboxes=True): + # pylint: disable=no-self-use + """Format display name based on details and args, returns str.""" + checkmark = '✓' if 'DISPLAY' in os.environ else '*' + display_name = f'{index if index else name[:1].upper()}: ' + # Add enabled status if necessary + if not no_checkboxes: + display_name += f'[{checkmark if details["Enabled"] else " "}] ' + # Add name + display_name += name + # Done + return display_name def _get_separator_string(self): """Format separator length based on name lengths, returns str.""" @@ -122,6 +134,29 @@ class Menu(): menu_lines = [str(line) for line in menu_lines] print('\n'.join(menu_lines)) + def _update_menu(self, single_selection=True): + """Update menu items in preparation for printing to screen.""" + index = 0 + + # Numbered sections + for section in (self.sets, self.toggles, self.options): + for name, details in section.items(): + index += 1 + details['Display Name'] = self._get_display_name( + name, + details, + index=index, + no_checkboxes=single_selection, + ) + + # Actions + for name, details in self.actions.items(): + details['Display Name'] = self._get_display_name( + name, + details, + no_checkboxes=True, + ) + def add_action(self, name, details=None): """Add action to menu.""" details = details if details else {} From bd3440daa97d7d8144f566c5f652266c0227a856 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 21:49:42 -0600 Subject: [PATCH 059/324] Updated Menu() * Added logic for disabled items * Use 'Display Name' in _show() --- scripts/wk/std.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index b155413b..09974f7a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -79,15 +79,21 @@ class Menu(): def _get_display_name(self, name, details, index=None, no_checkboxes=True): # pylint: disable=no-self-use """Format display name based on details and args, returns str.""" + disabled = details.get('Disabled', False) checkmark = '✓' if 'DISPLAY' in os.environ else '*' - display_name = f'{index if index else name[:1].upper()}: ' + clear_code = COLORS['CLEAR'] + color_code = COLORS['YELLOW'] if disabled else '' + display_name = f'{color_code}{index if index else name[:1].upper()}: ' # Add enabled status if necessary if not no_checkboxes: display_name += f'[{checkmark if details["Enabled"] else " "}] ' # Add name - display_name += name + if disabled: + display_name += f'{name} ({self.disabled_str}){clear_code}' + else: + display_name += name # Done return display_name @@ -111,30 +117,27 @@ class Menu(): menu_lines = [self.title, separator_string] # Sets & toggles - if self.sets: - for items in self.sets.items(): - menu_lines.append(items) - if self.toggles: - for items in self.toggles.items(): - menu_lines.append(items) + for section in (self.sets, self.toggles): + for details in section.values(): + menu_lines.append(details['Display Name']) if self.sets or self.toggles: menu_lines.append(separator_string) # Options + for details in self.options.values(): + menu_lines.append(details['Display Name']) if self.options: - for items in self.options.items(): - menu_lines.append(items) menu_lines.append(separator_string) # Actions - for items in self.actions.items(): - menu_lines.append(items) + for details in self.actions.values(): + menu_lines.append(details['Display Name']) # Show menu menu_lines = [str(line) for line in menu_lines] print('\n'.join(menu_lines)) - def _update_menu(self, single_selection=True): + def _update(self, single_selection=True): """Update menu items in preparation for printing to screen.""" index = 0 From 1f96ae5c53bc7e90a7b0a19ff82b2e38b8018da3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 22:04:42 -0600 Subject: [PATCH 060/324] Renamed _show() to _generate_menu_text() * It returns a string instead of printing the text --- scripts/wk/std.py | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 09974f7a..cde2595c 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -76,6 +76,32 @@ class Menu(): self.separator = '─' self.title = title + def _generate_menu_text(self): + """Generate menu text, returns str.""" + separator_string = self._get_separator_string() + menu_lines = [self.title, separator_string] + + # Sets & toggles + for section in (self.sets, self.toggles): + for details in section.values(): + menu_lines.append(details['Display Name']) + if self.sets or self.toggles: + menu_lines.append(separator_string) + + # Options + for details in self.options.values(): + menu_lines.append(details['Display Name']) + if self.options: + menu_lines.append(separator_string) + + # Actions + for details in self.actions.values(): + menu_lines.append(details['Display Name']) + + # Show menu + menu_lines = [str(line) for line in menu_lines] + return '\n'.join(menu_lines) + def _get_display_name(self, name, details, index=None, no_checkboxes=True): # pylint: disable=no-self-use """Format display name based on details and args, returns str.""" @@ -111,32 +137,6 @@ class Menu(): # Done return self.separator * separator_length - def _show(self): - """Print menu to screen.""" - separator_string = self._get_separator_string() - menu_lines = [self.title, separator_string] - - # Sets & toggles - for section in (self.sets, self.toggles): - for details in section.values(): - menu_lines.append(details['Display Name']) - if self.sets or self.toggles: - menu_lines.append(separator_string) - - # Options - for details in self.options.values(): - menu_lines.append(details['Display Name']) - if self.options: - menu_lines.append(separator_string) - - # Actions - for details in self.actions.values(): - menu_lines.append(details['Display Name']) - - # Show menu - menu_lines = [str(line) for line in menu_lines] - print('\n'.join(menu_lines)) - def _update(self, single_selection=True): """Update menu items in preparation for printing to screen.""" index = 0 From 34d5106804f793eef61d0aa1c35e084f9e63063b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 22:05:20 -0600 Subject: [PATCH 061/324] Added _get_valid_answers() to Menu() --- scripts/wk/std.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index cde2595c..6e3a0a2e 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -137,6 +137,23 @@ class Menu(): # Done return self.separator * separator_length + def _get_valid_answers(self): + """Get valid answers based on menu items, returns list.""" + valid_answers = [] + + # Numbered items + max_value = 0 + for section in (self.sets, self.toggles, self.options): + max_value += len(section) + valid_answers.extend([str(x+1) for x in range(max_value)]) + + # Action items + for name in self.actions.keys(): + valid_answers.append(name[:1].upper()) + + # Done + return valid_answers + def _update(self, single_selection=True): """Update menu items in preparation for printing to screen.""" index = 0 From 1542ba39cd206f98ba5244210d7293a9c77cf2f2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 22:12:09 -0600 Subject: [PATCH 062/324] Fixed _get_valid_answers() * Correctly omits disabled items --- scripts/wk/std.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 6e3a0a2e..de783b81 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -142,14 +142,17 @@ class Menu(): valid_answers = [] # Numbered items - max_value = 0 + index = 0 for section in (self.sets, self.toggles, self.options): - max_value += len(section) - valid_answers.extend([str(x+1) for x in range(max_value)]) + for details in section.values(): + index += 1 + if not details.get('Disabled', False): + valid_answers.append(str(index)) # Action items - for name in self.actions.keys(): - valid_answers.append(name[:1].upper()) + for name, details in self.actions.items(): + if not details.get('Disabled', False): + valid_answers.append(name[:1].upper()) # Done return valid_answers From 2b08654d7c0f2b7d3679848beba5f3bc273d79cc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 20 Aug 2019 22:34:38 -0600 Subject: [PATCH 063/324] Fixed _get_separator_length() * Use title line(s) and 'Display Name' instead of name * Menu()._update() is required to be run previously --- scripts/wk/std.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index de783b81..0c748c37 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -128,10 +128,14 @@ class Menu(): """Format separator length based on name lengths, returns str.""" separator_length = 0 + # Check title line(s) + for line in self.title.split('\n'): + separator_length = max(separator_length, len(line)) + # Loop over all item names for section in (self.actions, self.options, self.sets, self.toggles): - for name in section.keys(): - separator_length = max(separator_length, len(name)) + for details in section.values(): + separator_length = max(separator_length, len(details['Display Name'])) separator_length += 1 # Done From 849c53a62d4222e9020d3dd7aecb8fb6db019e6b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 22 Aug 2019 17:29:50 -0600 Subject: [PATCH 064/324] Include try_and_print result in log --- scripts/wk/std.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 0c748c37..3257210d 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -725,6 +725,7 @@ def try_and_print( raise # Done + LOG.info('Result: %s', result_msg.strip()) return { 'Failed': bool(f_exception), 'Exception': f_exception, From d7fc209e536496c0d082ce0ac115544d06d787c7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 7 Sep 2019 15:17:54 -0600 Subject: [PATCH 065/324] Added some doctests --- scripts/wk/log.py | 2 +- scripts/wk/std.py | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 13934b0d..e2ee6336 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -8,7 +8,7 @@ import pathlib import shutil import time -from . import cfg +from wk import cfg # Functions diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 3257210d..7905ce86 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -255,7 +255,20 @@ def beep(repeat=1): def bytes_to_string(size, decimals=0, use_binary=True): - """Convert size into a human-readable format, returns str.""" + """Convert size into a human-readable format, returns str. + + [Doctest] + >>> bytes_to_string(10) + '10 B' + >>> bytes_to_string(10_000_000) + '10 MiB' + >>> bytes_to_string(10_000_000, decimals=2) + '9.54 MiB' + >>> bytes_to_string(10_000_000, decimals=2, use_binary=False) + '10.00 MB' + >>> bytes_to_string(-10_000_000, decimals=4) + '-9.5367 MiB' + """ LOG.debug( 'size: %s, decimals: %s, use_binary: %s', size, @@ -414,7 +427,16 @@ def format_function_output(output, indent=INDENT, width=WIDTH): def get_exception(name): - """Get exception by name, returns exception object.""" + """Get exception by name, returns exception object. + + [Doctest] + >>> get_exception('AttributeError') + + >>> get_exception('CalledProcessError') + + >>> get_exception('GenericError') + + """ LOG.debug('Getting exception: %s', name) try: obj = getattr(sys.modules[__name__], name) From e52e90454da4778cdd17e074776019ad85e8b449 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 7 Sep 2019 15:44:38 -0600 Subject: [PATCH 066/324] Fix seperator length --- scripts/wk/std.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 7905ce86..53e2fadd 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -130,12 +130,13 @@ class Menu(): # Check title line(s) for line in self.title.split('\n'): - separator_length = max(separator_length, len(line)) + separator_length = max(separator_length, len(strip_colors(line))) # Loop over all item names for section in (self.actions, self.options, self.sets, self.toggles): for details in section.values(): - separator_length = max(separator_length, len(details['Display Name'])) + line = strip_colors(details['Display Name']) + separator_length = max(separator_length, len(line)) separator_length += 1 # Done From 68000272ea2af39944872ff5393bde48b600f32f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 16:14:36 -0700 Subject: [PATCH 067/324] Improved clear_screen() * Now uses subprocess.run() instead of os.system() * Avoids weird clear -> print issues * i.e. Missing newlines, etc --- scripts/wk/std.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 53e2fadd..09fadb28 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -8,12 +8,12 @@ import os import pathlib import platform import re +import subprocess import sys import time import traceback from collections import OrderedDict -from subprocess import CalledProcessError, CompletedProcess try: from termios import tcflush, TCIOFLUSH except ImportError: @@ -337,10 +337,8 @@ def choice(choices, prompt='答えろ！'): def clear_screen(): """Simple wrapper for clear/cls.""" - if os.name == 'nt': - os.system('cls') - else: - os.system('clear') + cmd = 'cls' if os.name == 'nt' else 'clear' + subprocess.run(cmd, check=False, stderr=subprocess.PIPE) def format_exception_message(_exception, indent=INDENT, width=WIDTH): @@ -354,7 +352,7 @@ def format_exception_message(_exception, indent=INDENT, width=WIDTH): # Use known argument index or first string found try: - if isinstance(_exception, CalledProcessError): + if isinstance(_exception, subprocess.CalledProcessError): message = _exception.stderr if not isinstance(message, str): message = message.decode('utf-8') @@ -403,7 +401,7 @@ def format_function_output(output, indent=INDENT, width=WIDTH): raise GenericWarning('No output') # Ensure we're working with a list - if isinstance(output, CompletedProcess): + if isinstance(output, subprocess.CompletedProcess): stdout = output.stdout if not isinstance(stdout, str): stdout = stdout.decode('utf8') From f1a1a158ba2aeff278bef047219f96854a6f7855 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 16:16:33 -0700 Subject: [PATCH 068/324] Added simple_select() to wk.std.Menu() * Allows user to select one entry from available entries --- scripts/wk/std.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 09fadb28..7494c3e5 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -162,6 +162,30 @@ class Menu(): # Done return valid_answers + def _resolve_selection(self, selection): + """Get menu item based on user selection, returns tuple.""" + resolved_selection = None + if selection.isnumeric(): + # Enumerate over numbered entries + entries = [ + *self.sets.items(), + *self.toggles.items(), + *self.options.items(), + ] + for _i, details in enumerate(entries): + if str(_i+1) == selection: + resolved_selection = (details) + break + else: + # Just check actions + for action, details in self.actions.items(): + if action.lower().startswith(selection.lower()): + resolved_selection = (action, details) + break + + # Done + return resolved_selection + def _update(self, single_selection=True): """Update menu items in preparation for printing to screen.""" index = 0 @@ -215,6 +239,24 @@ class Menu(): details['Enabled'] = details.get('Enabled', False) self.toggles[name] = details + def simple_select(self, prompt='Please make a selection...'): + """Display menu and make a single selection, returns str.""" + self._update() + menu_text = self._generate_menu_text() + valid_answers = self._get_valid_answers() + + # Loop until valid answer is given + while True: + clear_screen() + print(menu_text) + sleep(0.01) + answer = input_text(prompt).strip() + if answer.upper() in valid_answers: + break + + # Done + return self._resolve_selection(answer) + # Functions def abort(prompt='Aborted.', show_prompt=True, return_code=1): From 9da19f3702b5d7e05fe8fd8a10e64cd40e617db8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 17:49:44 -0700 Subject: [PATCH 069/324] Added support for hidden menu entries --- scripts/wk/std.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 7494c3e5..7fa5778a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -84,18 +84,24 @@ class Menu(): # Sets & toggles for section in (self.sets, self.toggles): for details in section.values(): + if details.get('Hidden', False): + continue menu_lines.append(details['Display Name']) if self.sets or self.toggles: menu_lines.append(separator_string) # Options for details in self.options.values(): + if details.get('Hidden', False): + continue menu_lines.append(details['Display Name']) if self.options: menu_lines.append(separator_string) # Actions for details in self.actions.values(): + if details.get('Hidden', False): + continue menu_lines.append(details['Display Name']) # Show menu @@ -135,6 +141,9 @@ class Menu(): # Loop over all item names for section in (self.actions, self.options, self.sets, self.toggles): for details in section.values(): + if details.get('Hidden', False): + # Skip hidden lines + continue line = strip_colors(details['Display Name']) separator_length = max(separator_length, len(line)) separator_length += 1 @@ -150,6 +159,9 @@ class Menu(): index = 0 for section in (self.sets, self.toggles, self.options): for details in section.values(): + if details.get('Hidden', False): + # Don't increment counter or add to valid_answers + continue index += 1 if not details.get('Disabled', False): valid_answers.append(str(index)) @@ -164,6 +176,7 @@ class Menu(): def _resolve_selection(self, selection): """Get menu item based on user selection, returns tuple.""" + offset = 1 resolved_selection = None if selection.isnumeric(): # Enumerate over numbered entries @@ -173,7 +186,9 @@ class Menu(): *self.options.items(), ] for _i, details in enumerate(entries): - if str(_i+1) == selection: + if details[1].get('Hidden', False): + offset -= 1 + elif str(_i+offset) == selection: resolved_selection = (details) break else: @@ -193,6 +208,9 @@ class Menu(): # Numbered sections for section in (self.sets, self.toggles, self.options): for name, details in section.items(): + if details.get('Hidden', False): + # Skip hidden lines and don't increment index + continue index += 1 details['Display Name'] = self._get_display_name( name, From 21b44d01ff33e65661bc3b27a236da98caaaefd9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 18:41:47 -0700 Subject: [PATCH 070/324] Added avanced_select() to Menu() * Renamed 'Enabled' to 'Selected' for clarity --- scripts/wk/std.py | 97 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 7fa5778a..0445f70b 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -66,13 +66,14 @@ class GenericWarning(Exception): class Menu(): """Object for tracking menu specific data and methods. - Menu items are added to an OrderedDict so the order is preserved.""" + Menu items are added to an OrderedDict so the order is preserved. + NOTE: This class and its methods assume all entry names are unique.""" def __init__(self, title='[Untitled Menu]'): self.actions = OrderedDict() self.options = OrderedDict() self.sets = OrderedDict() self.toggles = OrderedDict() - self.disabled_str = 'DISABLED' + self.disabled_str = 'Disabled' self.separator = '─' self.title = title @@ -119,7 +120,7 @@ class Menu(): # Add enabled status if necessary if not no_checkboxes: - display_name += f'[{checkmark if details["Enabled"] else " "}] ' + display_name += f'[{checkmark if details["Selected"] else " "}] ' # Add name if disabled: @@ -160,7 +161,7 @@ class Menu(): for section in (self.sets, self.toggles, self.options): for details in section.values(): if details.get('Hidden', False): - # Don't increment counter or add to valid_answers + # Don't increment index or add to valid_answers continue index += 1 if not details.get('Disabled', False): @@ -205,6 +206,17 @@ class Menu(): """Update menu items in preparation for printing to screen.""" index = 0 + # Fix selection status for sets + for set_details in self.sets.values(): + set_selected = True + set_targets = set_details['Targets'] + for option, option_details in self.options.items(): + if option in set_targets and not option_details['Selected']: + set_selected = False + elif option not in set_targets and option_details['Selected']: + set_selected = False + set_details['Selected'] = set_selected + # Numbered sections for section in (self.sets, self.toggles, self.options): for name, details in section.items(): @@ -227,22 +239,60 @@ class Menu(): no_checkboxes=True, ) + def _update_entry_selection_status(self, entry, toggle=True, status=None): + """Update entry selection status either directly or by toggling.""" + if entry in self.sets: + # Update targets not the set itself + new_status = not self.sets[entry]['Selected'] if toggle else status + targets = self.sets[entry]['Targets'] + self._update_set_selection_status(targets, new_status) + for section in (self.toggles, self.options, self.actions): + if entry in section: + if toggle: + section[entry]['Selected'] = not section[entry]['Selected'] + else: + section[entry]['Selected'] = status + + def _update_set_selection_status(self, targets, status): + """Select or deselect options based on targets and status.""" + for option, details in self.options.items(): + # If (new) status is True and this option is a target then select + # Otherwise deselect + details['Selected'] = status and option in targets + + def _user_select(self, prompt): + """Show menu and select an entry, returns str.""" + menu_text = self._generate_menu_text() + valid_answers = self._get_valid_answers() + + # Menu loop + while True: + clear_screen() + print(menu_text) + sleep(0.01) + answer = input_text(prompt).strip() + if answer.upper() in valid_answers: + break + + # Done + return answer + def add_action(self, name, details=None): """Add action to menu.""" details = details if details else {} - details['Enabled'] = details.get('Enabled', False) + details['Selected'] = details.get('Selected', False) self.actions[name] = details def add_option(self, name, details=None): """Add option to menu.""" details = details if details else {} - details['Enabled'] = details.get('Enabled', False) + details['Selected'] = details.get('Selected', False) self.options[name] = details def add_set(self, name, details=None): """Add set to menu.""" details = details if details else {} - details['Enabled'] = details.get('Enabled', False) + details['Selected'] = details.get('Selected', False) # Safety check if 'Targets' not in details: @@ -254,26 +304,33 @@ class Menu(): def add_toggle(self, name, details=None): """Add toggle to menu.""" details = details if details else {} - details['Enabled'] = details.get('Enabled', False) + details['Selected'] = details.get('Selected', False) self.toggles[name] = details - def simple_select(self, prompt='Please make a selection...'): - """Display menu and make a single selection, returns str.""" - self._update() - menu_text = self._generate_menu_text() - valid_answers = self._get_valid_answers() + def advanced_select(self, prompt='Please make a selection...'): + """Display menu and make multiple selections, returns tuple. - # Loop until valid answer is given + The menu can only be exited by selecting an action.""" while True: - clear_screen() - print(menu_text) - sleep(0.01) - answer = input_text(prompt).strip() - if answer.upper() in valid_answers: + # Resolve selection status for sets + self._update(single_selection=False) + user_selection = self._user_select(prompt) + if user_selection.isnumeric(): + # Update selection(s) + entry = self._resolve_selection(user_selection)[0] + self._update_entry_selection_status(entry) + else: + # Action selected break # Done - return self._resolve_selection(answer) + return self._resolve_selection(user_selection) + + def simple_select(self, prompt='Please make a selection...'): + """Display menu and make a single selection, returns tuple.""" + self._update() + user_selection = self._user_select(prompt) + return self._resolve_selection(user_selection) # Functions From 428bb5a05cdbc21283962eda1b5fce9c8b1efdf7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 18:45:07 -0700 Subject: [PATCH 071/324] Updated advanced_select() --- scripts/wk/std.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 0445f70b..60029737 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -312,19 +312,18 @@ class Menu(): The menu can only be exited by selecting an action.""" while True: - # Resolve selection status for sets self._update(single_selection=False) user_selection = self._user_select(prompt) + selected_entry = self._resolve_selection(user_selection) if user_selection.isnumeric(): # Update selection(s) - entry = self._resolve_selection(user_selection)[0] - self._update_entry_selection_status(entry) + self._update_entry_selection_status(selected_entry[0]) else: # Action selected break # Done - return self._resolve_selection(user_selection) + return selected_entry def simple_select(self, prompt='Please make a selection...'): """Display menu and make a single selection, returns tuple.""" From a59f20ac8bb4f0056f48b40f4cc84aeb28bb4afc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 19:44:56 -0700 Subject: [PATCH 072/324] Support optional extra separators in Menu() --- scripts/wk/std.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 60029737..060e74c2 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -87,6 +87,8 @@ class Menu(): for details in section.values(): if details.get('Hidden', False): continue + if details.get('Separator', False): + menu_lines.append(separator_string) menu_lines.append(details['Display Name']) if self.sets or self.toggles: menu_lines.append(separator_string) @@ -95,6 +97,8 @@ class Menu(): for details in self.options.values(): if details.get('Hidden', False): continue + if details.get('Separator', False): + menu_lines.append(separator_string) menu_lines.append(details['Display Name']) if self.options: menu_lines.append(separator_string) @@ -103,9 +107,12 @@ class Menu(): for details in self.actions.values(): if details.get('Hidden', False): continue + if details.get('Separator', False): + menu_lines.append(separator_string) menu_lines.append(details['Display Name']) # Show menu + menu_lines.append('') menu_lines = [str(line) for line in menu_lines] return '\n'.join(menu_lines) From 94dac676fe3cd7afe43210f3d1541917c699e9d1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 20:29:05 -0700 Subject: [PATCH 073/324] Updated formatting --- scripts/wk/std.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 060e74c2..efaa9d3c 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -67,7 +67,11 @@ class Menu(): """Object for tracking menu specific data and methods. Menu items are added to an OrderedDict so the order is preserved. - NOTE: This class and its methods assume all entry names are unique.""" + + ASSUMPTIONS: + 1. All entry names are unique. + 2. All action entry names start with different letters. + """ def __init__(self, title='[Untitled Menu]'): self.actions = OrderedDict() self.options = OrderedDict() @@ -314,10 +318,11 @@ class Menu(): details['Selected'] = details.get('Selected', False) self.toggles[name] = details - def advanced_select(self, prompt='Please make a selection...'): + def advanced_select(self, prompt='Please make a selection: '): """Display menu and make multiple selections, returns tuple. - The menu can only be exited by selecting an action.""" + NOTE: Menu is displayed until an action entry is selected. + """ while True: self._update(single_selection=False) user_selection = self._user_select(prompt) @@ -332,7 +337,7 @@ class Menu(): # Done return selected_entry - def simple_select(self, prompt='Please make a selection...'): + def simple_select(self, prompt='Please make a selection: '): """Display menu and make a single selection, returns tuple.""" self._update() user_selection = self._user_select(prompt) From 4d6fad82dbff1f777c25e31de8a34e453233fddb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 20:29:18 -0700 Subject: [PATCH 074/324] Adjusted menu index formatting * Right-align numbered and action text --- scripts/wk/std.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index efaa9d3c..18692a67 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -128,6 +128,8 @@ class Menu(): clear_code = COLORS['CLEAR'] color_code = COLORS['YELLOW'] if disabled else '' display_name = f'{color_code}{index if index else name[:1].upper()}: ' + if not (index and index >= 10): + display_name = f' {display_name}' # Add enabled status if necessary if not no_checkboxes: From 3d95d9c1a1224f3b59d76e4ef13ee12f0eb0a28c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 20:47:27 -0700 Subject: [PATCH 075/324] Fixed clear_screen() under Windows --- scripts/wk/std.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 18692a67..d63bc889 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -468,7 +468,7 @@ def choice(choices, prompt='答えろ！'): def clear_screen(): """Simple wrapper for clear/cls.""" cmd = 'cls' if os.name == 'nt' else 'clear' - subprocess.run(cmd, check=False, stderr=subprocess.PIPE) + subprocess.run(cmd, check=False, shell=True, stderr=subprocess.PIPE) def format_exception_message(_exception, indent=INDENT, width=WIDTH): From 8d9e264efce74d73f4fd14a65f5d54a3fbc87c35 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 21:02:56 -0700 Subject: [PATCH 076/324] Fix menu checkmark under macOS --- scripts/wk/std.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index d63bc889..9d3f77f6 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -124,7 +124,9 @@ class Menu(): # pylint: disable=no-self-use """Format display name based on details and args, returns str.""" disabled = details.get('Disabled', False) - checkmark = '✓' if 'DISPLAY' in os.environ else '*' + checkmark = '*' + if 'DISPLAY' in os.environ or sys.platform == 'darwin': + checkmark = '✓' clear_code = COLORS['CLEAR'] color_code = COLORS['YELLOW'] if disabled else '' display_name = f'{color_code}{index if index else name[:1].upper()}: ' From 7cb5ecd09f8ea3ade8d9b8564fd48ab840f36254 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 22:35:39 -0700 Subject: [PATCH 077/324] Added TryAndPrint() class * Replaces try_and_print() function * Moved several functions to TryAndPrint() class * _format_exception_message() * _format_function_output() * _get_exception() * Separates the formatting settings and the function paramters --- scripts/wk/std.py | 389 ++++++++++++++++++++++++---------------------- 1 file changed, 200 insertions(+), 189 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 9d3f77f6..82dcd8eb 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -348,6 +348,206 @@ class Menu(): return self._resolve_selection(user_selection) +class TryAndPrint(): + """Object used to standardize running functions and returning the result. + + The errors and warning attributes are used to allow fine-tuned results + based on exception names. + """ + def __init__(self, msg_bad='FAILED', msg_good='SUCCESS'): + self.indent = INDENT + self.msg_bad = msg_bad + self.msg_good = msg_good + self.width = WIDTH + self.list_errors = ['GenericError'] + self.list_warnings = ['GenericWarning'] + + def _format_exception_message(self, _exception): + """Format using the exception's args or name, returns str.""" + # pylint: disable=broad-except + LOG.debug( + 'Formatting exception: %s', + _exception.__class__.__name__, + ) + message = None + + # Use known argument index or first string found + try: + if isinstance(_exception, subprocess.CalledProcessError): + message = _exception.stderr + if not isinstance(message, str): + message = message.decode('utf-8') + message = message.strip() + elif isinstance(_exception, FileNotFoundError): + message = _exception.args[1] + elif isinstance(_exception, ZeroDivisionError): + message = 'ZeroDivisionError' + else: + for arg in _exception.args: + if isinstance(arg, str): + message = arg + break + except Exception: + # Just use the exception name instead + pass + + # Safety check + if not message: + try: + message = _exception.__class__.__name__ + except Exception: + message = 'UNKNOWN ERROR' + + # Fix multi-line messages + if '\n' in message: + try: + lines = [ + f'{" "*(self.indent+self.width)}{line.strip()}' + for line in message.splitlines() if line.strip() + ] + lines[0] = lines[0].strip() + message = '\n'.join(lines) + except Exception: + pass + + # Done + return message + + def _format_function_output(self, output): + """Format function output for use in try_and_print(), returns str.""" + LOG.debug('Formatting output: %s', output) + + if not output: + raise GenericWarning('No output') + + # Ensure we're working with a list + if isinstance(output, subprocess.CompletedProcess): + stdout = output.stdout + if not isinstance(stdout, str): + stdout = stdout.decode('utf8') + output = stdout.strip().splitlines() + else: + output = list(output) + + # Safety check + if not output: + # Going to ignore empty function output for now + LOG.error('Output is empty') + return 'UNKNOWN' + + # Build result_msg + result_msg = f'{output.pop(0)}' + if output: + output = [f'{" "*(self.indent+self.width)}{line}' for line in output] + result_msg += '\n' + '\n'.join(output) + + # Done + return result_msg + + def _get_exception(self, name): + # pylint: disable=no-self-use + """Get exception by name, returns exception object. + + [Doctest] + >>> self._get_exception('AttributeError') + + >>> self._get_exception('CalledProcessError') + + >>> self._get_exception('GenericError') + + """ + LOG.debug('Getting exception: %s', name) + try: + obj = getattr(sys.modules[__name__], name) + except AttributeError: + # Try builtin classes + obj = getattr(sys.modules['builtins'], name) + return obj + + def add_error(self, exception_name): + """Add exception name to error list.""" + if exception_name not in self.list_errors: + self.list_errors.append(exception_name) + + def add_warning(self, exception_name): + """Add exception name to warning list.""" + if exception_name not in self.list_warnings: + self.list_warnings.append(exception_name) + + def run_function( + self, message, function, *args, + catch_all=True, print_return=False, verbose=False, **kwargs): + # pylint: disable=catching-non-exception + """Run a function and print the results, returns results as dict. + + If catch_all is True then (nearly) all exceptions will be caught. + Otherwise if an exception occurs that wasn't specified it will be + re-raised. + + If print_return is True then the output from the function will be used + instead of msg_good, msg_bad, or exception text. The output should be + a list or a subprocess.CompletedProcess object. + + If verbose is True then exception names or messages will be used for + the result message. Otherwise it will simply be set to result_bad. + + args and kwargs are passed to the function. + """ + LOG.debug('function: %s.%s', function.__module__, function.__name__) + LOG.debug('args: %s', args) + LOG.debug('kwargs: %s', kwargs) + LOG.debug( + 'catch_all: %s, print_return: %s, verbose: %s', + catch_all, + print_return, + verbose, + ) + f_exception = None + output = None + result_msg = 'UNKNOWN' + + # Build exception tuples + e_exceptions = tuple(self._get_exception(e) for e in self.list_errors) + w_exceptions = tuple(self._get_exception(e) for e in self.list_warnings) + + # Run function and catch exceptions + print(f'{" "*self.indent}{message:<{self.width}}', end='', flush=True) + LOG.info('Running function: %s.%s', function.__module__, function.__name__) + try: + output = function(*args, **kwargs) + if print_return: + result_msg = self._format_function_output(output) + print(result_msg) + else: + print_success(self.msg_good) + except w_exceptions as _exception: + result_msg = self._format_exception_message(_exception) + print_warning(result_msg) + f_exception = _exception + except e_exceptions as _exception: + result_msg = self._format_exception_message(_exception) + print_error(result_msg) + f_exception = _exception + except Exception as _exception: # pylint: disable=broad-except + if verbose: + result_msg = self._format_exception_message(_exception) + else: + result_msg = self.msg_bad + print_error(result_msg) + f_exception = _exception + if not catch_all: + # Re-raise error as necessary + raise + + # Done + LOG.info('Result: %s', result_msg.strip()) + return { + 'Failed': bool(f_exception), + 'Exception': f_exception, + 'Output': output, + } + + # Functions def abort(prompt='Aborted.', show_prompt=True, return_code=1): """Abort script.""" @@ -473,110 +673,6 @@ def clear_screen(): subprocess.run(cmd, check=False, shell=True, stderr=subprocess.PIPE) -def format_exception_message(_exception, indent=INDENT, width=WIDTH): - """Format using the exception's args or name, returns str.""" - # pylint: disable=broad-except - LOG.debug( - 'Formatting exception: %s', - _exception.__class__.__name__, - ) - message = None - - # Use known argument index or first string found - try: - if isinstance(_exception, subprocess.CalledProcessError): - message = _exception.stderr - if not isinstance(message, str): - message = message.decode('utf-8') - message = message.strip() - elif isinstance(_exception, FileNotFoundError): - message = _exception.args[1] - elif isinstance(_exception, ZeroDivisionError): - message = 'ZeroDivisionError' - else: - for arg in _exception.args: - if isinstance(arg, str): - message = arg - break - except Exception: - # Just use the exception name instead - pass - - # Safety check - if not message: - try: - message = _exception.__class__.__name__ - except Exception: - message = 'UNKNOWN ERROR' - - # Fix multi-line messages - if '\n' in message: - try: - lines = [ - f'{" "*(indent+width)}{line.strip()}' - for line in message.splitlines() if line.strip() - ] - lines[0] = lines[0].strip() - message = '\n'.join(lines) - except Exception: - pass - - # Done - return message - - -def format_function_output(output, indent=INDENT, width=WIDTH): - """Format function output for use in try_and_print(), returns str.""" - LOG.debug('Formatting output: %s', output) - - if not output: - raise GenericWarning('No output') - - # Ensure we're working with a list - if isinstance(output, subprocess.CompletedProcess): - stdout = output.stdout - if not isinstance(stdout, str): - stdout = stdout.decode('utf8') - output = stdout.strip().splitlines() - else: - output = list(output) - - # Safety check - if not output: - # Going to ignore empty function output for now - LOG.error('Output is empty') - return 'UNKNOWN' - - # Build result_msg - result_msg = f'{output.pop(0)}' - if output: - output = [f'{" "*(indent+width)}{line}' for line in output] - result_msg += '\n' + '\n'.join(output) - - # Done - return result_msg - - -def get_exception(name): - """Get exception by name, returns exception object. - - [Doctest] - >>> get_exception('AttributeError') - - >>> get_exception('CalledProcessError') - - >>> get_exception('GenericError') - - """ - LOG.debug('Getting exception: %s', name) - try: - obj = getattr(sys.modules[__name__], name) - except AttributeError: - # Try builtin classes - obj = getattr(sys.modules['builtins'], name) - return obj - - def get_log_filepath(): """Get the log filepath from the root logger, returns pathlib.Path obj. @@ -801,91 +897,6 @@ def strip_colors(string): return string -def try_and_print( - message, function, *args, - msg_good='CS', msg_bad='NS', indent=INDENT, width=WIDTH, - w_exceptions=None, e_exceptions=None, - catch_all=True, print_return=False, verbose=False, - **kwargs): - # pylint: disable=catching-non-exception,unused-argument,too-many-locals - """Run a function and print the results, returns results as dict. - - If catch_all is True then (nearly) all exceptions will be caught. - Otherwise if an exception occurs that wasn't specified it will be - re-raised. - - If print_return is True then the output from the function will be used - instead of msg_good, msg_bad, or exception text. The output should be - a list or a subprocess.CompletedProcess object. - - If verbose is True then exception names or messages will be used for - the result message. Otherwise it will simply be set to result_bad. - - If specified w_exceptions and e_exceptions should be lists of - exception class names. Details from the excceptions will be used to - format more clear result messages. - """ - LOG.debug('function: %s.%s', function.__module__, function.__name__) - LOG.debug('args: %s', args) - LOG.debug('kwargs: %s', kwargs) - LOG.debug('w_exceptions: %s', w_exceptions) - LOG.debug('e_exceptions: %s', e_exceptions) - LOG.debug( - 'catch_all: %s, print_return: %s, verbose: %s', - catch_all, - print_return, - verbose, - ) - f_exception = None - output = None - result_msg = 'UNKNOWN' - - # Build tuples of exceptions - if not w_exceptions: - w_exceptions = ('GenericWarning',) - if not e_exceptions: - e_exceptions = ('GenericError',) - w_exceptions = tuple(get_exception(e) for e in w_exceptions) - e_exceptions = tuple(get_exception(e) for e in e_exceptions) - - # Run function and catch exceptions - print(f'{" "*indent}{message:<{width}}', end='', flush=True) - LOG.info('Running function: %s.%s', function.__module__, function.__name__) - try: - output = function(*args, **kwargs) - if print_return: - result_msg = format_function_output(output, indent, width) - print(result_msg) - else: - print_success(msg_good) - except w_exceptions as _exception: - result_msg = format_exception_message(_exception, indent, width) - print_warning(result_msg) - f_exception = _exception - except e_exceptions as _exception: - result_msg = format_exception_message(_exception, indent, width) - print_error(result_msg) - f_exception = _exception - except Exception as _exception: # pylint: disable=broad-except - if verbose: - result_msg = format_exception_message(_exception, indent, width) - else: - result_msg = msg_bad - print_error(result_msg) - f_exception = _exception - if not catch_all: - # Re-raise error as necessary - raise - - # Done - LOG.info('Result: %s', result_msg.strip()) - return { - 'Failed': bool(f_exception), - 'Exception': f_exception, - 'Output': output, - } - - def upload_debug_report(report, compress=True, reason='DEBUG'): """Upload debug report to CRASH_SERVER as specified in wk.cfg.main.""" LOG.info('Uploading debug report to %s', CRASH_SERVER.get('Name', '?')) From 8c8eea0f9a6bfccbb1c357938c30757754691c1b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Sep 2019 22:48:33 -0700 Subject: [PATCH 078/324] Adjusted pylint settings --- scripts/wk/std.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 82dcd8eb..e51b14e2 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -364,7 +364,6 @@ class TryAndPrint(): def _format_exception_message(self, _exception): """Format using the exception's args or name, returns str.""" - # pylint: disable=broad-except LOG.debug( 'Formatting exception: %s', _exception.__class__.__name__, @@ -387,7 +386,7 @@ class TryAndPrint(): if isinstance(arg, str): message = arg break - except Exception: + except Exception: # pylint: disable=broad-except # Just use the exception name instead pass @@ -395,7 +394,7 @@ class TryAndPrint(): if not message: try: message = _exception.__class__.__name__ - except Exception: + except Exception: # pylint: disable=broad-except message = 'UNKNOWN ERROR' # Fix multi-line messages @@ -407,7 +406,7 @@ class TryAndPrint(): ] lines[0] = lines[0].strip() message = '\n'.join(lines) - except Exception: + except Exception: # pylint: disable=broad-except pass # Done From b71f1d8d8029edeb4954798b7a00c38f741d83ad Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 13:33:59 -0700 Subject: [PATCH 079/324] Reordered functions --- scripts/wk/std.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index e51b14e2..fd20a286 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -125,7 +125,7 @@ class Menu(): """Format display name based on details and args, returns str.""" disabled = details.get('Disabled', False) checkmark = '*' - if 'DISPLAY' in os.environ or sys.platform == 'darwin': + if 'DISPLAY' in os.environ or platform.system() == 'Darwin': checkmark = '✓' clear_code = COLORS['CLEAR'] color_code = COLORS['YELLOW'] if disabled else '' @@ -672,24 +672,6 @@ def clear_screen(): subprocess.run(cmd, check=False, shell=True, stderr=subprocess.PIPE) -def get_log_filepath(): - """Get the log filepath from the root logger, returns pathlib.Path obj. - - NOTE: This will use the first handler baseFilename it finds (if any). - """ - log_filepath = None - root_logger = logging.getLogger() - - # Check handlers - for handler in root_logger.handlers: - if hasattr(handler, 'baseFilename'): - log_filepath = pathlib.Path(handler.baseFilename).resolve() - break - - # Done - return log_filepath - - def generate_debug_report(): """Generate debug report, returns str.""" import socket @@ -734,6 +716,24 @@ def generate_debug_report(): return '\n'.join(report) +def get_log_filepath(): + """Get the log filepath from the root logger, returns pathlib.Path obj. + + NOTE: This will use the first handler baseFilename it finds (if any). + """ + log_filepath = None + root_logger = logging.getLogger() + + # Check handlers + for handler in root_logger.handlers: + if hasattr(handler, 'baseFilename'): + log_filepath = pathlib.Path(handler.baseFilename).resolve() + break + + # Done + return log_filepath + + def input_text(prompt='Enter text'): """Get text from user, returns string.""" prompt = str(prompt) From 6f60006c9a27853570ded2b85e805b44d4507a89 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 14:35:32 -0700 Subject: [PATCH 080/324] Added get, kill, and wait process functions --- scripts/wk/std.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index fd20a286..4fa73fd8 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -21,6 +21,8 @@ except ImportError: # Not worried about this under Windows raise +import psutil + from wk.cfg.main import ( CRASH_SERVER, ENABLED_UPLOAD_DATA, @@ -734,6 +736,20 @@ def get_log_filepath(): return log_filepath +def get_procs(name, exact=True): + """Get process object(s) based on name, returns list of proc objects.""" + processes = [] + regex = f'^{name}$' if exact else name + + # Iterate over all processes + for proc in psutil.process_iter(): + if re.search(regex, proc.name(), re.IGNORECASE): + processes.append(proc) + + # Done + return processes + + def input_text(prompt='Enter text'): """Get text from user, returns string.""" prompt = str(prompt) @@ -756,6 +772,26 @@ def input_text(prompt='Enter text'): return response +def kill_procs(name, exact=True, force=False, timeout=30): + """Kill all processes matching name (case-insensitively). + + NOTE: Under Posix systems this will send SIGINT to allow processes + to gracefully exit. + + If force is True then it will wait until timeout specified and then + send SIGKILL to any processes still alive. + """ + target_procs = get_procs(name, exact=exact) + for proc in target_procs: + proc.terminate() + + # Force kill if necesary + if force: + results = psutil.wait_procs(target_procs, timeout=timeout) + for proc in results[1]: # Alive processes + proc.kill() + + def major_exception(): """Display traceback, optionally upload detailes, and exit.""" LOG.critical('Major exception encountered', exc_info=True) @@ -939,5 +975,15 @@ def upload_debug_report(report, compress=True, reason='DEBUG'): raise RuntimeError('Failed to upload report') +def wait_for_procs(name, exact=True, timeout=None): + """Wait for all process matching name.""" + target_procs = get_procs(name, exact=exact) + results = psutil.wait_procs(target_procs, timeout=timeout) + + # Raise exception if necessary + if results[1]: # Alive processes + raise psutil.TimeoutExpired(name=name, seconds=timeout) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 40413151c875e779fed31692245016aa1edb4e4a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 15:10:13 -0700 Subject: [PATCH 081/324] Added run_program() --- scripts/wk/std.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 4fa73fd8..6af49274 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,4 +1,5 @@ '''WizardKit: Standard Functions''' +# pylint: disable=too-many-lines # vim: sts=2 sw=2 ts=2 import itertools @@ -874,6 +875,31 @@ def print_warning(msg, **kwargs): print_colored([msg], ['YELLOW'], **kwargs) +def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): + """Run program and return a subprocess.CompletedProcess object.""" + cmd_kwargs = { + 'args': cmd, + 'check': check, + 'shell': shell, + } + + # Add additional kwargs if applicable + for key in ('cwd', 'encoding', 'errors', 'stderr', 'stdout'): + if key in kwargs: + cmd_kwargs[key] = kwargs[key] + + # Finalize cmd_kwargs + if pipe: + cmd_kwargs['stderr'] = subprocess.PIPE + cmd_kwargs['stdout'] = subprocess.PIPE + if not ('encoding' in cmd_kwargs or 'errors' in cmd_kwargs): + cmd_kwargs['encoding'] = 'utf-8' + cmd_kwargs['errors'] = 'ignore' + + # Ready to run program + return subprocess.run(**cmd_kwargs) + + def set_title(title): """Set window title.""" if os.name == 'nt': From 8ca59a029a4ca6ad4b96b3c2a571b9069fdb7a80 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 15:32:34 -0700 Subject: [PATCH 082/324] Added popen_program() --- scripts/wk/std.py | 75 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 6af49274..8a5065ca 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -589,6 +589,44 @@ def beep(repeat=1): repeat -= 1 +def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): + """Build kwargs for use by subprocess functions, returns dict. + + Specifically subprocess.run() and subprocess.Popen(). + NOTE: If no encoding specified then UTF-8 will be used. + """ + cmd_kwargs = { + 'args': cmd, + 'shell': shell, + } + + # Add additional kwargs if applicable + for key in ('check', 'cwd', 'encoding', 'errors', 'stderr', 'stdout'): + if key in kwargs: + cmd_kwargs[key] = kwargs[key] + + # Default to UTF-8 encoding + if not ('encoding' in cmd_kwargs or 'errors' in cmd_kwargs): + cmd_kwargs['encoding'] = 'utf-8' + cmd_kwargs['errors'] = 'ignore' + + # Start minimized + if minimized: + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = 6 + cmd_kwargs['startupinfo'] = startupinfo + + + # Pipe output + if pipe: + cmd_kwargs['stderr'] = subprocess.PIPE + cmd_kwargs['stdout'] = subprocess.PIPE + + # Done + return cmd_kwargs + + def bytes_to_string(size, decimals=0, use_binary=True): """Convert size into a human-readable format, returns str. @@ -826,6 +864,19 @@ def pause(prompt='Press Enter to continue... '): input_text(prompt) +def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): + """Run program and return a subprocess.Popen object.""" + cmd_kwargs = build_cmd_kwargs( + cmd, + minimized=minimized, + pipe=pipe, + shell=shell, + **kwargs) + + # Ready to run program + return subprocess.Popen(**cmd_kwargs) + + def print_colored(strings, colors, **kwargs): """Prints strings in the colors specified.""" LOG.debug('strings: %s, colors: %s, kwargs: %s', strings, colors, kwargs) @@ -877,24 +928,12 @@ def print_warning(msg, **kwargs): def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): """Run program and return a subprocess.CompletedProcess object.""" - cmd_kwargs = { - 'args': cmd, - 'check': check, - 'shell': shell, - } - - # Add additional kwargs if applicable - for key in ('cwd', 'encoding', 'errors', 'stderr', 'stdout'): - if key in kwargs: - cmd_kwargs[key] = kwargs[key] - - # Finalize cmd_kwargs - if pipe: - cmd_kwargs['stderr'] = subprocess.PIPE - cmd_kwargs['stdout'] = subprocess.PIPE - if not ('encoding' in cmd_kwargs or 'errors' in cmd_kwargs): - cmd_kwargs['encoding'] = 'utf-8' - cmd_kwargs['errors'] = 'ignore' + cmd_kwargs = build_cmd_kwargs( + cmd, + check=check, + pipe=pipe, + shell=shell, + **kwargs) # Ready to run program return subprocess.run(**cmd_kwargs) From 6af5f78fbb8cbebabe5db9c30436753ea2eab66a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 15:34:27 -0700 Subject: [PATCH 083/324] Dropping wk/exe.py * All intended functionality now in std.py --- scripts/wk/__init__.py | 1 - scripts/wk/exe.py | 0 2 files changed, 1 deletion(-) delete mode 100644 scripts/wk/exe.py diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 670e78d7..d9264d83 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -4,7 +4,6 @@ from sys import version_info as version from wk import cfg -from wk import exe from wk import hw from wk import io from wk import kit diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py deleted file mode 100644 index e69de29b..00000000 From 7a9971304384d2434d95a3d41ecd9ed5b0d21cef Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 16:28:47 -0700 Subject: [PATCH 084/324] Added non_clobbering_path() --- scripts/wk/io.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/scripts/wk/io.py b/scripts/wk/io.py index 469408c3..d991be4a 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -1,6 +1,35 @@ '''WizardKit: I/O Functions''' # vim: sts=2 sw=2 ts=2 +import pathlib + + +# Functions +def non_clobbering_path(path): + """Update path as needed to non-existing path, returns pathlib.Path.""" + path = pathlib.Path(path) + name = path.name + new_path = None + suffix = ''.join(path.suffixes) + name = name.replace(suffix, '') + + # Bail early + if not path.exists(): + return path + + # Find non-existant path + for _i in range(1000): + new_path = path.with_name(f'{name}_{_i}').with_suffix(suffix) + if not new_path.exists(): + break + + # Raise error if viable path not found + if not new_path: + raise FileExistsError(new_path) + + # Done + return new_path + if __name__ == '__main__': print("This file is not meant to be called directly.") From 010ac87de633d37b278aba0d87ca493fc4d5b94e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 16:50:06 -0700 Subject: [PATCH 085/324] Added delete_empty_folders() and delete_folder() --- scripts/wk/io.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/wk/io.py b/scripts/wk/io.py index d991be4a..d9637716 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -1,10 +1,36 @@ '''WizardKit: I/O Functions''' # vim: sts=2 sw=2 ts=2 +import os +import shutil + import pathlib # Functions +def delete_empty_folders(path): + """Recursively delete all empty folders in path.""" + # Delete empty subfolders first + for item in os.scandir(path): + if item.is_dir(): + delete_empty_folders(item.path) + + # Attempt to remove (top) path + try: + delete_folder(path, force=False) + except OSError: + # Assuming it's not empty + pass + + +def delete_folder(path, force=False): + """Delete folder if empty or if forced.""" + if force: + shutil.rmtree(path) + else: + os.rmdir(path) + + def non_clobbering_path(path): """Update path as needed to non-existing path, returns pathlib.Path.""" path = pathlib.Path(path) From 78e3765730e2a61c8eae2c85a57925fc180839ac Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 17:29:10 -0700 Subject: [PATCH 086/324] Added delete_item() --- scripts/wk/io.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/wk/io.py b/scripts/wk/io.py index d9637716..5ff75d7b 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -23,14 +23,31 @@ def delete_empty_folders(path): pass -def delete_folder(path, force=False): - """Delete folder if empty or if forced.""" +def delete_folder(path, force=False, ignore_errors=False): + """Delete folder if empty or if forced. + + NOTE: Exceptions are not caught by this function, + ignore_errors is passed to shutil.rmtree to allow partial deletions. + """ if force: - shutil.rmtree(path) + shutil.rmtree(path, ignore_errors=ignore_errors) else: os.rmdir(path) +def delete_item(path, force=False, ignore_errors=False): + """Delete file or folder, optionally recursively. + + NOTE: Exceptions are not caught by this function, + ignore_errors is passed to delete_folder to allow partial deletions. + """ + path = pathlib.Path(path) + if path.is_dir(): + delete_folder(path, force=force, ignore_errors=ignore_errors) + else: + os.remove(path) + + def non_clobbering_path(path): """Update path as needed to non-existing path, returns pathlib.Path.""" path = pathlib.Path(path) From b41027562a13d3ee411e82342a51ec24595bc086 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 18:47:52 -0700 Subject: [PATCH 087/324] Added SafeMode enter/exit sections --- .../outer_scripts_to_review/safemode_enter.py | 39 ----------------- .../outer_scripts_to_review/safemode_exit.py | 39 ----------------- scripts/safemode_enter.py | 37 ++++++++++++++++ scripts/safemode_exit.py | 37 ++++++++++++++++ scripts/wk/__init__.py | 2 +- scripts/wk/cfg/__init__.py | 2 +- scripts/wk/cfg/log.py | 2 +- scripts/wk/cfg/main.py | 4 +- scripts/wk/cfg/net.py | 2 +- scripts/wk/hw/__init__.py | 2 +- scripts/wk/io.py | 2 +- scripts/wk/kit/__init__.py | 2 +- scripts/wk/log.py | 2 +- scripts/wk/os/__init__.py | 8 +++- scripts/wk/os/win.py | 42 +++++++++++++++++++ scripts/wk/std.py | 4 +- scripts/wk/sw/__init__.py | 2 +- 17 files changed, 136 insertions(+), 92 deletions(-) delete mode 100644 scripts/outer_scripts_to_review/safemode_enter.py delete mode 100644 scripts/outer_scripts_to_review/safemode_exit.py create mode 100644 scripts/safemode_enter.py create mode 100644 scripts/safemode_exit.py create mode 100644 scripts/wk/os/win.py diff --git a/scripts/outer_scripts_to_review/safemode_enter.py b/scripts/outer_scripts_to_review/safemode_enter.py deleted file mode 100644 index de9ad119..00000000 --- a/scripts/outer_scripts_to_review/safemode_enter.py +++ /dev/null @@ -1,39 +0,0 @@ -# Wizard Kit: Enter SafeMode by editing the BCD - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.safemode import * -init_global_vars() -os.system('title {}: SafeMode Tool'.format(KIT_NAME_FULL)) - -if __name__ == '__main__': - try: - clear_screen() - print_info('{}: SafeMode Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': {'CalledProcessError': 'Unknown Error'}, - 'Warning': {}} - - if not ask('Enable booting to SafeMode (with Networking)?'): - abort() - - # Configure SafeMode - try_and_print(message='Set BCD option...', - function=enable_safemode, other_results=other_results) - try_and_print(message='Enable MSI in SafeMode...', - function=enable_safemode_msi, other_results=other_results) - - # Done - print_standard('\nDone.') - pause('Press Enter to reboot...') - reboot() - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/scripts/outer_scripts_to_review/safemode_exit.py b/scripts/outer_scripts_to_review/safemode_exit.py deleted file mode 100644 index 6c47b02d..00000000 --- a/scripts/outer_scripts_to_review/safemode_exit.py +++ /dev/null @@ -1,39 +0,0 @@ -# Wizard Kit: Exit SafeMode by editing the BCD - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.safemode import * -init_global_vars() -os.system('title {}: SafeMode Tool'.format(KIT_NAME_FULL)) - -if __name__ == '__main__': - try: - clear_screen() - print_info('{}: SafeMode Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': {'CalledProcessError': 'Unknown Error'}, - 'Warning': {}} - - if not ask('Disable booting to SafeMode?'): - abort() - - # Configure SafeMode - try_and_print(message='Remove BCD option...', - function=disable_safemode, other_results=other_results) - try_and_print(message='Disable MSI in SafeMode...', - function=disable_safemode_msi, other_results=other_results) - - # Done - print_standard('\nDone.') - pause('Press Enter to reboot...') - reboot() - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/scripts/safemode_enter.py b/scripts/safemode_enter.py new file mode 100644 index 00000000..87660560 --- /dev/null +++ b/scripts/safemode_enter.py @@ -0,0 +1,37 @@ +"""Wizard Kit: Enter SafeMode by editing the BCD""" +# vim: sts=2 sw=2 ts=2 + +import wk + + +def main(): + """Prompt user to enter safe mode.""" + title = f'{wk.cfg.main.KIT_NAME_FULL}: SafeMode Tool' + try_print = wk.std.TryAndPrint() + wk.std.clear_screen() + wk.std.set_title(title) + wk.std.print_info(title) + print('') + + # Ask + if not wk.std.ask('Enable booting to SafeMode (with Networking)?'): + wk.std.abort() + print('') + + # Configure SafeMode + try_print.run('Set BCD option...', wk.os.win.enable_safemode) + try_print.run('Enable MSI in SafeMode...', wk.os.win.enable_safemode_msi) + + # Done + print('Done.') + wk.std.pause('Press Enter to reboot...') + wk.std.run_program('shutdown -r -t 3'.split(), check=False) + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/safemode_exit.py b/scripts/safemode_exit.py new file mode 100644 index 00000000..a1bbfbd1 --- /dev/null +++ b/scripts/safemode_exit.py @@ -0,0 +1,37 @@ +"""Wizard Kit: Exit SafeMode by editing the BCD""" +# vim: sts=2 sw=2 ts=2 + +import wk + + +def main(): + """Prompt user to exit safe mode.""" + title = f'{wk.cfg.main.KIT_NAME_FULL}: SafeMode Tool' + try_print = wk.std.TryAndPrint() + wk.std.clear_screen() + wk.std.set_title(title) + wk.std.print_info(title) + print('') + + # Ask + if not wk.std.ask('Disable booting to SafeMode?'): + wk.std.abort() + print('') + + # Configure SafeMode + try_print.run('Remove BCD option...', wk.os.win.disable_safemode) + try_print.run('Disable MSI in SafeMode...', wk.os.win.disable_safemode_msi) + + # Done + print('Done.') + wk.std.pause('Press Enter to reboot...') + wk.std.run_program('shutdown -r -t 3'.split(), check=False) + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index d9264d83..959978c7 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -1,4 +1,4 @@ -'''WizardKit: wk module init''' +"""WizardKit: wk module init""" # vim: sts=2 sw=2 ts=2 from sys import version_info as version diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py index ff52308f..9538f1c8 100644 --- a/scripts/wk/cfg/__init__.py +++ b/scripts/wk/cfg/__init__.py @@ -1,4 +1,4 @@ -'''WizardKit: cfg module init''' +"""WizardKit: cfg module init""" from wk.cfg import log from wk.cfg import main diff --git a/scripts/wk/cfg/log.py b/scripts/wk/cfg/log.py index 925be4d3..a9a6e6ca 100644 --- a/scripts/wk/cfg/log.py +++ b/scripts/wk/cfg/log.py @@ -1,4 +1,4 @@ -'''WizardKit: Config - Log''' +"""WizardKit: Config - Log""" # vim: sts=2 sw=2 ts=2 diff --git a/scripts/wk/cfg/main.py b/scripts/wk/cfg/main.py index 6f5ac11f..2928f54a 100644 --- a/scripts/wk/cfg/main.py +++ b/scripts/wk/cfg/main.py @@ -1,6 +1,6 @@ -'''WizardKit: Config - Main +"""WizardKit: Config - Main -NOTE: A non-standard format is used for BASH/BATCH/PYTHON compatibility''' +NOTE: A non-standard format is used for BASH/BATCH/PYTHON compatibility""" # pylint: disable=bad-whitespace # vim: sts=2 sw=2 ts=2 diff --git a/scripts/wk/cfg/net.py b/scripts/wk/cfg/net.py index 8a84620c..344c695b 100644 --- a/scripts/wk/cfg/net.py +++ b/scripts/wk/cfg/net.py @@ -1,4 +1,4 @@ -'''WizardKit: Config - Net''' +"""WizardKit: Config - Net""" # vim: sts=2 sw=2 ts=2 diff --git a/scripts/wk/hw/__init__.py b/scripts/wk/hw/__init__.py index 6c19c9c7..2b0fbc10 100644 --- a/scripts/wk/hw/__init__.py +++ b/scripts/wk/hw/__init__.py @@ -1 +1 @@ -'''WizardKit: hw module init''' +"""WizardKit: hw module init""" diff --git a/scripts/wk/io.py b/scripts/wk/io.py index 5ff75d7b..846db77e 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -1,4 +1,4 @@ -'''WizardKit: I/O Functions''' +"""WizardKit: I/O Functions""" # vim: sts=2 sw=2 ts=2 import os diff --git a/scripts/wk/kit/__init__.py b/scripts/wk/kit/__init__.py index 61f06215..2fc43258 100644 --- a/scripts/wk/kit/__init__.py +++ b/scripts/wk/kit/__init__.py @@ -1 +1 @@ -'''WizardKit: kit module init''' +"""WizardKit: kit module init""" diff --git a/scripts/wk/log.py b/scripts/wk/log.py index e2ee6336..e930592b 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -1,4 +1,4 @@ -'''WizardKit: Log Functions''' +"""WizardKit: Log Functions""" # vim: sts=2 sw=2 ts=2 import atexit diff --git a/scripts/wk/os/__init__.py b/scripts/wk/os/__init__.py index da1056d8..5e58dbc8 100644 --- a/scripts/wk/os/__init__.py +++ b/scripts/wk/os/__init__.py @@ -1 +1,7 @@ -'''WizardKit: os module init''' +"""WizardKit: os module init""" +# vim: sts=2 sw=2 ts=2 + +import os + +if os.name == 'nt': + from wk.os import win diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py new file mode 100644 index 00000000..85073e46 --- /dev/null +++ b/scripts/wk/os/win.py @@ -0,0 +1,42 @@ +"""WizardKit: Windows Functions""" +# vim: sts=2 sw=2 ts=2 + +from wk.std import run_program + +# STATIC VARIABLES +REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' + + +# Functions +def disable_safemode(): + """Edit BCD to remove safeboot value.""" + cmd = ['bcdedit', '/deletevalue', '{default}', 'safeboot'] + run_program(cmd) + + +def disable_safemode_msi(): + """Disable MSI access under safemode.""" + cmd = ['reg', 'delete', REG_MSISERVER, '/f'] + run_program(cmd) + + +def enable_safemode(): + """Edit BCD to set safeboot as default.""" + cmd = ['bcdedit', '/set', '{default}', 'safeboot', 'network'] + run_program(cmd) + + +def enable_safemode_msi(): + """Enable MSI access under safemode.""" + cmd = ['reg', 'add', REG_MSISERVER, '/f'] + run_program(cmd) + cmd = [ + 'reg', 'add', REG_MSISERVER, '/ve', + '/t', 'REG_SZ', + '/d', 'Service', '/f', + ] + run_program(cmd) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 8a5065ca..d1633af6 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,4 +1,4 @@ -'''WizardKit: Standard Functions''' +"""WizardKit: Standard Functions""" # pylint: disable=too-many-lines # vim: sts=2 sw=2 ts=2 @@ -476,7 +476,7 @@ class TryAndPrint(): if exception_name not in self.list_warnings: self.list_warnings.append(exception_name) - def run_function( + def run( self, message, function, *args, catch_all=True, print_return=False, verbose=False, **kwargs): # pylint: disable=catching-non-exception diff --git a/scripts/wk/sw/__init__.py b/scripts/wk/sw/__init__.py index f64f5df6..1cc6236f 100644 --- a/scripts/wk/sw/__init__.py +++ b/scripts/wk/sw/__init__.py @@ -1 +1 @@ -'''WizardKit: sw module init''' +"""WizardKit: sw module init""" From 2678ce77da4e3160d328dab0b97823a63170a0f2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Sep 2019 19:34:06 -0700 Subject: [PATCH 088/324] Added SFC scan --- scripts/outer_scripts_to_review/sfc_scan.py | 40 ---------------- scripts/sfc_scan.py | 35 ++++++++++++++ scripts/wk/os/win.py | 51 ++++++++++++++++++++- 3 files changed, 85 insertions(+), 41 deletions(-) delete mode 100644 scripts/outer_scripts_to_review/sfc_scan.py create mode 100644 scripts/sfc_scan.py diff --git a/scripts/outer_scripts_to_review/sfc_scan.py b/scripts/outer_scripts_to_review/sfc_scan.py deleted file mode 100644 index ec85836a..00000000 --- a/scripts/outer_scripts_to_review/sfc_scan.py +++ /dev/null @@ -1,40 +0,0 @@ -# Wizard Kit: Check, and possibly repair, system file health via SFC - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.repairs import * -init_global_vars() -os.system('title {}: SFC Tool'.format(KIT_NAME_FULL)) -set_log_file('SFC Tool.log') - -if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: SFC Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - }, - 'Warning': { - 'GenericRepair': 'Repaired', - }} - if ask('Run a SFC scan now?'): - try_and_print(message='SFC scan...', - function=run_sfc_scan, other_results=other_results) - else: - abort() - - # Done - print_standard('\nDone.') - pause('Press Enter to exit...') - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/scripts/sfc_scan.py b/scripts/sfc_scan.py new file mode 100644 index 00000000..02d84f77 --- /dev/null +++ b/scripts/sfc_scan.py @@ -0,0 +1,35 @@ +"""Wizard Kit: Check, and possibly repair, system file health via SFC""" +# vim: sts=2 sw=2 ts=2 + +import wk + + +def main(): + """Run SFC and report result.""" + title = f'{wk.cfg.main.KIT_NAME_FULL}: SFC Tool' + try_print = wk.std.TryAndPrint() + wk.std.clear_screen() + wk.std.set_title(title) + wk.std.print_info(title) + print('') + + # Ask + if not wk.std.ask('Run a SFC scan now?'): + wk.std.abort() + print('') + + # Run + try_print.run('SFC scan...', wk.os.win.run_sfc_scan) + + # Done + print('Done') + wk.std.pause('Press Enter to exit...') + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index 85073e46..2f3731b7 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -1,9 +1,22 @@ """WizardKit: Windows Functions""" # vim: sts=2 sw=2 ts=2 -from wk.std import run_program +import logging +import os +import pathlib +import re +import time + +from wk import cfg +from wk.io import non_clobber_path +from wk.std import ( + GenericError, + GenericWarning, + run_program, + ) # STATIC VARIABLES +LOG = logging.getLogger(__name__) REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' @@ -38,5 +51,41 @@ def enable_safemode_msi(): run_program(cmd) +def run_sfc_scan(): + """Run SFC and save results.""" + cmd = ['sfc', '/scannow'] + log_path = pathlib.Path( + '{drive}/{short}/Logs/{date}/Tools/SFC.log'.format( + drive=os.environ.get('SYSTEMDRIVE', 'C:'), + short=cfg.main.KIT_NAME_SHORT, + date=time.strftime('%Y-%m-%d'), + )) + err_path = log_path.with_suffix('.err') + + # Run SFC + proc = run_program(cmd, check=False) + + # Fix paths + log_path = non_clobber_path(log_path) + err_path = non_clobber_path(err_path) + + # Save output + output = proc.stdout.replace('\0', '') + errors = proc.stderr.replace('\0', '') + os.makedirs(log_path.parent, exist_ok=True) + with open(log_path, 'w') as _f: + _f.write(output) + with open(err_path, 'w') as _f: + _f.write(errors) + + # Check result + if re.findall(r'did\s+not\s+find\s+any\s+integrity\s+violations', output): + pass + elif re.findall(r'successfully\s+repaired\s+them', output): + raise GenericWarning('Repaired') + else: + raise GenericError + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 304d81169804d50746c56895a00ca9b0be1b99b0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 18:49:56 -0700 Subject: [PATCH 089/324] Moved exe functions to a separate file --- scripts/safemode_enter.py | 2 +- scripts/safemode_exit.py | 2 +- scripts/wk/__init__.py | 1 + scripts/wk/exe.py | 120 ++++++++++++++++++++++++++++++++++++++ scripts/wk/os/win.py | 7 +-- scripts/wk/std.py | 111 ----------------------------------- 6 files changed, 125 insertions(+), 118 deletions(-) create mode 100644 scripts/wk/exe.py diff --git a/scripts/safemode_enter.py b/scripts/safemode_enter.py index 87660560..fffa585e 100644 --- a/scripts/safemode_enter.py +++ b/scripts/safemode_enter.py @@ -25,7 +25,7 @@ def main(): # Done print('Done.') wk.std.pause('Press Enter to reboot...') - wk.std.run_program('shutdown -r -t 3'.split(), check=False) + wk.exe.run_program('shutdown -r -t 3'.split(), check=False) if __name__ == '__main__': diff --git a/scripts/safemode_exit.py b/scripts/safemode_exit.py index a1bbfbd1..e46c9ade 100644 --- a/scripts/safemode_exit.py +++ b/scripts/safemode_exit.py @@ -25,7 +25,7 @@ def main(): # Done print('Done.') wk.std.pause('Press Enter to reboot...') - wk.std.run_program('shutdown -r -t 3'.split(), check=False) + wk.exe.run_program('shutdown -r -t 3'.split(), check=False) if __name__ == '__main__': diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 959978c7..511cf8a7 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -4,6 +4,7 @@ from sys import version_info as version from wk import cfg +from wk import exe from wk import hw from wk import io from wk import kit diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py new file mode 100644 index 00000000..9b454b3a --- /dev/null +++ b/scripts/wk/exe.py @@ -0,0 +1,120 @@ +"""WizardKit: Executable functions""" +#vim: sts=2 sw=2 ts=2 + +import re +import subprocess + +import psutil + + +# Functions +def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): + """Build kwargs for use by subprocess functions, returns dict. + + Specifically subprocess.run() and subprocess.Popen(). + NOTE: If no encoding specified then UTF-8 will be used. + """ + cmd_kwargs = { + 'args': cmd, + 'shell': shell, + } + + # Add additional kwargs if applicable + for key in ('check', 'cwd', 'encoding', 'errors', 'stderr', 'stdout'): + if key in kwargs: + cmd_kwargs[key] = kwargs[key] + + # Default to UTF-8 encoding + if not ('encoding' in cmd_kwargs or 'errors' in cmd_kwargs): + cmd_kwargs['encoding'] = 'utf-8' + cmd_kwargs['errors'] = 'ignore' + + # Start minimized + if minimized: + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = 6 + cmd_kwargs['startupinfo'] = startupinfo + + + # Pipe output + if pipe: + cmd_kwargs['stderr'] = subprocess.PIPE + cmd_kwargs['stdout'] = subprocess.PIPE + + # Done + return cmd_kwargs + + +def get_procs(name, exact=True): + """Get process object(s) based on name, returns list of proc objects.""" + processes = [] + regex = f'^{name}$' if exact else name + + # Iterate over all processes + for proc in psutil.process_iter(): + if re.search(regex, proc.name(), re.IGNORECASE): + processes.append(proc) + + # Done + return processes + + +def kill_procs(name, exact=True, force=False, timeout=30): + """Kill all processes matching name (case-insensitively). + + NOTE: Under Posix systems this will send SIGINT to allow processes + to gracefully exit. + + If force is True then it will wait until timeout specified and then + send SIGKILL to any processes still alive. + """ + target_procs = get_procs(name, exact=exact) + for proc in target_procs: + proc.terminate() + + # Force kill if necesary + if force: + results = psutil.wait_procs(target_procs, timeout=timeout) + for proc in results[1]: # Alive processes + proc.kill() + + +def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): + """Run program and return a subprocess.Popen object.""" + cmd_kwargs = build_cmd_kwargs( + cmd, + minimized=minimized, + pipe=pipe, + shell=shell, + **kwargs) + + # Ready to run program + return subprocess.Popen(**cmd_kwargs) + + +def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): + """Run program and return a subprocess.CompletedProcess object.""" + cmd_kwargs = build_cmd_kwargs( + cmd, + check=check, + pipe=pipe, + shell=shell, + **kwargs) + + # Ready to run program + return subprocess.run(**cmd_kwargs) + + +def wait_for_procs(name, exact=True, timeout=None): + """Wait for all process matching name.""" + target_procs = get_procs(name, exact=exact) + results = psutil.wait_procs(target_procs, timeout=timeout) + + # Raise exception if necessary + if results[1]: # Alive processes + raise psutil.TimeoutExpired(name=name, seconds=timeout) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index 2f3731b7..8c70944e 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -8,12 +8,9 @@ import re import time from wk import cfg +from wk.exe import run_program from wk.io import non_clobber_path -from wk.std import ( - GenericError, - GenericWarning, - run_program, - ) +from wk.std import GenericError, GenericWarning # STATIC VARIABLES LOG = logging.getLogger(__name__) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index d1633af6..36417f8b 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,5 +1,4 @@ """WizardKit: Standard Functions""" -# pylint: disable=too-many-lines # vim: sts=2 sw=2 ts=2 import itertools @@ -22,8 +21,6 @@ except ImportError: # Not worried about this under Windows raise -import psutil - from wk.cfg.main import ( CRASH_SERVER, ENABLED_UPLOAD_DATA, @@ -589,44 +586,6 @@ def beep(repeat=1): repeat -= 1 -def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): - """Build kwargs for use by subprocess functions, returns dict. - - Specifically subprocess.run() and subprocess.Popen(). - NOTE: If no encoding specified then UTF-8 will be used. - """ - cmd_kwargs = { - 'args': cmd, - 'shell': shell, - } - - # Add additional kwargs if applicable - for key in ('check', 'cwd', 'encoding', 'errors', 'stderr', 'stdout'): - if key in kwargs: - cmd_kwargs[key] = kwargs[key] - - # Default to UTF-8 encoding - if not ('encoding' in cmd_kwargs or 'errors' in cmd_kwargs): - cmd_kwargs['encoding'] = 'utf-8' - cmd_kwargs['errors'] = 'ignore' - - # Start minimized - if minimized: - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = 6 - cmd_kwargs['startupinfo'] = startupinfo - - - # Pipe output - if pipe: - cmd_kwargs['stderr'] = subprocess.PIPE - cmd_kwargs['stdout'] = subprocess.PIPE - - # Done - return cmd_kwargs - - def bytes_to_string(size, decimals=0, use_binary=True): """Convert size into a human-readable format, returns str. @@ -775,20 +734,6 @@ def get_log_filepath(): return log_filepath -def get_procs(name, exact=True): - """Get process object(s) based on name, returns list of proc objects.""" - processes = [] - regex = f'^{name}$' if exact else name - - # Iterate over all processes - for proc in psutil.process_iter(): - if re.search(regex, proc.name(), re.IGNORECASE): - processes.append(proc) - - # Done - return processes - - def input_text(prompt='Enter text'): """Get text from user, returns string.""" prompt = str(prompt) @@ -811,26 +756,6 @@ def input_text(prompt='Enter text'): return response -def kill_procs(name, exact=True, force=False, timeout=30): - """Kill all processes matching name (case-insensitively). - - NOTE: Under Posix systems this will send SIGINT to allow processes - to gracefully exit. - - If force is True then it will wait until timeout specified and then - send SIGKILL to any processes still alive. - """ - target_procs = get_procs(name, exact=exact) - for proc in target_procs: - proc.terminate() - - # Force kill if necesary - if force: - results = psutil.wait_procs(target_procs, timeout=timeout) - for proc in results[1]: # Alive processes - proc.kill() - - def major_exception(): """Display traceback, optionally upload detailes, and exit.""" LOG.critical('Major exception encountered', exc_info=True) @@ -864,19 +789,6 @@ def pause(prompt='Press Enter to continue... '): input_text(prompt) -def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): - """Run program and return a subprocess.Popen object.""" - cmd_kwargs = build_cmd_kwargs( - cmd, - minimized=minimized, - pipe=pipe, - shell=shell, - **kwargs) - - # Ready to run program - return subprocess.Popen(**cmd_kwargs) - - def print_colored(strings, colors, **kwargs): """Prints strings in the colors specified.""" LOG.debug('strings: %s, colors: %s, kwargs: %s', strings, colors, kwargs) @@ -926,19 +838,6 @@ def print_warning(msg, **kwargs): print_colored([msg], ['YELLOW'], **kwargs) -def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): - """Run program and return a subprocess.CompletedProcess object.""" - cmd_kwargs = build_cmd_kwargs( - cmd, - check=check, - pipe=pipe, - shell=shell, - **kwargs) - - # Ready to run program - return subprocess.run(**cmd_kwargs) - - def set_title(title): """Set window title.""" if os.name == 'nt': @@ -1040,15 +939,5 @@ def upload_debug_report(report, compress=True, reason='DEBUG'): raise RuntimeError('Failed to upload report') -def wait_for_procs(name, exact=True, timeout=None): - """Wait for all process matching name.""" - target_procs = get_procs(name, exact=exact) - results = psutil.wait_procs(target_procs, timeout=timeout) - - # Raise exception if necessary - if results[1]: # Alive processes - raise psutil.TimeoutExpired(name=name, seconds=timeout) - - if __name__ == '__main__': print("This file is not meant to be called directly.") From f55f0ba0167454bff7f00fe5b1a2942d07f603d3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 20:02:13 -0700 Subject: [PATCH 090/324] Adjusted logging in TryAndPrint() --- scripts/wk/cfg/log.py | 2 +- scripts/wk/std.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/wk/cfg/log.py b/scripts/wk/cfg/log.py index a9a6e6ca..06f79df4 100644 --- a/scripts/wk/cfg/log.py +++ b/scripts/wk/cfg/log.py @@ -10,7 +10,7 @@ DEBUG = { DEFAULT = { 'level': 'INFO', 'format': '[%(asctime)s %(levelname)s] %(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S %z', + 'datefmt': '%Y-%m-%d %H%M%z', } diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 36417f8b..96da0c08 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -463,6 +463,13 @@ class TryAndPrint(): obj = getattr(sys.modules['builtins'], name) return obj + def _log_result(self, message, result_msg): + """Log result text without color formatting.""" + log_text = f'{" "*self.indent}{message:<{self.width}}{result_msg}' + for line in log_text.splitlines(): + line = strip_colors(line) + LOG.info(line) + def add_error(self, exception_name): """Add exception name to error list.""" if exception_name not in self.list_errors: @@ -539,7 +546,7 @@ class TryAndPrint(): raise # Done - LOG.info('Result: %s', result_msg.strip()) + self._log_result(message, result_msg) return { 'Failed': bool(f_exception), 'Exception': f_exception, From 5925aca3c267af2f19dc9c04479484dc4c2dfe1d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 20:02:39 -0700 Subject: [PATCH 091/324] Allow strings to be passed to print_colored() --- scripts/wk/std.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 96da0c08..b78743c3 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -807,6 +807,12 @@ def print_colored(strings, colors, **kwargs): 'flush': kwargs.get('flush', False), } + # Convert to tuples if necessary + if isinstance(strings, str): + strings = (strings,) + if isinstance(colors, str): + colors = (colors,) + # Build new string with color escapes added for string, color in itertools.zip_longest(strings, colors): color_code = COLORS.get(color, clear_code) From 318f59c473f3d2fc327c4cdfea4e3af4e1ab0bfb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 20:23:13 -0700 Subject: [PATCH 092/324] Added logging to print functions --- scripts/wk/std.py | 64 +++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index b78743c3..5532bd1b 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -521,29 +521,34 @@ class TryAndPrint(): LOG.info('Running function: %s.%s', function.__module__, function.__name__) try: output = function(*args, **kwargs) - if print_return: - result_msg = self._format_function_output(output) - print(result_msg) - else: - print_success(self.msg_good) except w_exceptions as _exception: + # Warnings result_msg = self._format_exception_message(_exception) - print_warning(result_msg) + print_warning(result_msg, log=False) f_exception = _exception except e_exceptions as _exception: + # Exceptions result_msg = self._format_exception_message(_exception) - print_error(result_msg) + print_error(result_msg, log=False) f_exception = _exception except Exception as _exception: # pylint: disable=broad-except + # Unexpected exceptions if verbose: result_msg = self._format_exception_message(_exception) else: result_msg = self.msg_bad - print_error(result_msg) + print_error(result_msg, log=False) f_exception = _exception if not catch_all: # Re-raise error as necessary raise + else: + # Success + if print_return: + result_msg = self._format_function_output(output) + print(result_msg) + else: + print_success(self.msg_good, log=False) # Done self._log_result(message, result_msg) @@ -558,7 +563,6 @@ class TryAndPrint(): def abort(prompt='Aborted.', show_prompt=True, return_code=1): """Abort script.""" print_warning(prompt) - LOG.warning(prompt) if show_prompt: sleep(1) pause(prompt='Press Enter to exit... ') @@ -766,7 +770,7 @@ def input_text(prompt='Enter text'): def major_exception(): """Display traceback, optionally upload detailes, and exit.""" LOG.critical('Major exception encountered', exc_info=True) - print_error('Major exception') + print_error('Major exception', log=False) print_warning(SUPPORT_MESSAGE) print(traceback.format_exc()) @@ -780,10 +784,10 @@ def major_exception(): try: upload_debug_report(report, reason='CRASH') except Exception: #pylint: disable=broad-except - print_error('FAILED') + print_error('FAILED', log=False) LOG.error('Upload failed', exc_info=True) else: - print_success('SUCCESS') + print_success('SUCCESS', log=False) LOG.info('Upload successful') # Done @@ -821,34 +825,45 @@ def print_colored(strings, colors, **kwargs): print(msg, **print_options) -def print_error(msg, **kwargs): - """Prints message in RED.""" - LOG.debug('msg: %s, kwargs: %s', msg, kwargs) +def print_error(msg, log=True, **kwargs): + """Prints message in RED and log as ERROR.""" if 'file' not in kwargs: # Only set if not specified kwargs['file'] = sys.stderr print_colored([msg], ['RED'], **kwargs) + if log: + LOG.error(msg) -def print_info(msg, **kwargs): - """Prints message in BLUE.""" - LOG.debug('msg: %s, kwargs: %s', msg, kwargs) +def print_info(msg, log=True, **kwargs): + """Prints message in BLUE and log as INFO.""" print_colored([msg], ['BLUE'], **kwargs) + if log: + LOG.info(msg, log=True) -def print_success(msg, **kwargs): - """Prints message in GREEN.""" - LOG.debug('msg: %s, kwargs: %s', msg, kwargs) +def print_standard(msg, log=True, **kwargs): + """Prints message and log as INFO.""" + print(msg, log=True, **kwargs) + if log: + LOG.info(msg, log=True) + + +def print_success(msg, log=True, **kwargs): + """Prints message in GREEN and log as INFO.""" print_colored([msg], ['GREEN'], **kwargs) + if log: + LOG.info(msg, log=True) -def print_warning(msg, **kwargs): - """Prints message in YELLOW.""" - LOG.debug('msg: %s, kwargs: %s', msg, kwargs) +def print_warning(msg, log=True, **kwargs): + """Prints message in YELLOW and log as WARNING.""" if 'file' not in kwargs: # Only set if not specified kwargs['file'] = sys.stderr print_colored([msg], ['YELLOW'], **kwargs) + if log: + LOG.warning(msg) def set_title(title): @@ -920,7 +935,6 @@ def upload_debug_report(report, compress=True, reason='DEBUG'): # Check if the required server details are available if not all(CRASH_SERVER.get(key, False) for key in ('Name', 'Url', 'User')): msg = 'Server details missing, aborting upload.' - LOG.error(msg) print_error(msg) raise RuntimeError(msg) From f1775766e7e5cc03e3c6ea7da9cf7b9b0d75a4d6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 20:58:06 -0700 Subject: [PATCH 093/324] Updated wk.log.update_log_path() * The log file is now moved instead of copied * The new path can now be based on a new dir, name, or both --- scripts/wk/log.py | 44 ++++++++++++++++++++++++++------------------ scripts/wk/std.py | 8 ++++---- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index e930592b..bdba18df 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -59,39 +59,47 @@ def start(config=None): atexit.register(logging.shutdown) -def update_log_path(dest_dir, dest_name=''): - """Copies current log file to new dir and updates the root logger. +def update_log_path(dest_dir=None, dest_name=None): + """Moves current log file to new path and updates the root logger. NOTE: A timestamp and extension will be added to dest_name if provided. """ root_logger = logging.getLogger() cur_handler = root_logger.handlers[0] - dest = pathlib.Path(dest_dir) - dest = dest.expanduser() - if dest_name: - dest_name = f'{dest_name}_{time.strftime("%Y-%m-%d_%H%M%S%z")}.log' + cur_path = pathlib.Path(cur_handler.baseFilename).resolve() # Safety checks + if not (dest_dir or dest_name): + raise RuntimeError('Neither a directory nor name specified') if len(root_logger.handlers) > 1: raise RuntimeError('Multiple handlers not supported') if not isinstance(cur_handler, logging.FileHandler): raise RuntimeError('Only FileHandlers are supported') - # Copy original log to new location - source = pathlib.Path(cur_handler.baseFilename) - source = source.resolve() - if dest_name: - dest = dest.joinpath(dest_name) + # Update dir if specified or use current path + if dest_dir: + new_path = pathlib.Path(dest_dir).resolve() else: - dest = dest.joinpath(source.name) - dest = dest.resolve() - if dest.exists(): - raise FileExistsError(f'Refusing to clobber: {dest}') - os.makedirs(dest.parent, exist_ok=True) - shutil.copy(source, dest) + new_path = cur_path + + # Update name if specified + if dest_name: + new_path = new_path.with_name( + f'{dest_name}' + f'_{time.strftime("%Y-%m-%d_%H%M%S%z")}' + f'{"".join(cur_path.suffixes)}' + ) + else: + new_path = new_path.with_name(cur_path.name) + + # Copy original log to new location + if new_path.exists(): + raise FileExistsError(f'Refusing to clobber: {new_path}') + os.makedirs(new_path.parent, exist_ok=True) + shutil.move(cur_path, new_path) # Create new cur_handler (preserving formatter settings) - new_handler = logging.FileHandler(dest, mode='a') + new_handler = logging.FileHandler(new_path, mode='a') new_handler.setFormatter(cur_handler.formatter) # Replace current handler diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 5532bd1b..8c3b8c2b 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -839,21 +839,21 @@ def print_info(msg, log=True, **kwargs): """Prints message in BLUE and log as INFO.""" print_colored([msg], ['BLUE'], **kwargs) if log: - LOG.info(msg, log=True) + LOG.info(msg) def print_standard(msg, log=True, **kwargs): """Prints message and log as INFO.""" - print(msg, log=True, **kwargs) + print(msg, **kwargs) if log: - LOG.info(msg, log=True) + LOG.info(msg) def print_success(msg, log=True, **kwargs): """Prints message in GREEN and log as INFO.""" print_colored([msg], ['GREEN'], **kwargs) if log: - LOG.info(msg, log=True) + LOG.info(msg) def print_warning(msg, log=True, **kwargs): From 27d348bf9c4d16072d627e23b096e405ca9698c0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 21:36:39 -0700 Subject: [PATCH 094/324] Expanded debug log --- scripts/wk/exe.py | 29 ++++++++++++++++++++++++++++- scripts/wk/io.py | 19 ++++++++++++++++++- scripts/wk/os/win.py | 1 + scripts/wk/std.py | 1 + 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index 9b454b3a..a3a29477 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -1,12 +1,17 @@ """WizardKit: Executable functions""" #vim: sts=2 sw=2 ts=2 +import logging import re import subprocess import psutil +# STATIC VARIABLES +LOG = logging.getLogger(__name__) + + # Functions def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): """Build kwargs for use by subprocess functions, returns dict. @@ -14,6 +19,11 @@ def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): Specifically subprocess.run() and subprocess.Popen(). NOTE: If no encoding specified then UTF-8 will be used. """ + LOG.debug( + 'cmd: %s, minimized: %s, pipe: %s, shell: %s', + cmd, minimized, pipe, shell, + ) + LOG.debug('kwargs: %s', kwargs) cmd_kwargs = { 'args': cmd, 'shell': shell, @@ -43,11 +53,13 @@ def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): cmd_kwargs['stdout'] = subprocess.PIPE # Done + LOG.debug('cmd_kwargs: %s', cmd_kwargs) return cmd_kwargs def get_procs(name, exact=True): """Get process object(s) based on name, returns list of proc objects.""" + LOG.debug('name: %s, exact: %s', name, exact) processes = [] regex = f'^{name}$' if exact else name @@ -69,6 +81,10 @@ def kill_procs(name, exact=True, force=False, timeout=30): If force is True then it will wait until timeout specified and then send SIGKILL to any processes still alive. """ + LOG.debug( + 'name: %s, exact: %s, force: %s, timeout: %s', + name, exact, force, timeout, + ) target_procs = get_procs(name, exact=exact) for proc in target_procs: proc.terminate() @@ -80,8 +96,13 @@ def kill_procs(name, exact=True, force=False, timeout=30): proc.kill() -def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): +def popen_program(cmd, minimized=False, pipe=False, shell=False, **kwargs): """Run program and return a subprocess.Popen object.""" + LOG.debug( + 'cmd: %s, minimized: %s, pipe: %s, shell: %s', + cmd, minimized, pipe, shell, + ) + LOG.debug('kwargs: %s', kwargs) cmd_kwargs = build_cmd_kwargs( cmd, minimized=minimized, @@ -95,6 +116,11 @@ def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): """Run program and return a subprocess.CompletedProcess object.""" + LOG.debug( + 'cmd: %s, check: %s, pipe: %s, shell: %s', + cmd, check, pipe, shell, + ) + LOG.debug('kwargs: %s', kwargs) cmd_kwargs = build_cmd_kwargs( cmd, check=check, @@ -108,6 +134,7 @@ def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): def wait_for_procs(name, exact=True, timeout=None): """Wait for all process matching name.""" + LOG.debug('name: %s, exact: %s, timeout: %s', name, exact, timeout) target_procs = get_procs(name, exact=exact) results = psutil.wait_procs(target_procs, timeout=timeout) diff --git a/scripts/wk/io.py b/scripts/wk/io.py index 846db77e..fa6110c8 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -1,15 +1,20 @@ """WizardKit: I/O Functions""" # vim: sts=2 sw=2 ts=2 +import logging import os +import pathlib import shutil -import pathlib +# STATIC VARIABLES +LOG = logging.getLogger(__name__) # Functions def delete_empty_folders(path): """Recursively delete all empty folders in path.""" + LOG.debug('path: %s', path) + # Delete empty subfolders first for item in os.scandir(path): if item.is_dir(): @@ -29,6 +34,11 @@ def delete_folder(path, force=False, ignore_errors=False): NOTE: Exceptions are not caught by this function, ignore_errors is passed to shutil.rmtree to allow partial deletions. """ + LOG.debug( + 'path: %s, force: %s, ignore_errors: %s', + path, force, ignore_errors, + ) + if force: shutil.rmtree(path, ignore_errors=ignore_errors) else: @@ -41,6 +51,11 @@ def delete_item(path, force=False, ignore_errors=False): NOTE: Exceptions are not caught by this function, ignore_errors is passed to delete_folder to allow partial deletions. """ + LOG.debug( + 'path: %s, force: %s, ignore_errors: %s', + path, force, ignore_errors, + ) + path = pathlib.Path(path) if path.is_dir(): delete_folder(path, force=force, ignore_errors=ignore_errors) @@ -50,6 +65,7 @@ def delete_item(path, force=False, ignore_errors=False): def non_clobbering_path(path): """Update path as needed to non-existing path, returns pathlib.Path.""" + LOG.debug('path: %s', path) path = pathlib.Path(path) name = path.name new_path = None @@ -71,6 +87,7 @@ def non_clobbering_path(path): raise FileExistsError(new_path) # Done + LOG.debug('new path: %s', new_path) return new_path diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index 8c70944e..cd6dfe7c 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -12,6 +12,7 @@ from wk.exe import run_program from wk.io import non_clobber_path from wk.std import GenericError, GenericWarning + # STATIC VARIABLES LOG = logging.getLogger(__name__) REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 8c3b8c2b..42f29608 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -868,6 +868,7 @@ def print_warning(msg, log=True, **kwargs): def set_title(title): """Set window title.""" + LOG.debug('title: %s', title) if os.name == 'nt': os.system(f'title {title}') else: From 0636a032beaa971a84556600a94d3936306323f5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 21:51:39 -0700 Subject: [PATCH 095/324] Added threading functions --- scripts/wk/exe.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index a3a29477..6dfd978e 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -1,10 +1,13 @@ -"""WizardKit: Executable functions""" +"""WizardKit: Execution functions""" #vim: sts=2 sw=2 ts=2 import logging import re import subprocess +from threading import Thread +from queue import Queue, Empty + import psutil @@ -12,6 +15,38 @@ import psutil LOG = logging.getLogger(__name__) +# Classes +class NonBlockingStreamReader(): + """Class to allow non-blocking reads from a stream.""" + # pylint: disable=too-few-public-methods + # Credits: + ## https://gist.github.com/EyalAr/7915597 + ## https://stackoverflow.com/a/4896288 + + def __init__(self, stream): + self.stream = stream + self.queue = Queue() + + def populate_queue(stream, queue): + """Collect lines from stream and put them in queue.""" + while True: + line = stream.read(1) + if line: + queue.put(line) + + self.thread = start_thread( + populate_queue, + args=(self.stream, self.queue), + ) + + def read(self, timeout=None): + """Read from queue if possible, returns item from queue.""" + try: + return self.queue.get(block=timeout is not None, timeout=timeout) + except Empty: + return None + + # Functions def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): """Build kwargs for use by subprocess functions, returns dict. @@ -132,6 +167,14 @@ def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): return subprocess.run(**cmd_kwargs) +def start_thread(function, args=None, daemon=True): + """Run function as thread in background, returns Thread object.""" + args = args if args else [] + thread = Thread(target=function, args=args, daemon=daemon) + thread.start() + return thread + + def wait_for_procs(name, exact=True, timeout=None): """Wait for all process matching name.""" LOG.debug('name: %s, exact: %s, timeout: %s', name, exact, timeout) From 115a462f6e49509605a6bb5633bb6eeaa819aef0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 22:27:38 -0700 Subject: [PATCH 096/324] Added Windows activation functions --- scripts/{ => wk}/borrowed/acpi.py | 0 .../{ => wk}/borrowed/knownpaths-LICENSE.txt | 0 scripts/{ => wk}/borrowed/knownpaths.py | 0 scripts/wk/os/win.py | 59 ++++++++++++++++++- 4 files changed, 58 insertions(+), 1 deletion(-) rename scripts/{ => wk}/borrowed/acpi.py (100%) rename scripts/{ => wk}/borrowed/knownpaths-LICENSE.txt (100%) rename scripts/{ => wk}/borrowed/knownpaths.py (100%) diff --git a/scripts/borrowed/acpi.py b/scripts/wk/borrowed/acpi.py similarity index 100% rename from scripts/borrowed/acpi.py rename to scripts/wk/borrowed/acpi.py diff --git a/scripts/borrowed/knownpaths-LICENSE.txt b/scripts/wk/borrowed/knownpaths-LICENSE.txt similarity index 100% rename from scripts/borrowed/knownpaths-LICENSE.txt rename to scripts/wk/borrowed/knownpaths-LICENSE.txt diff --git a/scripts/borrowed/knownpaths.py b/scripts/wk/borrowed/knownpaths.py similarity index 100% rename from scripts/borrowed/knownpaths.py rename to scripts/wk/borrowed/knownpaths.py diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index cd6dfe7c..8e00ec1a 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -8,17 +8,54 @@ import re import time from wk import cfg +from wk.borrowed import acpi from wk.exe import run_program from wk.io import non_clobber_path -from wk.std import GenericError, GenericWarning +from wk.std import GenericError, GenericWarning, sleep # STATIC VARIABLES LOG = logging.getLogger(__name__) REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' +SLMGR = pathlib.Path(f'{os.environ("SYSTEMROOT")}/System32/slmgr.vbs') # Functions +def activate_with_bios(): + """Attempt to activate Windows with a key stored in the BIOS.""" + # Code borrowed from https://github.com/aeruder/get_win8key + ##################################################### + #script to query windows 8.x OEM key from PC firmware + #ACPI -> table MSDM -> raw content -> byte offset 56 to end + #ck, 03-Jan-2014 (christian@korneck.de) + ##################################################### + bios_key = None + table = b"MSDM" + if acpi.FindAcpiTable(table) is True: + rawtable = acpi.GetAcpiTable(table) + #http://msdn.microsoft.com/library/windows/hardware/hh673514 + #byte offset 36 from beginning + # = Microsoft 'software licensing data structure' + # / 36 + 20 bytes offset from beginning = Win Key + bios_key = rawtable[56:len(rawtable)].decode("utf-8") + if not bios_key: + raise GenericError('BIOS key not found.') + + # Install Key + cmd = ['cscript', '//nologo', SLMGR, '/ipk', bios_key] + run_program(cmd, check=False) + sleep(5) + + # Attempt activation + cmd = ['cscript', '//nologo', SLMGR, '/ato'] + run_program(cmd, check=False) + sleep(5) + + # Check status + if not windows_is_activated(): + raise GenericError('Activation Failed') + + def disable_safemode(): """Edit BCD to remove safeboot value.""" cmd = ['bcdedit', '/deletevalue', '{default}', 'safeboot'] @@ -49,6 +86,16 @@ def enable_safemode_msi(): run_program(cmd) +def get_activation_string(): + """Get activation status, returns str.""" + cmd = ['cscript', '//nologo', SLMGR, '/xpr'] + result = run_program(cmd, check=False) + act_str = result.stdout + act_str = act_str.splitlines()[1] + act_str = act_str.strip() + return act_str + + def run_sfc_scan(): """Run SFC and save results.""" cmd = ['sfc', '/scannow'] @@ -85,5 +132,15 @@ def run_sfc_scan(): raise GenericError +def windows_is_activated(): + """Check if Windows is activated via slmgr.vbs and return bool.""" + cmd = ['cscript', '//nologo', SLMGR, '/xpr'] + result = run_program(cmd, check=False) + act_str = result.stdout + + # Check result. + return bool(act_str and 'permanent' in act_str) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From f27f3024e85fbee0c777d11a689e2e38a9115ad8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Sep 2019 22:37:18 -0700 Subject: [PATCH 097/324] Added get_json_from_command() --- scripts/wk/exe.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index 6dfd978e..758b4ae1 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -1,6 +1,7 @@ """WizardKit: Execution functions""" #vim: sts=2 sw=2 ts=2 +import json import logging import re import subprocess @@ -92,6 +93,24 @@ def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): return cmd_kwargs +def get_json_from_command(cmd, check=True, encoding='utf-8', errors='ignore'): + """Capture JSON content from cmd output, returns dict. + + If the data can't be decoded then either an exception is raised + or an empty dict is returned depending on ignore_errors. + """ + json_data = {} + + try: + result = run_program(cmd, check=check, encoding=encoding, errors=errors) + json_data = json.loads(result.stdout) + except (subprocess.CalledProcessError, json.decoder.JSONDecodeError): + if errors != 'ignore': + raise + + return json_data + + def get_procs(name, exact=True): """Get process object(s) based on name, returns list of proc objects.""" LOG.debug('name: %s, exact: %s', name, exact) From 1cfd8fb7b4ade61eb2d3e9ef43dab1b661e11901 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Sep 2019 18:43:21 -0700 Subject: [PATCH 098/324] Added outer activation script --- scripts/activate.py | 31 ++++++++++ scripts/outer_scripts_to_review/activate.py | 63 --------------------- scripts/wk/io.py | 2 +- scripts/wk/os/__init__.py | 6 +- scripts/wk/os/win.py | 26 +++++---- 5 files changed, 50 insertions(+), 78 deletions(-) create mode 100644 scripts/activate.py delete mode 100644 scripts/outer_scripts_to_review/activate.py diff --git a/scripts/activate.py b/scripts/activate.py new file mode 100644 index 00000000..0affc19a --- /dev/null +++ b/scripts/activate.py @@ -0,0 +1,31 @@ +"""Wizard Kit: Activate Windows using a BIOS key""" +# vim: sts=2 sw=2 ts=2 + +import wk + + +def main(): + """Attempt to activate Windows and show result.""" + title = f'{wk.cfg.main.KIT_NAME_FULL}: Activation Tool' + try_print = wk.std.TryAndPrint() + wk.std.clear_screen() + wk.std.set_title(title) + wk.std.print_info(title) + print('') + + # Attempt activation + try_print.run('Attempting activation...', wk.os.win.activate_with_bios) + + # Done + print('') + print('Done.') + wk.std.pause('Press Enter to exit...') + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/outer_scripts_to_review/activate.py b/scripts/outer_scripts_to_review/activate.py deleted file mode 100644 index fa54fa5d..00000000 --- a/scripts/outer_scripts_to_review/activate.py +++ /dev/null @@ -1,63 +0,0 @@ -# Wizard Kit: Activate Windows using various methods - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.activation import * -init_global_vars() -os.system('title {}: Windows Activation Tool'.format(KIT_NAME_FULL)) - -if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: Windows Activation Tool\n'.format(KIT_NAME_FULL)) - # Bail early if already activated - if windows_is_activated(): - print_info('This system is already activated') - sleep(5) - exit_script() - other_results = { - 'Error': { - 'BIOSKeyNotFoundError': 'BIOS key not found.', - }} - - # Determine activation method - activation_methods = [ - {'Name': 'Activate with BIOS key', 'Function': activate_with_bios}, - ] - if global_vars['OS']['Version'] not in ('8', '8.1', '10'): - activation_methods[0]['Disabled'] = True - actions = [ - {'Name': 'Quit', 'Letter': 'Q'}, - ] - - while True: - selection = menu_select( - '{}: Windows Activation Menu'.format(KIT_NAME_FULL), - main_entries=activation_methods, action_entries=actions) - - if (selection.isnumeric()): - result = try_and_print( - message = activation_methods[int(selection)-1]['Name'], - function = activation_methods[int(selection)-1]['Function'], - other_results=other_results) - if result['CS']: - break - else: - sleep(2) - elif selection == 'Q': - exit_script() - - # Done - print_success('\nDone.') - pause("Press Enter to exit...") - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/scripts/wk/io.py b/scripts/wk/io.py index fa6110c8..cba8b3fa 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -63,7 +63,7 @@ def delete_item(path, force=False, ignore_errors=False): os.remove(path) -def non_clobbering_path(path): +def non_clobber_path(path): """Update path as needed to non-existing path, returns pathlib.Path.""" LOG.debug('path: %s', path) path = pathlib.Path(path) diff --git a/scripts/wk/os/__init__.py b/scripts/wk/os/__init__.py index 5e58dbc8..e3d66798 100644 --- a/scripts/wk/os/__init__.py +++ b/scripts/wk/os/__init__.py @@ -1,7 +1,9 @@ """WizardKit: os module init""" # vim: sts=2 sw=2 ts=2 -import os +import platform -if os.name == 'nt': +#if platform.system() == 'Darwin': +#if platform.system() == 'Linux': +if platform.system() == 'Windows': from wk.os import win diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index 8e00ec1a..b99a1e13 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -17,7 +17,7 @@ from wk.std import GenericError, GenericWarning, sleep # STATIC VARIABLES LOG = logging.getLogger(__name__) REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' -SLMGR = pathlib.Path(f'{os.environ("SYSTEMROOT")}/System32/slmgr.vbs') +SLMGR = pathlib.Path(f'{os.environ.get("SYSTEMROOT")}/System32/slmgr.vbs') # Functions @@ -41,6 +41,10 @@ def activate_with_bios(): if not bios_key: raise GenericError('BIOS key not found.') + # Check if activation is needed + if is_activated(): + raise GenericWarning('System already activated') + # Install Key cmd = ['cscript', '//nologo', SLMGR, '/ipk', bios_key] run_program(cmd, check=False) @@ -52,7 +56,7 @@ def activate_with_bios(): sleep(5) # Check status - if not windows_is_activated(): + if not is_activated(): raise GenericError('Activation Failed') @@ -96,6 +100,14 @@ def get_activation_string(): return act_str +def is_activated(): + """Check if Windows is activated via slmgr.vbs and return bool.""" + act_str = get_activation_string() + + # Check result. + return act_str and 'permanent' in act_str + + def run_sfc_scan(): """Run SFC and save results.""" cmd = ['sfc', '/scannow'] @@ -132,15 +144,5 @@ def run_sfc_scan(): raise GenericError -def windows_is_activated(): - """Check if Windows is activated via slmgr.vbs and return bool.""" - cmd = ['cscript', '//nologo', SLMGR, '/xpr'] - result = run_program(cmd, check=False) - act_str = result.stdout - - # Check result. - return bool(act_str and 'permanent' in act_str) - - if __name__ == '__main__': print("This file is not meant to be called directly.") From 9ee664bc2bbef5e31c1e50e395a5a6e21a52a325 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Sep 2019 21:26:48 -0700 Subject: [PATCH 099/324] Dropping CBS fix sections --- scripts/outer_scripts_to_review/cbs_fix.py | 43 ---------------------- 1 file changed, 43 deletions(-) delete mode 100644 scripts/outer_scripts_to_review/cbs_fix.py diff --git a/scripts/outer_scripts_to_review/cbs_fix.py b/scripts/outer_scripts_to_review/cbs_fix.py deleted file mode 100644 index 167f95aa..00000000 --- a/scripts/outer_scripts_to_review/cbs_fix.py +++ /dev/null @@ -1,43 +0,0 @@ -# Wizard Kit: Backup CBS Logs and prep CBS temp data for deletion - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.cleanup import * -from functions.data import * -init_global_vars() -os.system('title {}: CBS Cleanup'.format(KIT_NAME_FULL)) -set_log_file('CBS Cleanup.log') - -if __name__ == '__main__': - try: - # Prep - stay_awake() - clear_screen() - folder_path = r'{}\Backups'.format(KIT_NAME_SHORT) - dest = select_destination(folder_path=folder_path, - prompt='Which disk are we using for temp data and backup?') - - # Show details - print_info('{}: CBS Cleanup Tool\n'.format(KIT_NAME_FULL)) - show_data('Backup / Temp path:', dest) - print_standard('\n') - if (not ask('Proceed with CBS cleanup?')): - abort() - - # Run Cleanup - try_and_print(message='Running cleanup...', function=cleanup_cbs, - cs='Done', dest_folder=dest) - - # Done - print_standard('\nDone.') - pause("Press Enter to exit...") - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 From 972cb6fb66db109a7dcd542330f1242e48817842 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Sep 2019 21:27:28 -0700 Subject: [PATCH 100/324] Breaking updates to TryAndPrint() * Removed print_return argument * Instead if the function returns data assume it should be printed * Added ability to override msg_good for a single run() call --- scripts/wk/std.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 42f29608..a2b16659 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -482,7 +482,7 @@ class TryAndPrint(): def run( self, message, function, *args, - catch_all=True, print_return=False, verbose=False, **kwargs): + catch_all=True, msg_good=None, verbose=False, **kwargs): # pylint: disable=catching-non-exception """Run a function and print the results, returns results as dict. @@ -490,9 +490,11 @@ class TryAndPrint(): Otherwise if an exception occurs that wasn't specified it will be re-raised. - If print_return is True then the output from the function will be used - instead of msg_good, msg_bad, or exception text. The output should be - a list or a subprocess.CompletedProcess object. + If the function returns data it will be used instead of msg_good, + msg_bad, or exception text. + The output should be a list or a subprocess.CompletedProcess object. + + If msg_good is passed it will override self.msg_good for this call. If verbose is True then exception names or messages will be used for the result message. Otherwise it will simply be set to result_bad. @@ -503,9 +505,9 @@ class TryAndPrint(): LOG.debug('args: %s', args) LOG.debug('kwargs: %s', kwargs) LOG.debug( - 'catch_all: %s, print_return: %s, verbose: %s', + 'catch_all: %s, msg_good: %s, verbose: %s', catch_all, - print_return, + msg_good, verbose, ) f_exception = None @@ -544,11 +546,11 @@ class TryAndPrint(): raise else: # Success - if print_return: + if output: result_msg = self._format_function_output(output) print(result_msg) else: - print_success(self.msg_good, log=False) + print_success(msg_good if msg_good else self.msg_good, log=False) # Done self._log_result(message, result_msg) From 2ea0b4818ab8dcb1b480d156eaac9f58f1317d92 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Sep 2019 21:50:08 -0700 Subject: [PATCH 101/324] Updated run_sfc_scan() * Output is UTF-16, decode it as such * Simplifies section * Reworked checking the result * Use separate exceptions for corruption and general errors --- scripts/wk/os/win.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index b99a1e13..6cd72e46 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -4,7 +4,6 @@ import logging import os import pathlib -import re import time from wk import cfg @@ -120,28 +119,28 @@ def run_sfc_scan(): err_path = log_path.with_suffix('.err') # Run SFC - proc = run_program(cmd, check=False) + proc = run_program(cmd, check=False, encoding='utf-16') # Fix paths log_path = non_clobber_path(log_path) err_path = non_clobber_path(err_path) # Save output - output = proc.stdout.replace('\0', '') - errors = proc.stderr.replace('\0', '') os.makedirs(log_path.parent, exist_ok=True) with open(log_path, 'w') as _f: - _f.write(output) + _f.write(proc.stdout) with open(err_path, 'w') as _f: - _f.write(errors) + _f.write(proc.stderr) # Check result - if re.findall(r'did\s+not\s+find\s+any\s+integrity\s+violations', output): + if 'did not find any integrity violations' in proc.stdout: pass - elif re.findall(r'successfully\s+repaired\s+them', output): + elif 'successfully repaired' in proc.stdout: raise GenericWarning('Repaired') + elif 'found corrupt files' in proc.stdout: + raise GenericError('Corruption detected') else: - raise GenericError + raise OSError if __name__ == '__main__': From 60969f26ebacc657554e921b54dfde7a9436c329 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Sep 2019 23:27:41 -0700 Subject: [PATCH 102/324] Reworked setting log paths * Added DEFAULT_LOG_DIR and DEFAULT_LOG_NAME vars * Allows easier reuse of default values * Added format_log_path() * Uses default path/name unless dir/name specified * Added get_root_logger_path() * Returns the first fileHandler path found (if any) * update_log_path() now supports multiple handler scenarios --- scripts/wk/log.py | 108 +++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index bdba18df..5fe50ce4 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -9,6 +9,22 @@ import shutil import time from wk import cfg +from wk.io import non_clobber_path + +# STATIC VARIABLES +if os.name == 'nt': + # Example: "C:\WK\1955-11-05\WizardKit" + DEFAULT_LOG_DIR = ( + f'{os.environ.get("SYSTEMDRIVE", "C:")}/' + f'{cfg.main.KIT_NAME_SHORT}/' + f'{time.strftime("%Y-%m-%d")}/' + f'{cfg.main.KIT_NAME_FULL}' + ) + DEFAULT_LOG_NAME = '' +else: + # Example: "/home/tech/Logs" + DEFAULT_LOG_DIR = f'{os.path.expanduser("~")}/Logs' + DEFAULT_LOG_NAME = cfg.main.KIT_NAME_FULL # Functions @@ -24,23 +40,43 @@ def enable_debug_mode(): root_logger.setLevel('DEBUG') +def format_log_path(log_dir=None, log_name=None, tool=False, timestamp=True): + """Format path based on args passed, returns pathlib.Path obj.""" + log_path = pathlib.Path( + f'{log_dir if log_dir else DEFAULT_LOG_DIR}/' + f'{"Tools/" if tool else ""}' + f'{log_name if log_name else DEFAULT_LOG_NAME}' + f'{"_" if timestamp else ""}' + f'{time.strftime("%Y-%m-%d_%H%M%S%z") if timestamp else ""}' + '.log' + ) + log_path = log_path.resolve() + + # Avoid clobbering + log_path = non_clobber_path(log_path) + + # Done + return log_path + + +def get_root_logger_path(): + """Get path to log file from root logger, returns pathlib.Path obj.""" + log_path = None + root_logger = logging.getLogger() + + # Check all handlers and use the first fileHandler found + for handler in root_logger.handlers: + if isinstance(handler, logging.FileHandler): + log_path = pathlib.Path(handler.baseFilename).resolve() + break + + # Done + return log_path + + def start(config=None): """Configure and start logging using safe defaults.""" - if os.name == 'nt': - log_path = '{drive}/{short}/Logs/{date}/{full}/{datetime}.log'.format( - drive=os.environ.get('SYSTEMDRIVE', 'C:'), - short=cfg.main.KIT_NAME_SHORT, - date=time.strftime('%Y-%m-%d'), - full=cfg.main.KIT_NAME_FULL, - datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), - ) - else: - log_path = '{home}/Logs/{full}_{datetime}.log'.format( - home=os.path.expanduser('~'), - full=cfg.main.KIT_NAME_FULL, - datetime=time.strftime('%Y-%m-%d_%H%M%S%z'), - ) - log_path = pathlib.Path(log_path).resolve() + log_path = format_log_path() root_logger = logging.getLogger() # Safety checks @@ -59,38 +95,20 @@ def start(config=None): atexit.register(logging.shutdown) -def update_log_path(dest_dir=None, dest_name=None): - """Moves current log file to new path and updates the root logger. - - NOTE: A timestamp and extension will be added to dest_name if provided. - """ +def update_log_path(dest_dir=None, dest_name=None, timestamp=True): + """Moves current log file to new path and updates the root logger.""" root_logger = logging.getLogger() - cur_handler = root_logger.handlers[0] - cur_path = pathlib.Path(cur_handler.baseFilename).resolve() + cur_handler = None + cur_path = get_root_logger_path() + new_path = format_log_path(dest_dir, dest_name, timestamp=timestamp) - # Safety checks - if not (dest_dir or dest_name): - raise RuntimeError('Neither a directory nor name specified') - if len(root_logger.handlers) > 1: - raise RuntimeError('Multiple handlers not supported') - if not isinstance(cur_handler, logging.FileHandler): - raise RuntimeError('Only FileHandlers are supported') - - # Update dir if specified or use current path - if dest_dir: - new_path = pathlib.Path(dest_dir).resolve() - else: - new_path = cur_path - - # Update name if specified - if dest_name: - new_path = new_path.with_name( - f'{dest_name}' - f'_{time.strftime("%Y-%m-%d_%H%M%S%z")}' - f'{"".join(cur_path.suffixes)}' - ) - else: - new_path = new_path.with_name(cur_path.name) + # Get current logging file handler + for handler in root_logger.handlers: + if isinstance(handler, logging.FileHandler): + cur_handler = handler + break + if not cur_handler: + raise RuntimeError('Logging FileHandler not found') # Copy original log to new location if new_path.exists(): From ff1044a401512ecffe0869b340253da4c41801ee Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Sep 2019 23:31:15 -0700 Subject: [PATCH 103/324] Fixed logic error in non_clobber_path() * Before the '_1000' path would be returned incorrectly * If a non-existant path wasn't found that is --- scripts/wk/io.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/wk/io.py b/scripts/wk/io.py index cba8b3fa..d7841fda 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -78,8 +78,9 @@ def non_clobber_path(path): # Find non-existant path for _i in range(1000): - new_path = path.with_name(f'{name}_{_i}').with_suffix(suffix) - if not new_path.exists(): + test_path = path.with_name(f'{name}_{_i}').with_suffix(suffix) + if not test_path.exists(): + new_path = test_path break # Raise error if viable path not found From 4c35d7cb4e63a9af4b6dc92cd205489b65f925ba Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Sep 2019 23:34:02 -0700 Subject: [PATCH 104/324] Added CHKDSK functions --- scripts/wk/os/win.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index 6cd72e46..0bbb0bb4 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -4,6 +4,7 @@ import logging import os import pathlib +import platform import time from wk import cfg @@ -15,6 +16,7 @@ from wk.std import GenericError, GenericWarning, sleep # STATIC VARIABLES LOG = logging.getLogger(__name__) +OS_VERSION = float(platform.win32_ver()[0]) # TODO: Check if Win8.1 returns '8' REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' SLMGR = pathlib.Path(f'{os.environ.get("SYSTEMROOT")}/System32/slmgr.vbs') @@ -107,6 +109,26 @@ def is_activated(): return act_str and 'permanent' in act_str +def run_chkdsk_offline(): + """Set filesystem 'dirty bit' to force a CHKDSK during startup.""" + cmd = f'fsutil dirty set {os.environ.get("SYSTEMDRIVE")}' + proc = run_program(cmd.split(), check=False) + + # Check result + if proc.returncode > 0: + raise GenericError('Failed to set dirty bit.') + + +def run_chkdsk_online(): + """Run CHKDSK in a split window. + + NOTE: If run on Windows 8+ online repairs are attempted. + """ + cmd = ['CHKDSK', os.environ.get('SYSTEMDRIVE', 'C:')] + if OS_VERSION >= 8: + cmd.extend(['/scan', '/perf']) + + def run_sfc_scan(): """Run SFC and save results.""" cmd = ['sfc', '/scannow'] From 1ffabd864224c9b024f0d4b320379823c8bf0a4e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 23 Sep 2019 00:01:54 -0700 Subject: [PATCH 105/324] Updated wk.os.win --- scripts/wk/os/win.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index 0bbb0bb4..4c4dfa7f 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -11,6 +11,7 @@ from wk import cfg from wk.borrowed import acpi from wk.exe import run_program from wk.io import non_clobber_path +from wk.log import format_log_path from wk.std import GenericError, GenericWarning, sleep @@ -127,17 +128,30 @@ def run_chkdsk_online(): cmd = ['CHKDSK', os.environ.get('SYSTEMDRIVE', 'C:')] if OS_VERSION >= 8: cmd.extend(['/scan', '/perf']) + log_path = format_log_path(log_name='CHKDSK', timestamp=False, tool=True) + err_path = log_path.with_suffix('.err') + + # Run scan + proc = run_program(cmd, check=False) + + # Check result + if proc.returncode == 1: + raise GenericWarning('Repaired (or manually aborted)') + elif proc.returncode > 1: + raise GenericError('Issue(s) detected') + + # Save output + os.makedirs(log_path.parent, exist_ok=True) + with open(log_path, 'w') as _f: + _f.write(proc.stdout) + with open(err_path, 'w') as _f: + _f.write(proc.stderr) def run_sfc_scan(): """Run SFC and save results.""" cmd = ['sfc', '/scannow'] - log_path = pathlib.Path( - '{drive}/{short}/Logs/{date}/Tools/SFC.log'.format( - drive=os.environ.get('SYSTEMDRIVE', 'C:'), - short=cfg.main.KIT_NAME_SHORT, - date=time.strftime('%Y-%m-%d'), - )) + log_path = format_log_path(log_name='SFC', timestamp=False, tool=True) err_path = log_path.with_suffix('.err') # Run SFC From ad06fab8a27d36222eb8db987733941551e15a32 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Oct 2019 21:34:07 -0700 Subject: [PATCH 106/324] Updated log path options * Adjusted default log path * Support Windows Kit/Tool/General log paths --- scripts/wk/log.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 5fe50ce4..f1ae4ec1 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -17,14 +17,12 @@ if os.name == 'nt': DEFAULT_LOG_DIR = ( f'{os.environ.get("SYSTEMDRIVE", "C:")}/' f'{cfg.main.KIT_NAME_SHORT}/' - f'{time.strftime("%Y-%m-%d")}/' - f'{cfg.main.KIT_NAME_FULL}' + f'{time.strftime("%Y-%m-%d")}' ) - DEFAULT_LOG_NAME = '' else: # Example: "/home/tech/Logs" DEFAULT_LOG_DIR = f'{os.path.expanduser("~")}/Logs' - DEFAULT_LOG_NAME = cfg.main.KIT_NAME_FULL +DEFAULT_LOG_NAME = cfg.main.KIT_NAME_FULL # Functions @@ -40,10 +38,13 @@ def enable_debug_mode(): root_logger.setLevel('DEBUG') -def format_log_path(log_dir=None, log_name=None, tool=False, timestamp=True): +def format_log_path( + log_dir=None, log_name=None, timestamp=False, + kit=False, tool=False): """Format path based on args passed, returns pathlib.Path obj.""" log_path = pathlib.Path( f'{log_dir if log_dir else DEFAULT_LOG_DIR}/' + f'{cfg.main.KIT_NAME_FULL+"/" if kit else ""}' f'{"Tools/" if tool else ""}' f'{log_name if log_name else DEFAULT_LOG_NAME}' f'{"_" if timestamp else ""}' @@ -76,7 +77,7 @@ def get_root_logger_path(): def start(config=None): """Configure and start logging using safe defaults.""" - log_path = format_log_path() + log_path = format_log_path(timestamp=os.name != 'nt') root_logger = logging.getLogger() # Safety checks From a2017fa4150e81b8006aa6e3a393231a0a544421 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Oct 2019 21:40:56 -0700 Subject: [PATCH 107/324] Added check_disk.py --- scripts/check_disk.py | 49 +++++++++++++++++++++++++++++++++++++++++++ scripts/wk/os/win.py | 8 +++---- 2 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 scripts/check_disk.py diff --git a/scripts/check_disk.py b/scripts/check_disk.py new file mode 100644 index 00000000..01126124 --- /dev/null +++ b/scripts/check_disk.py @@ -0,0 +1,49 @@ +"""Wizard Kit: Check or repair the %SYSTEMDRIVE% filesystem via CHKDSK""" +# vim: sts=2 sw=2 ts=2 + +import os +import wk + + +def main(): + """Run or schedule CHKDSK and show result.""" + menu = wk.std.Menu(title=title) + title = f'{wk.cfg.main.KIT_NAME_FULL}: Check Disk Tool' + try_print = wk.std.TryAndPrint() + wk.std.clear_screen() + wk.std.set_title(title) + print('') + + # Add menu entries + menu.add_option('Offline scan') + menu.add_option('Online scan') + + # Show menu and make selection + selection = menu.simple_select() + + # Run or schedule scan + if 'Offline' in selection[0]: + function = wk.os.win.run_chkdsk_offline + msg_good = 'Scheduled' + else: + function = wk.os.win.run_chkdsk_online + msg_good = 'No issues detected' + try_print.run(f'CHKDSK ( + message={os.environ.get("SYSTEMDRIVE})...', + function=function, + msg_good=msg_good, + ) + + # Done + print('') + print('Done.') + wk.std.pause('Press Enter to exit...') + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index 4c4dfa7f..df341d75 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -5,9 +5,7 @@ import logging import os import pathlib import platform -import time -from wk import cfg from wk.borrowed import acpi from wk.exe import run_program from wk.io import non_clobber_path @@ -128,7 +126,7 @@ def run_chkdsk_online(): cmd = ['CHKDSK', os.environ.get('SYSTEMDRIVE', 'C:')] if OS_VERSION >= 8: cmd.extend(['/scan', '/perf']) - log_path = format_log_path(log_name='CHKDSK', timestamp=False, tool=True) + log_path = format_log_path(log_name='CHKDSK', tool=True) err_path = log_path.with_suffix('.err') # Run scan @@ -137,7 +135,7 @@ def run_chkdsk_online(): # Check result if proc.returncode == 1: raise GenericWarning('Repaired (or manually aborted)') - elif proc.returncode > 1: + if proc.returncode > 1: raise GenericError('Issue(s) detected') # Save output @@ -151,7 +149,7 @@ def run_chkdsk_online(): def run_sfc_scan(): """Run SFC and save results.""" cmd = ['sfc', '/scannow'] - log_path = format_log_path(log_name='SFC', timestamp=False, tool=True) + log_path = format_log_path(log_name='SFC', tool=True) err_path = log_path.with_suffix('.err') # Run SFC From e80a63ee51964007fafde12d0d27e85ea764856c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Oct 2019 22:25:54 -0700 Subject: [PATCH 108/324] Added show_data() --- scripts/wk/std.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index a2b16659..77b1ceab 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -877,6 +877,15 @@ def set_title(title): print_error('Setting the title is only supported under Windows.') +def show_data(message, data, color=None): + """Display info using standard WIDTH and INDENT.""" + colors = (None, color if color else None) + print_colored( + (f'{" "*INDENT}{message:<{WIDTH}}', data), + colors, + ) + + def sleep(seconds=2): """Simple wrapper for time.sleep.""" time.sleep(seconds) From 386299ce57be9468e1afbefb3abd5771043b835e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Oct 2019 22:26:10 -0700 Subject: [PATCH 109/324] Added network functions --- scripts/wk/net.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/scripts/wk/net.py b/scripts/wk/net.py index e69de29b..ba4d869e 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -0,0 +1,52 @@ +"""WizardKit: Net Functions""" +# vim: sts=2 sw=2 ts=2 + +import re + +import psutil + +from wk.exe import run_program +from wk.std import show_data + +# REGEX +REGEX_VALID_IP = re.compile( + r'(10.\d+.\d+.\d+' + r'|172.(1[6-9]|2\d|3[0-1])' + r'|192.168.\d+.\d+)', + re.IGNORECASE) + + +# Functions +def is_connected(): + """Check for a valid private IP.""" + devs = psutil.net_if_addrs() + for dev in devs.values(): + for family in dev: + if REGEX_VALID_IP.search(family.address): + # Valid IP found + return True + # Else + return False + +def show_valid_addresses(): + """Show all valid private IP addresses assigned to the system.""" + devs = psutil.net_if_addrs() + for dev, families in sorted(devs.items()): + for family in families: + if REGEX_VALID_IP.search(family.address): + # Valid IP found + show_data(message=dev, data=family.address) + + +def speedtest(): + """Run a network speedtest using speedtest-cli.""" + cmd = ['speedtest-cli', '--simple'] + proc = run_program(cmd, check=False) + output = [line.strip() for line in proc.stdout.splitlines()] + output = [line.split() for line in output] + output = [(a, float(b), c) for a, b, c in output] + return [f'{a:<10}{b:6.2f} {c}' for a, b, c in output] + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From cc483abd291972c764919c53f5d009a970a96339 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Oct 2019 22:31:54 -0700 Subject: [PATCH 110/324] Added ping() Added ping() Added ping() Added ping() Added ping() Added ping() Added ping() Added ping() Added ping() --- scripts/wk/net.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/wk/net.py b/scripts/wk/net.py index ba4d869e..ceed1cd3 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -28,6 +28,18 @@ def is_connected(): # Else return False + +def ping(addr='google.com'): + """Attempt to ping addr.""" + cmd = ( + 'ping', + '-n' if psutil.WINDOWS else '-c', + '2', + addr, + ) + run_program(cmd) + + def show_valid_addresses(): """Show all valid private IP addresses assigned to the system.""" devs = psutil.net_if_addrs() From 187a29ff6f56a53eb39be0ef3b915bdc5677cbca Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Oct 2019 22:44:36 -0700 Subject: [PATCH 111/324] Bugfix speedtest() --- scripts/wk/net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/net.py b/scripts/wk/net.py index ceed1cd3..509e708d 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -54,7 +54,7 @@ def speedtest(): """Run a network speedtest using speedtest-cli.""" cmd = ['speedtest-cli', '--simple'] proc = run_program(cmd, check=False) - output = [line.strip() for line in proc.stdout.splitlines()] + output = [line.strip() for line in proc.stdout.splitlines() if line.strip()] output = [line.split() for line in output] output = [(a, float(b), c) for a, b, c in output] return [f'{a:<10}{b:6.2f} {c}' for a, b, c in output] From 89f62562f06e2e292a1f81534e4f0c7c6b94e560 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Oct 2019 22:58:24 -0700 Subject: [PATCH 112/324] Removed reference sections from wk.prev --- scripts/wk.prev/functions/common.py | 520 --------------------------- scripts/wk.prev/functions/network.py | 53 --- 2 files changed, 573 deletions(-) delete mode 100644 scripts/wk.prev/functions/network.py diff --git a/scripts/wk.prev/functions/common.py b/scripts/wk.prev/functions/common.py index 689cc85f..f2e019d3 100644 --- a/scripts/wk.prev/functions/common.py +++ b/scripts/wk.prev/functions/common.py @@ -99,103 +99,6 @@ class WindowsUnsupportedError(Exception): # General functions -def abort(show_prompt=True): - """Abort script.""" - print_warning('Aborted.') - if show_prompt: - sleep(1) - pause(prompt='Press Enter to exit... ') - exit_script(1) - - -def ask(prompt='Kotaero!'): - """Prompt the user with a Y/N question, returns bool.""" - answer = None - prompt = '{} [Y/N]: '.format(prompt) - while answer is None: - tmp = input(prompt) - if re.search(r'^y(es|)$', tmp, re.IGNORECASE): - answer = True - elif re.search(r'^n(o|ope|)$', tmp, re.IGNORECASE): - answer = False - message = '{prompt}{answer_text}'.format( - prompt = prompt, - answer_text = 'Yes' if answer else 'No') - print_log(message=message) - return answer - - -def beep(repeat=1): - """Play system bell with optional repeat.""" - for i in range(repeat): - # Print bell char - print('\a') - sleep(0.5) - - -def choice(choices, prompt='Kotaero!'): - """Prompt the user with a choice question, returns str.""" - answer = None - choices = [str(c) for c in choices] - choices_short = {c[:1].upper(): c for c in choices} - prompt = '{} [{}]: '.format(prompt, '/'.join(choices)) - regex = '^({}|{})$'.format( - '|'.join([c[:1] for c in choices]), - '|'.join(choices)) - - # Get user's choice - while answer is None: - tmp = input(prompt) - if re.search(regex, tmp, re.IGNORECASE): - answer = tmp - - # Log result - message = '{prompt}{answer_text}'.format( - prompt = prompt, - answer_text = 'Yes' if answer else 'No') - print_log(message=message) - - # Fix answer formatting to match provided values - answer = choices_short[answer[:1].upper()] - - # Done - return answer - - -def clear_screen(): - """Simple wrapper for cls/clear.""" - if psutil.WINDOWS: - os.system('cls') - else: - os.system('clear') - - -def convert_to_bytes(size): - """Convert human-readable size str to bytes and return an int.""" - size = str(size) - tmp = re.search(r'(\d+\.?\d*)\s+([PTGMKB])B?', size.upper()) - if tmp: - size = float(tmp.group(1)) - units = tmp.group(2) - if units == 'P': - size *= 1024 ** 5 - if units == 'T': - size *= 1024 ** 4 - elif units == 'G': - size *= 1024 ** 3 - elif units == 'M': - size *= 1024 ** 2 - elif units == 'K': - size *= 1024 ** 1 - elif units == 'B': - size *= 1024 ** 0 - size = int(size) - else: - return -1 - - return size - - def exit_script(return_value=0): """Exits the script after some cleanup and opens the log (if set).""" # Remove dirs (if empty) @@ -260,16 +163,6 @@ def get_process(name=None): return proc -def get_simple_string(prompt='Enter string'): - """Get string from user (restricted character set), returns str.""" - simple_string = None - while simple_string is None: - _input = input('{}: '.format(prompt)) - if re.match(r"^(\w|-| |\.|')+$", _input, re.ASCII): - simple_string = _input.strip() - return simple_string - - def get_ticket_number(): """Get TicketNumber from user, save in LogDir, and return as str.""" if not ENABLED_TICKET_NUMBERS: @@ -287,50 +180,6 @@ def get_ticket_number(): return ticket_number -def human_readable_size(size, decimals=0): - """Convert size from bytes to a human-readable format, returns str.""" - # Prep string formatting - width = 3+decimals - if decimals > 0: - width += 1 - - # Convert size to int - try: - size = int(size) - except ValueError: - size = convert_to_bytes(size) - except TypeError: - size = -1 - - # Verify we have a valid size - if size < 0: - return '{size:>{width}} b'.format(size='???', width=width) - - # Convert to sensible units - if size >= 1024 ** 5: - size /= 1024 ** 5 - units = 'PB' - elif size >= 1024 ** 4: - size /= 1024 ** 4 - units = 'TB' - elif size >= 1024 ** 3: - size /= 1024 ** 3 - units = 'GB' - elif size >= 1024 ** 2: - size /= 1024 ** 2 - units = 'MB' - elif size >= 1024 ** 1: - size /= 1024 ** 1 - units = 'KB' - else: - size /= 1024 ** 0 - units = ' B' - - # Return - return '{size:>{width}.{decimals}f} {units}'.format( - size=size, width=width, decimals=decimals, units=units) - - def kill_process(name): """Kill any running caffeine.exe processes.""" for proc in psutil.process_iter(): @@ -338,243 +187,6 @@ def kill_process(name): proc.kill() -def major_exception(): - """Display traceback and exit""" - print_error('Major exception') - print_warning(SUPPORT_MESSAGE) - print(traceback.format_exc()) - print_log(traceback.format_exc()) - try: - upload_crash_details() - except GenericAbort: - # User declined upload - print_warning('Upload: Aborted') - sleep(10) - except GenericError: - # No log file or uploading disabled - sleep(10) - except: - print_error('Upload: NS') - sleep(10) - else: - print_success('Upload: CS') - pause('Press Enter to exit...') - exit_script(1) - - -def menu_select( - title='[Untitled Menu]', - prompt='Please make a selection', secret_actions=[], secret_exit=False, - main_entries=[], action_entries=[], disabled_label='DISABLED', - spacer=''): - """Display options in a menu and return selected option as a str.""" - # Bail early - if not main_entries and not action_entries: - raise Exception("MenuError: No items given") - - # Set title - if 'Title' in global_vars: - title = '{}\n\n{}'.format(global_vars['Title'], title) - - # Build menu - menu_splash = '{}\n{}\n'.format(title, spacer) - width = len(str(len(main_entries))) - valid_answers = [] - if secret_exit: - valid_answers.append('Q') - if secret_actions: - valid_answers.extend(secret_actions) - - # Add main entries - for i in range(len(main_entries)): - entry = main_entries[i] - # Add Spacer - if ('CRLF' in entry): - menu_splash += '{}\n'.format(spacer) - entry_str = '{number:>{width}}: {name}'.format( - number = i+1, - width = width, - name = entry.get('Display Name', entry['Name'])) - if entry.get('Disabled', False): - entry_str = '{YELLOW}{entry_str} ({disabled}){CLEAR}'.format( - entry_str = entry_str, - disabled = disabled_label, - **COLORS) - else: - valid_answers.append(str(i+1)) - menu_splash += '{}\n'.format(entry_str) - menu_splash += '{}\n'.format(spacer) - - # Add action entries - for entry in action_entries: - # Add Spacer - if ('CRLF' in entry): - menu_splash += '{}\n'.format(spacer) - valid_answers.append(entry['Letter']) - menu_splash += '{letter:>{width}}: {name}\n'.format( - letter = entry['Letter'].upper(), - width = len(str(len(action_entries))), - name = entry['Name']) - - answer = '' - - while (answer.upper() not in valid_answers): - clear_screen() - print(menu_splash) - answer = input('{}: '.format(prompt)) - - return answer.upper() - - -def non_clobber_rename(full_path): - """Append suffix to path, if necessary, to avoid clobbering path""" - new_path = full_path - _i = 1; - while os.path.exists(new_path): - new_path = '{path}_{i}'.format(i=_i, path=full_path) - _i += 1 - - return new_path - - -def pause(prompt='Press Enter to continue... '): - """Simple pause implementation.""" - if prompt[-1] != ' ': - prompt += ' ' - input(prompt) - - -def ping(addr='google.com'): - """Attempt to ping addr.""" - cmd = [ - 'ping', - '-n' if psutil.WINDOWS else '-c', - '2', - addr] - run_program(cmd) - - -def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): - """Run program and return a subprocess.Popen object.""" - cmd_kwargs = {'args': cmd, 'shell': shell} - for kw in ('encoding', 'errors'): - if kw in kwargs: - cmd_kwargs[kw] = kwargs[kw] - - if minimized: - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = 6 - cmd_kwargs['startupinfo'] = startupinfo - - if pipe: - cmd_kwargs.update({ - 'stdout': subprocess.PIPE, - 'stderr': subprocess.PIPE, - }) - - if 'cwd' in kwargs: - cmd_kwargs['cwd'] = kwargs['cwd'] - - return subprocess.Popen(**cmd_kwargs) - - -def print_error(*args, **kwargs): - """Prints message to screen in RED.""" - print_standard(*args, color=COLORS['RED'], **kwargs) - - -def print_info(*args, **kwargs): - """Prints message to screen in BLUE.""" - print_standard(*args, color=COLORS['BLUE'], **kwargs) - - -def print_standard(message='Generic info', - color=None, end='\n', timestamp=True, **kwargs): - """Prints message to screen and log (if set).""" - display_message = message - if color: - display_message = color + message + COLORS['CLEAR'] - # **COLORS is used below to support non-"standard" color printing - print(display_message.format(**COLORS), end=end, **kwargs) - print_log(message, end, timestamp) - - -def print_success(*args, **kwargs): - """Prints message to screen in GREEN.""" - print_standard(*args, color=COLORS['GREEN'], **kwargs) - - -def print_warning(*args, **kwargs): - """Prints message to screen in YELLOW.""" - print_standard(*args, color=COLORS['YELLOW'], **kwargs) - - -def print_log(message='', end='\n', timestamp=True): - """Writes message to a log if LogFile is set.""" - time_str = time.strftime("%Y-%m-%d %H%M%z: ") if timestamp else '' - if 'LogFile' in global_vars and global_vars['LogFile']: - with open(global_vars['LogFile'], 'a', encoding='utf-8') as f: - for line in message.splitlines(): - f.write('{timestamp}{line}{end}'.format( - timestamp = time_str, - line = line, - end = end)) - - -def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): - """Run program and return a subprocess.CompletedProcess object.""" - cmd = [c for c in cmd if c] - if shell: - cmd = ' '.join(cmd) - - cmd_kwargs = {'args': cmd, 'check': check, 'shell': shell} - for kw in ('encoding', 'errors'): - if kw in kwargs: - cmd_kwargs[kw] = kwargs[kw] - - if pipe: - cmd_kwargs.update({ - 'stdout': subprocess.PIPE, - 'stderr': subprocess.PIPE, - }) - - if 'cwd' in kwargs: - cmd_kwargs['cwd'] = kwargs['cwd'] - - return subprocess.run(**cmd_kwargs) - - -def set_title(title='[Some Title]'): - """Set title. - - Used for window title and menu titles.""" - global_vars['Title'] = title - os.system('title {}'.format(title)) - - -def show_data( - message='[Some message]', data='[Some data]', - indent=8, width=32, - info=False, warning=False, error=False): - """Display info with formatting.""" - message = '{indent}{message:<{width}}{data}'.format( - indent=' '*indent, width=width, message=message, data=data) - if error: - print_error(message) - elif warning: - print_warning(message) - elif info: - print_info(message) - else: - print_standard(message) - - -def sleep(seconds=2): - """Wait for a while.""" - time.sleep(seconds) - - def stay_awake(): """Prevent the system from sleeping or hibernating.""" # DISABLED due to VCR2008 dependency @@ -592,121 +204,6 @@ def stay_awake(): print_warning('Please set the power setting to High Performance.') -def strip_colors(s): - """Remove all ASCII color escapes from string, returns str.""" - for c in COLORS.values(): - s = s.replace(c, '') - return s - - -def get_exception(s): - """Get exception by name, returns Exception object.""" - try: - obj = getattr(sys.modules[__name__], s) - except AttributeError: - # Try builtin classes - obj = getattr(sys.modules['builtins'], s) - return obj - - -def try_and_print(message='Trying...', - function=None, cs='CS', ns='NS', other_results={}, - catch_all=True, print_return=False, silent_function=True, - indent=8, width=32, *args, **kwargs): - """Run function, print if successful or not, and return dict. - - other_results is in the form of - { - 'Warning': {'ExceptionClassName': 'Result Message'}, - 'Error': {'ExceptionClassName': 'Result Message'} - } - The the ExceptionClassNames will be excepted conditions - and the result string will be printed in the correct color. - catch_all=False will re-raise unspecified exceptions.""" - err = None - out = None - w_exceptions = other_results.get('Warning', {}).keys() - w_exceptions = tuple(get_exception(e) for e in w_exceptions) - e_exceptions = other_results.get('Error', {}).keys() - e_exceptions = tuple(get_exception(e) for e in e_exceptions) - w_results = other_results.get('Warning', {}) - e_results = other_results.get('Error', {}) - - # Run function and catch errors - print_standard('{indent}{message:<{width}}'.format( - indent=' '*indent, message=message, width=width), end='', flush=True) - try: - out = function(*args, **kwargs) - if print_return: - str_list = out - if isinstance(out, subprocess.CompletedProcess): - str_list = out.stdout.decode().strip().splitlines() - print_standard(str_list[0].strip(), timestamp=False) - for item in str_list[1:]: - print_standard('{indent}{item}'.format( - indent=' '*(indent+width), item=item.strip())) - elif silent_function: - print_success(cs, timestamp=False) - except w_exceptions as e: - _result = w_results.get(e.__class__.__name__, 'Warning') - print_warning(_result, timestamp=False) - err = e - except e_exceptions as e: - _result = e_results.get(e.__class__.__name__, 'Error') - print_error(_result, timestamp=False) - err = e - except Exception: - print_error(ns, timestamp=False) - err = traceback.format_exc() - - # Return or raise? - if err and not catch_all: - raise - else: - return {'CS': not bool(err), 'Error': err, 'Out': out} - - -def upload_crash_details(): - """Upload log and runtime data to the CRASH_SERVER. - - Intended for uploading to a public Nextcloud share.""" - if not ENABLED_UPLOAD_DATA: - raise GenericError - - import requests - if 'LogFile' in global_vars and global_vars['LogFile']: - if ask('Upload crash details to {}?'.format(CRASH_SERVER['Name'])): - with open(global_vars['LogFile']) as f: - data = '{}\n'.format(f.read()) - data += '#############################\n' - data += 'Runtime Details:\n\n' - data += 'sys.argv: {}\n\n'.format(sys.argv) - try: - data += generate_global_vars_report() - except Exception: - data += 'global_vars: {}\n'.format(global_vars) - filename = global_vars.get('LogFile', 'Unknown') - filename = re.sub(r'.*(\\|/)', '', filename) - filename += '.txt' - url = '{}/Crash_{}__{}'.format( - CRASH_SERVER['Url'], - global_vars.get('Date-Time', 'Unknown Date-Time'), - filename) - r = requests.put( - url, data=data, - headers={'X-Requested-With': 'XMLHttpRequest'}, - auth=(CRASH_SERVER['User'], CRASH_SERVER['Pass'])) - # Raise exception if upload NS - if not r.ok: - raise Exception - else: - # User said no - raise GenericAbort - else: - # No LogFile defined (or invalid LogFile) - raise GenericError - - def wait_for_process(name, poll_rate=3): """Wait for process by name.""" running = True @@ -931,23 +428,6 @@ def set_linux_vars(): } -def set_log_file(log_name): - """Sets global var LogFile and creates path as needed.""" - if psutil.LINUX: - folder_path = global_vars['LogDir'] - else: - folder_path = '{}{}{}'.format( - global_vars['LogDir'], - os.sep, - KIT_NAME_FULL) - log_file = '{}{}{}'.format( - folder_path, - os.sep, - log_name) - os.makedirs(folder_path, exist_ok=True) - global_vars['LogFile'] = log_file - - if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk.prev/functions/network.py b/scripts/wk.prev/functions/network.py deleted file mode 100644 index 5b5d4f52..00000000 --- a/scripts/wk.prev/functions/network.py +++ /dev/null @@ -1,53 +0,0 @@ -# Wizard Kit: Functions - Network - -import os -import shutil -import sys - -from functions.common import * - - -# REGEX -REGEX_VALID_IP = re.compile( - r'(10.\d+.\d+.\d+' - r'|172.(1[6-9]|2\d|3[0-1])' - r'|192.168.\d+.\d+)', - re.IGNORECASE) - - -def is_connected(): - """Check for a valid private IP.""" - devs = psutil.net_if_addrs() - for dev in devs.values(): - for family in dev: - if REGEX_VALID_IP.search(family.address): - # Valid IP found - return True - # Else - return False - - -def show_valid_addresses(): - """Show all valid private IP addresses assigned to the system.""" - devs = psutil.net_if_addrs() - for dev, families in sorted(devs.items()): - for family in families: - if REGEX_VALID_IP.search(family.address): - # Valid IP found - show_data(message=dev, data=family.address) - - -def speedtest(): - """Run a network speedtest using speedtest-cli.""" - result = run_program(['speedtest-cli', '--simple']) - output = [line.strip() for line in result.stdout.decode().splitlines() - if line.strip()] - output = [line.split() for line in output] - output = [(a, float(b), c) for a, b, c in output] - return ['{:10}{:6.2f} {}'.format(*line) for line in output] - - -if __name__ == '__main__': - print("This file is not meant to be called directly.") - -# vim: sts=2 sw=2 ts=2 From 193511d83bb528c91bea0623da3d0ab1383eee07 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 21 Oct 2019 18:51:32 -0700 Subject: [PATCH 113/324] Added color_string(), blink "colors", and more * The list of strings are now joined using ' ' by default * Instead of '' * Added YELLOW_BLINK and RED_BLINK escape codes * print_colored() now optionally logs the msg --- scripts/wk/std.py | 62 ++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 77b1ceab..d7706eab 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -32,14 +32,16 @@ from wk.cfg.main import ( # STATIC VARIABLES COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'ORANGE': '\033[31;1m', - 'GREEN': '\033[32m', - 'YELLOW': '\033[33m', - 'BLUE': '\033[34m', - 'PURPLE': '\033[35m', - 'CYAN': '\033[36m', + 'CLEAR': '\033[0m', + 'RED': '\033[31m', + 'RED_BLINK': '\033[31;5m', + 'ORANGE': '\033[31;1m', + 'GREEN': '\033[32m', + 'YELLOW': '\033[33m', + 'YELLOW_BLINK': '\033[33;5m', + 'BLUE': '\033[34m', + 'PURPLE': '\033[35m', + 'CYAN': '\033[36m', } LOG = logging.getLogger(__name__) REGEX_SIZE_STRING = re.compile( @@ -685,6 +687,26 @@ def clear_screen(): subprocess.run(cmd, check=False, shell=True, stderr=subprocess.PIPE) +def color_string(strings, colors, sep=' '): + """Build colored string using ANSI escapes, returns str.""" + clear_code = COLORS['CLEAR'] + msg = [] + + # Convert to tuples if necessary + if isinstance(strings, str): + strings = (strings,) + if isinstance(colors, str): + colors = (colors,) + + # Build new string with color escapes added + for string, color in itertools.zip_longest(strings, colors): + color_code = COLORS.get(color, clear_code) + msg.append(f'{color_code}{string}{clear_code}') + + # Done + return sep.join(msg) + + def generate_debug_report(): """Generate debug report, returns str.""" import socket @@ -805,26 +827,16 @@ def pause(prompt='Press Enter to continue... '): def print_colored(strings, colors, **kwargs): """Prints strings in the colors specified.""" LOG.debug('strings: %s, colors: %s, kwargs: %s', strings, colors, kwargs) - clear_code = COLORS['CLEAR'] - msg = '' + msg = color_string(strings, colors, sep=kwargs.get('sep', ' ')) print_options = { 'end': kwargs.get('end', '\n'), 'file': kwargs.get('file', sys.stdout), 'flush': kwargs.get('flush', False), } - # Convert to tuples if necessary - if isinstance(strings, str): - strings = (strings,) - if isinstance(colors, str): - colors = (colors,) - - # Build new string with color escapes added - for string, color in itertools.zip_longest(strings, colors): - color_code = COLORS.get(color, clear_code) - msg += f'{color_code}{string}{clear_code}' - print(msg, **print_options) + if kwargs.get('log', False): + LOG.info(strip_colors(msg)) def print_error(msg, log=True, **kwargs): @@ -832,14 +844,14 @@ def print_error(msg, log=True, **kwargs): if 'file' not in kwargs: # Only set if not specified kwargs['file'] = sys.stderr - print_colored([msg], ['RED'], **kwargs) + print_colored(msg, 'RED', **kwargs) if log: LOG.error(msg) def print_info(msg, log=True, **kwargs): """Prints message in BLUE and log as INFO.""" - print_colored([msg], ['BLUE'], **kwargs) + print_colored(msg, 'BLUE', **kwargs) if log: LOG.info(msg) @@ -853,7 +865,7 @@ def print_standard(msg, log=True, **kwargs): def print_success(msg, log=True, **kwargs): """Prints message in GREEN and log as INFO.""" - print_colored([msg], ['GREEN'], **kwargs) + print_colored(msg, 'GREEN', **kwargs) if log: LOG.info(msg) @@ -863,7 +875,7 @@ def print_warning(msg, log=True, **kwargs): if 'file' not in kwargs: # Only set if not specified kwargs['file'] = sys.stderr - print_colored([msg], ['YELLOW'], **kwargs) + print_colored(msg, 'YELLOW', **kwargs) if log: LOG.warning(msg) From 3ecf107c3992157c676d00c833a596228824fb54 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 21 Oct 2019 18:57:31 -0700 Subject: [PATCH 114/324] Adjusted print_colored() kwargs --- scripts/wk/std.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index d7706eab..eeede87e 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -824,10 +824,13 @@ def pause(prompt='Press Enter to continue... '): input_text(prompt) -def print_colored(strings, colors, **kwargs): +def print_colored(strings, colors, log=False, sep=' ', **kwargs): """Prints strings in the colors specified.""" - LOG.debug('strings: %s, colors: %s, kwargs: %s', strings, colors, kwargs) - msg = color_string(strings, colors, sep=kwargs.get('sep', ' ')) + LOG.debug( + 'strings: %s, colors: %s, sep: %s, kwargs: %s', + strings, colors, sep, kwargs, + ) + msg = color_string(strings, colors, sep=sep) print_options = { 'end': kwargs.get('end', '\n'), 'file': kwargs.get('file', sys.stdout), @@ -835,7 +838,7 @@ def print_colored(strings, colors, **kwargs): } print(msg, **print_options) - if kwargs.get('log', False): + if log: LOG.info(strip_colors(msg)) From 2b5254dd0c2c7d6193b4c96b58caeb998f1aeed9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 21 Oct 2019 18:57:53 -0700 Subject: [PATCH 115/324] Bugfix check_disk.py * Now passes pylint --- scripts/check_disk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/check_disk.py b/scripts/check_disk.py index 01126124..7e6810c9 100644 --- a/scripts/check_disk.py +++ b/scripts/check_disk.py @@ -7,8 +7,8 @@ import wk def main(): """Run or schedule CHKDSK and show result.""" - menu = wk.std.Menu(title=title) title = f'{wk.cfg.main.KIT_NAME_FULL}: Check Disk Tool' + menu = wk.std.Menu(title=title) try_print = wk.std.TryAndPrint() wk.std.clear_screen() wk.std.set_title(title) @@ -28,8 +28,8 @@ def main(): else: function = wk.os.win.run_chkdsk_online msg_good = 'No issues detected' - try_print.run(f'CHKDSK ( - message={os.environ.get("SYSTEMDRIVE})...', + try_print.run( + message=f'CHKDSK ({os.environ.get("SYSTEMDRIVE")})...', function=function, msg_good=msg_good, ) From c9b3794f0e327b30b3e3bf2b9e7393000051bf18 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 22 Oct 2019 18:43:04 -0700 Subject: [PATCH 116/324] Renamed some vars for consistency --- scripts/wk/exe.py | 4 ++-- scripts/wk/os/win.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index 758b4ae1..9a747e4c 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -102,8 +102,8 @@ def get_json_from_command(cmd, check=True, encoding='utf-8', errors='ignore'): json_data = {} try: - result = run_program(cmd, check=check, encoding=encoding, errors=errors) - json_data = json.loads(result.stdout) + proc = run_program(cmd, check=check, encoding=encoding, errors=errors) + json_data = json.loads(proc.stdout) except (subprocess.CalledProcessError, json.decoder.JSONDecodeError): if errors != 'ignore': raise diff --git a/scripts/wk/os/win.py b/scripts/wk/os/win.py index df341d75..f367e40f 100644 --- a/scripts/wk/os/win.py +++ b/scripts/wk/os/win.py @@ -93,8 +93,8 @@ def enable_safemode_msi(): def get_activation_string(): """Get activation status, returns str.""" cmd = ['cscript', '//nologo', SLMGR, '/xpr'] - result = run_program(cmd, check=False) - act_str = result.stdout + proc = run_program(cmd, check=False) + act_str = proc.stdout act_str = act_str.splitlines()[1] act_str = act_str.strip() return act_str From 6e557da370856ae9f62857cb7647aea06c10c14b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 22 Oct 2019 18:49:44 -0700 Subject: [PATCH 117/324] Added CpuRAM() object. --- scripts/wk/__init__.py | 1 + scripts/wk/obj.py | 91 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 scripts/wk/obj.py diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 511cf8a7..589cc28b 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -10,6 +10,7 @@ from wk import io from wk import kit from wk import log from wk import net +from wk import obj from wk import os from wk import std from wk import sw diff --git a/scripts/wk/obj.py b/scripts/wk/obj.py new file mode 100644 index 00000000..8507d6f0 --- /dev/null +++ b/scripts/wk/obj.py @@ -0,0 +1,91 @@ +"""WizardKit: Objects.""" +# vim: sts=2 sw=2 ts=2 + +import pathlib + +from collections import OrderedDict + +from wk.exe import get_json_from_command, run_program +from wk.std import bytes_to_string, color_string, string_to_bytes + +# Classes +class CpuRam(): + """Object for tracking CPU & RAM specific data.""" + def __init__(self): + self.lscpu = {} + self.tests = OrderedDict() + self.get_cpu_details() + self.get_ram_details() + self.name = self.lscpu.get('Model name', 'Unknown CPU') + self.description = self.name + + def get_cpu_details(self): + """Get CPU details from lscpu.""" + cmd = ['lscpu', '--json'] + json_data = get_json_from_command(cmd) + for line in json_data.get('lscpu', [{}]): + _field = line.get('field', '').replace(':', '') + _data = line.get('data', '') + if not (_field or _data): + # Skip + continue + self.lscpu[_field] = _data + + def get_ram_details(self): + """Get RAM details from dmidecode.""" + cmd = ['sudo', 'dmidecode', '--type', 'memory'] + manufacturer = 'UNKNOWN' + details = {'Total': 0} + size = 0 + + # Get DMI data + proc = run_program(cmd) + dmi_data = proc.stdout.splitlines() + + # Parse data + for line in dmi_data: + line = line.strip() + if line == 'Memory Device': + # Reset vars + manufacturer = 'UNKNOWN' + size = 0 + elif line.startswith('Size:'): + size = line.replace('Size: ', '') + size = string_to_bytes(size, assume_binary=True) + elif line.startswith('Manufacturer:'): + manufacturer = line.replace('Manufacturer: ', '') + if size <= 0: + # Skip non-populated slots + continue + description = f'{bytes_to_string(size)} {manufacturer}' + details['Total'] += size + if description in details: + details[description] += 1 + else: + details[description] = 1 + + # Save details + self.ram_total = bytes_to_string(details.pop('Total', 0)) + self.ram_dimms = [ + f'{count}x {desc}' for desc, count in sorted(details.items()) + ] + + def generate_cpu_report(self): + """Generate CPU report with data from all tests.""" + report = [] + report.append(color_string('Device', 'BLUE')) + report.append(f' {self.name}') + + # Include RAM details + report.append(color_string('RAM', 'BLUE')) + report.append(f' {self.ram_total} ({", ".join(self.ram_dimms)})') + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From 52d61226a0385d92daef5ba9a3a82edc9d85444d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 23 Oct 2019 15:23:50 -0700 Subject: [PATCH 118/324] Added Disk() obj --- scripts/wk/exe.py | 2 +- scripts/wk/obj.py | 128 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index 9a747e4c..753df2a0 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -97,7 +97,7 @@ def get_json_from_command(cmd, check=True, encoding='utf-8', errors='ignore'): """Capture JSON content from cmd output, returns dict. If the data can't be decoded then either an exception is raised - or an empty dict is returned depending on ignore_errors. + or an empty dict is returned depending on errors. """ json_data = {} diff --git a/scripts/wk/obj.py b/scripts/wk/obj.py index 8507d6f0..46cc1cd8 100644 --- a/scripts/wk/obj.py +++ b/scripts/wk/obj.py @@ -1,13 +1,23 @@ """WizardKit: Objects.""" # vim: sts=2 sw=2 ts=2 +import logging import pathlib +import re from collections import OrderedDict from wk.exe import get_json_from_command, run_program from wk.std import bytes_to_string, color_string, string_to_bytes +# STATIC VARIABLES +KEY_NVME = 'nvme_smart_health_information_log' +KEY_SMART = 'ata_smart_attributes' +LOG = logging.getLogger(__name__) +REGEX_POWER_ON_TIME = re.compile( + r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' + ) + # Classes class CpuRam(): """Object for tracking CPU & RAM specific data.""" @@ -87,5 +97,123 @@ class CpuRam(): return report +class Disk(): + """Object for tracking disk specific data.""" + def __init__(self, path): + self.attributes = {} + self.description = 'UNKNOWN' + self.lsblk = {} + self.nvme_smart_notes = {} + self.path = pathlib.Path(path).resolve() + self.smartctl = {} + self.tests = OrderedDict() + + # Update details + self.get_details() + self.enable_smart() + self.update_smart_details() + + def enable_smart(self): + """Try enabling SMART for this disk.""" + cmd = [ + 'sudo', + 'smartctl', + '--tolerance=permissive', + '--smart=on', + self.path, + ] + run_program(cmd, check=False) + + def get_details(self): + """Get details via lsblk. + + Required details default to generic descriptions and + are converted to the correct type. + """ + cmd = ['lsblk', '--bytes', '--json', '--output-all', '--paths', self.path] + json_data = get_json_from_command(cmd) + self.lsblk = json_data.get('blockdevices', [{}])[0] + + # Set necessary details + self.lsblk['model'] = self.lsblk.get('model', 'Unknown Model') + self.lsblk['name'] = self.lsblk.get('name', self.path) + self.lsblk['log-sec'] = self.lsblk.get('log-sec', 512) + self.lsblk['phy-sec'] = self.lsblk.get('phy-sec', 512) + self.lsblk['rota'] = self.lsblk.get('rota', True) + self.lsblk['serial'] = self.lsblk.get('serial', 'Unknown Serial') + self.lsblk['size'] = self.lsblk.get('size', -1) + self.lsblk['tran'] = self.lsblk.get('tran', '???') + self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') + + # Ensure certain attributes types + for attr in ['model', 'name', 'serial', 'tran']: + if not isinstance(self.lsblk[attr], str): + self.lsblk[attr] = str(self.lsblk[attr]) + for attr in ['log-sec', 'phy-sec', 'size']: + if not isinstance(self.lsblk[attr], int): + self.lsblk[attr] = int(self.lsblk[attr]) + + def get_labels(self): + """Build list of labels for this disk, returns list.""" + labels = [] + + # Add all labels from lsblk + for disk in [self.lsblk, *self.lsblk.get('children', [])]: + labels.append(disk.get('label', '')) + labels.append(disk.get('partlabel', '')) + + # Remove empty labels + labels = [str(label) for label in labels if label] + + # Done + return labels + + def update_smart_details(self): + """Update SMART details via smartctl.""" + self.attributes = {} + cmd = [ + 'sudo', + 'smartctl', + '--tolerance=verypermissive', + '--all', + '--json', + self.path, + ] + self.smartctl = get_json_from_command(cmd) + + # Check for attributes + if KEY_NVME in self.smartctl: + for name, value in self.smartctl[KEY_NVME].items(): + try: + self.attributes[name] = { + 'name': name, + 'raw': int(value), + 'raw_str': str(value), + } + except ValueError: + # Ignoring invalid attribute + LOG.error('Invalid NVMe attribute: %s %s', name, value) + elif KEY_SMART in self.smartctl: + for attribute in self.smartctl[KEY_SMART].get('table', {}): + try: + _id = int(attribute['id']) + except (KeyError, ValueError): + # Ignoring invalid attribute + LOG.error('Invalid SMART attribute: %s', attribute) + continue + name = str(attribute.get('name', 'UNKNOWN')).replace('_', ' ').title() + raw = int(attribute.get('raw', {}).get('value', -1)) + raw_str = attribute.get('raw', {}).get('string', 'UNKNOWN') + + # Fix power-on time + match = REGEX_POWER_ON_TIME.match(raw_str) + if _id == 9 and match: + raw = int(match.group(1)) + + # Add to dict + self.attributes[_id] = { + 'name': name, 'raw': raw, 'raw_str': raw_str} + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 07120b7dc42463b5dfe46b2a778e094035d7e888 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 23 Oct 2019 16:28:55 -0700 Subject: [PATCH 119/324] Fixed Disk() description and SMART data --- scripts/wk/obj.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/wk/obj.py b/scripts/wk/obj.py index 46cc1cd8..a0ce2db1 100644 --- a/scripts/wk/obj.py +++ b/scripts/wk/obj.py @@ -153,6 +153,12 @@ class Disk(): if not isinstance(self.lsblk[attr], int): self.lsblk[attr] = int(self.lsblk[attr]) + # Set description + self.description = '{size_str} ({tran}) {model} {serial}'.format( + size_str = bytes_to_string(self.lsblk['size'], use_binary=False), + **self.lsblk, + ) + def get_labels(self): """Build list of labels for this disk, returns list.""" labels = [] @@ -179,7 +185,7 @@ class Disk(): '--json', self.path, ] - self.smartctl = get_json_from_command(cmd) + self.smartctl = get_json_from_command(cmd, check=False) # Check for attributes if KEY_NVME in self.smartctl: From 70248ef0b5875a4efa4bb0819cdabf4407c37c27 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 23 Oct 2019 20:33:41 -0700 Subject: [PATCH 120/324] Added macOS support for CpuRam() object. --- scripts/wk/obj.py | 173 +++++++++++++++++++++++++++++++++------------- 1 file changed, 125 insertions(+), 48 deletions(-) diff --git a/scripts/wk/obj.py b/scripts/wk/obj.py index a0ce2db1..227afd15 100644 --- a/scripts/wk/obj.py +++ b/scripts/wk/obj.py @@ -3,6 +3,8 @@ import logging import pathlib +import platform +import plistlib import re from collections import OrderedDict @@ -13,6 +15,16 @@ from wk.std import bytes_to_string, color_string, string_to_bytes # STATIC VARIABLES KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' +KNOWN_RAM_VENDOR_IDS = { + # https://github.com/hewigovens/hewigovens.github.com/wiki/Memory-vendor-code + '0x014F': 'Transcend', + '0x2C00': 'Micron', + '0x802C': 'Micron', + '0x80AD': 'Hynix', + '0x80CE': 'Samsung', + '0xAD00': 'Hynix', + '0xCE00': 'Samsung', + } LOG = logging.getLogger(__name__) REGEX_POWER_ON_TIME = re.compile( r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' @@ -22,57 +34,58 @@ REGEX_POWER_ON_TIME = re.compile( class CpuRam(): """Object for tracking CPU & RAM specific data.""" def __init__(self): - self.lscpu = {} + self.description = 'Unknown' + self.details = {} + self.ram_total = 'Unknown' + self.ram_dimms = [] self.tests = OrderedDict() + + # Update details self.get_cpu_details() self.get_ram_details() - self.name = self.lscpu.get('Model name', 'Unknown CPU') - self.description = self.name def get_cpu_details(self): - """Get CPU details from lscpu.""" - cmd = ['lscpu', '--json'] - json_data = get_json_from_command(cmd) - for line in json_data.get('lscpu', [{}]): - _field = line.get('field', '').replace(':', '') - _data = line.get('data', '') - if not (_field or _data): - # Skip - continue - self.lscpu[_field] = _data + """Get CPU details using OS specific methods.""" + if platform.system() == 'Darwin': + cmd = 'sysctl -n machdep.cpu.brand_string'.split() + proc = run_program(cmd, check=False) + self.description = re.sub(r'\s+', ' ', proc.stdout.strip()) + elif platform.system() == 'Linux': + cmd = ['lscpu', '--json'] + json_data = get_json_from_command(cmd) + for line in json_data.get('lscpu', [{}]): + _field = line.get('field', '').replace(':', '') + _data = line.get('data', '') + if not (_field or _data): + # Skip + continue + self.details[_field] = _data + + self.description = self.details.get('Model name', '') + + # Replace empty description + if not self.description: + self.description = 'Unknown CPU' def get_ram_details(self): - """Get RAM details from dmidecode.""" - cmd = ['sudo', 'dmidecode', '--type', 'memory'] - manufacturer = 'UNKNOWN' + """Get RAM details using OS specific methods.""" + if platform.system() == 'Darwin': + dimm_list = get_ram_list_macos() + elif platform.system() == 'Linux': + dimm_list = get_ram_list_linux() + details = {'Total': 0} - size = 0 - - # Get DMI data - proc = run_program(cmd) - dmi_data = proc.stdout.splitlines() - - # Parse data - for line in dmi_data: - line = line.strip() - if line == 'Memory Device': - # Reset vars - manufacturer = 'UNKNOWN' - size = 0 - elif line.startswith('Size:'): - size = line.replace('Size: ', '') - size = string_to_bytes(size, assume_binary=True) - elif line.startswith('Manufacturer:'): - manufacturer = line.replace('Manufacturer: ', '') - if size <= 0: - # Skip non-populated slots - continue - description = f'{bytes_to_string(size)} {manufacturer}' - details['Total'] += size - if description in details: - details[description] += 1 - else: - details[description] = 1 + for dimm_details in dimm_list: + size, manufacturer = dimm_details + if size <= 0: + # Skip empty DIMMs + continue + description = f'{bytes_to_string(size)} {manufacturer}' + details['Total'] += size + if description in details: + details[description] += 1 + else: + details[description] = 1 # Save details self.ram_total = bytes_to_string(details.pop('Total', 0)) @@ -84,7 +97,7 @@ class CpuRam(): """Generate CPU report with data from all tests.""" report = [] report.append(color_string('Device', 'BLUE')) - report.append(f' {self.name}') + report.append(f' {self.description}') # Include RAM details report.append(color_string('RAM', 'BLUE')) @@ -101,7 +114,7 @@ class Disk(): """Object for tracking disk specific data.""" def __init__(self, path): self.attributes = {} - self.description = 'UNKNOWN' + self.description = 'Unknown' self.lsblk = {} self.nvme_smart_notes = {} self.path = pathlib.Path(path).resolve() @@ -155,7 +168,7 @@ class Disk(): # Set description self.description = '{size_str} ({tran}) {model} {serial}'.format( - size_str = bytes_to_string(self.lsblk['size'], use_binary=False), + size_str=bytes_to_string(self.lsblk['size'], use_binary=False), **self.lsblk, ) @@ -207,9 +220,9 @@ class Disk(): # Ignoring invalid attribute LOG.error('Invalid SMART attribute: %s', attribute) continue - name = str(attribute.get('name', 'UNKNOWN')).replace('_', ' ').title() + name = str(attribute.get('name', 'Unknown')).replace('_', ' ').title() raw = int(attribute.get('raw', {}).get('value', -1)) - raw_str = attribute.get('raw', {}).get('string', 'UNKNOWN') + raw_str = attribute.get('raw', {}).get('string', 'Unknown') # Fix power-on time match = REGEX_POWER_ON_TIME.match(raw_str) @@ -221,5 +234,69 @@ class Disk(): 'name': name, 'raw': raw, 'raw_str': raw_str} +# Functions +def get_ram_list_linux(): + """Get RAM list using dmidecode.""" + cmd = ['sudo', 'dmidecode', '--type', 'memory'] + dimm_list = [] + manufacturer = 'Unknown' + size = 0 + + # Get DMI data + proc = run_program(cmd) + dmi_data = proc.stdout.splitlines() + + # Parse data + for line in dmi_data: + line = line.strip() + if line == 'Memory Device': + # Reset vars + manufacturer = 'Unknown' + size = 0 + elif line.startswith('Size:'): + size = line.replace('Size: ', '') + size = string_to_bytes(size, assume_binary=True) + elif line.startswith('Manufacturer:'): + manufacturer = line.replace('Manufacturer: ', '') + dimm_list.append([size, manufacturer]) + + # Save details + return dimm_list + +def get_ram_list_macos(): + """Get RAM list using system_profiler.""" + dimm_list = [] + + # Get and parse plist data + cmd = [ + 'system_profiler', + '-xml', + 'SPMemoryDataType', + ] + proc = run_program(cmd, check=False, encoding=None, errors=None) + try: + plist_data = plistlib.loads(proc.stdout) + except (TypeError, ValueError): + # Ignore and return an empty list + return dimm_list + + # Check DIMM data + dimm_details = plist_data[0].get('_items', [{}])[0].get('_items', []) + for dimm in dimm_details: + manufacturer = dimm.get('dimm_manufacturer', None) + manufacturer = KNOWN_RAM_VENDOR_IDS.get(manufacturer, 'Unknown') + size = dimm.get('dimm_size', '0 GB') + try: + size = string_to_bytes(size, assume_binary=True) + except ValueError: + # Empty DIMM? + LOG.error('Invalid DIMM size: %s', size) + continue + dimm_list.append([size, manufacturer]) + + # Save details + return dimm_list + + if __name__ == '__main__': print("This file is not meant to be called directly.") From c0242ad55c957c42927e555acc77979b4232a023 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 25 Oct 2019 19:13:04 -0600 Subject: [PATCH 121/324] Added macOS support for disk details and SMART --- scripts/wk/obj.py | 116 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 25 deletions(-) diff --git a/scripts/wk/obj.py b/scripts/wk/obj.py index 227afd15..9305f6ad 100644 --- a/scripts/wk/obj.py +++ b/scripts/wk/obj.py @@ -115,7 +115,7 @@ class Disk(): def __init__(self, path): self.attributes = {} self.description = 'Unknown' - self.lsblk = {} + self.details = {} self.nvme_smart_notes = {} self.path = pathlib.Path(path).resolve() self.smartctl = {} @@ -138,38 +138,39 @@ class Disk(): run_program(cmd, check=False) def get_details(self): - """Get details via lsblk. + """Get disk details using OS specific methods. - Required details default to generic descriptions and - are converted to the correct type. + Required details default to generic descriptions + and are converted to the correct type. """ - cmd = ['lsblk', '--bytes', '--json', '--output-all', '--paths', self.path] - json_data = get_json_from_command(cmd) - self.lsblk = json_data.get('blockdevices', [{}])[0] + if platform.system() == 'Darwin': + self.details = get_disk_details_macos(self.path) + elif platform.system() == 'Linux': + self.details = get_disk_details_linux(self.path) # Set necessary details - self.lsblk['model'] = self.lsblk.get('model', 'Unknown Model') - self.lsblk['name'] = self.lsblk.get('name', self.path) - self.lsblk['log-sec'] = self.lsblk.get('log-sec', 512) - self.lsblk['phy-sec'] = self.lsblk.get('phy-sec', 512) - self.lsblk['rota'] = self.lsblk.get('rota', True) - self.lsblk['serial'] = self.lsblk.get('serial', 'Unknown Serial') - self.lsblk['size'] = self.lsblk.get('size', -1) - self.lsblk['tran'] = self.lsblk.get('tran', '???') - self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') + self.details['model'] = self.details.get('model', 'Unknown Model') + self.details['name'] = self.details.get('name', self.path) + self.details['log-sec'] = self.details.get('log-sec', 512) + self.details['phy-sec'] = self.details.get('phy-sec', 512) + self.details['proto'] = self.details.get('proto', '???') + self.details['proto'] = self.details['proto'].upper().replace('NVME', 'NVMe') + self.details['rota'] = self.details.get('rota', True) + self.details['serial'] = self.details.get('serial', 'Unknown Serial') + self.details['size'] = self.details.get('size', -1) # Ensure certain attributes types - for attr in ['model', 'name', 'serial', 'tran']: - if not isinstance(self.lsblk[attr], str): - self.lsblk[attr] = str(self.lsblk[attr]) + for attr in ['model', 'name', 'proto', 'serial']: + if not isinstance(self.details[attr], str): + self.details[attr] = str(self.details[attr]) for attr in ['log-sec', 'phy-sec', 'size']: - if not isinstance(self.lsblk[attr], int): - self.lsblk[attr] = int(self.lsblk[attr]) + if not isinstance(self.details[attr], int): + self.details[attr] = int(self.details[attr]) # Set description - self.description = '{size_str} ({tran}) {model} {serial}'.format( - size_str=bytes_to_string(self.lsblk['size'], use_binary=False), - **self.lsblk, + self.description = '{size_str} ({proto}) {model} {serial}'.format( + size_str=bytes_to_string(self.details['size'], use_binary=False), + **self.details, ) def get_labels(self): @@ -177,7 +178,7 @@ class Disk(): labels = [] # Add all labels from lsblk - for disk in [self.lsblk, *self.lsblk.get('children', [])]: + for disk in [self.details, *self.details.get('children', [])]: labels.append(disk.get('label', '')) labels.append(disk.get('partlabel', '')) @@ -235,6 +236,71 @@ class Disk(): # Functions +def get_disk_details_linux(path): + """Get disk details using lsblk, returns dict.""" + cmd = ['lsblk', '--bytes', '--json', '--output-all', '--paths', path] + json_data = get_json_from_command(cmd, check=False) + details = json_data.get('blockdevices', [{}])[0] + return details + + +def get_disk_details_macos(path): + """Get disk details using diskutil, returns dict.""" + details = {} + + # Get "list" details + cmd = ['diskutil', 'list', '-plist', path] + proc = run_program(cmd, check=False, encoding=None, errors=None) + try: + plist_data = plistlib.loads(proc.stdout) + except (TypeError, ValueError): + LOG.error('Failed to get diskutil list for %s', path) + # TODO: Figure this out + return details #Bail + + # Parse "list" details + details = plist_data.get('AllDisksAndPartitions', [{}])[0] + details['children'] = details.pop('Partitions', []) + details['path'] = path + for child in details['children']: + child['path'] = path.with_name(child.get('DeviceIdentifier', 'null')) + + # Get "info" details + for dev in [details, *details['children']]: + cmd = ['diskutil', 'info', '-plist', dev['path']] + proc = run_program(cmd, check=False, encoding=None, errors=None) + try: + plist_data = plistlib.loads(proc.stdout) + except (TypeError, ValueError): + LOG.error('Failed to get diskutil info for %s', path) + continue #Skip + + # Parse "info" details + dev.update(plist_data) + dev['size'] = dev.pop('Size', -1) + dev['phy-sec'] = dev.pop('DeviceBlockSize', 512) + dev['ssd'] = dev.pop('SolidState', False) + dev['proto'] = dev.pop('BusProtocol', '???') + dev['vendor'] = '' + dev['model'] = dev.pop('MediaName', 'Unknown') + dev['serial'] = get_disk_serial_macos(dev['path']) + dev['label'] = dev.pop('VolumeName', '') + dev['fstype'] = dev.pop('FilesystemType', '') + dev['mountpoint'] = dev.pop('MountPoint', '') + if not dev.get('WholeDisk', True): + dev['parent'] = dev.pop('ParentWholeDisk', None) + + # Done + return details + + +def get_disk_serial_macos(path): + """Get disk serial using system_profiler, returns str.""" + serial = 'Unknown Serial' + # TODO: Make it real + return serial + + def get_ram_list_linux(): """Get RAM list using dmidecode.""" cmd = ['sudo', 'dmidecode', '--type', 'memory'] From 6a1be5cf06a387aff38e100b479fbf0ddd32f217 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 28 Oct 2019 17:46:17 -0600 Subject: [PATCH 122/324] Moved wk.obj to wk.hw.obj * Done because the main classes are CpuRam() and Disk() * The rest are there for uniformity while working with HW objects --- scripts/wk/__init__.py | 1 - scripts/wk/hw/__init__.py | 2 ++ scripts/wk/{ => hw}/obj.py | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) rename scripts/wk/{ => hw}/obj.py (99%) diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 589cc28b..511cf8a7 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -10,7 +10,6 @@ from wk import io from wk import kit from wk import log from wk import net -from wk import obj from wk import os from wk import std from wk import sw diff --git a/scripts/wk/hw/__init__.py b/scripts/wk/hw/__init__.py index 2b0fbc10..dbc9ea7e 100644 --- a/scripts/wk/hw/__init__.py +++ b/scripts/wk/hw/__init__.py @@ -1 +1,3 @@ """WizardKit: hw module init""" + +from wk.hw import obj diff --git a/scripts/wk/obj.py b/scripts/wk/hw/obj.py similarity index 99% rename from scripts/wk/obj.py rename to scripts/wk/hw/obj.py index 9305f6ad..ac52c869 100644 --- a/scripts/wk/obj.py +++ b/scripts/wk/hw/obj.py @@ -1,4 +1,4 @@ -"""WizardKit: Objects.""" +"""WizardKit: Hardware objects (mostly).""" # vim: sts=2 sw=2 ts=2 import logging @@ -298,6 +298,7 @@ def get_disk_serial_macos(path): """Get disk serial using system_profiler, returns str.""" serial = 'Unknown Serial' # TODO: Make it real + str(path) return serial @@ -329,6 +330,7 @@ def get_ram_list_linux(): # Save details return dimm_list + def get_ram_list_macos(): """Get RAM list using system_profiler.""" dimm_list = [] From fbb480dae6ff79afaf5f0adbd41c288b46b2dc18 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 28 Oct 2019 17:55:37 -0600 Subject: [PATCH 123/324] Adjusted drive details * Use 'bus' instead of 'proto(col)' or 'tran' * I think it's a better description * Reordered details alphanumerically * Removed 'log-sec' from required details * Only concerned with the phy-sec --- scripts/wk/hw/obj.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index ac52c869..844896dc 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -149,26 +149,25 @@ class Disk(): self.details = get_disk_details_linux(self.path) # Set necessary details + self.details['bus'] = self.details.get('bus', '???') + self.details['bus'] = self.details['bus'].upper().replace('NVME', 'NVMe') self.details['model'] = self.details.get('model', 'Unknown Model') self.details['name'] = self.details.get('name', self.path) - self.details['log-sec'] = self.details.get('log-sec', 512) self.details['phy-sec'] = self.details.get('phy-sec', 512) - self.details['proto'] = self.details.get('proto', '???') - self.details['proto'] = self.details['proto'].upper().replace('NVME', 'NVMe') - self.details['rota'] = self.details.get('rota', True) self.details['serial'] = self.details.get('serial', 'Unknown Serial') self.details['size'] = self.details.get('size', -1) + self.details['ssd'] = self.details.get('ssd', False) # Ensure certain attributes types - for attr in ['model', 'name', 'proto', 'serial']: + for attr in ['bus', 'model', 'name', 'serial']: if not isinstance(self.details[attr], str): self.details[attr] = str(self.details[attr]) - for attr in ['log-sec', 'phy-sec', 'size']: + for attr in ['phy-sec', 'size']: if not isinstance(self.details[attr], int): self.details[attr] = int(self.details[attr]) # Set description - self.description = '{size_str} ({proto}) {model} {serial}'.format( + self.description = '{size_str} ({bus}) {model} {serial}'.format( size_str=bytes_to_string(self.details['size'], use_binary=False), **self.details, ) @@ -241,6 +240,8 @@ def get_disk_details_linux(path): cmd = ['lsblk', '--bytes', '--json', '--output-all', '--paths', path] json_data = get_json_from_command(cmd, check=False) details = json_data.get('blockdevices', [{}])[0] + details['bus'] = details.pop('tran', '???') + details['ssd'] = not details.pop('rota', True) return details @@ -277,16 +278,16 @@ def get_disk_details_macos(path): # Parse "info" details dev.update(plist_data) - dev['size'] = dev.pop('Size', -1) - dev['phy-sec'] = dev.pop('DeviceBlockSize', 512) - dev['ssd'] = dev.pop('SolidState', False) - dev['proto'] = dev.pop('BusProtocol', '???') - dev['vendor'] = '' - dev['model'] = dev.pop('MediaName', 'Unknown') - dev['serial'] = get_disk_serial_macos(dev['path']) - dev['label'] = dev.pop('VolumeName', '') + dev['bus'] = dev.pop('BusProtocol', '???') dev['fstype'] = dev.pop('FilesystemType', '') + dev['label'] = dev.pop('VolumeName', '') + dev['model'] = dev.pop('MediaName', 'Unknown') dev['mountpoint'] = dev.pop('MountPoint', '') + dev['phy-sec'] = dev.pop('DeviceBlockSize', 512) + dev['serial'] = get_disk_serial_macos(dev['path']) + dev['size'] = dev.pop('Size', -1) + dev['ssd'] = dev.pop('SolidState', False) + dev['vendor'] = '' if not dev.get('WholeDisk', True): dev['parent'] = dev.pop('ParentWholeDisk', None) From c7090e77c26840082554b28f01157aac1579bf0f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 28 Oct 2019 20:15:58 -0600 Subject: [PATCH 124/324] Added Disk().generate_report() * Uses new merged ATTRIBUTES config --- scripts/wk/cfg/__init__.py | 1 + scripts/wk/cfg/hw.py | 34 +++++++++++++++ scripts/wk/hw/obj.py | 85 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 scripts/wk/cfg/hw.py diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py index 9538f1c8..fc63cf2d 100644 --- a/scripts/wk/cfg/__init__.py +++ b/scripts/wk/cfg/__init__.py @@ -1,5 +1,6 @@ """WizardKit: cfg module init""" +from wk.cfg import hw from wk.cfg import log from wk.cfg import main from wk.cfg import net diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py new file mode 100644 index 00000000..972f6ce0 --- /dev/null +++ b/scripts/wk/cfg/hw.py @@ -0,0 +1,34 @@ +"""WizardKit: Config - Hardware""" +# pylint: disable=bad-whitespace,line-too-long +# vim: sts=2 sw=2 ts=2 + + +ATTRIBUTES = { + # NVMe + 'critical_warning': {'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'media_errors': {'Critical': False, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'power_on_hours': {'Critical': False, 'Ignore': True, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 'unsafe_shutdowns': {'Critical': False, 'Ignore': True, 'Warning': 1, 'Error': None, 'Maximum': None, }, + # SMART + 5: {'Hex': '05', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 9: {'Hex': '09', 'Critical': False, 'Ignore': True, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 10: {'Hex': '10', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 184: {'Hex': 'B8', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 187: {'Hex': 'BB', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 188: {'Hex': 'BC', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 196: {'Hex': 'C4', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 197: {'Hex': 'C5', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 198: {'Hex': 'C6', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 199: {'Hex': 'C7', 'Critical': False, 'Ignore': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 201: {'Hex': 'C9', 'Critical': False, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': 10000, }, + } +ATTRIBUTE_COLORS = ( + # NOTE: Ordered by ascending importance + ('Warning', 'YELLOW'), + ('Error', 'RED'), + ('Maximum', 'PURPLE'), + ) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 844896dc..ab30d258 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -1,4 +1,4 @@ -"""WizardKit: Hardware objects (mostly).""" +"""WizardKit: Hardware objects (mostly)""" # vim: sts=2 sw=2 ts=2 import logging @@ -9,6 +9,7 @@ import re from collections import OrderedDict +from wk.cfg.hw import ATTRIBUTES, ATTRIBUTE_COLORS from wk.exe import get_json_from_command, run_program from wk.std import bytes_to_string, color_string, string_to_bytes @@ -93,8 +94,8 @@ class CpuRam(): f'{count}x {desc}' for desc, count in sorted(details.items()) ] - def generate_cpu_report(self): - """Generate CPU report with data from all tests.""" + def generate_report(self): + """Generate CPU & RAM report, returns list.""" report = [] report.append(color_string('Device', 'BLUE')) report.append(f' {self.description}') @@ -116,7 +117,7 @@ class Disk(): self.attributes = {} self.description = 'Unknown' self.details = {} - self.nvme_smart_notes = {} + self.notes = {} self.path = pathlib.Path(path).resolve() self.smartctl = {} self.tests = OrderedDict() @@ -137,6 +138,77 @@ class Disk(): ] run_program(cmd, check=False) + def generate_attribute_report(self): + """Generate attribute report, returns list.""" + report = [] + for attr, value in sorted(self.attributes.items()): + note = '' + value_color = 'GREEN' + + # Skip attributes not in our list + if attr not in ATTRIBUTES: + continue + + # ID / Name + label = f'{attr:>3}' + if isinstance(attr, int): + # Assuming SMART, include hex ID and name + label += f' / {str(hex(attr))[2:].upper():0>2}: {value["name"]}' + label = f' {label.replace("_", " "):38}' + + # Value color + for threshold, color in ATTRIBUTE_COLORS: + if value['raw'] >= ATTRIBUTES.get(threshold, float('inf')): + value_color = color + if threshold == 'Maximum': + note = '(invalid?)' + + # 199/C7 warning + if str(attr) == '199' and value['raw'] > 0: + note = '(bad cable?)' + + # Build colored string and append to report + line = color_string( + [label, value['raw_str'], note], + [None, value_color, 'YELLOW'], + ) + report.append(line) + + # Done + return report + + + def generate_report(self): + """Generate Disk report, returns list.""" + report = [] + report.append(color_string(f'Device {self.path.name}', 'BLUE')) + report.append(f' {self.description}') + + # Attributes + if self.attributes: + report.append(color_string('Attributes', 'BLUE')) + report.extend(self.generate_attribute_report()) + else: + report.append( + color_string(' No NVMe or SMART data available', 'YELLOW')) + + # Notes + if self.notes: + report.append(color_string('Notes', 'BLUE')) + for note in sorted(self.notes.keys()): + report.append(f' {note}') + + # 4K alignment check + if not self.is_4k_aligned(): + report.append(color_string('Warning', 'YELLOW')) + report.append(' One or more partitions are not 4K aligned') + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + def get_details(self): """Get disk details using OS specific methods. @@ -187,6 +259,11 @@ class Disk(): # Done return labels + def is_4k_aligned(self): + """Check that all disk partitions are aligned, returns bool.""" + #TODO: Make real + return True + def update_smart_details(self): """Update SMART details via smartctl.""" self.attributes = {} From ae5e9b8f3424716e9057170a40334cd0887fd16f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 28 Oct 2019 20:45:30 -0600 Subject: [PATCH 125/324] Added 4K alignment check --- scripts/wk/hw/obj.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index ab30d258..3330f7f7 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -261,8 +261,12 @@ class Disk(): def is_4k_aligned(self): """Check that all disk partitions are aligned, returns bool.""" - #TODO: Make real - return True + aligned = True + if not platform.system() == 'Linux': + aligned = is_4k_aligned_linux(self.path, self.details['phy-sec']) + #TODO: Add checks for other OS + + return aligned def update_smart_details(self): """Update SMART details via smartctl.""" @@ -444,5 +448,27 @@ def get_ram_list_macos(): return dimm_list +def is_4k_aligned_linux(dev_path, physical_sector_size): + """Check partition alignment using lsblk, returns bool.""" + aligned = True + cmd = [ + 'sudo', + 'sfdisk', + '--json', + dev_path, + ] + + # Get partition details + json_data = get_json_from_command(cmd) + + # Check partitions + for part in json_data.get('partitiontable', {}).get('partitions', []): + offset = physical_sector_size * part.get('start', -1) + aligned = aligned and offset >= 0 and offset % 2096 == 0 + + # Done + return aligned + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 117df6158af8ad72f4477054a15d956470ea2831 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 28 Oct 2019 20:57:34 -0600 Subject: [PATCH 126/324] Fix attribute value colors --- scripts/wk/hw/obj.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 3330f7f7..2c0cb54c 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -158,7 +158,8 @@ class Disk(): # Value color for threshold, color in ATTRIBUTE_COLORS: - if value['raw'] >= ATTRIBUTES.get(threshold, float('inf')): + threshold_val = ATTRIBUTES.get(attr, {}).get(threshold, float('inf')) + if threshold_val and value['raw'] >= threshold_val: value_color = color if threshold == 'Maximum': note = '(invalid?)' From 2a019d09a0e17292c40c156c475597bd03089bce Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 31 Oct 2019 16:33:35 -0600 Subject: [PATCH 127/324] Updated Disk notes sections --- scripts/wk/hw/obj.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 2c0cb54c..f02ac5be 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -117,7 +117,7 @@ class Disk(): self.attributes = {} self.description = 'Unknown' self.details = {} - self.notes = {} + self.notes = [] self.path = pathlib.Path(path).resolve() self.smartctl = {} self.tests = OrderedDict() @@ -126,6 +126,16 @@ class Disk(): self.get_details() self.enable_smart() self.update_smart_details() + if not self.is_4k_aligned(): + self.add_note('One or more partitions are not 4K aligned', 'YELLOW') + + def add_note(self, note, color=None): + """Add note that will be included in the disk report.""" + if color: + note = color_string(note, color) + if note not in self.notes: + self.notes.append(note) + self.notes.sort() def enable_smart(self): """Try enabling SMART for this disk.""" @@ -182,27 +192,19 @@ class Disk(): def generate_report(self): """Generate Disk report, returns list.""" report = [] - report.append(color_string(f'Device {self.path.name}', 'BLUE')) + report.append(color_string(f'Device ({self.path.name})', 'BLUE')) report.append(f' {self.description}') # Attributes if self.attributes: report.append(color_string('Attributes', 'BLUE')) report.extend(self.generate_attribute_report()) - else: - report.append( - color_string(' No NVMe or SMART data available', 'YELLOW')) # Notes if self.notes: report.append(color_string('Notes', 'BLUE')) - for note in sorted(self.notes.keys()): - report.append(f' {note}') - - # 4K alignment check - if not self.is_4k_aligned(): - report.append(color_string('Warning', 'YELLOW')) - report.append(' One or more partitions are not 4K aligned') + for note in self.notes: + report.append(f' {note}') # Tests for test in self.tests.values(): @@ -315,6 +317,10 @@ class Disk(): self.attributes[_id] = { 'name': name, 'raw': raw, 'raw_str': raw_str} + # Add note if necessary + if not self.attributes: + self.add_note('No NVMe or SMART data available', 'YELLOW') + # Functions def get_disk_details_linux(path): From 5d6b7578d30e60268327c5606cc033035efea1a0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 31 Oct 2019 16:34:54 -0600 Subject: [PATCH 128/324] Fixed 4K alignment check under Linux --- scripts/wk/hw/obj.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index f02ac5be..ee24458d 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -265,7 +265,7 @@ class Disk(): def is_4k_aligned(self): """Check that all disk partitions are aligned, returns bool.""" aligned = True - if not platform.system() == 'Linux': + if platform.system() == 'Linux': aligned = is_4k_aligned_linux(self.path, self.details['phy-sec']) #TODO: Add checks for other OS @@ -471,7 +471,7 @@ def is_4k_aligned_linux(dev_path, physical_sector_size): # Check partitions for part in json_data.get('partitiontable', {}).get('partitions', []): offset = physical_sector_size * part.get('start', -1) - aligned = aligned and offset >= 0 and offset % 2096 == 0 + aligned = aligned and offset >= 0 and offset % 4096 == 0 # Done return aligned From d25b341eb3c2ae70a0da92d39def9e90b1b74753 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 31 Oct 2019 16:48:30 -0600 Subject: [PATCH 129/324] Added is_4k_aligned_macos() --- scripts/wk/hw/obj.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index ee24458d..70fe4657 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -265,7 +265,9 @@ class Disk(): def is_4k_aligned(self): """Check that all disk partitions are aligned, returns bool.""" aligned = True - if platform.system() == 'Linux': + if platform.system() == 'Darwin': + aligned = is_4k_aligned_macos(self.details) + elif platform.system() == 'Linux': aligned = is_4k_aligned_linux(self.path, self.details['phy-sec']) #TODO: Add checks for other OS @@ -455,6 +457,23 @@ def get_ram_list_macos(): return dimm_list +def is_4k_aligned_macos(disk_details): + """Check partition alignment using diskutil info, returns bool.""" + aligned = True + + # Check partitions + for part in disk_details.get('children', []): + offset = part.get('PartitionMapPartitionOffset', 0) + if not offset: + # Assuming offset couldn't be found and it defaulted to 0 + # NOTE: Just logging the error, not bailing + LOG.error('Failed to get partition offset for %s', part['path']) + aligned = aligned and offset >= 0 and offset % 4096 == 0 + + # Done + return aligned + + def is_4k_aligned_linux(dev_path, physical_sector_size): """Check partition alignment using lsblk, returns bool.""" aligned = True From 07fdbcdd7c74d01689120a0367c1167862d63a17 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 31 Oct 2019 18:28:34 -0600 Subject: [PATCH 130/324] Added Disk().safety_checks() * Raises an exception for blocking events * Removed "Ignore" column from ATTRIBUTES * Listed attributes should either be warnings or errors * Only 'Critical' attributes should block futher tests --- scripts/wk/cfg/hw.py | 30 +++++++++--------- scripts/wk/hw/obj.py | 72 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 972f6ce0..06554c9e 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -5,22 +5,22 @@ ATTRIBUTES = { # NVMe - 'critical_warning': {'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 'media_errors': {'Critical': False, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 'power_on_hours': {'Critical': False, 'Ignore': True, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, - 'unsafe_shutdowns': {'Critical': False, 'Ignore': True, 'Warning': 1, 'Error': None, 'Maximum': None, }, + 'critical_warning': {'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'media_errors': {'Critical': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'power_on_hours': {'Critical': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 'unsafe_shutdowns': {'Critical': False, 'Warning': 1, 'Error': None, 'Maximum': None, }, # SMART - 5: {'Hex': '05', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 9: {'Hex': '09', 'Critical': False, 'Ignore': True, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, - 10: {'Hex': '10', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 184: {'Hex': 'B8', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 187: {'Hex': 'BB', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 188: {'Hex': 'BC', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 196: {'Hex': 'C4', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 197: {'Hex': 'C5', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 198: {'Hex': 'C6', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 199: {'Hex': 'C7', 'Critical': False, 'Ignore': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 201: {'Hex': 'C9', 'Critical': False, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': 10000, }, + 5: {'Hex': '05', 'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 9: {'Hex': '09', 'Critical': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 10: {'Hex': '10', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 184: {'Hex': 'B8', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 187: {'Hex': 'BB', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 188: {'Hex': 'BC', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 196: {'Hex': 'C4', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 197: {'Hex': 'C5', 'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 198: {'Hex': 'C6', 'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 199: {'Hex': 'C7', 'Critical': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 201: {'Hex': 'C9', 'Critical': False, 'Warning': None, 'Error': 1, 'Maximum': 10000, }, } ATTRIBUTE_COLORS = ( # NOTE: Ordered by ascending importance diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 70fe4657..bed9ec0a 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -13,6 +13,7 @@ from wk.cfg.hw import ATTRIBUTES, ATTRIBUTE_COLORS from wk.exe import get_json_from_command, run_program from wk.std import bytes_to_string, color_string, string_to_bytes + # STATIC VARIABLES KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' @@ -31,6 +32,12 @@ REGEX_POWER_ON_TIME = re.compile( r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' ) + +# Exception Classes +class CriticalHardwareError(RuntimeError): + """Exception used for critical hardware failures.""" + + # Classes class CpuRam(): """Object for tracking CPU & RAM specific data.""" @@ -273,6 +280,71 @@ class Disk(): return aligned + def safety_checks(self): + """Run safety checks and raise an exception if necessary.""" + blocking_event_encountered = False + self.update_smart_details() + + # Attributes + for attr, value in self.attributes.items(): + # Skip unknown attributes + if attr not in ATTRIBUTES: + continue + + # Get thresholds + critical = ATTRIBUTES[attr].get('Critical', False) + err_thresh = ATTRIBUTES[attr].get('Error', None) + max_thresh = ATTRIBUTES[attr].get('Maximum', None) + if not max_thresh: + max_thresh = float('inf') + + # Skip non-critical attributes + if not critical: + continue + + # Skip informational attributes + if not err_thresh: + continue + + # Check attribute + if err_thresh <= value['raw'] < max_thresh: + blocking_event_encountered = True + msg = f'Failed attribute: {attr}' + LOG.error('%s %s', self.path, msg) + + # NVMe status + # TODO: See https://github.com/2Shirt/WizardKit/issues/130 + + # SMART overall assessment + smart_passed = True + try: + smart_passed = self.smartctl['smart_status']['passed'] + except (KeyError, TypeError): + # Assuming disk doesn't support SMART overall assessment + pass + if not smart_passed: + blocking_event_encountered = True + msg = 'SMART overall self-assessment: Failed' + self.add_note(msg, 'RED') + LOG.error('%s %s', self.path, msg) + + # SMART self-test status + test_status = '' + try: + test_status = self.smartctl['ata_smart_data']['self_test']['status'] + except (KeyError, TypeError): + # Assuming disk doesn't support SMART self-tests + pass + if 'remaining_percent' in test_status: + blocking_event_encountered = True + msg = 'SMART self-test in progress' + self.add_note(msg, 'RED') + LOG.error('%s %s', self.path, msg) + + # Raise exception if necessary + if blocking_event_encountered: + raise CriticalHardwareError(f'Critical error(s) for: {self.path}') + def update_smart_details(self): """Update SMART details via smartctl.""" self.attributes = {} From 93102b5144d33a655ff9b0b25607131d2c5e027d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 31 Oct 2019 19:19:52 -0600 Subject: [PATCH 131/324] Reworked checking Disk() attributes * Added separate Disk().check_attributes() function * Can be used to check all KNOWN_ATTRIBUTES or just blocking ones * Renamed ATTRIBUTES to KNOWN_ATTRIBUTES for clarity * Renamed 'Critical' column to 'Blocking' * Added '(Failed)' note to attribute report * Addresses issue #131 --- scripts/wk/cfg/hw.py | 38 ++++++++++------------- scripts/wk/hw/obj.py | 74 +++++++++++++++++++++++++++----------------- 2 files changed, 61 insertions(+), 51 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 06554c9e..27455bbd 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -3,31 +3,25 @@ # vim: sts=2 sw=2 ts=2 -ATTRIBUTES = { +KNOWN_ATTRIBUTES = { # NVMe - 'critical_warning': {'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 'media_errors': {'Critical': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 'power_on_hours': {'Critical': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, - 'unsafe_shutdowns': {'Critical': False, 'Warning': 1, 'Error': None, 'Maximum': None, }, + 'critical_warning': {'Blocking': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'media_errors': {'Blocking': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'power_on_hours': {'Blocking': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 'unsafe_shutdowns': {'Blocking': False, 'Warning': 1, 'Error': None, 'Maximum': None, }, # SMART - 5: {'Hex': '05', 'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 9: {'Hex': '09', 'Critical': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, - 10: {'Hex': '10', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 184: {'Hex': 'B8', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 187: {'Hex': 'BB', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 188: {'Hex': 'BC', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 196: {'Hex': 'C4', 'Critical': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, - 197: {'Hex': 'C5', 'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 198: {'Hex': 'C6', 'Critical': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 199: {'Hex': 'C7', 'Critical': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 201: {'Hex': 'C9', 'Critical': False, 'Warning': None, 'Error': 1, 'Maximum': 10000, }, + 5: {'Hex': '05', 'Blocking': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 9: {'Hex': '09', 'Blocking': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 10: {'Hex': '10', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 184: {'Hex': 'B8', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 187: {'Hex': 'BB', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 188: {'Hex': 'BC', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 196: {'Hex': 'C4', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 197: {'Hex': 'C5', 'Blocking': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 198: {'Hex': 'C6', 'Blocking': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 199: {'Hex': 'C7', 'Blocking': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 201: {'Hex': 'C9', 'Blocking': False, 'Warning': None, 'Error': 1, 'Maximum': 10000, }, } -ATTRIBUTE_COLORS = ( - # NOTE: Ordered by ascending importance - ('Warning', 'YELLOW'), - ('Error', 'RED'), - ('Maximum', 'PURPLE'), - ) if __name__ == '__main__': diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index bed9ec0a..8df8a459 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -9,12 +9,18 @@ import re from collections import OrderedDict -from wk.cfg.hw import ATTRIBUTES, ATTRIBUTE_COLORS +from wk.cfg.hw import KNOWN_ATTRIBUTES from wk.exe import get_json_from_command, run_program from wk.std import bytes_to_string, color_string, string_to_bytes # STATIC VARIABLES +ATTRIBUTE_COLORS = ( + # NOTE: Ordered by ascending importance + ('Warning', 'YELLOW'), + ('Error', 'RED'), + ('Maximum', 'PURPLE'), + ) KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' KNOWN_RAM_VENDOR_IDS = { @@ -144,6 +150,36 @@ class Disk(): self.notes.append(note) self.notes.sort() + def check_attributes(self, only_blocking=False): + """Check if any known attributes are failing, returns bool.""" + attributes_ok = True + for attr, value in self.attributes.items(): + # Skip unknown attributes + if attr not in KNOWN_ATTRIBUTES: + continue + + # Get thresholds + blocking_attribute = KNOWN_ATTRIBUTES[attr].get('Blocking', False) + err_thresh = KNOWN_ATTRIBUTES[attr].get('Error', None) + max_thresh = KNOWN_ATTRIBUTES[attr].get('Maximum', None) + if not max_thresh: + max_thresh = float('inf') + + # Skip non-blocking attributes if necessary + if only_blocking and not blocking_attribute: + continue + + # Skip informational attributes + if not err_thresh: + continue + + # Check attribute + if err_thresh <= value['raw'] < max_thresh: + attributes_ok = False + + # Done + return attributes_ok + def enable_smart(self): """Try enabling SMART for this disk.""" cmd = [ @@ -163,7 +199,7 @@ class Disk(): value_color = 'GREEN' # Skip attributes not in our list - if attr not in ATTRIBUTES: + if attr not in KNOWN_ATTRIBUTES: continue # ID / Name @@ -175,10 +211,12 @@ class Disk(): # Value color for threshold, color in ATTRIBUTE_COLORS: - threshold_val = ATTRIBUTES.get(attr, {}).get(threshold, float('inf')) + threshold_val = KNOWN_ATTRIBUTES[attr].get(threshold, None) if threshold_val and value['raw'] >= threshold_val: value_color = color - if threshold == 'Maximum': + if threshold == 'Error': + note = '(Failed)' + elif threshold == 'Maximum': note = '(invalid?)' # 199/C7 warning @@ -286,31 +324,9 @@ class Disk(): self.update_smart_details() # Attributes - for attr, value in self.attributes.items(): - # Skip unknown attributes - if attr not in ATTRIBUTES: - continue - - # Get thresholds - critical = ATTRIBUTES[attr].get('Critical', False) - err_thresh = ATTRIBUTES[attr].get('Error', None) - max_thresh = ATTRIBUTES[attr].get('Maximum', None) - if not max_thresh: - max_thresh = float('inf') - - # Skip non-critical attributes - if not critical: - continue - - # Skip informational attributes - if not err_thresh: - continue - - # Check attribute - if err_thresh <= value['raw'] < max_thresh: - blocking_event_encountered = True - msg = f'Failed attribute: {attr}' - LOG.error('%s %s', self.path, msg) + if not self.check_attributes(only_blocking=True): + blocking_event_encountered = True + LOG.error('%s: Blocked for failing attributes', self.path) # NVMe status # TODO: See https://github.com/2Shirt/WizardKit/issues/130 From e634d1691f86f338036d35962ac23adf5c2875e7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 1 Nov 2019 18:51:02 -0600 Subject: [PATCH 132/324] Added SMART self-test sections --- scripts/wk/hw/obj.py | 105 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 11 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 8df8a459..ddc9d2ab 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -11,7 +11,7 @@ from collections import OrderedDict from wk.cfg.hw import KNOWN_ATTRIBUTES from wk.exe import get_json_from_command, run_program -from wk.std import bytes_to_string, color_string, string_to_bytes +from wk.std import bytes_to_string, color_string, sleep, string_to_bytes # STATIC VARIABLES @@ -43,6 +43,9 @@ REGEX_POWER_ON_TIME = re.compile( class CriticalHardwareError(RuntimeError): """Exception used for critical hardware failures.""" +class SMARTNotSupportedError(TypeError): + """Exception used for disks lacking SMART support.""" + # Classes class CpuRam(): @@ -233,7 +236,6 @@ class Disk(): # Done return report - def generate_report(self): """Generate Disk report, returns list.""" report = [] @@ -307,6 +309,18 @@ class Disk(): # Done return labels + def get_smart_self_test_details(self): + """Shorthand to get deeply nested self-test details, returns dict.""" + details = {} + try: + details = self.smartctl['ata_smart_data']['self_test'] + except (KeyError, TypeError): + # Assuming disk lacks SMART support, ignore and return empty dict. + pass + + # Done + return details + def is_4k_aligned(self): """Check that all disk partitions are aligned, returns bool.""" aligned = True @@ -345,13 +359,8 @@ class Disk(): LOG.error('%s %s', self.path, msg) # SMART self-test status - test_status = '' - try: - test_status = self.smartctl['ata_smart_data']['self_test']['status'] - except (KeyError, TypeError): - # Assuming disk doesn't support SMART self-tests - pass - if 'remaining_percent' in test_status: + test_details = self.get_smart_self_test_details() + if 'remaining_percent' in test_details.get('status', ''): blocking_event_encountered = True msg = 'SMART self-test in progress' self.add_note(msg, 'RED') @@ -361,6 +370,80 @@ class Disk(): if blocking_event_encountered: raise CriticalHardwareError(f'Critical error(s) for: {self.path}') + def run_self_test(self, log_path): + """Run disk self-test and check if it passed, returns bool. + + NOTE: This function is here to reserve a place for future + NVMe self-tests announced in NVMe spec v1.3. + """ + result = self.run_smart_self_test(log_path) + return result + + def run_smart_self_test(self, log_path): + """Run SMART self-test and check if it passed, returns bool. + + NOTE: An exception will be raised if the disk lacks SMART support. + """ + finished = False + result = None + started = False + status_str = 'Starting self-test...' + test_details = self.get_smart_self_test_details() + test_minutes = 15 + + # Check if disk supports self-tests + if not test_details: + raise SMARTNotSupportedError( + f'SMART self-test not supported for {self.path}') + + # Get real test length + test_minutes = test_details.get('polling_minutes', {}).get('short', 5) + test_minutes = int(test_minutes) + 10 + + # Start test + cmd = [ + 'sudo', + 'smartctl', + '--tolerance=normal', + '--test=short', + self.path, + ] + run_program(cmd, check=False) + + # Monitor progress (in five second intervals) + for _i in range(int(test_minutes*60/5)): + sleep(5) + + # Update status + self.update_smart_details() + test_details = self.get_smart_self_test_details() + + # Check test progress + if started: + status_str = test_details.get('status', {}).get('string', 'Unknown') + status_str = status_str.capitalize() + + # Update log + with open(log_path, 'w') as _f: + _f.write(f'SMART self-test status for {self.path}:\n {status_str}') + + # Check if finished + if 'remaining_percent' not in test_details['status']: + finished = True + break + + elif 'remaining_percent' in test_details['status']: + started = True + + # Check result + if finished: + result = test_details.get('status', {}).get('passed', False) + elif started: + raise TimeoutError(f'SMART self-test timed out for {self.path}') + + # Done + return result + def update_smart_details(self): """Update SMART details via smartctl.""" self.attributes = {} @@ -433,9 +516,9 @@ def get_disk_details_macos(path): try: plist_data = plistlib.loads(proc.stdout) except (TypeError, ValueError): + # Invalid / corrupt plist data? return empty dict to avoid crash LOG.error('Failed to get diskutil list for %s', path) - # TODO: Figure this out - return details #Bail + return details # Parse "list" details details = plist_data.get('AllDisksAndPartitions', [{}])[0] From d933ff97425b90bb0aa8cdd5520b902afe5385da Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 1 Nov 2019 18:52:26 -0600 Subject: [PATCH 133/324] Reordered functions --- scripts/wk/hw/obj.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index ddc9d2ab..94bbdc54 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -61,6 +61,22 @@ class CpuRam(): self.get_cpu_details() self.get_ram_details() + def generate_report(self): + """Generate CPU & RAM report, returns list.""" + report = [] + report.append(color_string('Device', 'BLUE')) + report.append(f' {self.description}') + + # Include RAM details + report.append(color_string('RAM', 'BLUE')) + report.append(f' {self.ram_total} ({", ".join(self.ram_dimms)})') + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + def get_cpu_details(self): """Get CPU details using OS specific methods.""" if platform.system() == 'Darwin': @@ -110,22 +126,6 @@ class CpuRam(): f'{count}x {desc}' for desc, count in sorted(details.items()) ] - def generate_report(self): - """Generate CPU & RAM report, returns list.""" - report = [] - report.append(color_string('Device', 'BLUE')) - report.append(f' {self.description}') - - # Include RAM details - report.append(color_string('RAM', 'BLUE')) - report.append(f' {self.ram_total} ({", ".join(self.ram_dimms)})') - - # Tests - for test in self.tests.values(): - report.extend(test.report) - - return report - class Disk(): """Object for tracking disk specific data.""" From 0e9b1af56b100093fb6bb41ca0d7a04be7db159b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 7 Nov 2019 18:57:31 -0700 Subject: [PATCH 134/324] Added safety check to Disk().get_details() --- scripts/wk/hw/obj.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 94bbdc54..e1ced59f 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -286,7 +286,11 @@ class Disk(): self.details[attr] = str(self.details[attr]) for attr in ['phy-sec', 'size']: if not isinstance(self.details[attr], int): - self.details[attr] = int(self.details[attr]) + try: + self.details[attr] = int(self.details[attr]) + except (TypeError, ValueError): + LOG.error('Invalid disk %s: %s', attr, self.details[attr]) + self.details[attr] = -1 # Set description self.description = '{size_str} ({bus}) {model} {serial}'.format( From 12de0e5b84684a4ecc7f3fe0c00eb784918454f6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 7 Nov 2019 18:58:20 -0700 Subject: [PATCH 135/324] Inlcude RAM vendor ID for unknown IDs --- scripts/wk/hw/obj.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index e1ced59f..3bba408d 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -344,7 +344,7 @@ class Disk(): # Attributes if not self.check_attributes(only_blocking=True): blocking_event_encountered = True - LOG.error('%s: Blocked for failing attributes', self.path) + LOG.error('%s: Blocked for failing attribute(s)', self.path) # NVMe status # TODO: See https://github.com/2Shirt/WizardKit/issues/130 @@ -618,7 +618,9 @@ def get_ram_list_macos(): dimm_details = plist_data[0].get('_items', [{}])[0].get('_items', []) for dimm in dimm_details: manufacturer = dimm.get('dimm_manufacturer', None) - manufacturer = KNOWN_RAM_VENDOR_IDS.get(manufacturer, 'Unknown') + manufacturer = KNOWN_RAM_VENDOR_IDS.get( + manufacturer, + f'Unknown ({manufacturer})') size = dimm.get('dimm_size', '0 GB') try: size = string_to_bytes(size, assume_binary=True) From b162c99d6e5aa5f3897f0c7791edbe94246e02a7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 7 Nov 2019 20:40:50 -0700 Subject: [PATCH 136/324] Added BaseObj() class --- scripts/wk/hw/obj.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 3bba408d..495051de 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -48,9 +48,24 @@ class SMARTNotSupportedError(TypeError): # Classes -class CpuRam(): +class BaseObj(): + """Base object for tracking device data.""" + def __init__(self): + self.tests = OrderedDict() + + def all_tests_passed(self): + """Check if all tests passed, returns bool.""" + return all([results.passed for results in self.tests.values()]) + + def any_test_failed(self): + """Check if any test failed, returns bool.""" + return any([results.failed for results in self.tests.values()]) + + +class CpuRam(BaseObj): """Object for tracking CPU & RAM specific data.""" def __init__(self): + super().__init__() self.description = 'Unknown' self.details = {} self.ram_total = 'Unknown' @@ -127,9 +142,10 @@ class CpuRam(): ] -class Disk(): +class Disk(BaseObj): """Object for tracking disk specific data.""" def __init__(self, path): + super().__init__() self.attributes = {} self.description = 'Unknown' self.details = {} From a053931c17d62ee163487a3b6a2d9b4b2f286287 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 7 Nov 2019 20:46:21 -0700 Subject: [PATCH 137/324] Added Test() object --- scripts/wk/hw/obj.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 495051de..6395e573 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -515,6 +515,26 @@ class Disk(BaseObj): self.add_note('No NVMe or SMART data available', 'YELLOW') +class Test(): + """Object for tracking test specific data.""" + def __init__(self, dev, label): + self.dev = dev + self.disabled = False + self.failed = False + self.label = label + self.passed = False + self.report = [] + self.status = '' + + def set_status(self, status): + """Update status string.""" + if self.disabled: + # Don't change status if disabled + return + + self.status = status + + # Functions def get_disk_details_linux(path): """Get disk details using lsblk, returns dict.""" From b7c790438abfa151580eb4e48fd5f3f34f25915e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 8 Nov 2019 13:50:26 -0700 Subject: [PATCH 138/324] Updated launch-in-tmux * Don't exit shells, just the function * Don't leave dangling tmux sessions if possible * Restore original session/window name if using an active tmux session --- scripts/launch-in-tmux | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/launch-in-tmux b/scripts/launch-in-tmux index e737b574..1f774426 100755 --- a/scripts/launch-in-tmux +++ b/scripts/launch-in-tmux @@ -13,16 +13,16 @@ function ask() { done } -die () { +function err () { echo "$0:" "$@" >&2 - exit 1 + return 1 } function launch_in_tmux() { # Check for required vars - [[ -n "${SESSION_NAME:-}" ]] || die "Required variable missing (SESSION_NAME)" - [[ -n "${WINDOW_NAME:-}" ]] || die "Required variable missing (WINDOW_NAME)" - [[ -n "${TMUX_CMD:-}" ]] || die "Required variable missing (TMUX_CMD)" + [[ -n "${SESSION_NAME:-}" ]] || return $(err "Required variable missing (SESSION_NAME)") + [[ -n "${WINDOW_NAME:-}" ]] || return $(err "Required variable missing (WINDOW_NAME)") + [[ -n "${TMUX_CMD:-}" ]] || return $(err "Required variable missing (TMUX_CMD)") # Check for running session if tmux list-session | grep -q "$SESSION_NAME"; then @@ -32,11 +32,15 @@ function launch_in_tmux() { if [[ -n "${TMUX:-}" ]]; then # Running inside TMUX, switch to session tmux switch-client -t "$SESSION_NAME" + if ! jobs %% >/dev/null 2>&1; then + # No running jobs, try exiting abandoned tmux session + exit 0 + fi else # Running outside TMUX, attach to session tmux attach-session -t "$SESSION_NAME" fi - exit 0 + return 0 elif ask "Kill current session and start new session?"; then tmux kill-session -t "$SESSION_NAME" || \ die "Failed to kill session: $SESSION_NAME" @@ -45,18 +49,21 @@ function launch_in_tmux() { echo "" echo -n "Press Enter to exit... " read -r - exit 0 + return 1 fi fi - # Start/Rename session + # Start session if [[ -n "${TMUX:-}" ]]; then - # Running inside TMUX, rename session/window and open the menu + # Running inside TMUX, save current session/window names + ORIGINAL_SESSION_NAME="$(tmux display-message -p '#S')" + ORIGINAL_WINDOW_NAME="$(tmux display-message -p '#W')" tmux rename-session "$SESSION_NAME" tmux rename-window "$WINDOW_NAME" "$TMUX_CMD" "$@" - tmux rename-session "${SESSION_NAME}_DONE" - tmux rename-window "${WINDOW_NAME}_DONE" + # Restore previous session/window names + tmux rename-session "${ORIGINAL_SESSION_NAME}" + tmux rename-window "${ORIGINAL_WINDOW_NAME}" else # Running outside TMUX, start/attach to session tmux new-session -s "$SESSION_NAME" -n "$WINDOW_NAME" "$TMUX_CMD" "$@" From 3a2924bd510fbd481eccc9cf5a6494ead6c266e4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 8 Nov 2019 14:08:16 -0700 Subject: [PATCH 139/324] Added print_report() --- scripts/wk/std.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index eeede87e..0acdeb74 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -859,6 +859,14 @@ def print_info(msg, log=True, **kwargs): LOG.info(msg) +def print_report(report, log=True): + """Print report to screen and optionally to log.""" + for line in report: + print(line) + if log: + LOG.info(strip_colors(line)) + + def print_standard(msg, log=True, **kwargs): """Prints message and log as INFO.""" print(msg, **kwargs) From 205c5ed0fc98ef03d40420396294d743cb73d209 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 8 Nov 2019 16:00:27 -0700 Subject: [PATCH 140/324] Support model-specifc attribute thresholds * Addresses issue #142 --- scripts/wk/cfg/hw.py | 32 ++++++++++++++++++++- scripts/wk/hw/obj.py | 67 +++++++++++++++++++++++++------------------- scripts/wk/std.py | 1 + 3 files changed, 70 insertions(+), 30 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 27455bbd..0310ad42 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -2,8 +2,19 @@ # pylint: disable=bad-whitespace,line-too-long # vim: sts=2 sw=2 ts=2 +import re -KNOWN_ATTRIBUTES = { + +# STATIC VARIABLES +ATTRIBUTE_COLORS = ( + # NOTE: Ordered by ascending importance + ('Warning', 'YELLOW'), + ('Error', 'RED'), + ('Maximum', 'PURPLE'), + ) +KEY_NVME = 'nvme_smart_health_information_log' +KEY_SMART = 'ata_smart_attributes' +KNOWN_DISK_ATTRIBUTES = { # NVMe 'critical_warning': {'Blocking': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, 'media_errors': {'Blocking': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, @@ -22,6 +33,25 @@ KNOWN_ATTRIBUTES = { 199: {'Hex': 'C7', 'Blocking': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, 201: {'Hex': 'C9', 'Blocking': False, 'Warning': None, 'Error': 1, 'Maximum': 10000, }, } +KNOWN_DISK_MODELS = { + # model_regex: model_attributes + r'CT(250|500|1000|2000)MX500SSD(1|4)': { + 197: {'Warning': 1, 'Error': 2, 'Note': '(MX500 thresholds)',}, + }, + } +KNOWN_RAM_VENDOR_IDS = { + # https://github.com/hewigovens/hewigovens.github.com/wiki/Memory-vendor-code + '0x014F': 'Transcend', + '0x2C00': 'Micron', + '0x802C': 'Micron', + '0x80AD': 'Hynix', + '0x80CE': 'Samsung', + '0xAD00': 'Hynix', + '0xCE00': 'Samsung', + } +REGEX_POWER_ON_TIME = re.compile( + r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' + ) if __name__ == '__main__': diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 6395e573..7b3c5f47 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -9,34 +9,21 @@ import re from collections import OrderedDict -from wk.cfg.hw import KNOWN_ATTRIBUTES +from wk.cfg.hw import ( + ATTRIBUTE_COLORS, + KEY_NVME, + KEY_SMART, + KNOWN_DISK_ATTRIBUTES, + KNOWN_DISK_MODELS, + KNOWN_RAM_VENDOR_IDS, + REGEX_POWER_ON_TIME, + ) from wk.exe import get_json_from_command, run_program from wk.std import bytes_to_string, color_string, sleep, string_to_bytes # STATIC VARIABLES -ATTRIBUTE_COLORS = ( - # NOTE: Ordered by ascending importance - ('Warning', 'YELLOW'), - ('Error', 'RED'), - ('Maximum', 'PURPLE'), - ) -KEY_NVME = 'nvme_smart_health_information_log' -KEY_SMART = 'ata_smart_attributes' -KNOWN_RAM_VENDOR_IDS = { - # https://github.com/hewigovens/hewigovens.github.com/wiki/Memory-vendor-code - '0x014F': 'Transcend', - '0x2C00': 'Micron', - '0x802C': 'Micron', - '0x80AD': 'Hynix', - '0x80CE': 'Samsung', - '0xAD00': 'Hynix', - '0xCE00': 'Samsung', - } LOG = logging.getLogger(__name__) -REGEX_POWER_ON_TIME = re.compile( - r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' - ) # Exception Classes @@ -172,15 +159,16 @@ class Disk(BaseObj): def check_attributes(self, only_blocking=False): """Check if any known attributes are failing, returns bool.""" attributes_ok = True + known_attributes = get_known_disk_attributes(self.details['model']) for attr, value in self.attributes.items(): # Skip unknown attributes - if attr not in KNOWN_ATTRIBUTES: + if attr not in known_attributes: continue # Get thresholds - blocking_attribute = KNOWN_ATTRIBUTES[attr].get('Blocking', False) - err_thresh = KNOWN_ATTRIBUTES[attr].get('Error', None) - max_thresh = KNOWN_ATTRIBUTES[attr].get('Maximum', None) + blocking_attribute = known_attributes[attr].get('Blocking', False) + err_thresh = known_attributes[attr].get('Error', None) + max_thresh = known_attributes[attr].get('Maximum', None) if not max_thresh: max_thresh = float('inf') @@ -212,15 +200,19 @@ class Disk(BaseObj): def generate_attribute_report(self): """Generate attribute report, returns list.""" + known_attributes = get_known_disk_attributes(self.details['model']) report = [] for attr, value in sorted(self.attributes.items()): note = '' value_color = 'GREEN' # Skip attributes not in our list - if attr not in KNOWN_ATTRIBUTES: + if attr not in known_attributes: continue + # Check for attribute note + note = known_attributes[attr].get('Note', '') + # ID / Name label = f'{attr:>3}' if isinstance(attr, int): @@ -230,11 +222,11 @@ class Disk(BaseObj): # Value color for threshold, color in ATTRIBUTE_COLORS: - threshold_val = KNOWN_ATTRIBUTES[attr].get(threshold, None) + threshold_val = known_attributes[attr].get(threshold, None) if threshold_val and value['raw'] >= threshold_val: value_color = color if threshold == 'Error': - note = '(Failed)' + note = '(failed)' elif threshold == 'Maximum': note = '(invalid?)' @@ -604,6 +596,23 @@ def get_disk_serial_macos(path): return serial +def get_known_disk_attributes(model): + """Get known NVMe/SMART attributes (model specific), returns str.""" + known_attributes = KNOWN_DISK_ATTRIBUTES.copy() + + # Apply model-specific data + for regex, data in KNOWN_DISK_MODELS.items(): + if re.search(regex, model): + for attr, thresholds in data.items(): + if attr in known_attributes: + known_attributes[attr].update(thresholds) + else: + known_attributes[attr] = thresholds + + # Done + return known_attributes + + def get_ram_list_linux(): """Get RAM list using dmidecode.""" cmd = ['sudo', 'dmidecode', '--type', 'memory'] diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 0acdeb74..bb5a5b4a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -1,4 +1,5 @@ """WizardKit: Standard Functions""" +# pylint: disable=too-many-lines # vim: sts=2 sw=2 ts=2 import itertools From 27b75ab8e9c1deacbb73a0186e577bf0c885c11a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 8 Nov 2019 16:01:58 -0700 Subject: [PATCH 141/324] Added README.md with pylint config info --- scripts/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..e6854ca9 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,6 @@ +## pylint ## + +These scripts use two spaces per indent instead of the default four. As such you will need to update your pylintrc file or run like this: + +`pylint --indent-after-paren=2 --indent-string=' ' wk` + From 920f48104962fa20ada3eddf0bb4295fb40abcae Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 9 Nov 2019 14:25:22 -0700 Subject: [PATCH 142/324] Adjusted DEBUG log date format --- scripts/wk/cfg/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/cfg/log.py b/scripts/wk/cfg/log.py index 06f79df4..f724b011 100644 --- a/scripts/wk/cfg/log.py +++ b/scripts/wk/cfg/log.py @@ -5,7 +5,7 @@ DEBUG = { 'level': 'DEBUG', 'format': '[%(asctime)s %(levelname)s] [%(name)s.%(funcName)s] %(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S %z', + 'datefmt': '%Y-%m-%d %H%M%S%z', } DEFAULT = { 'level': 'INFO', From 177401ecc85feb926243d92a374f73cc2a2194c5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 9 Nov 2019 17:28:43 -0700 Subject: [PATCH 143/324] Small formatting adjustment --- scripts/wk/cfg/main.py | 1 + scripts/wk/io.py | 1 + scripts/wk/log.py | 1 + scripts/wk/net.py | 1 + 4 files changed, 4 insertions(+) diff --git a/scripts/wk/cfg/main.py b/scripts/wk/cfg/main.py index 2928f54a..aef91b61 100644 --- a/scripts/wk/cfg/main.py +++ b/scripts/wk/cfg/main.py @@ -39,5 +39,6 @@ CRASH_SERVER = { 'Headers': {'X-Requested-With': 'XMLHttpRequest'}, } + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/io.py b/scripts/wk/io.py index d7841fda..bbaa9532 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -10,6 +10,7 @@ import shutil # STATIC VARIABLES LOG = logging.getLogger(__name__) + # Functions def delete_empty_folders(path): """Recursively delete all empty folders in path.""" diff --git a/scripts/wk/log.py b/scripts/wk/log.py index f1ae4ec1..6ac43f39 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -11,6 +11,7 @@ import time from wk import cfg from wk.io import non_clobber_path + # STATIC VARIABLES if os.name == 'nt': # Example: "C:\WK\1955-11-05\WizardKit" diff --git a/scripts/wk/net.py b/scripts/wk/net.py index 509e708d..40d7e52b 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -8,6 +8,7 @@ import psutil from wk.exe import run_program from wk.std import show_data + # REGEX REGEX_VALID_IP = re.compile( r'(10.\d+.\d+.\d+' From 05d6fb762c7f619ad56697f733d53be550c205ad Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 9 Nov 2019 17:29:31 -0700 Subject: [PATCH 144/324] Added wk/tmux.py --- scripts/wk/__init__.py | 1 + scripts/wk/tmux.py | 202 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 scripts/wk/tmux.py diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 511cf8a7..2356adb5 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -13,6 +13,7 @@ from wk import net from wk import os from wk import std from wk import sw +from wk import tmux # Check env diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py new file mode 100644 index 00000000..9120a033 --- /dev/null +++ b/scripts/wk/tmux.py @@ -0,0 +1,202 @@ +"""WizardKit: tmux Functions""" +# vim: sts=2 sw=2 ts=2 + +import logging +import pathlib + +from wk.exe import run_program + + +# STATIC_VARIABLES +LOG = logging.getLogger(__name__) + + +# Functions +def capture_pane(pane_id=None): + """Capture text from current or target pane, returns str.""" + cmd = ['tmux', 'capture-pane', '-p'] + if pane_id: + cmd.extend(['-t', pane_id]) + + # Capture and return + proc = run_program(cmd, check=False) + return proc.stdout.strip() + + +def get_pane_size(pane_id=None): + """Get current or target pane size, returns tuple.""" + cmd = ['tmux', 'display', '-p'] + if pane_id: + cmd.extend(['-t', pane_id]) + cmd.append('#{pane_width} #{pane_height}') + + # Get resolution + proc = run_program(cmd, check=False) + width, height = proc.stdout.strip().split() + width = int(width) + height = int(height) + + # Done + return (width, height) + + +def kill_all_panes(pane_id=None): + """Kill all panes except for the current or target pane.""" + cmd = ['tmux', 'kill-pane', '-a'] + if pane_id: + cmd.extend(['-t', pane_id]) + + # Kill + run_program(cmd, check=False) + + +def kill_pane(*pane_ids): + """Kill pane(s) by id.""" + cmd = ['tmux', 'kill-pane', '-t'] + + # Iterate over all passed pane IDs + for pane_id in pane_ids: + run_program(cmd+[pane_id], check=False) + + +def poll_pane(pane_id): + """Check if pane exists, returns bool.""" + cmd = ['tmux', 'list-panes', '-F', '#D'] + + # Get list of panes + proc = run_program(cmd, check=False) + existant_panes = proc.stdout.splitlines() + + # Check if pane exists + return pane_id in existant_panes + + +def prep_action( + cmd=None, working_dir=None, text=None, watch_file=None, watch_cmd='cat'): + """Prep action to perform during a tmux call, returns list. + + This will prep for running a basic command, displaying text on screen, + or monitoring a file. The last option uses cat by default but can be + overridden by using the watch_cmd. + """ + action_cmd = [] + if working_dir: + action_cmd.extend(['-c', working_dir]) + + if cmd: + # Basic command + action_cmd.append(cmd) + elif text: + # Display text + action_cmd.extend([ + 'watch', + '--color', + '--exec', + '--no-title', + '--interval', '1', + 'echo', '-e', text, + ]) + elif watch_file: + # Monitor file + prep_file(watch_file) + action_cmd.extend([ + 'watch', + '--color', + '--no-title', + '--interval', '1', + ]) + if watch_cmd == 'cat': + action_cmd.append('cat') + elif watch_cmd == 'tail': + action_cmd.extend(['tail', '--follow']) + action_cmd.append(watch_file) + else: + LOG.error('No action specified') + raise RuntimeError('No action specified') + + # Done + return action_cmd + + +def prep_file(path): + """Check if file exists and create empty file if not.""" + path = pathlib.Path(path).resolve() + try: + path.touch(exist_ok=False) + except FileExistsError: + # Leave existing files alone + pass + + +def resize_pane(pane_id=None, width=None, height=None): + """Resize current or target pane.""" + cmd = ['tmux', 'resize-pane'] + + # Safety checks + if not poll_pane(pane_id): + LOG.error('tmux pane %s not found', pane_id) + raise RuntimeError(f'tmux pane {pane_id} not found') + if not (width or height): + LOG.error('Neither width nor height specified') + raise RuntimeError('Neither width nor height specified') + + # Finish building cmd + if pane_id: + cmd.extend(['-t', pane_id]) + if width: + cmd.extend(['-x', str(width)]) + if height: + cmd.extend(['-y', str(height)]) + + # Resize + run_program(cmd, check=False) + + +def split_window( + lines=None, percent=None, + behind=False, vertical=False, + target_id=None, **action): + """Split tmux window, run action, and return pane_id as str.""" + cmd = ['tmux', 'split-window', '-d', '-PF', '#D'] + pane_id = None + + # Safety checks + if not (lines or percent): + LOG.error('Neither lines nor percent specified') + raise RuntimeError('Neither lines nor percent specified') + + # New pane placement + if behind: + cmd.append('-b') + if vertical: + cmd.append('-v') + else: + cmd.append('-h') + if target_id: + cmd.extend(['-t', target_id]) + + # New pane size + if lines: + cmd.extend(['-l', str(lines)]) + elif percent: + cmd.extend(['-p', str(percent)]) + + # New pane action + cmd.extend(prep_action(**action)) + + # Run and return pane_id + proc = run_program(cmd, check=False) + return proc.stdout.strip() + + +def respawn_pane(pane_id, **action): + """Respawn pane with action.""" + cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + cmd.extend(prep_action(**action)) + + # Respawn + run_program(cmd, check=False) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From 8f663072f6549c33dd8345615e17d963549e43e5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 9 Nov 2019 19:25:30 -0700 Subject: [PATCH 145/324] Added HW-Diags audio test, menu, and launcher --- scripts/hw-diags | 11 +++ scripts/hw-diags.py | 25 +++++++ scripts/wk/hw/__init__.py | 1 + scripts/wk/hw/diags.py | 143 ++++++++++++++++++++++++++++++++++++++ scripts/wk/tmux.py | 1 - 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100755 scripts/hw-diags create mode 100755 scripts/hw-diags.py create mode 100644 scripts/wk/hw/diags.py diff --git a/scripts/hw-diags b/scripts/hw-diags new file mode 100755 index 00000000..feb87fc8 --- /dev/null +++ b/scripts/hw-diags @@ -0,0 +1,11 @@ +#!/bin/bash +# +## Wizard Kit: HW Diagnostics Launcher + +source ./launch-in-tmux + +SESSION_NAME="hw-diags" +WINDOW_NAME="Hardware Diagnostics" +TMUX_CMD="./hw-diags.py" + +launch_in_tmux "$@" diff --git a/scripts/hw-diags.py b/scripts/hw-diags.py new file mode 100755 index 00000000..26ba499f --- /dev/null +++ b/scripts/hw-diags.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Wizard Kit: Hardware Diagnostics""" +# vim: sts=2 sw=2 ts=2 + +import wk + + +def main(): + """Run hardware diagnostics.""" + state = wk.hw.diags.State() + wk.hw.diags.main_menu() + + # Done + print('') + print('Done.') + wk.std.pause('Press Enter to exit...') + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/hw/__init__.py b/scripts/wk/hw/__init__.py index dbc9ea7e..b28bffad 100644 --- a/scripts/wk/hw/__init__.py +++ b/scripts/wk/hw/__init__.py @@ -1,3 +1,4 @@ """WizardKit: hw module init""" +from wk.hw import diags from wk.hw import obj diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py new file mode 100644 index 00000000..a27ba087 --- /dev/null +++ b/scripts/wk/hw/diags.py @@ -0,0 +1,143 @@ +"""WizardKit: Hardware diagnostics""" +# vim: sts=2 sw=2 ts=2 + +import logging +import pathlib +import platform + +from collections import OrderedDict +from docopt import docopt + +from wk.cfg.main import KIT_NAME_FULL +from wk.exe import run_program +from wk.std import ( + Menu, + clear_screen, + color_string, + pause, + print_error, + print_info, + print_standard, + print_warning, + sleep, + ) + + +# STATIC VARIABLES +DOCSTRING = f'''{KIT_NAME_FULL}: Hardware Diagnostics + +Usage: + hw-diags + hw-diags (-q | --quick) + hw-diags (-h | --help) + +Options: + -h --help Show this page + -q --quick Skip menu and perform a quick check +''' +LOG = logging.getLogger(__name__) +MENU_ACTIONS = ( + 'Audio Test', + 'Keyboard Test', + 'Network Test', + 'Start', + 'Quit') +MENU_OPTIONS = ( + 'CPU & Cooling', + 'Disk Attributes', + 'Disk Self-Test', + 'Disk Surface Scan', + 'Disk I/O Benchmark', +) +MENU_OPTIONS_QUICK = ('Disk Attributes',) +MENU_SETS = { + 'Full Diagnostic': (*MENU_OPTIONS,), + 'Disk Diagnostic': ( + 'Disk Attributes', + 'Disk Self-Test', + 'Disk Surface Scan', + 'Disk I/O Benchmark', + ), + 'Disk Diagnostic (Quick)': ('Disk Attributes',), +} +MENU_TOGGLES = [] + + +# Classes +class State(): + """Object for tracking hardware diagnostic data.""" + def __init__(self): + self.tests = OrderedDict() + + +# Functions +def audio_test(): + """Run an OS-specific audio test.""" + if platform.system() == 'Linux': + audio_test_linux() + # TODO: Add tests for other OS + + +def audio_test_linux(): + """Run an audio test using amixer and speaker-test.""" + clear_screen() + print_standard('Audio test') + print_standard('') + + # Set volume + for source in ('Master', 'PCM'): + cmd = f'amixer -q set "{source}" 80% unmute'.split() + run_program(cmd, check=False) + + # Run audio tests + for mode in ('pink', 'wav'): + cmd = f'speaker-test -c 2 -l 1 -t {mode}'.split() + run_program(cmd, check=False, pipe=False) + + +def main_menu(): + """Main menu for hardware diagnostics.""" + args = docopt(DOCSTRING) + menu = Menu() + + # Build menu + menu.title = color_string( + strings=['Hardware Diagnostics', 'Main Menu'], + colors=['GREEN', None], + sep='\n', + ) + for action in MENU_ACTIONS: + menu.add_action(action) + for option in MENU_OPTIONS: + menu.add_option(option, {'Selected': True}) + for toggle in MENU_TOGGLES: + menu.add_toggle(toggle, {'Selected': True}) + for name, targets in MENU_SETS.items(): + menu.add_set(name, {'Targets': targets}) + menu.actions['Start']['Separator'] = True + + # Check if running a quick check + if args['--quick']: + for name in menu.options.keys(): + # Only select quick option(s) + menu.options[name]['Selected'] = name in MENU_OPTIONS_QUICK + + # Compatibility checks + if platform.system() != 'Linux': + for name in ('Audio Test', 'Keyboard Test', 'Network Test'): + menu.actions[name]['Disabled'] = True + + # Show menu + while True: + selection = menu.advanced_select() + if 'Audio Test' in selection: + audio_test() + elif 'Quit' in selection: + break + print(f'Sel: {selection}') + print('') + pause() + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 9120a033..186878d9 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -158,7 +158,6 @@ def split_window( target_id=None, **action): """Split tmux window, run action, and return pane_id as str.""" cmd = ['tmux', 'split-window', '-d', '-PF', '#D'] - pane_id = None # Safety checks if not (lines or percent): From 2b06375f713c44eecaa2c7803d345789151e71dc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 9 Nov 2019 19:35:23 -0700 Subject: [PATCH 146/324] Removed unused hw-diags and hw-diags-audio scripts --- scripts/outer_scripts_to_review/hw-diags | 11 ----- .../outer_scripts_to_review/hw-diags-audio | 42 ------------------- 2 files changed, 53 deletions(-) delete mode 100755 scripts/outer_scripts_to_review/hw-diags delete mode 100755 scripts/outer_scripts_to_review/hw-diags-audio diff --git a/scripts/outer_scripts_to_review/hw-diags b/scripts/outer_scripts_to_review/hw-diags deleted file mode 100755 index 70f84db4..00000000 --- a/scripts/outer_scripts_to_review/hw-diags +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -## Wizard Kit: HW Diagnostics Launcher - -source launch-in-tmux - -SESSION_NAME="hw-diags" -WINDOW_NAME="Hardware Diagnostics" -TMUX_CMD="hw-diags-menu" - -launch_in_tmux "$@" diff --git a/scripts/outer_scripts_to_review/hw-diags-audio b/scripts/outer_scripts_to_review/hw-diags-audio deleted file mode 100755 index e581330f..00000000 --- a/scripts/outer_scripts_to_review/hw-diags-audio +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: HW Diagnostics - Audio - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.common import * -init_global_vars() - -if __name__ == '__main__': - try: - # Prep - clear_screen() - print_standard('Hardware Diagnostics: Audio\n') - - # Set volume - try: - run_program('amixer -q set "Master" 80% unmute'.split()) - run_program('amixer -q set "PCM" 90% unmute'.split()) - except subprocess.CalledProcessError: - print_error('Failed to set volume') - - # Run tests - for mode in ['pink', 'wav']: - run_program( - cmd = 'speaker-test -c 2 -l 1 -t {}'.format(mode).split(), - check = False, - pipe = False) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 From ce3a98028aff480face132ff34d6befb7f048116 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 14:29:55 -0700 Subject: [PATCH 147/324] Fixed wk.std.show_data() alignment --- scripts/wk/std.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index bb5a5b4a..6c2862a8 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -907,6 +907,8 @@ def show_data(message, data, color=None): print_colored( (f'{" "*INDENT}{message:<{WIDTH}}', data), colors, + log=True, + sep='', ) From 100757ba696c312cbb8514331515370c64e9e906 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 14:47:56 -0700 Subject: [PATCH 148/324] Added network_test() --- scripts/hw-diags.py | 5 ----- scripts/wk/hw/diags.py | 41 ++++++++++++++++++++++++++++++++++++++--- scripts/wk/net.py | 17 +++++++++++------ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/scripts/hw-diags.py b/scripts/hw-diags.py index 26ba499f..b70413ea 100755 --- a/scripts/hw-diags.py +++ b/scripts/hw-diags.py @@ -10,11 +10,6 @@ def main(): state = wk.hw.diags.State() wk.hw.diags.main_menu() - # Done - print('') - print('Done.') - wk.std.pause('Press Enter to exit...') - if __name__ == '__main__': try: diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index a27ba087..2f80e1e3 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -10,8 +10,15 @@ from docopt import docopt from wk.cfg.main import KIT_NAME_FULL from wk.exe import run_program +from wk.net import ( + connected_to_private_network, + ping, + show_valid_addresses, + speedtest, + ) from wk.std import ( Menu, + TryAndPrint, clear_screen, color_string, pause, @@ -132,11 +139,39 @@ def main_menu(): selection = menu.advanced_select() if 'Audio Test' in selection: audio_test() + if 'Network Test' in selection: + network_test() elif 'Quit' in selection: break - print(f'Sel: {selection}') - print('') - pause() + + +def network_test(): + """Run network tests.""" + clear_screen() + try_and_print = TryAndPrint() + result = try_and_print.run( + 'Network connection...', connected_to_private_network, msg_good='OK') + + # Bail if not connected + if result['Failed']: + print_warning('Please connect to a network and try again') + pause('Press Enter to return to main menu...') + return + + # Show IP address(es) + show_valid_addresses() + + # Ping tests + try_and_print.run( + 'Internet connection...', ping, msg_good='OK', addr='8.8.8.8') + try_and_print.run( + 'DNS resolution...', ping, msg_good='OK', addr='google.com') + + # Speedtest + try_and_print.run('Speedtest...', speedtest) + + # Done + pause('Press Enter to return to main menu...') if __name__ == '__main__': diff --git a/scripts/wk/net.py b/scripts/wk/net.py index 40d7e52b..69e9a345 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -6,7 +6,7 @@ import re import psutil from wk.exe import run_program -from wk.std import show_data +from wk.std import GenericError, show_data # REGEX @@ -18,16 +18,21 @@ REGEX_VALID_IP = re.compile( # Functions -def is_connected(): - """Check for a valid private IP.""" +def connected_to_private_network(): + """Check if connected to a private network. + + This checks for a valid private IP assigned to this system. + If one isn't found then an exception is raised. + """ devs = psutil.net_if_addrs() for dev in devs.values(): for family in dev: if REGEX_VALID_IP.search(family.address): # Valid IP found - return True - # Else - return False + return + + # No valid IP found + raise GenericError('Not connected to a network') def ping(addr='google.com'): From fe228a5edcff01f04ea611b9b5ac90df16d42446 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 15:17:00 -0700 Subject: [PATCH 149/324] Added keyboard_test() --- scripts/wk/hw/diags.py | 59 +++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 2f80e1e3..4432e437 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -102,17 +102,18 @@ def audio_test_linux(): run_program(cmd, check=False, pipe=False) -def main_menu(): - """Main menu for hardware diagnostics.""" - args = docopt(DOCSTRING) +def build_menu(quick_mode=False): + """Build main menu, returns wk.std.Menu.""" menu = Menu() - # Build menu + # Set title menu.title = color_string( strings=['Hardware Diagnostics', 'Main Menu'], colors=['GREEN', None], sep='\n', ) + + # Add actions, options, etc for action in MENU_ACTIONS: menu.add_action(action) for option in MENU_OPTIONS: @@ -123,8 +124,8 @@ def main_menu(): menu.add_set(name, {'Targets': targets}) menu.actions['Start']['Separator'] = True - # Check if running a quick check - if args['--quick']: + # Update default selections for quick mode if necessary + if quick_mode: for name in menu.options.keys(): # Only select quick option(s) menu.options[name]['Selected'] = name in MENU_OPTIONS_QUICK @@ -134,16 +135,54 @@ def main_menu(): for name in ('Audio Test', 'Keyboard Test', 'Network Test'): menu.actions[name]['Disabled'] = True + # Done + return menu + + +def keyboard_test(): + """Test keyboard using xev.""" + cmd = ['xev', '-event', 'keyboard'] + clear_screen() + run_program(cmd, check=False, pipe=False) + + +def main_menu(): + """Main menu for hardware diagnostics.""" + args = docopt(DOCSTRING) + menu = build_menu(args['--quick']) + # Show menu while True: + action = None selection = menu.advanced_select() + + # Set action if 'Audio Test' in selection: - audio_test() - if 'Network Test' in selection: - network_test() - elif 'Quit' in selection: + action = audio_test + elif 'Keyboard Test' in selection: + action = keyboard_test + elif 'Network Test' in selection: + action = network_test + + # Run simple test + if action: + try: + action() + except KeyboardInterrupt: + print_warning('Aborted.') + print_standard('') + pause('Press Enter to return to main menu...') + + # Quit + if 'Quit' in selection: break + # Start diagnostics + if 'Start' in selection: + #TODO + #run_diags() + pass + def network_test(): """Run network tests.""" From 2520126905c71f7edcb78e0cff50ae8a0bc773a1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 16:22:04 -0700 Subject: [PATCH 150/324] Use broader imports for wk/hw/diags.py --- scripts/hw-diags.py | 2 +- scripts/wk/hw/diags.py | 68 ++++++++++++++++-------------------------- 2 files changed, 26 insertions(+), 44 deletions(-) diff --git a/scripts/hw-diags.py b/scripts/hw-diags.py index b70413ea..6b34f324 100755 --- a/scripts/hw-diags.py +++ b/scripts/hw-diags.py @@ -8,7 +8,7 @@ import wk def main(): """Run hardware diagnostics.""" state = wk.hw.diags.State() - wk.hw.diags.main_menu() + wk.hw.diags.main() if __name__ == '__main__': diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 4432e437..aa0e43c2 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -8,26 +8,8 @@ import platform from collections import OrderedDict from docopt import docopt +from wk import exe, net, std from wk.cfg.main import KIT_NAME_FULL -from wk.exe import run_program -from wk.net import ( - connected_to_private_network, - ping, - show_valid_addresses, - speedtest, - ) -from wk.std import ( - Menu, - TryAndPrint, - clear_screen, - color_string, - pause, - print_error, - print_info, - print_standard, - print_warning, - sleep, - ) # STATIC VARIABLES @@ -87,27 +69,27 @@ def audio_test(): def audio_test_linux(): """Run an audio test using amixer and speaker-test.""" - clear_screen() - print_standard('Audio test') - print_standard('') + std.clear_screen() + std.print_standard('Audio test') + std.print_standard('') # Set volume for source in ('Master', 'PCM'): cmd = f'amixer -q set "{source}" 80% unmute'.split() - run_program(cmd, check=False) + exe.run_program(cmd, check=False) # Run audio tests for mode in ('pink', 'wav'): cmd = f'speaker-test -c 2 -l 1 -t {mode}'.split() - run_program(cmd, check=False, pipe=False) + exe.run_program(cmd, check=False, pipe=False) def build_menu(quick_mode=False): """Build main menu, returns wk.std.Menu.""" - menu = Menu() + menu = std.Menu() # Set title - menu.title = color_string( + menu.title = std.color_string( strings=['Hardware Diagnostics', 'Main Menu'], colors=['GREEN', None], sep='\n', @@ -142,12 +124,12 @@ def build_menu(quick_mode=False): def keyboard_test(): """Test keyboard using xev.""" cmd = ['xev', '-event', 'keyboard'] - clear_screen() - run_program(cmd, check=False, pipe=False) + std.clear_screen() + exe.run_program(cmd, check=False, pipe=False) -def main_menu(): - """Main menu for hardware diagnostics.""" +def main(): + """Main function for hardware diagnostics.""" args = docopt(DOCSTRING) menu = build_menu(args['--quick']) @@ -169,9 +151,9 @@ def main_menu(): try: action() except KeyboardInterrupt: - print_warning('Aborted.') - print_standard('') - pause('Press Enter to return to main menu...') + std.print_warning('Aborted.') + std.print_standard('') + std.pause('Press Enter to return to main menu...') # Quit if 'Quit' in selection: @@ -186,31 +168,31 @@ def main_menu(): def network_test(): """Run network tests.""" - clear_screen() - try_and_print = TryAndPrint() + std.clear_screen() + try_and_print = std.TryAndPrint() result = try_and_print.run( - 'Network connection...', connected_to_private_network, msg_good='OK') + 'Network connection...', net.connected_to_private_network, msg_good='OK') # Bail if not connected if result['Failed']: - print_warning('Please connect to a network and try again') - pause('Press Enter to return to main menu...') + std.print_warning('Please connect to a network and try again') + std.pause('Press Enter to return to main menu...') return # Show IP address(es) - show_valid_addresses() + net.show_valid_addresses() # Ping tests try_and_print.run( - 'Internet connection...', ping, msg_good='OK', addr='8.8.8.8') + 'Internet connection...', net.ping, msg_good='OK', addr='8.8.8.8') try_and_print.run( - 'DNS resolution...', ping, msg_good='OK', addr='google.com') + 'DNS resolution...', net.ping, msg_good='OK', addr='google.com') # Speedtest - try_and_print.run('Speedtest...', speedtest) + try_and_print.run('Speedtest...', net.speedtest) # Done - pause('Press Enter to return to main menu...') + std.pause('Press Enter to return to main menu...') if __name__ == '__main__': From 76a501af85930421ce107fba1c8e54878baa088a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 17:04:45 -0700 Subject: [PATCH 151/324] Added State() and tmux sections to wk/hw/diags --- scripts/wk/cfg/hw.py | 1 + scripts/wk/hw/diags.py | 118 ++++++++++++++++++++++++++++++++++++----- scripts/wk/std.py | 7 +-- 3 files changed, 110 insertions(+), 16 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 0310ad42..2247b8dc 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -52,6 +52,7 @@ KNOWN_RAM_VENDOR_IDS = { REGEX_POWER_ON_TIME = re.compile( r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' ) +TMUX_SIDE_WIDTH = 20 if __name__ == '__main__': diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index aa0e43c2..e61ee054 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -4,11 +4,13 @@ import logging import pathlib import platform +import time from collections import OrderedDict from docopt import docopt -from wk import exe, net, std +from wk import exe, net, std, tmux +from wk.cfg.hw import TMUX_SIDE_WIDTH from wk.cfg.main import KIT_NAME_FULL @@ -49,14 +51,80 @@ MENU_SETS = { ), 'Disk Diagnostic (Quick)': ('Disk Attributes',), } -MENU_TOGGLES = [] +MENU_TOGGLES = ( + 'Skip USB Benchmarks', + ) # Classes class State(): """Object for tracking hardware diagnostic data.""" def __init__(self): - self.tests = OrderedDict() + self.cpu = None + self.disks = [] + self.panes = {} + self.tests = OrderedDict({ + 'CPU & Cooling': { + 'Enabled': False, + 'Function': cpu_mprime_test, + 'Objects': [], + }, + 'Disk Attributes': { + 'Enabled': False, + 'Function': disk_attribute_check, + 'Objects': [], + }, + 'Disk Self-Test': { + 'Enabled': False, + 'Function': disk_self_test, + 'Objects': [], + }, + 'Disk Surface Scan': { + 'Enabled': False, + 'Function': disk_surface_scan, + 'Objects': [], + }, + 'Disk I/O Benchmark': { + 'Enabled': False, + 'Function': disk_io_benchmark, + 'Objects': [], + }, + }) + self.top_text = std.color_string('Hardware Diagnostics', 'GREEN') + self.init_tmux() + + def init_tmux(self): + """Initialize tmux layout.""" + tmux.kill_all_panes() + + # Top + self.panes['Top'] = tmux.split_window( + behind=True, + lines=2, + vertical=True, + text=self.top_text, + ) + + # Started + self.panes['Started'] = tmux.split_window( + lines=TMUX_SIDE_WIDTH, + target_id=self.panes['Top'], + text=std.color_string( + ['Started', time.strftime("%Y-%m-%d %H:%M %Z")], + ['BLUE', None], + sep='\n', + ), + ) + + # Progress + self.panes['Progress'] = tmux.split_window( + lines=TMUX_SIDE_WIDTH, + text=' ', + ) + + def update_top_pane(self, text): + """Update top pane with text.""" + tmux.respawn_pane(self.panes['Top'], text=f'{self.top_text}\n{text}') # Functions @@ -70,8 +138,6 @@ def audio_test(): def audio_test_linux(): """Run an audio test using amixer and speaker-test.""" std.clear_screen() - std.print_standard('Audio test') - std.print_standard('') # Set volume for source in ('Master', 'PCM'): @@ -86,14 +152,7 @@ def audio_test_linux(): def build_menu(quick_mode=False): """Build main menu, returns wk.std.Menu.""" - menu = std.Menu() - - # Set title - menu.title = std.color_string( - strings=['Hardware Diagnostics', 'Main Menu'], - colors=['GREEN', None], - sep='\n', - ) + menu = std.Menu(title=None) # Add actions, options, etc for action in MENU_ACTIONS: @@ -121,6 +180,36 @@ def build_menu(quick_mode=False): return menu +def cpu_mprime_test(): + """CPU & cooling check using Prime95.""" + #TODO: p95 + std.print_warning('TODO: p95') + + +def disk_attribute_check(): + """Disk attribute check.""" + #TODO: at + std.print_warning('TODO: at') + + +def disk_io_benchmark(): + """Disk I/O benchmark using dd.""" + #TODO: io + std.print_warning('TODO: io') + + +def disk_self_test(): + """Disk self-test if available.""" + #TODO: st + std.print_warning('TODO: st') + + +def disk_surface_scan(): + """Disk surface scan using badblocks.""" + #TODO: bb + std.print_warning('TODO: bb') + + def keyboard_test(): """Test keyboard using xev.""" cmd = ['xev', '-event', 'keyboard'] @@ -132,9 +221,11 @@ def main(): """Main function for hardware diagnostics.""" args = docopt(DOCSTRING) menu = build_menu(args['--quick']) + state = State() # Show menu while True: + state.update_top_pane('Main Menu') action = None selection = menu.advanced_select() @@ -148,6 +239,7 @@ def main(): # Run simple test if action: + state.update_top_pane(selection[0]) try: action() except KeyboardInterrupt: diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 6c2862a8..2428b168 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -87,7 +87,7 @@ class Menu(): def _generate_menu_text(self): """Generate menu text, returns str.""" separator_string = self._get_separator_string() - menu_lines = [self.title, separator_string] + menu_lines = [self.title, separator_string] if self.title else [] # Sets & toggles for section in (self.sets, self.toggles): @@ -154,8 +154,9 @@ class Menu(): separator_length = 0 # Check title line(s) - for line in self.title.split('\n'): - separator_length = max(separator_length, len(strip_colors(line))) + if self.title: + for line in self.title.split('\n'): + separator_length = max(separator_length, len(strip_colors(line))) # Loop over all item names for section in (self.actions, self.options, self.sets, self.toggles): From 964885d63cf3d92b52af2ed321aba2c4a7a5295c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 17:38:03 -0700 Subject: [PATCH 152/324] Ensure tmux panes are closed atexit for hw-diags --- scripts/hw-diags.py | 8 +------- scripts/wk/hw/diags.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/hw-diags.py b/scripts/hw-diags.py index 6b34f324..e3719875 100755 --- a/scripts/hw-diags.py +++ b/scripts/hw-diags.py @@ -5,15 +5,9 @@ import wk -def main(): - """Run hardware diagnostics.""" - state = wk.hw.diags.State() - wk.hw.diags.main() - - if __name__ == '__main__': try: - main() + wk.hw.diags.main() except SystemExit: raise except: #pylint: disable=bare-except diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index e61ee054..d1667c01 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -1,6 +1,7 @@ """WizardKit: Hardware diagnostics""" # vim: sts=2 sw=2 ts=2 +import atexit import logging import pathlib import platform @@ -14,6 +15,9 @@ from wk.cfg.hw import TMUX_SIDE_WIDTH from wk.cfg.main import KIT_NAME_FULL +# atexit functions +atexit.register(tmux.kill_all_panes) + # STATIC VARIABLES DOCSTRING = f'''{KIT_NAME_FULL}: Hardware Diagnostics @@ -102,7 +106,7 @@ class State(): behind=True, lines=2, vertical=True, - text=self.top_text, + text=f'{self.top_text}\nMain Menu', ) # Started @@ -225,7 +229,6 @@ def main(): # Show menu while True: - state.update_top_pane('Main Menu') action = None selection = menu.advanced_select() @@ -257,6 +260,9 @@ def main(): #run_diags() pass + # Reset top pane + state.update_top_pane('Main Menu') + def network_test(): """Run network tests.""" From 0b6cd1cb6cae3992dc931dfe1557046e4ac49b3d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 17:42:04 -0700 Subject: [PATCH 153/324] Added secret menu options in hw-diags --- scripts/wk/hw/diags.py | 37 +++++++++++++++++++++++++++++++++++++ scripts/wk/tmux.py | 10 ++++++++++ 2 files changed, 47 insertions(+) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index d1667c01..9c31b24f 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -37,6 +37,10 @@ MENU_ACTIONS = ( 'Network Test', 'Start', 'Quit') +MENU_ACTIONS_SECRET = ( + 'Matrix', + 'Tubes', + ) MENU_OPTIONS = ( 'CPU & Cooling', 'Disk Attributes', @@ -161,6 +165,8 @@ def build_menu(quick_mode=False): # Add actions, options, etc for action in MENU_ACTIONS: menu.add_action(action) + for action in MENU_ACTIONS_SECRET: + menu.add_action(action, {'Hidden': True}) for option in MENU_OPTIONS: menu.add_option(option, {'Selected': True}) for toggle in MENU_TOGGLES: @@ -179,6 +185,9 @@ def build_menu(quick_mode=False): if platform.system() != 'Linux': for name in ('Audio Test', 'Keyboard Test', 'Network Test'): menu.actions[name]['Disabled'] = True + if platform.system() not in ('Darwin', 'Linux'): + for name in ('Matrix', 'Tubes'): + menu.actions[name]['Disabled'] = True # Done return menu @@ -250,6 +259,13 @@ def main(): std.print_standard('') std.pause('Press Enter to return to main menu...') + # Secrets + if 'Matrix' in selection: + screensaver('matrix') + elif 'Tubes' in selection: + # Tubes ≈≈ Pipes? + screensaver('pipes') + # Quit if 'Quit' in selection: break @@ -293,5 +309,26 @@ def network_test(): std.pause('Press Enter to return to main menu...') +def screensaver(name): + """Show screensaver""" + if name == 'matrix': + cmd = ['cmatrix', '-abs'] + elif name == 'pipes': + cmd = [ + 'pipes' if platform.system() == 'Linux' else 'pipes.sh', + '-t', '0', + '-t', '1', + '-t', '2', + '-t', '3', + '-t', '5', + '-R', '-r', '4000', + ] + + # Switch pane to fullscreen and start screensaver + tmux.zoom_pane() + exe.run_program(cmd, check=False, pipe=False) + tmux.zoom_pane() + + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 186878d9..525b42a8 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -197,5 +197,15 @@ def respawn_pane(pane_id, **action): run_program(cmd, check=False) +def zoom_pane(pane_id=None): + """Toggle zoom status for current or target pane.""" + cmd = ['tmux', 'resize-pane', '-Z'] + if pane_id: + cmd.extend(['-t', pane_id]) + + # Toggle + run_program(cmd, check=False) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 6963d2ae717deaf90868df2ad5ab84538a4201fd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 18:01:07 -0700 Subject: [PATCH 154/324] Fix echo usage under macOS --- scripts/wk/tmux.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 525b42a8..2b726c2a 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -3,6 +3,7 @@ import logging import pathlib +import platform from wk.exe import run_program @@ -88,14 +89,18 @@ def prep_action( action_cmd.append(cmd) elif text: # Display text + echo_cmd = ['echo'] + if platform.system() == 'Linux': + echo_cmd.append('-e') action_cmd.extend([ 'watch', '--color', '--exec', '--no-title', '--interval', '1', - 'echo', '-e', text, ]) + action_cmd.extend(echo_cmd) + action_cmd.append(text) elif watch_file: # Monitor file prep_file(watch_file) From 0cbc858cf44381deae74bd58370cae5806eed220 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 18:05:06 -0700 Subject: [PATCH 155/324] Intentionally crash if not running inside tmux --- scripts/wk/hw/diags.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 9c31b24f..6405cf05 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -3,6 +3,7 @@ import atexit import logging +import os import pathlib import platform import time @@ -233,6 +234,13 @@ def keyboard_test(): def main(): """Main function for hardware diagnostics.""" args = docopt(DOCSTRING) + + # Safety check + if 'TMUX' not in os.environ: + LOG.error('tmux session not found') + raise RuntimeError('tmux session not found') + + # Init menu = build_menu(args['--quick']) state = State() From 196e2adc824459e828f7f09a5eb59ad388b44c47 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 19:10:35 -0700 Subject: [PATCH 156/324] Added tmux layout maintenance sections * Support both threading and signal based calls * Should provide a smoother UIX under Linux & macOS --- scripts/wk/cfg/hw.py | 13 ++++++++++ scripts/wk/hw/diags.py | 32 +++++++++++++++++++++++- scripts/wk/tmux.py | 56 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 2247b8dc..1e848eb6 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -4,6 +4,8 @@ import re +from collections import OrderedDict + # STATIC VARIABLES ATTRIBUTE_COLORS = ( @@ -53,6 +55,17 @@ REGEX_POWER_ON_TIME = re.compile( r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' ) TMUX_SIDE_WIDTH = 20 +TMUX_LAYOUT = OrderedDict({ + 'Top': {'height': 2, 'Check': True}, + 'Started': {'width': TMUX_SIDE_WIDTH, 'Check': True}, + 'Progress': {'width': TMUX_SIDE_WIDTH, 'Check': True}, + # Testing panes + 'Prime95': {'height': 11, 'Check': False}, + 'Temps': {'height': 1000, 'Check': False}, + 'SMART': {'height': 3, 'Check': True}, + 'badblocks': {'height': 5, 'Check': True}, + 'I/O Benchmark': {'height': 1000, 'Check': False}, + }) if __name__ == '__main__': diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 6405cf05..8e8a301f 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -6,13 +6,14 @@ import logging import os import pathlib import platform +import signal import time from collections import OrderedDict from docopt import docopt from wk import exe, net, std, tmux -from wk.cfg.hw import TMUX_SIDE_WIDTH +from wk.cfg.hw import TMUX_LAYOUT, TMUX_SIDE_WIDTH from wk.cfg.main import KIT_NAME_FULL @@ -100,7 +101,36 @@ class State(): }, }) self.top_text = std.color_string('Hardware Diagnostics', 'GREEN') + + # Init tmux and start a background process to maintain layout self.init_tmux() + if hasattr(signal, 'SIGWINCH'): + # Use signal handling + signal.signal(signal.SIGWINCH, self.fix_tmux_layout) + else: + exe.start_thread(self.fix_tmux_layout_loop) + + def fix_tmux_layout(self, forced=True, signum=None, frame=None): + # pylint: disable=unused-argument + """Fix tmux layout based on TMUX_LAYOUT. + + NOTE: To support being called by both a signal and a thread + signum and frame must be valid aguments. + """ + try: + tmux.fix_layout(self.panes, TMUX_LAYOUT, forced=forced) + except RuntimeError: + # Assuming self.panes changed while running + pass + + def fix_tmux_layout_loop(self): + """Fix tmux layout on a loop. + + NOTE: This should be called as a thread. + """ + while True: + self.fix_tmux_layout(forced=False) + std.sleep(1) def init_tmux(self): """Initialize tmux layout.""" diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 2b726c2a..84068ab5 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -24,6 +24,27 @@ def capture_pane(pane_id=None): return proc.stdout.strip() +def fix_layout(panes, layout, forced=False): + """Fix pane sizes based on layout.""" + if not (forced or layout_needs_fixed(panes, layout)): + # Layout should be fine + return + + # Update panes + for name, data in layout.items(): + # Skip missing panes + if name not in panes: + continue + + # Resize pane + pane_id = panes[name] + try: + resize_pane(pane_id, **data) + except RuntimeError: + # Assuming pane was closed just before resizing + pass + + def get_pane_size(pane_id=None): """Get current or target pane size, returns tuple.""" cmd = ['tmux', 'display', '-p'] @@ -60,6 +81,32 @@ def kill_pane(*pane_ids): run_program(cmd+[pane_id], check=False) +def layout_needs_fixed(panes, layout): + """Check if layout needs fixed, returns bool.""" + needs_fixed = False + + # Check panes + for name, data in layout.items(): + # Skip unpredictably sized panes + if not data.get('Check', False): + continue + + # Skip missing panes + if name not in panes: + continue + + # Check pane size + pane_id = panes[name] + width, height = get_pane_size(pane_id) + if data.get('width', False) and data['width'] != width: + needs_fixed = True + if data.get('height', False) and data['height'] != height: + needs_fixed = True + + # Done + return needs_fixed + + def poll_pane(pane_id): """Check if pane exists, returns bool.""" cmd = ['tmux', 'list-panes', '-F', '#D'] @@ -133,8 +180,13 @@ def prep_file(path): pass -def resize_pane(pane_id=None, width=None, height=None): - """Resize current or target pane.""" +def resize_pane(pane_id=None, width=None, height=None, **kwargs): + # pylint: disable=unused-argument + """Resize current or target pane. + + NOTE: kwargs is only here to make calling this function easier + by dropping any extra kwargs passed. + """ cmd = ['tmux', 'resize-pane'] # Safety checks From 906826d752eec11ec1913982256a9875e093e352 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 20:21:15 -0700 Subject: [PATCH 157/324] Updated TryAndPrint() * Don't log function name unless in debug mode * Log msg_good instead of UNKNOWN for non-failed functions with no output * Avoid issue if function returns int --- scripts/wk/std.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 2428b168..a9b5d54a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -430,7 +430,10 @@ class TryAndPrint(): stdout = stdout.decode('utf8') output = stdout.strip().splitlines() else: - output = list(output) + try: + output = list(output) + except TypeError: + output = [output] # Safety check if not output: @@ -524,7 +527,7 @@ class TryAndPrint(): # Run function and catch exceptions print(f'{" "*self.indent}{message:<{self.width}}', end='', flush=True) - LOG.info('Running function: %s.%s', function.__module__, function.__name__) + LOG.debug('Running function: %s.%s', function.__module__, function.__name__) try: output = function(*args, **kwargs) except w_exceptions as _exception: @@ -554,7 +557,8 @@ class TryAndPrint(): result_msg = self._format_function_output(output) print(result_msg) else: - print_success(msg_good if msg_good else self.msg_good, log=False) + result_msg = msg_good if msg_good else self.msg_good + print_success(result_msg, log=False) # Done self._log_result(message, result_msg) From 21dfeac20b50fae743d02cafe5dbd7c1a292f32d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 20:22:48 -0700 Subject: [PATCH 158/324] Expanded logging in wk.hw.diags --- scripts/wk/hw/diags.py | 25 +++++++++++++++++-------- scripts/wk/tmux.py | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 8e8a301f..c7dcb333 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -12,16 +12,15 @@ import time from collections import OrderedDict from docopt import docopt -from wk import exe, net, std, tmux -from wk.cfg.hw import TMUX_LAYOUT, TMUX_SIDE_WIDTH -from wk.cfg.main import KIT_NAME_FULL +from wk import cfg, exe, log, net, std, tmux # atexit functions atexit.register(tmux.kill_all_panes) +#TODO: Add state/dev data dump debug function # STATIC VARIABLES -DOCSTRING = f'''{KIT_NAME_FULL}: Hardware Diagnostics +DOCSTRING = f'''{cfg.main.KIT_NAME_FULL}: Hardware Diagnostics Usage: hw-diags @@ -112,13 +111,13 @@ class State(): def fix_tmux_layout(self, forced=True, signum=None, frame=None): # pylint: disable=unused-argument - """Fix tmux layout based on TMUX_LAYOUT. + """Fix tmux layout based on cfg.hw.TMUX_LAYOUT. NOTE: To support being called by both a signal and a thread signum and frame must be valid aguments. """ try: - tmux.fix_layout(self.panes, TMUX_LAYOUT, forced=forced) + tmux.fix_layout(self.panes, cfg.hw.TMUX_LAYOUT, forced=forced) except RuntimeError: # Assuming self.panes changed while running pass @@ -146,7 +145,7 @@ class State(): # Started self.panes['Started'] = tmux.split_window( - lines=TMUX_SIDE_WIDTH, + lines=cfg.hw.TMUX_SIDE_WIDTH, target_id=self.panes['Top'], text=std.color_string( ['Started', time.strftime("%Y-%m-%d %H:%M %Z")], @@ -157,7 +156,7 @@ class State(): # Progress self.panes['Progress'] = tmux.split_window( - lines=TMUX_SIDE_WIDTH, + lines=cfg.hw.TMUX_SIDE_WIDTH, text=' ', ) @@ -176,6 +175,7 @@ def audio_test(): def audio_test_linux(): """Run an audio test using amixer and speaker-test.""" + LOG.info('Audio Test') std.clear_screen() # Set volume @@ -226,36 +226,42 @@ def build_menu(quick_mode=False): def cpu_mprime_test(): """CPU & cooling check using Prime95.""" + LOG.info('CPU Test (Prime95)') #TODO: p95 std.print_warning('TODO: p95') def disk_attribute_check(): """Disk attribute check.""" + LOG.info('Disk Attribute Check') #TODO: at std.print_warning('TODO: at') def disk_io_benchmark(): """Disk I/O benchmark using dd.""" + LOG.info('Disk I/O Benchmark (dd)') #TODO: io std.print_warning('TODO: io') def disk_self_test(): """Disk self-test if available.""" + LOG.info('Disk Self-Test') #TODO: st std.print_warning('TODO: st') def disk_surface_scan(): """Disk surface scan using badblocks.""" + LOG.info('Disk Surface Scan (badblocks)') #TODO: bb std.print_warning('TODO: bb') def keyboard_test(): """Test keyboard using xev.""" + LOG.info('Keyboard Test (xev)') cmd = ['xev', '-event', 'keyboard'] std.clear_screen() exe.run_program(cmd, check=False, pipe=False) @@ -264,6 +270,7 @@ def keyboard_test(): def main(): """Main function for hardware diagnostics.""" args = docopt(DOCSTRING) + log.update_log_path(dest_name='Hardware-Diagnostics', timestamp=True) # Safety check if 'TMUX' not in os.environ: @@ -320,6 +327,7 @@ def main(): def network_test(): """Run network tests.""" + LOG.info('Network Test') std.clear_screen() try_and_print = std.TryAndPrint() result = try_and_print.run( @@ -349,6 +357,7 @@ def network_test(): def screensaver(name): """Show screensaver""" + LOG.info('Screensaver (%s)', name) if name == 'matrix': cmd = ['cmatrix', '-abs'] elif name == 'pipes': diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 84068ab5..ad210308 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -191,7 +191,7 @@ def resize_pane(pane_id=None, width=None, height=None, **kwargs): # Safety checks if not poll_pane(pane_id): - LOG.error('tmux pane %s not found', pane_id) + LOG.debug('tmux pane %s not found', pane_id) raise RuntimeError(f'tmux pane {pane_id} not found') if not (width or height): LOG.error('Neither width nor height specified') From ee7d656f2aaa6228695e3fef416b02c15425b015 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 20:47:59 -0700 Subject: [PATCH 159/324] Delete log atexit if empty --- scripts/wk/log.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 6ac43f39..8fb105a8 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -76,6 +76,23 @@ def get_root_logger_path(): return log_path +def remove_empty_log(): + """Remove log if empty.""" + is_empty = False + + # Check if log is empty + log_path = get_root_logger_path() + try: + is_empty = log_path and log_path.exists() and log_path.stat().st_size == 0 + except (FileNotFoundError, AttributeError): + # File doesn't exist or couldn't verify it's empty + pass + + # Delete log + if is_empty: + log_path.unlink() + + def start(config=None): """Configure and start logging using safe defaults.""" log_path = format_log_path(timestamp=os.name != 'nt') @@ -94,6 +111,7 @@ def start(config=None): logging.basicConfig(filename=log_path, **config) # Register shutdown to run atexit + atexit.register(remove_empty_log) atexit.register(logging.shutdown) From 72905f9cccabeda610d066c248df62bb4c1a5e9f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 10 Nov 2019 21:05:41 -0700 Subject: [PATCH 160/324] Added CLI options to hw-diags --- scripts/wk/hw/diags.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index c7dcb333..dcfc4622 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -23,11 +23,11 @@ atexit.register(tmux.kill_all_panes) DOCSTRING = f'''{cfg.main.KIT_NAME_FULL}: Hardware Diagnostics Usage: - hw-diags - hw-diags (-q | --quick) + hw-diags [options] hw-diags (-h | --help) Options: + -c --cli Force CLI mode -h --help Show this page -q --quick Skip menu and perform a quick check ''' @@ -189,7 +189,7 @@ def audio_test_linux(): exe.run_program(cmd, check=False, pipe=False) -def build_menu(quick_mode=False): +def build_menu(cli_mode=False, quick_mode=False): """Build main menu, returns wk.std.Menu.""" menu = std.Menu(title=None) @@ -212,6 +212,11 @@ def build_menu(quick_mode=False): # Only select quick option(s) menu.options[name]['Selected'] = name in MENU_OPTIONS_QUICK + # Add CLI actions if necessary + if cli_mode or 'DISPLAY' not in os.environ: + menu.add_action('Reboot') + menu.add_action('Power Off') + # Compatibility checks if platform.system() != 'Linux': for name in ('Audio Test', 'Keyboard Test', 'Network Test'): @@ -278,7 +283,7 @@ def main(): raise RuntimeError('tmux session not found') # Init - menu = build_menu(args['--quick']) + menu = build_menu(cli_mode=args['--cli'], quick_mode=args['--quick']) state = State() # Show menu @@ -314,6 +319,12 @@ def main(): # Quit if 'Quit' in selection: break + elif 'Reboot' in selection: + cmd = ['/usr/local/bin/wk-power-command', 'reboot'] + exe.run_program(cmd, check=False) + elif 'Power Off' in selection: + cmd = ['/usr/local/bin/wk-power-command', 'poweroff'] + exe.run_program(cmd, check=False) # Start diagnostics if 'Start' in selection: From 49c0ce9a6204d11ad41091da4cfc22cc57d709ff Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 11 Nov 2019 17:29:58 -0700 Subject: [PATCH 161/324] Support layouts with multiple panes of same type --- scripts/wk/tmux.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index ad210308..10b3b5a1 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -36,13 +36,16 @@ def fix_layout(panes, layout, forced=False): if name not in panes: continue - # Resize pane - pane_id = panes[name] - try: - resize_pane(pane_id, **data) - except RuntimeError: - # Assuming pane was closed just before resizing - pass + # Resize pane(s) + pane_list = panes[name] + if isinstance(pane_list, str): + pane_list = [pane_list] + for pane_id in pane_list: + try: + resize_pane(pane_id, **data) + except RuntimeError: + # Assuming pane was closed just before resizing + pass def get_pane_size(pane_id=None): From dc030ab0765a5e954c4827c522265bf1c43b8bff Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 11 Nov 2019 21:29:21 -0700 Subject: [PATCH 162/324] Added initial version of wk.hw.sensors * Supports Linux and macOS * Only initial temp, no updates yet --- scripts/wk/cfg/hw.py | 56 +++++++++ scripts/wk/hw/__init__.py | 1 + scripts/wk/hw/diags.py | 1 + scripts/wk/hw/sensors.py | 247 ++++++++++++++++++++++++++++++++++++++ scripts/wk/std.py | 1 + 5 files changed, 306 insertions(+) create mode 100644 scripts/wk/hw/sensors.py diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 1e848eb6..4b74f1ad 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -54,6 +54,62 @@ KNOWN_RAM_VENDOR_IDS = { REGEX_POWER_ON_TIME = re.compile( r'^(\d+)([Hh].*|\s+\(\d+\s+\d+\s+\d+\).*)' ) +SMC_IDS = { + # Sources: https://github.com/beltex/SMCKit/blob/master/SMCKit/SMC.swift + # http://www.opensource.apple.com/source/net_snmp/ + # https://github.com/jedda/OSX-Monitoring-Tools + 'TA0P': {'CPU Temp': False, 'Source': 'Ambient temp'}, + 'TA0S': {'CPU Temp': False, 'Source': 'PCIE Slot 1 Ambient'}, + 'TA1P': {'CPU Temp': False, 'Source': 'Ambient temp'}, + 'TA1S': {'CPU Temp': False, 'Source': 'PCIE Slot 1 PCB'}, + 'TA2S': {'CPU Temp': False, 'Source': 'PCIE Slot 2 Ambient'}, + 'TA3S': {'CPU Temp': False, 'Source': 'PCIE Slot 2 PCB'}, + 'TC0C': {'CPU Temp': True, 'Source': 'CPU Core 0'}, + 'TC0D': {'CPU Temp': True, 'Source': 'CPU die temp'}, + 'TC0H': {'CPU Temp': True, 'Source': 'CPU heatsink temp'}, + 'TC0P': {'CPU Temp': True, 'Source': 'CPU Ambient 1'}, + 'TC1C': {'CPU Temp': True, 'Source': 'CPU Core 1'}, + 'TC1P': {'CPU Temp': True, 'Source': 'CPU Ambient 2'}, + 'TC2C': {'CPU Temp': True, 'Source': 'CPU B Core 0'}, + 'TC2P': {'CPU Temp': True, 'Source': 'CPU B Ambient 1'}, + 'TC3C': {'CPU Temp': True, 'Source': 'CPU B Core 1'}, + 'TC3P': {'CPU Temp': True, 'Source': 'CPU B Ambient 2'}, + 'TCAC': {'CPU Temp': True, 'Source': 'CPU core from PCECI'}, + 'TCAH': {'CPU Temp': True, 'Source': 'CPU HeatSink'}, + 'TCBC': {'CPU Temp': True, 'Source': 'CPU B core from PCECI'}, + 'TCBH': {'CPU Temp': True, 'Source': 'CPU HeatSink'}, + 'Te1P': {'CPU Temp': False, 'Source': 'PCIE ambient temp'}, + 'Te1S': {'CPU Temp': False, 'Source': 'PCIE slot 1'}, + 'Te2S': {'CPU Temp': False, 'Source': 'PCIE slot 2'}, + 'Te3S': {'CPU Temp': False, 'Source': 'PCIE slot 3'}, + 'Te4S': {'CPU Temp': False, 'Source': 'PCIE slot 4'}, + 'TG0C': {'CPU Temp': False, 'Source': 'Mezzanine GPU Core'}, + 'TG0P': {'CPU Temp': False, 'Source': 'Mezzanine GPU Exhaust'}, + 'TH0P': {'CPU Temp': False, 'Source': 'Drive Bay 0'}, + 'TH1P': {'CPU Temp': False, 'Source': 'Drive Bay 1'}, + 'TH2P': {'CPU Temp': False, 'Source': 'Drive Bay 2'}, + 'TH3P': {'CPU Temp': False, 'Source': 'Drive Bay 3'}, + 'TH4P': {'CPU Temp': False, 'Source': 'Drive Bay 4'}, + 'TM0P': {'CPU Temp': False, 'Source': 'CPU DIMM Exit Ambient'}, + 'Tp0C': {'CPU Temp': False, 'Source': 'PSU1 Inlet Ambient'}, + 'Tp0P': {'CPU Temp': False, 'Source': 'PSU1 Inlet Ambient'}, + 'Tp1C': {'CPU Temp': False, 'Source': 'PSU1 Secondary Component'}, + 'Tp1P': {'CPU Temp': False, 'Source': 'PSU1 Primary Component'}, + 'Tp2P': {'CPU Temp': False, 'Source': 'PSU1 Secondary Component'}, + 'Tp3P': {'CPU Temp': False, 'Source': 'PSU2 Inlet Ambient'}, + 'Tp4P': {'CPU Temp': False, 'Source': 'PSU2 Primary Component'}, + 'Tp5P': {'CPU Temp': False, 'Source': 'PSU2 Secondary Component'}, + 'TS0C': {'CPU Temp': False, 'Source': 'CPU B DIMM Exit Ambient'}, + } +TEMP_COLORS = { + float('-inf'): 'CYAN', + 00: 'BLUE', + 60: 'GREEN', + 70: 'YELLOW', + 80: 'ORANGE', + 90: 'RED', + 100: 'ORANGE_RED', + } TMUX_SIDE_WIDTH = 20 TMUX_LAYOUT = OrderedDict({ 'Top': {'height': 2, 'Check': True}, diff --git a/scripts/wk/hw/__init__.py b/scripts/wk/hw/__init__.py index b28bffad..17b6df35 100644 --- a/scripts/wk/hw/__init__.py +++ b/scripts/wk/hw/__init__.py @@ -2,3 +2,4 @@ from wk.hw import diags from wk.hw import obj +from wk.hw import sensors diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index dcfc4622..5cfda66a 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -273,6 +273,7 @@ def keyboard_test(): def main(): + # pylint: disable=too-many-branches """Main function for hardware diagnostics.""" args = docopt(DOCSTRING) log.update_log_path(dest_name='Hardware-Diagnostics', timestamp=True) diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py new file mode 100644 index 00000000..4f6598e1 --- /dev/null +++ b/scripts/wk/hw/sensors.py @@ -0,0 +1,247 @@ +"""WizardKit: Hardware sensors""" +# vim: sts=2 sw=2 ts=2 + +import json +import logging +import platform +import re + +from subprocess import CalledProcessError + +from wk.cfg.hw import SMC_IDS, TEMP_COLORS +from wk.exe import run_program +from wk.std import color_string + + +# STATIC VARIABLES +LOG = logging.getLogger(__name__) +LM_SENSORS_CPU_REGEX = re.compile(r'(core|k\d+)temp', re.IGNORECASE) +SMC_REGEX = re.compile( + r'^\s*(?P\w{4})' + r'\s+\[(?P.*)\]' + r'\s+(?P.*?)' + r'\s*\(bytes (?P.*)\)$' + ) + + +# Error Classes +class ThermalLimitReachedError(RuntimeError): + """Raised when the thermal threshold is reached.""" + + +# Classes +class Sensors(): + """Class for holding sensor specific data.""" + def __init__(self): + self.data = get_sensor_data() + + def clear_temps(self): + """Clear saved temps but keep structure""" + for adapters in self.data.values(): + for sources in adapters.values(): + for source_data in sources.values(): + source_data['Temps'] = [] + + def generate_report(self, *temp_labels, colored=True, only_cpu=False): + """Generate report based on given temp_labels, returns list.""" + report = [] + + for section, adapters in sorted(self.data.items()): + if only_cpu and not section.startswith('CPU'): + continue + + # Ugly section + for adapter, sources in sorted(adapters.items()): + report.append(fix_sensor_name(adapter)) + for source, source_data in sorted(sources.items()): + line = f'{fix_sensor_name(source):18} ' + for label in temp_labels: + if label != 'Current': + line += f' {label.lower()}: ' + line += get_temp_str( + source_data.get(label, '???'), + colored=colored, + ) + report.append(line) + if not only_cpu: + report.append('') + + # Handle empty reports + if not report: + report = [ + color_string('WARNING: No sensors found', 'YELLOW'), + '', + 'Please monitor temps manually', + ] + + # Done + return report + + +# Functions +def fix_sensor_name(name): + """Cleanup sensor name, returns str.""" + name = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', name, re.IGNORECASE) + name = name.title() + name = name.replace('Coretemp', 'CoreTemp') + name = name.replace('Acpi', 'ACPI') + name = name.replace('ACPItz', 'ACPI TZ') + name = name.replace('Isa ', 'ISA ') + name = name.replace('Pci ', 'PCI ') + name = name.replace('Id ', 'ID ') + name = re.sub(r'(\D+)(\d+)', r'\1 \2', name, re.IGNORECASE) + name = re.sub(r'^K (\d+)Temp', r'AMD K\1 Temps', name, re.IGNORECASE) + name = re.sub(r'T(ctl|die)', r'CPU (T\1)', name, re.IGNORECASE) + return name + + +def get_lm_sensor_data(): + """Get sensor data via lm_sensors, returns dict.""" + raw_lm_sensor_data = get_raw_lm_sensor_data() + sensor_data = {'CPUTemps': {}, 'Others': {}} + + # Parse lm_sensor data + for adapter, sources in raw_lm_sensor_data.items(): + section = 'Others' + if LM_SENSORS_CPU_REGEX.search(adapter): + section = 'CPUTemps' + sensor_data[section][adapter] = {} + sources.pop('Adapter', None) + + # Find current temp and add to dict + ## current temp is labeled xxxx_input + for source, labels in sources.items(): + for label, temp in labels.items(): + if label.startswith('fan') or label.startswith('in'): + # Skip fan RPMs and voltages + continue + if 'input' in label: + sensor_data[section][adapter][source] = { + 'Current': temp, + 'Label': label, + 'Max': temp, + 'Temps': [temp], + } + + # Remove empty adapters + if not sensor_data[section][adapter]: + sensor_data[section].pop(adapter) + + # Remove empty sections + for adapters in sensor_data.values(): + adapters = {source: source_data for source, source_data in adapters.items() + if source_data} + + # Done + return sensor_data + + +def get_raw_lm_sensor_data(): + """Get raw sensor data via lm_sensors, returns dict.""" + raw_lm_sensor_data = {} + cmd = ['sensors', '-j'] + + # Get raw data + try: + proc = run_program(cmd) + except CalledProcessError: + # Assuming no sensors available, return empty dict + return {} + + # Workaround for bad sensors + raw_data = [] + for line in proc.stdout.splitlines(): + if line.strip() == ',': + # Assuming malformatted line caused by missing data + continue + raw_data.append(line) + + # Parse JSON data + try: + raw_lm_sensor_data = json.loads('\n'.join(raw_data)) + except json.JSONDecodeError: + # Still broken, just return the empty dict + pass + + # Done + return raw_lm_sensor_data + + +def get_sensor_data(): + """Get sensor data via OS-specific means, returns dict.""" + sensor_data = {} + if platform.system() == 'Darwin': + sensor_data = get_smc_sensor_data() + elif platform.system() == 'Linux': + sensor_data = get_lm_sensor_data() + + return sensor_data + + +def get_smc_sensor_data(): + """Get sensor data via SMC, returns dict. + + NOTE: The data is structured like the lm_sensor data. + """ + cmd = ['smc', '-l'] + sensor_data = {'CPUTemps': {'smc': {}}, 'Others': {'smc': {}}} + + # Parse SMC data + proc = run_program(cmd) + for line in proc.stdout.splitlines(): + tmp = SMC_REGEX.match(line.strip()) + if tmp: + value = tmp.group('Value') + try: + LOG.debug('Invalid sensor: %s', tmp.group('ID')) + value = float(value) + except (TypeError, ValueError): + # Skip this sensor + continue + + # Only add known sensor IDs + sensor_id = tmp.group('ID') + if sensor_id not in SMC_IDS: + continue + + # Add to dict + section = 'Others' + if SMC_IDS[sensor_id].get('CPUTemp', False): + section = 'CPUTemps' + source = SMC_IDS[sensor_id]['Source'] + sensor_data[section]['smc'][source] = { + 'Current': value, + 'Label': sensor_id, + 'Max': value, + 'Temps': [value], + } + + # Done + return sensor_data + + +def get_temp_str(temp, colored=True): + """Get colored string based on temp, returns str.""" + temp_color = None + + # Safety check + try: + temp = float(temp) + except (TypeError, ValueError): + # Invalid temp? + return color_string(temp, 'PURPLE') + + # Determine color + if colored: + for threshold, color in sorted(TEMP_COLORS.items(), reverse=True): + if temp >= threshold: + temp_color = color + break + + # Done + return color_string(f'{"-" if temp < 0 else ""}{temp:2.0f}°C', temp_color) + + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/std.py b/scripts/wk/std.py index a9b5d54a..32a107f6 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -37,6 +37,7 @@ COLORS = { 'RED': '\033[31m', 'RED_BLINK': '\033[31;5m', 'ORANGE': '\033[31;1m', + 'ORANGE_RED': '\033[1;31;41m', 'GREEN': '\033[32m', 'YELLOW': '\033[33m', 'YELLOW_BLINK': '\033[33;5m', From b15c01ac372c0e1fbe87f2eaa7dbc3e531c168e7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 11 Nov 2019 21:47:55 -0700 Subject: [PATCH 163/324] Fixed sensor sections under macOS --- scripts/wk/hw/sensors.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 4f6598e1..92147381 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -54,7 +54,7 @@ class Sensors(): for adapter, sources in sorted(adapters.items()): report.append(fix_sensor_name(adapter)) for source, source_data in sorted(sources.items()): - line = f'{fix_sensor_name(source):18} ' + line = f'{fix_sensor_name(source):25} ' for label in temp_labels: if label != 'Current': line += f' {label.lower()}: ' @@ -83,12 +83,14 @@ def fix_sensor_name(name): """Cleanup sensor name, returns str.""" name = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', name, re.IGNORECASE) name = name.title() - name = name.replace('Coretemp', 'CoreTemp') - name = name.replace('Acpi', 'ACPI') name = name.replace('ACPItz', 'ACPI TZ') + name = name.replace('Acpi', 'ACPI') + name = name.replace('Coretemp', 'CoreTemp') + name = name.replace('Cpu', 'CPU') + name = name.replace('Id ', 'ID ') name = name.replace('Isa ', 'ISA ') name = name.replace('Pci ', 'PCI ') - name = name.replace('Id ', 'ID ') + name = name.replace('Smc', 'SMC') name = re.sub(r'(\D+)(\d+)', r'\1 \2', name, re.IGNORECASE) name = re.sub(r'^K (\d+)Temp', r'AMD K\1 Temps', name, re.IGNORECASE) name = re.sub(r'T(ctl|die)', r'CPU (T\1)', name, re.IGNORECASE) @@ -184,7 +186,7 @@ def get_smc_sensor_data(): NOTE: The data is structured like the lm_sensor data. """ cmd = ['smc', '-l'] - sensor_data = {'CPUTemps': {'smc': {}}, 'Others': {'smc': {}}} + sensor_data = {'CPUTemps': {'SMC (CPU)': {}}, 'Others': {'SMC (Other)': {}}} # Parse SMC data proc = run_program(cmd) @@ -206,10 +208,12 @@ def get_smc_sensor_data(): # Add to dict section = 'Others' - if SMC_IDS[sensor_id].get('CPUTemp', False): + adapter = 'SMC (Other)' + if SMC_IDS[sensor_id].get('CPU Temp', False): section = 'CPUTemps' + adapter = 'SMC (CPU)' source = SMC_IDS[sensor_id]['Source'] - sensor_data[section]['smc'][source] = { + sensor_data[section][adapter][source] = { 'Current': value, 'Label': sensor_id, 'Max': value, From 4ecdc80e4ce48877441ebb45788ee4a7c1310bfa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 11 Nov 2019 22:18:09 -0700 Subject: [PATCH 164/324] Added sensor update sections --- scripts/wk/cfg/hw.py | 2 + scripts/wk/hw/sensors.py | 84 ++++++++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 16 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 4b74f1ad..791ff48b 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -14,6 +14,8 @@ ATTRIBUTE_COLORS = ( ('Error', 'RED'), ('Maximum', 'PURPLE'), ) +CPU_FAILURE_TEMP = 90 +CPU_THERMAL_LIMIT = 99 KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' KNOWN_DISK_ATTRIBUTES = { diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 92147381..2380fbf1 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -8,7 +8,7 @@ import re from subprocess import CalledProcessError -from wk.cfg.hw import SMC_IDS, TEMP_COLORS +from wk.cfg.hw import CPU_THERMAL_LIMIT, SMC_IDS, TEMP_COLORS from wk.exe import run_program from wk.std import color_string @@ -77,6 +77,58 @@ class Sensors(): # Done return report + def update_sensor_data(self, exit_on_thermal_limit=True): + """Update sensor data via OS-specific means.""" + if platform.system() == 'Darwin': + self.update_sensor_data_macos(exit_on_thermal_limit) + elif platform.system() == 'Linux': + self.update_sensor_data_linux(exit_on_thermal_limit) + + def update_sensor_data_linux(self, exit_on_thermal_limit=True): + """Update sensor data via lm_sensors.""" + lm_sensor_data = get_sensor_data_lm() + for section, adapters in self.data.items(): + for adapter, sources in adapters.items(): + for source, source_data in sources.items(): + try: + label = source_data['Label'] + temp = lm_sensor_data[adapter][source][label] + source_data['Current'] = temp + source_data['Max'] = max(temp, source_data['Max']) + source_data['Temps'].append(temp) + except KeyError: + # Dumb workaround for Dell sensors with changing source names + pass + + # Raise exception if thermal limit reached + if exit_on_thermal_limit and section == 'CPUTemps': + if source_data['Current'] >= CPU_THERMAL_LIMIT: + raise ThermalLimitReachedError('CPU temps reached limit') + + def update_sensor_data_macos(self, exit_on_thermal_limit=True): + """Update sensor data via SMC.""" + for section, adapters in self.data.items(): + for sources in adapters.values(): + for source_data in sources.values(): + cmd = ['smc', '-k', source_data['Label'], '-r'] + proc = run_program(cmd) + match = SMC_REGEX.match(proc.stdout.strip()) + try: + temp = float(match.group('Value')) + except (TypeError, ValueError): + LOG.error('Failed to update temp %s', source_data['Label']) + continue + + # Update source + source_data['Current'] = temp + source_data['Max'] = max(temp, source_data['Max']) + source_data['Temps'].append(temp) + + # Raise exception if thermal limit reached + if exit_on_thermal_limit and section == 'CPUTemps': + if source_data['Current'] >= CPU_THERMAL_LIMIT: + raise ThermalLimitReachedError('CPU temps reached limit') + # Functions def fix_sensor_name(name): @@ -97,9 +149,20 @@ def fix_sensor_name(name): return name -def get_lm_sensor_data(): +def get_sensor_data(): + """Get sensor data via OS-specific means, returns dict.""" + sensor_data = {} + if platform.system() == 'Darwin': + sensor_data = get_sensor_data_macos() + elif platform.system() == 'Linux': + sensor_data = get_sensor_data_linux() + + return sensor_data + + +def get_sensor_data_linux(): """Get sensor data via lm_sensors, returns dict.""" - raw_lm_sensor_data = get_raw_lm_sensor_data() + raw_lm_sensor_data = get_sensor_data_lm() sensor_data = {'CPUTemps': {}, 'Others': {}} # Parse lm_sensor data @@ -138,7 +201,7 @@ def get_lm_sensor_data(): return sensor_data -def get_raw_lm_sensor_data(): +def get_sensor_data_lm(): """Get raw sensor data via lm_sensors, returns dict.""" raw_lm_sensor_data = {} cmd = ['sensors', '-j'] @@ -169,18 +232,7 @@ def get_raw_lm_sensor_data(): return raw_lm_sensor_data -def get_sensor_data(): - """Get sensor data via OS-specific means, returns dict.""" - sensor_data = {} - if platform.system() == 'Darwin': - sensor_data = get_smc_sensor_data() - elif platform.system() == 'Linux': - sensor_data = get_lm_sensor_data() - - return sensor_data - - -def get_smc_sensor_data(): +def get_sensor_data_macos(): """Get sensor data via SMC, returns dict. NOTE: The data is structured like the lm_sensor data. From e3d0902c458ad309b166f3538879ab5ce7956cdb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 11 Nov 2019 23:22:47 -0700 Subject: [PATCH 165/324] Updated wk.hw.sensors * Added monitor_to_file() * Added save_average_temps() --- scripts/wk/hw/sensors.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 2380fbf1..7307818b 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -10,7 +10,7 @@ from subprocess import CalledProcessError from wk.cfg.hw import CPU_THERMAL_LIMIT, SMC_IDS, TEMP_COLORS from wk.exe import run_program -from wk.std import color_string +from wk.std import color_string, sleep # STATIC VARIABLES @@ -77,6 +77,32 @@ class Sensors(): # Done return report + def monitor_to_file(self, path): + """Write report to path every second until stopped.""" + while True: + self.update_sensor_data() + report = self.generate_report('Current', 'Max') + with open(path, 'w') as _f: + _f.write('\n'.join(report)) + sleep(1) + + def save_average_temps(self, temp_label, seconds=10): + # pylint: disable=unused-variable + """Save average temps under temp_label over provided seconds..""" + self.clear_temps() + + # Get temps + for i in range(seconds): + self.update_sensor_data() + sleep(1) + + # Calculate averages + for adapters in self.data.values(): + for sources in adapters.values(): + for source_data in sources.values(): + temps = source_data['Temps'] + source_data[temp_label] = sum(temps) / len(temps) + def update_sensor_data(self, exit_on_thermal_limit=True): """Update sensor data via OS-specific means.""" if platform.system() == 'Darwin': From 4bd4536cfdc2645ffb3e3c58883b5bd1e21fe6cd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 11 Nov 2019 23:57:48 -0700 Subject: [PATCH 166/324] Avoid using the unicode degree symbol under macOS * The (home)brew watch command butchers the unicode? --- scripts/wk/hw/sensors.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 7307818b..dcf2ca43 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -304,6 +304,7 @@ def get_sensor_data_macos(): def get_temp_str(temp, colored=True): """Get colored string based on temp, returns str.""" + degree_symbol = '*' if platform.system() == 'Darwin' else '°' temp_color = None # Safety check @@ -321,7 +322,10 @@ def get_temp_str(temp, colored=True): break # Done - return color_string(f'{"-" if temp < 0 else ""}{temp:2.0f}°C', temp_color) + return color_string( + f'{"-" if temp < 0 else ""}{temp:2.0f}{degree_symbol}C', + temp_color, + ) From 9b5d9e1186b51682d6e59f61c4a6965affc033a3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 12 Nov 2019 10:36:34 -0700 Subject: [PATCH 167/324] Added watch-mac and reverted previous commit * This allows the degree symbol to be displayed correctly * (At least in iTerm2) --- scripts/watch-mac | 11 +++++++++++ scripts/wk/hw/sensors.py | 6 +----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100755 scripts/watch-mac diff --git a/scripts/watch-mac b/scripts/watch-mac new file mode 100755 index 00000000..81029734 --- /dev/null +++ b/scripts/watch-mac @@ -0,0 +1,11 @@ +#!/bin/zsh +# +## watch-like utility + +WATCH_FILE="${1}" + +while :; do + echo -n "\e[100A" + cat "${WATCH_FILE}" + sleep 1s +done diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index dcf2ca43..7307818b 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -304,7 +304,6 @@ def get_sensor_data_macos(): def get_temp_str(temp, colored=True): """Get colored string based on temp, returns str.""" - degree_symbol = '*' if platform.system() == 'Darwin' else '°' temp_color = None # Safety check @@ -322,10 +321,7 @@ def get_temp_str(temp, colored=True): break # Done - return color_string( - f'{"-" if temp < 0 else ""}{temp:2.0f}{degree_symbol}C', - temp_color, - ) + return color_string(f'{"-" if temp < 0 else ""}{temp:2.0f}°C', temp_color) From 4e6b2cd4da75d1a363da736e9c944cf6e2829390 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 12 Nov 2019 17:32:55 -0700 Subject: [PATCH 168/324] Started work on per-pass log handling in hw-diags --- scripts/wk/hw/diags.py | 26 ++++++++++++++++++++++++++ scripts/wk/log.py | 15 ++++++++++----- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 5cfda66a..05b1c9fa 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -71,6 +71,7 @@ class State(): def __init__(self): self.cpu = None self.disks = [] + self.log_dir = None self.panes = {} self.tests = OrderedDict({ 'CPU & Cooling': { @@ -131,6 +132,27 @@ class State(): self.fix_tmux_layout(forced=False) std.sleep(1) + def init_diags(self): + """Initialize diagnostic pass.""" + # Reset objects + self.disks.clear() + for test_data in self.tests.values(): + test_data['Objects'].clear() + + # Set log + self.log_dir = log.format_log_path() + self.log_dir = pathlib.Path( + f'{self.log_dir.parent}/' + f'Hardware-Diagnostics_{time.strftime("%Y-%m-%d_%H%M%z")}/' + ) + log.update_log_path( + dest_dir=self.log_dir, + dest_name='main', + keep_history=False, + timestamp=False, + ) + std.print_info('Starting Hardware Diagnostics') + def init_tmux(self): """Initialize tmux layout.""" tmux.kill_all_panes() @@ -367,6 +389,10 @@ def network_test(): std.pause('Press Enter to return to main menu...') +def run_diags(state): + """Run selected diagnostics.""" + + def screensaver(name): """Show screensaver""" LOG.info('Screensaver (%s)', name) diff --git a/scripts/wk/log.py b/scripts/wk/log.py index 8fb105a8..ee512c95 100644 --- a/scripts/wk/log.py +++ b/scripts/wk/log.py @@ -115,12 +115,14 @@ def start(config=None): atexit.register(logging.shutdown) -def update_log_path(dest_dir=None, dest_name=None, timestamp=True): +def update_log_path( + dest_dir=None, dest_name=None, keep_history=True, timestamp=True): """Moves current log file to new path and updates the root logger.""" root_logger = logging.getLogger() cur_handler = None cur_path = get_root_logger_path() new_path = format_log_path(dest_dir, dest_name, timestamp=timestamp) + os.makedirs(new_path.parent, exist_ok=True) # Get current logging file handler for handler in root_logger.handlers: @@ -131,10 +133,13 @@ def update_log_path(dest_dir=None, dest_name=None, timestamp=True): raise RuntimeError('Logging FileHandler not found') # Copy original log to new location - if new_path.exists(): - raise FileExistsError(f'Refusing to clobber: {new_path}') - os.makedirs(new_path.parent, exist_ok=True) - shutil.move(cur_path, new_path) + if keep_history: + if new_path.exists(): + raise FileExistsError(f'Refusing to clobber: {new_path}') + shutil.move(cur_path, new_path) + + # Remove old log if empty + remove_empty_log() # Create new cur_handler (preserving formatter settings) new_handler = logging.FileHandler(new_path, mode='a') From 1054794af32adc568cd5d0819c52fa8283cf8ad1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 12 Nov 2019 19:56:39 -0700 Subject: [PATCH 169/324] Added get_disks() * This calls either get_disks_linux() or get_disks_macos() --- scripts/wk/hw/diags.py | 85 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 05b1c9fa..1ffb1162 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -6,6 +6,8 @@ import logging import os import pathlib import platform +import plistlib +import re import signal import time @@ -13,6 +15,7 @@ from collections import OrderedDict from docopt import docopt from wk import cfg, exe, log, net, std, tmux +from wk.hw import obj as hw_obj # atexit functions @@ -63,6 +66,10 @@ MENU_SETS = { MENU_TOGGLES = ( 'Skip USB Benchmarks', ) +WK_LABEL_REGEX = re.compile( + fr'{cfg.main.KIT_NAME_SHORT}_(LINUX|UFD)', + re.IGNORECASE, + ) # Classes @@ -153,6 +160,12 @@ class State(): ) std.print_info('Starting Hardware Diagnostics') + # Add CPU + self.cpu = hw_obj.CpuRam() + + # Add disks + self.disks = get_disks() + def init_tmux(self): """Initialize tmux layout.""" tmux.kill_all_panes() @@ -286,6 +299,78 @@ def disk_surface_scan(): std.print_warning('TODO: bb') +def get_disks(): + """Get disks using OS-specific methods, returns list.""" + disks = [] + if platform.system() == 'Darwin': + disks = get_disks_macos() + elif platform.system() == 'Linux': + disks = get_disks_linux() + + # Done + return disks + + +def get_disks_linux(): + """Get disks via lsblk, returns list.""" + cmd = ['lsblk', '--json', '--nodeps', '--paths'] + disks = [] + + # Add valid disks + json_data = exe.get_json_from_command(cmd) + for disk in json_data.get('blockdevices', []): + disk_obj = hw_obj.Disk(disk['name']) + skip = False + + # Skip loopback devices, optical devices, etc + if disk_obj.details['type'] != 'disk': + skip = True + + # Skip WK disks + for label in disk_obj.get_labels(): + if WK_LABEL_REGEX.search(label): + skip = True + + # Add disk + if not skip: + disks.append(disk_obj) + + # Done + return disks + + +def get_disks_macos(): + """Get disks via diskutil, returns list.""" + cmd = ['diskutil', 'list', '-plist', 'physical'] + disks = [] + + # Get info from diskutil + proc = exe.run_program(cmd, encoding=None, errors=None) + try: + plist_data = plistlib.loads(proc.stdout) + except (TypeError, ValueError): + # Invalid / corrupt plist data? return empty list to avoid crash + LOG.error('Failed to get diskutil list') + return disks + + # Add valid disks + for disk in plist_data['WholeDisks']: + disk_obj = hw_obj.Disk(disk) + skip = False + + # Skip WK disks + for label in disk_obj.get_labels(): + if WK_LABEL_REGEX.search(label): + skip = True + + # Add disk + if not skip: + disks.append(disk_obj) + + # Done + return disks + + def keyboard_test(): """Test keyboard using xev.""" LOG.info('Keyboard Test (xev)') From d4ca575426a07ee29ae8918441049598789bbe55 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 12 Nov 2019 20:06:18 -0700 Subject: [PATCH 170/324] Fix get_disks_macos() --- scripts/wk/hw/diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 1ffb1162..37ca7cb6 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -355,7 +355,7 @@ def get_disks_macos(): # Add valid disks for disk in plist_data['WholeDisks']: - disk_obj = hw_obj.Disk(disk) + disk_obj = hw_obj.Disk(f'/dev/{disk}') skip = False # Skip WK disks From aa5b5cd9b77de2d1cfaf15a2705d61e404f8b188 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 12 Nov 2019 21:10:11 -0700 Subject: [PATCH 171/324] Selecting and running (dummy) tests now working --- scripts/wk/hw/diags.py | 95 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 37ca7cb6..82d0c689 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -139,7 +139,7 @@ class State(): self.fix_tmux_layout(forced=False) std.sleep(1) - def init_diags(self): + def init_diags(self, menu): """Initialize diagnostic pass.""" # Reset objects self.disks.clear() @@ -160,12 +160,32 @@ class State(): ) std.print_info('Starting Hardware Diagnostics') - # Add CPU + # Add HW Objects self.cpu = hw_obj.CpuRam() - - # Add disks self.disks = get_disks() + # Add test objects + for name, details in menu.options.items(): + self.tests[name]['Enabled'] = details['Selected'] + if not details['Selected']: + continue + if 'CPU' in name: + test_obj = hw_obj.Test(dev=self.cpu, label=name) + self.cpu.tests[name] = test_obj + self.tests[name]['Objects'].append(test_obj) + elif 'Disk' in name: + for disk in self.disks: + test_obj = hw_obj.Test(dev=disk, label=disk.path.name) + disk.tests[name] = test_obj + self.tests[name]['Objects'].append(test_obj) + + # No disks detected? + if not self.tests[name]['Objects']: + test_obj = hw_obj.Test(dev=None, label='') + test_obj.set_status('N/A') + test_obj.disabled = True + self.tests[name]['Objects'].append(test_obj) + def init_tmux(self): """Initialize tmux layout.""" tmux.kill_all_panes() @@ -264,39 +284,44 @@ def build_menu(cli_mode=False, quick_mode=False): return menu -def cpu_mprime_test(): +def cpu_mprime_test(state, test_objects): """CPU & cooling check using Prime95.""" LOG.info('CPU Test (Prime95)') #TODO: p95 std.print_warning('TODO: p95') + std.pause() -def disk_attribute_check(): +def disk_attribute_check(state, test_objects): """Disk attribute check.""" LOG.info('Disk Attribute Check') #TODO: at std.print_warning('TODO: at') + std.pause() -def disk_io_benchmark(): +def disk_io_benchmark(state, test_objects): """Disk I/O benchmark using dd.""" LOG.info('Disk I/O Benchmark (dd)') #TODO: io std.print_warning('TODO: io') + std.pause() -def disk_self_test(): +def disk_self_test(state, test_objects): """Disk self-test if available.""" LOG.info('Disk Self-Test') #TODO: st std.print_warning('TODO: st') + std.pause() -def disk_surface_scan(): +def disk_surface_scan(state, test_objects): """Disk surface scan using badblocks.""" LOG.info('Disk Surface Scan (badblocks)') #TODO: bb std.print_warning('TODO: bb') + std.pause() def get_disks(): @@ -394,6 +419,12 @@ def main(): menu = build_menu(cli_mode=args['--cli'], quick_mode=args['--quick']) state = State() + # Quick Mode + if args['--quick']: + std.clear_screen() + run_diags(state, menu, quick_mode=True) + return + # Show menu while True: action = None @@ -436,9 +467,7 @@ def main(): # Start diagnostics if 'Start' in selection: - #TODO - #run_diags() - pass + run_diags(state, menu, quick_mode=False) # Reset top pane state.update_top_pane('Main Menu') @@ -474,8 +503,48 @@ def network_test(): std.pause('Press Enter to return to main menu...') -def run_diags(state): +def run_diags(state, menu, quick_mode=False): """Run selected diagnostics.""" + aborted = False + state.init_diags(menu) + + # Just return if no tests were selected + if not any([details['Enabled'] for details in state.tests.values()]): + std.print_warning('No tests selected?') + std.pause() + return + + # Run tests + for details in state.tests.values(): + if not details['Enabled']: + # Skip disabled tests + continue + + # Run test(s) + function = details['Function'] + try: + function(state, details['Objects']) + except std.GenericAbort: + aborted = True + # Restart tmux + state.init_tmux() + break + + # Handle aborts + if aborted: + for details in state.tests.values(): + for test_obj in details['Objects']: + if test_obj.status == 'Pending': + test_obj.set_status('Aborted') + + # Show results + #TODO: Show results + + # Done + if quick_mode: + std.pause('Press Enter to exit...') + else: + std.pause('Press Enter to return to main menu...') def screensaver(name): From e18b62528125f958af19d256ff8777398f543c85 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 13 Nov 2019 11:05:44 -0700 Subject: [PATCH 172/324] Updated ClassicStartSkin source URL --- scripts/wk.prev/settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk.prev/settings/sources.py b/scripts/wk.prev/settings/sources.py index 2abf079f..1605d5f2 100644 --- a/scripts/wk.prev/settings/sources.py +++ b/scripts/wk.prev/settings/sources.py @@ -12,7 +12,7 @@ SOURCE_URLS = { 'BlueScreenView32': 'http://www.nirsoft.net/utils/bluescreenview.zip', 'BlueScreenView64': 'http://www.nirsoft.net/utils/bluescreenview-x64.zip', 'Caffeine': 'http://www.zhornsoftware.co.uk/caffeine/caffeine.zip', - 'ClassicStartSkin': 'http://www.classicshell.net/forum/download/file.php?id=3001&sid=9a195960d98fd754867dcb63d9315335', + 'ClassicStartSkin': 'https://coddec.github.io/Classic-Shell/www.classicshell.net/forum/download/fileb1ba.php?id=3001', 'Du': 'https://download.sysinternals.com/files/DU.zip', 'ERUNT': 'http://www.aumha.org/downloads/erunt.zip', 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.935.x86.en-US.zip', From 0eadb784bbc4fff02250fc79937b9b9dc23a7cf5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 13 Nov 2019 11:14:55 -0700 Subject: [PATCH 173/324] Updated get_ram_list_linux() --- scripts/wk/hw/obj.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 7b3c5f47..6cc8e3a5 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -633,7 +633,11 @@ def get_ram_list_linux(): size = 0 elif line.startswith('Size:'): size = line.replace('Size: ', '') - size = string_to_bytes(size, assume_binary=True) + try: + size = string_to_bytes(size, assume_binary=True) + except ValueError: + # Assuming empty module + size = 0 elif line.startswith('Manufacturer:'): manufacturer = line.replace('Manufacturer: ', '') dimm_list.append([size, manufacturer]) From 46a6dda0ff071de96c432713cde9e6d01312deaa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 13 Nov 2019 17:47:52 -0700 Subject: [PATCH 174/324] Prime95 workflow mostly done --- scripts/wk/cfg/hw.py | 3 +- scripts/wk/hw/diags.py | 131 +++++++++++++++++++++++++++++++++++++-- scripts/wk/hw/sensors.py | 14 ++++- scripts/wk/tmux.py | 22 ++++--- 4 files changed, 150 insertions(+), 20 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 791ff48b..fa303cba 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -15,6 +15,7 @@ ATTRIBUTE_COLORS = ( ('Maximum', 'PURPLE'), ) CPU_FAILURE_TEMP = 90 +CPU_TEST_MINUTES = 7 CPU_THERMAL_LIMIT = 99 KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' @@ -118,8 +119,8 @@ TMUX_LAYOUT = OrderedDict({ 'Started': {'width': TMUX_SIDE_WIDTH, 'Check': True}, 'Progress': {'width': TMUX_SIDE_WIDTH, 'Check': True}, # Testing panes - 'Prime95': {'height': 11, 'Check': False}, 'Temps': {'height': 1000, 'Check': False}, + 'Prime95': {'height': 11, 'Check': False}, 'SMART': {'height': 3, 'Check': True}, 'badblocks': {'height': 5, 'Check': True}, 'I/O Benchmark': {'height': 1000, 'Check': False}, diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 82d0c689..068a6242 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -16,6 +16,7 @@ from docopt import docopt from wk import cfg, exe, log, net, std, tmux from wk.hw import obj as hw_obj +from wk.hw import sensors as hw_sensors # atexit functions @@ -78,6 +79,7 @@ class State(): def __init__(self): self.cpu = None self.disks = [] + self.layout = cfg.hw.TMUX_LAYOUT.copy() self.log_dir = None self.panes = {} self.tests = OrderedDict({ @@ -111,11 +113,13 @@ class State(): # Init tmux and start a background process to maintain layout self.init_tmux() - if hasattr(signal, 'SIGWINCH'): - # Use signal handling - signal.signal(signal.SIGWINCH, self.fix_tmux_layout) - else: - exe.start_thread(self.fix_tmux_layout_loop) + #TODO: Fix SIGWINCH? + #if hasattr(signal, 'SIGWINCH'): + # # Use signal handling + # signal.signal(signal.SIGWINCH, self.fix_tmux_layout) + #else: + # exe.start_thread(self.fix_tmux_layout_loop) + exe.start_thread(self.fix_tmux_layout_loop) def fix_tmux_layout(self, forced=True, signum=None, frame=None): # pylint: disable=unused-argument @@ -125,7 +129,7 @@ class State(): signum and frame must be valid aguments. """ try: - tmux.fix_layout(self.panes, cfg.hw.TMUX_LAYOUT, forced=forced) + tmux.fix_layout(self.panes, self.layout, forced=forced) except RuntimeError: # Assuming self.panes changed while running pass @@ -143,6 +147,8 @@ class State(): """Initialize diagnostic pass.""" # Reset objects self.disks.clear() + self.layout.clear() + self.layout.update(cfg.hw.TMUX_LAYOUT) for test_data in self.tests.values(): test_data['Objects'].clear() @@ -287,6 +293,81 @@ def build_menu(cli_mode=False, quick_mode=False): def cpu_mprime_test(state, test_objects): """CPU & cooling check using Prime95.""" LOG.info('CPU Test (Prime95)') + thermal_abort = False + prime_log = pathlib.Path(f'{state.log_dir}/prime.log') + test = test_objects[0] + + # Bail early + if test.disabled: + return + + # Prep + dev = test.dev + test.set_status('Working') + state.update_top_pane(dev.description) + + # Start sensors monitor + sensors = hw_sensors.Sensors() + sensors_out = pathlib.Path(f'{state.log_dir}/sensors.out') + sensors_thread = exe.start_thread( + sensors.monitor_to_file, args=(sensors_out,)) + + # Create monitor and worker panes + state.panes['Prime95'] = tmux.split_window( + lines=10, vertical=True, watch_file=prime_log) + state.panes['Temps'] = tmux.split_window( + behind=True, percent=80, vertical=True, watch_file=sensors_out) + tmux.resize_pane(height=3) + state.panes['Current'] = '' + state.layout['Current'] = {'height': 3, 'Check': True} + + # Get idle temps + std.clear_screen() + std.print_standard('Saving idle temps...') + sensors.save_average_temps(temp_label='Idle', seconds=5) + + # Stress CPU + std.print_info('Starting stress test') + std.print_warning('If running too hot, press CTRL+c to abort the test') + set_apple_fan_speed('max') + #RUN: mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "prime.log" + try: + print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) + except KeyboardInterrupt: + test.set_status('Aborted') + except hw_sensors.ThermalLimitReachedError: + test.set_status('Failed') + test.failed = True + thermal_abort = True + + # Stop Prime95 + #TODO kill p95 + tmux.kill_pane(state.panes.pop('Prime95', None)) + + # Get cooldown temp + set_apple_fan_speed('auto') + std.clear_screen() + std.print_standard('Letting CPU cooldown...') + std.sleep(5) + std.print_standard('Saving cooldown temps...') + sensors.save_average_temps(temp_label='Cooldown', seconds=5) + + # Check results and build report + #TODO + + # Stop sensors monitor + sensors_out.with_suffix('.stop').touch() + sensors_thread.join() + + # Cleanup + state.panes.pop('Current', None) + tmux.kill_pane(state.panes.pop('Temps', None)) + + + + + + #TODO: p95 std.print_warning('TODO: p95') std.pause() @@ -503,6 +584,26 @@ def network_test(): std.pause('Press Enter to return to main menu...') +def print_countdown(seconds): + """Print countdown to screen.""" + time_limit = seconds + for i in range(seconds): + sec_left = (seconds - i) % 60 + min_left = int((seconds - i) / 60) + + out_str = '\r' + if min_left: + out_str += f'{min_left} minute{"s" if min_left != 1 else ""}, ' + out_str += f'{sec_left} second{"s" if sec_left != 1 else ""}' + out_str += ' remaining' + + print(f'{out_str:<40}', end='', flush=True) + std.sleep(1) + + # Done + print('') + + def run_diags(state, menu, quick_mode=False): """Run selected diagnostics.""" aborted = False @@ -569,5 +670,23 @@ def screensaver(name): tmux.zoom_pane() +def set_apple_fan_speed(speed): + """Set Apple fan speed.""" + cmd = None + + # Check + if speed not in ('auto', 'max'): + raise RuntimeError(f'Invalid speed {speed}') + + # Set cmd + if platform.system() == 'Linux': + cmd = ['apple-fans', speed] + #TODO: Add method for use under macOS + + # Run cmd + if cmd: + exe.run_program(cmd, check=False) + + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 7307818b..fce71d32 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -3,6 +3,7 @@ import json import logging +import pathlib import platform import re @@ -77,14 +78,21 @@ class Sensors(): # Done return report - def monitor_to_file(self, path): + def monitor_to_file(self, out_path): """Write report to path every second until stopped.""" + stop_path = pathlib.Path(out_path).resolve().with_suffix('.stop') while True: self.update_sensor_data() report = self.generate_report('Current', 'Max') - with open(path, 'w') as _f: + with open(out_path, 'w') as _f: _f.write('\n'.join(report)) - sleep(1) + + # Check if we should stop + if stop_path.exists(): + break + + # Sleep before next loop + sleep(0.5) def save_average_temps(self, temp_label, seconds=10): # pylint: disable=unused-variable diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 10b3b5a1..8639db6f 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -41,6 +41,8 @@ def fix_layout(panes, layout, forced=False): if isinstance(pane_list, str): pane_list = [pane_list] for pane_id in pane_list: + if name == 'Current': + pane_id = None try: resize_pane(pane_id, **data) except RuntimeError: @@ -98,13 +100,16 @@ def layout_needs_fixed(panes, layout): if name not in panes: continue - # Check pane size - pane_id = panes[name] - width, height = get_pane_size(pane_id) - if data.get('width', False) and data['width'] != width: - needs_fixed = True - if data.get('height', False) and data['height'] != height: - needs_fixed = True + # Check pane size(s) + pane_list = panes[name] + if isinstance(pane_list, str): + pane_list = [pane_list] + for pane_id in pane_list: + width, height = get_pane_size(pane_id) + if data.get('width', False) and data['width'] != width: + needs_fixed = True + if data.get('height', False) and data['height'] != height: + needs_fixed = True # Done return needs_fixed @@ -193,9 +198,6 @@ def resize_pane(pane_id=None, width=None, height=None, **kwargs): cmd = ['tmux', 'resize-pane'] # Safety checks - if not poll_pane(pane_id): - LOG.debug('tmux pane %s not found', pane_id) - raise RuntimeError(f'tmux pane {pane_id} not found') if not (width or height): LOG.error('Neither width nor height specified') raise RuntimeError('Neither width nor height specified') From 1a91f72d8c31e5c42f599e6d66c605701f50109a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 13 Nov 2019 19:45:53 -0700 Subject: [PATCH 175/324] Running and stopping Prime95 working --- scripts/wk/exe.py | 12 ++++++++++++ scripts/wk/hw/diags.py | 44 +++++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index 753df2a0..c5c1718b 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -47,6 +47,18 @@ class NonBlockingStreamReader(): except Empty: return None + def save_to_file(self, proc, out_path): + """Continuously save output to file while proc is running.""" + while proc.poll() is None: + out = b'' + out_bytes = b'' + while out is not None: + out = self.read(0.1) + if out: + out_bytes += out + with open(out_path, 'a') as _f: + _f.write(out_bytes.decode('utf-8', errors='ignore')) + # Functions def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 068a6242..4a83a5bf 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -9,6 +9,7 @@ import platform import plistlib import re import signal +import subprocess import time from collections import OrderedDict @@ -156,7 +157,7 @@ class State(): self.log_dir = log.format_log_path() self.log_dir = pathlib.Path( f'{self.log_dir.parent}/' - f'Hardware-Diagnostics_{time.strftime("%Y-%m-%d_%H%M%z")}/' + f'Hardware-Diagnostics_{time.strftime("%Y-%m-%d_%H%M%S%z")}/' ) log.update_log_path( dest_dir=self.log_dir, @@ -291,6 +292,8 @@ def build_menu(cli_mode=False, quick_mode=False): def cpu_mprime_test(state, test_objects): + # pylint: disable=too-many-statements + #TODO: Fix above? """CPU & cooling check using Prime95.""" LOG.info('CPU Test (Prime95)') thermal_abort = False @@ -330,7 +333,26 @@ def cpu_mprime_test(state, test_objects): std.print_info('Starting stress test') std.print_warning('If running too hot, press CTRL+c to abort the test') set_apple_fan_speed('max') - #RUN: mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "prime.log" + proc_mprime = subprocess.Popen( + ['mprime', '-t'], + bufsize=1, + cwd=state.log_dir, + stdout=subprocess.PIPE, + ) + proc_grep = subprocess.Popen( + 'grep --ignore-case --invert-match --line-buffered stress.txt'.split(), + bufsize=1, + stdin=proc_mprime.stdout, + stdout=subprocess.PIPE, + ) + proc_mprime.stdout.close() + save_nsbr = exe.NonBlockingStreamReader(proc_grep.stdout) + save_thread = exe.start_thread( + save_nsbr.save_to_file, + args=(proc_grep, prime_log), + ) + + # Show countdown try: print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) except KeyboardInterrupt: @@ -341,7 +363,10 @@ def cpu_mprime_test(state, test_objects): thermal_abort = True # Stop Prime95 - #TODO kill p95 + proc_mprime.send_signal(signal.SIGINT) + std.sleep(1) + proc_mprime.kill() + save_thread.join() tmux.kill_pane(state.panes.pop('Prime95', None)) # Get cooldown temp @@ -353,7 +378,7 @@ def cpu_mprime_test(state, test_objects): sensors.save_average_temps(temp_label='Cooldown', seconds=5) # Check results and build report - #TODO + std.print_report(sensors.generate_report('Current', 'Idle', 'Max','Cooldown')) # Stop sensors monitor sensors_out.with_suffix('.stop').touch() @@ -363,13 +388,7 @@ def cpu_mprime_test(state, test_objects): state.panes.pop('Current', None) tmux.kill_pane(state.panes.pop('Temps', None)) - - - - - #TODO: p95 - std.print_warning('TODO: p95') std.pause() @@ -586,18 +605,17 @@ def network_test(): def print_countdown(seconds): """Print countdown to screen.""" - time_limit = seconds for i in range(seconds): sec_left = (seconds - i) % 60 min_left = int((seconds - i) / 60) - out_str = '\r' + out_str = '\r ' if min_left: out_str += f'{min_left} minute{"s" if min_left != 1 else ""}, ' out_str += f'{sec_left} second{"s" if sec_left != 1 else ""}' out_str += ' remaining' - print(f'{out_str:<40}', end='', flush=True) + print(f'{out_str:<42}', end='', flush=True) std.sleep(1) # Done From 45086c90bb2f981fd2494e7b4b8e3a21ded28e3b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 14 Nov 2019 19:13:21 -0700 Subject: [PATCH 176/324] Prime95 test fully functional --- scripts/wk/hw/diags.py | 157 +++++++++++++++++++++++++++------------ scripts/wk/hw/sensors.py | 35 ++++++++- 2 files changed, 140 insertions(+), 52 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 4a83a5bf..96e05951 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -8,7 +8,6 @@ import pathlib import platform import plistlib import re -import signal import subprocess import time @@ -270,7 +269,7 @@ def build_menu(cli_mode=False, quick_mode=False): # Update default selections for quick mode if necessary if quick_mode: - for name in menu.options.keys(): + for name in menu.options: # Only select quick option(s) menu.options[name]['Selected'] = name in MENU_OPTIONS_QUICK @@ -291,29 +290,79 @@ def build_menu(cli_mode=False, quick_mode=False): return menu +def check_mprime_results(test_obj, working_dir): + """Check mprime log files to determine if test passed.""" + passing_lines = {} + warning_lines = {} + + def _read_file(log_name): + """Read file and split into lines, returns list.""" + lines = [] + try: + with open(f'{working_dir}/{log_name}', 'r') as _f: + lines = _f.readlines() + except FileNotFoundError: + # File may be missing on older systems + lines = [] + + return lines + + # results.txt (check if failed) + for line in _read_file('results.txt'): + line = line.strip() + if re.search(r'(error|fail)', line, re.IGNORECASE): + warning_lines[line] = None + + # print.log (check if passed) + for line in _read_file('prime.log'): + line = line.strip() + match = re.search( + r'(completed.*(\d+) errors, (\d+) warnings)', line, re.IGNORECASE) + if match: + if int(match.group(2)) + int(match.group(3)) > 0: + # Errors and/or warnings encountered + warning_lines[match.group(1).capitalize()] = None + else: + # No errors/warnings + passing_lines[match.group(1).capitalize()] = None + + # Update status + if warning_lines: + test_obj.failed = True + test_obj.set_status('Failed') + elif passing_lines and 'Aborted' not in test_obj.status: + test_obj.passed = True + test_obj.set_status('Passed') + else: + test_obj.set_status('Unknown') + + # Update report + for line in passing_lines: + test_obj.report.append(f' {line}') + for line in warning_lines: + test_obj.report.append(std.color_string(f' {line}', 'YELLOW')) + if not (passing_lines or warning_lines): + test_obj.report.append(std.color_string(' Unknown result', 'YELLOW')) + + def cpu_mprime_test(state, test_objects): - # pylint: disable=too-many-statements - #TODO: Fix above? """CPU & cooling check using Prime95.""" LOG.info('CPU Test (Prime95)') - thermal_abort = False prime_log = pathlib.Path(f'{state.log_dir}/prime.log') - test = test_objects[0] + sensors_out = pathlib.Path(f'{state.log_dir}/sensors.out') + test_obj = test_objects[0] # Bail early - if test.disabled: + if test_obj.disabled: return # Prep - dev = test.dev - test.set_status('Working') - state.update_top_pane(dev.description) + state.update_top_pane(test_obj.dev.description) + test_obj.set_status('Working') # Start sensors monitor sensors = hw_sensors.Sensors() - sensors_out = pathlib.Path(f'{state.log_dir}/sensors.out') - sensors_thread = exe.start_thread( - sensors.monitor_to_file, args=(sensors_out,)) + sensors.start_background_monitor(sensors_out) # Create monitor and worker panes state.panes['Prime95'] = tmux.split_window( @@ -333,44 +382,27 @@ def cpu_mprime_test(state, test_objects): std.print_info('Starting stress test') std.print_warning('If running too hot, press CTRL+c to abort the test') set_apple_fan_speed('max') - proc_mprime = subprocess.Popen( - ['mprime', '-t'], - bufsize=1, - cwd=state.log_dir, - stdout=subprocess.PIPE, - ) - proc_grep = subprocess.Popen( - 'grep --ignore-case --invert-match --line-buffered stress.txt'.split(), - bufsize=1, - stdin=proc_mprime.stdout, - stdout=subprocess.PIPE, - ) - proc_mprime.stdout.close() - save_nsbr = exe.NonBlockingStreamReader(proc_grep.stdout) - save_thread = exe.start_thread( - save_nsbr.save_to_file, - args=(proc_grep, prime_log), - ) + proc_mprime = start_mprime_thread(state.log_dir, prime_log) # Show countdown try: - print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) + #print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) + print_countdown(seconds=7) except KeyboardInterrupt: - test.set_status('Aborted') + test_obj.set_status('Aborted') except hw_sensors.ThermalLimitReachedError: - test.set_status('Failed') - test.failed = True - thermal_abort = True + test_obj.failed = True + test_obj.set_status('Failed') # Stop Prime95 - proc_mprime.send_signal(signal.SIGINT) - std.sleep(1) - proc_mprime.kill() - save_thread.join() - tmux.kill_pane(state.panes.pop('Prime95', None)) + proc_mprime.terminate() + try: + proc_mprime.wait(timeout=5) + except subprocess.TimeoutExpired: + proc_mprime.kill() + set_apple_fan_speed('auto') # Get cooldown temp - set_apple_fan_speed('auto') std.clear_screen() std.print_standard('Letting CPU cooldown...') std.sleep(5) @@ -378,18 +410,21 @@ def cpu_mprime_test(state, test_objects): sensors.save_average_temps(temp_label='Cooldown', seconds=5) # Check results and build report - std.print_report(sensors.generate_report('Current', 'Idle', 'Max','Cooldown')) - - # Stop sensors monitor - sensors_out.with_suffix('.stop').touch() - sensors_thread.join() + test_obj.report.append(std.color_string('Prime95', 'BLUE')) + check_mprime_results(test_obj=test_obj, working_dir=state.log_dir) + test_obj.report.append(std.color_string('Temps', 'BLUE')) + for line in sensors.generate_report( + 'Idle', 'Max', 'Cooldown', only_cpu=True): + test_obj.report.append(f' {line}') # Cleanup + sensors.stop_background_monitor() state.panes.pop('Current', None) + tmux.kill_pane(state.panes.pop('Prime95', None)) tmux.kill_pane(state.panes.pop('Temps', None)) #TODO: p95 - std.pause() + std.print_report(test_obj.report) def disk_attribute_check(state, test_objects): @@ -706,5 +741,31 @@ def set_apple_fan_speed(speed): exe.run_program(cmd, check=False) +def start_mprime_thread(working_dir, log_path): + """Start mprime and save filtered output to log, returns Popen object.""" + proc_mprime = subprocess.Popen( + ['mprime', '-t'], + bufsize=1, + cwd=working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + proc_grep = subprocess.Popen( + 'grep --ignore-case --invert-match --line-buffered stress.txt'.split(), + bufsize=1, + stdin=proc_mprime.stdout, + stdout=subprocess.PIPE, + ) + proc_mprime.stdout.close() + save_nsbr = exe.NonBlockingStreamReader(proc_grep.stdout) + exe.start_thread( + save_nsbr.save_to_file, + args=(proc_grep, log_path), + ) + + # Return objects + return proc_mprime + + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index fce71d32..9956bf1a 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -10,7 +10,7 @@ import re from subprocess import CalledProcessError from wk.cfg.hw import CPU_THERMAL_LIMIT, SMC_IDS, TEMP_COLORS -from wk.exe import run_program +from wk.exe import run_program, start_thread from wk.std import color_string, sleep @@ -23,6 +23,7 @@ SMC_REGEX = re.compile( r'\s+(?P.*?)' r'\s*\(bytes (?P.*)\)$' ) +SENSOR_SOURCE_WIDTH = 25 if platform.system() == 'Darwin' else 20 # Error Classes @@ -34,7 +35,9 @@ class ThermalLimitReachedError(RuntimeError): class Sensors(): """Class for holding sensor specific data.""" def __init__(self): + self.background_thread = None self.data = get_sensor_data() + self.out_path = None def clear_temps(self): """Clear saved temps but keep structure""" @@ -55,7 +58,7 @@ class Sensors(): for adapter, sources in sorted(adapters.items()): report.append(fix_sensor_name(adapter)) for source, source_data in sorted(sources.items()): - line = f'{fix_sensor_name(source):25} ' + line = f'{fix_sensor_name(source):{SENSOR_SOURCE_WIDTH}} ' for label in temp_labels: if label != 'Current': line += f' {label.lower()}: ' @@ -78,12 +81,16 @@ class Sensors(): # Done return report - def monitor_to_file(self, out_path): + def monitor_to_file(self, out_path, temp_labels=None): """Write report to path every second until stopped.""" stop_path = pathlib.Path(out_path).resolve().with_suffix('.stop') + if not temp_labels: + temp_labels = ('Current', 'Max') + + # Start loop while True: self.update_sensor_data() - report = self.generate_report('Current', 'Max') + report = self.generate_report(*temp_labels) with open(out_path, 'w') as _f: _f.write('\n'.join(report)) @@ -111,6 +118,26 @@ class Sensors(): temps = source_data['Temps'] source_data[temp_label] = sum(temps) / len(temps) + def start_background_monitor(self, out_path, temp_labels=None): + """Start background thread to save report to file.""" + if self.background_thread: + raise RuntimeError('Background thread already running') + + self.out_path = pathlib.Path(out_path) + self.background_thread = start_thread( + self.monitor_to_file, + args=(out_path, temp_labels), + ) + + def stop_background_monitor(self): + """Stop background thread.""" + self.out_path.with_suffix('.stop').touch() + self.background_thread.join() + + # Reset vars to None + self.background_thread = None + self.out_path = None + def update_sensor_data(self, exit_on_thermal_limit=True): """Update sensor data via OS-specific means.""" if platform.system() == 'Darwin': From fec2473b9330d3d3d41a91ebb57ce514cb44c403 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 14 Nov 2019 19:16:10 -0700 Subject: [PATCH 177/324] Fixed Prime95 test length --- scripts/wk/hw/diags.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 96e05951..c39041e6 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -386,8 +386,7 @@ def cpu_mprime_test(state, test_objects): # Show countdown try: - #print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) - print_countdown(seconds=7) + print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) except KeyboardInterrupt: test_obj.set_status('Aborted') except hw_sensors.ThermalLimitReachedError: From 402c4359a1e0077416f36931dda372b5a217d0a2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 14 Nov 2019 20:16:15 -0700 Subject: [PATCH 178/324] Split Prime95 and cooling Test() objects --- scripts/wk/cfg/hw.py | 2 +- scripts/wk/hw/diags.py | 65 ++++++++++++++++++++++++++++------------ scripts/wk/hw/sensors.py | 24 +++++++++++++-- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index fa303cba..9418bca3 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -14,9 +14,9 @@ ATTRIBUTE_COLORS = ( ('Error', 'RED'), ('Maximum', 'PURPLE'), ) +CPU_CRITICAL_TEMP = 99 CPU_FAILURE_TEMP = 90 CPU_TEST_MINUTES = 7 -CPU_THERMAL_LIMIT = 99 KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' KNOWN_DISK_ATTRIBUTES = { diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index c39041e6..c5bebd66 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -176,9 +176,14 @@ class State(): if not details['Selected']: continue if 'CPU' in name: - test_obj = hw_obj.Test(dev=self.cpu, label=name) - self.cpu.tests[name] = test_obj - self.tests[name]['Objects'].append(test_obj) + # Create two Test objects which will both be used by cpu_mprime_test + # NOTE: Prime95 should be added first + test_mprime_obj = hw_obj.Test(dev=self.cpu, label='Prime95') + test_cooling_obj = hw_obj.Test(dev=self.cpu, label='Cooling') + self.cpu.tests[name] = test_mprime_obj + self.cpu.tests[name] = test_cooling_obj + self.tests[name]['Objects'].append(test_mprime_obj) + self.tests[name]['Objects'].append(test_cooling_obj) elif 'Disk' in name: for disk in self.disks: test_obj = hw_obj.Test(dev=disk, label=disk.path.name) @@ -290,8 +295,28 @@ def build_menu(cli_mode=False, quick_mode=False): return menu +def check_cooling_results(test_obj, sensors): + """Check cooling results and update test_obj.""" + max_temp = sensors.cpu_max_temp() + + # Check temps + if not max_temp: + test_obj.set_status('Unknown') + elif max_temp >= cfg.hw.CPU_FAILURE_TEMP: + test_obj.failed = True + test_obj.set_status('Failed') + elif 'Aborted' not in test_obj.status: + test_obj.passed = True + test_obj.set_status('Passed') + + # Add temps to report + for line in sensors.generate_report( + 'Idle', 'Max', 'Cooldown', only_cpu=True): + test_obj.report.append(f' {line}') + + def check_mprime_results(test_obj, working_dir): - """Check mprime log files to determine if test passed.""" + """Check mprime log files and update test_obj.""" passing_lines = {} warning_lines = {} @@ -350,15 +375,16 @@ def cpu_mprime_test(state, test_objects): LOG.info('CPU Test (Prime95)') prime_log = pathlib.Path(f'{state.log_dir}/prime.log') sensors_out = pathlib.Path(f'{state.log_dir}/sensors.out') - test_obj = test_objects[0] + test_mprime_obj, test_cooling_obj = test_objects # Bail early - if test_obj.disabled: + if test_cooling_obj.disabled or test_mprime_obj.disabled: return # Prep - state.update_top_pane(test_obj.dev.description) - test_obj.set_status('Working') + state.update_top_pane(test_mprime_obj.dev.description) + test_cooling_obj.set_status('Working') + test_mprime_obj.set_status('Working') # Start sensors monitor sensors = hw_sensors.Sensors() @@ -388,10 +414,10 @@ def cpu_mprime_test(state, test_objects): try: print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) except KeyboardInterrupt: - test_obj.set_status('Aborted') + test_cooling_obj.set_status('Aborted') + test_mprime_obj.set_status('Aborted') except hw_sensors.ThermalLimitReachedError: - test_obj.failed = True - test_obj.set_status('Failed') + test_mprime_obj.set_status('Aborted') # Stop Prime95 proc_mprime.terminate() @@ -408,13 +434,13 @@ def cpu_mprime_test(state, test_objects): std.print_standard('Saving cooldown temps...') sensors.save_average_temps(temp_label='Cooldown', seconds=5) - # Check results and build report - test_obj.report.append(std.color_string('Prime95', 'BLUE')) - check_mprime_results(test_obj=test_obj, working_dir=state.log_dir) - test_obj.report.append(std.color_string('Temps', 'BLUE')) - for line in sensors.generate_report( - 'Idle', 'Max', 'Cooldown', only_cpu=True): - test_obj.report.append(f' {line}') + # Check Prime95 results + test_mprime_obj.report.append(std.color_string('Prime95', 'BLUE')) + check_mprime_results(test_obj=test_mprime_obj, working_dir=state.log_dir) + + # Check Cooling results + test_cooling_obj.report.append(std.color_string('Temps', 'BLUE')) + check_cooling_results(test_obj=test_cooling_obj, sensors=sensors) # Cleanup sensors.stop_background_monitor() @@ -423,7 +449,8 @@ def cpu_mprime_test(state, test_objects): tmux.kill_pane(state.panes.pop('Temps', None)) #TODO: p95 - std.print_report(test_obj.report) + std.print_report(test_mprime_obj.report) + std.print_report(test_cooling_obj.report) def disk_attribute_check(state, test_objects): diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 9956bf1a..763f14fa 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -9,7 +9,7 @@ import re from subprocess import CalledProcessError -from wk.cfg.hw import CPU_THERMAL_LIMIT, SMC_IDS, TEMP_COLORS +from wk.cfg.hw import CPU_CRITICAL_TEMP, SMC_IDS, TEMP_COLORS from wk.exe import run_program, start_thread from wk.std import color_string, sleep @@ -46,6 +46,24 @@ class Sensors(): for source_data in sources.values(): source_data['Temps'] = [] + def cpu_max_temp(self): + """Get max temp from any CPU source, returns float. + + NOTE: If no temps are found this returns zero. + """ + max_temp = 0.0 + + # Check all CPU Temps + for section, adapters in self.data.items(): + if not section.startswith('CPU'): + continue + for sources in adapters.values(): + for source_data in sources.values(): + max_temp = max(max_temp, source_data.get('Max', 0)) + + # Done + return max_temp + def generate_report(self, *temp_labels, colored=True, only_cpu=False): """Generate report based on given temp_labels, returns list.""" report = [] @@ -163,7 +181,7 @@ class Sensors(): # Raise exception if thermal limit reached if exit_on_thermal_limit and section == 'CPUTemps': - if source_data['Current'] >= CPU_THERMAL_LIMIT: + if source_data['Current'] >= CPU_CRITICAL_TEMP: raise ThermalLimitReachedError('CPU temps reached limit') def update_sensor_data_macos(self, exit_on_thermal_limit=True): @@ -187,7 +205,7 @@ class Sensors(): # Raise exception if thermal limit reached if exit_on_thermal_limit and section == 'CPUTemps': - if source_data['Current'] >= CPU_THERMAL_LIMIT: + if source_data['Current'] >= CPU_CRITICAL_TEMP: raise ThermalLimitReachedError('CPU temps reached limit') From 79371a3fa5d756737113c43585aea48bdcd5ecb5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 14 Nov 2019 20:43:44 -0700 Subject: [PATCH 179/324] Added results screen to hw-diags --- scripts/wk/hw/diags.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index c5bebd66..336c802a 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -448,10 +448,6 @@ def cpu_mprime_test(state, test_objects): tmux.kill_pane(state.panes.pop('Prime95', None)) tmux.kill_pane(state.panes.pop('Temps', None)) - #TODO: p95 - std.print_report(test_mprime_obj.report) - std.print_report(test_cooling_obj.report) - def disk_attribute_check(state, test_objects): """Disk attribute check.""" @@ -718,7 +714,7 @@ def run_diags(state, menu, quick_mode=False): test_obj.set_status('Aborted') # Show results - #TODO: Show results + show_results(state) # Done if quick_mode: @@ -767,6 +763,32 @@ def set_apple_fan_speed(speed): exe.run_program(cmd, check=False) +def show_results(state): + """Show test results by device.""" + std.clear_screen() + state.update_top_pane('Results') + + # CPU Tests + cpu_tests_enabled = [data['Enabled'] for name, data in state.tests.items() + if name.startswith('CPU')] + if any(cpu_tests_enabled): + std.print_success('CPU:') + std.print_report(state.cpu.generate_report()) + std.print_standard(' ') + + # Disk Tests + disk_tests_enabled = [data['Enabled'] for name, data in state.tests.items() + if name.startswith('Disk')] + if any(disk_tests_enabled): + std.print_success(f'Disk{"s" if len(state.disks) > 1 else ""}:') + for disk in state.disks: + std.print_report(disk.generate_report()) + std.print_standard(' ') + if not state.disks: + std.print_warning('No devices') + std.print_standard(' ') + + def start_mprime_thread(working_dir, log_path): """Start mprime and save filtered output to log, returns Popen object.""" proc_mprime = subprocess.Popen( From 4cadb913e806ffeccc71985cb7a033f2f53c0c66 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 30 Nov 2019 21:29:24 -0700 Subject: [PATCH 180/324] Dropped bufsize=1 due to Python 3.8 warning --- scripts/wk/hw/diags.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 336c802a..75ce2c34 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -793,14 +793,12 @@ def start_mprime_thread(working_dir, log_path): """Start mprime and save filtered output to log, returns Popen object.""" proc_mprime = subprocess.Popen( ['mprime', '-t'], - bufsize=1, cwd=working_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) proc_grep = subprocess.Popen( 'grep --ignore-case --invert-match --line-buffered stress.txt'.split(), - bufsize=1, stdin=proc_mprime.stdout, stdout=subprocess.PIPE, ) From aa3b69f6fab855714fc17e535fdfadbea1059ed0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 30 Nov 2019 22:43:10 -0700 Subject: [PATCH 181/324] Added progress pane logic --- scripts/wk/hw/diags.py | 55 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 75ce2c34..5aa67a98 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -67,6 +67,17 @@ MENU_SETS = { MENU_TOGGLES = ( 'Skip USB Benchmarks', ) +STATUS_COLORS = { + 'Aborted': 'YELLOW', + 'Denied': 'RED', + 'ERROR': 'RED', + 'FAIL': 'RED', + 'N/A': 'YELLOW', + 'PASS': 'GREEN', + 'TimedOut': 'RED', + 'Unknown': 'YELLOW', + 'Working': 'YELLOW', + } WK_LABEL_REGEX = re.compile( fr'{cfg.main.KIT_NAME_SHORT}_(LINUX|UFD)', re.IGNORECASE, @@ -145,6 +156,8 @@ class State(): def init_diags(self, menu): """Initialize diagnostic pass.""" + std.print_info('Starting Hardware Diagnostics') + # Reset objects self.disks.clear() self.layout.clear() @@ -164,7 +177,13 @@ class State(): keep_history=False, timestamp=False, ) - std.print_info('Starting Hardware Diagnostics') + + # Progress Pane + self.update_progress_pane() + tmux.respawn_pane( + pane_id=self.panes['Progress'], + watch_file=f'{self.log_dir}/progress.out', + ) # Add HW Objects self.cpu = hw_obj.CpuRam() @@ -226,6 +245,32 @@ class State(): text=' ', ) + def update_progress_pane(self): + """Update progress pane.""" + report = [] + width = cfg.hw.TMUX_SIDE_WIDTH + + for name, details in self.tests.items(): + if not details['Enabled']: + continue + + # Add test details + report.append(std.color_string(name, 'BLUE')) + for test_obj in details['Objects']: + report.append(std.color_string( + [test_obj.label, f'{test_obj.status:>{width-len(test_obj.label)}}'], + [None, STATUS_COLORS.get(test_obj.status, None)], + sep='', + )) + + # Add spacer + report.append(' ') + + # Write to progress file + out_path = pathlib.Path(f'{self.log_dir}/progress.out') + with open(out_path, 'w') as _f: + _f.write('\n'.join(report)) + def update_top_pane(self, text): """Update top pane with text.""" tmux.respawn_pane(self.panes['Top'], text=f'{self.top_text}\n{text}') @@ -391,6 +436,7 @@ def cpu_mprime_test(state, test_objects): sensors.start_background_monitor(sensors_out) # Create monitor and worker panes + state.update_progress_pane() state.panes['Prime95'] = tmux.split_window( lines=10, vertical=True, watch_file=prime_log) state.panes['Temps'] = tmux.split_window( @@ -443,6 +489,7 @@ def cpu_mprime_test(state, test_objects): check_cooling_results(test_obj=test_cooling_obj, sensors=sensors) # Cleanup + state.update_progress_pane() sensors.stop_background_monitor() state.panes.pop('Current', None) tmux.kill_pane(state.panes.pop('Prime95', None)) @@ -613,14 +660,14 @@ def main(): screensaver('pipes') # Quit - if 'Quit' in selection: - break - elif 'Reboot' in selection: + if 'Reboot' in selection: cmd = ['/usr/local/bin/wk-power-command', 'reboot'] exe.run_program(cmd, check=False) elif 'Power Off' in selection: cmd = ['/usr/local/bin/wk-power-command', 'poweroff'] exe.run_program(cmd, check=False) + elif 'Quit' in selection: + break # Start diagnostics if 'Start' in selection: From 7796189d14bf304cb808170d488f9d1ccb658172 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 2 Dec 2019 17:54:48 -0700 Subject: [PATCH 182/324] Clear screen before all diag functions --- scripts/wk/hw/diags.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 5aa67a98..62669a03 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -287,7 +287,6 @@ def audio_test(): def audio_test_linux(): """Run an audio test using amixer and speaker-test.""" LOG.info('Audio Test') - std.clear_screen() # Set volume for source in ('Master', 'PCM'): @@ -446,7 +445,6 @@ def cpu_mprime_test(state, test_objects): state.layout['Current'] = {'height': 3, 'Check': True} # Get idle temps - std.clear_screen() std.print_standard('Saving idle temps...') sensors.save_average_temps(temp_label='Idle', seconds=5) @@ -474,7 +472,6 @@ def cpu_mprime_test(state, test_objects): set_apple_fan_speed('auto') # Get cooldown temp - std.clear_screen() std.print_standard('Letting CPU cooldown...') std.sleep(5) std.print_standard('Saving cooldown temps...') @@ -604,7 +601,6 @@ def keyboard_test(): """Test keyboard using xev.""" LOG.info('Keyboard Test (xev)') cmd = ['xev', '-event', 'keyboard'] - std.clear_screen() exe.run_program(cmd, check=False, pipe=False) @@ -625,7 +621,6 @@ def main(): # Quick Mode if args['--quick']: - std.clear_screen() run_diags(state, menu, quick_mode=True) return @@ -680,7 +675,6 @@ def main(): def network_test(): """Run network tests.""" LOG.info('Network Test') - std.clear_screen() try_and_print = std.TryAndPrint() result = try_and_print.run( 'Network connection...', net.connected_to_private_network, msg_good='OK') @@ -746,6 +740,7 @@ def run_diags(state, menu, quick_mode=False): # Run test(s) function = details['Function'] try: + std.clear_screen() function(state, details['Objects']) except std.GenericAbort: aborted = True From c520b5a86565edc2afa2313c587809a682d6972d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 2 Dec 2019 17:55:05 -0700 Subject: [PATCH 183/324] Update for Python 3.8 pylint alerts --- scripts/wk/exe.py | 1 + scripts/wk/hw/diags.py | 4 ++++ scripts/wk/std.py | 6 ++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index c5c1718b..b1d1927c 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -181,6 +181,7 @@ def popen_program(cmd, minimized=False, pipe=False, shell=False, **kwargs): def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): + # pylint: disable=subprocess-run-check """Run program and return a subprocess.CompletedProcess object.""" LOG.debug( 'cmd: %s, check: %s, pipe: %s, shell: %s', diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 62669a03..41d784f7 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -497,6 +497,7 @@ def disk_attribute_check(state, test_objects): """Disk attribute check.""" LOG.info('Disk Attribute Check') #TODO: at + LOG.debug('%s, %s', state, test_objects) std.print_warning('TODO: at') std.pause() @@ -505,6 +506,7 @@ def disk_io_benchmark(state, test_objects): """Disk I/O benchmark using dd.""" LOG.info('Disk I/O Benchmark (dd)') #TODO: io + LOG.debug('%s, %s', state, test_objects) std.print_warning('TODO: io') std.pause() @@ -513,6 +515,7 @@ def disk_self_test(state, test_objects): """Disk self-test if available.""" LOG.info('Disk Self-Test') #TODO: st + LOG.debug('%s, %s', state, test_objects) std.print_warning('TODO: st') std.pause() @@ -521,6 +524,7 @@ def disk_surface_scan(state, test_objects): """Disk surface scan using badblocks.""" LOG.info('Disk Surface Scan (badblocks)') #TODO: bb + LOG.debug('%s, %s', state, test_objects) std.print_warning('TODO: bb') std.pause() diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 32a107f6..d0eecde4 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -9,12 +9,16 @@ import os import pathlib import platform import re +import socket import subprocess import sys import time import traceback from collections import OrderedDict + +import requests + try: from termios import tcflush, TCIOFLUSH except ImportError: @@ -716,7 +720,6 @@ def color_string(strings, colors, sep=' '): def generate_debug_report(): """Generate debug report, returns str.""" - import socket platform_function_list = ( 'architecture', 'machine', @@ -971,7 +974,6 @@ def strip_colors(string): def upload_debug_report(report, compress=True, reason='DEBUG'): """Upload debug report to CRASH_SERVER as specified in wk.cfg.main.""" LOG.info('Uploading debug report to %s', CRASH_SERVER.get('Name', '?')) - import requests headers = CRASH_SERVER.get('Headers', {'X-Requested-With': 'XMLHttpRequest'}) if compress: headers['Content-Type'] = 'application/octet-stream' From 499053708231a5cbe11d3399c4aa66869284cb61 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 2 Dec 2019 20:11:02 -0700 Subject: [PATCH 184/324] Handle critical temps correctly in mprime sections * Moved ThermalLimitReachedError catches to wk.hw.sensors * Before they would never be caught and would never stop the script * Added cpu_reached_critical_temp() to wk.hw.sensors * This allows us to check if it happened without exceptions * Added thermal_action to wk.hw.sensors * This is run when ThermalLimitReachedError(s) are caught * Stop print_countdown if mprime is terminated * This is required since it may be killed in the background --- scripts/wk/hw/diags.py | 70 +++++++++++++++++++++++++++------------- scripts/wk/hw/sensors.py | 41 +++++++++++++++++++---- 2 files changed, 81 insertions(+), 30 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 41d784f7..602d3d16 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -68,15 +68,15 @@ MENU_TOGGLES = ( 'Skip USB Benchmarks', ) STATUS_COLORS = { + 'Passed': 'GREEN', 'Aborted': 'YELLOW', - 'Denied': 'RED', - 'ERROR': 'RED', - 'FAIL': 'RED', 'N/A': 'YELLOW', - 'PASS': 'GREEN', - 'TimedOut': 'RED', 'Unknown': 'YELLOW', 'Working': 'YELLOW', + 'Denied': 'RED', + 'ERROR': 'RED', + 'Failed': 'RED', + 'TimedOut': 'RED', } WK_LABEL_REGEX = re.compile( fr'{cfg.main.KIT_NAME_SHORT}_(LINUX|UFD)', @@ -417,6 +417,7 @@ def check_mprime_results(test_obj, working_dir): def cpu_mprime_test(state, test_objects): """CPU & cooling check using Prime95.""" LOG.info('CPU Test (Prime95)') + aborted = False prime_log = pathlib.Path(f'{state.log_dir}/prime.log') sensors_out = pathlib.Path(f'{state.log_dir}/sensors.out') test_mprime_obj, test_cooling_obj = test_objects @@ -432,7 +433,10 @@ def cpu_mprime_test(state, test_objects): # Start sensors monitor sensors = hw_sensors.Sensors() - sensors.start_background_monitor(sensors_out) + sensors.start_background_monitor( + sensors_out, + thermal_action=('killall', 'mprime'), + ) # Create monitor and worker panes state.update_progress_pane() @@ -450,28 +454,27 @@ def cpu_mprime_test(state, test_objects): # Stress CPU std.print_info('Starting stress test') - std.print_warning('If running too hot, press CTRL+c to abort the test') set_apple_fan_speed('max') - proc_mprime = start_mprime_thread(state.log_dir, prime_log) + proc_mprime = start_mprime(state.log_dir, prime_log) # Show countdown + print('') try: - print_countdown(seconds=cfg.hw.CPU_TEST_MINUTES*60) + print_countdown(proc=proc_mprime, seconds=cfg.hw.CPU_TEST_MINUTES*60) except KeyboardInterrupt: - test_cooling_obj.set_status('Aborted') - test_mprime_obj.set_status('Aborted') - except hw_sensors.ThermalLimitReachedError: - test_mprime_obj.set_status('Aborted') + aborted = True # Stop Prime95 - proc_mprime.terminate() - try: - proc_mprime.wait(timeout=5) - except subprocess.TimeoutExpired: - proc_mprime.kill() - set_apple_fan_speed('auto') + stop_mprime(proc_mprime) + + # Update progress if necessary + if sensors.cpu_reached_critical_temp() or aborted: + test_cooling_obj.set_status('Aborted') + test_mprime_obj.set_status('Aborted') + state.update_progress_pane() # Get cooldown temp + std.clear_screen() std.print_standard('Letting CPU cooldown...') std.sleep(5) std.print_standard('Saving cooldown temps...') @@ -705,8 +708,8 @@ def network_test(): std.pause('Press Enter to return to main menu...') -def print_countdown(seconds): - """Print countdown to screen.""" +def print_countdown(proc, seconds): + """Print countdown to screen while proc is alive.""" for i in range(seconds): sec_left = (seconds - i) % 60 min_left = int((seconds - i) / 60) @@ -718,7 +721,17 @@ def print_countdown(seconds): out_str += ' remaining' print(f'{out_str:<42}', end='', flush=True) - std.sleep(1) + try: + proc.wait(1) + except KeyboardInterrupt: + # Stop countdown + break + except subprocess.TimeoutExpired: + # proc still going, continue + pass + if proc.poll() is not None: + # proc exited, stop countdown + break # Done print('') @@ -835,8 +848,9 @@ def show_results(state): std.print_standard(' ') -def start_mprime_thread(working_dir, log_path): +def start_mprime(working_dir, log_path): """Start mprime and save filtered output to log, returns Popen object.""" + set_apple_fan_speed('max') proc_mprime = subprocess.Popen( ['mprime', '-t'], cwd=working_dir, @@ -859,5 +873,15 @@ def start_mprime_thread(working_dir, log_path): return proc_mprime +def stop_mprime(proc): + """Stop mprime gracefully, then forcefully as needed.""" + proc_mprime.terminate() + try: + proc_mprime.wait(timeout=5) + except subprocess.TimeoutExpired: + proc_mprime.kill() + set_apple_fan_speed('auto') + + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 763f14fa..dcf304be 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -64,6 +64,22 @@ class Sensors(): # Done return max_temp + def cpu_reached_critical_temp(self): + """Check if CPU reached CPU_CRITICAL_TEMP, returns bool.""" + for section, adapters in self.data.items(): + if not section.startswith('CPU'): + # Limit to CPU temps + continue + + # Ugly section + for sources in adapters.values(): + for source_data in sources.values(): + if source_data.get('Max', -1) >= CPU_CRITICAL_TEMP: + return True + + # Didn't return above so temps are within the threshold + return False + def generate_report(self, *temp_labels, colored=True, only_cpu=False): """Generate report based on given temp_labels, returns list.""" report = [] @@ -72,7 +88,7 @@ class Sensors(): if only_cpu and not section.startswith('CPU'): continue - # Ugly section + # Ugly section for adapter, sources in sorted(adapters.items()): report.append(fix_sensor_name(adapter)) for source, source_data in sorted(sources.items()): @@ -99,15 +115,22 @@ class Sensors(): # Done return report - def monitor_to_file(self, out_path, temp_labels=None): - """Write report to path every second until stopped.""" + def monitor_to_file(self, out_path, temp_labels=None, thermal_action=None): + """Write report to path every second until stopped. + + thermal_action is a cmd to run if ThermalLimitReachedError is caught. + """ stop_path = pathlib.Path(out_path).resolve().with_suffix('.stop') if not temp_labels: temp_labels = ('Current', 'Max') # Start loop while True: - self.update_sensor_data() + try: + self.update_sensor_data() + except ThermalLimitReachedError: + if thermal_action: + run_program(thermal_action, check=False) report = self.generate_report(*temp_labels) with open(out_path, 'w') as _f: _f.write('\n'.join(report)) @@ -136,15 +159,19 @@ class Sensors(): temps = source_data['Temps'] source_data[temp_label] = sum(temps) / len(temps) - def start_background_monitor(self, out_path, temp_labels=None): - """Start background thread to save report to file.""" + def start_background_monitor( + self, out_path, temp_labels=None, thermal_action=None): + """Start background thread to save report to file. + + thermal_action is a cmd to run if ThermalLimitReachedError is caught. + """ if self.background_thread: raise RuntimeError('Background thread already running') self.out_path = pathlib.Path(out_path) self.background_thread = start_thread( self.monitor_to_file, - args=(out_path, temp_labels), + args=(out_path, temp_labels, thermal_action), ) def stop_background_monitor(self): From 4dc41aec27a9a274584f890d61907922732bfbed Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 2 Dec 2019 20:31:33 -0700 Subject: [PATCH 185/324] Bugfix: mprime typos --- scripts/wk/hw/diags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 602d3d16..6a382a97 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -873,9 +873,9 @@ def start_mprime(working_dir, log_path): return proc_mprime -def stop_mprime(proc): +def stop_mprime(proc_mprime): """Stop mprime gracefully, then forcefully as needed.""" - proc_mprime.terminate() + proc_mprime.terminate() try: proc_mprime.wait(timeout=5) except subprocess.TimeoutExpired: From e041125c20ae73c38026140061de354284f62725 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 2 Dec 2019 21:02:12 -0700 Subject: [PATCH 186/324] Added hw-sensors --- scripts/hw-sensors | 46 ++++++++++++++++++++++++++++++++++++++++ scripts/wk/hw/sensors.py | 11 ++++++---- 2 files changed, 53 insertions(+), 4 deletions(-) create mode 100755 scripts/hw-sensors diff --git a/scripts/hw-sensors b/scripts/hw-sensors new file mode 100755 index 00000000..d4665466 --- /dev/null +++ b/scripts/hw-sensors @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Wizard Kit: Hardware Sensors""" +# vim: sts=2 sw=2 ts=2 + +import platform + +import wk + + +def main(): + """Show sensor data on screen.""" + sensors = wk.hw.sensors.Sensors() + if platform.system() == 'Darwin': + wk.std.clear_screen() + while True: + print('\033[100A', end='') + sensors.update_sensor_data() + wk.std.print_report(sensors.generate_report('Current', 'Max')) + wk.std.sleep(1) + elif platform.system() == 'Linux': + proc = wk.exe.run_program(cmd=['mktemp']) + sensors.start_background_monitor( + out_path=proc.stdout.strip(), + exit_on_thermal_limit=False, + temp_labels=('Current', 'Max'), + ) + watch_cmd = [ + 'watch', + '--color', + '--exec', + '--no-title', + '--interval', '1', + 'cat', + proc.stdout.strip(), + ] + wk.exe.run_program(watch_cmd, check=False, pipe=False) + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + pass + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index dcf304be..9b321674 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -115,7 +115,9 @@ class Sensors(): # Done return report - def monitor_to_file(self, out_path, temp_labels=None, thermal_action=None): + def monitor_to_file( + self, out_path, + exit_on_thermal_limit=True, temp_labels=None, thermal_action=None): """Write report to path every second until stopped. thermal_action is a cmd to run if ThermalLimitReachedError is caught. @@ -127,7 +129,7 @@ class Sensors(): # Start loop while True: try: - self.update_sensor_data() + self.update_sensor_data(exit_on_thermal_limit) except ThermalLimitReachedError: if thermal_action: run_program(thermal_action, check=False) @@ -160,7 +162,8 @@ class Sensors(): source_data[temp_label] = sum(temps) / len(temps) def start_background_monitor( - self, out_path, temp_labels=None, thermal_action=None): + self, out_path, + exit_on_thermal_limit=True, temp_labels=None, thermal_action=None): """Start background thread to save report to file. thermal_action is a cmd to run if ThermalLimitReachedError is caught. @@ -171,7 +174,7 @@ class Sensors(): self.out_path = pathlib.Path(out_path) self.background_thread = start_thread( self.monitor_to_file, - args=(out_path, temp_labels, thermal_action), + args=(out_path, exit_on_thermal_limit, temp_labels, thermal_action), ) def stop_background_monitor(self): From d1005ad0a97982a3344bb2fc1efcde3f34056634 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 2 Dec 2019 22:47:09 -0700 Subject: [PATCH 187/324] Updated sensor name formatting --- scripts/wk/hw/sensors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index 9b321674..a23a8575 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -244,8 +244,8 @@ def fix_sensor_name(name): """Cleanup sensor name, returns str.""" name = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', name, re.IGNORECASE) name = name.title() - name = name.replace('ACPItz', 'ACPI TZ') name = name.replace('Acpi', 'ACPI') + name = name.replace('ACPItz', 'ACPI TZ') name = name.replace('Coretemp', 'CoreTemp') name = name.replace('Cpu', 'CPU') name = name.replace('Id ', 'ID ') @@ -255,6 +255,7 @@ def fix_sensor_name(name): name = re.sub(r'(\D+)(\d+)', r'\1 \2', name, re.IGNORECASE) name = re.sub(r'^K (\d+)Temp', r'AMD K\1 Temps', name, re.IGNORECASE) name = re.sub(r'T(ctl|die)', r'CPU (T\1)', name, re.IGNORECASE) + name = re.sub(r'\s+', ' ', name) return name From c0b66067584f1d790f5f8289c82f8ebb375a2f90 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 3 Dec 2019 14:47:57 -0700 Subject: [PATCH 188/324] Stop Prime95 with INT signal instead of TERM --- scripts/wk/hw/diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 6a382a97..1a9758f9 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -435,7 +435,7 @@ def cpu_mprime_test(state, test_objects): sensors = hw_sensors.Sensors() sensors.start_background_monitor( sensors_out, - thermal_action=('killall', 'mprime'), + thermal_action=('killall', 'mprime', '-INT'), ) # Create monitor and worker panes From 6da34c1f2beebb3b88465d37c86415dc55dc581b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 3 Dec 2019 15:03:02 -0700 Subject: [PATCH 189/324] Only register tmux atexit when running HW Diags * Prevents unintended killing of tmux panes when importing wk or wk.hw --- scripts/wk/hw/diags.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 1a9758f9..316d2f8f 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -19,10 +19,6 @@ from wk.hw import obj as hw_obj from wk.hw import sensors as hw_sensors -# atexit functions -atexit.register(tmux.kill_all_panes) -#TODO: Add state/dev data dump debug function - # STATIC VARIABLES DOCSTRING = f'''{cfg.main.KIT_NAME_FULL}: Hardware Diagnostics @@ -623,6 +619,8 @@ def main(): raise RuntimeError('tmux session not found') # Init + atexit.register(tmux.kill_all_panes) + #TODO: Add state/dev data dump debug function menu = build_menu(cli_mode=args['--cli'], quick_mode=args['--quick']) state = State() From 445523e5f14117c3f5b91739b80d18b377fb5563 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 3 Dec 2019 15:18:10 -0700 Subject: [PATCH 190/324] Fix aborting Prime95 test --- scripts/wk/hw/diags.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 316d2f8f..dc98af20 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -721,9 +721,6 @@ def print_countdown(proc, seconds): print(f'{out_str:<42}', end='', flush=True) try: proc.wait(1) - except KeyboardInterrupt: - # Stop countdown - break except subprocess.TimeoutExpired: # proc still going, continue pass From b71bca45777d0c6af571c37f22de5b2f97ac40e6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 3 Dec 2019 16:31:26 -0700 Subject: [PATCH 191/324] Updated disk_attribute_check() --- scripts/wk/hw/diags.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index dc98af20..09a39197 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -495,10 +495,19 @@ def cpu_mprime_test(state, test_objects): def disk_attribute_check(state, test_objects): """Disk attribute check.""" LOG.info('Disk Attribute Check') - #TODO: at - LOG.debug('%s, %s', state, test_objects) - std.print_warning('TODO: at') - std.pause() + for test in test_objects: + if not test.dev.attributes: + # No NVMe/SMART data + test.set_status('N/A') + continue + + if test.dev.check_attributes(): + test.set_status('Passed') + else: + test.set_status('Failed') + + # Done + state.update_progress_pane() def disk_io_benchmark(state, test_objects): From 65c08ad9727b5aae8b34c031dd6e73a45b0964f4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 3 Dec 2019 17:36:52 -0700 Subject: [PATCH 192/324] Updated disk_self_test() * Parallel self-tests! --- scripts/wk/hw/diags.py | 75 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 09a39197..dbabb471 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -522,10 +522,77 @@ def disk_io_benchmark(state, test_objects): def disk_self_test(state, test_objects): """Disk self-test if available.""" LOG.info('Disk Self-Test') - #TODO: st - LOG.debug('%s, %s', state, test_objects) - std.print_warning('TODO: st') - std.pause() + aborted = False + threads = [] + state.panes['SMART'] = [] + + def _run_self_test(test_obj, log_path): + """Run self-test and handle exceptions.""" + result = None + + try: + test_obj.passed = test_obj.dev.run_self_test(log_path) + test_obj.failed = not test_obj.passed + except TimeoutError: + test_obj.failed = True + result = 'TimedOut' + except hw_obj.SMARTNotSupportedError: + result = 'N/A' + + # Set status + if result: + test_obj.set_status(result) + else: + if test_obj.failed: + test_obj.set_status('Failed') + elif test_obj.passed: + test_obj.set_status('Passed') + else: + test_obj.set_status('Unknown') + + # Run self-tests + std.print_info(f'Starting self-test{"s" if len(test_objects) > 1 else ""}') + for test in test_objects: + test.set_status('Working') + test_log = f'{state.log_dir}/{test.dev.path.name}_selftest.log' + + # Start thread + threads.append(exe.start_thread(_run_self_test, args=(test, test_log))) + + # Show progress + if threads[-1].is_alive(): + state.panes['SMART'].append( + tmux.split_window(lines=3, vertical=True, watch_file=test_log), + ) + + # Wait for all tests to complete + state.update_progress_pane() + try: + while True: + if any([t.is_alive() for t in threads]): + std.sleep(1) + else: + break + except KeyboardInterrupt: + aborted = True + + # Save report(s) + for test in test_objects: + if test.status != 'N/A': + test_details = test.dev.get_smart_self_test_details() + test_result = test_details.get('status', {}).get('string', 'Unknown') + test.report.append(std.color_string('Self-Test', 'BLUE')) + test.report.append(f' {test_result}') + if aborted and not (test.passed or test.failed): + test.report.append(std.color_string(' Aborted', 'YELLOW')) + elif test.status == 'TimedOut': + test.report.append(std.color_string(' TimedOut', 'YELLOW')) + + # Cleanup + state.update_progress_pane() + for pane in state.panes['SMART']: + tmux.kill_pane(pane) + state.panes.pop('SMART', None) def disk_surface_scan(state, test_objects): From fb4b44fefb62c88170db2786516610af7d3a2991 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 3 Dec 2019 18:16:33 -0700 Subject: [PATCH 193/324] Fixed temps pane under macOS --- scripts/wk/hw/diags.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index dbabb471..a17d2309 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -438,8 +438,12 @@ def cpu_mprime_test(state, test_objects): state.update_progress_pane() state.panes['Prime95'] = tmux.split_window( lines=10, vertical=True, watch_file=prime_log) - state.panes['Temps'] = tmux.split_window( - behind=True, percent=80, vertical=True, watch_file=sensors_out) + if platform.system() == 'Darwin': + state.panes['Temps'] = tmux.split_window( + behind=True, percent=80, vertical=True, cmd='./hw-sensors') + elif platform.system() == 'Linux': + state.panes['Temps'] = tmux.split_window( + behind=True, percent=80, vertical=True, watch_file=sensors_out) tmux.resize_pane(height=3) state.panes['Current'] = '' state.layout['Current'] = {'height': 3, 'Check': True} From da7c12bb618b3242919ff9c31c7f5defbb9f3dda Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 5 Dec 2019 14:20:17 -0700 Subject: [PATCH 194/324] Don't use dummy test objects when no disks avail --- scripts/wk/hw/diags.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index a17d2309..05b06c9f 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -205,13 +205,6 @@ class State(): disk.tests[name] = test_obj self.tests[name]['Objects'].append(test_obj) - # No disks detected? - if not self.tests[name]['Objects']: - test_obj = hw_obj.Test(dev=None, label='') - test_obj.set_status('N/A') - test_obj.disabled = True - self.tests[name]['Objects'].append(test_obj) - def init_tmux(self): """Initialize tmux layout.""" tmux.kill_all_panes() From 6167d0d78d5326f09aa8790ae348f1e01fca35e8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 5 Dec 2019 14:24:57 -0700 Subject: [PATCH 195/324] Get disk serial numbers under macOS --- scripts/wk/hw/obj.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 6cc8e3a5..87f72072 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -516,7 +516,7 @@ class Test(): self.label = label self.passed = False self.report = [] - self.status = '' + self.status = 'Pending' def set_status(self, status): """Update status string.""" @@ -590,10 +590,9 @@ def get_disk_details_macos(path): def get_disk_serial_macos(path): """Get disk serial using system_profiler, returns str.""" - serial = 'Unknown Serial' - # TODO: Make it real - str(path) - return serial + cmd = ['sudo', 'smartctl', '--info', '--json', path] + smart_info = get_json_from_command(cmd) + return smart_info.get('serial_number', 'Unknown Serial') def get_known_disk_attributes(model): From 76772be422b1a5a2ddfcaf7c60d1f0fd155550d5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 5 Dec 2019 22:20:26 -0700 Subject: [PATCH 196/324] Added badblocks sections * Supports running in parallel * Dropped NonBlockingStreamReader usage --- scripts/wk/cfg/hw.py | 4 +- scripts/wk/hw/diags.py | 105 ++++++++++++++++++++++++++++++++++++++--- scripts/wk/tmux.py | 14 +++--- 3 files changed, 109 insertions(+), 14 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 9418bca3..ddd73edf 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -14,6 +14,8 @@ ATTRIBUTE_COLORS = ( ('Error', 'RED'), ('Maximum', 'PURPLE'), ) +# NOTE: Force 4K read block size for disks >= 3TB +BADBLOCKS_LARGE_DISK = 3*1024**4 CPU_CRITICAL_TEMP = 99 CPU_FAILURE_TEMP = 90 CPU_TEST_MINUTES = 7 @@ -122,7 +124,7 @@ TMUX_LAYOUT = OrderedDict({ 'Temps': {'height': 1000, 'Check': False}, 'Prime95': {'height': 11, 'Check': False}, 'SMART': {'height': 3, 'Check': True}, - 'badblocks': {'height': 5, 'Check': True}, + 'badblocks': {'height': 3, 'Check': True}, 'I/O Benchmark': {'height': 1000, 'Check': False}, }) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 05b06c9f..f7cc562c 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -1,4 +1,5 @@ """WizardKit: Hardware diagnostics""" +# pylint: disable=too-many-lines # vim: sts=2 sw=2 ts=2 import atexit @@ -518,7 +519,7 @@ def disk_io_benchmark(state, test_objects): def disk_self_test(state, test_objects): """Disk self-test if available.""" - LOG.info('Disk Self-Test') + LOG.info('Disk Self-Test(s)') aborted = False threads = [] state.panes['SMART'] = [] @@ -548,6 +549,9 @@ def disk_self_test(state, test_objects): test_obj.set_status('Unknown') # Run self-tests + state.update_top_pane( + f'Disk self-test{"s" if len(test_objects) > 1 else ""}', + ) std.print_info(f'Starting self-test{"s" if len(test_objects) > 1 else ""}') for test in test_objects: test.set_status('Working') @@ -593,12 +597,101 @@ def disk_self_test(state, test_objects): def disk_surface_scan(state, test_objects): - """Disk surface scan using badblocks.""" + """Read-only disk surface scan using badblocks.""" LOG.info('Disk Surface Scan (badblocks)') - #TODO: bb - LOG.debug('%s, %s', state, test_objects) - std.print_warning('TODO: bb') - std.pause() + threads = [] + state.panes['badblocks'] = [] + + def _run_surface_scan(test_obj, log_path): + """Run surface scan and handle exceptions.""" + block_size = '1024' + dev = test_obj.dev + test_obj.report.append(std.color_string('badblocks', 'BLUE')) + test_obj.set_status('Working') + + # Increase block size if necessary + if (dev.details['phy-sec'] == 4096 + or dev.details['size'] >= cfg.hw.BADBLOCKS_LARGE_DISK): + block_size = '4096' + + # Start scan + cmd = ['sudo', 'badblocks', '-sv', '-b', block_size, '-e', '1', dev.path] + with open(log_path, 'a') as _f: + size_str = std.bytes_to_string(dev.details["size"], use_binary=False) + _f.write( + f'[{dev.path.name} {size_str}]\n', + ) + _f.flush() + exe.run_program( + cmd, + check=False, + pipe=False, + stderr=subprocess.STDOUT, + stdout=_f, + ) + + # Check results + with open(log_path, 'a') as _f: + for line in _f.readlines(): + line = line.strip() + if not line or line.startswith('Checking'): + # Skip + continue + if re.search(f'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): + test_obj.passed = True + test_obj.report.append(f' {line}') + test_obj.set_status('Passed') + else: + test_obj.failed = True + test_obj.report.append(f' {std.color_string(line, "YELLOW")}') + test_obj.set_status('Failed') + if not (test_obj.passed or test_obj.failed): + test_obj.set_status('Unknown') + + # Run surface scans + state.update_top_pane( + f'Disk Surface Scan{"s" if len(test_objects) > 1 else ""}', + ) + std.print_info( + f'Starting disk surface scan{"s" if len(test_objects) > 1 else ""}', + ) + for test in test_objects: + test_log = f'{state.log_dir}/{test.dev.path.name}_badblocks.log' + + # Start thread + threads.append(exe.start_thread(_run_surface_scan, args=(test, test_log))) + + # Show progress + if threads[-1].is_alive(): + state.panes['badblocks'].append( + tmux.split_window( + lines=3, + vertical=True, + watch_cmd='tail', + watch_file=test_log, + ), + ) + + # Wait for all tests to complete + try: + while True: + if any([t.is_alive() for t in threads]): + state.update_progress_pane() + std.sleep(5) + else: + break + except KeyboardInterrupt: + # Handle aborts + for test in test_objects: + if not (test.passed or test.failed or test.status == 'Unknown'): + test.set_status('Aborted') + test.report.append(std.color_string('Aborted', 'YELLOW')) + + # Cleanup + state.update_progress_pane() + for pane in state.panes['badblocks']: + tmux.kill_pane(pane) + state.panes.pop('badblocks', None) def get_disks(): diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 8639db6f..c4dfb2cd 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -159,14 +159,14 @@ def prep_action( elif watch_file: # Monitor file prep_file(watch_file) - action_cmd.extend([ - 'watch', - '--color', - '--no-title', - '--interval', '1', - ]) if watch_cmd == 'cat': - action_cmd.append('cat') + action_cmd.extend([ + 'watch', + '--color', + '--no-title', + '--interval', '1', + 'cat', + ]) elif watch_cmd == 'tail': action_cmd.extend(['tail', '--follow']) action_cmd.append(watch_file) From d173d317e314ad46c4af97eeb8cfcada1bb8e16f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 5 Dec 2019 22:57:13 -0700 Subject: [PATCH 197/324] Updated badblocks section * Start tests in reverse order (so they appear in order on screen) * Fixed report parsing --- scripts/wk/hw/diags.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index f7cc562c..5bc33ba0 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -553,7 +553,7 @@ def disk_self_test(state, test_objects): f'Disk self-test{"s" if len(test_objects) > 1 else ""}', ) std.print_info(f'Starting self-test{"s" if len(test_objects) > 1 else ""}') - for test in test_objects: + for test in reversed(test_objects): test.set_status('Working') test_log = f'{state.log_dir}/{test.dev.path.name}_selftest.log' @@ -631,10 +631,13 @@ def disk_surface_scan(state, test_objects): ) # Check results - with open(log_path, 'a') as _f: + with open(log_path, 'r') as _f: + # Skip first line + _f.readline() + # Check the rest for line in _f.readlines(): line = line.strip() - if not line or line.startswith('Checking'): + if not line or line.startswith('Checking' or line.startswith('[')): # Skip continue if re.search(f'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): @@ -655,7 +658,7 @@ def disk_surface_scan(state, test_objects): std.print_info( f'Starting disk surface scan{"s" if len(test_objects) > 1 else ""}', ) - for test in test_objects: + for test in reversed(test_objects): test_log = f'{state.log_dir}/{test.dev.path.name}_badblocks.log' # Start thread From e1ef9db6b657ce167dc628452859c6f5597fc002 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 5 Dec 2019 23:02:08 -0700 Subject: [PATCH 198/324] Color disk labels in badblocks panes --- scripts/wk/hw/diags.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 5bc33ba0..e6fad52a 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -619,7 +619,10 @@ def disk_surface_scan(state, test_objects): with open(log_path, 'a') as _f: size_str = std.bytes_to_string(dev.details["size"], use_binary=False) _f.write( - f'[{dev.path.name} {size_str}]\n', + std.color_string( + [dev.path.name, size_str, '\n'], + ['BLUE', 'CYAN', None], + ), ) _f.flush() exe.run_program( From 56a99a8a4e02477849f9a41dbb37f6ac9c6dcd99 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 12:25:48 -0700 Subject: [PATCH 199/324] Avoid crash if tmux pane closes while getting size --- scripts/wk/tmux.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index c4dfb2cd..5c6440b8 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -105,7 +105,11 @@ def layout_needs_fixed(panes, layout): if isinstance(pane_list, str): pane_list = [pane_list] for pane_id in pane_list: - width, height = get_pane_size(pane_id) + try: + width, height = get_pane_size(pane_id) + except ValueError: + # Pane may have disappeared during this loop + continue if data.get('width', False) and data['width'] != width: needs_fixed = True if data.get('height', False) and data['height'] != height: From 564745f03bf96a4fcbcc32bbfda6b255f70e0663 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 13:00:34 -0700 Subject: [PATCH 200/324] Adjusted wk.std.input_text() * Should hopefully reduce the duplicate prompts --- scripts/wk/std.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index d0eecde4..30a3c839 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -785,18 +785,20 @@ def input_text(prompt='Enter text'): response = None if prompt[-1:] != ' ': prompt += ' ' + print(prompt, end='', flush=True) while response is None: if os.name == 'posix': # Flush input to (hopefully) avoid EOFError tcflush(sys.stdin, TCIOFLUSH) try: - response = input(prompt) + response = input() LOG.debug('%s%s', prompt, response) except EOFError: # Ignore and try again - LOG.warning('Exception occured', exc_info=True) - print('', flush=True) + #LOG.warning('Exception occured', exc_info=True) + #print('', end='', flush=True) + pass return response From b45dc74e5a6ba87df4f45c967926108be4ee6e75 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 13:01:31 -0700 Subject: [PATCH 201/324] Start logging after updating log path --- scripts/wk/hw/diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index e6fad52a..13892ebb 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -153,7 +153,6 @@ class State(): def init_diags(self, menu): """Initialize diagnostic pass.""" - std.print_info('Starting Hardware Diagnostics') # Reset objects self.disks.clear() @@ -174,6 +173,7 @@ class State(): keep_history=False, timestamp=False, ) + std.print_info('Starting Hardware Diagnostics') # Progress Pane self.update_progress_pane() From a76d7775fdf284975ef4871a2d6a3921c3257335 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 13:02:57 -0700 Subject: [PATCH 202/324] Updated badblocks sections * Increaded pane height to 5 * Updated pass/fail/unknown logic * Reduced lines included in reports --- scripts/wk/cfg/hw.py | 2 +- scripts/wk/hw/diags.py | 40 ++++++++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index ddd73edf..5571ec1a 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -124,7 +124,7 @@ TMUX_LAYOUT = OrderedDict({ 'Temps': {'height': 1000, 'Check': False}, 'Prime95': {'height': 11, 'Check': False}, 'SMART': {'height': 3, 'Check': True}, - 'badblocks': {'height': 3, 'Check': True}, + 'badblocks': {'height': 5, 'Check': True}, 'I/O Benchmark': {'height': 1000, 'Check': False}, }) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 13892ebb..171ed232 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -32,6 +32,10 @@ Options: -h --help Show this page -q --quick Skip menu and perform a quick check ''' +BADBLOCKS_REGEX = re.compile( + r'^Pass completed, (\d+) bad blocks found. .(\d+)/(\d+)/(\d+) errors', + re.IGNORECASE, + ) LOG = logging.getLogger(__name__) MENU_ACTIONS = ( 'Audio Test', @@ -620,8 +624,9 @@ def disk_surface_scan(state, test_objects): size_str = std.bytes_to_string(dev.details["size"], use_binary=False) _f.write( std.color_string( - [dev.path.name, size_str, '\n'], - ['BLUE', 'CYAN', None], + ['[', dev.path.name, ' ', size_str, ']\n'], + [None, 'BLUE', None, 'CYAN', None], + sep='', ), ) _f.flush() @@ -635,22 +640,23 @@ def disk_surface_scan(state, test_objects): # Check results with open(log_path, 'r') as _f: - # Skip first line - _f.readline() - # Check the rest for line in _f.readlines(): - line = line.strip() - if not line or line.startswith('Checking' or line.startswith('[')): + line = std.strip_colors(line.strip()) + if not line or line.startswith('Checking') or line.startswith('['): # Skip continue - if re.search(f'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): - test_obj.passed = True - test_obj.report.append(f' {line}') - test_obj.set_status('Passed') + match = BADBLOCKS_REGEX.search(line) + if match: + if all([s == '0' for s in match.groups()]): + test_obj.passed = True + test_obj.report.append(f' {line}') + test_obj.set_status('Passed') + else: + test_obj.failed = True + test_obj.report.append(f' {std.color_string(line, "YELLOW")}') + test_obj.set_status('Failed') else: - test_obj.failed = True test_obj.report.append(f' {std.color_string(line, "YELLOW")}') - test_obj.set_status('Failed') if not (test_obj.passed or test_obj.failed): test_obj.set_status('Unknown') @@ -671,7 +677,7 @@ def disk_surface_scan(state, test_objects): if threads[-1].is_alive(): state.panes['badblocks'].append( tmux.split_window( - lines=3, + lines=5, vertical=True, watch_cmd='tail', watch_file=test_log, @@ -687,11 +693,12 @@ def disk_surface_scan(state, test_objects): else: break except KeyboardInterrupt: + std.sleep(0.5) # Handle aborts for test in test_objects: - if not (test.passed or test.failed or test.status == 'Unknown'): + if not (test.passed or test.failed): test.set_status('Aborted') - test.report.append(std.color_string('Aborted', 'YELLOW')) + test.report.append(std.color_string(' Aborted', 'YELLOW')) # Cleanup state.update_progress_pane() @@ -991,6 +998,7 @@ def set_apple_fan_speed(speed): def show_results(state): """Show test results by device.""" + std.sleep(0.5) std.clear_screen() state.update_top_pane('Results') From 2a4b68c222a2e3881705de5fc6919b3c74c78538 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 13:19:47 -0700 Subject: [PATCH 203/324] Fixed tail usage under macOS --- scripts/wk/tmux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 5c6440b8..478a81d7 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -172,7 +172,7 @@ def prep_action( 'cat', ]) elif watch_cmd == 'tail': - action_cmd.extend(['tail', '--follow']) + action_cmd.extend(['tail', '-f']) action_cmd.append(watch_file) else: LOG.error('No action specified') From 2c732885c6a5980298fe1884c30bf01c5c048695 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 13:21:36 -0700 Subject: [PATCH 204/324] Revert "Adjusted wk.std.input_text()" This reverts commit 564745f03bf96a4fcbcc32bbfda6b255f70e0663. --- scripts/wk/std.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 30a3c839..d0eecde4 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -785,20 +785,18 @@ def input_text(prompt='Enter text'): response = None if prompt[-1:] != ' ': prompt += ' ' - print(prompt, end='', flush=True) while response is None: if os.name == 'posix': # Flush input to (hopefully) avoid EOFError tcflush(sys.stdin, TCIOFLUSH) try: - response = input() + response = input(prompt) LOG.debug('%s%s', prompt, response) except EOFError: # Ignore and try again - #LOG.warning('Exception occured', exc_info=True) - #print('', end='', flush=True) - pass + LOG.warning('Exception occured', exc_info=True) + print('', flush=True) return response From c7585d17f05b17474cd546821d5273580ea3d2ed Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 15:02:06 -0700 Subject: [PATCH 205/324] Added graph functions --- scripts/wk/__init__.py | 1 + scripts/wk/graph.py | 120 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 scripts/wk/graph.py diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index 2356adb5..e24443d0 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -5,6 +5,7 @@ from sys import version_info as version from wk import cfg from wk import exe +from wk import graph from wk import hw from wk import io from wk import kit diff --git a/scripts/wk/graph.py b/scripts/wk/graph.py new file mode 100644 index 00000000..06f905f7 --- /dev/null +++ b/scripts/wk/graph.py @@ -0,0 +1,120 @@ +"""WizardKit: Graph Functions""" +# pylint: disable=bad-whitespace +# vim: sts=2 sw=2 ts=2 + +import logging + +from wk.std import color_string + + +# STATIC VARIABLES +LOG = logging.getLogger(__name__) +ALT_TEST_SIZE_FACTOR = 0.01 +BLOCK_SIZE = 512 * 1024 +CHUNK_SIZE = 32 * 1024**2 +GRAPH_HORIZONTAL = ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█') +GRAPH_WIDTH = 40 +GRAPH_VERTICAL = ( + '▏', '▎', '▍', '▌', + '▋', '▊', '▉', '█', + '█▏', '█▎', '█▍', '█▌', + '█▋', '█▊', '█▉', '██', + '██▏', '██▎', '██▍', '██▌', + '██▋', '██▊', '██▉', '███', + '███▏', '███▎', '███▍', '███▌', + '███▋', '███▊', '███▉', '████', + ) +MINIMUM_TEST_SIZE = 10 * 1024**3 +# SCALE_STEPS: These scales allow showing differences between HDDs and SSDs +# on the same graph. +SCALE_STEPS = { + 8: [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], + 16: [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], + 32: [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], + } +# THRESHOLDS: These are the rate_list (in MB/s) used to color graphs +THRESH_FAIL = 65 * 1024**2 +THRESH_WARN = 135 * 1024**2 +THRESH_GREAT = 750 * 1024**2 + + +# Functions +def generate_horizontal_graph(rate_list, oneline=False): + """Generate horizontal graph from rate_list, returns list.""" + graph = ['', '', '', ''] + scale = 8 if oneline else 32 + + # Build graph + for rate in merge_rates(rate_list): + step = get_graph_step(rate, scale=scale) + + # Set color + rate_color = None + if rate < THRESH_FAIL: + rate_color = 'RED' + elif rate < THRESH_WARN: + rate_color = 'YELLOW' + elif rate > THRESH_GREAT: + rate_color = 'GREEN' + + # Build graph + full_block = color_string((GRAPH_HORIZONTAL[-1],), (rate_color,)) + if step >= 24: + graph[0] += color_string((GRAPH_HORIZONTAL[step-24],), (rate_color,)) + graph[1] += full_block + graph[2] += full_block + graph[3] += full_block + elif step >= 16: + graph[0] += ' ' + graph[1] += color_string((GRAPH_HORIZONTAL[step-16],), (rate_color,)) + graph[2] += full_block + graph[3] += full_block + elif step >= 8: + graph[0] += ' ' + graph[1] += ' ' + graph[2] += color_string((GRAPH_HORIZONTAL[step-8],), (rate_color,)) + graph[3] += full_block + else: + graph[0] += ' ' + graph[1] += ' ' + graph[2] += ' ' + graph[3] += color_string((GRAPH_HORIZONTAL[step],), (rate_color,)) + + # Done + if oneline: + graph = graph[-1:] + return graph + + +def get_graph_step(rate, scale=16): + """Get graph step based on rate and scale, returns int.""" + rate_in_mb = rate / (1024**2) + step = 0 + + # Iterate over scale_steps backwards + for _r in range(scale-1, -1, -1): + if rate_in_mb >= SCALE_STEPS[scale][_r]: + step = _r + break + + # Done + return step + + +def merge_rates(rates, graph_width=GRAPH_WIDTH): + """Merge rates to have entries equal to the width, returns list.""" + merged_rates = [] + offset = 0 + slice_width = int(len(rates) / graph_width) + + # Merge rates + for _i in range(graph_width): + merged_rates.append(sum(rates[offset:offset+slice_width])/slice_width) + offset += slice_width + + # Done + return merged_rates + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From a0b07cbfde1e07f767e7bde1f4c71a316c903af7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 19:10:36 -0700 Subject: [PATCH 206/324] Added I/O Benchmark sections --- scripts/wk/cfg/hw.py | 9 +- scripts/wk/graph.py | 47 ++++++-- scripts/wk/hw/diags.py | 264 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 298 insertions(+), 22 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index 5571ec1a..ebc52737 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -15,7 +15,7 @@ ATTRIBUTE_COLORS = ( ('Maximum', 'PURPLE'), ) # NOTE: Force 4K read block size for disks >= 3TB -BADBLOCKS_LARGE_DISK = 3*1024**4 +BADBLOCKS_LARGE_DISK = 3 * 1024**4 CPU_CRITICAL_TEMP = 99 CPU_FAILURE_TEMP = 90 CPU_TEST_MINUTES = 7 @@ -115,6 +115,13 @@ TEMP_COLORS = { 90: 'RED', 100: 'ORANGE_RED', } +# THRESHOLDS: Rates used to determine HDD/SSD pass/fail +THRESH_HDD_MIN = 50 * 1024**2 +THRESH_HDD_AVG_HIGH = 75 * 1024**2 +THRESH_HDD_AVG_LOW = 65 * 1024**2 +THRESH_SSD_MIN = 90 * 1024**2 +THRESH_SSD_AVG_HIGH = 135 * 1024**2 +THRESH_SSD_AVG_LOW = 100 * 1024**2 TMUX_SIDE_WIDTH = 20 TMUX_LAYOUT = OrderedDict({ 'Top': {'height': 2, 'Check': True}, diff --git a/scripts/wk/graph.py b/scripts/wk/graph.py index 06f905f7..1bcb9c27 100644 --- a/scripts/wk/graph.py +++ b/scripts/wk/graph.py @@ -9,11 +9,7 @@ from wk.std import color_string # STATIC VARIABLES LOG = logging.getLogger(__name__) -ALT_TEST_SIZE_FACTOR = 0.01 -BLOCK_SIZE = 512 * 1024 -CHUNK_SIZE = 32 * 1024**2 GRAPH_HORIZONTAL = ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█') -GRAPH_WIDTH = 40 GRAPH_VERTICAL = ( '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█', @@ -24,7 +20,6 @@ GRAPH_VERTICAL = ( '███▏', '███▎', '███▍', '███▌', '███▋', '███▊', '███▉', '████', ) -MINIMUM_TEST_SIZE = 10 * 1024**3 # SCALE_STEPS: These scales allow showing differences between HDDs and SSDs # on the same graph. SCALE_STEPS = { @@ -39,13 +34,13 @@ THRESH_GREAT = 750 * 1024**2 # Functions -def generate_horizontal_graph(rate_list, oneline=False): +def generate_horizontal_graph(rate_list, graph_width=40, oneline=False): """Generate horizontal graph from rate_list, returns list.""" graph = ['', '', '', ''] scale = 8 if oneline else 32 # Build graph - for rate in merge_rates(rate_list): + for rate in merge_rates(rate_list, graph_width=graph_width): step = get_graph_step(rate, scale=scale) # Set color @@ -101,7 +96,7 @@ def get_graph_step(rate, scale=16): return step -def merge_rates(rates, graph_width=GRAPH_WIDTH): +def merge_rates(rates, graph_width=40): """Merge rates to have entries equal to the width, returns list.""" merged_rates = [] offset = 0 @@ -116,5 +111,41 @@ def merge_rates(rates, graph_width=GRAPH_WIDTH): return merged_rates +def vertical_graph_line(percent, rate, scale=32): + """Build colored graph string using thresholds, returns str.""" + color_bar = None + color_rate = None + step = get_graph_step(rate, scale=scale) + + # Set colors + if rate < THRESH_FAIL: + color_bar = 'RED' + color_rate = 'YELLOW' + elif rate < THRESH_WARN: + color_bar = 'YELLOW' + color_rate = 'YELLOW' + elif rate > THRESH_GREAT: + color_bar = 'GREEN' + color_rate = 'GREEN' + + # Build string + line = color_string( + strings=( + f'{percent:5.1f}%', + f'{GRAPH_VERTICAL[step]:<4}', + f'{rate/(1000**2):6.1f} MB/s', + ), + colors=( + None, + color_bar, + color_rate, + ), + sep=' ', + ) + + # Done + return line + + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 171ed232..f9aea016 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -15,7 +15,7 @@ import time from collections import OrderedDict from docopt import docopt -from wk import cfg, exe, log, net, std, tmux +from wk import cfg, exe, graph, log, net, std, tmux from wk.hw import obj as hw_obj from wk.hw import sensors as hw_sensors @@ -32,11 +32,17 @@ Options: -h --help Show this page -q --quick Skip menu and perform a quick check ''' +LOG = logging.getLogger(__name__) BADBLOCKS_REGEX = re.compile( r'^Pass completed, (\d+) bad blocks found. .(\d+)/(\d+)/(\d+) errors', re.IGNORECASE, ) -LOG = logging.getLogger(__name__) +IO_GRAPH_WIDTH = 40 +IO_ALT_TEST_SIZE_FACTOR = 0.01 +IO_BLOCK_SIZE = 512 * 1024 +IO_CHUNK_SIZE = 32 * 1024**2 +IO_MINIMUM_TEST_SIZE = 10 * 1024**3 +IO_RATE_REGEX = re.compile(r'(?P\d+) bytes.* (?P\S+) s,') MENU_ACTIONS = ( 'Audio Test', 'Keyboard Test', @@ -72,6 +78,7 @@ STATUS_COLORS = { 'Passed': 'GREEN', 'Aborted': 'YELLOW', 'N/A': 'YELLOW', + 'Skipped': 'YELLOW', 'Unknown': 'YELLOW', 'Working': 'YELLOW', 'Denied': 'RED', @@ -85,6 +92,11 @@ WK_LABEL_REGEX = re.compile( ) +# Error Classes +class DeviceTooSmallError(RuntimeError): + """Raised when a device is too small to test.""" + + # Classes class State(): """Object for tracking hardware diagnostic data.""" @@ -333,6 +345,58 @@ def build_menu(cli_mode=False, quick_mode=False): return menu +def calc_io_dd_values(dev_size): + """Calculate I/O benchmark dd values, returns dict. + + Calculations: + The minimum dev size is IO_GRAPH_WIDTH * IO_CHUNK_SIZE + (e.g. 1.25 GB for a width of 40 and a chunk size of 32MB) + + read_total is the area to be read in bytes + If the dev is < IO_MINIMUM_TEST_SIZE then it's the whole dev + Else it's the larger of IO_MINIMUM_TEST_SIZE or the alt test size + (determined by dev * IO_ALT_TEST_SIZE_FACTOR) + + read_chunks is the number of groups of IO_CHUNK_SIZE in test_obj.dev + This number is reduced to a multiple of IO_GRAPH_WIDTH in order + to allow for the data to be condensed cleanly + + read_blocks is the chunk size in number of blocks + (e.g. 64 if block size is 512KB and chunk size is 32MB + + skip_total is the number of IO_BLOCK_SIZE groups not tested + skip_blocks is the number of blocks to skip per IO_CHUNK_SIZE + skip_extra_rate is how often to add an additional skip block + This is needed to ensure an even testing across the dev + This is calculated by using the fractional amount left off + of the skip_blocks variable + """ + read_total = min(IO_MINIMUM_TEST_SIZE, dev_size) + read_total = max(read_total, dev_size*IO_ALT_TEST_SIZE_FACTOR) + read_chunks = int(read_total // IO_CHUNK_SIZE) + read_chunks -= read_chunks % IO_GRAPH_WIDTH + if read_chunks < IO_GRAPH_WIDTH: + raise DeviceTooSmallError + read_blocks = int(IO_CHUNK_SIZE / IO_BLOCK_SIZE) + read_total = read_chunks * IO_CHUNK_SIZE + skip_total = int((dev_size - read_total) // IO_BLOCK_SIZE) + skip_blocks = int((skip_total / read_chunks) // 1) + skip_extra_rate = 0 + try: + skip_extra_rate = 1 + int(1 / ((skip_total / read_chunks) % 1)) + except ZeroDivisionError: + # skip_extra_rate == 0 is fine + pass + + # Done + return { + 'Read Chunks': read_chunks, + 'Read Blocks': read_blocks, + 'Skip Blocks': skip_blocks, + 'Skip Extra': skip_extra_rate, + } + + def check_cooling_results(test_obj, sensors): """Check cooling results and update test_obj.""" max_temp = sensors.cpu_max_temp() @@ -353,6 +417,51 @@ def check_cooling_results(test_obj, sensors): test_obj.report.append(f' {line}') +def check_io_benchmark_results(test_obj, rate_list, graph_width): + """Generate colored report using rate_list, returns list of str.""" + avg_read = sum(rate_list) / len(rate_list) + min_read = min(rate_list) + max_read = max(rate_list) + if test_obj.dev.details['ssd']: + thresh_min = cfg.hw.THRESH_SSD_MIN + thresh_avg_high = cfg.hw.THRESH_SSD_AVG_HIGH + thresh_avg_low = cfg.hw.THRESH_SSD_AVG_LOW + else: + thresh_min = cfg.hw.THRESH_HDD_MIN + thresh_avg_high = cfg.hw.THRESH_HDD_AVG_HIGH + thresh_avg_low = cfg.hw.THRESH_HDD_AVG_LOW + + # Add horizontal graph to report + for line in graph.generate_horizontal_graph(rate_list, graph_width): + if not std.strip_colors(line).strip(): + # Skip empty lines + continue + test_obj.report.append(line) + + # Add read rates to report + test_obj.report.append( + f'Read speeds avg: {avg_read/(1000**2):3.1f}' + f' min: {min_read/(1000**2):3.1f}' + f' max: {max_read/(1000**2):3.1f}' + ) + + # Compare against thresholds + if min_read <= thresh_min and avg_read <= thresh_avg_high: + test_obj.failed = True + elif avg_read <= thresh_avg_low: + test_obj.failed = True + else: + test_obj.passed = True + + # Set status + if test_obj.failed: + test_obj.set_status('Failed') + elif test_obj.passed: + test_obj.set_status('Passed') + else: + test_obj.set_status('Unknown') + + def check_mprime_results(test_obj, working_dir): """Check mprime log files and update test_obj.""" passing_lines = {} @@ -512,16 +621,135 @@ def disk_attribute_check(state, test_objects): state.update_progress_pane() -def disk_io_benchmark(state, test_objects): +def disk_io_benchmark(state, test_objects, skip_usb=True): + # pylint: disable=too-many-statements """Disk I/O benchmark using dd.""" LOG.info('Disk I/O Benchmark (dd)') - #TODO: io - LOG.debug('%s, %s', state, test_objects) - std.print_warning('TODO: io') - std.pause() + aborted = False + + def _run_io_benchmark(test_obj, log_path): + """Run I/O benchmark and handle exceptions.""" + offset = 0 + read_rates = [] + test_obj.report.append(std.color_string('I/O Benchmark', 'BLUE')) + + # Get dd values or bail + try: + dd_values = calc_io_dd_values(test_obj.dev.details['size']) + except DeviceTooSmallError: + test_obj.set_status('N/A') + test_obj.report.append( + std.color_string('Disk too small to test', 'YELLOW'), + ) + return + + # Run dd read tests + for _i in range(dd_values['Read Chunks']): + _i += 1 + + # Build cmd + skip = dd_values['Skip Blocks'] + if dd_values['Skip Extra'] and _i % dd_values['Skip Extra'] == 0: + skip += 1 + cmd = [ + 'sudo', 'dd', + f'bs={IO_BLOCK_SIZE}', + f'skip={offset+skip}', + f'count={dd_values["Read Blocks"]}', + f'if={test_obj.dev.path}', + 'of=/dev/null', + ] + if platform.system() == 'Linux': + cmd.append('iflag=direct') + + # Run and get read rate + try: + proc = exe.run_program( + cmd, + pipe=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + except PermissionError: + # Since we're using sudo we can't kill dd + # Assuming this happened during a CTRL+c + raise KeyboardInterrupt + match = IO_RATE_REGEX.search(proc.stdout) + if match: + read_rates.append( + int(match.group('bytes')) / float(match.group('seconds')), + ) + match.group(1) + + # Show progress + with open(log_path, 'a') as _f: + if _i % 5 == 0: + percent = (_i / dd_values['Read Chunks']) * 100 + _f.write(f' {graph.vertical_graph_line(percent, read_rates[-1])}\n') + + # Update offset + offset += dd_values['Read Blocks'] + skip + + # Check results + check_io_benchmark_results(test_obj, read_rates, IO_GRAPH_WIDTH) + + # Run benchmarks + state.update_top_pane( + f'Disk I/O Benchmark{"s" if len(test_objects) > 1 else ""}', + ) + state.panes['I/O Benchmark'] = tmux.split_window( + percent=75, + vertical=True, + text=' ', + ) + for test in test_objects: + if test.disabled: + # Skip + continue + + # Skip USB devices if requested + if skip_usb and test.dev.details['bus'] == 'USB': + test.set_status('Skipped') + continue + + # Start benchmark + if not aborted: + std.clear_screen() + std.print_report(test.dev.generate_report()) + test.set_status('Working') + test_log = f'{state.log_dir}/{test.dev.path.name}_benchmark.out' + tmux.respawn_pane( + state.panes['I/O Benchmark'], + watch_cmd='tail', + watch_file=test_log, + ) + state.update_progress_pane() + try: + _run_io_benchmark(test, test_log) + except KeyboardInterrupt: + aborted = True + except (subprocess.CalledProcessError, TypeError, ValueError) as err: + # Something went wrong + test.set_status('ERROR') + print(' ') + print(err) + std.pause('lolwut?') + + # Mark test(s) aborted if necessary + if aborted: + test.set_status('Aborted') + test.report.append(std.color_string(' Aborted', 'YELLOW')) + + # Update progress after each test + state.update_progress_pane() + + # Cleanup + state.update_progress_pane() + tmux.kill_pane(state.panes.pop('I/O Benchmark', None)) def disk_self_test(state, test_objects): + # pylint: disable=too-many-statements """Disk self-test if available.""" LOG.info('Disk Self-Test(s)') aborted = False @@ -558,10 +786,13 @@ def disk_self_test(state, test_objects): ) std.print_info(f'Starting self-test{"s" if len(test_objects) > 1 else ""}') for test in reversed(test_objects): - test.set_status('Working') - test_log = f'{state.log_dir}/{test.dev.path.name}_selftest.log' + if test.disabled: + # Skip + continue # Start thread + test.set_status('Working') + test_log = f'{state.log_dir}/{test.dev.path.name}_selftest.log' threads.append(exe.start_thread(_run_self_test, args=(test, test_log))) # Show progress @@ -601,6 +832,7 @@ def disk_self_test(state, test_objects): def disk_surface_scan(state, test_objects): + # pylint: disable=too-many-statements """Read-only disk surface scan using badblocks.""" LOG.info('Disk Surface Scan (badblocks)') threads = [] @@ -668,9 +900,12 @@ def disk_surface_scan(state, test_objects): f'Starting disk surface scan{"s" if len(test_objects) > 1 else ""}', ) for test in reversed(test_objects): - test_log = f'{state.log_dir}/{test.dev.path.name}_badblocks.log' + if test.disabled: + # Skip + continue # Start thread + test_log = f'{state.log_dir}/{test.dev.path.name}_badblocks.log' threads.append(exe.start_thread(_run_surface_scan, args=(test, test_log))) # Show progress @@ -923,16 +1158,19 @@ def run_diags(state, menu, quick_mode=False): return # Run tests - for details in state.tests.values(): + for name, details in state.tests.items(): if not details['Enabled']: # Skip disabled tests continue # Run test(s) function = details['Function'] + args = [details['Objects']] + if name == 'Disk I/O Benchmark': + args.append(menu.toggles['Skip USB Benchmarks']['Selected']) + std.clear_screen() try: - std.clear_screen() - function(state, details['Objects']) + function(state, *args) except std.GenericAbort: aborted = True # Restart tmux From 1f74b0b989ea499a3bf0c3a32c9ab49b48a63f12 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 19:18:40 -0700 Subject: [PATCH 207/324] Use "RAW" disks under macOS --- scripts/wk/hw/diags.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index f9aea016..1a36317b 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -629,6 +629,10 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): def _run_io_benchmark(test_obj, log_path): """Run I/O benchmark and handle exceptions.""" + dev_path = test_obj.dev.path + platform.system() == 'Darwin': + # Use "RAW" disks under macOS + dev_path = dev_path.with_name(f'r{dev_path.name}') offset = 0 read_rates = [] test_obj.report.append(std.color_string('I/O Benchmark', 'BLUE')) @@ -656,7 +660,7 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): f'bs={IO_BLOCK_SIZE}', f'skip={offset+skip}', f'count={dd_values["Read Blocks"]}', - f'if={test_obj.dev.path}', + f'if={dev_path}', 'of=/dev/null', ] if platform.system() == 'Linux': From 8f909182d3add53970604fed67f315afad946152 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 19:21:24 -0700 Subject: [PATCH 208/324] Bugfix: typo and batch catch --- scripts/wk/hw/diags.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 1a36317b..dbf339eb 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -630,7 +630,7 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): def _run_io_benchmark(test_obj, log_path): """Run I/O benchmark and handle exceptions.""" dev_path = test_obj.dev.path - platform.system() == 'Darwin': + if platform.system() == 'Darwin': # Use "RAW" disks under macOS dev_path = dev_path.with_name(f'r{dev_path.name}') offset = 0 @@ -734,10 +734,8 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): aborted = True except (subprocess.CalledProcessError, TypeError, ValueError) as err: # Something went wrong + LOG.error('%s', err) test.set_status('ERROR') - print(' ') - print(err) - std.pause('lolwut?') # Mark test(s) aborted if necessary if aborted: From 126aaae8ba93c5529c086ff5d0f96445e9fd6ad2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 19:31:32 -0700 Subject: [PATCH 209/324] Fix IO_RATE_REGEX under macOS --- scripts/wk/hw/diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index dbf339eb..d101ba2d 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -42,7 +42,7 @@ IO_ALT_TEST_SIZE_FACTOR = 0.01 IO_BLOCK_SIZE = 512 * 1024 IO_CHUNK_SIZE = 32 * 1024**2 IO_MINIMUM_TEST_SIZE = 10 * 1024**3 -IO_RATE_REGEX = re.compile(r'(?P\d+) bytes.* (?P\S+) s,') +IO_RATE_REGEX = re.compile(r'(?P\d+) bytes.* (?P\S+) s(,|ecs )') MENU_ACTIONS = ( 'Audio Test', 'Keyboard Test', From 7d66b723ca4bed52dfb51b12c6a11dad9853f4e2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 6 Dec 2019 19:34:53 -0700 Subject: [PATCH 210/324] Update test report on ERROR --- scripts/wk/hw/diags.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index d101ba2d..f646df5a 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -42,7 +42,9 @@ IO_ALT_TEST_SIZE_FACTOR = 0.01 IO_BLOCK_SIZE = 512 * 1024 IO_CHUNK_SIZE = 32 * 1024**2 IO_MINIMUM_TEST_SIZE = 10 * 1024**3 -IO_RATE_REGEX = re.compile(r'(?P\d+) bytes.* (?P\S+) s(,|ecs )') +IO_RATE_REGEX = re.compile( + r'(?P\d+) bytes.* (?P\S+) s(?:,|ecs )', + ) MENU_ACTIONS = ( 'Audio Test', 'Keyboard Test', @@ -736,6 +738,7 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): # Something went wrong LOG.error('%s', err) test.set_status('ERROR') + test.report.append(std.color_string(' Unknown Error', 'RED')) # Mark test(s) aborted if necessary if aborted: From c09cd0c9c27ab4be7d35cbcdedff0db4cfca8f84 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 8 Dec 2019 15:29:18 -0700 Subject: [PATCH 211/324] Added disk safety check before each test * Includes possible workaround for SMART self-test TimedOut errors --- scripts/wk/hw/diags.py | 63 ++++++++++++++++++++++++++++++++++++++++++ scripts/wk/hw/obj.py | 19 +++++++------ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index f646df5a..5fda3ade 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -147,6 +147,51 @@ class State(): # exe.start_thread(self.fix_tmux_layout_loop) exe.start_thread(self.fix_tmux_layout_loop) + def disk_safety_checks(self, prep=False, wait_for_self_tests=True): + """Run disk safety checks.""" + self_tests_in_progress = False + for disk in self.disks: + disable_tests = False + try: + disk.safety_checks() + except hw_obj.CriticalHardwareError: + disable_tests = True + if 'Disk Attributes' in disk.tests: + disk.tests['Disk Attributes'].failed = True + disk.tests['Disk Attributes'].set_status('Failed') + except hw_obj.SMARTSelfTestInProgressError: + if prep: + std.print_warning(f'SMART self-test(s) in progress for {disk.path}') + if std.ask('Continue with all tests disabled for this device?'): + disable_tests = True + else: + std.print_standard('Diagnostics aborted.') + std.print_standard(' ') + std.pause('Press Enter to exit...') + raise SystemExit(1) + elif wait_for_self_tests: + self_tests_in_progress = True + else: + # Other tests will NOT be disabled + LOG.warning('SMART data may not be reliable for: %s', disk.path) + # Add note to report + if 'Disk Self-Test' in disk.tests: + disk.tests['Disk Self-Test'].failed = True + disk.tests['Disk Self-Test'].report.append( + std.color_string('Please manually review SMART data', 'YELLOW'), + ) + + # Disable tests if necessary + if disable_tests: + disable_disk_tests(disk) + + # Wait for self-test(s) + if self_tests_in_progress: + std.print_warning('SMART self-test(s) in progress') + std.print_standard('Waiting 60 seconds before continuing...') + std.sleep(60) + self.disk_safety_checks(wait_for_self_tests=False) + def fix_tmux_layout(self, forced=True, signum=None, frame=None): # pylint: disable=unused-argument """Fix tmux layout based on cfg.hw.TMUX_LAYOUT. @@ -224,6 +269,9 @@ class State(): disk.tests[name] = test_obj self.tests[name]['Objects'].append(test_obj) + # Run safety checks + #self.disk_safety_checks(prep=True) + def init_tmux(self): """Initialize tmux layout.""" tmux.kill_all_panes() @@ -605,6 +653,17 @@ def cpu_mprime_test(state, test_objects): tmux.kill_pane(state.panes.pop('Temps', None)) +def disable_disk_tests(disk): + """Disable remaining tests for disk.""" + LOG.warning('Disabling further tests for: %s', disk.path) + for name, test in disk.tests.items(): + if name == 'Disk Attributes': + continue + if test.status in ('Pending', 'Working'): + test.set_status('Denied') + test.disabled = True + + def disk_attribute_check(state, test_objects): """Disk attribute check.""" LOG.info('Disk Attribute Check') @@ -1168,6 +1227,10 @@ def run_diags(state, menu, quick_mode=False): # Skip disabled tests continue + # Run safety checks + if name.startswith('Disk') and name != 'Disk Attributes': + state.disk_safety_checks() + # Run test(s) function = details['Function'] args = [details['Objects']] diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 87f72072..bb5bd89e 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -33,6 +33,9 @@ class CriticalHardwareError(RuntimeError): class SMARTNotSupportedError(TypeError): """Exception used for disks lacking SMART support.""" +class SMARTSelfTestInProgressError(RuntimeError): + """Exception used when a SMART self-test is in progress.""" + # Classes class BaseObj(): @@ -370,17 +373,16 @@ class Disk(BaseObj): self.add_note(msg, 'RED') LOG.error('%s %s', self.path, msg) + # Raise blocking exception if necessary + if blocking_event_encountered: + raise CriticalHardwareError(f'Critical error(s) for: {self.path}') + # SMART self-test status test_details = self.get_smart_self_test_details() if 'remaining_percent' in test_details.get('status', ''): - blocking_event_encountered = True - msg = 'SMART self-test in progress' - self.add_note(msg, 'RED') - LOG.error('%s %s', self.path, msg) - - # Raise exception if necessary - if blocking_event_encountered: - raise CriticalHardwareError(f'Critical error(s) for: {self.path}') + msg = f'SMART self-test in progress for: {self.path}' + LOG.error(msg) + raise SMARTSelfTestInProgressError(msg) def run_self_test(self, log_path): """Run disk self-test and check if it passed, returns bool. @@ -508,6 +510,7 @@ class Disk(BaseObj): class Test(): + # pylint: disable=too-few-public-methods """Object for tracking test specific data.""" def __init__(self, dev, label): self.dev = dev From 9dc8329dec8ee2edfc8d8b8795fe07d50471fef4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 8 Dec 2019 16:37:37 -0700 Subject: [PATCH 212/324] Updated self-test sections * Improved abort handling * Always include report if state.tests['Disk Self-Test'] is enabled * Send abort command via smartctl if aborting self-test(s) --- scripts/wk/hw/diags.py | 52 +++++++++++++++++++++++++++++------------- scripts/wk/hw/obj.py | 5 ++++ 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 5fda3ade..9e7a656c 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -152,6 +152,11 @@ class State(): self_tests_in_progress = False for disk in self.disks: disable_tests = False + + # Skip already disabled devices + if all([test.disabled for test in disk.tests.values()]): + continue + try: disk.safety_checks() except hw_obj.CriticalHardwareError: @@ -270,7 +275,7 @@ class State(): self.tests[name]['Objects'].append(test_obj) # Run safety checks - #self.disk_safety_checks(prep=True) + self.disk_safety_checks(prep=True) def init_tmux(self): """Initialize tmux layout.""" @@ -567,6 +572,28 @@ def check_mprime_results(test_obj, working_dir): test_obj.report.append(std.color_string(' Unknown result', 'YELLOW')) +def check_self_test_results(test_obj, aborted=False): + """Check SMART self-test results.""" + test_obj.report.append(std.color_string('Self-Test', 'BLUE')) + if test_obj.disabled or test_obj.status == 'Denied': + test_obj.report.append(std.color_string(f' {test_obj.status}', 'RED')) + elif test_obj.status == 'N/A' or not test_obj.dev.attributes: + test_obj.report.append(std.color_string(f' {test_obj.status}', 'YELLOW')) + else: + # Not updating SMART data here to preserve the test status for the report + # For instance if the test was aborted the report should inlcude the last + # known progress instead of just "was aborted buy host" + test_details = test_obj.dev.get_smart_self_test_details() + test_result = test_details.get('status', {}).get('string', 'Unknown') + test_obj.report.append(f' {test_result}') + if aborted and not (test_obj.passed or test_obj.failed): + test_obj.report.append(std.color_string(' Aborted', 'YELLOW')) + test_obj.set_status('Aborted') + elif test_obj.status == 'TimedOut': + test_obj.report.append(std.color_string(' TimedOut', 'YELLOW')) + test_obj.set_status('TimedOut') + + def cpu_mprime_test(state, test_objects): """CPU & cooling check using Prime95.""" LOG.info('CPU Test (Prime95)') @@ -654,14 +681,12 @@ def cpu_mprime_test(state, test_objects): def disable_disk_tests(disk): - """Disable remaining tests for disk.""" - LOG.warning('Disabling further tests for: %s', disk.path) + """Disable all tests for disk.""" + LOG.warning('Disabling all tests for: %s', disk.path) for name, test in disk.tests.items(): - if name == 'Disk Attributes': - continue if test.status in ('Pending', 'Working'): test.set_status('Denied') - test.disabled = True + test.disabled = True def disk_attribute_check(state, test_objects): @@ -875,18 +900,13 @@ def disk_self_test(state, test_objects): break except KeyboardInterrupt: aborted = True + for test in test_objects: + test.dev.abort_self_test() + std.sleep(0.5) # Save report(s) for test in test_objects: - if test.status != 'N/A': - test_details = test.dev.get_smart_self_test_details() - test_result = test_details.get('status', {}).get('string', 'Unknown') - test.report.append(std.color_string('Self-Test', 'BLUE')) - test.report.append(f' {test_result}') - if aborted and not (test.passed or test.failed): - test.report.append(std.color_string(' Aborted', 'YELLOW')) - elif test.status == 'TimedOut': - test.report.append(std.color_string(' TimedOut', 'YELLOW')) + check_self_test_results(test, aborted=aborted) # Cleanup state.update_progress_pane() @@ -995,7 +1015,7 @@ def disk_surface_scan(state, test_objects): std.sleep(0.5) # Handle aborts for test in test_objects: - if not (test.passed or test.failed): + if not (test.disabled or test.passed or test.failed): test.set_status('Aborted') test.report.append(std.color_string(' Aborted', 'YELLOW')) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index bb5bd89e..bd2d18a1 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -151,6 +151,11 @@ class Disk(BaseObj): if not self.is_4k_aligned(): self.add_note('One or more partitions are not 4K aligned', 'YELLOW') + def abort_self_test(self): + """Abort currently running non-captive self-test.""" + cmd = ['sudo', 'smartctl', '--abort', self.path] + run_program(cmd, check=False) + def add_note(self, note, color=None): """Add note that will be included in the disk report.""" if color: From 376a9e92ba0d9d252b9f667a961d8c7b99ead56b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 8 Dec 2019 16:42:34 -0700 Subject: [PATCH 213/324] Supress a couple pylint warnings --- scripts/wk/hw/diags.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 9e7a656c..355f2424 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -148,6 +148,7 @@ class State(): exe.start_thread(self.fix_tmux_layout_loop) def disk_safety_checks(self, prep=False, wait_for_self_tests=True): + # pylint: disable=too-many-branches """Run disk safety checks.""" self_tests_in_progress = False for disk in self.disks: @@ -683,7 +684,7 @@ def cpu_mprime_test(state, test_objects): def disable_disk_tests(disk): """Disable all tests for disk.""" LOG.warning('Disabling all tests for: %s', disk.path) - for name, test in disk.tests.items(): + for test in disk.tests.values(): if test.status in ('Pending', 'Working'): test.set_status('Denied') test.disabled = True From 6071470b6af55071aac6e02d7b8c0d3a5665bb76 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 8 Dec 2019 16:52:20 -0700 Subject: [PATCH 214/324] Add note to disk report for critical HW error(s) --- scripts/wk/hw/diags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 355f2424..c4b81e9b 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -162,6 +162,7 @@ class State(): disk.safety_checks() except hw_obj.CriticalHardwareError: disable_tests = True + disk.add_note('Critical hardware error detected.', 'RED') if 'Disk Attributes' in disk.tests: disk.tests['Disk Attributes'].failed = True disk.tests['Disk Attributes'].set_status('Failed') From 82341dbbb356cefb4903c083d8c80c46ecf48cce Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 8 Dec 2019 17:02:10 -0700 Subject: [PATCH 215/324] Moved disk safety checks to after the test * This way failures during the last test should be caught --- scripts/wk/hw/diags.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index c4b81e9b..285fa93d 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -1249,10 +1249,6 @@ def run_diags(state, menu, quick_mode=False): # Skip disabled tests continue - # Run safety checks - if name.startswith('Disk') and name != 'Disk Attributes': - state.disk_safety_checks() - # Run test(s) function = details['Function'] args = [details['Objects']] @@ -1267,6 +1263,10 @@ def run_diags(state, menu, quick_mode=False): state.init_tmux() break + # Run safety checks + if name.startswith('Disk'): + state.disk_safety_checks(wait_for_self_tests=name != 'Disk Attributes') + # Handle aborts if aborted: for details in state.tests.values(): From 6bc4ce3c0b1f664d9cd82750ed75064d50b77802 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 14:29:28 -0700 Subject: [PATCH 216/324] Add Maximum value for power on hours --- scripts/wk/cfg/hw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wk/cfg/hw.py b/scripts/wk/cfg/hw.py index ebc52737..59cd7248 100644 --- a/scripts/wk/cfg/hw.py +++ b/scripts/wk/cfg/hw.py @@ -25,11 +25,11 @@ KNOWN_DISK_ATTRIBUTES = { # NVMe 'critical_warning': {'Blocking': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, 'media_errors': {'Blocking': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 'power_on_hours': {'Blocking': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 'power_on_hours': {'Blocking': False, 'Warning': 17532, 'Error': 26298, 'Maximum': 100000,}, 'unsafe_shutdowns': {'Blocking': False, 'Warning': 1, 'Error': None, 'Maximum': None, }, # SMART 5: {'Hex': '05', 'Blocking': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, - 9: {'Hex': '09', 'Blocking': False, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 9: {'Hex': '09', 'Blocking': False, 'Warning': 17532, 'Error': 26298, 'Maximum': 100000,}, 10: {'Hex': '10', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, 184: {'Hex': 'B8', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, 187: {'Hex': 'BB', 'Blocking': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, From 081658550b784df70fcf4da3c9841e89807d6d55 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 15:55:30 -0700 Subject: [PATCH 217/324] Added debug report sections * HW-Diags debug reports are saved after showing results or atexit --- scripts/wk/__init__.py | 1 + scripts/wk/debug.py | 45 ++++++++++++++++++++++++++++++++++++++++++ scripts/wk/hw/diags.py | 38 +++++++++++++++++++++++++++++++---- 3 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 scripts/wk/debug.py diff --git a/scripts/wk/__init__.py b/scripts/wk/__init__.py index e24443d0..b6a11b56 100644 --- a/scripts/wk/__init__.py +++ b/scripts/wk/__init__.py @@ -4,6 +4,7 @@ from sys import version_info as version from wk import cfg +from wk import debug from wk import exe from wk import graph from wk import hw diff --git a/scripts/wk/debug.py b/scripts/wk/debug.py new file mode 100644 index 00000000..437ab0f8 --- /dev/null +++ b/scripts/wk/debug.py @@ -0,0 +1,45 @@ +"""WizardKit: Debug Functions""" +# pylint: disable=invalid-name +# vim: sts=2 sw=2 ts=2 + + +# Classes +class Debug(): + # pylint: disable=too-few-public-methods + """Object used when dumping debug data.""" + def method(self): + """Dummy method used to identify functions vs data.""" + + +# STATIC VARIABLES +DEBUG_CLASS = Debug() +METHOD_TYPE = type(DEBUG_CLASS.method) + + +# Functions +def generate_object_report(obj, indent=0): + """Generate debug report for obj, returns list.""" + report = [] + + # Dump object data + for name in dir(obj): + attr = getattr(obj, name) + + # Skip methods and private attributes + if isinstance(attr, METHOD_TYPE) or name.startswith('_'): + continue + + # Add attribute to report (expanded if necessary) + if isinstance(attr, dict): + report.append(f'{name}:') + for key, value in sorted(attr.items()): + report.append(f'{" "*(indent+1)}{key}: {str(value)}') + else: + report.append(f'{" "*indent}{name}: {str(attr)}') + + # Done + return report + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 285fa93d..b9d2470d 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -15,7 +15,7 @@ import time from collections import OrderedDict from docopt import docopt -from wk import cfg, exe, graph, log, net, std, tmux +from wk import cfg, debug, exe, graph, log, net, std, tmux from wk.hw import obj as hw_obj from wk.hw import sensors as hw_sensors @@ -266,8 +266,8 @@ class State(): # NOTE: Prime95 should be added first test_mprime_obj = hw_obj.Test(dev=self.cpu, label='Prime95') test_cooling_obj = hw_obj.Test(dev=self.cpu, label='Cooling') - self.cpu.tests[name] = test_mprime_obj - self.cpu.tests[name] = test_cooling_obj + self.cpu.tests[test_mprime_obj.label] = test_mprime_obj + self.cpu.tests[test_cooling_obj.label] = test_cooling_obj self.tests[name]['Objects'].append(test_mprime_obj) self.tests[name]['Objects'].append(test_cooling_obj) elif 'Disk' in name: @@ -308,6 +308,34 @@ class State(): text=' ', ) + def save_debug_reports(self): + """Save debug reports to disk.""" + LOG.info('Saving debug reports') + debug_dir = pathlib.Path(f'{self.log_dir}/debug') + if not debug_dir.exists(): + debug_dir.mkdir() + + # State (self) + with open(f'{debug_dir}/state.report', 'a') as _f: + _f.write('\n'.join(debug.generate_object_report(self))) + + # CPU/RAM + with open(f'{debug_dir}/cpu.report', 'a') as _f: + _f.write('\n'.join(debug.generate_object_report(self.cpu))) + _f.write('\n\n[Tests]') + for name, test in self.cpu.tests.items(): + _f.write(f'\n{name}:\n') + _f.write('\n'.join(debug.generate_object_report(test, indent=1))) + + # Disks + for disk in self.disks: + with open(f'{debug_dir}/disk_{disk.path.name}.report', 'a') as _f: + _f.write('\n'.join(debug.generate_object_report(disk))) + _f.write('\n\n[Tests]') + for name, test in disk.tests.items(): + _f.write(f'\n{name}:\n') + _f.write('\n'.join(debug.generate_object_report(test, indent=1))) + def update_progress_pane(self): """Update progress pane.""" report = [] @@ -1120,7 +1148,6 @@ def main(): # Init atexit.register(tmux.kill_all_panes) - #TODO: Add state/dev data dump debug function menu = build_menu(cli_mode=args['--cli'], quick_mode=args['--quick']) state = State() @@ -1235,6 +1262,7 @@ def print_countdown(proc, seconds): def run_diags(state, menu, quick_mode=False): """Run selected diagnostics.""" aborted = False + atexit.register(state.save_debug_reports) state.init_diags(menu) # Just return if no tests were selected @@ -1278,6 +1306,8 @@ def run_diags(state, menu, quick_mode=False): show_results(state) # Done + state.save_debug_reports() + atexit.unregister(state.save_debug_reports) if quick_mode: std.pause('Press Enter to exit...') else: From cc85e3e8ed8da62d7ff0e209e944625376db06ca Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 16:32:35 -0700 Subject: [PATCH 218/324] Improve abort handling --- scripts/wk/hw/diags.py | 58 ++++++++++++++++++++++++++++++++---------- scripts/wk/hw/obj.py | 8 ++++++ 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index b9d2470d..00f1d77c 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -147,6 +147,25 @@ class State(): # exe.start_thread(self.fix_tmux_layout_loop) exe.start_thread(self.fix_tmux_layout_loop) + def abort_testing(self): + """Set unfinished tests as aborted and cleanup tmux panes.""" + for details in self.tests.values(): + for test in details['Objects']: + if test.status in ('Pending', 'Working'): + test.set_status('Aborted') + + # Cleanup tmux + self.panes.pop('Current', None) + for key, pane_ids in self.panes.copy().items(): + if key in ('Top', 'Started', 'Progress'): + continue + if isinstance(pane_ids, str): + tmux.kill_pane(self.panes.pop(key)) + else: + for _id in pane_ids: + tmux.kill_pane(_id) + self.panes.pop(key) + def disk_safety_checks(self, prep=False, wait_for_self_tests=True): # pylint: disable=too-many-branches """Run disk safety checks.""" @@ -190,7 +209,7 @@ class State(): # Disable tests if necessary if disable_tests: - disable_disk_tests(disk) + disk.disable_disk_tests() # Wait for self-test(s) if self_tests_in_progress: @@ -622,9 +641,14 @@ def check_self_test_results(test_obj, aborted=False): elif test_obj.status == 'TimedOut': test_obj.report.append(std.color_string(' TimedOut', 'YELLOW')) test_obj.set_status('TimedOut') + else: + test_obj.failed = not test_obj.passed + if test_obj.failed: + test_obj.set_status('Failed') def cpu_mprime_test(state, test_objects): + # pylint: disable=too-many-statements """CPU & cooling check using Prime95.""" LOG.info('CPU Test (Prime95)') aborted = False @@ -709,14 +733,9 @@ def cpu_mprime_test(state, test_objects): tmux.kill_pane(state.panes.pop('Prime95', None)) tmux.kill_pane(state.panes.pop('Temps', None)) - -def disable_disk_tests(disk): - """Disable all tests for disk.""" - LOG.warning('Disabling all tests for: %s', disk.path) - for test in disk.tests.values(): - if test.status in ('Pending', 'Working'): - test.set_status('Denied') - test.disabled = True + # Done + if aborted: + raise std.GenericAbort('Aborted') def disk_attribute_check(state, test_objects): @@ -866,6 +885,10 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): state.update_progress_pane() tmux.kill_pane(state.panes.pop('I/O Benchmark', None)) + # Done + if aborted: + raise std.GenericAbort('Aborted') + def disk_self_test(state, test_objects): # pylint: disable=too-many-statements @@ -881,7 +904,6 @@ def disk_self_test(state, test_objects): try: test_obj.passed = test_obj.dev.run_self_test(log_path) - test_obj.failed = not test_obj.passed except TimeoutError: test_obj.failed = True result = 'TimedOut' @@ -944,11 +966,16 @@ def disk_self_test(state, test_objects): tmux.kill_pane(pane) state.panes.pop('SMART', None) + # Done + if aborted: + raise std.GenericAbort('Aborted') + def disk_surface_scan(state, test_objects): # pylint: disable=too-many-statements """Read-only disk surface scan using badblocks.""" LOG.info('Disk Surface Scan (badblocks)') + aborted = False threads = [] state.panes['badblocks'] = [] @@ -1042,6 +1069,7 @@ def disk_surface_scan(state, test_objects): else: break except KeyboardInterrupt: + aborted = True std.sleep(0.5) # Handle aborts for test in test_objects: @@ -1055,6 +1083,10 @@ def disk_surface_scan(state, test_objects): tmux.kill_pane(pane) state.panes.pop('badblocks', None) + # Done + if aborted: + raise std.GenericAbort('Aborted') + def get_disks(): """Get disks using OS-specific methods, returns list.""" @@ -1285,10 +1317,10 @@ def run_diags(state, menu, quick_mode=False): std.clear_screen() try: function(state, *args) - except std.GenericAbort: + except (KeyboardInterrupt, std.GenericAbort): aborted = True - # Restart tmux - state.init_tmux() + state.abort_testing() + state.update_progress_pane() break # Run safety checks diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index bd2d18a1..fe1405d0 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -195,6 +195,14 @@ class Disk(BaseObj): # Done return attributes_ok + def disable_disk_tests(self): + """Disable all tests.""" + LOG.warning('Disabling all tests for: %s', self.path) + for test in self.tests.values(): + if test.status in ('Pending', 'Working'): + test.set_status('Denied') + test.disabled = True + def enable_smart(self): """Try enabling SMART for this disk.""" cmd = [ From 23c99084b5741ba97471de4aede8d757351fa3b2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 16:48:15 -0700 Subject: [PATCH 219/324] Drop SIGWINCH sections --- scripts/wk/hw/diags.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 00f1d77c..347640d5 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -139,12 +139,6 @@ class State(): # Init tmux and start a background process to maintain layout self.init_tmux() - #TODO: Fix SIGWINCH? - #if hasattr(signal, 'SIGWINCH'): - # # Use signal handling - # signal.signal(signal.SIGWINCH, self.fix_tmux_layout) - #else: - # exe.start_thread(self.fix_tmux_layout_loop) exe.start_thread(self.fix_tmux_layout_loop) def abort_testing(self): @@ -218,13 +212,9 @@ class State(): std.sleep(60) self.disk_safety_checks(wait_for_self_tests=False) - def fix_tmux_layout(self, forced=True, signum=None, frame=None): + def fix_tmux_layout(self, forced=True): # pylint: disable=unused-argument - """Fix tmux layout based on cfg.hw.TMUX_LAYOUT. - - NOTE: To support being called by both a signal and a thread - signum and frame must be valid aguments. - """ + """Fix tmux layout based on cfg.hw.TMUX_LAYOUT.""" try: tmux.fix_layout(self.panes, self.layout, forced=forced) except RuntimeError: From 8e5bfa12f4a2f10611a6554b036e2417e130a012 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 16:48:35 -0700 Subject: [PATCH 220/324] Added NVMe SMART status checks * Addresses issue #130 --- scripts/wk/hw/obj.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index fe1405d0..bc1df561 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -24,6 +24,11 @@ from wk.std import bytes_to_string, color_string, sleep, string_to_bytes # STATIC VARIABLES LOG = logging.getLogger(__name__) +NVME_WARNING_KEYS = ( + 'spare_below_threshold', + 'reliability_degraded', + 'volatile_memory_backup_failed', + ) # Exception Classes @@ -371,7 +376,17 @@ class Disk(BaseObj): LOG.error('%s: Blocked for failing attribute(s)', self.path) # NVMe status - # TODO: See https://github.com/2Shirt/WizardKit/issues/130 + nvme_status = self.smartctl.get('smart_status', {}).get('nvme', {}) + if nvme_status.get('media_read_only', False): + blocking_event_encountered = True + msg = 'Media has been placed in read-only mode' + self.add_note(msg, 'RED') + LOG.error('%s %s', self.path, msg) + for key in NVME_WARNING_KEYS: + if nvme_status.get(key, False): + msg = key.replace('_', ' ') + self.add_note(msg, 'YELLOW') + LOG.warning('%s %s', self.path, msg) # SMART overall assessment smart_passed = True From e623185d963edf953f63a8338bfe12b5da3f2d8e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 17:09:56 -0700 Subject: [PATCH 221/324] Removed old HW script wrappers --- scripts/hw-drive-info | 39 ++++++++++ scripts/{outer_scripts_to_review => }/hw-info | 72 +++++++++---------- .../hw-diags-iobenchmark | 18 ----- scripts/outer_scripts_to_review/hw-diags-menu | 66 ----------------- .../outer_scripts_to_review/hw-diags-network | 48 ------------- .../outer_scripts_to_review/hw-diags-prime95 | 19 ----- scripts/outer_scripts_to_review/hw-drive-info | 39 ---------- scripts/outer_scripts_to_review/hw-sensors | 10 --- .../hw-sensors-monitor | 36 ---------- 9 files changed, 75 insertions(+), 272 deletions(-) create mode 100755 scripts/hw-drive-info rename scripts/{outer_scripts_to_review => }/hw-info (53%) delete mode 100755 scripts/outer_scripts_to_review/hw-diags-iobenchmark delete mode 100755 scripts/outer_scripts_to_review/hw-diags-menu delete mode 100755 scripts/outer_scripts_to_review/hw-diags-network delete mode 100755 scripts/outer_scripts_to_review/hw-diags-prime95 delete mode 100755 scripts/outer_scripts_to_review/hw-drive-info delete mode 100755 scripts/outer_scripts_to_review/hw-sensors delete mode 100755 scripts/outer_scripts_to_review/hw-sensors-monitor diff --git a/scripts/hw-drive-info b/scripts/hw-drive-info new file mode 100755 index 00000000..a1c051d8 --- /dev/null +++ b/scripts/hw-drive-info @@ -0,0 +1,39 @@ +#!/bin/bash +# + +BLUE='\033[34m' +CLEAR='\033[0m' +IFS=$'\n' + +# List devices +for line in $(lsblk -do NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL); do + if [[ "${line:0:4}" == "NAME" ]]; then + echo -e "${BLUE}${line}${CLEAR}" + else + echo "${line}" + fi +done +echo "" + +# List loopback devices +if [[ "$(losetup -l | wc -l)" > 0 ]]; then + for line in $(losetup -lO NAME,PARTSCAN,RO,BACK-FILE); do + if [[ "${line:0:4}" == "NAME" ]]; then + echo -e "${BLUE}${line}${CLEAR}" + else + echo "${line}" | sed -r 's#/dev/(loop[0-9]+)#\1 #' + fi + done + echo "" +fi + +# List partitions +for line in $(lsblk -o NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT); do + if [[ "${line:0:4}" == "NAME" ]]; then + echo -e "${BLUE}${line}${CLEAR}" + else + echo "${line}" + fi +done +echo "" + diff --git a/scripts/outer_scripts_to_review/hw-info b/scripts/hw-info similarity index 53% rename from scripts/outer_scripts_to_review/hw-info rename to scripts/hw-info index 8321e7aa..8a909c67 100755 --- a/scripts/outer_scripts_to_review/hw-info +++ b/scripts/hw-info @@ -9,20 +9,20 @@ YELLOW="\e[33m" BLUE="\e[34m" function print_in_columns() { - string="$1" - label="$(echo "$string" | sed -r 's/^\s*(.*:).*/\1/')" - value="$(echo "$string" | sed -r 's/^\s*.*:\s*(.*)/\1/')" - printf ' %-18s%s\n' "$label" "$value" + string="$1" + label="$(echo "$string" | sed -r 's/^\s*(.*:).*/\1/')" + value="$(echo "$string" | sed -r 's/^\s*.*:\s*(.*)/\1/')" + printf ' %-18s%s\n' "$label" "$value" } function print_dmi_value() { - name="$1" - file="/sys/devices/virtual/dmi/id/$2" - value="UNKNOWN" - if [[ -e "$file" ]]; then - value="$(cat "$file")" - fi - print_in_columns "$name: $value" + name="$1" + file="/sys/devices/virtual/dmi/id/$2" + value="UNKNOWN" + if [[ -e "$file" ]]; then + value="$(cat "$file")" + fi + print_in_columns "$name: $value" } # System @@ -50,58 +50,58 @@ echo "" # Processor echo -e "${BLUE}Processor${CLEAR}" lscpu | grep -E '^(Arch|CPU.s.|Core|Thread|Model name|Virt)' \ - | sed -r 's/\(s\)(.*:)/s\1 /' \ - | sed -r 's/CPUs: /Threads:/' \ - | sed -r 's/^(.*:) / \1/' + | sed -r 's/\(s\)(.*:)/s\1 /' \ + | sed -r 's/CPUs: /Threads:/' \ + | sed -r 's/^(.*:) / \1/' echo "" # Memory echo -e "${BLUE}Memory${CLEAR}" first_device="True" while read -r line; do - if [[ "$line" == "Memory Device" ]]; then - if [[ "$first_device" == "True" ]]; then - first_device="False" - else - # Add space between devices - echo "" - fi + if [[ "$line" == "Memory Device" ]]; then + if [[ "$first_device" == "True" ]]; then + first_device="False" else - print_in_columns "$line" + # Add space between devices + echo "" fi + else + print_in_columns "$line" + fi done <<< $(sudo dmidecode -t memory \ - | grep -E '^(Memory Device|\s+(Type|Size|Speed|Manuf.*|Locator|Part Number):)') + | grep -E '^(Memory Device|\s+(Type|Size|Speed|Manuf.*|Locator|Part Number):)') echo "" # Graphics echo -e "${BLUE}Graphics${CLEAR}" -lspci | grep 'VGA' | sed -r 's/^.*:/ Device: /' \ - | sed 's/Intel Corporation/Intel/' \ - | sed 's/Generation Core Processor Family/Gen/' \ - | sed 's/Integrated Graphics Controller.*/iGPU/' -glxinfo 2>/dev/null | grep 'OpenGL renderer' | sed -r 's/^.*:/ OpenGL Renderer: /' \ - | sed 's/Mesa DRI //' +lspci | grep 'VGA' | sed -r 's/^.*:/ Device: /' \ + | sed 's/Intel Corporation/Intel/' \ + | sed 's/Generation Core Processor Family/Gen/' \ + | sed 's/Integrated Graphics Controller.*/iGPU/' +glxinfo 2>/dev/null | grep 'OpenGL renderer' | sed -r 's/^.*:/ OpenGL Renderer: /' \ + | sed 's/Mesa DRI //' echo "" # Audio echo -e "${BLUE}Audio${CLEAR}" while read -r line; do - if [[ "$line" =~ .*no.soundcards.found.* ]]; then - echo " No soundcards found" - else - print_in_columns "$line" - fi + if [[ "$line" = .*no.soundcards.found.* ]]; then + echo " No soundcards found" + else + print_in_columns "$line" + fi done <<< $(aplay -l 2>&1 | grep -Ei '(^card|no soundcards found)' | sed -r 's/.*\[(.*)\].*\[(.*)\].*/\1: \2/') echo "" # Network echo -e "${BLUE}Network${CLEAR}" lspci | grep -Ei '(ethernet|network|wireless|wifi)' \ - | sed -r 's/.*: (.*)$/ \1/' + | sed -r 's/.*: (.*)$/ \1/' echo "" # Drives echo -e "${BLUE}Drives${CLEAR}" -hw-drive-info | sed 's/^/ /' +hw-drive-info | sed 's/^/ /' echo "" diff --git a/scripts/outer_scripts_to_review/hw-diags-iobenchmark b/scripts/outer_scripts_to_review/hw-diags-iobenchmark deleted file mode 100755 index 6821b1a4..00000000 --- a/scripts/outer_scripts_to_review/hw-diags-iobenchmark +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -## Wizard Kit: HW Diagnostics - Benchmarks - -function usage { - echo "Usage: ${0} device log-file" - echo " e.g. ${0} /dev/sda /tmp/tmp.XXXXXXX/benchmarks.log" -} - -# Bail early -if [ ! -b "${1}" ]; then - usage - exit 1 -fi - -# Run Benchmarks -echo 3 | sudo tee -a /proc/sys/vm/drop_caches >/dev/null 2>&1 -sudo dd bs=4M if="${1}" of=/dev/null status=progress 2>&1 | tee -a "${2}" diff --git a/scripts/outer_scripts_to_review/hw-diags-menu b/scripts/outer_scripts_to_review/hw-diags-menu deleted file mode 100755 index 7a122ae7..00000000 --- a/scripts/outer_scripts_to_review/hw-diags-menu +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: HW Diagnostics - Menu - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.hw_diags import * -from functions.tmux import * -init_global_vars() - -if __name__ == '__main__': - # Show menu - try: - state = State() - menu_diags(state, sys.argv) - except KeyboardInterrupt: - print_standard(' ') - print_warning('Aborted') - print_standard(' ') - sleep(1) - pause('Press Enter to exit...') - except SystemExit as sys_exit: - tmux_switch_client() - exit_script(sys_exit.code) - except: - # Cleanup - tmux_kill_all_panes() - - if DEBUG_MODE: - # Custom major exception - print_standard(' ') - print_error('Major exception') - print_warning(SUPPORT_MESSAGE) - print(traceback.format_exc()) - print_log(traceback.format_exc()) - - # Save debug reports and upload data - try_and_print( - message='Saving debug reports...', - function=save_debug_reports, - state=state, global_vars=global_vars) - question = 'Upload crash details to {}?'.format(CRASH_SERVER['Name']) - if ENABLED_UPLOAD_DATA and ask(question): - try_and_print( - message='Uploading Data...', - function=upload_logdir, - global_vars=global_vars) - - # Done - sleep(1) - pause('Press Enter to exit...') - exit_script(1) - - else: - # "Normal" major exception - major_exception() - - # Done - tmux_kill_all_panes() - tmux_switch_client() - exit_script() - -# vim: sts=2 sw=2 ts=2 diff --git a/scripts/outer_scripts_to_review/hw-diags-network b/scripts/outer_scripts_to_review/hw-diags-network deleted file mode 100755 index 138ea67e..00000000 --- a/scripts/outer_scripts_to_review/hw-diags-network +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: HW Diagnostics - Network - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.network import * - - -def check_connection(): - if not is_connected(): - # Raise to cause NS in try_and_print() - raise Exception - - -if __name__ == '__main__': - try: - # Prep - clear_screen() - print_standard('Hardware Diagnostics: Network\n') - - # Connect - print_standard('Initializing...') - connect_to_network() - - # Tests - try_and_print( - message='Network connection:', function=check_connection, cs='OK') - show_valid_addresses() - try_and_print(message='Internet connection:', function=ping, - addr='8.8.8.8', cs='OK') - try_and_print(message='DNS Resolution:', function=ping, cs='OK') - try_and_print(message='Speedtest:', function=speedtest, - print_return=True) - - # Done - print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/scripts/outer_scripts_to_review/hw-diags-prime95 b/scripts/outer_scripts_to_review/hw-diags-prime95 deleted file mode 100755 index 4927da76..00000000 --- a/scripts/outer_scripts_to_review/hw-diags-prime95 +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# -## Wizard Kit: HW Diagnostics - Prime95 - -function usage { - echo "Usage: $0 log-dir" - echo " e.g. $0 /tmp/tmp.7Mh5f1RhSL9001" -} - -# Bail early -if [ ! -d "$1" ]; then - usage - exit 1 -fi - -# Run Prime95 -cd "$1" -mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "prime.log" - diff --git a/scripts/outer_scripts_to_review/hw-drive-info b/scripts/outer_scripts_to_review/hw-drive-info deleted file mode 100755 index df1e1748..00000000 --- a/scripts/outer_scripts_to_review/hw-drive-info +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# - -BLUE='\033[34m' -CLEAR='\033[0m' -IFS=$'\n' - -# List devices -for line in $(lsblk -do NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL); do - if [[ "${line:0:4}" == "NAME" ]]; then - echo -e "${BLUE}${line}${CLEAR}" - else - echo "${line}" - fi -done -echo "" - -# List loopback devices -if [[ "$(losetup -l | wc -l)" > 0 ]]; then - for line in $(losetup -lO NAME,PARTSCAN,RO,BACK-FILE); do - if [[ "${line:0:4}" == "NAME" ]]; then - echo -e "${BLUE}${line}${CLEAR}" - else - echo "${line}" | sed -r 's#/dev/(loop[0-9]+)#\1 #' - fi - done - echo "" -fi - -# List partitions -for line in $(lsblk -o NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT); do - if [[ "${line:0:4}" == "NAME" ]]; then - echo -e "${BLUE}${line}${CLEAR}" - else - echo "${line}" - fi -done -echo "" - diff --git a/scripts/outer_scripts_to_review/hw-sensors b/scripts/outer_scripts_to_review/hw-sensors deleted file mode 100755 index 39ca7147..00000000 --- a/scripts/outer_scripts_to_review/hw-sensors +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# -## Wizard Kit: Sensor monitoring tool - -WINDOW_NAME="Hardware Sensors" -MONITOR="hw-sensors-monitor" - -# Start session -tmux new-session -n "$WINDOW_NAME" "$MONITOR" - diff --git a/scripts/outer_scripts_to_review/hw-sensors-monitor b/scripts/outer_scripts_to_review/hw-sensors-monitor deleted file mode 100755 index ffdbbad3..00000000 --- a/scripts/outer_scripts_to_review/hw-sensors-monitor +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: Sensor monitoring tool - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.sensors import * -from functions.tmux import * -init_global_vars(silent=True) - -if __name__ == '__main__': - background = False - try: - if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): - background = True - monitor_file = sys.argv[1] - monitor_pane = None - else: - result = run_program(['mktemp']) - monitor_file = result.stdout.decode().strip() - if not background: - monitor_pane = tmux_split_window( - percent=1, vertical=True, watch=monitor_file) - cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] - run_program(cmd, check=False) - monitor_sensors(monitor_pane, monitor_file) - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 From ca001ed831e9ccf8076f7fea871439a9c6b23816 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 17:11:54 -0700 Subject: [PATCH 222/324] Restrict hw-drive-info and hw-info to Linux --- scripts/hw-drive-info | 10 ++++++++++ scripts/hw-info | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/scripts/hw-drive-info b/scripts/hw-drive-info index a1c051d8..76a0aa27 100755 --- a/scripts/hw-drive-info +++ b/scripts/hw-drive-info @@ -5,6 +5,16 @@ BLUE='\033[34m' CLEAR='\033[0m' IFS=$'\n' +# Check if running under Linux +os_name="$(uname -s)" +if [[ "$os_name" == "Darwin" ]]; then + os_name="macOS" +fi +if [[ "$os_name" != "Linux" ]]; then + echo "This script is not supported under $os_name." 1>&2 + exit 1 +fi + # List devices for line in $(lsblk -do NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL); do if [[ "${line:0:4}" == "NAME" ]]; then diff --git a/scripts/hw-info b/scripts/hw-info index 8a909c67..2cd5f848 100755 --- a/scripts/hw-info +++ b/scripts/hw-info @@ -25,6 +25,16 @@ function print_dmi_value() { print_in_columns "$name: $value" } +# Check if running under Linux +os_name="$(uname -s)" +if [[ "$os_name" == "Darwin" ]]; then + os_name="macOS" +fi +if [[ "$os_name" != "Linux" ]]; then + echo "This script is not supported under $os_name." 1>&2 + exit 1 +fi + # System echo -e "${BLUE}System Information${CLEAR}" print_dmi_value "Vendor" "sys_vendor" From 442ed991bb4cbff92f252f2dd9b81bcf1a36dbc3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 17:12:42 -0700 Subject: [PATCH 223/324] Remove shebang and exec mod from hw-diags.py --- scripts/hw-diags.py | 1 - 1 file changed, 1 deletion(-) mode change 100755 => 100644 scripts/hw-diags.py diff --git a/scripts/hw-diags.py b/scripts/hw-diags.py old mode 100755 new mode 100644 index e3719875..eba99820 --- a/scripts/hw-diags.py +++ b/scripts/hw-diags.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Wizard Kit: Hardware Diagnostics""" # vim: sts=2 sw=2 ts=2 From b25b15f1952781db6816c3750f1309f03f2ce346 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 17:32:40 -0700 Subject: [PATCH 224/324] Set PYTHONPATH --- setup/linux/include/airootfs/etc/skel/.bashrc | 3 +++ setup/linux/include/airootfs/etc/skel/.zshrc | 1 + 2 files changed, 4 insertions(+) diff --git a/setup/linux/include/airootfs/etc/skel/.bashrc b/setup/linux/include/airootfs/etc/skel/.bashrc index cb37e84b..0c12a187 100644 --- a/setup/linux/include/airootfs/etc/skel/.bashrc +++ b/setup/linux/include/airootfs/etc/skel/.bashrc @@ -12,3 +12,6 @@ PS1='[\u@\h \W]\$ ' # Update LS_COLORS eval $(dircolors ~/.dircolors) + +# WizardKit +export PYTHONPATH='/usr/local/bin' diff --git a/setup/linux/include/airootfs/etc/skel/.zshrc b/setup/linux/include/airootfs/etc/skel/.zshrc index 59a747eb..a5a2298b 100644 --- a/setup/linux/include/airootfs/etc/skel/.zshrc +++ b/setup/linux/include/airootfs/etc/skel/.zshrc @@ -9,3 +9,4 @@ source $ZSH/oh-my-zsh.sh # Wizard Kit . $HOME/.aliases eval $(dircolors ~/.dircolors) +export PYTHONPATH="/usr/local/bin" From 2770f85e01bb0b4101ea32084e538fa2e1afade7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 17:50:36 -0700 Subject: [PATCH 225/324] Moved server definitions to wk/cfg/net.py --- scripts/wk/cfg/main.py | 12 ++---------- scripts/wk/cfg/net.py | 29 +++++++++++++++++++++++++++++ scripts/wk/std.py | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/scripts/wk/cfg/main.py b/scripts/wk/cfg/main.py index aef91b61..99a3b2a1 100644 --- a/scripts/wk/cfg/main.py +++ b/scripts/wk/cfg/main.py @@ -1,6 +1,7 @@ """WizardKit: Config - Main -NOTE: A non-standard format is used for BASH/BATCH/PYTHON compatibility""" +NOTE: Non-standard formating is used for BASH/BATCH/PYTHON compatibility +""" # pylint: disable=bad-whitespace # vim: sts=2 sw=2 ts=2 @@ -30,15 +31,6 @@ TECH_PASSWORD='Abracadabra' LINUX_TIME_ZONE='America/Denver' WINDOWS_TIME_ZONE='Mountain Standard Time' -# Misc -CRASH_SERVER = { - 'Name': 'CrashServer', - 'Url': '', - 'User': '', - 'Pass': '', - 'Headers': {'X-Requested-With': 'XMLHttpRequest'}, - } - if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/cfg/net.py b/scripts/wk/cfg/net.py index 344c695b..cdcf7082 100644 --- a/scripts/wk/cfg/net.py +++ b/scripts/wk/cfg/net.py @@ -1,6 +1,35 @@ """WizardKit: Config - Net""" +# pylint: disable=bad-whitespace # vim: sts=2 sw=2 ts=2 +# Servers +BACKUP_SERVERS = { + 'Server One': { + 'IP': '10.0.0.10', + 'Share': 'Backups', + 'RO-User': 'restore', + 'RO-Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', + }, + 'Server Two': { + 'IP': '10.0.0.11', + 'Share': 'Backups', + 'RO-User': 'restore', + 'RO-Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', + }, + } +CRASH_SERVER = { + 'Name': 'CrashServer', + 'Url': '', + 'User': '', + 'Pass': '', + 'Headers': {'X-Requested-With': 'XMLHttpRequest'}, + } + + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/std.py b/scripts/wk/std.py index d0eecde4..c82d13be 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -27,12 +27,12 @@ except ImportError: raise from wk.cfg.main import ( - CRASH_SERVER, ENABLED_UPLOAD_DATA, INDENT, SUPPORT_MESSAGE, WIDTH, ) +from wk.cfg.net import CRASH_SERVER # STATIC VARIABLES From 9c7914fc3d9fff77863c9c2599e6c0a63072503a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 19:23:44 -0700 Subject: [PATCH 226/324] Added mount_backup_shares & mount_network_share --- scripts/wk/cfg/net.py | 4 +- scripts/wk/net.py | 92 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/scripts/wk/cfg/net.py b/scripts/wk/cfg/net.py index cdcf7082..5e42dd84 100644 --- a/scripts/wk/cfg/net.py +++ b/scripts/wk/cfg/net.py @@ -6,7 +6,7 @@ # Servers BACKUP_SERVERS = { 'Server One': { - 'IP': '10.0.0.10', + 'Address': '10.0.0.10', 'Share': 'Backups', 'RO-User': 'restore', 'RO-Pass': 'Abracadabra', @@ -14,7 +14,7 @@ BACKUP_SERVERS = { 'RW-Pass': 'Abracadabra', }, 'Server Two': { - 'IP': '10.0.0.11', + 'Address': 'servertwo.example.com', 'Share': 'Backups', 'RO-User': 'restore', 'RO-Pass': 'Abracadabra', diff --git a/scripts/wk/net.py b/scripts/wk/net.py index 69e9a345..b03644d9 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -1,6 +1,9 @@ """WizardKit: Net Functions""" # vim: sts=2 sw=2 ts=2 +import os +import pathlib +import platform import re import psutil @@ -8,6 +11,8 @@ import psutil from wk.exe import run_program from wk.std import GenericError, show_data +from wk.cfg.net import BACKUP_SERVERS + # REGEX REGEX_VALID_IP = re.compile( @@ -35,6 +40,93 @@ def connected_to_private_network(): raise GenericError('Not connected to a network') +def is_mounted(details): + """Check if dev/share/etc is mounted, returns bool.""" + #TODO: Make real + if not details: + return False + return False + + +def mount_backup_shares(read_write=False): + """Mount backup shares using OS specific methods.""" + report = [] + for name, details in BACKUP_SERVERS.items(): + mount_point = None + mount_str = f'{name}/{details["Share"]}' + + # Prep mount point + if platform.system() in ('Darwin', 'Linux'): + mount_point = pathlib.Path(f'/Backups/{name}') + if not mount_point.exists(): + # Script should be run as user so sudo is required + run_program(['sudo', 'mkdir', mount_point]) + + # Check if already mounted + if is_mounted(details): + report.append(f'(Already) Mounted {mount_str}') + # Skip to next share + continue + + # Mount share + proc = mount_network_share(details, mount_point, read_write=read_write) + if proc.returncode: + report.append(f'Failed to Mount {mount_str}') + else: + report.append(f'Mounted {mount_str}') + + # Done + return report + + +def mount_network_share(details, mount_point=None, read_write=False): + """Mount network share using OS specific methods.""" + cmd = None + address = details['Address'] + share = details['Share'] + username = details['RO-User'] + password = details['RO-Pass'] + if read_write: + username = details['RW-User'] + password = details['RW-Pass'] + + # Build OS-specific command + if platform.system() == 'Darwin': + cmd = [ + 'sudo', + 'mount', + '-t', 'smbfs', + '-o', f'{"rw" if read_write else "ro"}', + f'//{username}:{password}@{address}/{share}', + mount_point, + ] + elif platform.system() == 'Linux': + cmd = [ + 'sudo', + 'mount', + '-t', 'cifs', + '-o', ( + f'{"rw" if read_write else "ro"}' + f',uid={os.getuid()}' + f',gid={os.getgid()}' + f',username={username}' + f',{"password=" if password else "guest"}{password}' + ), + f'//{address}/{share}', + mount_point + ] + elif platform.system() == 'Windows': + cmd = ['net', 'use'] + if mount_point: + cmd.append(f'{mount_point}:') + cmd.append(f'/user:{username}') + cmd.append(fr'\\{address}\{share}') + cmd.append(password) + + # Mount share + return run_program(cmd, check=False) + + def ping(addr='google.com'): """Attempt to ping addr.""" cmd = ( From 0472166c097cab74043baa17daad128fca86d871 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 19:42:02 -0700 Subject: [PATCH 227/324] Added share mount check logic --- scripts/wk/net.py | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/scripts/wk/net.py b/scripts/wk/net.py index b03644d9..5d3a9d63 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -8,7 +8,7 @@ import re import psutil -from wk.exe import run_program +from wk.exe import get_json_from_command, run_program from wk.std import GenericError, show_data from wk.cfg.net import BACKUP_SERVERS @@ -40,14 +40,6 @@ def connected_to_private_network(): raise GenericError('Not connected to a network') -def is_mounted(details): - """Check if dev/share/etc is mounted, returns bool.""" - #TODO: Make real - if not details: - return False - return False - - def mount_backup_shares(read_write=False): """Mount backup shares using OS specific methods.""" report = [] @@ -63,7 +55,7 @@ def mount_backup_shares(read_write=False): run_program(['sudo', 'mkdir', mount_point]) # Check if already mounted - if is_mounted(details): + if share_is_mounted(details): report.append(f'(Already) Mounted {mount_str}') # Skip to next share continue @@ -138,6 +130,41 @@ def ping(addr='google.com'): run_program(cmd) +def share_is_mounted(details): + """Check if dev/share/etc is mounted, returns bool.""" + mounted = False + + if platform.system() == 'Darwin': + # Weak and naive text search + proc = run_program(['mount'], check=False) + for line in proc.stdout.splitlines(): + if f'{details["Address"]}/{details["Share"]}' in line: + mounted = True + break + elif platform.system() == 'Linux': + cmd = [ + 'findmnt', + '--list', + '--json', + '--invert', + '--types', ( + 'autofs,binfmt_misc,bpf,cgroup,cgroup2,configfs,debugfs,devpts,' + 'devtmpfs,hugetlbfs,mqueue,proc,pstore,securityfs,sysfs,tmpfs' + ), + '--output', 'SOURCE', + ] + mount_data = get_json_from_command(cmd) + for row in mount_data.get('filesystems', []): + if row['source'] == f'//{details["Address"]}/{details["Share"]}': + mounted = True + break + #TODO: Check mount status under Windows + #elif platform.system() == 'Windows': + + # Done + return mounted + + def show_valid_addresses(): """Show all valid private IP addresses assigned to the system.""" devs = psutil.net_if_addrs() From 82827b7a0d6138825eb111a8e471bd826e9f6aa2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 19:57:47 -0700 Subject: [PATCH 228/324] Avoid crash under macOS --- scripts/wk/net.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/wk/net.py b/scripts/wk/net.py index 5d3a9d63..f8732af4 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -50,9 +50,13 @@ def mount_backup_shares(read_write=False): # Prep mount point if platform.system() in ('Darwin', 'Linux'): mount_point = pathlib.Path(f'/Backups/{name}') - if not mount_point.exists(): - # Script should be run as user so sudo is required - run_program(['sudo', 'mkdir', mount_point]) + try: + if not mount_point.exists(): + # Script should be run as user so sudo is required + run_program(['sudo', 'mkdir', '-p', mount_point]) + except OSError: + # Assuming permission denied under macOS + pass # Check if already mounted if share_is_mounted(details): From 77190137f621d419021a94b5c300e33def98ed6e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 20:10:57 -0700 Subject: [PATCH 229/324] Added mount-backup-shares wrapper --- scripts/mount-backup-shares | 30 ++++++++++++++++++++++++++++++ scripts/wk/net.py | 4 +++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 scripts/mount-backup-shares diff --git a/scripts/mount-backup-shares b/scripts/mount-backup-shares new file mode 100755 index 00000000..69ae4a58 --- /dev/null +++ b/scripts/mount-backup-shares @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Wizard Kit: Mount Backup Shares""" +# pylint: disable=invalid-name +# vim: sts=2 sw=2 ts=2 + +import wk + + +# Functions +def main(): + """Attempt to mount backup shares and print report.""" + wk.std.print_info('Mounting Backup Shares') + report = wk.net.mount_backup_shares() + for line in report: + color = 'GREEN' + line = f' {line}' + if 'Failed' in line: + color = 'RED' + elif 'Already' in line: + color = 'YELLOW' + print(wk.std.color_string(line, color)) + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/net.py b/scripts/wk/net.py index f8732af4..011a83d7 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -45,7 +45,7 @@ def mount_backup_shares(read_write=False): report = [] for name, details in BACKUP_SERVERS.items(): mount_point = None - mount_str = f'{name}/{details["Share"]}' + mount_str = f'//{name}/{details["Share"]}' # Prep mount point if platform.system() in ('Darwin', 'Linux'): @@ -57,6 +57,8 @@ def mount_backup_shares(read_write=False): except OSError: # Assuming permission denied under macOS pass + if mount_point: + mount_str += f' to {mount_point}' # Check if already mounted if share_is_mounted(details): From 32628880241fc3e6387098ff2803869a89dda838 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 20:50:17 -0700 Subject: [PATCH 230/324] Added unmount network share sections --- scripts/wk/net.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/scripts/wk/net.py b/scripts/wk/net.py index 011a83d7..d63d606c 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -45,7 +45,7 @@ def mount_backup_shares(read_write=False): report = [] for name, details in BACKUP_SERVERS.items(): mount_point = None - mount_str = f'//{name}/{details["Share"]}' + mount_str = f'{name} (//{details["Address"]}/{details["Share"]})' # Prep mount point if platform.system() in ('Darwin', 'Linux'): @@ -191,5 +191,53 @@ def speedtest(): return [f'{a:<10}{b:6.2f} {c}' for a, b, c in output] +def unmount_backup_shares(): + """Unmount backup shares.""" + report = [] + for name, details in BACKUP_SERVERS.items(): + kwargs = {} + source_str = f'{name} (//{details["Address"]}/{details["Share"]})' + + # Check if mounted + if not share_is_mounted(details): + report.append(f'Not mounted {source_str}') + continue + + # Build OS specific kwargs + if platform.system() in ('Darwin', 'Linux'): + kwargs['mount_point'] = f'/Backups/{name}' + elif platform.system() == 'Windows': + kwargs['details'] = details + + # Unmount and add to report + proc = unmount_network_share(**kwargs) + if proc.returncode: + report.append(f'Failed to unmount {source_str}') + else: + report.append(f'Unmounted {source_str}') + + # Done + return report + + +def unmount_network_share(details=None, mount_point=None): + """Unmount network share""" + cmd = [] + + # Build OS specific command + if platform.system() in ('Darwin', 'Linux'): + cmd = ['sudo', 'umount', mount_point] + elif platform.system() == 'Windows': + cmd = ['net', 'use'] + if mount_point: + cmd.append(f'{mount_point}:') + elif details: + cmd.append(fr'\\{details["Address"]}\{details["Share"]}') + cmd.append('/delete') + + # Unmount share + return run_program(cmd, check=False) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 07cb287eb019f250dc1bf18f039c940755eb60dc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 20:53:42 -0700 Subject: [PATCH 231/324] Updated wk.net.connected_to_private_network() * Can either return True/False or return None/raise Exception * Added network check to mount_backup_shares() --- scripts/hw-diags.py | 1 + scripts/wk/hw/diags.py | 6 +++++- scripts/wk/net.py | 28 +++++++++++++++++++++++----- 3 files changed, 29 insertions(+), 6 deletions(-) mode change 100644 => 100755 scripts/hw-diags.py diff --git a/scripts/hw-diags.py b/scripts/hw-diags.py old mode 100644 new mode 100755 index eba99820..e3719875 --- a/scripts/hw-diags.py +++ b/scripts/hw-diags.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Wizard Kit: Hardware Diagnostics""" # vim: sts=2 sw=2 ts=2 diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 347640d5..0cc4f42f 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -1231,7 +1231,11 @@ def network_test(): LOG.info('Network Test') try_and_print = std.TryAndPrint() result = try_and_print.run( - 'Network connection...', net.connected_to_private_network, msg_good='OK') + message='Network connection...', + function=net.connected_to_private_network, + msg_good='OK', + raise_on_error=True, + ) # Bail if not connected if result['Failed']: diff --git a/scripts/wk/net.py b/scripts/wk/net.py index d63d606c..c0e967ef 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -23,21 +23,35 @@ REGEX_VALID_IP = re.compile( # Functions -def connected_to_private_network(): - """Check if connected to a private network. +def connected_to_private_network(raise_on_error=False): + """Check if connected to a private network, returns bool. This checks for a valid private IP assigned to this system. - If one isn't found then an exception is raised. + + NOTE: If one isn't found and raise_on_error=True then an exception is raised. + NOTE 2: If one is found and raise_on_error=True then None is returned. """ + connected = False + + # Check IPs devs = psutil.net_if_addrs() for dev in devs.values(): for family in dev: if REGEX_VALID_IP.search(family.address): # Valid IP found - return + connected = True + break + if connected: + break # No valid IP found - raise GenericError('Not connected to a network') + if not connected and raise_on_error: + raise GenericError('Not connected to a network') + + # Done + if raise_on_error: + connected = None + return connected def mount_backup_shares(read_write=False): @@ -88,6 +102,10 @@ def mount_network_share(details, mount_point=None, read_write=False): username = details['RW-User'] password = details['RW-Pass'] + # Network check + if not connected_to_private_network(): + raise RuntimeError('Not connected to a network') + # Build OS-specific command if platform.system() == 'Darwin': cmd = [ From d0eee811291c5ad03ca74c141102e2035221260c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 20:54:53 -0700 Subject: [PATCH 232/324] Added unmount-backup-shares wrapper --- scripts/unmount-backup-shares | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 scripts/unmount-backup-shares diff --git a/scripts/unmount-backup-shares b/scripts/unmount-backup-shares new file mode 100755 index 00000000..a71a5c7b --- /dev/null +++ b/scripts/unmount-backup-shares @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Wizard Kit: Unmount Backup Shares""" +# pylint: disable=invalid-name +# vim: sts=2 sw=2 ts=2 + +import wk + + +# Functions +def main(): + """Attempt to mount backup shares and print report.""" + wk.std.print_info('Unmounting Backup Shares') + report = wk.net.unmount_backup_shares() + for line in report: + color = 'GREEN' + line = f' {line}' + if 'Not mounted' in line: + color = 'YELLOW' + print(wk.std.color_string(line, color)) + + +if __name__ == '__main__': + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() From 15f1a5badab65313457474dbfcc01f3efcefa207 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 9 Dec 2019 20:56:17 -0700 Subject: [PATCH 233/324] Removed old mount-backup-shares wrapper --- .../mount-backup-shares | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100755 scripts/outer_scripts_to_review/mount-backup-shares diff --git a/scripts/outer_scripts_to_review/mount-backup-shares b/scripts/outer_scripts_to_review/mount-backup-shares deleted file mode 100755 index 0d8b7fd3..00000000 --- a/scripts/outer_scripts_to_review/mount-backup-shares +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: Backup share mount tool - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.data import * -from functions.network import * -init_global_vars() - -if __name__ == '__main__': - try: - # Prep - clear_screen() - - # Mount - if is_connected(): - mount_backup_shares(read_write=True) - else: - # Couldn't connect - print_error('ERROR: No network connectivity.') - - # Done - print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 From 7a880e2ee7268bb0d216bad0b03e36da66093924 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 10 Dec 2019 15:56:12 -0700 Subject: [PATCH 234/324] Added initial ddrescue sections * Very early outline, very broken still --- scripts/ddrescue-tui | 21 ++++ scripts/ddrescue-tui.py | 14 +++ scripts/wk/cfg/ddrescue.py | 58 +++++++++ scripts/wk/hw/ddrescue.py | 252 +++++++++++++++++++++++++++++++++++++ 4 files changed, 345 insertions(+) create mode 100755 scripts/ddrescue-tui create mode 100755 scripts/ddrescue-tui.py create mode 100644 scripts/wk/cfg/ddrescue.py create mode 100644 scripts/wk/hw/ddrescue.py diff --git a/scripts/ddrescue-tui b/scripts/ddrescue-tui new file mode 100755 index 00000000..03e8eee2 --- /dev/null +++ b/scripts/ddrescue-tui @@ -0,0 +1,21 @@ +#!/bin/bash +# +## Wizard Kit: ddrescue TUI Launcher + +# Check if running under Linux +os_name="$(uname -s)" +if [[ "$os_name" == "Darwin" ]]; then + os_name="macOS" +fi +if [[ "$os_name" != "Linux" ]]; then + echo "This script is not supported under $os_name." 1>&2 + exit 1 +fi + +source ./launch-in-tmux + +SESSION_NAME="ddrescue-tui" +WINDOW_NAME="ddrescue TUI" +TMUX_CMD="./ddrescue-tui.py" + +launch_in_tmux "$@" diff --git a/scripts/ddrescue-tui.py b/scripts/ddrescue-tui.py new file mode 100755 index 00000000..89584890 --- /dev/null +++ b/scripts/ddrescue-tui.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Wizard Kit: ddrescue TUI""" +# vim: sts=2 sw=2 ts=2 + +import wk + + +if __name__ == '__main__': + try: + wk.hw.ddrescue.main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py new file mode 100644 index 00000000..ee9cb72a --- /dev/null +++ b/scripts/wk/cfg/ddrescue.py @@ -0,0 +1,58 @@ +"""WizardKit: Config - ddrescue""" +# pylint: disable=bad-whitespace,line-too-long +# vim: sts=2 sw=2 ts=2 + +import re + +from collections import OrderedDict + + +# General +MAP_DIR = '/Backups/ddrescue-tui' +RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] +RECOMMENDED_MAP_FSTYPES = ['cifs', 'ext2', 'ext3', 'ext4', 'vfat', 'xfs'] + +# Layout +SIDE_PANE_WIDTH = 21 +TMUX_LAYOUT = OrderedDict({ + 'Source': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, +}) + +# ddrescue +AUTO_PASS_1_THRESHOLD = 95 +AUTO_PASS_2_THRESHOLD = 98 +DDRESCUE_SETTINGS = { + '--binary-prefixes': {'Enabled': True, 'Hidden': True, }, + '--data-preview': {'Enabled': True, 'Value': '5', 'Hidden': True, }, + '--idirect': {'Enabled': True, }, + '--odirect': {'Enabled': True, }, + '--max-read-rate': {'Enabled': False, 'Value': '1MiB', }, + '--min-read-rate': {'Enabled': True, 'Value': '64KiB', }, + '--reopen-on-error': {'Enabled': True, }, + '--retry-passes': {'Enabled': True, 'Value': '0', }, + '--test-mode': {'Enabled': False, 'Value': 'test.map', }, + '--timeout': {'Enabled': True, 'Value': '5m', }, + '-vvvv': {'Enabled': True, 'Hidden': True, }, + } +ETOC_REFRESH_RATE = 30 # in seconds +REGEX_DDRESCUE_LOG = re.compile( + r'^\s*(?P\S+):\s+' + r'(?P\d+)\s+' + r'(?P[PTGMKB])i?B?', + re.IGNORECASE, + ) +REGEX_REMAINING_TIME = re.compile( + r'remaining time:' + r'\s*((?P\d+)d)?' + r'\s*((?P\d+)h)?' + r'\s*((?P\d+)m)?' + r'\s*((?P\d+)s)?' + r'\s*(?Pn/a)?', + re.IGNORECASE + ) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py new file mode 100644 index 00000000..51759035 --- /dev/null +++ b/scripts/wk/hw/ddrescue.py @@ -0,0 +1,252 @@ +"""WizardKit: ddrescue TUI""" +# pylint: disable=too-many-lines +# vim: sts=2 sw=2 ts=2 + +import atexit +import logging +import os +import pathlib +import platform +import plistlib +import re +import subprocess +import time + +from collections import OrderedDict +from docopt import docopt + +from wk import cfg, debug, exe, graph, log, net, std, tmux +from wk.hw import obj as hw_obj +from wk.hw import sensors as hw_sensors + + +# STATIC VARIABLES +DOCSTRING = f'''{cfg.main.KIT_NAME_FULL}: ddrescue TUI + +Usage: + ddrescue-tui + ddrescue-tui (clone|image) [ []] + ddrescue-tui (-h | --help) + +Options: + -h --help Show this page +''' +LOG = logging.getLogger(__name__) +MENU_ACTIONS = ( + 'Start', + std.color_string(['Change settings', '(experts only)'], [None, 'YELLOW']), + 'Quit') +MENU_TOGGLES = { + 'Auto continue (if recovery % over threshold)': True, + 'Retry (mark non-rescued sectors "non-tried")': False, + 'Reverse direction': False, + } +STATUS_COLORS = { + 'Passed': 'GREEN', + 'Aborted': 'YELLOW', + 'Skipped': 'YELLOW', + 'Working': 'YELLOW', + 'ERROR': 'RED', + } + + +# Classes +class State(): + """Object for tracking hardware diagnostic data.""" + def __init__(self): + #TODO + self.block_pairs = [] + self.disks = [] + self.layout = cfg.ddrescue.TMUX_LAYOUT.copy() + self.log_dir = None + self.panes = {} + + # Init tmux and start a background process to maintain layout + self.init_tmux() + exe.start_thread(self.fix_tmux_layout_loop) + + def fix_tmux_layout(self, forced=True): + # pylint: disable=unused-argument + """Fix tmux layout based on cfg.ddrescue.TMUX_LAYOUT.""" + try: + tmux.fix_layout(self.panes, self.layout, forced=forced) + except RuntimeError: + # Assuming self.panes changed while running + pass + + def fix_tmux_layout_loop(self): + """Fix tmux layout on a loop. + + NOTE: This should be called as a thread. + """ + while True: + self.fix_tmux_layout(forced=False) + std.sleep(1) + + def init_tmux(self): + """Initialize tmux layout.""" + tmux.kill_all_panes() + + # Source / Dest + self.update_top_panes() + + # Started + self.panes['Started'] = tmux.split_window( + lines=cfg.ddrescue.TMUX_SIDE_WIDTH, + target_id=self.panes['Top'], + text=std.color_string( + ['Started', time.strftime("%Y-%m-%d %H:%M %Z")], + ['BLUE', None], + sep='\n', + ), + ) + + # Progress + self.panes['Progress'] = tmux.split_window( + lines=cfg.ddrescue.TMUX_SIDE_WIDTH, + text=' ', + ) + + def save_debug_reports(self): + """Save debug reports to disk.""" + LOG.info('Saving debug reports') + debug_dir = pathlib.Path(f'{self.log_dir}/debug') + if not debug_dir.exists(): + debug_dir.mkdir() + + # State (self) + with open(f'{debug_dir}/state.report', 'a') as _f: + _f.write('\n'.join(debug.generate_object_report(self))) + + # Block pairs + for _bp in self.block_pairs: + with open(f'{debug_dir}/bp_part#.report', 'a') as _f: + _f.write('\n'.join(debug.generate_object_report(_bp))) + + def update_progress_pane(self): + """Update progress pane.""" + report = [] + width = cfg.ddrescue.TMUX_SIDE_WIDTH + + #TODO + + # Write to progress file + out_path = pathlib.Path(f'{self.log_dir}/progress.out') + with open(out_path, 'w') as _f: + _f.write('\n'.join(report)) + + def update_top_panes(self, text): + """(Re)create top source/destination panes.""" + #TODO + self.panes['Source'] = tmux.split_window( + behind=True, + lines=2, + vertical=True, + text=std.color_string( + ['Source', f'TODO'], + ['BLUE', None], + sep='\n', + ), + ) + + # Destination + self.panes['Destination'] = tmux.split_window( + percent=50, + vertical=False, + target_id=self.panes['Source'], + text=std.color_string( + ['Destination', f'TODO'], + ['BLUE', None], + sep='\n', + ), + ) + + +# Functions +def build_main_menu(): + """Build main menu, returns wk.std.Menu.""" + menu = std.Menu(title=std.color_string('ddrescue TUI: Main Menu', 'GREEN')) + + # Add actions, options, etc + for action in MENU_ACTIONS: + menu.add_action(action) + for toggle, selected in MENU_TOGGLES.items(): + menu.add_toggle(toggle, {'Selected': selected}) + menu.actions['Start']['Separator'] = True + + # Done + return menu + + +def build_settings_menu(): + """Build settings menu, returns wk.std.Menu.""" + #TODO Fixme + title_text = [ + std.color_string('ddrescue TUI: Exper Settings', 'GREEN'), + ' ', + std.color_string( + ['These settings can cause', 'MAJOR DAMAGE', 'to drives'], + ['YELLOW', 'RED', 'YELLOW'], + ), + 'Please read the manual before making changes', + ] + menu = std.Menu(title='\n'join(title_text)) + + # Add actions, options, etc + menu.add_action('Main Menu') + for name, details in cfg.ddrescue.DDRESCUE_SETTINGS.items(): + menu.add_option(name, details) + menu.actions['Main Menu']['Separator'] = True + + # Done + return menu + + +def main(): + # pylint: disable=too-many-branches + """Main function for hardware diagnostics.""" + args = docopt(DOCSTRING) + log.update_log_path(dest_name='ddrescue-TUI', timestamp=True) + + # Safety check + if 'TMUX' not in os.environ: + LOG.error('tmux session not found') + raise RuntimeError('tmux session not found') + + # Init + atexit.register(tmux.kill_all_panes) + main_menu = build_main_menu() + settings_menu = build_settings_menu() + state = State() + + # Show menu + while True: + action = None + selection = menu.advanced_select() + + # Start diagnostics + if 'Start' in selection: + run_diags(state, menu, quick_mode=False) + + # Quit + if 'Quit' in selection: + break + + +def run_recovery(state, main_menu, settings_menu): + """Run recovery passes.""" + aborted = False + atexit.register(state.save_debug_reports) + state.init_recovery(menu) + + # TODO + # Run ddrescue + + # Done + state.save_debug_reports() + atexit.unregister(state.save_debug_reports) + std.pause('Press Enter to return to main menu...') + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From 5445df8e62771a3d109a088748a5e678de54edc8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 12 Dec 2019 15:52:09 -0700 Subject: [PATCH 235/324] Added settings_select() to Menu() * Supports ddrescue-tui style toggle/change usage --- scripts/wk/std.py | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index c82d13be..89164726 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -128,18 +128,22 @@ class Menu(): menu_lines = [str(line) for line in menu_lines] return '\n'.join(menu_lines) - def _get_display_name(self, name, details, index=None, no_checkboxes=True): + def _get_display_name(self, name, details, + index=None, no_checkboxes=True, setting_item=False): # pylint: disable=no-self-use """Format display name based on details and args, returns str.""" disabled = details.get('Disabled', False) + if setting_item and not details['Selected']: + # Display item in YELLOW + disabled = True checkmark = '*' if 'DISPLAY' in os.environ or platform.system() == 'Darwin': checkmark = '✓' - clear_code = COLORS['CLEAR'] - color_code = COLORS['YELLOW'] if disabled else '' - display_name = f'{color_code}{index if index else name[:1].upper()}: ' + display_name = f'{index if index else name[:1].upper()}: ' if not (index and index >= 10): display_name = f' {display_name}' + if setting_item and 'Value' in details: + name = f'{name} = {details["Value"]}' # Add enabled status if necessary if not no_checkboxes: @@ -147,7 +151,7 @@ class Menu(): # Add name if disabled: - display_name += f'{name} ({self.disabled_str}){clear_code}' + display_name += color_string(f'{name} ({self.disabled_str})', 'YELLOW') else: display_name += name @@ -226,7 +230,7 @@ class Menu(): # Done return resolved_selection - def _update(self, single_selection=True): + def _update(self, single_selection=True, settings_mode=False): """Update menu items in preparation for printing to screen.""" index = 0 @@ -253,6 +257,7 @@ class Menu(): details, index=index, no_checkboxes=single_selection, + setting_item=settings_mode, ) # Actions @@ -350,6 +355,34 @@ class Menu(): # Done return selected_entry + def settings_select(self, prompt='Please make a selection: '): + """Display menu and make multiple selections, returns tuple. + + NOTE: Menu is displayed until an action entry is selected. + """ + choice_kwargs = { + 'choices': ['T', 'C'], + 'prompt': 'Toggle or change value?', + } + + while True: + self._update(single_selection=True, settings_mode=True) + user_selection = self._user_select(prompt) + selected_entry = self._resolve_selection(user_selection) + if user_selection.isnumeric(): + if 'Value' in selected_entry[-1] and choice(**choice_kwargs) == 'C': + # Change + selected_entry[-1[-1]]['Value'] = input_text('Enter new value: ') + else: + # Toggle + self._update_entry_selection_status(selected_entry[0]) + else: + # Action selected + break + + # Done + return selected_entry + def simple_select(self, prompt='Please make a selection: '): """Display menu and make a single selection, returns tuple.""" self._update() From 045d2b25714d84c81e41244ee4fb2f8bdbd15c12 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 12 Dec 2019 16:39:42 -0700 Subject: [PATCH 236/324] Updates ddrescue-tui tmux sections --- scripts/wk/cfg/__init__.py | 1 + scripts/wk/cfg/ddrescue.py | 8 ++--- scripts/wk/hw/__init__.py | 1 + scripts/wk/hw/ddrescue.py | 72 +++++++++++++++++++++++++++++++------- 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py index fc63cf2d..d86f7245 100644 --- a/scripts/wk/cfg/__init__.py +++ b/scripts/wk/cfg/__init__.py @@ -1,5 +1,6 @@ """WizardKit: cfg module init""" +from wk.cfg import ddrescue from wk.cfg import hw from wk.cfg import log from wk.cfg import main diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index ee9cb72a..482b683c 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -13,11 +13,11 @@ RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] RECOMMENDED_MAP_FSTYPES = ['cifs', 'ext2', 'ext3', 'ext4', 'vfat', 'xfs'] # Layout -SIDE_PANE_WIDTH = 21 +TMUX_SIDE_WIDTH = 21 TMUX_LAYOUT = OrderedDict({ - 'Source': {'y': 2, 'Check': True}, - 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, - 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Source': {'height': 2, 'Check': True}, + 'Started': {'width': TMUX_SIDE_WIDTH, 'Check': True}, + 'Progress': {'width': TMUX_SIDE_WIDTH, 'Check': True}, }) # ddrescue diff --git a/scripts/wk/hw/__init__.py b/scripts/wk/hw/__init__.py index 17b6df35..ad2b04cb 100644 --- a/scripts/wk/hw/__init__.py +++ b/scripts/wk/hw/__init__.py @@ -1,5 +1,6 @@ """WizardKit: hw module init""" +from wk.hw import ddrescue from wk.hw import diags from wk.hw import obj from wk.hw import sensors diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 51759035..dba4b74f 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -41,6 +41,11 @@ MENU_TOGGLES = { 'Retry (mark non-rescued sectors "non-tried")': False, 'Reverse direction': False, } +PANE_RATIOS = ( + 12, # SMART + 22, # ddrescue progress + 4, # Journal (kernel messages) + ) STATUS_COLORS = { 'Passed': 'GREEN', 'Aborted': 'YELLOW', @@ -54,12 +59,13 @@ STATUS_COLORS = { class State(): """Object for tracking hardware diagnostic data.""" def __init__(self): - #TODO self.block_pairs = [] + self.destination = None self.disks = [] self.layout = cfg.ddrescue.TMUX_LAYOUT.copy() self.log_dir = None self.panes = {} + self.source = None # Init tmux and start a background process to maintain layout self.init_tmux() @@ -68,12 +74,28 @@ class State(): def fix_tmux_layout(self, forced=True): # pylint: disable=unused-argument """Fix tmux layout based on cfg.ddrescue.TMUX_LAYOUT.""" + needs_fixed = tmux.layout_needs_fixed(self.panes, self.layout) + + # Main layout fix try: tmux.fix_layout(self.panes, self.layout, forced=forced) except RuntimeError: # Assuming self.panes changed while running pass + # Source/Destination + if forced or needs_fixed: + self.update_top_panes() + + # SMART/Journal + height = tmux.get_pane_size(self.panes['Progress'])[1] - 2 + p_ratios = [int((x/sum(PANE_RATIOS)) * height) for x in PANE_RATIOS] + if 'SMART' in self.panes: + tmux.resize_pane(self.panes['SMART'], height=p_ratios[0]) + tmux.resize_pane(height=p_ratios[1]) + if 'Journal' in self.panes: + tmux.resize_pane(self.panes['Journal'], height=p_ratios[2]) + def fix_tmux_layout_loop(self): """Fix tmux layout on a loop. @@ -87,13 +109,18 @@ class State(): """Initialize tmux layout.""" tmux.kill_all_panes() - # Source / Dest - self.update_top_panes() + # Source (placeholder) + self.panes['Source'] = tmux.split_window( + behind=True, + lines=2, + text=' ', + vertical=True, + ) # Started self.panes['Started'] = tmux.split_window( lines=cfg.ddrescue.TMUX_SIDE_WIDTH, - target_id=self.panes['Top'], + target_id=self.panes['Source'], text=std.color_string( ['Started', time.strftime("%Y-%m-%d %H:%M %Z")], ['BLUE', None], @@ -101,6 +128,9 @@ class State(): ), ) + # Source / Dest + self.update_top_panes() + # Progress self.panes['Progress'] = tmux.split_window( lines=cfg.ddrescue.TMUX_SIDE_WIDTH, @@ -135,27 +165,45 @@ class State(): with open(out_path, 'w') as _f: _f.write('\n'.join(report)) - def update_top_panes(self, text): + def update_top_panes(self): """(Re)create top source/destination panes.""" - #TODO - self.panes['Source'] = tmux.split_window( - behind=True, - lines=2, - vertical=True, + width = tmux.get_pane_size()[0] + width = int(width / 2) - 1 + + # Kill destination pane + if 'Destination' in self.panes: + tmux.kill_pane(self.panes.pop('Destination')) + + # Source + source_str = ' ' + if self.source: + source_str = f'{self.source.path} {self.source.description}' + if len(source_str) > width: + source_str = f'{source_str[:width-3]}...' + tmux.respawn_pane( + self.panes['Source'], text=std.color_string( - ['Source', f'TODO'], + ['Source', source_str], ['BLUE', None], sep='\n', ), ) # Destination + dest_str = '' + if self.destination: + dest_str = f'{self.destination.path} {self.destination.description}' + if len(dest_str) > width: + if self.destination.path.is_dir(): + dest_str = f'...{dest_str[-width+3:]}' + else: + dest_str = f'{dest_str[:width-3]}...' self.panes['Destination'] = tmux.split_window( percent=50, vertical=False, target_id=self.panes['Source'], text=std.color_string( - ['Destination', f'TODO'], + ['Destination', dest_str], ['BLUE', None], sep='\n', ), From 3a8c052d5afbc3f29abbb41b555cddcd43579b01 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 12 Dec 2019 16:43:23 -0700 Subject: [PATCH 237/324] Updated ddrescue menu sections * Support loading presets --- scripts/wk/cfg/ddrescue.py | 37 ++++++++++++++++++-------- scripts/wk/hw/ddrescue.py | 54 +++++++++++++++++++++++++++++--------- scripts/wk/std.py | 7 ++--- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index 482b683c..2de2fc88 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -24,17 +24,32 @@ TMUX_LAYOUT = OrderedDict({ AUTO_PASS_1_THRESHOLD = 95 AUTO_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { - '--binary-prefixes': {'Enabled': True, 'Hidden': True, }, - '--data-preview': {'Enabled': True, 'Value': '5', 'Hidden': True, }, - '--idirect': {'Enabled': True, }, - '--odirect': {'Enabled': True, }, - '--max-read-rate': {'Enabled': False, 'Value': '1MiB', }, - '--min-read-rate': {'Enabled': True, 'Value': '64KiB', }, - '--reopen-on-error': {'Enabled': True, }, - '--retry-passes': {'Enabled': True, 'Value': '0', }, - '--test-mode': {'Enabled': False, 'Value': 'test.map', }, - '--timeout': {'Enabled': True, 'Value': '5m', }, - '-vvvv': {'Enabled': True, 'Hidden': True, }, + 'Default': { + '--binary-prefixes': {'Selected': True, 'Hidden': True, }, + '--data-preview': {'Selected': True, 'Value': '5', 'Hidden': True, }, + '--idirect': {'Selected': True, }, + '--odirect': {'Selected': True, }, + '--max-error-rate': {'Selected': True, 'Value': '100MiB', }, + '--max-read-rate': {'Selected': False, 'Value': '1MiB', }, + '--min-read-rate': {'Selected': True, 'Value': '64KiB', }, + '--reopen-on-error': {'Selected': True, }, + '--retry-passes': {'Selected': True, 'Value': '0', }, + '--test-mode': {'Selected': False, 'Value': 'test.map', }, + '--timeout': {'Selected': True, 'Value': '30m', }, + '-vvvv': {'Selected': True, 'Hidden': True, }, + }, + 'Fast': { + '--max-error-rate': {'Selected': True, 'Value': '32MiB', }, + '--min-read-rate': {'Selected': True, 'Value': '1MiB', }, + '--reopen-on-error': {'Selected': False, }, + '--timeout': {'Selected': True, 'Value': '5m', }, + }, + 'Safe': { + '--max-read-rate': {'Selected': True, 'Value': '64MiB', }, + '--min-read-rate': {'Selected': True, 'Value': '1KiB', }, + '--reopen-on-error': {'Selected': True, }, + '--timeout': {'Selected': False, 'Value': '30m', }, + }, } ETOC_REFRESH_RATE = 30 # in seconds REGEX_DDRESCUE_LOG = re.compile( diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index dba4b74f..6358b899 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -34,7 +34,7 @@ Options: LOG = logging.getLogger(__name__) MENU_ACTIONS = ( 'Start', - std.color_string(['Change settings', '(experts only)'], [None, 'YELLOW']), + f'Change settings {std.color_string("(experts only)", "YELLOW")}', 'Quit') MENU_TOGGLES = { 'Auto continue (if recovery % over threshold)': True, @@ -46,6 +46,11 @@ PANE_RATIOS = ( 22, # ddrescue progress 4, # Journal (kernel messages) ) +SETTING_PRESETS = ( + 'Default', + 'Fast', + 'Safe', + ) STATUS_COLORS = { 'Passed': 'GREEN', 'Aborted': 'YELLOW', @@ -214,21 +219,20 @@ class State(): def build_main_menu(): """Build main menu, returns wk.std.Menu.""" menu = std.Menu(title=std.color_string('ddrescue TUI: Main Menu', 'GREEN')) + menu.separator = ' ' # Add actions, options, etc for action in MENU_ACTIONS: menu.add_action(action) for toggle, selected in MENU_TOGGLES.items(): menu.add_toggle(toggle, {'Selected': selected}) - menu.actions['Start']['Separator'] = True # Done return menu -def build_settings_menu(): +def build_settings_menu(silent=True): """Build settings menu, returns wk.std.Menu.""" - #TODO Fixme title_text = [ std.color_string('ddrescue TUI: Exper Settings', 'GREEN'), ' ', @@ -238,13 +242,29 @@ def build_settings_menu(): ), 'Please read the manual before making changes', ] - menu = std.Menu(title='\n'join(title_text)) + menu = std.Menu(title='\n'.join(title_text)) + menu.separator = ' ' + preset = 'Default' + if not silent: + # Ask which preset to use + print(f'Available ddrescue presets: {" / ".join(SETTING_PRESETS)}') + preset = std.choice(SETTING_PRESETS, 'Please select a preset:') - # Add actions, options, etc + # Fix selection + for _p in SETTING_PRESETS: + if _p.startswith(preset): + preset = _p + + # Add default settings menu.add_action('Main Menu') - for name, details in cfg.ddrescue.DDRESCUE_SETTINGS.items(): - menu.add_option(name, details) - menu.actions['Main Menu']['Separator'] = True + menu.add_action('Load Preset') + for name, details in cfg.ddrescue.DDRESCUE_SETTINGS['Default'].items(): + menu.add_option(name, details.copy()) + + # Update settings using preset + if preset != 'Default': + for name, details in cfg.ddrescue.DDRESCUE_SETTINGS[preset].items(): + menu.options[name].update(details.copy()) # Done return menu @@ -270,11 +290,21 @@ def main(): # Show menu while True: action = None - selection = menu.advanced_select() + selection = main_menu.advanced_select() - # Start diagnostics + # Change settings + if 'Change settings' in selection[0]: + while True: + selection = settings_menu.settings_select() + if 'Load Preset' in selection: + # Rebuild settings menu using preset + settings_menu = build_settings_menu(silent=False) + else: + break + + # Start recovery if 'Start' in selection: - run_diags(state, menu, quick_mode=False) + run_recovery(state, main_menu, settings_menu) # Quit if 'Quit' in selection: diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 89164726..babc0505 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -128,9 +128,10 @@ class Menu(): menu_lines = [str(line) for line in menu_lines] return '\n'.join(menu_lines) - def _get_display_name(self, name, details, + def _get_display_name( + self, name, details, index=None, no_checkboxes=True, setting_item=False): - # pylint: disable=no-self-use + # pylint: disable=no-self-use,too-many-arguments """Format display name based on details and args, returns str.""" disabled = details.get('Disabled', False) if setting_item and not details['Selected']: @@ -372,7 +373,7 @@ class Menu(): if user_selection.isnumeric(): if 'Value' in selected_entry[-1] and choice(**choice_kwargs) == 'C': # Change - selected_entry[-1[-1]]['Value'] = input_text('Enter new value: ') + selected_entry[-1]['Value'] = input_text('Enter new value: ') else: # Toggle self._update_entry_selection_status(selected_entry[0]) From 48a6b3200bae96f199222bf83ae64964bf9401fa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 12 Dec 2019 17:22:02 -0700 Subject: [PATCH 238/324] Added init_recovery() * Set mode * Select/verify source/dest --- scripts/wk/hw/ddrescue.py | 75 +++++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 6358b899..cf2435fb 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -15,7 +15,7 @@ import time from collections import OrderedDict from docopt import docopt -from wk import cfg, debug, exe, graph, log, net, std, tmux +from wk import cfg, debug, exe, log, net, std, tmux from wk.hw import obj as hw_obj from wk.hw import sensors as hw_sensors @@ -72,7 +72,7 @@ class State(): self.panes = {} self.source = None - # Init tmux and start a background process to maintain layout + # Start a background process to maintain layout self.init_tmux() exe.start_thread(self.fix_tmux_layout_loop) @@ -110,6 +110,41 @@ class State(): self.fix_tmux_layout(forced=False) std.sleep(1) + def init_recovery(self, docopt_args): + """Select source/dest and set env.""" + + # Set log + self.log_dir = log.format_log_path() + self.log_dir = pathlib.Path( + f'{self.log_dir.parent}/' + f'ddrescue-TUI_{time.strftime("%Y-%m-%d_%H%M%S%z")}/' + ) + log.update_log_path( + dest_dir=self.log_dir, + dest_name='main', + keep_history=True, + timestamp=False, + ) + + # Update progress pane + tmux.respawn_pane( + pane_id=self.panes['Progress'], + watch_file=f'{self.log_dir}/progress.out', + ) + + # Set mode + mode = set_mode(docopt_args) + + # Select source + # TODO + + # Select destination + # TODO + + # Update panes + self.update_progress_pane() + self.update_top_panes() + def init_tmux(self): """Initialize tmux layout.""" tmux.kill_all_panes() @@ -136,7 +171,7 @@ class State(): # Source / Dest self.update_top_panes() - # Progress + # Progress (placeholder) self.panes['Progress'] = tmux.split_window( lines=cfg.ddrescue.TMUX_SIDE_WIDTH, text=' ', @@ -271,8 +306,7 @@ def build_settings_menu(silent=True): def main(): - # pylint: disable=too-many-branches - """Main function for hardware diagnostics.""" + """Main function for ddrescue TUI.""" args = docopt(DOCSTRING) log.update_log_path(dest_name='ddrescue-TUI', timestamp=True) @@ -286,6 +320,7 @@ def main(): main_menu = build_main_menu() settings_menu = build_settings_menu() state = State() + state.init_recovery(args) # Show menu while True: @@ -313,18 +348,44 @@ def main(): def run_recovery(state, main_menu, settings_menu): """Run recovery passes.""" - aborted = False atexit.register(state.save_debug_reports) - state.init_recovery(menu) + + # Start SMART/Journal + # TODO # TODO # Run ddrescue + # Stop SMART/Journal + # TODO + # Done state.save_debug_reports() atexit.unregister(state.save_debug_reports) std.pause('Press Enter to return to main menu...') +def set_mode(docopt_args): + """Set mode from docopt_args or user selection, returns str.""" + mode = None + + # Check docopt_args + if docopt_args['clone']: + mode = 'Clone' + elif docopt_args['image']: + mode = 'Image' + + # Ask user if necessary + if not mode: + answer = std.choice(['C', 'I'], 'Are we cloning or imaging?') + if answer == 'C': + mode = 'Clone' + else: + mode = 'Image' + + # Done + return mode + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 3733da17fc98ecb686bc1f21e913720e1d383af7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 12 Dec 2019 18:36:57 -0700 Subject: [PATCH 239/324] Moved get_disks() from wk/hw/diags to wk/hw/obj --- scripts/wk/hw/diags.py | 78 +----------------------------------------- scripts/wk/hw/obj.py | 69 +++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 77 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 0cc4f42f..58f8825f 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -88,10 +88,6 @@ STATUS_COLORS = { 'Failed': 'RED', 'TimedOut': 'RED', } -WK_LABEL_REGEX = re.compile( - fr'{cfg.main.KIT_NAME_SHORT}_(LINUX|UFD)', - re.IGNORECASE, - ) # Error Classes @@ -263,7 +259,7 @@ class State(): # Add HW Objects self.cpu = hw_obj.CpuRam() - self.disks = get_disks() + self.disks = hw_obj.get_disks(skip_kits=True) # Add test objects for name, details in menu.options.items(): @@ -1078,78 +1074,6 @@ def disk_surface_scan(state, test_objects): raise std.GenericAbort('Aborted') -def get_disks(): - """Get disks using OS-specific methods, returns list.""" - disks = [] - if platform.system() == 'Darwin': - disks = get_disks_macos() - elif platform.system() == 'Linux': - disks = get_disks_linux() - - # Done - return disks - - -def get_disks_linux(): - """Get disks via lsblk, returns list.""" - cmd = ['lsblk', '--json', '--nodeps', '--paths'] - disks = [] - - # Add valid disks - json_data = exe.get_json_from_command(cmd) - for disk in json_data.get('blockdevices', []): - disk_obj = hw_obj.Disk(disk['name']) - skip = False - - # Skip loopback devices, optical devices, etc - if disk_obj.details['type'] != 'disk': - skip = True - - # Skip WK disks - for label in disk_obj.get_labels(): - if WK_LABEL_REGEX.search(label): - skip = True - - # Add disk - if not skip: - disks.append(disk_obj) - - # Done - return disks - - -def get_disks_macos(): - """Get disks via diskutil, returns list.""" - cmd = ['diskutil', 'list', '-plist', 'physical'] - disks = [] - - # Get info from diskutil - proc = exe.run_program(cmd, encoding=None, errors=None) - try: - plist_data = plistlib.loads(proc.stdout) - except (TypeError, ValueError): - # Invalid / corrupt plist data? return empty list to avoid crash - LOG.error('Failed to get diskutil list') - return disks - - # Add valid disks - for disk in plist_data['WholeDisks']: - disk_obj = hw_obj.Disk(f'/dev/{disk}') - skip = False - - # Skip WK disks - for label in disk_obj.get_labels(): - if WK_LABEL_REGEX.search(label): - skip = True - - # Add disk - if not skip: - disks.append(disk_obj) - - # Done - return disks - - def keyboard_test(): """Test keyboard using xev.""" LOG.info('Keyboard Test (xev)') diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index bc1df561..82b52e22 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -18,6 +18,7 @@ from wk.cfg.hw import ( KNOWN_RAM_VENDOR_IDS, REGEX_POWER_ON_TIME, ) +from wk.cfg.main import KIT_NAME_SHORT from wk.exe import get_json_from_command, run_program from wk.std import bytes_to_string, color_string, sleep, string_to_bytes @@ -29,6 +30,10 @@ NVME_WARNING_KEYS = ( 'reliability_degraded', 'volatile_memory_backup_failed', ) +WK_LABEL_REGEX = re.compile( + fr'{KIT_NAME_SHORT}_(LINUX|UFD)', + re.IGNORECASE, + ) # Exception Classes @@ -626,6 +631,70 @@ def get_disk_serial_macos(path): return smart_info.get('serial_number', 'Unknown Serial') +def get_disks(skip_kits=False): + """Get disks using OS-specific methods, returns list.""" + disks = [] + if platform.system() == 'Darwin': + disks = get_disks_macos() + elif platform.system() == 'Linux': + disks = get_disks_linux() + + # Skip WK disks + if skip_kits: + disks = [ + disk_obj for disk_obj in disks + if not any( + [WK_LABEL_REGEX.search(label) for label in disk_obj.get_labels()] + ) + ] + + # Done + return disks + + +def get_disks_linux(): + """Get disks via lsblk, returns list.""" + cmd = ['lsblk', '--json', '--nodeps', '--paths'] + disks = [] + + # Add valid disks + json_data = get_json_from_command(cmd) + for disk in json_data.get('blockdevices', []): + disk_obj = Disk(disk['name']) + + # Skip loopback devices, optical devices, etc + if disk_obj.details['type'] != 'disk': + continue + + # Add disk + disks.append(disk_obj) + + # Done + return disks + + +def get_disks_macos(): + """Get disks via diskutil, returns list.""" + cmd = ['diskutil', 'list', '-plist', 'physical'] + disks = [] + + # Get info from diskutil + proc = exe.run_program(cmd, encoding=None, errors=None) + try: + plist_data = plistlib.loads(proc.stdout) + except (TypeError, ValueError): + # Invalid / corrupt plist data? return empty list to avoid crash + LOG.error('Failed to get diskutil list') + return disks + + # Add valid disks + for disk in plist_data['WholeDisks']: + disks.append(Disk(f'/dev/{disk}')) + + # Done + return disks + + def get_known_disk_attributes(model): """Get known NVMe/SMART attributes (model specific), returns str.""" known_attributes = KNOWN_DISK_ATTRIBUTES.copy() From b746cda6e7990cc8f852c979ebb3a55838dd1576 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 12 Dec 2019 18:46:44 -0700 Subject: [PATCH 240/324] Bugfix --- scripts/wk/hw/diags.py | 1 - scripts/wk/hw/obj.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 58f8825f..0c0c642a 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -7,7 +7,6 @@ import logging import os import pathlib import platform -import plistlib import re import subprocess import time diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 82b52e22..442f79fa 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -679,7 +679,7 @@ def get_disks_macos(): disks = [] # Get info from diskutil - proc = exe.run_program(cmd, encoding=None, errors=None) + proc = run_program(cmd, encoding=None, errors=None) try: plist_data = plistlib.loads(proc.stdout) except (TypeError, ValueError): From da5f521f923a398c0a23d3f355d3899b362d2b49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 12 Dec 2019 19:29:32 -0700 Subject: [PATCH 241/324] Added wk.hw.ddrescue.select_disk() --- scripts/wk/hw/ddrescue.py | 71 ++++++++++++++++++++++++++++++--------- scripts/wk/std.py | 2 +- 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index cf2435fb..9abfd4f0 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -93,6 +93,9 @@ class State(): self.update_top_panes() # SMART/Journal + if 'Progress' not in self.panes: + # Assumning we're still selecting source/dest + return height = tmux.get_pane_size(self.panes['Progress'])[1] - 2 p_ratios = [int((x/sum(PANE_RATIOS)) * height) for x in PANE_RATIOS] if 'SMART' in self.panes: @@ -112,6 +115,7 @@ class State(): def init_recovery(self, docopt_args): """Select source/dest and set env.""" + std.clear_screen() # Set log self.log_dir = log.format_log_path() @@ -126,22 +130,31 @@ class State(): timestamp=False, ) - # Update progress pane - tmux.respawn_pane( - pane_id=self.panes['Progress'], - watch_file=f'{self.log_dir}/progress.out', - ) - # Set mode mode = set_mode(docopt_args) # Select source - # TODO + try: + self.source = select_disk() + except std.GenericAbort: + std.abort() # Select destination - # TODO + if mode == 'Clone': + try: + self.destination = select_disk(self.source.path) + except std.GenericAbort: + std.abort() + elif mode == 'Image': + #TODO + std.print_error('Not implemented yet.') + std.abort() # Update panes + self.panes['Progress'] = tmux.split_window( + lines=cfg.ddrescue.TMUX_SIDE_WIDTH, + watch_file=f'{self.log_dir}/progress.out', + ) self.update_progress_pane() self.update_top_panes() @@ -171,12 +184,6 @@ class State(): # Source / Dest self.update_top_panes() - # Progress (placeholder) - self.panes['Progress'] = tmux.split_window( - lines=cfg.ddrescue.TMUX_SIDE_WIDTH, - text=' ', - ) - def save_debug_reports(self): """Save debug reports to disk.""" LOG.info('Saving debug reports') @@ -269,7 +276,7 @@ def build_main_menu(): def build_settings_menu(silent=True): """Build settings menu, returns wk.std.Menu.""" title_text = [ - std.color_string('ddrescue TUI: Exper Settings', 'GREEN'), + std.color_string('ddrescue TUI: Expert Settings', 'GREEN'), ' ', std.color_string( ['These settings can cause', 'MAJOR DAMAGE', 'to drives'], @@ -365,6 +372,40 @@ def run_recovery(state, main_menu, settings_menu): std.pause('Press Enter to return to main menu...') +def select_disk(skip_disk=None): + """Select disk from list, returns Disk().""" + disks = hw_obj.get_disks() + menu = std.Menu( + title=std.color_string('ddrescue TUI: Source Selection', 'GREEN'), + ) + menu.disabled_str = 'Already selected' + menu.separator = ' ' + menu.add_action('Quit') + if skip_disk: + skip_disk = str(skip_disk) + for disk in disks: + disable_option = skip_disk and disk.path.match(skip_disk) + size = disk.details["size"] + menu.add_option( + name=( + f'{str(disk.path):<12} ' + f'{disk.details["bus"]:<5} ' + f'{std.bytes_to_string(size, decimals=1, use_binary=False):<8} ' + f'{disk.details["model"]} ' + f'{disk.details["serial"]}' + ), + details={'Disabled': disable_option, 'Object': disk}, + ) + + # Get selection + selection = menu.simple_select('Please select the source: ') + if 'Quit' in selection: + raise std.GenericAbort() + + # Done + return selection[-1]['Object'] + + def set_mode(docopt_args): """Set mode from docopt_args or user selection, returns str.""" mode = None diff --git a/scripts/wk/std.py b/scripts/wk/std.py index babc0505..ac197b67 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -613,7 +613,7 @@ def abort(prompt='Aborted.', show_prompt=True, return_code=1): """Abort script.""" print_warning(prompt) if show_prompt: - sleep(1) + sleep(0.5) pause(prompt='Press Enter to exit... ') sys.exit(return_code) From 6bfee9504322256c5deceab51f063374d03468de Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Dec 2019 16:38:24 -0700 Subject: [PATCH 242/324] Support coloring pathlib.Path objects --- scripts/wk/std.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index ac197b67..cd5e9dc1 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -738,9 +738,9 @@ def color_string(strings, colors, sep=' '): msg = [] # Convert to tuples if necessary - if isinstance(strings, str): + if isinstance(strings, (str, pathlib.Path)): strings = (strings,) - if isinstance(colors, str): + if isinstance(colors, (str, pathlib.Path)): colors = (colors,) # Build new string with color escapes added From cb7d0da8168c2f059d92d1a507e8f2b2ca4a0d89 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Dec 2019 18:36:56 -0700 Subject: [PATCH 243/324] Drop pause in launch-in-tmux --- scripts/launch-in-tmux | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/launch-in-tmux b/scripts/launch-in-tmux index 1f774426..2be76959 100755 --- a/scripts/launch-in-tmux +++ b/scripts/launch-in-tmux @@ -46,9 +46,6 @@ function launch_in_tmux() { die "Failed to kill session: $SESSION_NAME" else echo "Aborted." - echo "" - echo -n "Press Enter to exit... " - read -r return 1 fi fi From bc2c3a2c80992846d3514d1ab03f0205d79146f7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Dec 2019 18:37:36 -0700 Subject: [PATCH 244/324] Expanded source/dest disk selection sections --- scripts/wk/hw/ddrescue.py | 77 ++++++++++++++++++++++++++++++--------- scripts/wk/hw/obj.py | 12 ++++-- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 9abfd4f0..3f0ca7ba 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -134,21 +134,19 @@ class State(): mode = set_mode(docopt_args) # Select source - try: - self.source = select_disk() - except std.GenericAbort: - std.abort() + self.source = get_object(docopt_args['']) + if not self.source: + self.source = select_disk('Source') # Select destination - if mode == 'Clone': - try: - self.destination = select_disk(self.source.path) - except std.GenericAbort: + self.destination = get_object(docopt_args['']) + if not self.destination: + if mode == 'Clone': + self.destination = select_disk('Destination', self.source) + elif mode == 'Image': + #TODO + std.print_error('Not implemented yet.') std.abort() - elif mode == 'Image': - #TODO - std.print_error('Not implemented yet.') - std.abort() # Update panes self.panes['Progress'] = tmux.split_window( @@ -312,6 +310,39 @@ def build_settings_menu(silent=True): return menu +def get_object(path): + """Get object based on path, returns obj.""" + obj = None + + # Bail early + if not path: + return obj + + # Check path + path = pathlib.Path(path).resolve() + if path.is_block_device() or path.is_char_device(): + obj = hw_obj.Disk(path) + + # Child/Parent check + parent = obj.details['parent'] + if parent: + std.print_warning(f'"{obj.path}" is a child device') + if std.ask(f'Use parent device "{parent}" instead?'): + obj = hw_obj.Disk(parent) + elif path.is_dir(): + #TODO + std.print_error('Not implemented yet.') + std.abort() + elif path.is_file(): + # Assuming image file, setup loopback dev + #TODO + std.print_error('Not implemented yet.') + std.abort() + + # Done + return obj + + def main(): """Main function for ddrescue TUI.""" args = docopt(DOCSTRING) @@ -327,7 +358,10 @@ def main(): main_menu = build_main_menu() settings_menu = build_settings_menu() state = State() - state.init_recovery(args) + try: + state.init_recovery(args) + except std.GenericAbort: + std.abort() # Show menu while True: @@ -372,20 +406,27 @@ def run_recovery(state, main_menu, settings_menu): std.pause('Press Enter to return to main menu...') -def select_disk(skip_disk=None): +def select_disk(prompt, skip_disk=None): """Select disk from list, returns Disk().""" disks = hw_obj.get_disks() menu = std.Menu( - title=std.color_string('ddrescue TUI: Source Selection', 'GREEN'), + title=std.color_string(f'ddrescue TUI: {prompt} Selection', 'GREEN'), ) menu.disabled_str = 'Already selected' menu.separator = ' ' menu.add_action('Quit') - if skip_disk: - skip_disk = str(skip_disk) for disk in disks: - disable_option = skip_disk and disk.path.match(skip_disk) + disable_option = False size = disk.details["size"] + + # Check if option should be disabled + if skip_disk: + parent = skip_disk.details.get('parent', None) + if (disk.path.samefile(skip_disk.path) + or (parent and disk.path.samefile(parent))): + disable_option = True + + # Add to menu menu.add_option( name=( f'{str(disk.path):<12} ' diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 442f79fa..177f34bf 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -305,7 +305,7 @@ class Disk(BaseObj): self.details = get_disk_details_linux(self.path) # Set necessary details - self.details['bus'] = self.details.get('bus', '???') + self.details['bus'] = str(self.details.get('bus', '???')) self.details['bus'] = self.details['bus'].upper().replace('NVME', 'NVMe') self.details['model'] = self.details.get('model', 'Unknown Model') self.details['name'] = self.details.get('name', self.path) @@ -569,8 +569,14 @@ def get_disk_details_linux(path): cmd = ['lsblk', '--bytes', '--json', '--output-all', '--paths', path] json_data = get_json_from_command(cmd, check=False) details = json_data.get('blockdevices', [{}])[0] - details['bus'] = details.pop('tran', '???') - details['ssd'] = not details.pop('rota', True) + + # Fix details + for dev in [details, *details.get('children', [])]: + dev['bus'] = dev.pop('tran', '???') + dev['parent'] = dev.pop('pkname', None) + dev['ssd'] = not dev.pop('rota', True) + + # Done return details From c72372d55ca60b9c0a1ec8c83513eb632e75d026 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Dec 2019 19:01:26 -0700 Subject: [PATCH 245/324] Replaced platform.system() with PLATFORM var * Better? --- scripts/wk/hw/ddrescue.py | 2 +- scripts/wk/hw/diags.py | 20 ++++++++++---------- scripts/wk/hw/obj.py | 29 +++++++++++++++++------------ scripts/wk/hw/sensors.py | 13 ++++++------- scripts/wk/net.py | 25 ++++++++++++------------- scripts/wk/std.py | 3 ++- scripts/wk/tmux.py | 4 ++-- 7 files changed, 50 insertions(+), 46 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 3f0ca7ba..e999bd75 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -6,7 +6,6 @@ import atexit import logging import os import pathlib -import platform import plistlib import re import subprocess @@ -46,6 +45,7 @@ PANE_RATIOS = ( 22, # ddrescue progress 4, # Journal (kernel messages) ) +PLATFORM = std.PLATFORM SETTING_PRESETS = ( 'Default', 'Fast', diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 0c0c642a..32264b08 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -6,7 +6,6 @@ import atexit import logging import os import pathlib -import platform import re import subprocess import time @@ -75,6 +74,7 @@ MENU_SETS = { MENU_TOGGLES = ( 'Skip USB Benchmarks', ) +PLATFORM = std.PLATFORM STATUS_COLORS = { 'Passed': 'GREEN', 'Aborted': 'YELLOW', @@ -374,7 +374,7 @@ class State(): # Functions def audio_test(): """Run an OS-specific audio test.""" - if platform.system() == 'Linux': + if PLATFORM == 'Linux': audio_test_linux() # TODO: Add tests for other OS @@ -423,10 +423,10 @@ def build_menu(cli_mode=False, quick_mode=False): menu.add_action('Power Off') # Compatibility checks - if platform.system() != 'Linux': + if PLATFORM != 'Linux': for name in ('Audio Test', 'Keyboard Test', 'Network Test'): menu.actions[name]['Disabled'] = True - if platform.system() not in ('Darwin', 'Linux'): + if PLATFORM not in ('Darwin', 'Linux'): for name in ('Matrix', 'Tubes'): menu.actions[name]['Disabled'] = True @@ -661,10 +661,10 @@ def cpu_mprime_test(state, test_objects): state.update_progress_pane() state.panes['Prime95'] = tmux.split_window( lines=10, vertical=True, watch_file=prime_log) - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': state.panes['Temps'] = tmux.split_window( behind=True, percent=80, vertical=True, cmd='./hw-sensors') - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': state.panes['Temps'] = tmux.split_window( behind=True, percent=80, vertical=True, watch_file=sensors_out) tmux.resize_pane(height=3) @@ -750,7 +750,7 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): def _run_io_benchmark(test_obj, log_path): """Run I/O benchmark and handle exceptions.""" dev_path = test_obj.dev.path - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': # Use "RAW" disks under macOS dev_path = dev_path.with_name(f'r{dev_path.name}') offset = 0 @@ -783,7 +783,7 @@ def disk_io_benchmark(state, test_objects, skip_usb=True): f'if={dev_path}', 'of=/dev/null', ] - if platform.system() == 'Linux': + if PLATFORM == 'Linux': cmd.append('iflag=direct') # Run and get read rate @@ -1270,7 +1270,7 @@ def screensaver(name): cmd = ['cmatrix', '-abs'] elif name == 'pipes': cmd = [ - 'pipes' if platform.system() == 'Linux' else 'pipes.sh', + 'pipes' if PLATFORM == 'Linux' else 'pipes.sh', '-t', '0', '-t', '1', '-t', '2', @@ -1294,7 +1294,7 @@ def set_apple_fan_speed(speed): raise RuntimeError(f'Invalid speed {speed}') # Set cmd - if platform.system() == 'Linux': + if PLATFORM == 'Linux': cmd = ['apple-fans', speed] #TODO: Add method for use under macOS diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 177f34bf..933bc610 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -3,7 +3,6 @@ import logging import pathlib -import platform import plistlib import re @@ -20,7 +19,13 @@ from wk.cfg.hw import ( ) from wk.cfg.main import KIT_NAME_SHORT from wk.exe import get_json_from_command, run_program -from wk.std import bytes_to_string, color_string, sleep, string_to_bytes +from wk.std import ( + PLATFORM, + bytes_to_string, + color_string, + sleep, + string_to_bytes, + ) # STATIC VARIABLES @@ -94,11 +99,11 @@ class CpuRam(BaseObj): def get_cpu_details(self): """Get CPU details using OS specific methods.""" - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': cmd = 'sysctl -n machdep.cpu.brand_string'.split() proc = run_program(cmd, check=False) self.description = re.sub(r'\s+', ' ', proc.stdout.strip()) - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': cmd = ['lscpu', '--json'] json_data = get_json_from_command(cmd) for line in json_data.get('lscpu', [{}]): @@ -117,9 +122,9 @@ class CpuRam(BaseObj): def get_ram_details(self): """Get RAM details using OS specific methods.""" - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': dimm_list = get_ram_list_macos() - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': dimm_list = get_ram_list_linux() details = {'Total': 0} @@ -299,9 +304,9 @@ class Disk(BaseObj): Required details default to generic descriptions and are converted to the correct type. """ - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': self.details = get_disk_details_macos(self.path) - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': self.details = get_disk_details_linux(self.path) # Set necessary details @@ -362,9 +367,9 @@ class Disk(BaseObj): def is_4k_aligned(self): """Check that all disk partitions are aligned, returns bool.""" aligned = True - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': aligned = is_4k_aligned_macos(self.details) - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': aligned = is_4k_aligned_linux(self.path, self.details['phy-sec']) #TODO: Add checks for other OS @@ -640,9 +645,9 @@ def get_disk_serial_macos(path): def get_disks(skip_kits=False): """Get disks using OS-specific methods, returns list.""" disks = [] - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': disks = get_disks_macos() - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': disks = get_disks_linux() # Skip WK disks diff --git a/scripts/wk/hw/sensors.py b/scripts/wk/hw/sensors.py index a23a8575..77cbcfa3 100644 --- a/scripts/wk/hw/sensors.py +++ b/scripts/wk/hw/sensors.py @@ -4,14 +4,13 @@ import json import logging import pathlib -import platform import re from subprocess import CalledProcessError from wk.cfg.hw import CPU_CRITICAL_TEMP, SMC_IDS, TEMP_COLORS from wk.exe import run_program, start_thread -from wk.std import color_string, sleep +from wk.std import PLATFORM, color_string, sleep # STATIC VARIABLES @@ -23,7 +22,7 @@ SMC_REGEX = re.compile( r'\s+(?P.*?)' r'\s*\(bytes (?P.*)\)$' ) -SENSOR_SOURCE_WIDTH = 25 if platform.system() == 'Darwin' else 20 +SENSOR_SOURCE_WIDTH = 25 if PLATFORM == 'Darwin' else 20 # Error Classes @@ -188,9 +187,9 @@ class Sensors(): def update_sensor_data(self, exit_on_thermal_limit=True): """Update sensor data via OS-specific means.""" - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': self.update_sensor_data_macos(exit_on_thermal_limit) - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': self.update_sensor_data_linux(exit_on_thermal_limit) def update_sensor_data_linux(self, exit_on_thermal_limit=True): @@ -262,9 +261,9 @@ def fix_sensor_name(name): def get_sensor_data(): """Get sensor data via OS-specific means, returns dict.""" sensor_data = {} - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': sensor_data = get_sensor_data_macos() - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': sensor_data = get_sensor_data_linux() return sensor_data diff --git a/scripts/wk/net.py b/scripts/wk/net.py index c0e967ef..8b930c3b 100644 --- a/scripts/wk/net.py +++ b/scripts/wk/net.py @@ -3,13 +3,12 @@ import os import pathlib -import platform import re import psutil from wk.exe import get_json_from_command, run_program -from wk.std import GenericError, show_data +from wk.std import PLATFORM, GenericError, show_data from wk.cfg.net import BACKUP_SERVERS @@ -62,7 +61,7 @@ def mount_backup_shares(read_write=False): mount_str = f'{name} (//{details["Address"]}/{details["Share"]})' # Prep mount point - if platform.system() in ('Darwin', 'Linux'): + if PLATFORM in ('Darwin', 'Linux'): mount_point = pathlib.Path(f'/Backups/{name}') try: if not mount_point.exists(): @@ -107,7 +106,7 @@ def mount_network_share(details, mount_point=None, read_write=False): raise RuntimeError('Not connected to a network') # Build OS-specific command - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': cmd = [ 'sudo', 'mount', @@ -116,7 +115,7 @@ def mount_network_share(details, mount_point=None, read_write=False): f'//{username}:{password}@{address}/{share}', mount_point, ] - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': cmd = [ 'sudo', 'mount', @@ -131,7 +130,7 @@ def mount_network_share(details, mount_point=None, read_write=False): f'//{address}/{share}', mount_point ] - elif platform.system() == 'Windows': + elif PLATFORM == 'Windows': cmd = ['net', 'use'] if mount_point: cmd.append(f'{mount_point}:') @@ -158,14 +157,14 @@ def share_is_mounted(details): """Check if dev/share/etc is mounted, returns bool.""" mounted = False - if platform.system() == 'Darwin': + if PLATFORM == 'Darwin': # Weak and naive text search proc = run_program(['mount'], check=False) for line in proc.stdout.splitlines(): if f'{details["Address"]}/{details["Share"]}' in line: mounted = True break - elif platform.system() == 'Linux': + elif PLATFORM == 'Linux': cmd = [ 'findmnt', '--list', @@ -183,7 +182,7 @@ def share_is_mounted(details): mounted = True break #TODO: Check mount status under Windows - #elif platform.system() == 'Windows': + #elif PLATFORM == 'Windows': # Done return mounted @@ -222,9 +221,9 @@ def unmount_backup_shares(): continue # Build OS specific kwargs - if platform.system() in ('Darwin', 'Linux'): + if PLATFORM in ('Darwin', 'Linux'): kwargs['mount_point'] = f'/Backups/{name}' - elif platform.system() == 'Windows': + elif PLATFORM == 'Windows': kwargs['details'] = details # Unmount and add to report @@ -243,9 +242,9 @@ def unmount_network_share(details=None, mount_point=None): cmd = [] # Build OS specific command - if platform.system() in ('Darwin', 'Linux'): + if PLATFORM in ('Darwin', 'Linux'): cmd = ['sudo', 'umount', mount_point] - elif platform.system() == 'Windows': + elif PLATFORM == 'Windows': cmd = ['net', 'use'] if mount_point: cmd.append(f'{mount_point}:') diff --git a/scripts/wk/std.py b/scripts/wk/std.py index cd5e9dc1..f8254ea0 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -50,6 +50,7 @@ COLORS = { 'CYAN': '\033[36m', } LOG = logging.getLogger(__name__) +PLATFORM = platform.system() REGEX_SIZE_STRING = re.compile( r'(?P\-?\d+\.?\d*)\s*(?P[PTGMKB])(?PI?)B?' ) @@ -138,7 +139,7 @@ class Menu(): # Display item in YELLOW disabled = True checkmark = '*' - if 'DISPLAY' in os.environ or platform.system() == 'Darwin': + if 'DISPLAY' in os.environ or PLATFORM == 'Darwin': checkmark = '✓' display_name = f'{index if index else name[:1].upper()}: ' if not (index and index >= 10): diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 478a81d7..2f5b32e7 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -3,9 +3,9 @@ import logging import pathlib -import platform from wk.exe import run_program +from wk.std import PLATFORM # STATIC_VARIABLES @@ -149,7 +149,7 @@ def prep_action( elif text: # Display text echo_cmd = ['echo'] - if platform.system() == 'Linux': + if PLATFORM == 'Linux': echo_cmd.append('-e') action_cmd.extend([ 'watch', From 3fc9a843fcfb8d06bf387288a27ab25480bceb8b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 13 Dec 2019 20:04:15 -0700 Subject: [PATCH 246/324] Added select_disk_parts() * Differentiate between all parts selected and whole disk selected --- scripts/wk/hw/ddrescue.py | 59 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index e999bd75..aac0c1a2 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -137,6 +137,7 @@ class State(): self.source = get_object(docopt_args['']) if not self.source: self.source = select_disk('Source') + source_parts = select_disk_parts(mode, self.source) # Select destination self.destination = get_object(docopt_args['']) @@ -408,6 +409,7 @@ def run_recovery(state, main_menu, settings_menu): def select_disk(prompt, skip_disk=None): """Select disk from list, returns Disk().""" + std.print_info('Scanning disks...') disks = hw_obj.get_disks() menu = std.Menu( title=std.color_string(f'ddrescue TUI: {prompt} Selection', 'GREEN'), @@ -439,7 +441,7 @@ def select_disk(prompt, skip_disk=None): ) # Get selection - selection = menu.simple_select('Please select the source: ') + selection = menu.simple_select() if 'Quit' in selection: raise std.GenericAbort() @@ -447,6 +449,61 @@ def select_disk(prompt, skip_disk=None): return selection[-1]['Object'] +def select_disk_parts(prompt, disk): + """Select disk parts from list, returns list of Disk().""" + menu = std.Menu( + title=std.color_string(f'ddrescue TUI: Part Selection', 'GREEN'), + ) + menu.separator = ' ' + menu.add_action('Proceed') + menu.add_action('Quit') + object_list = [] + + # Add parts + whole_disk_str = f'{str(disk.path):<14} (Whole device)' + for part in disk.details.get('children', []): + size = part["size"] + name = ( + f'{str(part["path"]):<14} ' + f'({std.bytes_to_string(size, decimals=1, use_binary=False):>6})' + ) + menu.add_option(name, details={'Selected': True, 'Path': part['path']}) + + # Add whole disk if necessary + if not menu.options: + menu.add_option(whole_disk_str, {'Selected': True, 'Path': disk.path}) + menu.title += '\n\n' + menu.title += std.color_string(' No partitions detected.', 'YELLOW') + + # Get selection + selection = menu.advanced_select( + f'Please select the parts to {prompt.lower()}: ', + ) + if 'Quit' in selection: + raise std.GenericAbort() + + # Build list of Disk() object_list + for option in menu.options.values(): + if option['Selected']: + object_list.append(option['Path']) + + # Check if whole disk selected + if len(object_list) == len(disk.details.get('children', [])): + # NOTE: This is not true if the disk has no partitions + msg = f'Preserve partition table and unused space in {prompt.lower()}?' + if std.ask(msg): + # Replace part list with whole disk obj + object_list = [disk.path] + + # Convert object_list to hw_obj.Disk() objects + print(' ') + std.print_info('Getting disk/partition details...') + object_list = [hw_obj.Disk(path) for path in object_list] + + # Done + return object_list + + def set_mode(docopt_args): """Set mode from docopt_args or user selection, returns str.""" mode = None From c3245c92da34284635d917bf87f3ac450bceb48f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Dec 2019 19:28:52 -0700 Subject: [PATCH 247/324] Handle passing dir/file paths to ddrescue-tui --- scripts/wk/hw/ddrescue.py | 55 +++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index aac0c1a2..0611eb12 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -216,6 +216,32 @@ class State(): width = tmux.get_pane_size()[0] width = int(width / 2) - 1 + def _format_string(obj, width): + """Format source/dest string using obj and width, returns str.""" + string = '' + + # Build base string + if isinstance(obj, hw_obj.Disk): + string = f'{obj.path} {obj.description}' + elif obj.is_dir(): + string = f'{obj}/' + elif obj.is_file(): + size_str = std.bytes_to_string( + obj.stat().st_size, + decimals=0, + use_binary=False) + string = f'{obj.name} {size_str}' + + # Adjust for width + if len(string) > width: + if hasattr(obj, 'is_dir') and obj.is_dir(): + string = f'...{string[-width+3:]}' + else: + string = f'{string[:width-3]}...' + + # Done + return string + # Kill destination pane if 'Destination' in self.panes: tmux.kill_pane(self.panes.pop('Destination')) @@ -223,9 +249,7 @@ class State(): # Source source_str = ' ' if self.source: - source_str = f'{self.source.path} {self.source.description}' - if len(source_str) > width: - source_str = f'{source_str[:width-3]}...' + source_str = _format_string(self.source, width) tmux.respawn_pane( self.panes['Source'], text=std.color_string( @@ -238,12 +262,7 @@ class State(): # Destination dest_str = '' if self.destination: - dest_str = f'{self.destination.path} {self.destination.description}' - if len(dest_str) > width: - if self.destination.path.is_dir(): - dest_str = f'...{dest_str[-width+3:]}' - else: - dest_str = f'{dest_str[:width-3]}...' + dest_str = _format_string(self.destination, width) self.panes['Destination'] = tmux.split_window( percent=50, vertical=False, @@ -330,14 +349,12 @@ def get_object(path): std.print_warning(f'"{obj.path}" is a child device') if std.ask(f'Use parent device "{parent}" instead?'): obj = hw_obj.Disk(parent) - elif path.is_dir(): - #TODO - std.print_error('Not implemented yet.') - std.abort() - elif path.is_file(): - # Assuming image file, setup loopback dev - #TODO - std.print_error('Not implemented yet.') + elif path.is_dir() or path.is_file(): + obj = path + + # Abort if obj not set + if not obj: + std.print_error(f'Invalid source/dest path: {path}') std.abort() # Done @@ -459,6 +476,10 @@ def select_disk_parts(prompt, disk): menu.add_action('Quit') object_list = [] + # Bail early if child device selected + if disk.details.get('parent', False): + return [disk] + # Add parts whole_disk_str = f'{str(disk.path):<14} (Whole device)' for part in disk.details.get('children', []): From b20e6cc4ad86b2503283f0e3b62396f18a186661 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Dec 2019 20:47:06 -0700 Subject: [PATCH 248/324] Mount passed filepath as raw image * Also unmount atexit --- scripts/wk/hw/ddrescue.py | 93 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 0611eb12..91061c59 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -349,8 +349,12 @@ def get_object(path): std.print_warning(f'"{obj.path}" is a child device') if std.ask(f'Use parent device "{parent}" instead?'): obj = hw_obj.Disk(parent) - elif path.is_dir() or path.is_file(): + elif path.is_dir(): obj = path + elif path.is_file(): + # Assuming file is a raw image, mounting + loop_path = mount_raw_image(path) + obj = hw_obj.Disk(loop_path) # Abort if obj not set if not obj: @@ -405,6 +409,79 @@ def main(): break +def mount_raw_image(path): + """Mount raw image using OS specific methods, returns pathlib.Path.""" + loopback_path = None + + if PLATFORM == 'Darwin': + loopback_path = mount_raw_image_macos(path) + elif PLATFORM == 'Linux': + loopback_path = mount_raw_image_linux(path) + + # Check + if not loopback_path: + std.print_error(f'Failed to mount image: {path}') + + # Register unmount atexit + atexit.register(unmount_loopback_device, loopback_path) + + # Done + return loopback_path + + +def mount_raw_image_linux(path): + """Mount raw image using losetup, returns pathlib.Path.""" + loopback_path = None + + # Mount using losetup + cmd = [ + 'sudo', + 'losetup', + '--find', + '--partscan', + '--show', + path, + ] + proc = exe.run_program(cmd, check=False) + + # Check result + if proc.returncode == 0: + loopback_path = proc.stdout.strip() + + # Done + return loopback_path + +def mount_raw_image_macos(path): + """Mount raw image using hdiutil, returns pathlib.Path.""" + loopback_path = None + plist_data = {} + + # Mount using hdiutil + # plistdata['system-entities'][{}...] + cmd = [ + 'hdiutil', 'attach', + '-imagekey', 'diskimage-class=CRawDiskImage', + '-nomount', + '-plist', + '-readonly', + path, + ] + proc = exe.run_program(cmd, check=False, encoding=None, errors=None) + + # Check result + try: + plist_data = plistlib.loads(proc.stdout) + except plistlib.InvalidFileException: + return None + for dev in plist_data.get('system-entities', []): + dev_path = dev.get('dev-entry', '') + if re.match(r'^/dev/disk\d+$', dev_path): + loopback_path = dev_path + + # Done + return loopback_path + + def run_recovery(state, main_menu, settings_menu): """Run recovery passes.""" atexit.register(state.save_debug_reports) @@ -547,5 +624,19 @@ def set_mode(docopt_args): return mode +def unmount_loopback_device(path): + """Unmount loopback device using OS specific methods.""" + cmd = [] + + # Build OS specific cmd + if PLATFORM == 'Darwin': + cmd = ['hdiutil', 'detach', path] + elif PLATFORM == 'Linux': + cmd = ['sudo', 'losetup', '--detach', path] + + # Unmount loopback device + exe.run_program(cmd, check=False) + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 2fb2c3fa6e0cde6f9a1769755dcfdbd05fbfa6a8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 19 Dec 2019 11:31:32 -0700 Subject: [PATCH 249/324] Adjusted loopback device descriptions --- scripts/wk/hw/ddrescue.py | 6 +++--- scripts/wk/hw/obj.py | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 91061c59..ce4ad70a 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -545,9 +545,9 @@ def select_disk(prompt, skip_disk=None): def select_disk_parts(prompt, disk): """Select disk parts from list, returns list of Disk().""" - menu = std.Menu( - title=std.color_string(f'ddrescue TUI: Part Selection', 'GREEN'), - ) + title = std.color_string(f'ddrescue TUI: Partition Selection', 'GREEN') + title += f'\n\nDisk: {disk.path} {disk.description}' + menu = std.Menu(title) menu.separator = ' ' menu.add_action('Proceed') menu.add_action('Quit') diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 933bc610..f473afaf 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -310,8 +310,9 @@ class Disk(BaseObj): self.details = get_disk_details_linux(self.path) # Set necessary details - self.details['bus'] = str(self.details.get('bus', '???')) - self.details['bus'] = self.details['bus'].upper().replace('NVME', 'NVMe') + self.details['bus'] = str(self.details.get('bus', '???')).upper() + self.details['bus'] = self.details['bus'].replace('IMAGE', 'Image') + self.details['bus'] = self.details['bus'].replace('NVME', 'NVMe') self.details['model'] = self.details.get('model', 'Unknown Model') self.details['name'] = self.details.get('name', self.path) self.details['phy-sec'] = self.details.get('phy-sec', 512) @@ -580,6 +581,10 @@ def get_disk_details_linux(path): dev['bus'] = dev.pop('tran', '???') dev['parent'] = dev.pop('pkname', None) dev['ssd'] = not dev.pop('rota', True) + if 'loop' in str(path) and dev['bus'] is None: + dev['bus'] = 'Image' + dev['model'] = '' + dev['serial'] = '' # Done return details From 59ef06f402df54f11cfbb2f7d07d1ee28f66d20c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 19 Dec 2019 11:59:34 -0700 Subject: [PATCH 250/324] Added select_path() --- scripts/wk/hw/ddrescue.py | 63 ++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index ce4ad70a..2ea0e4a2 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -145,9 +145,7 @@ class State(): if mode == 'Clone': self.destination = select_disk('Destination', self.source) elif mode == 'Image': - #TODO - std.print_error('Not implemented yet.') - std.abort() + self.destination = select_path('Destination') # Update panes self.panes['Progress'] = tmux.split_window( @@ -359,7 +357,7 @@ def get_object(path): # Abort if obj not set if not obj: std.print_error(f'Invalid source/dest path: {path}') - std.abort() + raise std.GenericAbort() # Done return obj @@ -549,7 +547,9 @@ def select_disk_parts(prompt, disk): title += f'\n\nDisk: {disk.path} {disk.description}' menu = std.Menu(title) menu.separator = ' ' - menu.add_action('Proceed') + menu.add_action('All') + menu.add_action('None') + menu.add_action('Proceed', {'Separator': True}) menu.add_action('Quit') object_list = [] @@ -574,11 +574,20 @@ def select_disk_parts(prompt, disk): menu.title += std.color_string(' No partitions detected.', 'YELLOW') # Get selection - selection = menu.advanced_select( - f'Please select the parts to {prompt.lower()}: ', - ) - if 'Quit' in selection: - raise std.GenericAbort() + while True: + selection = menu.advanced_select( + f'Please select the parts to {prompt.lower()}: ', + ) + if 'All' in selection: + for option in menu.options.values(): + option['Selected'] = True + elif 'None' in selection: + for option in menu.options.values(): + option['Selected'] = False + elif 'Proceed' in selection: + break + elif 'Quit' in selection: + raise std.GenericAbort() # Build list of Disk() object_list for option in menu.options.values(): @@ -602,6 +611,40 @@ def select_disk_parts(prompt, disk): return object_list +def select_path(prompt): + """Select path, returns pathlib.Path.""" + invalid = False + menu = std.Menu( + title=std.color_string(f'ddrescue TUI: {prompt} Path Selection', 'GREEN'), + ) + menu.separator = ' ' + menu.add_action('Quit') + menu.add_option(f'Current directory') + menu.add_option('Enter manually') + path = None + + # Make selection + selection = menu.simple_select() + if 'Current directory' in selection: + path = os.getcwd() + elif 'Enter manually' in selection: + path = std.input_text('Please enter path: ') + elif 'Quit' in selection: + raise std.GenericAbort() + + # Check + try: + path = pathlib.Path(path).resolve() + except TypeError: + invalid = True + if invalid or not path.is_dir(): + std.print_error(f'Invalid path: {path}') + raise std.GenericAbort() + + # Done + return path + + def set_mode(docopt_args): """Set mode from docopt_args or user selection, returns str.""" mode = None From 0f0c47bbe48be6406db2d9580edf3b6f4b52eb8a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 20 Dec 2019 12:54:42 -0700 Subject: [PATCH 251/324] Force selecting at least one partition/device --- scripts/wk/hw/ddrescue.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 2ea0e4a2..6e5bc06b 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -138,6 +138,8 @@ class State(): if not self.source: self.source = select_disk('Source') source_parts = select_disk_parts(mode, self.source) + self.update_top_panes() + std.pause() # Select destination self.destination = get_object(docopt_args['']) @@ -146,6 +148,7 @@ class State(): self.destination = select_disk('Destination', self.source) elif mode == 'Image': self.destination = select_path('Destination') + self.update_top_panes() # Update panes self.panes['Progress'] = tmux.split_window( @@ -153,7 +156,6 @@ class State(): watch_file=f'{self.log_dir}/progress.out', ) self.update_progress_pane() - self.update_top_panes() def init_tmux(self): """Initialize tmux layout.""" @@ -585,7 +587,9 @@ def select_disk_parts(prompt, disk): for option in menu.options.values(): option['Selected'] = False elif 'Proceed' in selection: - break + if any([option['Selected'] for option in menu.options.values()]): + # At least one partition/device selected/device selected + break elif 'Quit' in selection: raise std.GenericAbort() From 428d2555388ab1af159cea790632cf43fdac2a8f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 21 Dec 2019 16:53:55 -0700 Subject: [PATCH 252/324] Added selection confirmation sections --- scripts/wk/hw/ddrescue.py | 220 +++++++++++++++++++++++++++++++++++--- 1 file changed, 203 insertions(+), 17 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 6e5bc06b..9a8406bd 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -76,6 +76,47 @@ class State(): self.init_tmux() exe.start_thread(self.fix_tmux_layout_loop) + def confirm_selections(self, mode, prompt): + """Show selection details and prompt for confirmation.""" + report = [] + + # Source + report.append(std.color_string('Source', 'GREEN')) + report.extend(build_object_report(self.source)) + report.append(' ') + + # Destination + report.append(std.color_string('Destination', 'GREEN')) + if mode == 'Clone': + report[-1] += std.color_string(' (ALL DATA WILL BE DELETED)', 'RED') + report.extend(build_object_report(self.destination)) + report.append(' ') + + # Block pairs + if self.block_pairs: + # Show mapping + # TODO + + # Show deletion warning if required + if mode == 'Clone': + report.append(std.color_string('WARNING', 'YELLOW')) + report.append( + 'All data will be deleted from the destination listed above.', + ) + report.append( + std.color_string( + ['This is irreversible and will lead to', 'DATA LOSS.'], + ['YELLOW', 'RED'], + ), + ) + report.append(' ') + + # Prompt user + std.clear_screen() + std.print_report(report) + if not std.ask(prompt): + raise std.GenericAbort() + def fix_tmux_layout(self, forced=True): # pylint: disable=unused-argument """Fix tmux layout based on cfg.ddrescue.TMUX_LAYOUT.""" @@ -139,7 +180,6 @@ class State(): self.source = select_disk('Source') source_parts = select_disk_parts(mode, self.source) self.update_top_panes() - std.pause() # Select destination self.destination = get_object(docopt_args['']) @@ -157,6 +197,28 @@ class State(): ) self.update_progress_pane() + # Confirmation #1 + self.confirm_selections(mode, 'Are these selections correct?') + + # Set working dir + # TODO + + # Add block pairs + # NOTE: Destination is not updated + # Load settings/maps + # Ask about boot partition + # Create pairs using paths + # TODO + + # Confirmation #2 + self.confirm_selections(mode, 'Start recovery?') + + # Prep destination + # if cloning and not resuming format destination + + # Done + # Ready for main menu + def init_tmux(self): """Initialize tmux layout.""" tmux.kill_all_panes() @@ -276,6 +338,110 @@ class State(): # Functions +def build_directory_report(path): + """Build directory report, returns list.""" + path = str(path) + report = [] + + # Get details + if PLATFORM == 'Linux': + cmd = [ + 'findmnt', + '--output', 'SIZE,AVAIL,USED,FSTYPE,OPTIONS', + '--target', path, + ] + proc = exe.run_program(cmd) + width = len(path) + 1 + for line in proc.stdout.splitlines(): + line = line.replace('\n', '') + if 'FSTYPE' in line: + line = std.color_string(f'{"PATH":<{width}}{line}', 'BLUE') + else: + line = f'{path:<{width}}{line}' + report.append(line) + else: + # TODO Get dir details under macOS + report.append(std.color_string('PATH', 'BLUE')) + report.append(str(path)) + + # Done + return report + + +def build_disk_report(dev): + """Build device report, returns list.""" + children = dev.details.get('children', []) + report = [] + + # Get widths + widths = { + 'fstype': max(6, len(str(dev.details.get('fstype', '')))), + 'label': max(5, len(str(dev.details.get('label', '')))), + 'name': max(4, len(dev.path.name)), + } + for child in children: + widths['fstype'] = max(widths['fstype'], len(str(child['fstype']))) + widths['label'] = max(widths['label'], len(str(child['label']))) + widths['name'] = max( + widths['name'], + len(child['name'].replace('/dev/', '')), + ) + widths = {k: v+1 for k, v in widths.items()} + + # Disk details + report.append(f'{dev.path.name} {dev.description}') + report.append(' ') + dev_fstype = dev.details.get('fstype', '') + dev_label = dev.details.get('label', '') + dev_name = dev.path.name + dev_size = std.bytes_to_string(dev.details["size"], use_binary=False) + + # Partition details + report.append( + std.color_string( + ( + f'{"NAME":<{widths["name"]}}' + f'{" " if children else ""}' + f'{"SIZE":<7}' + f'{"FSTYPE":<{widths["fstype"]}}' + f'{"LABEL":<{widths["label"]}}' + ), + 'BLUE', + ), + ) + report.append( + f'{dev_name if dev_name else "":<{widths["name"]}}' + f'{" " if children else ""}' + f'{dev_size:>6} ' + f'{dev_fstype if dev_fstype else "":<{widths["fstype"]}}' + f'{dev_label if dev_label else "":<{widths["label"]}}' + ) + for child in children: + fstype = child['fstype'] + label = child['label'] + name = child['name'].replace('/dev/', '') + size = std.bytes_to_string(child["size"], use_binary=False) + report.append( + f'{name if name else "":<{widths["name"]}}' + f'{size:>6} ' + f'{fstype if fstype else "":<{widths["fstype"]}}' + f'{label if label else "":<{widths["label"]}}' + ) + + # Indent children + if len(children) > 1: + report = [ + *report[:4], + *[f'├─{line}' for line in report[4:-1]], + f'└─{report[-1]}', + ] + elif len(children) == 1: + report[-1] = f'└─{report[-1]}' + + # Done + return report + + def build_main_menu(): """Build main menu, returns wk.std.Menu.""" menu = std.Menu(title=std.color_string('ddrescue TUI: Main Menu', 'GREEN')) @@ -291,6 +457,22 @@ def build_main_menu(): return menu +def build_object_report(obj): + """Build object report, returns list.""" + report = [] + + # Get details based on object given + if hasattr(obj, 'is_dir') and obj.is_dir(): + # Directory report + report = build_directory_report(obj) + else: + # Device report + report = build_disk_report(obj) + + # Done + return report + + def build_settings_menu(silent=True): """Build settings menu, returns wk.std.Menu.""" title_text = [ @@ -555,6 +737,25 @@ def select_disk_parts(prompt, disk): menu.add_action('Quit') object_list = [] + def _select_parts(menu): + """Loop over selection menu until at least one partition selected.""" + while True: + selection = menu.advanced_select( + f'Please select the parts to {prompt.lower()}: ', + ) + if 'All' in selection: + for option in menu.options.values(): + option['Selected'] = True + elif 'None' in selection: + for option in menu.options.values(): + option['Selected'] = False + elif 'Proceed' in selection: + if any([option['Selected'] for option in menu.options.values()]): + # At least one partition/device selected/device selected + break + elif 'Quit' in selection: + raise std.GenericAbort() + # Bail early if child device selected if disk.details.get('parent', False): return [disk] @@ -576,22 +777,7 @@ def select_disk_parts(prompt, disk): menu.title += std.color_string(' No partitions detected.', 'YELLOW') # Get selection - while True: - selection = menu.advanced_select( - f'Please select the parts to {prompt.lower()}: ', - ) - if 'All' in selection: - for option in menu.options.values(): - option['Selected'] = True - elif 'None' in selection: - for option in menu.options.values(): - option['Selected'] = False - elif 'Proceed' in selection: - if any([option['Selected'] for option in menu.options.values()]): - # At least one partition/device selected/device selected - break - elif 'Quit' in selection: - raise std.GenericAbort() + _select_parts(menu) # Build list of Disk() object_list for option in menu.options.values(): From 0f2007f5dc33e69c4c91f45ddc169bff4a1c55d6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Dec 2019 16:14:03 -0700 Subject: [PATCH 253/324] Set working directory for ddrescue TUI * If cloning use backup server share * If imaging use destination directory * If a preferred directory and fstype can't be used then warn the user --- scripts/wk/cfg/ddrescue.py | 5 --- scripts/wk/hw/ddrescue.py | 84 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index 2de2fc88..6097d7b9 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -7,11 +7,6 @@ import re from collections import OrderedDict -# General -MAP_DIR = '/Backups/ddrescue-tui' -RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] -RECOMMENDED_MAP_FSTYPES = ['cifs', 'ext2', 'ext3', 'ext4', 'vfat', 'xfs'] - # Layout TMUX_SIDE_WIDTH = 21 TMUX_LAYOUT = OrderedDict({ diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 9a8406bd..fe90c061 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -46,6 +46,8 @@ PANE_RATIOS = ( 4, # Journal (kernel messages) ) PLATFORM = std.PLATFORM +RECOMMENDED_FSTYPES = re.compile(r'^(ext[234]|ntfs|xfs)$') +RECOMMENDED_MAP_FSTYPES = re.compile(r'^(cifs|ext[234]|ntfs|vfat|xfs)$') SETTING_PRESETS = ( 'Default', 'Fast', @@ -76,7 +78,7 @@ class State(): self.init_tmux() exe.start_thread(self.fix_tmux_layout_loop) - def confirm_selections(self, mode, prompt): + def confirm_selections(self, mode, prompt, map_dir=None): """Show selection details and prompt for confirmation.""" report = [] @@ -111,6 +113,26 @@ class State(): ) report.append(' ') + # Map dir + if map_dir: + report.append(std.color_string('Map Save Directory', 'GREEN')) + report.append(str(map_dir)) + report.append(' ') + if map_dir and not fstype_is_ok(map_dir, map_dir=True): + report.append( + std.color_string( + 'Map file(s) are being saved to a non-recommended filesystem.', + 'YELLOW', + ), + ) + report.append( + std.color_string( + ['This is strongly discouraged and may lead to', 'DATA LOSS'], + [None, 'RED'], + ), + ) + report.append(' ') + # Prompt user std.clear_screen() std.print_report(report) @@ -201,7 +223,13 @@ class State(): self.confirm_selections(mode, 'Are these selections correct?') # Set working dir - # TODO + working_dir = get_working_dir(mode, self.destination) + if working_dir: + LOG.info('Set working directory to: %s', working_dir) + os.chdir(working_dir) + else: + LOG.error('Failed to set preferred working directory') + working_dir = pathlib.Path(os.getcwd()) # Add block pairs # NOTE: Destination is not updated @@ -211,7 +239,7 @@ class State(): # TODO # Confirmation #2 - self.confirm_selections(mode, 'Start recovery?') + self.confirm_selections(mode, 'Start recovery?', map_dir=working_dir) # Prep destination # if cloning and not resuming format destination @@ -512,6 +540,36 @@ def build_settings_menu(silent=True): return menu +def fstype_is_ok(path, map_dir=False): + """Check if filesystem type is acceptable, returns bool.""" + is_ok = False + fstype = None + + # Get fstype + if PLATFORM == 'Darwin': + # TODO: leave as None for now + pass + elif PLATFORM == 'Linux': + cmd = [ + 'findmnt', + '--noheadings', + '--output', 'FSTYPE', + '--target', path, + ] + proc = exe.run_program(cmd, check=False) + fstype = proc.stdout + fstype = fstype.strip().lower() + + # Check fstype + if map_dir: + is_ok = RECOMMENDED_MAP_FSTYPES.match(fstype) + else: + is_ok = RECOMMENDED_FSTYPES.match(fstype) + + # Done + return is_ok + + def get_object(path): """Get object based on path, returns obj.""" obj = None @@ -547,6 +605,26 @@ def get_object(path): return obj +def get_working_dir(mode, destination): + """Get working directory using mode and destination, returns path.""" + working_dir = None + if mode == 'Clone': + net.mount_backup_shares(read_write=True) + for server in cfg.net.BACKUP_SERVERS: + path = pathlib.Path(f'/Backups/{server}') + if path.exists() and fstype_is_ok(path, map_dir=True): + # Acceptable path found + working_dir = path + break + else: + path = pathlib.Path(destination).resolve() + if path.exists() and fstype_is_ok(path, map_dir=False): + working_dir = path + + # Done + return working_dir + + def main(): """Main function for ddrescue TUI.""" args = docopt(DOCSTRING) From 6d6380dc6a4215c5f7f938d68788fdee6c0579c2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Dec 2019 18:49:07 -0700 Subject: [PATCH 254/324] Added clone load/save and add block pair sections --- scripts/wk/hw/ddrescue.py | 147 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 142 insertions(+), 5 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index fe90c061..21dd52bd 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -3,6 +3,7 @@ # vim: sts=2 sw=2 ts=2 import atexit +import json import logging import os import pathlib @@ -30,6 +31,15 @@ Usage: Options: -h --help Show this page ''' +CLONE_SETTINGS = { + 'Source': None, + 'Destination': None, + 'Needs Format': False, + 'Table Type': None, + 'Partition Mapping': [ + # (5, 1) ## Clone source partition #5 to destination partition #1 + ], + } LOG = logging.getLogger(__name__) MENU_ACTIONS = ( 'Start', @@ -78,6 +88,101 @@ class State(): self.init_tmux() exe.start_thread(self.fix_tmux_layout_loop) + def add_clone_block_pairs(self, source_parts, working_dir): + """Add device to device block pairs and set settings if necessary.""" + part_prefix = '' + if re.search(r'(loop|mmc|nvme)', self.destination.path.name): + part_prefix = 'p' + settings = {} + + def _check_settings(settings): + """Check settings for issues and update as necessary.""" + if settings: + bail = False + for key in ('model', 'serial'): + if settings['Source'][key] != self.source.details[key]: + std.print_error(f"Clone settings don't match source {key}") + bail = True + if settings['Destination'][key] != self.destination.details[key]: + std.print_error(f"Clone settings don't match destination {key}") + bail = True + if bail: + raise std.GenericAbort() + + # Update settings + if not settings: + settings = CLONE_SETTINGS.copy() + if not settings['Source']: + settings['Source'] = { + 'model': self.source.details['model'], + 'serial': self.source.details['serial'], + } + if not settings['Destination']: + settings['Destination'] = { + 'model': self.destination.details['model'], + 'serial': self.destination.details['serial'], + } + + # Done + return settings + + # Clone settings + settings = self.load_settings(working_dir) + settings = _check_settings(settings) + + # Add pairs + if not self.source.path.samefile(source_parts[0].path): + # One or more partitions selected for cloning + if settings['Partition Mapping']: + for part_map in settings['Partition Mapping']: + bp_source = pathlib.Path( + f'{self.source.path}{part_prefix}{part_map[0]}', + ) + bp_dest = pathlib.Path( + f'{self.destination.path}{part_prefix}{part_map[1]}', + ) + # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + else: + # New run and new settings + offset = 0 + if (std.ask('Does the source disk contain an OS?') + and std.ask('Create an empty boot partition on the clone?')): + offset = 2 + settings['Needs Format'] = True + settings['Table Type'] = 'GPT' + if std.choice(['G', 'M'], 'GPT or MBR partition table?') == 'M': + offset = 1 + settings['Table Type'] = 'MBR' + + # Add pairs + for dest_num, part in enumerate(source_parts): + dest_num += offset + 1 + bp_source = part.path + bp_dest = pathlib.Path( + f'{self.destination.path}{part_prefix}{dest_num}', + ) + # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + + # Add to settings file + source_num = re.sub(r'^.*?(\d+)$', r'\1', part.path.name) + settings['Partition Mapping'].append([source_num, dest_num]) + + # Save settings + self.save_settings(settings, working_dir) + + else: + # Whole device or forced single partition selected, skip settings + bp_source = self.source.path + bp_dest = self.destination.path + # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + + def add_image_block_pairs(self, source_parts, working_dir): + """Add device to image file block pairs.""" + for part in source_parts: + bp_source = part.path + bp_dest = pathlib.Path(f'{self.destination.path}/{part_TODO}.dd') + # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + def confirm_selections(self, mode, prompt, map_dir=None): """Show selection details and prompt for confirmation.""" report = [] @@ -232,11 +337,10 @@ class State(): working_dir = pathlib.Path(os.getcwd()) # Add block pairs - # NOTE: Destination is not updated - # Load settings/maps - # Ask about boot partition - # Create pairs using paths - # TODO + if mode == 'Clone': + self.add_clone_block_pairs(source_parts, working_dir) + else: + self.add_image_block_pairs(source_parts, working_dir) # Confirmation #2 self.confirm_selections(mode, 'Start recovery?', map_dir=working_dir) @@ -273,6 +377,25 @@ class State(): # Source / Dest self.update_top_panes() + def load_settings(self, working_dir): + # pylint: disable=no-self-use + """Load settings from previous run, returns dict.""" + settings = {} + settings_file = pathlib.Path(f'{working_dir}/clone.json') + + # Try loading JSON data + if settings_file.exists(): + with open(settings_file, 'r') as _f: + try: + settings = json.loads(_f.read()) + except (OSError, json.JSONDecodeError): + LOG.error('Failed to load clone settings') + std.print_error('Invalid clone settings detected.') + raise std.GenericAbort() + + # Done + return settings + def save_debug_reports(self): """Save debug reports to disk.""" LOG.info('Saving debug reports') @@ -289,6 +412,19 @@ class State(): with open(f'{debug_dir}/bp_part#.report', 'a') as _f: _f.write('\n'.join(debug.generate_object_report(_bp))) + def save_settings(self, settings, working_dir): + # pylint: disable=no-self-use + """Save settings for future runs.""" + settings_file = pathlib.Path(f'{working_dir}/clone.json') + + # Try saving JSON data + try: + with open(settings_file, 'w') as _f: + json.dump(settings, _f) + except OSError: + std.print_error('Failed to save clone settings') + raise std.GenericAbort() + def update_progress_pane(self): """Update progress pane.""" report = [] @@ -609,6 +745,7 @@ def get_working_dir(mode, destination): """Get working directory using mode and destination, returns path.""" working_dir = None if mode == 'Clone': + std.print_info('Mounting backup shares...') net.mount_backup_shares(read_write=True) for server in cfg.net.BACKUP_SERVERS: path = pathlib.Path(f'/Backups/{server}') From 1ed630997151d4e28a20a14f9e629029d663d73e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Dec 2019 19:05:32 -0700 Subject: [PATCH 255/324] Include selected source parts in 1st confirmation --- scripts/wk/hw/ddrescue.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 21dd52bd..bb65feb6 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -183,7 +183,7 @@ class State(): bp_dest = pathlib.Path(f'{self.destination.path}/{part_TODO}.dd') # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) - def confirm_selections(self, mode, prompt, map_dir=None): + def confirm_selections(self, mode, prompt, map_dir=None, source_parts=None): """Show selection details and prompt for confirmation.""" report = [] @@ -238,6 +238,20 @@ class State(): ) report.append(' ') + # Source part(s) selected + if source_parts: + report.append(std.color_string('Source Part(s) selected', 'GREEN')) + if self.source.path.samefile(source_parts[0].path): + report.append('Whole Disk') + else: + report.append(std.color_string(f'{"NAME":<9} SIZE', 'BLUE')) + for part in source_parts: + report.append( + f'{part.path.name:<9} ' + f'{std.bytes_to_string(part.details["size"], use_binary=False)}' + ) + report.append(' ') + # Prompt user std.clear_screen() std.print_report(report) @@ -245,7 +259,6 @@ class State(): raise std.GenericAbort() def fix_tmux_layout(self, forced=True): - # pylint: disable=unused-argument """Fix tmux layout based on cfg.ddrescue.TMUX_LAYOUT.""" needs_fixed = tmux.layout_needs_fixed(self.panes, self.layout) @@ -325,7 +338,11 @@ class State(): self.update_progress_pane() # Confirmation #1 - self.confirm_selections(mode, 'Are these selections correct?') + self.confirm_selections( + mode=mode, + prompt='Are these selections correct?', + source_parts=source_parts, + ) # Set working dir working_dir = get_working_dir(mode, self.destination) From f71cc8ad682828a7a658a6c75ec3f58300728994 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Dec 2019 21:24:02 -0700 Subject: [PATCH 256/324] Expanded block pair sections and confirmations --- scripts/wk/hw/ddrescue.py | 162 ++++++++++++++++++++++++++++---------- 1 file changed, 121 insertions(+), 41 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index bb65feb6..28c3ba9d 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -73,6 +73,70 @@ STATUS_COLORS = { # Classes +class BlockPair(): + """Object for tracking source to dest recovery data.""" + def __init__(self, source, destination, model, working_dir): + """Initialize BlockPair() + + NOTE: source should be a wk.hw.obj.Disk() object + and destination should be a pathlib.Path() object. + """ + self.source = source.path + self.destination = destination + self.map_data = {} + self.map_path = None + self.size = source.details['size'] + + # Set map file + # e.g. '(Clone|Image)_Model[_p#]_Size[_Label].map' + map_name = model + if source.details['parent']: + part_num = re.sub(r"^.*?(\d+)$", r"\1", source.path.name) + map_name += f'_p{part_num}' + size_str = std.bytes_to_string( + size=source.details["size"], + use_binary=False, + ) + map_name += f'_{size_str.replace(" ", "")}' + if source.details.get('label', ''): + map_name += f'_{source.details["label"]}' + map_name = map_name.replace(' ', '_') + map_name = map_name.replace('/', '_') + if destination.is_dir(): + # Imaging + self.map_path = pathlib.Path(f'{destination}/Image_{map_name}.map') + self.destination = self.map_path.with_suffix('.dd') + else: + # Cloning + self.map_path = pathlib.Path(f'{working_dir}/Clone_{map_name}.map') + + # Read map file + self.load_map_data() + + def load_map_data(self): + """Load map data from file. + + NOTE: If the file is missing it is assumed that recovery hasn't + started yet so default values will be returned instead. + """ + self.map_data = {} + + # Read file + if self.map_path.exists(): + with open(self.map_path, 'r') as _f: + #TODO + pass + + def pass_complete(self, pass_num): + """Check if pass_num is complete based on map data, returns bool.""" + complete = False + + # TODO + + # Done + return complete + + class State(): """Object for tracking hardware diagnostic data.""" def __init__(self): @@ -88,6 +152,19 @@ class State(): self.init_tmux() exe.start_thread(self.fix_tmux_layout_loop) + def add_block_pair(self, source, destination, working_dir): + """Add BlockPair object and run safety checks.""" + self.block_pairs.append( + BlockPair( + source=source, + destination=destination, + model=self.source.details['model'], + working_dir=working_dir, + )) + + # Safety Checks + # TODO + def add_clone_block_pairs(self, source_parts, working_dir): """Add device to device block pairs and set settings if necessary.""" part_prefix = '' @@ -135,13 +212,13 @@ class State(): # One or more partitions selected for cloning if settings['Partition Mapping']: for part_map in settings['Partition Mapping']: - bp_source = pathlib.Path( + bp_source = hw_obj.Disk( f'{self.source.path}{part_prefix}{part_map[0]}', ) bp_dest = pathlib.Path( f'{self.destination.path}{part_prefix}{part_map[1]}', ) - # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + self.add_block_pair(bp_source, bp_dest, working_dir) else: # New run and new settings offset = 0 @@ -157,11 +234,10 @@ class State(): # Add pairs for dest_num, part in enumerate(source_parts): dest_num += offset + 1 - bp_source = part.path bp_dest = pathlib.Path( f'{self.destination.path}{part_prefix}{dest_num}', ) - # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + self.add_block_pair(part, bp_dest, working_dir) # Add to settings file source_num = re.sub(r'^.*?(\d+)$', r'\1', part.path.name) @@ -172,16 +248,14 @@ class State(): else: # Whole device or forced single partition selected, skip settings - bp_source = self.source.path bp_dest = self.destination.path - # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + self.add_block_pair(self.source, bp_dest, working_dir) def add_image_block_pairs(self, source_parts, working_dir): """Add device to image file block pairs.""" for part in source_parts: - bp_source = part.path - bp_dest = pathlib.Path(f'{self.destination.path}/{part_TODO}.dd') - # TODO: add bp(bp_source, bp_dest, map_dir=working_dir) + bp_dest = self.destination + self.add_block_pair(part, bp_dest, working_dir) def confirm_selections(self, mode, prompt, map_dir=None, source_parts=None): """Show selection details and prompt for confirmation.""" @@ -199,44 +273,50 @@ class State(): report.extend(build_object_report(self.destination)) report.append(' ') + # Show deletion warning if necessary + # NOTE: The check for block_pairs is to limit this section + # to the second confirmation + if mode == 'Clone' and self.block_pairs: + report.append(std.color_string('WARNING', 'YELLOW')) + report.append( + 'All data will be deleted from the destination listed above.', + ) + report.append( + std.color_string( + ['This is irreversible and will lead to', 'DATA LOSS.'], + ['YELLOW', 'RED'], + ), + ) + report.append(' ') + # Block pairs if self.block_pairs: - # Show mapping - # TODO - - # Show deletion warning if required - if mode == 'Clone': - report.append(std.color_string('WARNING', 'YELLOW')) - report.append( - 'All data will be deleted from the destination listed above.', - ) - report.append( - std.color_string( - ['This is irreversible and will lead to', 'DATA LOSS.'], - ['YELLOW', 'RED'], - ), - ) - report.append(' ') + report.append(std.color_string('Block Pairs', 'GREEN')) + # TODO Move to separate function and include resume messages + for pair in self.block_pairs: + # Show mapping + report.append(f'{pair.source.name} --> {pair.destination.name}') + report.append(' ') # Map dir if map_dir: report.append(std.color_string('Map Save Directory', 'GREEN')) - report.append(str(map_dir)) - report.append(' ') - if map_dir and not fstype_is_ok(map_dir, map_dir=True): - report.append( - std.color_string( - 'Map file(s) are being saved to a non-recommended filesystem.', - 'YELLOW', - ), - ) - report.append( - std.color_string( - ['This is strongly discouraged and may lead to', 'DATA LOSS'], - [None, 'RED'], - ), - ) + report.append(f'{map_dir}/') report.append(' ') + if not fstype_is_ok(map_dir, map_dir=True): + report.append( + std.color_string( + 'Map file(s) are being saved to a non-recommended filesystem.', + 'YELLOW', + ), + ) + report.append( + std.color_string( + ['This is strongly discouraged and may lead to', 'DATA LOSS'], + [None, 'RED'], + ), + ) + report.append(' ') # Source part(s) selected if source_parts: @@ -521,7 +601,7 @@ class State(): # Functions def build_directory_report(path): """Build directory report, returns list.""" - path = str(path) + path = f'{path}/' report = [] # Get details From 44b6c4eedb4fcb43093dea71a1c3260ce1cd9922 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Dec 2019 21:24:55 -0700 Subject: [PATCH 257/324] Disable network servers by default --- scripts/wk/cfg/net.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/wk/cfg/net.py b/scripts/wk/cfg/net.py index 5e42dd84..f530bb6e 100644 --- a/scripts/wk/cfg/net.py +++ b/scripts/wk/cfg/net.py @@ -5,29 +5,29 @@ # Servers BACKUP_SERVERS = { - 'Server One': { - 'Address': '10.0.0.10', - 'Share': 'Backups', - 'RO-User': 'restore', - 'RO-Pass': 'Abracadabra', - 'RW-User': 'backup', - 'RW-Pass': 'Abracadabra', - }, - 'Server Two': { - 'Address': 'servertwo.example.com', - 'Share': 'Backups', - 'RO-User': 'restore', - 'RO-Pass': 'Abracadabra', - 'RW-User': 'backup', - 'RW-Pass': 'Abracadabra', - }, + #'Server One': { + # 'Address': '10.0.0.10', + # 'Share': 'Backups', + # 'RO-User': 'restore', + # 'RO-Pass': 'Abracadabra', + # 'RW-User': 'backup', + # 'RW-Pass': 'Abracadabra', + # }, + #'Server Two': { + # 'Address': 'servertwo.example.com', + # 'Share': 'Backups', + # 'RO-User': 'restore', + # 'RO-Pass': 'Abracadabra', + # 'RW-User': 'backup', + # 'RW-Pass': 'Abracadabra', + # }, } CRASH_SERVER = { - 'Name': 'CrashServer', - 'Url': '', - 'User': '', - 'Pass': '', - 'Headers': {'X-Requested-With': 'XMLHttpRequest'}, + #'Name': 'CrashServer', + #'Url': '', + #'User': '', + #'Pass': '', + #'Headers': {'X-Requested-With': 'XMLHttpRequest'}, } From e7e3261b0a31945b51e1cdbd401c57d1f9702bb5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Dec 2019 16:38:42 -0700 Subject: [PATCH 258/324] Fixed partition separators --- scripts/wk/hw/ddrescue.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 28c3ba9d..c88472c6 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -167,9 +167,8 @@ class State(): def add_clone_block_pairs(self, source_parts, working_dir): """Add device to device block pairs and set settings if necessary.""" - part_prefix = '' - if re.search(r'(loop|mmc|nvme)', self.destination.path.name): - part_prefix = 'p' + source_sep = get_partition_separator(self.source.path.name) + dest_sep = get_partition_separator(self.destination.path.name) settings = {} def _check_settings(settings): @@ -213,10 +212,10 @@ class State(): if settings['Partition Mapping']: for part_map in settings['Partition Mapping']: bp_source = hw_obj.Disk( - f'{self.source.path}{part_prefix}{part_map[0]}', + f'{self.source.path}{source_sep}{part_map[0]}', ) bp_dest = pathlib.Path( - f'{self.destination.path}{part_prefix}{part_map[1]}', + f'{self.destination.path}{dest_sep}{part_map[1]}', ) self.add_block_pair(bp_source, bp_dest, working_dir) else: @@ -235,7 +234,7 @@ class State(): for dest_num, part in enumerate(source_parts): dest_num += offset + 1 bp_dest = pathlib.Path( - f'{self.destination.path}{part_prefix}{dest_num}', + f'{self.destination.path}{dest_sep}{dest_num}', ) self.add_block_pair(part, bp_dest, working_dir) @@ -838,6 +837,15 @@ def get_object(path): return obj +def get_partition_separator(name): + """Get partition separator based on device name, returns str.""" + separator = '' + if re.search(r'(loop|mmc|nvme)', name, re.IGNORECASE): + separator = 'p' + + return separator + + def get_working_dir(mode, destination): """Get working directory using mode and destination, returns path.""" working_dir = None From 67bb9223aaf7f06157387991832937463d64fe94 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Dec 2019 16:42:18 -0700 Subject: [PATCH 259/324] Moved block pair report to new function --- scripts/wk/hw/ddrescue.py | 54 +++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index c88472c6..b3881666 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -221,8 +221,7 @@ class State(): else: # New run and new settings offset = 0 - if (std.ask('Does the source disk contain an OS?') - and std.ask('Create an empty boot partition on the clone?')): + if std.ask('Create an empty Windows boot partition on the clone?'): offset = 2 settings['Needs Format'] = True settings['Table Type'] = 'GPT' @@ -256,7 +255,8 @@ class State(): bp_dest = self.destination self.add_block_pair(part, bp_dest, working_dir) - def confirm_selections(self, mode, prompt, map_dir=None, source_parts=None): + def confirm_selections( + self, mode, prompt, working_dir=None, source_parts=None): """Show selection details and prompt for confirmation.""" report = [] @@ -290,19 +290,19 @@ class State(): # Block pairs if self.block_pairs: - report.append(std.color_string('Block Pairs', 'GREEN')) - # TODO Move to separate function and include resume messages - for pair in self.block_pairs: - # Show mapping - report.append(f'{pair.source.name} --> {pair.destination.name}') - report.append(' ') + report.extend( + build_block_pair_report( + self.block_pairs, + self.load_settings(working_dir) if mode == 'Clone' else {}, + ), + ) # Map dir - if map_dir: + if working_dir: report.append(std.color_string('Map Save Directory', 'GREEN')) - report.append(f'{map_dir}/') + report.append(f'{working_dir}/') report.append(' ') - if not fstype_is_ok(map_dir, map_dir=True): + if not fstype_is_ok(working_dir, map_dir=True): report.append( std.color_string( 'Map file(s) are being saved to a non-recommended filesystem.', @@ -439,7 +439,7 @@ class State(): self.add_image_block_pairs(source_parts, working_dir) # Confirmation #2 - self.confirm_selections(mode, 'Start recovery?', map_dir=working_dir) + self.confirm_selections(mode, 'Start recovery?', working_dir=working_dir) # Prep destination # if cloning and not resuming format destination @@ -598,6 +598,34 @@ class State(): # Functions +def build_block_pair_report(block_pairs, settings): + """Build block pair report, returns list.""" + report = [] + if block_pairs: + report.append(std.color_string('Block Pairs', 'GREEN')) + else: + # Bail early + return report + + # Show block pair mapping + if settings and settings['Needs Format']: + if settings['Table Type'] == 'GPT': + report.append(f'{" —— ":<9} --> EFI System Partition') + report.append(f'{" —— ":<9} --> Microsoft Reserved Partition') + elif settings['Table Type'] == 'MBR': + report.append(f'{" —— ":<9} --> System Reserved') + for pair in block_pairs: + report.append(f'{pair.source.name:<9} --> {pair.destination.name}') + report.append(' ') + + # Show resume messages as necessary + # TODO If settings --> Add loaded settings msg + # TODO If anything recovered --> Add resume msg + + # Done + return report + + def build_directory_report(path): """Build directory report, returns list.""" path = f'{path}/' From 4c50a1fb8a852ff4e899356cdbba4bc68ec94934 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Dec 2019 16:43:07 -0700 Subject: [PATCH 260/324] Added first run flag to clone settings * If the loaded settings are for a non-attempted recovery discard settings --- scripts/wk/hw/ddrescue.py | 42 +++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index b3881666..7538a5fa 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -34,6 +34,7 @@ Options: CLONE_SETTINGS = { 'Source': None, 'Destination': None, + 'First Run': True, 'Needs Format': False, 'Table Type': None, 'Partition Mapping': [ @@ -174,16 +175,20 @@ class State(): def _check_settings(settings): """Check settings for issues and update as necessary.""" if settings: - bail = False - for key in ('model', 'serial'): - if settings['Source'][key] != self.source.details[key]: - std.print_error(f"Clone settings don't match source {key}") - bail = True - if settings['Destination'][key] != self.destination.details[key]: - std.print_error(f"Clone settings don't match destination {key}") - bail = True - if bail: - raise std.GenericAbort() + if settings['First Run']: + # Previous run aborted before starting recovery, settings discarded + settings = {} + else: + bail = False + for key in ('model', 'serial'): + if settings['Source'][key] != self.source.details[key]: + std.print_error(f"Clone settings don't match source {key}") + bail = True + if settings['Destination'][key] != self.destination.details[key]: + std.print_error(f"Clone settings don't match destination {key}") + bail = True + if bail: + raise std.GenericAbort() # Update settings if not settings: @@ -619,7 +624,22 @@ def build_block_pair_report(block_pairs, settings): report.append(' ') # Show resume messages as necessary - # TODO If settings --> Add loaded settings msg + if settings: + if not settings['First Run']: + report.append( + std.color_string( + ['NOTE:', 'Clone settings loaded from previous run.'], + ['BLUE', None], + ), + ) + if settings['Needs Format'] and settings['Table Type']: + msg = f'Destination will be formatted using {settings["Table Type"]}' + report.append( + std.color_string( + ['NOTE:', msg], + ['BLUE', None], + ), + ) # TODO If anything recovered --> Add resume msg # Done From f542b62f3ce386f6582d8bef478925a06bfcb9ac Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Dec 2019 16:44:26 -0700 Subject: [PATCH 261/324] Use source model name in clone settings save file --- scripts/wk/hw/ddrescue.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 7538a5fa..6db309a2 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -482,7 +482,9 @@ class State(): # pylint: disable=no-self-use """Load settings from previous run, returns dict.""" settings = {} - settings_file = pathlib.Path(f'{working_dir}/clone.json') + settings_file = pathlib.Path( + f'{working_dir}/Clone_{self.source.details["model"]}.json', + ) # Try loading JSON data if settings_file.exists(): @@ -516,7 +518,9 @@ class State(): def save_settings(self, settings, working_dir): # pylint: disable=no-self-use """Save settings for future runs.""" - settings_file = pathlib.Path(f'{working_dir}/clone.json') + settings_file = pathlib.Path( + f'{working_dir}/Clone_{self.source.details["model"]}.json', + ) # Try saving JSON data try: From ef6abce6ab9d1655b3ef85a1e37e6390cc70d854 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Dec 2019 17:35:38 -0700 Subject: [PATCH 262/324] Skip source partition selection if using JSON data --- scripts/wk/hw/ddrescue.py | 124 +++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 6db309a2..a073209f 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -166,65 +166,34 @@ class State(): # Safety Checks # TODO - def add_clone_block_pairs(self, source_parts, working_dir): + def add_clone_block_pairs(self, working_dir): """Add device to device block pairs and set settings if necessary.""" source_sep = get_partition_separator(self.source.path.name) dest_sep = get_partition_separator(self.destination.path.name) settings = {} - def _check_settings(settings): - """Check settings for issues and update as necessary.""" - if settings: - if settings['First Run']: - # Previous run aborted before starting recovery, settings discarded - settings = {} - else: - bail = False - for key in ('model', 'serial'): - if settings['Source'][key] != self.source.details[key]: - std.print_error(f"Clone settings don't match source {key}") - bail = True - if settings['Destination'][key] != self.destination.details[key]: - std.print_error(f"Clone settings don't match destination {key}") - bail = True - if bail: - raise std.GenericAbort() - - # Update settings - if not settings: - settings = CLONE_SETTINGS.copy() - if not settings['Source']: - settings['Source'] = { - 'model': self.source.details['model'], - 'serial': self.source.details['serial'], - } - if not settings['Destination']: - settings['Destination'] = { - 'model': self.destination.details['model'], - 'serial': self.destination.details['serial'], - } - - # Done - return settings - # Clone settings settings = self.load_settings(working_dir) - settings = _check_settings(settings) # Add pairs - if not self.source.path.samefile(source_parts[0].path): - # One or more partitions selected for cloning - if settings['Partition Mapping']: - for part_map in settings['Partition Mapping']: - bp_source = hw_obj.Disk( - f'{self.source.path}{source_sep}{part_map[0]}', - ) - bp_dest = pathlib.Path( - f'{self.destination.path}{dest_sep}{part_map[1]}', - ) - self.add_block_pair(bp_source, bp_dest, working_dir) + if settings['Partition Mapping']: + # Resume previous run, load pairs from settings file + for part_map in settings['Partition Mapping']: + bp_source = hw_obj.Disk( + f'{self.source.path}{source_sep}{part_map[0]}', + ) + bp_dest = pathlib.Path( + f'{self.destination.path}{dest_sep}{part_map[1]}', + ) + self.add_block_pair(bp_source, bp_dest, working_dir) + else: + source_parts = select_disk_parts('Clone', self.source) + if self.source.path.samefile(source_parts[0].path): + # Whole disk (or single partition via args), skip settings + bp_dest = self.destination.path + self.add_block_pair(self.source, bp_dest, working_dir) else: - # New run and new settings + # New run, use new settings file offset = 0 if std.ask('Create an empty Windows boot partition on the clone?'): offset = 2 @@ -246,13 +215,8 @@ class State(): source_num = re.sub(r'^.*?(\d+)$', r'\1', part.path.name) settings['Partition Mapping'].append([source_num, dest_num]) - # Save settings - self.save_settings(settings, working_dir) - - else: - # Whole device or forced single partition selected, skip settings - bp_dest = self.destination.path - self.add_block_pair(self.source, bp_dest, working_dir) + # Save settings + self.save_settings(settings, working_dir) def add_image_block_pairs(self, source_parts, working_dir): """Add device to image file block pairs.""" @@ -298,9 +262,10 @@ class State(): report.extend( build_block_pair_report( self.block_pairs, - self.load_settings(working_dir) if mode == 'Clone' else {}, + self.load_settings(working_dir, False) if mode == 'Clone' else {}, ), ) + report.append(' ') # Map dir if working_dir: @@ -381,6 +346,7 @@ class State(): def init_recovery(self, docopt_args): """Select source/dest and set env.""" std.clear_screen() + source_parts = [] # Set log self.log_dir = log.format_log_path() @@ -402,7 +368,6 @@ class State(): self.source = get_object(docopt_args['']) if not self.source: self.source = select_disk('Source') - source_parts = select_disk_parts(mode, self.source) self.update_top_panes() # Select destination @@ -439,8 +404,9 @@ class State(): # Add block pairs if mode == 'Clone': - self.add_clone_block_pairs(source_parts, working_dir) + self.add_clone_block_pairs(working_dir) else: + source_parts = select_disk_parts(mode, self.source) self.add_image_block_pairs(source_parts, working_dir) # Confirmation #2 @@ -478,8 +444,7 @@ class State(): # Source / Dest self.update_top_panes() - def load_settings(self, working_dir): - # pylint: disable=no-self-use + def load_settings(self, working_dir, discard_unused_settings=False): """Load settings from previous run, returns dict.""" settings = {} settings_file = pathlib.Path( @@ -496,6 +461,37 @@ class State(): std.print_error('Invalid clone settings detected.') raise std.GenericAbort() + # Check settings + if settings: + if settings['First Run'] and discard_unused_settings: + # Previous run aborted before starting recovery, discard settings + settings = {} + else: + bail = False + for key in ('model', 'serial'): + if settings['Source'][key] != self.source.details[key]: + std.print_error(f"Clone settings don't match source {key}") + bail = True + if settings['Destination'][key] != self.destination.details[key]: + std.print_error(f"Clone settings don't match destination {key}") + bail = True + if bail: + raise std.GenericAbort() + + # Update settings + if not settings: + settings = CLONE_SETTINGS.copy() + if not settings['Source']: + settings['Source'] = { + 'model': self.source.details['model'], + 'serial': self.source.details['serial'], + } + if not settings['Destination']: + settings['Destination'] = { + 'model': self.destination.details['model'], + 'serial': self.destination.details['serial'], + } + # Done return settings @@ -625,10 +621,10 @@ def build_block_pair_report(block_pairs, settings): report.append(f'{" —— ":<9} --> System Reserved') for pair in block_pairs: report.append(f'{pair.source.name:<9} --> {pair.destination.name}') - report.append(' ') # Show resume messages as necessary if settings: + report.append(' ') if not settings['First Run']: report.append( std.color_string( @@ -646,6 +642,10 @@ def build_block_pair_report(block_pairs, settings): ) # TODO If anything recovered --> Add resume msg + # Remove double line-break + if report[-1] == ' ': + report.pop(-1) + # Done return report From 20ffa0c6db5229aa6f337fe7c01d592ab947367d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Dec 2019 18:01:04 -0700 Subject: [PATCH 263/324] Added --start-fresh argument --- scripts/wk/hw/ddrescue.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index a073209f..780d81c4 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -9,13 +9,14 @@ import os import pathlib import plistlib import re +import shutil import subprocess import time from collections import OrderedDict from docopt import docopt -from wk import cfg, debug, exe, log, net, std, tmux +from wk import cfg, debug, exe, io, log, net, std, tmux from wk.hw import obj as hw_obj from wk.hw import sensors as hw_sensors @@ -25,11 +26,12 @@ DOCSTRING = f'''{cfg.main.KIT_NAME_FULL}: ddrescue TUI Usage: ddrescue-tui - ddrescue-tui (clone|image) [ []] + ddrescue-tui [options] (clone|image) [ []] ddrescue-tui (-h | --help) Options: -h --help Show this page + --start-fresh Ignore previous runs and start new recovery ''' CLONE_SETTINGS = { 'Source': None, @@ -224,6 +226,23 @@ class State(): bp_dest = self.destination self.add_block_pair(part, bp_dest, working_dir) + def clean_working_dir(self, working_dir): + """Clean working directory to ensure a fresh recovery session. + + NOTE: Data from previous sessions will be preserved + in a backup directory. + """ + backup_directory = pathlib.Path(f'{working_dir}/prev') + backup_directory = io.non_clobber_path(backup_directory) + backup_directory.mkdir() + + # Move settings, maps, etc to backup_directory + for entry in os.scandir(working_directory): + if entry.name.endswith(('.dd', '.json', '.map')): + new_path = f'{backup_directory}/{entry.name}' + new_path = io.non_clobber_path(new_path) + shutil.move(entry.path, new_path) + def confirm_selections( self, mode, prompt, working_dir=None, source_parts=None): """Show selection details and prompt for confirmation.""" @@ -402,6 +421,10 @@ class State(): LOG.error('Failed to set preferred working directory') working_dir = pathlib.Path(os.getcwd()) + # Start fresh if requested + if docopt_args['--start-fresh']: + self.clean_working_dir(working_dir) + # Add block pairs if mode == 'Clone': self.add_clone_block_pairs(working_dir) From c083a124ad1101b29d640f7ce8aa08e063ba5080 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Dec 2019 17:00:41 -0700 Subject: [PATCH 264/324] Adjusted wk.std.input_text() again * Dropped tcflush usage for simplicity * Readded the prompt usage from 564745f03bf96a4fcbcc32bbfda6b255f70e0663 --- scripts/wk/std.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index f8254ea0..88d70d25 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -19,13 +19,6 @@ from collections import OrderedDict import requests -try: - from termios import tcflush, TCIOFLUSH -except ImportError: - if os.name == 'posix': - # Not worried about this under Windows - raise - from wk.cfg.main import ( ENABLED_UPLOAD_DATA, INDENT, @@ -820,13 +813,11 @@ def input_text(prompt='Enter text'): response = None if prompt[-1:] != ' ': prompt += ' ' + print(prompt, end='', flush=True) while response is None: - if os.name == 'posix': - # Flush input to (hopefully) avoid EOFError - tcflush(sys.stdin, TCIOFLUSH) try: - response = input(prompt) + response = input() LOG.debug('%s%s', prompt, response) except EOFError: # Ignore and try again From 20787da275fd21f1f6e66dc885cc275cc2ebc1ac Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Dec 2019 17:19:26 -0700 Subject: [PATCH 265/324] Optionally disallow empty responses to input_text --- scripts/wk/std.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 88d70d25..ab72314a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -807,7 +807,7 @@ def get_log_filepath(): return log_filepath -def input_text(prompt='Enter text'): +def input_text(prompt='Enter text', allow_empty_response=True): """Get text from user, returns string.""" prompt = str(prompt) response = None @@ -823,6 +823,11 @@ def input_text(prompt='Enter text'): # Ignore and try again LOG.warning('Exception occured', exc_info=True) print('', flush=True) + if not allow_empty_response: + if response is None or not response.strip(): + # The None check here is used to avoid a TypeError if response is None + print(f'\r{prompt}', end='', flush=True) + response = None return response From bd3601e0c8786990974e1a3db5d3c699b88f024c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Dec 2019 17:35:58 -0700 Subject: [PATCH 266/324] Ask for ticket ID and use in working_dir --- scripts/wk/hw/ddrescue.py | 44 ++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 780d81c4..fdc6422e 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -232,14 +232,14 @@ class State(): NOTE: Data from previous sessions will be preserved in a backup directory. """ - backup_directory = pathlib.Path(f'{working_dir}/prev') - backup_directory = io.non_clobber_path(backup_directory) - backup_directory.mkdir() + backup_dir = pathlib.Path(f'{working_dir}/prev') + backup_dir = io.non_clobber_path(backup_dir) + backup_dir.mkdir() - # Move settings, maps, etc to backup_directory - for entry in os.scandir(working_directory): + # Move settings, maps, etc to backup_dir + for entry in os.scandir(working_dir): if entry.name.endswith(('.dd', '.json', '.map')): - new_path = f'{backup_directory}/{entry.name}' + new_path = f'{backup_dir}/{entry.name}' new_path = io.non_clobber_path(new_path) shutil.move(entry.path, new_path) @@ -414,12 +414,7 @@ class State(): # Set working dir working_dir = get_working_dir(mode, self.destination) - if working_dir: - LOG.info('Set working directory to: %s', working_dir) - os.chdir(working_dir) - else: - LOG.error('Failed to set preferred working directory') - working_dir = pathlib.Path(os.getcwd()) + os.chdir(working_dir) # Start fresh if requested if docopt_args['--start-fresh']: @@ -923,7 +918,20 @@ def get_partition_separator(name): def get_working_dir(mode, destination): """Get working directory using mode and destination, returns path.""" + ticket_id = None working_dir = None + + # Set ticket ID + while ticket_id is None: + ticket_id = std.input_text( + prompt='Please enter ticket ID:', + allow_empty_response=False, + ) + ticket_id = ticket_id.replace(' ', '_') + if not re.match(r'^\d+', ticket_id): + ticket_id = None + + # Use preferred path if possible if mode == 'Clone': std.print_info('Mounting backup shares...') net.mount_backup_shares(read_write=True) @@ -938,6 +946,18 @@ def get_working_dir(mode, destination): if path.exists() and fstype_is_ok(path, map_dir=False): working_dir = path + # Default to current dir if necessary + if not working_dir: + LOG.error('Failed to set preferred working directory') + working_dir = pathlib.Path(os.getcwd()) + + # Set subdir using ticket ID + working_dir = working_dir.joinpath(ticket_id) + LOG.info('Set working directory to: %s', working_dir) + + # Create directory + working_dir.mkdir(parents=True, exist_ok=True) + # Done return working_dir From fc0a37999bdee8f95ae9c85bfbdba6e73b38a1be Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Dec 2019 19:27:08 -0700 Subject: [PATCH 267/324] Added size safety check to ddrescue TUI --- scripts/wk/hw/ddrescue.py | 61 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index fdc6422e..ca0fad40 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -16,6 +16,8 @@ import time from collections import OrderedDict from docopt import docopt +import psutil + from wk import cfg, debug, exe, io, log, net, std, tmux from wk.hw import obj as hw_obj from wk.hw import sensors as hw_sensors @@ -36,6 +38,7 @@ Options: CLONE_SETTINGS = { 'Source': None, 'Destination': None, + 'Create Boot Partition': False, 'First Run': True, 'Needs Format': False, 'Table Type': None, @@ -175,7 +178,7 @@ class State(): settings = {} # Clone settings - settings = self.load_settings(working_dir) + settings = self.load_settings(working_dir, discard_unused_settings=True) # Add pairs if settings['Partition Mapping']: @@ -196,10 +199,11 @@ class State(): self.add_block_pair(self.source, bp_dest, working_dir) else: # New run, use new settings file + settings['Needs Format'] = True offset = 0 if std.ask('Create an empty Windows boot partition on the clone?'): offset = 2 - settings['Needs Format'] = True + settings['Create Boot Partition'] = True settings['Table Type'] = 'GPT' if std.choice(['G', 'M'], 'GPT or MBR partition table?') == 'M': offset = 1 @@ -281,7 +285,7 @@ class State(): report.extend( build_block_pair_report( self.block_pairs, - self.load_settings(working_dir, False) if mode == 'Clone' else {}, + self.load_settings(working_dir) if mode == 'Clone' else {}, ), ) report.append(' ') @@ -427,6 +431,9 @@ class State(): source_parts = select_disk_parts(mode, self.source) self.add_image_block_pairs(source_parts, working_dir) + # Safety Check + self.safety_check(mode, working_dir) + # Confirmation #2 self.confirm_selections(mode, 'Start recovery?', working_dir=working_dir) @@ -513,6 +520,52 @@ class State(): # Done return settings + def safety_check(self, mode, working_dir): + """Run safety check and abort if necessary.""" + required_size = sum([pair.size for pair in self.block_pairs]) + + # Increase required_size if necessary + if mode == 'Clone' and settings['Needs Format']: + settings = self.load_settings(working_dir) + if settings['Table Type'] == 'GPT': + # Below is the size calculation for the GPT + # 1 LBA for the protective MBR + # 33 LBAs each for the primary and backup GPT tables + # Source: https://en.wikipedia.org/wiki/GUID_Partition_Table + required_size += (1 + 33 + 33) * self.destination.details['phy-sec'] + if settings['Create Boot Partition']: + # 384MiB EFI System Partition and a 16MiB MS Reserved partition + required_size += (384 + 16) * 1024**2 + else: + # MBR only requires one LBA but adding a full 4096 bytes anyway + required_size += 4096 + if settings['Create Boot Partition']: + # 100MiB System Reserved partition + required_size += 100 * 1024**2 + + # Reduce required_size if necessary + if mode == 'Image': + for pair in self.block_pairs: + if pair.destination.exists(): + # NOTE: This uses the "max space" of the destination + # i.e. not the apparent size which is smaller for sparse files + # While this can result in an out-of-space error it's better + # than nothing. + required_size -= pair.destination.stat().st_size + + # Check destination size + if mode == 'Clone': + destination_size = self.destination.details['size'] + error_msg = 'A larger destination disk is required' + else: + # NOTE: Adding an extra 5% here to better ensure it will fit + destination_size = psutil.disk_usage(self.destination).free + destination_size *= 1.05 + error_msg = 'Not enough free space on the destination' + if required_size > destination_size: + std.print_error(error_msg) + raise std.GenericAbort() + def save_debug_reports(self): """Save debug reports to disk.""" LOG.info('Saving debug reports') @@ -631,7 +684,7 @@ def build_block_pair_report(block_pairs, settings): return report # Show block pair mapping - if settings and settings['Needs Format']: + if settings and settings['Create Boot Partition']: if settings['Table Type'] == 'GPT': report.append(f'{" —— ":<9} --> EFI System Partition') report.append(f'{" —— ":<9} --> Microsoft Reserved Partition') From 7d7dc7063063fd6e694a9445f85c08ff5de49bd8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Dec 2019 20:16:35 -0700 Subject: [PATCH 268/324] Added map data loading sections --- scripts/wk/hw/ddrescue.py | 74 +++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index ca0fad40..02ce3596 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -10,10 +10,8 @@ import pathlib import plistlib import re import shutil -import subprocess import time -from collections import OrderedDict from docopt import docopt import psutil @@ -46,6 +44,12 @@ CLONE_SETTINGS = { # (5, 1) ## Clone source partition #5 to destination partition #1 ], } +DDRESCUE_LOG_REGEX = re.compile( + r'^\s*(?P\S+):\s+' + r'(?P\d+)\s+' + r'(?P[PTGMKB]i?B?)', + re.IGNORECASE, + ) LOG = logging.getLogger(__name__) MENU_ACTIONS = ( 'Start', @@ -119,19 +123,51 @@ class BlockPair(): # Read map file self.load_map_data() + def get_rescued_size(self): + """Get rescued size using map data. + + NOTE: Returns 0 if no map data is available. + """ + self.load_map_data() + return self.map_data.get('rescued', 0) + def load_map_data(self): """Load map data from file. NOTE: If the file is missing it is assumed that recovery hasn't started yet so default values will be returned instead. """ - self.map_data = {} + data = {'full recovery': False, 'pass completed': False} - # Read file - if self.map_path.exists(): - with open(self.map_path, 'r') as _f: - #TODO - pass + # Get output from ddrescuelog + cmd = [ + 'ddrescuelog', + '--binary-prefixes', + '--show-status', + self.map_path, + ] + proc = exe.run_program(cmd, check=False) + + # Parse output + for line in proc.stdout.splitlines(): + _r = DDRESCUE_LOG_REGEX.search(line) + if _r: + data[_r.group('key')] = std.string_to_bytes( + f'{_r.group("size")} {_r.group("unit")}', + ) + data['pass completed'] = 'current status: finished' in line.lower() + + # Check if 100% done + cmd = [ + 'ddrescuelog', + '--done-status', + self.map_path, + ] + proc = exe.run_program(cmd, check=False) + data['full recovery'] = proc.returncode == 0 + + # Done + self.map_data.update(data) def pass_complete(self, pass_num): """Check if pass_num is complete based on map data, returns bool.""" @@ -418,7 +454,6 @@ class State(): # Set working dir working_dir = get_working_dir(mode, self.destination) - os.chdir(working_dir) # Start fresh if requested if docopt_args['--start-fresh']: @@ -523,10 +558,10 @@ class State(): def safety_check(self, mode, working_dir): """Run safety check and abort if necessary.""" required_size = sum([pair.size for pair in self.block_pairs]) + settings = self.load_settings(working_dir) if mode == 'Clone' else {} # Increase required_size if necessary - if mode == 'Clone' and settings['Needs Format']: - settings = self.load_settings(working_dir) + if mode == 'Clone' and settings.get('Needs Format', False): if settings['Table Type'] == 'GPT': # Below is the size calculation for the GPT # 1 LBA for the protective MBR @@ -711,7 +746,14 @@ def build_block_pair_report(block_pairs, settings): ['BLUE', None], ), ) - # TODO If anything recovered --> Add resume msg + if any([pair.get_rescued_size() > 0 for pair in block_pairs]): + report.append(' ') + report.append( + std.color_string( + ['NOTE:', 'Resume data loaded from map file(s).'], + ['BLUE', None], + ), + ) # Remove double line-break if report[-1] == ' ': @@ -902,7 +944,7 @@ def fstype_is_ok(path, map_dir=False): # Get fstype if PLATFORM == 'Darwin': - # TODO: leave as None for now + # TODO: Determine fstype under macOS pass elif PLATFORM == 'Linux': cmd = [ @@ -1005,13 +1047,15 @@ def get_working_dir(mode, destination): working_dir = pathlib.Path(os.getcwd()) # Set subdir using ticket ID - working_dir = working_dir.joinpath(ticket_id) - LOG.info('Set working directory to: %s', working_dir) + if mode == 'Clone': + working_dir = working_dir.joinpath(ticket_id) # Create directory working_dir.mkdir(parents=True, exist_ok=True) + os.chdir(working_dir) # Done + LOG.info('Set working directory to: %s', working_dir) return working_dir From fa398015232c67b92d6f3bce38f7147a569ef306 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Dec 2019 20:18:17 -0700 Subject: [PATCH 269/324] Adjusted block pair report --- scripts/wk/hw/ddrescue.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 02ce3596..351c88bd 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -712,6 +712,7 @@ class State(): def build_block_pair_report(block_pairs, settings): """Build block pair report, returns list.""" report = [] + notes = [] if block_pairs: report.append(std.color_string('Block Pairs', 'GREEN')) else: @@ -730,9 +731,8 @@ def build_block_pair_report(block_pairs, settings): # Show resume messages as necessary if settings: - report.append(' ') if not settings['First Run']: - report.append( + notes.append( std.color_string( ['NOTE:', 'Clone settings loaded from previous run.'], ['BLUE', None], @@ -740,24 +740,24 @@ def build_block_pair_report(block_pairs, settings): ) if settings['Needs Format'] and settings['Table Type']: msg = f'Destination will be formatted using {settings["Table Type"]}' - report.append( + notes.append( std.color_string( ['NOTE:', msg], ['BLUE', None], ), ) if any([pair.get_rescued_size() > 0 for pair in block_pairs]): - report.append(' ') - report.append( + notes.append( std.color_string( ['NOTE:', 'Resume data loaded from map file(s).'], ['BLUE', None], ), ) - # Remove double line-break - if report[-1] == ' ': - report.pop(-1) + # Add notes to report + if notes: + report.append(' ') + report.extend(notes) # Done return report From 6ad68c37d48fb9edf73997fb56d8c6249e5f5044 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 29 Dec 2019 19:29:09 -0700 Subject: [PATCH 270/324] Added update_progress_pane() * Still a WIP --- scripts/wk/hw/ddrescue.py | 75 ++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 351c88bd..c93a50ae 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -438,13 +438,6 @@ class State(): self.destination = select_path('Destination') self.update_top_panes() - # Update panes - self.panes['Progress'] = tmux.split_window( - lines=cfg.ddrescue.TMUX_SIDE_WIDTH, - watch_file=f'{self.log_dir}/progress.out', - ) - self.update_progress_pane() - # Confirmation #1 self.confirm_selections( mode=mode, @@ -452,6 +445,13 @@ class State(): source_parts=source_parts, ) + # Update panes + self.panes['Progress'] = tmux.split_window( + lines=cfg.ddrescue.TMUX_SIDE_WIDTH, + watch_file=f'{self.log_dir}/progress.out', + ) + self.update_progress_pane('Idle') + # Set working dir working_dir = get_working_dir(mode, self.destination) @@ -476,6 +476,7 @@ class State(): # if cloning and not resuming format destination # Done + self.update_progress_pane('Idle') # Ready for main menu def init_tmux(self): @@ -632,12 +633,54 @@ class State(): std.print_error('Failed to save clone settings') raise std.GenericAbort() - def update_progress_pane(self): + def update_progress_pane(self, status): """Update progress pane.""" report = [] + separator = '─────────────────────' width = cfg.ddrescue.TMUX_SIDE_WIDTH + # TODO: Finish update_progress_pane() - #TODO + # Status + report.append(std.color_string(f'{"Status":^{width}}', 'BLUE')) + report.append(f'{status:^{width}}') + report.append(separator) + + # Overall progress + if self.block_pairs: + total_rescued = sum( + [pair.map_data.get('rescued', 0) for pair in self.block_pairs], + ) + total_rescued = 400 * 1024**2 + total_size = sum([pair.size for pair in self.block_pairs]) + percent = total_rescued / total_size + report.append(std.color_string('Overall Progress', 'BLUE')) + report.append( + std.color_string( + ['Rescued:', f'{100*total_rescued/total_size:>{width-11}.2f} %'], + [None, get_percent_color(percent)], + ), + ) + report.append( + std.color_string( + f'{std.bytes_to_string(total_rescued, decimals=2):>{width}}', + get_percent_color(percent), + ), + ) + report.append(separator) + + # Block pair progress + for pair in self.block_pairs: + report.append(std.color_string(pair.source, 'BLUE')) + report.append(f'Pass 1 {"TODO":>{width-7}}') + report.append(f'Pass 2 {"TODO":>{width-7}}') + report.append(f'Pass 3 {"TODO":>{width-7}}') + report.append(' ') + + # EToC + if status in ('In Progress', 'NEEDS ATTENTION'): + report.append(separator) + report.append(std.color_string('Estimated Pass Finish', 'BLUE')) + report.append('TODO') # Write to progress file out_path = pathlib.Path(f'{self.log_dir}/progress.out') @@ -1011,6 +1054,20 @@ def get_partition_separator(name): return separator +def get_percent_color(percent): + """Get color based on percentage, returns str.""" + color = 'RED' + if percent > 100: + color = 'PURPLE' + elif percent >= 99: + color = 'GREEN' + elif percent >= 90: + color = 'YELLOW' + + # Done + return color + + def get_working_dir(mode, destination): """Get working directory using mode and destination, returns path.""" ticket_id = None From 89de1d52bbd8dda6d7efbd40f1f3c6da496d3ee6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 16:37:49 -0700 Subject: [PATCH 271/324] Updated BlockPair __init__() and pass_complete() --- scripts/wk/hw/ddrescue.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index c93a50ae..163f7837 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -12,6 +12,7 @@ import re import shutil import time +from collections import OrderedDict from docopt import docopt import psutil @@ -96,6 +97,11 @@ class BlockPair(): self.map_data = {} self.map_path = None self.size = source.details['size'] + self.status = OrderedDict({ + 'read': 'Pending', + 'trim': 'Pending', + 'scrape': 'Pending', + }) # Set map file # e.g. '(Clone|Image)_Model[_p#]_Size[_Label].map' @@ -123,6 +129,17 @@ class BlockPair(): # Read map file self.load_map_data() + # Set initial status + percent = 100 * self.map_data.get('rescued', 0) / self.size + for name in self.status.keys(): + if self.pass_complete(name): + self.status[name] = percent + else: + # Stop checking + if percent > 0: + self.status[name] = percent + break + def get_rescued_size(self): """Get rescued size using map data. @@ -169,11 +186,26 @@ class BlockPair(): # Done self.map_data.update(data) - def pass_complete(self, pass_num): + def pass_complete(self, pass_name): """Check if pass_num is complete based on map data, returns bool.""" complete = False + pending_size = 0 - # TODO + # Check map data + if self.map_data.get('full recovery', False): + complete = True + elif 'non-tried' not in self.map_data: + # Assuming recovery has not been attempted yet + complete = False + else: + # Check that current and previous passes are complete + pending_size = self.map_data['non-tried'] + if pass_name in ('trim', 'scrape'): + pending_size += self.map_data['non-trimmed'] + if pass_name == 'scrape': + pending_size += self.map_data['non-scraped'] + if pending_size == 0: + complete = True # Done return complete From 631449e40ae79703cd98d8dc3f49efe92521ef0a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 16:40:03 -0700 Subject: [PATCH 272/324] Added format_status_string() --- scripts/wk/hw/ddrescue.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 163f7837..5c0e2de5 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1012,6 +1012,40 @@ def build_settings_menu(silent=True): return menu +def format_status_string(status, width): + """Format colored status string, returns str.""" + color = None + percent = -1 + status_str = str(status) + + # Check if status is percentage + try: + percent = float(status_str) + except ValueError: + # Assuming status is text + pass + + # Format status + if percent >= 0: + # Percentage + color = get_percent_color(percent) + status_str = f'{percent:{width-2}.2f} %' + if '100.00' in status_str and percent < 100: + # Always round down to 99.99% + status_str = f'{"99.99 %":>{width}}' + else: + # Text + color = STATUS_COLORS.get(status_str, None) + status_str = f'{status_str:>{width}}' + + # Add color if necessary + if color: + status_str = std.color_string(status_str, color) + + # Done + return status_str + + def fstype_is_ok(path, map_dir=False): """Check if filesystem type is acceptable, returns bool.""" is_ok = False @@ -1088,13 +1122,15 @@ def get_partition_separator(name): def get_percent_color(percent): """Get color based on percentage, returns str.""" - color = 'RED' + color = None if percent > 100: color = 'PURPLE' elif percent >= 99: color = 'GREEN' elif percent >= 90: color = 'YELLOW' + elif percent > 0: + color = 'RED' # Done return color From 0ddafe8a428412632b79f7e0f2ace956e78f20b2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 16:40:28 -0700 Subject: [PATCH 273/324] Updated side pane sections --- scripts/wk/hw/ddrescue.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 5c0e2de5..c7b349d0 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -502,13 +502,13 @@ class State(): self.safety_check(mode, working_dir) # Confirmation #2 + self.update_progress_pane('Idle') self.confirm_selections(mode, 'Start recovery?', working_dir=working_dir) # Prep destination # if cloning and not resuming format destination # Done - self.update_progress_pane('Idle') # Ready for main menu def init_tmux(self): @@ -665,16 +665,23 @@ class State(): std.print_error('Failed to save clone settings') raise std.GenericAbort() - def update_progress_pane(self, status): + def update_progress_pane(self, overall_status): """Update progress pane.""" report = [] separator = '─────────────────────' width = cfg.ddrescue.TMUX_SIDE_WIDTH - # TODO: Finish update_progress_pane() # Status report.append(std.color_string(f'{"Status":^{width}}', 'BLUE')) - report.append(f'{status:^{width}}') + if 'NEEDS ATTENTION' in overall_status: + report.append( + std.color_string( + f'{overall_status:^{width}}', + 'YELLOW_BLINK', + ), + ) + else: + report.append(f'{overall_status:^{width}}') report.append(separator) # Overall progress @@ -682,15 +689,11 @@ class State(): total_rescued = sum( [pair.map_data.get('rescued', 0) for pair in self.block_pairs], ) - total_rescued = 400 * 1024**2 total_size = sum([pair.size for pair in self.block_pairs]) - percent = total_rescued / total_size + percent = 100 * total_rescued / total_size report.append(std.color_string('Overall Progress', 'BLUE')) report.append( - std.color_string( - ['Rescued:', f'{100*total_rescued/total_size:>{width-11}.2f} %'], - [None, get_percent_color(percent)], - ), + f'Rescued: {format_status_string(percent, width=width-9)}', ) report.append( std.color_string( @@ -703,13 +706,16 @@ class State(): # Block pair progress for pair in self.block_pairs: report.append(std.color_string(pair.source, 'BLUE')) - report.append(f'Pass 1 {"TODO":>{width-7}}') - report.append(f'Pass 2 {"TODO":>{width-7}}') - report.append(f'Pass 3 {"TODO":>{width-7}}') + for name, status in pair.status.items(): + name = name.title() + report.append( + f'{name}{format_status_string(status, width=width-len(name))}', + ) report.append(' ') # EToC - if status in ('In Progress', 'NEEDS ATTENTION'): + # TODO: Finish update_progress_pane() [EToC] + if overall_status in ('Active', 'NEEDS ATTENTION'): report.append(separator) report.append(std.color_string('Estimated Pass Finish', 'BLUE')) report.append('TODO') From e7fbc2172107fd224fa9c2a1d1bf1da0daf849de Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 17:43:15 -0700 Subject: [PATCH 274/324] Added EToC logic --- scripts/wk/cfg/ddrescue.py | 16 ------------- scripts/wk/hw/ddrescue.py | 49 +++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index 6097d7b9..4a9c8e02 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -46,22 +46,6 @@ DDRESCUE_SETTINGS = { '--timeout': {'Selected': False, 'Value': '30m', }, }, } -ETOC_REFRESH_RATE = 30 # in seconds -REGEX_DDRESCUE_LOG = re.compile( - r'^\s*(?P\S+):\s+' - r'(?P\d+)\s+' - r'(?P[PTGMKB])i?B?', - re.IGNORECASE, - ) -REGEX_REMAINING_TIME = re.compile( - r'remaining time:' - r'\s*((?P\d+)d)?' - r'\s*((?P\d+)h)?' - r'\s*((?P\d+)m)?' - r'\s*((?P\d+)s)?' - r'\s*(?Pn/a)?', - re.IGNORECASE - ) if __name__ == '__main__': diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index c7b349d0..aace7514 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -3,11 +3,13 @@ # vim: sts=2 sw=2 ts=2 import atexit +import datetime import json import logging import os import pathlib import plistlib +import pytz import re import shutil import time @@ -51,6 +53,15 @@ DDRESCUE_LOG_REGEX = re.compile( r'(?P[PTGMKB]i?B?)', re.IGNORECASE, ) +REGEX_REMAINING_TIME = re.compile( + r'remaining time:' + r'\s*((?P\d+)d)?' + r'\s*((?P\d+)h)?' + r'\s*((?P\d+)m)?' + r'\s*((?P\d+)s)?' + r'\s*(?Pn/a)?', + re.IGNORECASE + ) LOG = logging.getLogger(__name__) MENU_ACTIONS = ( 'Start', @@ -81,6 +92,7 @@ STATUS_COLORS = { 'Working': 'YELLOW', 'ERROR': 'RED', } +TIMEZONE = pytz.timezone(cfg.main.LINUX_TIME_ZONE) # Classes @@ -434,6 +446,35 @@ class State(): self.fix_tmux_layout(forced=False) std.sleep(1) + def get_etoc(self): + """Get EToC from ddrescue output, returns str.""" + delta = None + delta_dict = {} + etoc = 'Unknown' + now = datetime.datetime.now(tz=TIMEZONE) + output = tmux.capture_pane() + + # Search for EToC delta + matches = re.findall(f'remaining time:.*$', output, re.MULTILINE) + if matches: + match = REGEX_REMAINING_TIME.search(matches[-1]) + if match.group('na'): + etoc = 'N/A' + else: + for key in ('days', 'hours', 'minutes', 'seconds'): + delta_dict[key] = match.group(key) + delta_dict = {k: int(v) if v else 0 for k, v in delta_dict.items()} + delta = datetime.timedelta(**delta_dict) + + # Calc EToC if delta found + if delta: + etoc_datetime = now + delta + etoc = etoc_datetime.strftime('%Y-%m-%d %H:%M %Z') + + # Done + return etoc + + def init_recovery(self, docopt_args): """Select source/dest and set env.""" std.clear_screen() @@ -716,9 +757,13 @@ class State(): # EToC # TODO: Finish update_progress_pane() [EToC] if overall_status in ('Active', 'NEEDS ATTENTION'): + etoc = self.get_etoc() report.append(separator) report.append(std.color_string('Estimated Pass Finish', 'BLUE')) - report.append('TODO') + if overall_status == 'NEEDS ATTENTION' or etoc == 'N/A': + report.append(std.color_string('N/A', 'YELLOW')) + else: + report.append(etoc) # Write to progress file out_path = pathlib.Path(f'{self.log_dir}/progress.out') @@ -1310,6 +1355,7 @@ def mount_raw_image_macos(path): def run_recovery(state, main_menu, settings_menu): """Run recovery passes.""" atexit.register(state.save_debug_reports) + state.update_progress_pane('Active') # Start SMART/Journal # TODO @@ -1324,6 +1370,7 @@ def run_recovery(state, main_menu, settings_menu): state.save_debug_reports() atexit.unregister(state.save_debug_reports) std.pause('Press Enter to return to main menu...') + state.update_progress_pane('Idle') def select_disk(prompt, skip_disk=None): From d9561a0159f0d70d1cc72fa1bbf27c32734f0ed5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 18:47:35 -0700 Subject: [PATCH 275/324] pylint cleanup --- scripts/wk/hw/ddrescue.py | 112 ++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 59 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index aace7514..b7da2cfa 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -9,7 +9,6 @@ import logging import os import pathlib import plistlib -import pytz import re import shutil import time @@ -18,10 +17,10 @@ from collections import OrderedDict from docopt import docopt import psutil +import pytz from wk import cfg, debug, exe, io, log, net, std, tmux from wk.hw import obj as hw_obj -from wk.hw import sensors as hw_sensors # STATIC VARIABLES @@ -248,9 +247,6 @@ class State(): working_dir=working_dir, )) - # Safety Checks - # TODO - def add_clone_block_pairs(self, working_dir): """Add device to device block pairs and set settings if necessary.""" source_sep = get_partition_separator(self.source.path.name) @@ -310,22 +306,6 @@ class State(): bp_dest = self.destination self.add_block_pair(part, bp_dest, working_dir) - def clean_working_dir(self, working_dir): - """Clean working directory to ensure a fresh recovery session. - - NOTE: Data from previous sessions will be preserved - in a backup directory. - """ - backup_dir = pathlib.Path(f'{working_dir}/prev') - backup_dir = io.non_clobber_path(backup_dir) - backup_dir.mkdir() - - # Move settings, maps, etc to backup_dir - for entry in os.scandir(working_dir): - if entry.name.endswith(('.dd', '.json', '.map')): - new_path = f'{backup_dir}/{entry.name}' - new_path = io.non_clobber_path(new_path) - shutil.move(entry.path, new_path) def confirm_selections( self, mode, prompt, working_dir=None, source_parts=None): @@ -446,35 +426,6 @@ class State(): self.fix_tmux_layout(forced=False) std.sleep(1) - def get_etoc(self): - """Get EToC from ddrescue output, returns str.""" - delta = None - delta_dict = {} - etoc = 'Unknown' - now = datetime.datetime.now(tz=TIMEZONE) - output = tmux.capture_pane() - - # Search for EToC delta - matches = re.findall(f'remaining time:.*$', output, re.MULTILINE) - if matches: - match = REGEX_REMAINING_TIME.search(matches[-1]) - if match.group('na'): - etoc = 'N/A' - else: - for key in ('days', 'hours', 'minutes', 'seconds'): - delta_dict[key] = match.group(key) - delta_dict = {k: int(v) if v else 0 for k, v in delta_dict.items()} - delta = datetime.timedelta(**delta_dict) - - # Calc EToC if delta found - if delta: - etoc_datetime = now + delta - etoc = etoc_datetime.strftime('%Y-%m-%d %H:%M %Z') - - # Done - return etoc - - def init_recovery(self, docopt_args): """Select source/dest and set env.""" std.clear_screen() @@ -530,7 +481,7 @@ class State(): # Start fresh if requested if docopt_args['--start-fresh']: - self.clean_working_dir(working_dir) + clean_working_dir(working_dir) # Add block pairs if mode == 'Clone': @@ -546,7 +497,7 @@ class State(): self.update_progress_pane('Idle') self.confirm_selections(mode, 'Start recovery?', working_dir=working_dir) - # Prep destination + # TODO: Prep destination # if cloning and not resuming format destination # Done @@ -716,11 +667,8 @@ class State(): report.append(std.color_string(f'{"Status":^{width}}', 'BLUE')) if 'NEEDS ATTENTION' in overall_status: report.append( - std.color_string( - f'{overall_status:^{width}}', - 'YELLOW_BLINK', - ), - ) + std.color_string(f'{overall_status:^{width}}', 'YELLOW_BLINK'), + ) else: report.append(f'{overall_status:^{width}}') report.append(separator) @@ -755,9 +703,8 @@ class State(): report.append(' ') # EToC - # TODO: Finish update_progress_pane() [EToC] if overall_status in ('Active', 'NEEDS ATTENTION'): - etoc = self.get_etoc() + etoc = get_etoc() report.append(separator) report.append(std.color_string('Estimated Pass Finish', 'BLUE')) if overall_status == 'NEEDS ATTENTION' or etoc == 'N/A': @@ -1063,6 +1010,24 @@ def build_settings_menu(silent=True): return menu +def clean_working_dir(working_dir): + """Clean working directory to ensure a fresh recovery session. + + NOTE: Data from previous sessions will be preserved + in a backup directory. + """ + backup_dir = pathlib.Path(f'{working_dir}/prev') + backup_dir = io.non_clobber_path(backup_dir) + backup_dir.mkdir() + + # Move settings, maps, etc to backup_dir + for entry in os.scandir(working_dir): + if entry.name.endswith(('.dd', '.json', '.map')): + new_path = f'{backup_dir}/{entry.name}' + new_path = io.non_clobber_path(new_path) + shutil.move(entry.path, new_path) + + def format_status_string(status, width): """Format colored status string, returns str.""" color = None @@ -1127,6 +1092,35 @@ def fstype_is_ok(path, map_dir=False): return is_ok +def get_etoc(): + """Get EToC from ddrescue output, returns str.""" + delta = None + delta_dict = {} + etoc = 'Unknown' + now = datetime.datetime.now(tz=TIMEZONE) + output = tmux.capture_pane() + + # Search for EToC delta + matches = re.findall(f'remaining time:.*$', output, re.MULTILINE) + if matches: + match = REGEX_REMAINING_TIME.search(matches[-1]) + if match.group('na'): + etoc = 'N/A' + else: + for key in ('days', 'hours', 'minutes', 'seconds'): + delta_dict[key] = match.group(key) + delta_dict = {k: int(v) if v else 0 for k, v in delta_dict.items()} + delta = datetime.timedelta(**delta_dict) + + # Calc EToC if delta found + if delta: + etoc_datetime = now + delta + etoc = etoc_datetime.strftime('%Y-%m-%d %H:%M %Z') + + # Done + return etoc + + def get_object(path): """Get object based on path, returns obj.""" obj = None From bcd46d4017c3e3296edde8a66b83eae6a1b8c56d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 19:25:46 -0700 Subject: [PATCH 276/324] Added SMART/Journal panes --- scripts/wk/cfg/ddrescue.py | 2 -- scripts/wk/hw/ddrescue.py | 34 ++++++++++++++++++++++------------ scripts/wk/hw/obj.py | 10 ++++++---- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index 4a9c8e02..e617bae5 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -2,8 +2,6 @@ # pylint: disable=bad-whitespace,line-too-long # vim: sts=2 sw=2 ts=2 -import re - from collections import OrderedDict diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index b7da2cfa..c85ad733 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -405,17 +405,19 @@ class State(): if forced or needs_fixed: self.update_top_panes() - # SMART/Journal + # Return if Progress pane not present if 'Progress' not in self.panes: - # Assumning we're still selecting source/dest return - height = tmux.get_pane_size(self.panes['Progress'])[1] - 2 - p_ratios = [int((x/sum(PANE_RATIOS)) * height) for x in PANE_RATIOS] - if 'SMART' in self.panes: - tmux.resize_pane(self.panes['SMART'], height=p_ratios[0]) - tmux.resize_pane(height=p_ratios[1]) - if 'Journal' in self.panes: - tmux.resize_pane(self.panes['Journal'], height=p_ratios[2]) + + # SMART/Journal + if forced or needs_fixed: + height = tmux.get_pane_size(self.panes['Progress'])[1] - 2 + p_ratios = [int((x/sum(PANE_RATIOS)) * height) for x in PANE_RATIOS] + if 'SMART' in self.panes: + tmux.resize_pane(self.panes['SMART'], height=p_ratios[0]) + tmux.resize_pane(height=p_ratios[1]) + if 'Journal' in self.panes: + tmux.resize_pane(self.panes['Journal'], height=p_ratios[2]) def fix_tmux_layout_loop(self): """Fix tmux layout on a loop. @@ -1349,16 +1351,24 @@ def mount_raw_image_macos(path): def run_recovery(state, main_menu, settings_menu): """Run recovery passes.""" atexit.register(state.save_debug_reports) - state.update_progress_pane('Active') # Start SMART/Journal - # TODO + state.panes['SMART'] = tmux.split_window( + behind=True, lines=12, vertical=True, + watch_file=f'{state.log_dir}/smart.out', + ) + state.panes['Journal'] = tmux.split_window( + lines=4, vertical=True, cmd='journalctl --dmesg --follow', + ) # TODO # Run ddrescue + state.update_progress_pane('Active') # Stop SMART/Journal - # TODO + for pane in ('SMART', 'Journal'): + if pane in state.panes: + tmux.kill_pane(state.panes.pop(pane)) # Done state.save_debug_reports() diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index f473afaf..490d96f7 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -275,15 +275,17 @@ class Disk(BaseObj): # Done return report - def generate_report(self): + def generate_report(self, header=True): """Generate Disk report, returns list.""" report = [] - report.append(color_string(f'Device ({self.path.name})', 'BLUE')) - report.append(f' {self.description}') + if header: + report.append(color_string(f'Device ({self.path.name})', 'BLUE')) + report.append(f' {self.description}') # Attributes if self.attributes: - report.append(color_string('Attributes', 'BLUE')) + if header: + report.append(color_string('Attributes', 'BLUE')) report.extend(self.generate_attribute_report()) # Notes From e88e4ab3ebf644e2f8546a40d42561d2b2f5ef1c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 19:29:32 -0700 Subject: [PATCH 277/324] Added ddrescue settings sections --- scripts/wk/cfg/ddrescue.py | 1 + scripts/wk/hw/ddrescue.py | 41 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index e617bae5..500f0056 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -27,6 +27,7 @@ DDRESCUE_SETTINGS = { '--min-read-rate': {'Selected': True, 'Value': '64KiB', }, '--reopen-on-error': {'Selected': True, }, '--retry-passes': {'Selected': True, 'Value': '0', }, + '--reverse': {'Selected': False, }, '--test-mode': {'Selected': False, 'Value': 'test.map', }, '--timeout': {'Selected': True, 'Value': '30m', }, '-vvvv': {'Selected': True, 'Hidden': True, }, diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index c85ad733..cf1220a4 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -69,7 +69,6 @@ MENU_ACTIONS = ( MENU_TOGGLES = { 'Auto continue (if recovery % over threshold)': True, 'Retry (mark non-rescued sectors "non-tried")': False, - 'Reverse direction': False, } PANE_RATIOS = ( 12, # SMART @@ -998,8 +997,8 @@ def build_settings_menu(silent=True): preset = _p # Add default settings - menu.add_action('Main Menu') menu.add_action('Load Preset') + menu.add_action('Main Menu') for name, details in cfg.ddrescue.DDRESCUE_SETTINGS['Default'].items(): menu.add_option(name, details.copy()) @@ -1094,6 +1093,26 @@ def fstype_is_ok(path, map_dir=False): return is_ok +def get_ddrescue_settings(main_menu, settings_menu): + """Get ddrescue settings from menu selections, returns list.""" + settings = [] + + # Check menu selections + for name, details in main_menu.toggles.items(): + if 'Retry' in name and details['Selected']: + settings.append('--retrim') + settings.append('--try-again') + for name, details in settings_menu.options.items(): + if details['Selected']: + if 'Value' in details: + settings.append(f'{name}={details["Value"]}') + else: + settings.append(name) + + # Done + return settings + + def get_etoc(): """Get EToC from ddrescue output, returns str.""" delta = None @@ -1351,6 +1370,14 @@ def mount_raw_image_macos(path): def run_recovery(state, main_menu, settings_menu): """Run recovery passes.""" atexit.register(state.save_debug_reports) + attempted_recovery = False + auto_continue = False + + # Get settings + for name, details in main_menu.toggles.items(): + if 'Auto continue' in name and details['Selected']: + auto_continue = True + settings = get_ddrescue_settings(main_menu, settings_menu) # Start SMART/Journal state.panes['SMART'] = tmux.split_window( @@ -1361,9 +1388,19 @@ def run_recovery(state, main_menu, settings_menu): lines=4, vertical=True, cmd='journalctl --dmesg --follow', ) + # Check if retrying + if '--retrim' in settings: + for pair in state.block_pairs: + for name in pair.status.keys(): + pair.status[name] = 'Pending' + # TODO # Run ddrescue state.update_progress_pane('Active') + print('ddrescue settings:') + for arg in settings: + print(f' {arg}') + std.pause('Run ddrescue pass?') # Stop SMART/Journal for pane in ('SMART', 'Journal'): From df6f3ba8e1fbcd1e99516f5a590cf40f9e310d20 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 30 Dec 2019 20:21:37 -0700 Subject: [PATCH 278/324] Added initial ddrescue pass logic --- scripts/wk/cfg/ddrescue.py | 8 +++-- scripts/wk/hw/ddrescue.py | 72 +++++++++++++++++++++++++++++++------- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index 500f0056..68f8ae43 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -14,8 +14,12 @@ TMUX_LAYOUT = OrderedDict({ }) # ddrescue -AUTO_PASS_1_THRESHOLD = 95 -AUTO_PASS_2_THRESHOLD = 98 +AUTO_PASS_THRESHOLDS = { + # NOTE: The scrape key is set to infinity to force a break + 'read': 95, + 'trim': 98, + 'scrape': float('inf'), + } DDRESCUE_SETTINGS = { 'Default': { '--binary-prefixes': {'Selected': True, 'Hidden': True, }, diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index cf1220a4..fe78e3ff 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -140,7 +140,7 @@ class BlockPair(): self.load_map_data() # Set initial status - percent = 100 * self.map_data.get('rescued', 0) / self.size + percent = self.get_percent_recovered() for name in self.status.keys(): if self.pass_complete(name): self.status[name] = percent @@ -150,6 +150,10 @@ class BlockPair(): self.status[name] = percent break + def get_percent_recovered(self): + """Get percent rescued from map_data, returns float.""" + return 100 * self.map_data.get('rescued', 0) / self.size + def get_rescued_size(self): """Get rescued size using map data. @@ -581,6 +585,23 @@ class State(): # Done return settings + def pass_above_threshold(self, pass_name): + """Check if all block_pairs meet the pass threshold, returns bool.""" + threshold = cfg.ddrescue.AUTO_PASS_THRESHOLDS[pass_name] + return all( + [p.get_percent_recovered() >= threshold for p in self.block_pairs], + ) + + def pass_complete(self, pass_name): + """Check if all block_pairs completed pass_name, returns bool.""" + return all([p.pass_complete(pass_name) for p in self.block_pairs]) + + def retry_all_passes(self): + """Set all statuses to Pending.""" + for pair in self.block_pairs: + for name in pair.status.keys(): + pair.status[name] = 'Pending' + def safety_check(self, mode, working_dir): """Run safety check and abort if necessary.""" required_size = sum([pair.size for pair in self.block_pairs]) @@ -687,8 +708,8 @@ class State(): ) report.append( std.color_string( - f'{std.bytes_to_string(total_rescued, decimals=2):>{width}}', - get_percent_color(percent), + [f'{std.bytes_to_string(total_rescued, decimals=2):>{width}}'], + [get_percent_color(percent)], ), ) report.append(separator) @@ -1367,6 +1388,17 @@ def mount_raw_image_macos(path): return loopback_path +def run_ddrescue(state, block_pair, settings): + """Run ddrescue using passed settings.""" + state.update_progress_pane('Active') + print('Running ddrescue') + print(f' {block_pair.source} --> {block_pair.destination}') + print('Using these settings:') + for _s in settings: + print(f' {_s}') + std.pause() + + def run_recovery(state, main_menu, settings_menu): """Run recovery passes.""" atexit.register(state.save_debug_reports) @@ -1390,17 +1422,31 @@ def run_recovery(state, main_menu, settings_menu): # Check if retrying if '--retrim' in settings: - for pair in state.block_pairs: - for name in pair.status.keys(): - pair.status[name] = 'Pending' + state.retry_all_passes() - # TODO - # Run ddrescue - state.update_progress_pane('Active') - print('ddrescue settings:') - for arg in settings: - print(f' {arg}') - std.pause('Run ddrescue pass?') + # Run pass(es) + for pass_name in ('read', 'trim', 'scrape'): + if state.pass_complete(pass_name): + # Skip to next pass + # NOTE: This bypasses auto_continue + continue + + # Run ddrescue + for pair in state.block_pairs: + if not pair.pass_complete(pass_name): + attempted_recovery = True + run_ddrescue(state, pair, settings) + + # Continue or return to menu + all_complete = state.pass_complete(pass_name) + all_above_threshold = state.pass_above_threshold(pass_name) + if not (all_complete and all_above_threshold and auto_continue): + break + + # Show warning if nothing was done + if not attempted_recovery: + std.print_warning('No actions performed') + std.print_standard(' ') # Stop SMART/Journal for pane in ('SMART', 'Journal'): From f45a10395faa699466b23a44793c7bf8363eca35 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Jan 2020 15:02:11 -0700 Subject: [PATCH 279/324] Added --force-local-map option --- scripts/wk/hw/ddrescue.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index fe78e3ff..99b8f075 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -33,6 +33,8 @@ Usage: Options: -h --help Show this page + -s --dry-run Print commands to be used instead of running them + --force-local-map Skip mounting shares and save map to current dir --start-fresh Ignore previous runs and start new recovery ''' CLONE_SETTINGS = { @@ -482,7 +484,9 @@ class State(): self.update_progress_pane('Idle') # Set working dir - working_dir = get_working_dir(mode, self.destination) + working_dir = get_working_dir( + mode, self.destination, force_local=docopt_args['--force-local-map'], + ) # Start fresh if requested if docopt_args['--start-fresh']: @@ -1223,7 +1227,7 @@ def get_percent_color(percent): return color -def get_working_dir(mode, destination): +def get_working_dir(mode, destination, force_local=False): """Get working directory using mode and destination, returns path.""" ticket_id = None working_dir = None @@ -1239,7 +1243,11 @@ def get_working_dir(mode, destination): ticket_id = None # Use preferred path if possible - if mode == 'Clone': + if mode == 'Image': + path = pathlib.Path(destination).resolve() + if path.exists() and fstype_is_ok(path, map_dir=False): + working_dir = path + elif mode == 'Clone' and not force_local: std.print_info('Mounting backup shares...') net.mount_backup_shares(read_write=True) for server in cfg.net.BACKUP_SERVERS: @@ -1248,10 +1256,6 @@ def get_working_dir(mode, destination): # Acceptable path found working_dir = path break - else: - path = pathlib.Path(destination).resolve() - if path.exists() and fstype_is_ok(path, map_dir=False): - working_dir = path # Default to current dir if necessary if not working_dir: @@ -1274,9 +1278,10 @@ def get_working_dir(mode, destination): def main(): """Main function for ddrescue TUI.""" args = docopt(DOCSTRING) + args['--dry-run'] = True # TODO: Remove dry-run safety net log.update_log_path(dest_name='ddrescue-TUI', timestamp=True) - # Safety check + # Check if running inside tmux if 'TMUX' not in os.environ: LOG.error('tmux session not found') raise RuntimeError('tmux session not found') From 1dacdd46375204b93855098a7357272e81beb5d0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Jan 2020 15:02:23 -0700 Subject: [PATCH 280/324] Abort if an invalid image destination selected --- scripts/wk/hw/ddrescue.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 99b8f075..3780714a 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1244,7 +1244,11 @@ def get_working_dir(mode, destination, force_local=False): # Use preferred path if possible if mode == 'Image': - path = pathlib.Path(destination).resolve() + try: + path = pathlib.Path(destination).resolve() + except TypeError: + std.print_error(f'Invalid destination: {destination}') + raise std.GenericAbort() if path.exists() and fstype_is_ok(path, map_dir=False): working_dir = path elif mode == 'Clone' and not force_local: From 764d35836bc9a9d2c9371a2a8c653b7fb485ac4a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Jan 2020 15:06:16 -0700 Subject: [PATCH 281/324] Force running all passes if retry selected --- scripts/wk/hw/ddrescue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 3780714a..f06d1fb8 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1435,8 +1435,8 @@ def run_recovery(state, main_menu, settings_menu): # Run pass(es) for pass_name in ('read', 'trim', 'scrape'): - if state.pass_complete(pass_name): - # Skip to next pass + if not '--retrim' in settings and state.pass_complete(pass_name): + # Skip to next pass (unless retry selected) # NOTE: This bypasses auto_continue continue From 6dc887b04eea8b7fa6ae275daed986f6031e3926 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 15:50:26 -0700 Subject: [PATCH 282/324] Added initial disk formatting sections --- scripts/wk/hw/ddrescue.py | 101 ++++++++++++++++++++++++++++++++++++-- scripts/wk/hw/obj.py | 2 + 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index f06d1fb8..7e985a27 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -6,6 +6,7 @@ import atexit import datetime import json import logging +import math import os import pathlib import plistlib @@ -257,6 +258,7 @@ class State(): source_sep = get_partition_separator(self.source.path.name) dest_sep = get_partition_separator(self.destination.path.name) settings = {} + source_parts = [] # Clone settings settings = self.load_settings(working_dir, discard_unused_settings=True) @@ -305,6 +307,9 @@ class State(): # Save settings self.save_settings(settings, working_dir) + # Done + return source_parts + def add_image_block_pairs(self, source_parts, working_dir): """Add device to image file block pairs.""" for part in source_parts: @@ -494,7 +499,7 @@ class State(): # Add block pairs if mode == 'Clone': - self.add_clone_block_pairs(working_dir) + source_parts = self.add_clone_block_pairs(working_dir) else: source_parts = select_disk_parts(mode, self.source) self.add_image_block_pairs(source_parts, working_dir) @@ -506,8 +511,11 @@ class State(): self.update_progress_pane('Idle') self.confirm_selections(mode, 'Start recovery?', working_dir=working_dir) - # TODO: Prep destination - # if cloning and not resuming format destination + # Prep destination + if mode == 'Clone': + self.prep_destination( + source_parts, working_dir, dry_run=docopt_args['--dry-run'], + ) # Done # Ready for main menu @@ -600,6 +608,91 @@ class State(): """Check if all block_pairs completed pass_name, returns bool.""" return all([p.pass_complete(pass_name) for p in self.block_pairs]) + def prep_destination(self, source_parts, working_dir, dry_run=False): + """Prep destination as necessary.""" + dest_prefix = str(self.destination.path) + dest_prefix += get_partition_separator(self.destination.path.name) + esp_type = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B' + msr_type = 'E3C9E316-0B5C-4DB8-817D-F92DF00215AE' + part_num = 0 + sfdisk_script = [] + settings = self.load_settings(working_dir) + + # Bail early + if not settings['Needs Format']: + return + print(f'{source_parts=}') + print(f'{working_dir=}') + print(f'{dry_run=}') + print(settings) + std.pause() + + # Add partition table settings + if settings['Table Type'] == 'GPT': + sfdisk_script.append('label: gpt') + else: + sfdisk_script.append('label: dos') + sfdisk_script.append('unit: sectors') + sfdisk_script.append('') + + # Add boot partition if requested + if settings['Create Boot Partition']: + if settings['Table Type'] == 'GPT': + part_num += 1 + sfdisk_script.append( + f'{dest_prefix}{part_num} : ' + f'size=384MiB, type={esp_type}, name="EFI System"', + ) + part_num += 1 + sfdisk_script.append( + f'{dest_prefix}{part_num} : ' + f'size=16MiB, type={msr_type}, name="Microsoft Reserved"', + ) + elif settings['Table Type'] == 'MBR': + part_num += 1 + sfdisk_script.append( + f'{dest_prefix}{part_num} : ' + f'size=100MiB, type=7, name="System Reserved"', + ) + + # Add selected partition(s) + for part in source_parts: + line = '' + num_sectors = part.details['size'] / self.destination.details['log-sec'] + num_sectors = math.ceil(num_sectors) + part_num += 1 + + # Build sfdisk line for part + # TODO: Move to separate function to support both DOS/GPT types + line = f'{dest_prefix}{part_num} : ' + line += f'size={num_sectors}, type={part.details["parttype"].upper()}' + if part.details['partlabel']: + line += f', name="{part.details["partlabel"]}"' + if part.details['partuuid']: + line += f', uuid={part.details["partuuid"].upper()}' + + # Add line to script + sfdisk_script.append(line) + + # Save sfdisk script + script_path = f'{working_dir}/sfdisk_{self.destination.path.name}.script' + with open(script_path, 'w') as _f: + _f.write('\n'.join(sfdisk_script)) + + # Format disk + if dry_run: + std.print_warning('Not formatting disk during dry run') + std.print_info('Script for sfdisk:') + std.print_report(sfdisk_script) + std.pause() + else: + # TODO + pass + + # Update settings + settings['Needs Format'] = not dry_run + self.save_settings(settings, working_dir) + def retry_all_passes(self): """Set all statuses to Pending.""" for pair in self.block_pairs: @@ -1435,7 +1528,7 @@ def run_recovery(state, main_menu, settings_menu): # Run pass(es) for pass_name in ('read', 'trim', 'scrape'): - if not '--retrim' in settings and state.pass_complete(pass_name): + if '--retrim' not in settings and state.pass_complete(pass_name): # Skip to next pass (unless retry selected) # NOTE: This bypasses auto_continue continue diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 490d96f7..1a7421b6 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -1,5 +1,6 @@ """WizardKit: Hardware objects (mostly)""" # vim: sts=2 sw=2 ts=2 +# TODO: Get log-sec data under Linux and macOS import logging import pathlib @@ -315,6 +316,7 @@ class Disk(BaseObj): self.details['bus'] = str(self.details.get('bus', '???')).upper() self.details['bus'] = self.details['bus'].replace('IMAGE', 'Image') self.details['bus'] = self.details['bus'].replace('NVME', 'NVMe') + self.details['log-sec'] = self.details.get('log-sec', 512) self.details['model'] = self.details.get('model', 'Unknown Model') self.details['name'] = self.details.get('name', self.path) self.details['phy-sec'] = self.details.get('phy-sec', 512) From 9702d7665fd0e8ba01befe70919ac7e82a5dd3c0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 19:54:18 -0700 Subject: [PATCH 283/324] Added limited support for converting MBR/GPT types --- scripts/wk/cfg/ddrescue.py | 12 +++++ scripts/wk/hw/ddrescue.py | 91 ++++++++++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 24 deletions(-) diff --git a/scripts/wk/cfg/ddrescue.py b/scripts/wk/cfg/ddrescue.py index 68f8ae43..c0d9f6dc 100644 --- a/scripts/wk/cfg/ddrescue.py +++ b/scripts/wk/cfg/ddrescue.py @@ -49,6 +49,18 @@ DDRESCUE_SETTINGS = { '--timeout': {'Selected': False, 'Value': '30m', }, }, } +PARTITION_TYPES = { + 'GPT': { + 'NTFS': 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7', # Basic Data Partition + 'VFAT': 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7', # Basic Data Partition + 'EXFAT': 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7', # Basic Data Partition + }, + 'MBR': { + 'EXFAT': 7, # 0x7 + 'NTFS': 7, # 0x7 + 'VFAT': 11, # 0xb + }, + } if __name__ == '__main__': diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 7e985a27..67892cf3 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -621,11 +621,6 @@ class State(): # Bail early if not settings['Needs Format']: return - print(f'{source_parts=}') - print(f'{working_dir=}') - print(f'{dry_run=}') - print(settings) - std.pause() # Add partition table settings if settings['Table Type'] == 'GPT': @@ -640,39 +635,46 @@ class State(): if settings['Table Type'] == 'GPT': part_num += 1 sfdisk_script.append( - f'{dest_prefix}{part_num} : ' - f'size=384MiB, type={esp_type}, name="EFI System"', + build_sfdisk_partition_line( + table_type='GPT', + dev_path=f'{dest_prefix}{part_num}', + size='384MiB', + details={'parttype': esp_type, 'partlabel': 'EFI System'}, + ), ) part_num += 1 sfdisk_script.append( - f'{dest_prefix}{part_num} : ' - f'size=16MiB, type={msr_type}, name="Microsoft Reserved"', + build_sfdisk_partition_line( + table_type=settings['Table Type'], + dev_path=f'{dest_prefix}{part_num}', + size='16MiB', + details={'parttype': msr_type, 'partlabel': 'Microsoft Reserved'}, + ), ) elif settings['Table Type'] == 'MBR': part_num += 1 sfdisk_script.append( - f'{dest_prefix}{part_num} : ' - f'size=100MiB, type=7, name="System Reserved"', + build_sfdisk_partition_line( + table_type='MBR', + dev_path=f'{dest_prefix}{part_num}', + size='100MiB', + details={'parttype': '0x7', 'partlabel': 'System Reserved'}, + ), ) # Add selected partition(s) for part in source_parts: - line = '' num_sectors = part.details['size'] / self.destination.details['log-sec'] num_sectors = math.ceil(num_sectors) part_num += 1 - - # Build sfdisk line for part - # TODO: Move to separate function to support both DOS/GPT types - line = f'{dest_prefix}{part_num} : ' - line += f'size={num_sectors}, type={part.details["parttype"].upper()}' - if part.details['partlabel']: - line += f', name="{part.details["partlabel"]}"' - if part.details['partuuid']: - line += f', uuid={part.details["partuuid"].upper()}' - - # Add line to script - sfdisk_script.append(line) + sfdisk_script.append( + build_sfdisk_partition_line( + table_type=settings['Table Type'], + dev_path=f'{dest_prefix}{part_num}', + size=num_sectors, + details=part.details, + ), + ) # Save sfdisk script script_path = f'{working_dir}/sfdisk_{self.destination.path.name}.script' @@ -1129,6 +1131,47 @@ def build_settings_menu(silent=True): return menu +def build_sfdisk_partition_line(table_type, dev_path, size, details): + """Build sfdisk partition line using passed details, returns str.""" + line = f'{dev_path} : size={size}' + dest_type = '' + source_filesystem = str(details.get('fstype', '')).upper() + source_table_type = '' + source_type = details.get('parttype', '') + + # Set dest type + if re.match(r'^0x\w+$', source_type): + # Both source and dest are MBR + source_table_type = 'MBR' + if table_type == 'MBR': + dest_type = source_type.replace('0x', '').lower() + elif re.match(r'^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}$', source_type): + # Source is a GPT type + source_table_type = 'GPT' + if table_type == 'GPT': + dest_type = source_type.upper() + if not dest_type: + # Assuming changing table types, set based on FS + if source_filesystem in cfg.ddrescue.PARTITION_TYPES[table_type]: + dest_type = cfg.ddrescue.PARTITION_TYPES[table_type][source_filesystem] + line += f', type={dest_type}' + + # Safety Check + if not dest_type: + std.print_error(f'Failed to determine partition type for: {dev_path}') + raise std.GenericAbort() + + # Add extra details + if details.get('partlabel', ''): + line += f', name="{details["partlabel"]}"' + if details.get('partuuid', '') and source_table_type == table_type: + # Only add UUID if source/dest table types match + line += f', uuid={details["partuuid"].upper()}' + + # Done + return line + + def clean_working_dir(working_dir): """Clean working directory to ensure a fresh recovery session. From 5d0ed475a6d15e80b46aba43c3a6fd4928c699ec Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 20:07:46 -0700 Subject: [PATCH 284/324] Added option to match source partition table type --- scripts/wk/hw/ddrescue.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 67892cf3..0204ae55 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -285,12 +285,19 @@ class State(): settings['Needs Format'] = True offset = 0 if std.ask('Create an empty Windows boot partition on the clone?'): - offset = 2 settings['Create Boot Partition'] = True - settings['Table Type'] = 'GPT' - if std.choice(['G', 'M'], 'GPT or MBR partition table?') == 'M': - offset = 1 + user_choice = std.choice( + ['G', 'M', 'S'], + 'Use GPT, MBR, or match Source type?', + ) + if user_choice == 'G': + settings['Table Type'] = 'GPT' + elif user_choice == 'M': settings['Table Type'] = 'MBR' + else: + # Match source type + settings['Table Type'] = get_table_type(self.source) + offset = 2 if settings['Table Type'] == 'GPT' else 1 # Add pairs for dest_num, part in enumerate(source_parts): @@ -1363,6 +1370,24 @@ def get_percent_color(percent): return color +def get_table_type(disk): + """Get disk partition table type, returns str. + + NOTE: If resulting table type is not GPT or MBR + then an exception is raised. + """ + table_type = str(disk.details.get('pttype', '')).upper() + table_type = table_type.replace('DOS', 'MBR') + + # Check type + if table_type not in ('GPT', 'MBR'): + std.print_error(f'Unsupported partition table type: {table_type}') + raise std.GenericAbort() + + # Done + return table_type + + def get_working_dir(mode, destination, force_local=False): """Get working directory using mode and destination, returns path.""" ticket_id = None From ac04a3ddc5647e855f888f645b1cf1d6af7bc85e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 21:14:25 -0700 Subject: [PATCH 285/324] Added another safety check for block pairs * Needed one more check since clone pairs were assumed to be okay --- scripts/wk/hw/ddrescue.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 0204ae55..a80d158b 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -227,6 +227,19 @@ class BlockPair(): # Done return complete + def safety_check(self, dry_run=False): + """Run safety check and abort if necessary.""" + dest_size = -1 + if self.destination.exists(): + dest_size = self.destination.stat().st_size + + # Raise exception if necessary + if dry_run: + std.print_warning(f'Assuming destination is okay ({self.destination})') + elif dest_size < self.size: + std.print_error('Invalid destination: {self.destination}') + raise std.GenericAbort() + class State(): """Object for tracking hardware diagnostic data.""" @@ -323,7 +336,6 @@ class State(): bp_dest = self.destination self.add_block_pair(part, bp_dest, working_dir) - def confirm_selections( self, mode, prompt, working_dir=None, source_parts=None): """Show selection details and prompt for confirmation.""" @@ -524,8 +536,9 @@ class State(): source_parts, working_dir, dry_run=docopt_args['--dry-run'], ) - # Done - # Ready for main menu + # Safety Check #2 + for pair in self.block_pairs: + pair.safety_check(dry_run=docopt_args['--dry-run']) def init_tmux(self): """Initialize tmux layout.""" @@ -689,12 +702,7 @@ class State(): _f.write('\n'.join(sfdisk_script)) # Format disk - if dry_run: - std.print_warning('Not formatting disk during dry run') - std.print_info('Script for sfdisk:') - std.print_report(sfdisk_script) - std.pause() - else: + if not dry_run: # TODO pass From 9ae88102828c9ab72a8ea5968f785c241b51afd3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 21:22:47 -0700 Subject: [PATCH 286/324] Added real disk format section * --dry-run=True safety wheels still engaged --- scripts/wk/exe.py | 2 +- scripts/wk/hw/ddrescue.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/wk/exe.py b/scripts/wk/exe.py index b1d1927c..fe2f1b0d 100644 --- a/scripts/wk/exe.py +++ b/scripts/wk/exe.py @@ -78,7 +78,7 @@ def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs): } # Add additional kwargs if applicable - for key in ('check', 'cwd', 'encoding', 'errors', 'stderr', 'stdout'): + for key in 'check cwd encoding errors stderr stdin stdout'.split(): if key in kwargs: cmd_kwargs[key] = kwargs[key] diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index a80d158b..c84eafb7 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -701,10 +701,20 @@ class State(): with open(script_path, 'w') as _f: _f.write('\n'.join(sfdisk_script)) + # Skip real format for dry runs + if dry_run: + return + # Format disk - if not dry_run: - # TODO - pass + with open(script_path, 'r') as _f: + proc = exe.run_program( + cmd=['sudo', 'sfdisk', self.destination.path], + stdin=_f, + check=False, + ) + if proc.returncode != 0: + std.print_error('Error(s) encoundtered while formatting destination') + raise std.GenericAbort() # Update settings settings['Needs Format'] = not dry_run From 48eb4c13d7b466eb618afe37ab911533855a0e92 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 21:57:40 -0700 Subject: [PATCH 287/324] Better handle non-iterables in color_string() --- scripts/wk/std.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index ab72314a..37df3e21 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -737,6 +737,13 @@ def color_string(strings, colors, sep=' '): if isinstance(colors, (str, pathlib.Path)): colors = (colors,) + # Convert to strings if necessary + try: + iterator = iter(strings) + except TypeError: + # Assuming single element passed, convert to string + strings = (str(strings),) + # Build new string with color escapes added for string, color in itertools.zip_longest(strings, colors): color_code = COLORS.get(color, clear_code) From 299b075eef71a28ad382cf8043690e9ba2354e4b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 22:32:18 -0700 Subject: [PATCH 288/324] Fixed BlockPair().safety_check() --- scripts/wk/hw/ddrescue.py | 19 ++++++++++--------- scripts/wk/std.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index c84eafb7..68fc7f12 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -227,17 +227,17 @@ class BlockPair(): # Done return complete - def safety_check(self, dry_run=False): + def safety_check(self): """Run safety check and abort if necessary.""" dest_size = -1 if self.destination.exists(): - dest_size = self.destination.stat().st_size + dest_obj = hw_obj.Disk(self.destination) + dest_size = dest_obj.details['size'] + del dest_obj # Raise exception if necessary - if dry_run: - std.print_warning(f'Assuming destination is okay ({self.destination})') - elif dest_size < self.size: - std.print_error('Invalid destination: {self.destination}') + if dest_size < self.size: + std.print_error(f'Invalid destination: {self.destination}') raise std.GenericAbort() @@ -537,8 +537,9 @@ class State(): ) # Safety Check #2 - for pair in self.block_pairs: - pair.safety_check(dry_run=docopt_args['--dry-run']) + if not docopt_args['--dry-run']: + for pair in self.block_pairs: + pair.safety_check() def init_tmux(self): """Initialize tmux layout.""" @@ -717,7 +718,7 @@ class State(): raise std.GenericAbort() # Update settings - settings['Needs Format'] = not dry_run + settings['Needs Format'] = False self.save_settings(settings, working_dir) def retry_all_passes(self): diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 37df3e21..e98b9d5d 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -739,7 +739,7 @@ def color_string(strings, colors, sep=' '): # Convert to strings if necessary try: - iterator = iter(strings) + iter(strings) except TypeError: # Assuming single element passed, convert to string strings = (str(strings),) From 2983eb9bd32b2fdd5034364a7f8780b7cf4874f9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 23:24:23 -0700 Subject: [PATCH 289/324] Updated run_ddrescue() * Added SMART pane logic --- scripts/wk/hw/ddrescue.py | 58 +++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 68fc7f12..9e0276dd 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -240,6 +240,15 @@ class BlockPair(): std.print_error(f'Invalid destination: {self.destination}') raise std.GenericAbort() + def update_progress(self, pass_name): + """Update progress via map data.""" + self.load_map_data() + + # Update status + percent = self.get_percent_recovered() + if percent > 0: + self.status[pass_name] = percent + class State(): """Object for tracking hardware diagnostic data.""" @@ -732,6 +741,11 @@ class State(): required_size = sum([pair.size for pair in self.block_pairs]) settings = self.load_settings(working_dir) if mode == 'Clone' else {} + # Check dest SMART if cloning + if mode == 'Clone': + # TODO: Check dest SMART + pass + # Increase required_size if necessary if mode == 'Clone' and settings.get('Needs Format', False): if settings['Table Type'] == 'GPT': @@ -1577,14 +1591,37 @@ def mount_raw_image_macos(path): return loopback_path -def run_ddrescue(state, block_pair, settings): +def run_ddrescue(state, block_pair, pass_name, settings): """Run ddrescue using passed settings.""" state.update_progress_pane('Active') - print('Running ddrescue') - print(f' {block_pair.source} --> {block_pair.destination}') - print('Using these settings:') - for _s in settings: - print(f' {_s}') + i = 0 + + while True: + # Update SMART pane (every 30 seconds) + if i % 30 == 0: + now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M %Z') + with open(f'{state.log_dir}/smart.out', 'w') as _f: + _f.write( + std.color_string( + ['SMART Attributes', f'Updated: {now}\n'], + ['BLUE', 'YELLOW'], + sep='\t\t', + ), + ) + _f.write('\n'.join(state.source.generate_report(header=False))) + + # Update progress + block_pair.update_progress(pass_name) + state.update_progress_pane('Active') + + # Run + # TODO: Make ddrescue calls real + print('Running ddrescue') + print(f' {block_pair.source} --> {block_pair.destination}') + print('Using these settings:') + for _s in settings: + print(f' {_s}') + break std.pause() @@ -1622,14 +1659,19 @@ def run_recovery(state, main_menu, settings_menu): # Run ddrescue for pair in state.block_pairs: + abort = False if not pair.pass_complete(pass_name): attempted_recovery = True - run_ddrescue(state, pair, settings) + try: + run_ddrescue(state, pair, pass_name, settings) + except KeyboardInterrupt: + abort = True + break # Continue or return to menu all_complete = state.pass_complete(pass_name) all_above_threshold = state.pass_above_threshold(pass_name) - if not (all_complete and all_above_threshold and auto_continue): + if abort or not (all_complete and all_above_threshold and auto_continue): break # Show warning if nothing was done From c22c3da493812fbb0a218c8965f92b7eeae23225 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Jan 2020 23:33:21 -0700 Subject: [PATCH 290/324] Expanded safety checks * Added destination NVMe/SMART checks --- scripts/wk/hw/ddrescue.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 9e0276dd..5b3f77cc 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -532,8 +532,10 @@ class State(): source_parts = select_disk_parts(mode, self.source) self.add_image_block_pairs(source_parts, working_dir) - # Safety Check - self.safety_check(mode, working_dir) + # Safety Checks #1 + if mode == 'Clone': + self.safety_check_destination() + self.safety_check_size(mode, working_dir) # Confirmation #2 self.update_progress_pane('Idle') @@ -545,7 +547,7 @@ class State(): source_parts, working_dir, dry_run=docopt_args['--dry-run'], ) - # Safety Check #2 + # Safety Checks #2 if not docopt_args['--dry-run']: for pair in self.block_pairs: pair.safety_check() @@ -736,16 +738,22 @@ class State(): for name in pair.status.keys(): pair.status[name] = 'Pending' - def safety_check(self, mode, working_dir): - """Run safety check and abort if necessary.""" + def safety_check_destination(self): + """Run safety checks for destination and abort if necessary.""" + try: + self.destination.safety_checks() + except hw_obj.CriticalHardwareError: + std.print_error( + f'Critical error(s) detected for: {self.destination.path}', + ) + raise std.GenericAbort() + + + def safety_check_size(self, mode, working_dir): + """Run size safety check and abort if necessary.""" required_size = sum([pair.size for pair in self.block_pairs]) settings = self.load_settings(working_dir) if mode == 'Clone' else {} - # Check dest SMART if cloning - if mode == 'Clone': - # TODO: Check dest SMART - pass - # Increase required_size if necessary if mode == 'Clone' and settings.get('Needs Format', False): if settings['Table Type'] == 'GPT': From 2b18da7244a00478bf88007fc274213b3ad330d2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 00:47:33 -0700 Subject: [PATCH 291/324] Added real ddrescue command logic * Still needs testing!! * Set all dry_run keywords to default to True --- scripts/wk/hw/ddrescue.py | 132 ++++++++++++++++++++++++++++++-------- 1 file changed, 105 insertions(+), 27 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 5b3f77cc..722fcdcf 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -12,6 +12,7 @@ import pathlib import plistlib import re import shutil +import subprocess import time from collections import OrderedDict @@ -640,7 +641,7 @@ class State(): """Check if all block_pairs completed pass_name, returns bool.""" return all([p.pass_complete(pass_name) for p in self.block_pairs]) - def prep_destination(self, source_parts, working_dir, dry_run=False): + def prep_destination(self, source_parts, working_dir, dry_run=True): """Prep destination as necessary.""" dest_prefix = str(self.destination.path) dest_prefix += get_partition_separator(self.destination.path.name) @@ -1005,6 +1006,30 @@ def build_block_pair_report(block_pairs, settings): return report +def build_ddrescue_cmd(block_pair, pass_name, settings): + """Build ddrescue cmd using passed details, returns list.""" + cmd = ['sudo', 'ddrescue'] + if (block_pair.destination.is_block_device() + or block_pair.destination.is_char_device()): + cmd.append('--force') + if pass_name == 'read': + cmd.extend(['--no-trim', '--no-scrape']) + elif pass_name == 'trim': + # Allow trimming + cmd.append('--no-scrape') + elif pass_name == 'scrape': + # Allow trimming and scraping + pass + cmd.extend(settings) + cmd.append(block_pair.source) + cmd.append(block_pair.destination) + cmd.append(block_pair.map_path) + + # Done + LOG.debug('ddrescue cmd: %s', cmd) + return cmd + + def build_directory_report(path): """Build directory report, returns list.""" path = f'{path}/' @@ -1519,7 +1544,7 @@ def main(): # Start recovery if 'Start' in selection: - run_recovery(state, main_menu, settings_menu) + run_recovery(state, main_menu, settings_menu, dry_run=args['--dry-run']) # Quit if 'Quit' in selection: @@ -1599,41 +1624,94 @@ def mount_raw_image_macos(path): return loopback_path -def run_ddrescue(state, block_pair, pass_name, settings): +def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): """Run ddrescue using passed settings.""" + cmd = build_ddrescue_cmd(block_pair, pass_name, settings) + proc = None state.update_progress_pane('Active') - i = 0 + std.clear_screen() + warning_message = '' + def _update_smart_pane(iteration): + """Update SMART pane every 30 seconds.""" + if iteration % 30 != 0: + return + state.source.update_smart_details() + now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M %Z') + with open(f'{state.log_dir}/smart.out', 'w') as _f: + _f.write( + std.color_string( + ['SMART Attributes', f'Updated: {now}\n'], + ['BLUE', 'YELLOW'], + sep='\t\t', + ), + ) + _f.write('\n'.join(state.source.generate_report(header=False))) + + # Dry run + if dry_run: + std.print_info('ddrescue cmd:') + for _c in cmd: + std.print_standard(f' {_c}') + std.pause() + return + + # Start ddrescue + proc = exe.popen_program(cmd) + + # ddrescue loop + _i = 0 while True: - # Update SMART pane (every 30 seconds) - if i % 30 == 0: - now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M %Z') - with open(f'{state.log_dir}/smart.out', 'w') as _f: - _f.write( - std.color_string( - ['SMART Attributes', f'Updated: {now}\n'], - ['BLUE', 'YELLOW'], - sep='\t\t', - ), - ) - _f.write('\n'.join(state.source.generate_report(header=False))) + _update_smart_pane(_i) + _i += 1 # Update progress block_pair.update_progress(pass_name) state.update_progress_pane('Active') - # Run - # TODO: Make ddrescue calls real - print('Running ddrescue') - print(f' {block_pair.source} --> {block_pair.destination}') - print('Using these settings:') - for _s in settings: - print(f' {_s}') - break - std.pause() + # Check if complete + try: + proc.wait(timeout=1) + except KeyboardInterrupt: + # Wait a bit to let ddrescue exit safely + warning_message = 'Aborted' + proc.wait(timeout=10) + proc.terminate() + break + except subprocess.TimeoutExpired: + # Continue to next loop to update panes + pass + else: + # Done + break + + # Update progress + # NOTE: Using 'Active' here to avoid flickering between block pairs + block_pair.update_progress(pass_name) + state.update_progress_pane('Active') + + # Check result + if proc.poll(): + # True if return code is non-zero (poll() returns None if still running) + warning_message = 'Error(s) encountered, see message above' + if warning_message: + print(' ') + print(' ') + std.print_error('DDRESCUE PROCESS HALTED') + print(' ') + std.print_warning(warning_message) + + # Needs attention? + if str(proc.poll()) != '0': + state.update_progress_pane('NEEDS ATTENTION') + std.pause('Press Enter to return to main menu...') + + # Aborted? + if 'Aborted' in warning_message: + raise std.GenericAbort() -def run_recovery(state, main_menu, settings_menu): +def run_recovery(state, main_menu, settings_menu, dry_run=True): """Run recovery passes.""" atexit.register(state.save_debug_reports) attempted_recovery = False @@ -1671,7 +1749,7 @@ def run_recovery(state, main_menu, settings_menu): if not pair.pass_complete(pass_name): attempted_recovery = True try: - run_ddrescue(state, pair, pass_name, settings) + run_ddrescue(state, pair, pass_name, settings, dry_run=dry_run) except KeyboardInterrupt: abort = True break From 4f2b31c705345c20b2fa5fad734f9fb6157425bb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 01:14:06 -0700 Subject: [PATCH 292/324] Avoid crash while stopping ddrescue * Killall is needed because of sudo --- scripts/wk/hw/ddrescue.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 722fcdcf..b67babc6 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1675,14 +1675,15 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): except KeyboardInterrupt: # Wait a bit to let ddrescue exit safely warning_message = 'Aborted' - proc.wait(timeout=10) - proc.terminate() + std.sleep(2) + exe.run_program(['sudo', 'killall', 'ddrescue'], check=False) break except subprocess.TimeoutExpired: # Continue to next loop to update panes pass else: # Done + std.sleep(1) break # Update progress From a4b5e81ef1500fbf2138852a181f989f9772c866 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 16:08:38 -0700 Subject: [PATCH 293/324] Made working_dir a State() variable --- scripts/wk/hw/ddrescue.py | 74 +++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index b67babc6..30821e47 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -256,27 +256,27 @@ class State(): def __init__(self): self.block_pairs = [] self.destination = None - self.disks = [] self.layout = cfg.ddrescue.TMUX_LAYOUT.copy() self.log_dir = None self.panes = {} self.source = None + self.working_dir = None # Start a background process to maintain layout self.init_tmux() exe.start_thread(self.fix_tmux_layout_loop) - def add_block_pair(self, source, destination, working_dir): + def add_block_pair(self, source, destination): """Add BlockPair object and run safety checks.""" self.block_pairs.append( BlockPair( source=source, destination=destination, model=self.source.details['model'], - working_dir=working_dir, + working_dir=self.working_dir, )) - def add_clone_block_pairs(self, working_dir): + def add_clone_block_pairs(self): """Add device to device block pairs and set settings if necessary.""" source_sep = get_partition_separator(self.source.path.name) dest_sep = get_partition_separator(self.destination.path.name) @@ -284,7 +284,7 @@ class State(): source_parts = [] # Clone settings - settings = self.load_settings(working_dir, discard_unused_settings=True) + settings = self.load_settings(discard_unused_settings=True) # Add pairs if settings['Partition Mapping']: @@ -296,13 +296,13 @@ class State(): bp_dest = pathlib.Path( f'{self.destination.path}{dest_sep}{part_map[1]}', ) - self.add_block_pair(bp_source, bp_dest, working_dir) + self.add_block_pair(bp_source, bp_dest) else: source_parts = select_disk_parts('Clone', self.source) if self.source.path.samefile(source_parts[0].path): # Whole disk (or single partition via args), skip settings bp_dest = self.destination.path - self.add_block_pair(self.source, bp_dest, working_dir) + self.add_block_pair(self.source, bp_dest) else: # New run, use new settings file settings['Needs Format'] = True @@ -328,26 +328,25 @@ class State(): bp_dest = pathlib.Path( f'{self.destination.path}{dest_sep}{dest_num}', ) - self.add_block_pair(part, bp_dest, working_dir) + self.add_block_pair(part, bp_dest) # Add to settings file source_num = re.sub(r'^.*?(\d+)$', r'\1', part.path.name) settings['Partition Mapping'].append([source_num, dest_num]) # Save settings - self.save_settings(settings, working_dir) + self.save_settings(settings) # Done return source_parts - def add_image_block_pairs(self, source_parts, working_dir): + def add_image_block_pairs(self, source_parts): """Add device to image file block pairs.""" for part in source_parts: bp_dest = self.destination - self.add_block_pair(part, bp_dest, working_dir) + self.add_block_pair(part, bp_dest) - def confirm_selections( - self, mode, prompt, working_dir=None, source_parts=None): + def confirm_selections(self, mode, prompt, source_parts=None): """Show selection details and prompt for confirmation.""" report = [] @@ -384,17 +383,17 @@ class State(): report.extend( build_block_pair_report( self.block_pairs, - self.load_settings(working_dir) if mode == 'Clone' else {}, + self.load_settings() if mode == 'Clone' else {}, ), ) report.append(' ') # Map dir - if working_dir: + if self.working_dir: report.append(std.color_string('Map Save Directory', 'GREEN')) - report.append(f'{working_dir}/') + report.append(f'{self.working_dir}/') report.append(' ') - if not fstype_is_ok(working_dir, map_dir=True): + if not fstype_is_ok(self.working_dir, map_dir=True): report.append( std.color_string( 'Map file(s) are being saved to a non-recommended filesystem.', @@ -518,35 +517,33 @@ class State(): self.update_progress_pane('Idle') # Set working dir - working_dir = get_working_dir( + self.working_dir = get_working_dir( mode, self.destination, force_local=docopt_args['--force-local-map'], ) # Start fresh if requested if docopt_args['--start-fresh']: - clean_working_dir(working_dir) + clean_working_dir(self.working_dir) # Add block pairs if mode == 'Clone': - source_parts = self.add_clone_block_pairs(working_dir) + source_parts = self.add_clone_block_pairs() else: source_parts = select_disk_parts(mode, self.source) - self.add_image_block_pairs(source_parts, working_dir) + self.add_image_block_pairs(source_parts) # Safety Checks #1 if mode == 'Clone': self.safety_check_destination() - self.safety_check_size(mode, working_dir) + self.safety_check_size(mode) # Confirmation #2 self.update_progress_pane('Idle') - self.confirm_selections(mode, 'Start recovery?', working_dir=working_dir) + self.confirm_selections(mode, 'Start recovery?') # Prep destination if mode == 'Clone': - self.prep_destination( - source_parts, working_dir, dry_run=docopt_args['--dry-run'], - ) + self.prep_destination(source_parts, dry_run=docopt_args['--dry-run']) # Safety Checks #2 if not docopt_args['--dry-run']: @@ -579,11 +576,11 @@ class State(): # Source / Dest self.update_top_panes() - def load_settings(self, working_dir, discard_unused_settings=False): + def load_settings(self, discard_unused_settings=False): """Load settings from previous run, returns dict.""" settings = {} settings_file = pathlib.Path( - f'{working_dir}/Clone_{self.source.details["model"]}.json', + f'{self.working_dir}/Clone_{self.source.details["model"]}.json', ) # Try loading JSON data @@ -641,7 +638,7 @@ class State(): """Check if all block_pairs completed pass_name, returns bool.""" return all([p.pass_complete(pass_name) for p in self.block_pairs]) - def prep_destination(self, source_parts, working_dir, dry_run=True): + def prep_destination(self, source_parts, dry_run=True): """Prep destination as necessary.""" dest_prefix = str(self.destination.path) dest_prefix += get_partition_separator(self.destination.path.name) @@ -649,7 +646,7 @@ class State(): msr_type = 'E3C9E316-0B5C-4DB8-817D-F92DF00215AE' part_num = 0 sfdisk_script = [] - settings = self.load_settings(working_dir) + settings = self.load_settings() # Bail early if not settings['Needs Format']: @@ -710,7 +707,10 @@ class State(): ) # Save sfdisk script - script_path = f'{working_dir}/sfdisk_{self.destination.path.name}.script' + script_path = ( + f'{self.working_dir}/' + f'sfdisk_{self.destination.path.name}.script' + ) with open(script_path, 'w') as _f: _f.write('\n'.join(sfdisk_script)) @@ -731,7 +731,7 @@ class State(): # Update settings settings['Needs Format'] = False - self.save_settings(settings, working_dir) + self.save_settings(settings) def retry_all_passes(self): """Set all statuses to Pending.""" @@ -750,10 +750,10 @@ class State(): raise std.GenericAbort() - def safety_check_size(self, mode, working_dir): + def safety_check_size(self, mode): """Run size safety check and abort if necessary.""" required_size = sum([pair.size for pair in self.block_pairs]) - settings = self.load_settings(working_dir) if mode == 'Clone' else {} + settings = self.load_settings() if mode == 'Clone' else {} # Increase required_size if necessary if mode == 'Clone' and settings.get('Needs Format', False): @@ -812,11 +812,11 @@ class State(): with open(f'{debug_dir}/bp_part#.report', 'a') as _f: _f.write('\n'.join(debug.generate_object_report(_bp))) - def save_settings(self, settings, working_dir): + def save_settings(self, settings): # pylint: disable=no-self-use """Save settings for future runs.""" settings_file = pathlib.Path( - f'{working_dir}/Clone_{self.source.details["model"]}.json', + f'{self.working_dir}/Clone_{self.source.details["model"]}.json', ) # Try saving JSON data @@ -1676,7 +1676,7 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): # Wait a bit to let ddrescue exit safely warning_message = 'Aborted' std.sleep(2) - exe.run_program(['sudo', 'killall', 'ddrescue'], check=False) + exe.run_program(['sudo', 'kill', str(proc.pid)], check=False) break except subprocess.TimeoutExpired: # Continue to next loop to update panes From 276e2e0ddab263c712abce346ff168f46fc33fb1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 16:38:48 -0700 Subject: [PATCH 294/324] Made mode a State() variable --- scripts/wk/hw/ddrescue.py | 48 ++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 30821e47..6e77a01d 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -256,8 +256,8 @@ class State(): def __init__(self): self.block_pairs = [] self.destination = None - self.layout = cfg.ddrescue.TMUX_LAYOUT.copy() self.log_dir = None + self.mode = None self.panes = {} self.source = None self.working_dir = None @@ -346,7 +346,7 @@ class State(): bp_dest = self.destination self.add_block_pair(part, bp_dest) - def confirm_selections(self, mode, prompt, source_parts=None): + def confirm_selections(self, prompt, source_parts=None): """Show selection details and prompt for confirmation.""" report = [] @@ -357,7 +357,7 @@ class State(): # Destination report.append(std.color_string('Destination', 'GREEN')) - if mode == 'Clone': + if self.mode == 'Clone': report[-1] += std.color_string(' (ALL DATA WILL BE DELETED)', 'RED') report.extend(build_object_report(self.destination)) report.append(' ') @@ -365,7 +365,7 @@ class State(): # Show deletion warning if necessary # NOTE: The check for block_pairs is to limit this section # to the second confirmation - if mode == 'Clone' and self.block_pairs: + if self.mode == 'Clone' and self.block_pairs: report.append(std.color_string('WARNING', 'YELLOW')) report.append( 'All data will be deleted from the destination listed above.', @@ -383,7 +383,7 @@ class State(): report.extend( build_block_pair_report( self.block_pairs, - self.load_settings() if mode == 'Clone' else {}, + self.load_settings() if self.mode == 'Clone' else {}, ), ) report.append(' ') @@ -430,11 +430,12 @@ class State(): def fix_tmux_layout(self, forced=True): """Fix tmux layout based on cfg.ddrescue.TMUX_LAYOUT.""" - needs_fixed = tmux.layout_needs_fixed(self.panes, self.layout) + layout = cfg.ddrescue.TMUX_LAYOUT + needs_fixed = tmux.layout_needs_fixed(self.panes, layout) # Main layout fix try: - tmux.fix_layout(self.panes, self.layout, forced=forced) + tmux.fix_layout(self.panes, layout, forced=forced) except RuntimeError: # Assuming self.panes changed while running pass @@ -485,7 +486,7 @@ class State(): ) # Set mode - mode = set_mode(docopt_args) + self.mode = set_mode(docopt_args) # Select source self.source = get_object(docopt_args['']) @@ -496,15 +497,14 @@ class State(): # Select destination self.destination = get_object(docopt_args['']) if not self.destination: - if mode == 'Clone': + if self.mode == 'Clone': self.destination = select_disk('Destination', self.source) - elif mode == 'Image': + elif self.mode == 'Image': self.destination = select_path('Destination') self.update_top_panes() # Confirmation #1 self.confirm_selections( - mode=mode, prompt='Are these selections correct?', source_parts=source_parts, ) @@ -518,7 +518,9 @@ class State(): # Set working dir self.working_dir = get_working_dir( - mode, self.destination, force_local=docopt_args['--force-local-map'], + self.mode, + self.destination, + force_local=docopt_args['--force-local-map'], ) # Start fresh if requested @@ -526,23 +528,23 @@ class State(): clean_working_dir(self.working_dir) # Add block pairs - if mode == 'Clone': + if self.mode == 'Clone': source_parts = self.add_clone_block_pairs() else: - source_parts = select_disk_parts(mode, self.source) + source_parts = select_disk_parts(self.mode, self.source) self.add_image_block_pairs(source_parts) # Safety Checks #1 - if mode == 'Clone': + if self.mode == 'Clone': self.safety_check_destination() - self.safety_check_size(mode) + self.safety_check_size() # Confirmation #2 self.update_progress_pane('Idle') - self.confirm_selections(mode, 'Start recovery?') + self.confirm_selections('Start recovery?') # Prep destination - if mode == 'Clone': + if self.mode == 'Clone': self.prep_destination(source_parts, dry_run=docopt_args['--dry-run']) # Safety Checks #2 @@ -750,13 +752,13 @@ class State(): raise std.GenericAbort() - def safety_check_size(self, mode): + def safety_check_size(self): """Run size safety check and abort if necessary.""" required_size = sum([pair.size for pair in self.block_pairs]) - settings = self.load_settings() if mode == 'Clone' else {} + settings = self.load_settings() if self.mode == 'Clone' else {} # Increase required_size if necessary - if mode == 'Clone' and settings.get('Needs Format', False): + if self.mode == 'Clone' and settings.get('Needs Format', False): if settings['Table Type'] == 'GPT': # Below is the size calculation for the GPT # 1 LBA for the protective MBR @@ -774,7 +776,7 @@ class State(): required_size += 100 * 1024**2 # Reduce required_size if necessary - if mode == 'Image': + if self.mode == 'Image': for pair in self.block_pairs: if pair.destination.exists(): # NOTE: This uses the "max space" of the destination @@ -784,7 +786,7 @@ class State(): required_size -= pair.destination.stat().st_size # Check destination size - if mode == 'Clone': + if self.mode == 'Clone': destination_size = self.destination.details['size'] error_msg = 'A larger destination disk is required' else: From eb702577ae93c658f785f66bed404d08b9aa2840 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 16:45:47 -0700 Subject: [PATCH 295/324] Mark clones as started to allow resuming --- scripts/wk/hw/ddrescue.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 6e77a01d..a1abd2f0 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -629,6 +629,23 @@ class State(): # Done return settings + def mark_started(self): + """Edit clone settings, if applicable, to mark recovery as started.""" + # Skip if not cloning + if self.mode != 'Clone': + return + + # Skip if not using settings + # i.e. Cloning whole disk (or single partition via args) + if self.source.path.samefile(self.block_pairs[0].source): + return + + # Update settings + settings = self.load_settings() + if settings.get('First Run', False): + settings['First Run'] = False + self.save_settings(settings) + def pass_above_threshold(self, pass_name): """Check if all block_pairs meet the pass threshold, returns bool.""" threshold = cfg.ddrescue.AUTO_PASS_THRESHOLDS[pass_name] @@ -1751,6 +1768,7 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): abort = False if not pair.pass_complete(pass_name): attempted_recovery = True + state.mark_started() try: run_ddrescue(state, pair, pass_name, settings, dry_run=dry_run) except KeyboardInterrupt: From 6eaf5c2bc2e6424eec769113eefa96995524f53e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 17:25:12 -0700 Subject: [PATCH 296/324] Get accurate size from ddrescuelog * Reported size is off by one sector in some cases --- scripts/wk/hw/ddrescue.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index a1abd2f0..b3a599b0 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -53,7 +53,8 @@ CLONE_SETTINGS = { DDRESCUE_LOG_REGEX = re.compile( r'^\s*(?P\S+):\s+' r'(?P\d+)\s+' - r'(?P[PTGMKB]i?B?)', + r'(?P[PTGMKB]i?B?)' + r'.*\(\s*(?P\d+\.?\d*)%\)$', re.IGNORECASE, ) REGEX_REMAINING_TIME = re.compile( @@ -187,9 +188,13 @@ class BlockPair(): for line in proc.stdout.splitlines(): _r = DDRESCUE_LOG_REGEX.search(line) if _r: - data[_r.group('key')] = std.string_to_bytes( - f'{_r.group("size")} {_r.group("unit")}', - ) + if _r.group('key') == 'rescued' and _r.group('percent') == '100': + # Fix rounding errors from ddrescuelog output + data['rescued'] = self.size + else: + data[_r.group('key')] = std.string_to_bytes( + f'{_r.group("size")} {_r.group("unit")}', + ) data['pass completed'] = 'current status: finished' in line.lower() # Check if 100% done @@ -1302,6 +1307,7 @@ def format_status_string(status, width): status_str = f'{percent:{width-2}.2f} %' if '100.00' in status_str and percent < 100: # Always round down to 99.99% + LOG.warning('Rounding down to 99.99 from %s', percent) status_str = f'{"99.99 %":>{width}}' else: # Text From 097360ca0abc7c5273d262ab763d87f474cf56b9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 17:32:51 -0700 Subject: [PATCH 297/324] Always ask GPT/MBR/Source when formatting a disk * This fixes cloning partitions when not creating a boot partiton --- scripts/wk/hw/ddrescue.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index b3a599b0..bdf300bf 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -312,19 +312,19 @@ class State(): # New run, use new settings file settings['Needs Format'] = True offset = 0 + user_choice = std.choice( + ['G', 'M', 'S'], + 'Format clone using GPT, MBR, or match Source type?', + ) + if user_choice == 'G': + settings['Table Type'] = 'GPT' + elif user_choice == 'M': + settings['Table Type'] = 'MBR' + else: + # Match source type + settings['Table Type'] = get_table_type(self.source) if std.ask('Create an empty Windows boot partition on the clone?'): settings['Create Boot Partition'] = True - user_choice = std.choice( - ['G', 'M', 'S'], - 'Use GPT, MBR, or match Source type?', - ) - if user_choice == 'G': - settings['Table Type'] = 'GPT' - elif user_choice == 'M': - settings['Table Type'] = 'MBR' - else: - # Match source type - settings['Table Type'] = get_table_type(self.source) offset = 2 if settings['Table Type'] == 'GPT' else 1 # Add pairs @@ -1249,7 +1249,7 @@ def build_sfdisk_partition_line(table_type, dev_path, size, details): dest_type = source_type.upper() if not dest_type: # Assuming changing table types, set based on FS - if source_filesystem in cfg.ddrescue.PARTITION_TYPES[table_type]: + if source_filesystem in cfg.ddrescue.PARTITION_TYPES.get(table_type, {}): dest_type = cfg.ddrescue.PARTITION_TYPES[table_type][source_filesystem] line += f', type={dest_type}' From 848ccc3ef113657ca680d2d45b88fa99186e2461 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 17:54:55 -0700 Subject: [PATCH 298/324] Made several State() functions "private" --- scripts/wk/hw/ddrescue.py | 143 +++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index bdf300bf..dd01cc2a 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -268,10 +268,10 @@ class State(): self.working_dir = None # Start a background process to maintain layout - self.init_tmux() - exe.start_thread(self.fix_tmux_layout_loop) + self._init_tmux() + exe.start_thread(self._fix_tmux_layout_loop) - def add_block_pair(self, source, destination): + def _add_block_pair(self, source, destination): """Add BlockPair object and run safety checks.""" self.block_pairs.append( BlockPair( @@ -281,6 +281,71 @@ class State(): working_dir=self.working_dir, )) + def _fix_tmux_layout(self, forced=True): + """Fix tmux layout based on cfg.ddrescue.TMUX_LAYOUT.""" + layout = cfg.ddrescue.TMUX_LAYOUT + needs_fixed = tmux.layout_needs_fixed(self.panes, layout) + + # Main layout fix + try: + tmux.fix_layout(self.panes, layout, forced=forced) + except RuntimeError: + # Assuming self.panes changed while running + pass + + # Source/Destination + if forced or needs_fixed: + self.update_top_panes() + + # Return if Progress pane not present + if 'Progress' not in self.panes: + return + + # SMART/Journal + if forced or needs_fixed: + height = tmux.get_pane_size(self.panes['Progress'])[1] - 2 + p_ratios = [int((x/sum(PANE_RATIOS)) * height) for x in PANE_RATIOS] + if 'SMART' in self.panes: + tmux.resize_pane(self.panes['SMART'], height=p_ratios[0]) + tmux.resize_pane(height=p_ratios[1]) + if 'Journal' in self.panes: + tmux.resize_pane(self.panes['Journal'], height=p_ratios[2]) + + def _fix_tmux_layout_loop(self): + """Fix tmux layout on a loop. + + NOTE: This should be called as a thread. + """ + while True: + self._fix_tmux_layout(forced=False) + std.sleep(1) + + def _init_tmux(self): + """Initialize tmux layout.""" + tmux.kill_all_panes() + + # Source (placeholder) + self.panes['Source'] = tmux.split_window( + behind=True, + lines=2, + text=' ', + vertical=True, + ) + + # Started + self.panes['Started'] = tmux.split_window( + lines=cfg.ddrescue.TMUX_SIDE_WIDTH, + target_id=self.panes['Source'], + text=std.color_string( + ['Started', time.strftime("%Y-%m-%d %H:%M %Z")], + ['BLUE', None], + sep='\n', + ), + ) + + # Source / Dest + self.update_top_panes() + def add_clone_block_pairs(self): """Add device to device block pairs and set settings if necessary.""" source_sep = get_partition_separator(self.source.path.name) @@ -301,13 +366,13 @@ class State(): bp_dest = pathlib.Path( f'{self.destination.path}{dest_sep}{part_map[1]}', ) - self.add_block_pair(bp_source, bp_dest) + self._add_block_pair(bp_source, bp_dest) else: source_parts = select_disk_parts('Clone', self.source) if self.source.path.samefile(source_parts[0].path): # Whole disk (or single partition via args), skip settings bp_dest = self.destination.path - self.add_block_pair(self.source, bp_dest) + self._add_block_pair(self.source, bp_dest) else: # New run, use new settings file settings['Needs Format'] = True @@ -333,7 +398,7 @@ class State(): bp_dest = pathlib.Path( f'{self.destination.path}{dest_sep}{dest_num}', ) - self.add_block_pair(part, bp_dest) + self._add_block_pair(part, bp_dest) # Add to settings file source_num = re.sub(r'^.*?(\d+)$', r'\1', part.path.name) @@ -349,7 +414,7 @@ class State(): """Add device to image file block pairs.""" for part in source_parts: bp_dest = self.destination - self.add_block_pair(part, bp_dest) + self._add_block_pair(part, bp_dest) def confirm_selections(self, prompt, source_parts=None): """Show selection details and prompt for confirmation.""" @@ -433,44 +498,7 @@ class State(): if not std.ask(prompt): raise std.GenericAbort() - def fix_tmux_layout(self, forced=True): - """Fix tmux layout based on cfg.ddrescue.TMUX_LAYOUT.""" - layout = cfg.ddrescue.TMUX_LAYOUT - needs_fixed = tmux.layout_needs_fixed(self.panes, layout) - # Main layout fix - try: - tmux.fix_layout(self.panes, layout, forced=forced) - except RuntimeError: - # Assuming self.panes changed while running - pass - - # Source/Destination - if forced or needs_fixed: - self.update_top_panes() - - # Return if Progress pane not present - if 'Progress' not in self.panes: - return - - # SMART/Journal - if forced or needs_fixed: - height = tmux.get_pane_size(self.panes['Progress'])[1] - 2 - p_ratios = [int((x/sum(PANE_RATIOS)) * height) for x in PANE_RATIOS] - if 'SMART' in self.panes: - tmux.resize_pane(self.panes['SMART'], height=p_ratios[0]) - tmux.resize_pane(height=p_ratios[1]) - if 'Journal' in self.panes: - tmux.resize_pane(self.panes['Journal'], height=p_ratios[2]) - - def fix_tmux_layout_loop(self): - """Fix tmux layout on a loop. - - NOTE: This should be called as a thread. - """ - while True: - self.fix_tmux_layout(forced=False) - std.sleep(1) def init_recovery(self, docopt_args): """Select source/dest and set env.""" @@ -557,32 +585,6 @@ class State(): for pair in self.block_pairs: pair.safety_check() - def init_tmux(self): - """Initialize tmux layout.""" - tmux.kill_all_panes() - - # Source (placeholder) - self.panes['Source'] = tmux.split_window( - behind=True, - lines=2, - text=' ', - vertical=True, - ) - - # Started - self.panes['Started'] = tmux.split_window( - lines=cfg.ddrescue.TMUX_SIDE_WIDTH, - target_id=self.panes['Source'], - text=std.color_string( - ['Started', time.strftime("%Y-%m-%d %H:%M %Z")], - ['BLUE', None], - sep='\n', - ), - ) - - # Source / Dest - self.update_top_panes() - def load_settings(self, discard_unused_settings=False): """Load settings from previous run, returns dict.""" settings = {} @@ -773,7 +775,6 @@ class State(): ) raise std.GenericAbort() - def safety_check_size(self): """Run size safety check and abort if necessary.""" required_size = sum([pair.size for pair in self.block_pairs]) From 4a2b18e4f7b3c83e1a3c6d342e628743d79637c0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 17:55:23 -0700 Subject: [PATCH 299/324] Added confirmation to Quit if recovery < 100% --- scripts/wk/hw/ddrescue.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index dd01cc2a..a05c7476 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -498,7 +498,15 @@ class State(): if not std.ask(prompt): raise std.GenericAbort() + def get_percent_recovered(self): + """Get total percent rescued from block_pairs, returns float.""" + total_rescued = self.get_rescued_size() + total_size = sum([pair.size for pair in self.block_pairs]) + return 100 * total_rescued / total_size + def get_rescued_size(self): + """Get total rescued size from all block pairs, returns int.""" + return sum([pair.get_rescued_size() for pair in self.block_pairs]) def init_recovery(self, docopt_args): """Select source/dest and set env.""" @@ -870,11 +878,8 @@ class State(): # Overall progress if self.block_pairs: - total_rescued = sum( - [pair.map_data.get('rescued', 0) for pair in self.block_pairs], - ) - total_size = sum([pair.size for pair in self.block_pairs]) - percent = 100 * total_rescued / total_size + total_rescued = self.get_rescued_size() + percent = self.get_percent_recovered() report.append(std.color_string('Overall Progress', 'BLUE')) report.append( f'Rescued: {format_status_string(percent, width=width-9)}', @@ -1574,7 +1579,14 @@ def main(): # Quit if 'Quit' in selection: - break + total_percent = state.get_percent_recovered() + if total_percent == 100: + break + + # Recovey < 100% + std.print_warning('Recovery is less than 100%') + if std.ask('Are you sure you want to quit?'): + break def mount_raw_image(path): From 30a5df8a00e95b42e127cc8b6ad0bf50565014db Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 18:36:40 -0700 Subject: [PATCH 300/324] Fix timezone in SMART pane --- scripts/wk/hw/ddrescue.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index a05c7476..5e7e858e 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -674,6 +674,9 @@ class State(): def prep_destination(self, source_parts, dry_run=True): """Prep destination as necessary.""" + # TODO: Split into Linux and macOS + # logical sector size is not easily found under macOS + # It might be easier to rewrite this section using macOS tools dest_prefix = str(self.destination.path) dest_prefix += get_partition_separator(self.destination.path.name) esp_type = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B' @@ -1675,7 +1678,7 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): if iteration % 30 != 0: return state.source.update_smart_details() - now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M %Z') + now = datetime.datetime.now(tz=TIMEZONE).strftime('%Y-%m-%d %H:%M %Z') with open(f'{state.log_dir}/smart.out', 'w') as _f: _f.write( std.color_string( From e6e51498dd9bff8332012ec2ad86c3fb5617bfa8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 3 Jan 2020 18:36:53 -0700 Subject: [PATCH 301/324] Clear ddrescue pane every minute --- scripts/wk/hw/ddrescue.py | 16 ++++++++-------- scripts/wk/hw/obj.py | 1 - scripts/wk/tmux.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 5e7e858e..d61b9763 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1673,10 +1673,8 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): std.clear_screen() warning_message = '' - def _update_smart_pane(iteration): + def _update_smart_pane(): """Update SMART pane every 30 seconds.""" - if iteration % 30 != 0: - return state.source.update_smart_details() now = datetime.datetime.now(tz=TIMEZONE).strftime('%Y-%m-%d %H:%M %Z') with open(f'{state.log_dir}/smart.out', 'w') as _f: @@ -1691,10 +1689,7 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): # Dry run if dry_run: - std.print_info('ddrescue cmd:') - for _c in cmd: - std.print_standard(f' {_c}') - std.pause() + LOG.info('ddrescue cmd: %s', cmd) return # Start ddrescue @@ -1703,7 +1698,12 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): # ddrescue loop _i = 0 while True: - _update_smart_pane(_i) + if _i % 30 == 0: + # Update SMART pane + _update_smart_pane() + if _i % 60 == 0: + # Clear ddrescue pane + tmux.clear_pane() _i += 1 # Update progress diff --git a/scripts/wk/hw/obj.py b/scripts/wk/hw/obj.py index 1a7421b6..205d857a 100644 --- a/scripts/wk/hw/obj.py +++ b/scripts/wk/hw/obj.py @@ -1,6 +1,5 @@ """WizardKit: Hardware objects (mostly)""" # vim: sts=2 sw=2 ts=2 -# TODO: Get log-sec data under Linux and macOS import logging import pathlib diff --git a/scripts/wk/tmux.py b/scripts/wk/tmux.py index 2f5b32e7..b6d32848 100644 --- a/scripts/wk/tmux.py +++ b/scripts/wk/tmux.py @@ -24,6 +24,16 @@ def capture_pane(pane_id=None): return proc.stdout.strip() +def clear_pane(pane_id=None): + """Clear pane buffer for current or target pane.""" + cmd = ['tmux', 'send-keys', '-R'] + if pane_id: + cmd.extend(['-t', pane_id]) + + # Clear pane + run_program(cmd, check=False) + + def fix_layout(panes, layout, forced=False): """Fix pane sizes based on layout.""" if not (forced or layout_needs_fixed(panes, layout)): From a68e52322c83e7b50d5653e0a7d72ec179b00508 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 15:41:15 -0700 Subject: [PATCH 302/324] Fixed aborting when multiple parts selected --- scripts/wk/hw/ddrescue.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index d61b9763..3ad91cc8 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1713,8 +1713,10 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): # Check if complete try: proc.wait(timeout=1) + break except KeyboardInterrupt: # Wait a bit to let ddrescue exit safely + LOG.warning('ddrescue stopped by user') warning_message = 'Aborted' std.sleep(2) exe.run_program(['sudo', 'kill', str(proc.pid)], check=False) @@ -1747,9 +1749,6 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): if str(proc.poll()) != '0': state.update_progress_pane('NEEDS ATTENTION') std.pause('Press Enter to return to main menu...') - - # Aborted? - if 'Aborted' in warning_message: raise std.GenericAbort() @@ -1780,20 +1779,22 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): # Run pass(es) for pass_name in ('read', 'trim', 'scrape'): + abort = False + + # Skip to next pass (unless retry selected) if '--retrim' not in settings and state.pass_complete(pass_name): - # Skip to next pass (unless retry selected) # NOTE: This bypasses auto_continue continue # Run ddrescue for pair in state.block_pairs: - abort = False if not pair.pass_complete(pass_name): attempted_recovery = True state.mark_started() try: run_ddrescue(state, pair, pass_name, settings, dry_run=dry_run) - except KeyboardInterrupt: + except (KeyboardInterrupt, std.GenericAbort): + LOG.warning('User stopped recovery (2)') abort = True break @@ -1801,6 +1802,7 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): all_complete = state.pass_complete(pass_name) all_above_threshold = state.pass_above_threshold(pass_name) if abort or not (all_complete and all_above_threshold and auto_continue): + LOG.warning('Recovery halted') break # Show warning if nothing was done @@ -1816,7 +1818,6 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): # Done state.save_debug_reports() atexit.unregister(state.save_debug_reports) - std.pause('Press Enter to return to main menu...') state.update_progress_pane('Idle') From c71e30e4fb05879c2c5ae9ed5b460710499645fc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 15:58:43 -0700 Subject: [PATCH 303/324] Adjusted debug reports --- scripts/wk/hw/ddrescue.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 3ad91cc8..c050ac85 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -36,7 +36,7 @@ Usage: Options: -h --help Show this page -s --dry-run Print commands to be used instead of running them - --force-local-map Skip mounting shares and save map to current dir + --force-local-map Skip mounting shares and save map to local drive --start-fresh Ignore previous runs and start new recovery ''' CLONE_SETTINGS = { @@ -841,12 +841,16 @@ class State(): # State (self) with open(f'{debug_dir}/state.report', 'a') as _f: + _f.write('[Debug report]\n') _f.write('\n'.join(debug.generate_object_report(self))) + _f.write('\n') # Block pairs for _bp in self.block_pairs: - with open(f'{debug_dir}/bp_part#.report', 'a') as _f: + with open(f'{debug_dir}/block_pairs.report', 'a') as _f: + _f.write('[Debug report]\n') _f.write('\n'.join(debug.generate_object_report(_bp))) + _f.write('\n') def save_settings(self, settings): # pylint: disable=no-self-use @@ -1794,7 +1798,6 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): try: run_ddrescue(state, pair, pass_name, settings, dry_run=dry_run) except (KeyboardInterrupt, std.GenericAbort): - LOG.warning('User stopped recovery (2)') abort = True break From 4acdab8c0ff943d96db3c4e9c52c6d7561cf0afa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 16:53:56 -0700 Subject: [PATCH 304/324] Mark passes Skipped as appropriate --- scripts/wk/hw/ddrescue.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index c050ac85..b7ba57e6 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -246,6 +246,11 @@ class BlockPair(): std.print_error(f'Invalid destination: {self.destination}') raise std.GenericAbort() + def skip_pass(self, pass_name): + """Mark pass as skipped if applicable.""" + if self.status[pass_name] == 'Pending': + self.status[pass_name] = 'Skipped' + def update_progress(self, pass_name): """Update progress via map data.""" self.load_map_data() @@ -255,6 +260,13 @@ class BlockPair(): if percent > 0: self.status[pass_name] = percent + # Mark future passes as skipped if applicable + if percent == 100: + if pass_name == 'read': + self.status['trim'] = 'Skipped' + if pass_name in ('read', 'trim'): + self.status['scrape'] = 'Skipped' + class State(): """Object for tracking hardware diagnostic data.""" @@ -853,7 +865,6 @@ class State(): _f.write('\n') def save_settings(self, settings): - # pylint: disable=no-self-use """Save settings for future runs.""" settings_file = pathlib.Path( f'{self.working_dir}/Clone_{self.source.details["model"]}.json', @@ -867,6 +878,12 @@ class State(): std.print_error('Failed to save clone settings') raise std.GenericAbort() + def skip_pass(self, pass_name): + """Mark block_pairs as skipped if applicable.""" + for pair in self.block_pairs: + if pair.status[pass_name] == 'Pending': + pair.status[pass_name] = 'Skipped' + def update_progress_pane(self, overall_status): """Update progress pane.""" report = [] @@ -1672,7 +1689,6 @@ def mount_raw_image_macos(path): def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True): """Run ddrescue using passed settings.""" cmd = build_ddrescue_cmd(block_pair, pass_name, settings) - proc = None state.update_progress_pane('Active') std.clear_screen() warning_message = '' @@ -1788,11 +1804,14 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): # Skip to next pass (unless retry selected) if '--retrim' not in settings and state.pass_complete(pass_name): # NOTE: This bypasses auto_continue + state.skip_pass(pass_name) continue # Run ddrescue for pair in state.block_pairs: - if not pair.pass_complete(pass_name): + if '--retrim' not in settings and pair.pass_complete(pass_name): + pair.skip_pass(pass_name) + else: attempted_recovery = True state.mark_started() try: From 470524dfff8249979e956b29de41d53f5c20afc3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 16:54:28 -0700 Subject: [PATCH 305/324] Added pause after "No actions performed" message --- scripts/wk/hw/ddrescue.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index b7ba57e6..e54d2c60 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1599,6 +1599,7 @@ def main(): # Start recovery if 'Start' in selection: + std.clear_screen() run_recovery(state, main_menu, settings_menu, dry_run=args['--dry-run']) # Quit @@ -1827,16 +1828,17 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): LOG.warning('Recovery halted') break - # Show warning if nothing was done - if not attempted_recovery: - std.print_warning('No actions performed') - std.print_standard(' ') - # Stop SMART/Journal for pane in ('SMART', 'Journal'): if pane in state.panes: tmux.kill_pane(state.panes.pop(pane)) + # Show warning if nothing was done + if not attempted_recovery: + std.print_warning('No actions performed') + std.print_standard(' ') + std.pause('Press Enter to return to main menu...') + # Done state.save_debug_reports() atexit.unregister(state.save_debug_reports) From 5926c3170d6541c7c952517b0eca313a9b3ce364 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 18:01:39 -0700 Subject: [PATCH 306/324] Reworked retry sections * Edit the map file directly instead of using --retrim and --try-again * Allows for more accurate pass status reporting * Allows for simpler pass break/continue logic * Create the map file before running ddrescue * Allows file to be edited by the current user instead of just root/ddrescue * Added check for empty map files * Avoids incorrectly marking a pass as complete --- scripts/wk/hw/ddrescue.py | 89 +++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index e54d2c60..5300a29a 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -140,20 +140,10 @@ class BlockPair(): else: # Cloning self.map_path = pathlib.Path(f'{working_dir}/Clone_{map_name}.map') - - # Read map file - self.load_map_data() + self.map_path.touch() # Set initial status - percent = self.get_percent_recovered() - for name in self.status.keys(): - if self.pass_complete(name): - self.status[name] = percent - else: - # Stop checking - if percent > 0: - self.status[name] = percent - break + self.set_initial_status() def get_percent_recovered(self): """Get percent rescued from map_data, returns float.""" @@ -197,14 +187,16 @@ class BlockPair(): ) data['pass completed'] = 'current status: finished' in line.lower() - # Check if 100% done - cmd = [ - 'ddrescuelog', - '--done-status', - self.map_path, - ] - proc = exe.run_program(cmd, check=False) - data['full recovery'] = proc.returncode == 0 + # Check if 100% done (only if map is present and non-zero size + # NOTE: ddrescuelog returns 0 (i.e. 100% done) for empty files + if self.map_path.exists() and self.map_path.stat().st_size != 0: + cmd = [ + 'ddrescuelog', + '--done-status', + self.map_path, + ] + proc = exe.run_program(cmd, check=False) + data['full recovery'] = proc.returncode == 0 # Done self.map_data.update(data) @@ -246,6 +238,19 @@ class BlockPair(): std.print_error(f'Invalid destination: {self.destination}') raise std.GenericAbort() + def set_initial_status(self): + """Read map data and set initial statuses.""" + self.load_map_data() + percent = self.get_percent_recovered() + for name in self.status.keys(): + if self.pass_complete(name): + self.status[name] = percent + else: + # Stop checking + if percent > 0: + self.status[name] = percent + break + def skip_pass(self, pass_name): """Mark pass as skipped if applicable.""" if self.status[pass_name] == 'Pending': @@ -783,11 +788,30 @@ class State(): self.save_settings(settings) def retry_all_passes(self): - """Set all statuses to Pending.""" + """Prep block_pairs for a retry recovery attempt.""" + bad_statuses = ('*', '/', '-') for pair in self.block_pairs: + map_data = [] + + # Reset status strings for name in pair.status.keys(): pair.status[name] = 'Pending' + # Mark all non-trimmed, non-scraped, and bad areas as non-tried + with open(pair.map_path, 'r') as _f: + for line in _f.readlines(): + line = line.strip() + if line.startswith('0x') and line.endswith(bad_statuses): + line = f'{line[:-1]}?' + map_data.append(line) + + # Save updated map + with open(pair.map_path, 'w') as _f: + _f.write('\n'.join(map_data)) + + # Reinitialize status + pair.set_initial_status() + def safety_check_destination(self): """Run safety checks for destination and abort if necessary.""" try: @@ -1382,15 +1406,11 @@ def fstype_is_ok(path, map_dir=False): return is_ok -def get_ddrescue_settings(main_menu, settings_menu): +def get_ddrescue_settings(settings_menu): """Get ddrescue settings from menu selections, returns list.""" settings = [] # Check menu selections - for name, details in main_menu.toggles.items(): - if 'Retry' in name and details['Selected']: - settings.append('--retrim') - settings.append('--try-again') for name, details in settings_menu.options.items(): if details['Selected']: if 'Value' in details: @@ -1783,7 +1803,10 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): for name, details in main_menu.toggles.items(): if 'Auto continue' in name and details['Selected']: auto_continue = True - settings = get_ddrescue_settings(main_menu, settings_menu) + if 'Retry' in name and details['Selected']: + details['Selected'] = False + state.retry_all_passes() + settings = get_ddrescue_settings(settings_menu) # Start SMART/Journal state.panes['SMART'] = tmux.split_window( @@ -1794,25 +1817,19 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True): lines=4, vertical=True, cmd='journalctl --dmesg --follow', ) - # Check if retrying - if '--retrim' in settings: - state.retry_all_passes() - # Run pass(es) for pass_name in ('read', 'trim', 'scrape'): abort = False - # Skip to next pass (unless retry selected) - if '--retrim' not in settings and state.pass_complete(pass_name): + # Skip to next pass + if state.pass_complete(pass_name): # NOTE: This bypasses auto_continue state.skip_pass(pass_name) continue # Run ddrescue for pair in state.block_pairs: - if '--retrim' not in settings and pair.pass_complete(pass_name): - pair.skip_pass(pass_name) - else: + if not pair.pass_complete(pass_name): attempted_recovery = True state.mark_started() try: From 64645cdf1f47edad0135679e53d17c10a43960fb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 18:06:57 -0700 Subject: [PATCH 307/324] Expanded logging (slightly) --- scripts/wk/hw/ddrescue.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 5300a29a..90e5075b 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -770,9 +770,11 @@ class State(): # Skip real format for dry runs if dry_run: + LOG.info('Dry run, refusing to format destination') return # Format disk + LOG.warning('Formatting destination: %s', self.destination.path) with open(script_path, 'r') as _f: proc = exe.run_program( cmd=['sudo', 'sfdisk', self.destination.path], @@ -790,6 +792,9 @@ class State(): def retry_all_passes(self): """Prep block_pairs for a retry recovery attempt.""" bad_statuses = ('*', '/', '-') + LOG.warning('Updating block_pairs for retry') + + # Update all block_pairs for pair in self.block_pairs: map_data = [] From 383b7c331a893071faa7b71d51656a1b2faa5d8e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 18:07:28 -0700 Subject: [PATCH 308/324] Safety wheels are off --- scripts/wk/hw/ddrescue.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 90e5075b..64b49989 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -1589,7 +1589,6 @@ def get_working_dir(mode, destination, force_local=False): def main(): """Main function for ddrescue TUI.""" args = docopt(DOCSTRING) - args['--dry-run'] = True # TODO: Remove dry-run safety net log.update_log_path(dest_name='ddrescue-TUI', timestamp=True) # Check if running inside tmux From 168c0a50df03a6b91912eeb4f3332db6db1e3de3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 18:18:39 -0700 Subject: [PATCH 309/324] Removed old ddrescue-tui launcher --- scripts/outer_scripts_to_review/ddrescue-tui | 11 ---- .../outer_scripts_to_review/ddrescue-tui-menu | 64 ------------------- 2 files changed, 75 deletions(-) delete mode 100755 scripts/outer_scripts_to_review/ddrescue-tui delete mode 100755 scripts/outer_scripts_to_review/ddrescue-tui-menu diff --git a/scripts/outer_scripts_to_review/ddrescue-tui b/scripts/outer_scripts_to_review/ddrescue-tui deleted file mode 100755 index 6ee8ad57..00000000 --- a/scripts/outer_scripts_to_review/ddrescue-tui +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -## Wizard Kit: ddrescue TUI Launcher - -source launch-in-tmux - -SESSION_NAME="ddrescue-tui" -WINDOW_NAME="ddrescue TUI" -TMUX_CMD="ddrescue-tui-menu" - -launch_in_tmux "$@" diff --git a/scripts/outer_scripts_to_review/ddrescue-tui-menu b/scripts/outer_scripts_to_review/ddrescue-tui-menu deleted file mode 100755 index eab8cd3f..00000000 --- a/scripts/outer_scripts_to_review/ddrescue-tui-menu +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: TUI for ddrescue cloning and imaging - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.ddrescue import * -from functions.hw_diags import * -init_global_vars() - -if __name__ == '__main__': - try: - # Prep - clear_screen() - args = list(sys.argv) - run_mode = '' - source_path = None - dest_path = None - - # Parse args - try: - script_name = os.path.basename(args.pop(0)) - run_mode = str(args.pop(0)).lower() - source_path = args.pop(0) - dest_path = args.pop(0) - except IndexError: - # We'll set the missing paths later - pass - - # Show usage - if re.search(r'-+(h|help)', str(sys.argv), re.IGNORECASE): - show_usage(script_name) - exit_script() - - # Start cloning/imaging - if run_mode in ('clone', 'image'): - menu_ddrescue(source_path, dest_path, run_mode) - else: - if not re.search(r'^-*(h|help\?)', run_mode, re.IGNORECASE): - print_error('Invalid mode.') - - # Done - print_standard('\nDone.') - pause("Press Enter to exit...") - tmux_switch_client() - exit_script() - except GenericAbort: - abort() - except GenericError as ge: - msg = 'Generic Error' - if str(ge): - msg = str(ge) - print_error(msg) - abort() - except SystemExit as sys_exit: - tmux_switch_client() - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 From b79deefdd66244a02c1d1e27e919b409d8a41a83 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 18:28:43 -0700 Subject: [PATCH 310/324] Fix map name when using loopback devices --- scripts/wk/hw/ddrescue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 64b49989..9695a715 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -120,7 +120,9 @@ class BlockPair(): # Set map file # e.g. '(Clone|Image)_Model[_p#]_Size[_Label].map' - map_name = model + map_name = model if model else 'None' + if source.details['bus'] == 'Image': + map_name = 'Image' if source.details['parent']: part_num = re.sub(r"^.*?(\d+)$", r"\1", source.path.name) map_name += f'_p{part_num}' From 72787d5c24aedfa876ecd5725f1ca4338d6ce020 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 18:42:55 -0700 Subject: [PATCH 311/324] Fix destination checks when imaging --- scripts/wk/hw/ddrescue.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/wk/hw/ddrescue.py b/scripts/wk/hw/ddrescue.py index 9695a715..abd63955 100644 --- a/scripts/wk/hw/ddrescue.py +++ b/scripts/wk/hw/ddrescue.py @@ -139,6 +139,7 @@ class BlockPair(): # Imaging self.map_path = pathlib.Path(f'{destination}/Image_{map_name}.map') self.destination = self.map_path.with_suffix('.dd') + self.destination.touch() else: # Cloning self.map_path = pathlib.Path(f'{working_dir}/Clone_{map_name}.map') @@ -235,8 +236,8 @@ class BlockPair(): dest_size = dest_obj.details['size'] del dest_obj - # Raise exception if necessary - if dest_size < self.size: + # Check destination size if cloning + if not self.destination.is_file() and dest_size < self.size: std.print_error(f'Invalid destination: {self.destination}') raise std.GenericAbort() From fdad48f6130e193f6538ba721b6d61c7270543ac Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 21:00:36 -0700 Subject: [PATCH 312/324] Fixed wk.std.color_string() --- scripts/wk/std.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index e98b9d5d..97c60d4a 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -743,6 +743,11 @@ def color_string(strings, colors, sep=' '): except TypeError: # Assuming single element passed, convert to string strings = (str(strings),) + try: + iter(colors) + except TypeError: + # Assuming single element passed, convert to string + colors = (str(colors),) # Build new string with color escapes added for string, color in itertools.zip_longest(strings, colors): From 945ae941fa7811e832c0db073c6466e5bdad3d02 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 21:01:41 -0700 Subject: [PATCH 313/324] Added mount-all-volumes sections * Still need to add the CoreStorage logic --- scripts/mount-all-volumes | 34 +++++++++++ scripts/wk/os/__init__.py | 3 +- scripts/wk/os/linux.py | 118 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100755 scripts/mount-all-volumes create mode 100644 scripts/wk/os/linux.py diff --git a/scripts/mount-all-volumes b/scripts/mount-all-volumes new file mode 100755 index 00000000..83da855f --- /dev/null +++ b/scripts/mount-all-volumes @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Wizard Kit: Mount all volumes""" +# vim: sts=2 sw=2 ts=2 + +import wk + + +# Functions +def main(): + """Mount all volumes and show results.""" + wk.std.print_standard(f'{wk.cfg.main.KIT_NAME_FULL}: Volume mount tool') + wk.std.print_standard(' ') + + # Mount volumes and get report + wk.std.print_standard('Mounting volumes...') + report = wk.os.linux.mount_volumes() + report = [f' {line}' for line in report] + + # Show results + wk.std.print_info('Results') + wk.std.print_report(report) + + +if __name__ == '__main__': + if wk.std.PLATFORM != 'Linux': + os_name = wk.std.PLATFORM.replace('Darwin', 'macOS') + wk.std.print_error(f'This script is not supported under {os_name}.') + wk.std.abort() + try: + main() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/os/__init__.py b/scripts/wk/os/__init__.py index e3d66798..1b9b21bd 100644 --- a/scripts/wk/os/__init__.py +++ b/scripts/wk/os/__init__.py @@ -4,6 +4,7 @@ import platform #if platform.system() == 'Darwin': -#if platform.system() == 'Linux': +if platform.system() == 'Linux': + from wk.os import linux if platform.system() == 'Windows': from wk.os import win diff --git a/scripts/wk/os/linux.py b/scripts/wk/os/linux.py new file mode 100644 index 00000000..8d6aea47 --- /dev/null +++ b/scripts/wk/os/linux.py @@ -0,0 +1,118 @@ +"""WizardKit: Linux Functions""" +# vim: sts=2 sw=2 ts=2 + +import logging + +from wk import std +from wk.exe import run_program +from wk.hw.obj import Disk + + +# STATIC VARIABLES +LOG = logging.getLogger(__name__) +UUID_CORESTORAGE = '53746f72-6167-11aa-aa11-00306543ecac' + + +# Functions +def mount_volumes(device_path=None, read_write=False, scan_corestorage=False): + """Mount all detected volumes, returns list. + + NOTE: If device_path is specified then only volumes + under that path will be mounted. + """ + report = [] + volumes = [] + containers = [] + + # Get list of volumes + cmd = [ + 'lsblk', + '--list', + '--noheadings', + '--output=name', + '--paths', + ] + if device_path: + cmd.append(device_path) + proc = run_program(cmd, check=False) + for line in sorted(proc.stdout.splitlines()): + volumes.append(Disk(line.strip())) + + # Get list of CoreStorage containers + containers = [ + vol for vol in volumes if vol.details.get('parttype', '') == UUID_CORESTORAGE + ] + + # Scan CoreStorage containers + if scan_corestorage: + if containers: + std.print_warning( + f'Detected CoreStorage container{"s" if len(containers) > 1 else ""}', + ) + std.print_standard('Scanning for inner volume(s)...') + for container in containers: + volumes.extend(scan_corestorage_container(container)) + + # Mount volumes + for vol in volumes: + already_mounted = vol.details.get('mountpoint', '') + result = f'{vol.details["name"].replace("/dev/mapper/", ""):<20}' + + # Parent devices + if vol.details.get('children', False): + if vol.details.get('fstype', ''): + result += vol.details['fstype'] + if vol.details.get('label', ''): + result += f' "{vol.details["label"]}"' + report.append(std.color_string(result, 'BLUE')) + continue + + # Attempt to mount volume + if not already_mounted: + cmd = [ + 'udevil', + 'mount', + '-o', 'rw' if read_write else 'ro', + vol.path, + ] + proc = run_program(cmd, check=False) + if proc.returncode: + result += 'Failed to mount' + report.append(std.color_string(result, 'RED')) + continue + + # Add size to result + vol.get_details() + vol.details['fsused'] = vol.details.get('fsused', -1) + vol.details['fsavail'] = vol.details.get('fsavail', -1) + result += f'{"Mounted on "+vol.details.get("mountpoint", "?"):<40}' + result = ( + f'{result} ({vol.details.get("fstype", "Unknown FS")+",":<5} ' + f'{std.bytes_to_string(vol.details["fsused"], decimals=1):>9} used, ' + f'{std.bytes_to_string(vol.details["fsavail"], decimals=1):>9} free)' + ) + report.append( + std.color_string( + result, + 'YELLOW' if already_mounted else None, + ), + ) + + # Done + return report + + +def scan_corestorage_container(container, timeout=300): + """Scan CoreStorage container for inner volumes, returns list.""" + inner_volumes = [] + + #TODO: Add testdisk logic to scan CoreStorage + if container or timeout: + pass + + # Done + return inner_volumes + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From b75326aeeea2bcab496f7e9f82b9d1c660b42e5e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 21:04:24 -0700 Subject: [PATCH 314/324] Added indent option to wk.std.print_report() --- scripts/wk/std.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/wk/std.py b/scripts/wk/std.py index 97c60d4a..ba44f18e 100644 --- a/scripts/wk/std.py +++ b/scripts/wk/std.py @@ -912,9 +912,11 @@ def print_info(msg, log=True, **kwargs): LOG.info(msg) -def print_report(report, log=True): +def print_report(report, indent=None, log=True): """Print report to screen and optionally to log.""" for line in report: + if indent: + line = f'{" "*indent}{line}' print(line) if log: LOG.info(strip_colors(line)) From 7bf03749ecb810b3917a3d26fb1b85b2fd063d47 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 21:35:42 -0700 Subject: [PATCH 315/324] Added CoreStorage scanning logic * Still needs tested --- scripts/mount-all-volumes | 3 +-- scripts/wk/os/linux.py | 52 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/scripts/mount-all-volumes b/scripts/mount-all-volumes index 83da855f..fb703011 100755 --- a/scripts/mount-all-volumes +++ b/scripts/mount-all-volumes @@ -14,11 +14,10 @@ def main(): # Mount volumes and get report wk.std.print_standard('Mounting volumes...') report = wk.os.linux.mount_volumes() - report = [f' {line}' for line in report] # Show results wk.std.print_info('Results') - wk.std.print_report(report) + wk.std.print_report(report, indent=2) if __name__ == '__main__': diff --git a/scripts/wk/os/linux.py b/scripts/wk/os/linux.py index 8d6aea47..14e6015b 100644 --- a/scripts/wk/os/linux.py +++ b/scripts/wk/os/linux.py @@ -2,9 +2,12 @@ # vim: sts=2 sw=2 ts=2 import logging +import pathlib +import re +import subprocess from wk import std -from wk.exe import run_program +from wk.exe import popen_program, run_program from wk.hw.obj import Disk @@ -14,6 +17,12 @@ UUID_CORESTORAGE = '53746f72-6167-11aa-aa11-00306543ecac' # Functions +def make_temp_file(): + """Make temporary file, returns pathlib.Path() obj.""" + proc = run_program(['mktemp'], check=False) + return pathlib.Path(proc.stdout.strip()) + + def mount_volumes(device_path=None, read_write=False, scan_corestorage=False): """Mount all detected volumes, returns list. @@ -104,11 +113,46 @@ def mount_volumes(device_path=None, read_write=False, scan_corestorage=False): def scan_corestorage_container(container, timeout=300): """Scan CoreStorage container for inner volumes, returns list.""" + # TODO: Test Scanning CoreStorage containers + detected_volumes = {} inner_volumes = [] + log_path = make_temp_file() - #TODO: Add testdisk logic to scan CoreStorage - if container or timeout: - pass + # Run scan via TestDisk + cmd = [ + 'sudo', 'testdisk', + '/logname', log_path, + '/debug', + '/log', + '/cmd', container.path, 'partition_none,analyze', + ] + proc = popen_program(cmd) + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # Failed to find any volumes, stop scan + run_program(['sudo', 'kill', proc.pid], check=False) + + # Check results + if proc.returncode == 0 and log_path.exists(): + results = log_path.read_text(encoding='utf-8', errors='ignore') + for line in results.splitlines(): + line = line.lower().strip() + match = re.match(r'^.*echo "([^"]+)" . dmsetup create test(\d)$', line) + if match: + cs_name = f'CoreStorage_{container.path.name}_{match.group(2)}' + detected_volumes[cs_name] = match.group(1) + + # Create mapper device(s) if necessary + for name, cmd in detected_volumes.items(): + cmd_file = make_temp_file() + cmd_file.write_text(cmd) + proc = run_program( + cmd=['sudo', 'dmsetup', 'create', name, cmd_file], + check=False, + ) + if proc.returncode == 0: + inner_volumes.append(Disk(f'/dev/mapper/{name}')) # Done return inner_volumes From 703783406a8e207552fe6de0f9594d11f45f605c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 4 Jan 2020 21:36:44 -0700 Subject: [PATCH 316/324] Removed old mount-all-volumes script --- .../outer_scripts_to_review/mount-all-volumes | 38 ------------------- 1 file changed, 38 deletions(-) delete mode 100755 scripts/outer_scripts_to_review/mount-all-volumes diff --git a/scripts/outer_scripts_to_review/mount-all-volumes b/scripts/outer_scripts_to_review/mount-all-volumes deleted file mode 100755 index 5b34c579..00000000 --- a/scripts/outer_scripts_to_review/mount-all-volumes +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: Volume mount tool - -import os -import sys - -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from functions.data import * -init_global_vars() - -if __name__ == '__main__': - try: - # Prep - clear_screen() - print_standard('{}: Volume mount tool'.format(KIT_NAME_FULL)) - - # Mount volumes - report = mount_volumes(all_devices=True) - - # Print report - print_info('\nResults') - for vol_name, vol_data in sorted(report.items()): - show_data(indent=4, width=20, **vol_data['show_data']) - - # Done - print_standard('\nDone.') - if 'gui' in sys.argv: - pause("Press Enter to exit...") - popen_program(['nohup', 'thunar', '/media'], pipe=True) - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 From 8f31e5bd67538669819acf8ba824f311ef64af6a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 6 Jan 2020 20:26:57 -0700 Subject: [PATCH 317/324] Added I/O functions for building UFDs --- scripts/wk.prev/functions/ufd.py | 105 ------------------------------- scripts/wk/io.py | 99 +++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 105 deletions(-) diff --git a/scripts/wk.prev/functions/ufd.py b/scripts/wk.prev/functions/ufd.py index 32f08201..ae2e27ca 100644 --- a/scripts/wk.prev/functions/ufd.py +++ b/scripts/wk.prev/functions/ufd.py @@ -10,35 +10,6 @@ from collections import OrderedDict from functions.common import * -def case_insensitive_search(path, item): - """Search path for item case insensitively, returns str.""" - regex_match = '^{}$'.format(item) - real_path = '' - - # Quick check first - if os.path.exists('{}/{}'.format(path, item)): - real_path = '{}{}{}'.format( - path, - '' if path == '/' else '/', - item, - ) - - # Check all items in dir - for entry in os.scandir(path): - if re.match(regex_match, entry.name, re.IGNORECASE): - real_path = '{}{}{}'.format( - path, - '' if path == '/' else '/', - entry.name, - ) - - # Done - if not real_path: - raise FileNotFoundError('{}/{}'.format(path, item)) - - return real_path - - def confirm_selections(args): """Ask tech to confirm selections, twice if necessary.""" if not ask('Is the above information correct?'): @@ -100,33 +71,6 @@ def find_first_partition(dev_path): return part_path -def find_path(path): - """Find path case-insensitively, returns pathlib.Path obj.""" - path_obj = pathlib.Path(path).resolve() - - # Quick check first - if path_obj.exists(): - return path_obj - - # Fix case - parts = path_obj.relative_to('/').parts - real_path = '/' - for part in parts: - try: - real_path = case_insensitive_search(real_path, part) - except NotADirectoryError: - # Reclassify error - raise FileNotFoundError(path) - - # Raise error if path doesn't exist - path_obj = pathlib.Path(real_path) - if not path_obj.exists(): - raise FileNotFoundError(path_obj) - - # Done - return path_obj - - def get_user_home(user): """Get path to user's home dir, returns str.""" home_dir = None @@ -279,55 +223,6 @@ def prep_device(dev_path, label, use_mbr=False, indent=2): ) -def recursive_copy(source, dest, overwrite=False): - """Copy source to dest recursively. - - NOTE: This uses rsync style source/dest syntax. - If the source has a trailing slash then it's contents are copied, - otherwise the source itself is copied. - - Examples assuming "ExDir/ExFile.txt" exists: - recursive_copy("ExDir", "Dest/") results in "Dest/ExDir/ExFile.txt" - recursive_copy("ExDir/", "Dest/") results in "Dest/ExFile.txt" - - NOTE 2: dest does not use find_path because it might not exist. - """ - copy_contents = source.endswith('/') - source = find_path(source) - dest = pathlib.Path(dest).resolve().joinpath(source.name) - os.makedirs(dest.parent, exist_ok=True) - - if source.is_dir(): - if copy_contents: - # Trailing slash syntax - for item in os.scandir(source): - recursive_copy(item.path, dest.parent, overwrite=overwrite) - elif not dest.exists(): - # No conflict, copying whole tree (no merging needed) - shutil.copytree(source, dest) - elif not dest.is_dir(): - # Refusing to replace file with dir - raise FileExistsError('Refusing to replace file: {}'.format(dest)) - else: - # Dest exists and is a dir, merge dirs - for item in os.scandir(source): - recursive_copy(item.path, dest, overwrite=overwrite) - elif source.is_file(): - if not dest.exists(): - # No conflict, copying file - shutil.copy2(source, dest) - elif not dest.is_file(): - # Refusing to replace dir with file - raise FileExistsError('Refusing to replace dir: {}'.format(dest)) - elif overwrite: - # Dest file exists, deleting and replacing file - os.remove(dest) - shutil.copy2(source, dest) - else: - # Refusing to delete file when overwrite=False - raise FileExistsError('Refusing to delete file: {}'.format(dest)) - - def remove_arch(): """Remove arch dir from UFD. diff --git a/scripts/wk/io.py b/scripts/wk/io.py index bbaa9532..a29ccee6 100644 --- a/scripts/wk/io.py +++ b/scripts/wk/io.py @@ -4,6 +4,7 @@ import logging import os import pathlib +import re import shutil @@ -12,6 +13,54 @@ LOG = logging.getLogger(__name__) # Functions +def case_insensitive_path(path): + """Find path case-insensitively, returns pathlib.Path obj.""" + given_path = pathlib.Path(path).resolve() + real_path = None + + # Quick check + if given_path.exists(): + return given_path + + # Search for real path + parts = list(given_path.parts) + real_path = parts.pop(0) + for part in parts: + try: + real_path = case_insensitive_search(real_path, part) + except NotADirectoryError: + # Reclassify error + raise FileNotFoundError(given_path) + real_path = pathlib.Path(real_path) + + # Done + return real_path + + +def case_insensitive_search(path, item): + """Search path for item case insensitively, returns pathlib.Path obj.""" + path = pathlib.Path(path).resolve() + given_path = path.joinpath(item) + real_path = None + regex = fr'^{item}' + + # Quick check + if given_path.exists(): + return given_path + + # Check all items in path + for entry in os.scandir(path): + if re.match(regex, entry.name, re.IGNORECASE): + real_path = path.joinpath(entry.name) + + # Raise exception if necessary + if not real_path: + raise FileNotFoundError(given_path) + + # Done + return real_path + + def delete_empty_folders(path): """Recursively delete all empty folders in path.""" LOG.debug('path: %s', path) @@ -93,5 +142,55 @@ def non_clobber_path(path): return new_path +def recursive_copy(source, dest, overwrite=False): + """Copy source to dest recursively. + + NOTE: This uses rsync style source/dest syntax. + If the source has a trailing slash then it's contents are copied, + otherwise the source itself is copied. + + Examples assuming "ExDir/ExFile.txt" exists: + recursive_copy("ExDir", "Dest/") results in "Dest/ExDir/ExFile.txt" + recursive_copy("ExDir/", "Dest/") results in "Dest/ExFile.txt" + + NOTE 2: dest does not use find_path because it might not exist. + """ + copy_contents = str(source).endswith(('/', '\\')) + source = case_insensitive_path(source) + dest = pathlib.Path(dest).resolve().joinpath(source.name) + os.makedirs(dest.parent, exist_ok=True) + + # Recursively copy source to dest + if source.is_dir(): + if copy_contents: + # Trailing slash syntax + for item in os.scandir(source): + recursive_copy(item.path, dest.parent, overwrite=overwrite) + elif not dest.exists(): + # No conflict, copying whole tree (no merging needed) + shutil.copytree(source, dest) + elif not dest.is_dir(): + # Refusing to replace file with dir + raise FileExistsError(f'Refusing to replace file: {dest}') + else: + # Dest exists and is a dir, merge dirs + for item in os.scandir(source): + recursive_copy(item.path, dest, overwrite=overwrite) + elif source.is_file(): + if not dest.exists(): + # No conflict, copying file + shutil.copy2(source, dest) + elif not dest.is_file(): + # Refusing to replace dir with file + raise FileExistsError(f'Refusing to replace dir: {dest}') + elif overwrite: + # Dest file exists, deleting and replacing file + os.remove(dest) + shutil.copy2(source, dest) + else: + # Refusing to delete file when overwrite=False + raise FileExistsError(f'Refusing to delete file: {dest}') + + if __name__ == '__main__': print("This file is not meant to be called directly.") From c135d686df207ce09b9dc8a227ece96d584c5184 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 6 Jan 2020 20:27:59 -0700 Subject: [PATCH 318/324] Added Linux functions for building UFDs --- scripts/wk.prev/functions/ufd.py | 30 ---------------------- scripts/wk/os/linux.py | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/scripts/wk.prev/functions/ufd.py b/scripts/wk.prev/functions/ufd.py index ae2e27ca..b5e8cf01 100644 --- a/scripts/wk.prev/functions/ufd.py +++ b/scripts/wk.prev/functions/ufd.py @@ -71,31 +71,6 @@ def find_first_partition(dev_path): return part_path -def get_user_home(user): - """Get path to user's home dir, returns str.""" - home_dir = None - cmd = ['getent', 'passwd', user] - result = run_program(cmd, encoding='utf-8', errors='ignore', check=False) - try: - home_dir = result.stdout.split(':')[5] - except Exception: - # Just use HOME from ENV (or '/root' if that fails) - home_dir = os.environ.get('HOME', '/root') - - return home_dir - - -def get_user_name(): - """Get real user name, returns str.""" - user = None - if 'SUDO_USER' in os.environ: - user = os.environ.get('SUDO_USER', 'Unknown') - else: - user = os.environ.get('USER', 'Unknown') - - return user - - def hide_items(ufd_dev, items): """Set FAT32 hidden flag for items.""" # pylint: disable=invalid-name @@ -231,11 +206,6 @@ def remove_arch(): shutil.rmtree(find_path('/mnt/UFD/arch')) -def running_as_root(): - """Check if running with effective UID of 0, returns bool.""" - return os.geteuid() == 0 - - def show_selections(args, sources, ufd_dev, ufd_sources): """Show selections including non-specified options.""" diff --git a/scripts/wk/os/linux.py b/scripts/wk/os/linux.py index 14e6015b..c089ceb8 100644 --- a/scripts/wk/os/linux.py +++ b/scripts/wk/os/linux.py @@ -2,6 +2,7 @@ # vim: sts=2 sw=2 ts=2 import logging +import os import pathlib import re import subprocess @@ -17,6 +18,44 @@ UUID_CORESTORAGE = '53746f72-6167-11aa-aa11-00306543ecac' # Functions +def get_user_home(user): + """Get path to user's home dir, returns pathlib.Path obj.""" + home = None + + # Get path from user details + cmd = ['getent', 'passwd', user] + proc = run_program(cmd, check=False) + try: + home = proc.stdout.split(':')[5] + except IndexError: + # Try using environment variable + home = os.environ.get('HOME') + + # Raise exception if necessary + if not home: + raise RuntimeError(f'Failed to find home for: {user}') + + # Done + return pathlib.Path(home) + + +def get_user_name(): + """Get real user name, returns str.""" + user = None + + # Query environment + user = os.environ.get('SUDO_USER') + if not user: + user = os.environ.get('USER') + + # Raise exception if necessary + if not user: + raise RuntimeError('Failed to determine user') + + # Done + return user + + def make_temp_file(): """Make temporary file, returns pathlib.Path() obj.""" proc = run_program(['mktemp'], check=False) @@ -111,6 +150,11 @@ def mount_volumes(device_path=None, read_write=False, scan_corestorage=False): return report +def running_as_root(): + """Check if running with effective UID of 0, returns bool.""" + return os.geteuid() == 0 + + def scan_corestorage_container(container, timeout=300): """Scan CoreStorage container for inner volumes, returns list.""" # TODO: Test Scanning CoreStorage containers From b0b0b612a103d2ffae512857794652260801b303 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 6 Jan 2020 20:58:46 -0700 Subject: [PATCH 319/324] Added Linux mount and unmount functions * If not running with root priviledges then udevil is used. --- scripts/wk.prev/functions/ufd.py | 19 ------------- scripts/wk/os/linux.py | 46 +++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/scripts/wk.prev/functions/ufd.py b/scripts/wk.prev/functions/ufd.py index b5e8cf01..ceae0b75 100644 --- a/scripts/wk.prev/functions/ufd.py +++ b/scripts/wk.prev/functions/ufd.py @@ -128,19 +128,6 @@ def is_valid_path(path_obj, path_type): return valid_path -def mount(mount_source, mount_point, read_write=False): - """Mount mount_source on mount_point.""" - os.makedirs(mount_point, exist_ok=True) - cmd = [ - 'mount', - mount_source, - mount_point, - '-o', - 'rw' if read_write else 'ro', - ] - run_program(cmd) - - def prep_device(dev_path, label, use_mbr=False, indent=2): """Format device in preparation for applying the WizardKit components @@ -250,12 +237,6 @@ def show_selections(args, sources, ufd_dev, ufd_sources): print_standard(' ') -def unmount(mount_point): - """Unmount mount_point.""" - cmd = ['umount', mount_point] - run_program(cmd) - - def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): """Update boot files for UFD usage""" configs = [] diff --git a/scripts/wk/os/linux.py b/scripts/wk/os/linux.py index c089ceb8..253b1dd0 100644 --- a/scripts/wk/os/linux.py +++ b/scripts/wk/os/linux.py @@ -62,6 +62,27 @@ def make_temp_file(): return pathlib.Path(proc.stdout.strip()) +def mount(source, mount_point=None, read_write=False): + """Mount source (on mount_point if provided). + + NOTE: If not running_as_root() then udevil will be used. + """ + cmd = [ + 'mount', + '-o', 'rw' if read_write else 'ro', + source, + ] + if not running_as_root(): + cmd.insert(0, 'udevil') + if mount_point: + cmd.append(mount_point) + + # Run mount command + proc = run_program(cmd, check=False) + if not proc.returncode == 0: + raise RuntimeError(f'Failed to mount: {source} on {mount_point}') + + def mount_volumes(device_path=None, read_write=False, scan_corestorage=False): """Mount all detected volumes, returns list. @@ -117,12 +138,7 @@ def mount_volumes(device_path=None, read_write=False, scan_corestorage=False): # Attempt to mount volume if not already_mounted: - cmd = [ - 'udevil', - 'mount', - '-o', 'rw' if read_write else 'ro', - vol.path, - ] + mount(vol.path, read_write=read_write) proc = run_program(cmd, check=False) if proc.returncode: result += 'Failed to mount' @@ -202,5 +218,23 @@ def scan_corestorage_container(container, timeout=300): return inner_volumes +def unmount(source_or_mountpoint): + """Unmount source_or_mountpoint. + + NOTE: If not running_as_root() then udevil will be used. + """ + cmd = [ + 'umount', + source_or_mountpoint, + ] + if not running_as_root(): + cmd.insert(0, 'udevil') + + # Run unmount command + proc = run_program(cmd, check=False) + if not proc.returncode == 0: + raise RuntimeError(f'Failed to unmount: {source_or_mountpoint}') + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 142ad75744aaa718fb1d94bba2eff2c783279dd3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 7 Jan 2020 21:58:04 -0700 Subject: [PATCH 320/324] Added remaining UFD functions --- .../{outer_scripts_to_review => }/build-ufd | 2 +- scripts/wk/kit/__init__.py | 6 + scripts/{wk.prev/functions => wk/kit}/ufd.py | 206 ++++++++++-------- 3 files changed, 119 insertions(+), 95 deletions(-) rename scripts/{outer_scripts_to_review => }/build-ufd (98%) rename scripts/{wk.prev/functions => wk/kit}/ufd.py (59%) diff --git a/scripts/outer_scripts_to_review/build-ufd b/scripts/build-ufd similarity index 98% rename from scripts/outer_scripts_to_review/build-ufd rename to scripts/build-ufd index 45c3ff35..04958a83 100755 --- a/scripts/outer_scripts_to_review/build-ufd +++ b/scripts/build-ufd @@ -51,7 +51,7 @@ if __name__ == '__main__': sources = verify_sources(args, UFD_SOURCES) show_selections(args, sources, ufd_dev, UFD_SOURCES) if not args['--force']: - confirm_selections(args) + confirm_selections(update=args['--update']) # Prep UFD if not args['--update']: diff --git a/scripts/wk/kit/__init__.py b/scripts/wk/kit/__init__.py index 2fc43258..0869d9c6 100644 --- a/scripts/wk/kit/__init__.py +++ b/scripts/wk/kit/__init__.py @@ -1 +1,7 @@ """WizardKit: kit module init""" +# vim: sts=2 sw=2 ts=2 + +import platform + +if platform.system() == 'Linux': + from wk.kit import ufd diff --git a/scripts/wk.prev/functions/ufd.py b/scripts/wk/kit/ufd.py similarity index 59% rename from scripts/wk.prev/functions/ufd.py rename to scripts/wk/kit/ufd.py index ceae0b75..0d83b3d0 100644 --- a/scripts/wk.prev/functions/ufd.py +++ b/scripts/wk/kit/ufd.py @@ -1,32 +1,43 @@ -"""Wizard Kit: Functions - UFD""" -# pylint: disable=broad-except,wildcard-import +"""WizardKit: UFD Functions""" # vim: sts=2 sw=2 ts=2 +# TODO: Replace some lsblk usage with hw_obj? +# TODO: Needs testing +import logging import os -import re import shutil -import pathlib + from collections import OrderedDict -from functions.common import * + +from wk import io, std +from wk.exe import run_program +from wk.os import linux -def confirm_selections(args): +# STATIC VARIABLES +LOG = logging.getLogger(__name__) + + +# Functions +def confirm_selections(update=False): """Ask tech to confirm selections, twice if necessary.""" - if not ask('Is the above information correct?'): - abort(False) - ## Safety check - if not args['--update']: - print_standard(' ') - print_warning('SAFETY CHECK') - print_standard( - 'All data will be DELETED from the disk and partition(s) listed above.') - print_standard( - 'This is irreversible and will lead to {RED}DATA LOSS.{CLEAR}'.format( - **COLORS)) - if not ask('Asking again to confirm, is this correct?'): - abort(False) + if not std.ask('Is the above information correct?'): + std.abort() - print_standard(' ') + # Safety check + if not update: + std.print_standard(' ') + std.print_warning('SAFETY CHECK') + std.print_standard( + 'All data will be DELETED from the disk and partition(s) listed above.') + std.print_colored( + ['This is irreversible and will lead to', 'DATA LOSS'], + [None, 'RED'], + ) + if not std.ask('Asking again to confirm, is this correct?'): + std.abort() + + std.print_standard(' ') def copy_source(source, items, overwrite=False): @@ -35,28 +46,29 @@ def copy_source(source, items, overwrite=False): # Mount source if necessary if is_image: - mount(source, '/mnt/Source') + linux.mount(source, '/mnt/Source') # Copy items for i_source, i_dest in items: - i_source = '{}{}'.format( - '/mnt/Source' if is_image else source, - i_source, - ) - i_dest = '/mnt/UFD{}'.format(i_dest) + i_source = f'{"/mnt/Source" if is_image else source}{i_source}' + i_dest = f'/mnt/UFD{i_dest}' try: - recursive_copy(i_source, i_dest, overwrite=overwrite) + io.recursive_copy(i_source, i_dest, overwrite=overwrite) except FileNotFoundError: # Going to assume (hope) that this is fine pass # Unmount source if necessary if is_image: - unmount('/mnt/Source') + linux.unmount('/mnt/Source') def find_first_partition(dev_path): - """Find path to first partition of dev, returns str.""" + """Find path to first partition of dev, returns str. + + NOTE: This assumes the dev was just partitioned with + a single partition. + """ cmd = [ 'lsblk', '--list', @@ -65,23 +77,25 @@ def find_first_partition(dev_path): '--paths', dev_path, ] - result = run_program(cmd, encoding='utf-8', errors='ignore') - part_path = result.stdout.splitlines()[-1].strip() + # Run cmd + proc = run_program(cmd) + part_path = proc.stdout.splitlines()[-1].strip() + + # Done return part_path def hide_items(ufd_dev, items): """Set FAT32 hidden flag for items.""" - # pylint: disable=invalid-name - with open('/root/.mtoolsrc', 'w') as f: - f.write('drive U: file="{}"\n'.format( - find_first_partition(ufd_dev))) - f.write('mtools_skip_check=1\n') + first_partition = find_first_partition(ufd_dev) + with open('/root/.mtoolsrc', 'w') as _f: + _f.write(f'drive U: file="{first_partition}"\n') + _f.write('mtools_skip_check=1\n') # Hide items for item in items: - cmd = ['yes | mattrib +h "U:/{}"'.format(item)] + cmd = [f'yes | mattrib +h "U:/{item}"'] run_program(cmd, check=False, shell=True) @@ -91,10 +105,8 @@ def install_syslinux_to_dev(ufd_dev, use_mbr): 'dd', 'bs=440', 'count=1', - 'if=/usr/lib/syslinux/bios/{}.bin'.format( - 'mbr' if use_mbr else 'gptmbr', - ), - 'of={}'.format(ufd_dev), + f'if=/usr/lib/syslinux/bios/{"mbr" if use_mbr else "gptmbr"}.bin', + f'of={ufd_dev}', ] run_program(cmd) @@ -128,7 +140,7 @@ def is_valid_path(path_obj, path_type): return valid_path -def prep_device(dev_path, label, use_mbr=False, indent=2): +def prep_device(dev_path, label, use_mbr=False): """Format device in preparation for applying the WizardKit components This is done is four steps: @@ -137,35 +149,44 @@ def prep_device(dev_path, label, use_mbr=False, indent=2): 3. Set boot flag 4. Format partition (FAT32, 4K aligned) """ + try_print = std.TryAndPrint() + # Zero-out first 64MB - cmd = 'dd bs=4M count=16 if=/dev/zero of={}'.format(dev_path).split() - try_and_print( - indent=indent, - message='Zeroing first 64MB...', + cmd = [ + 'dd', + 'bs=4M', + 'count=16', + 'if=/dev/zero', + f'of={dev_path}', + ] + try_print.run( + message='Zeroing first 64MiB...', function=run_program, cmd=cmd, ) # Create partition table - cmd = 'parted {} --script -- mklabel {} mkpart primary fat32 4MiB {}'.format( - dev_path, - 'msdos' if use_mbr else 'gpt', + cmd = [ + 'parted', dev_path, + '--script', + '--', + 'mklabel', 'msdos' if use_mbr else 'gpt', '-1s' if use_mbr else '-4MiB', - ).split() - try_and_print( - indent=indent, + ] + try_print.run( message='Creating partition table...', function=run_program, cmd=cmd, ) # Set boot flag - cmd = 'parted {} set 1 {} on'.format( - dev_path, + cmd = [ + 'parted', dev_path, + 'set', '1', 'boot' if use_mbr else 'legacy_boot', - ).split() - try_and_print( - indent=indent, + 'on', + ] + try_print.run( message='Setting boot flag...', function=run_program, cmd=cmd, @@ -173,12 +194,12 @@ def prep_device(dev_path, label, use_mbr=False, indent=2): # Format partition cmd = [ - 'mkfs.vfat', '-F', '32', + 'mkfs.vfat', + '-F', '32', '-n', label, find_first_partition(dev_path), ] - try_and_print( - indent=indent, + try_print.run( message='Formatting partition...', function=run_program, cmd=cmd, @@ -190,51 +211,48 @@ def remove_arch(): This ensures a clean installation to the UFD and resets the boot files """ - shutil.rmtree(find_path('/mnt/UFD/arch')) + shutil.rmtree(io.case_insensitive_path('/mnt/UFD/arch')) def show_selections(args, sources, ufd_dev, ufd_sources): """Show selections including non-specified options.""" # Sources - print_info('Sources') + std.print_info('Sources') for label in ufd_sources.keys(): if label in sources: - print_standard(' {label:<18} {path}'.format( - label=label+':', - path=sources[label], - )) + std.print_standard(f' {label+":":<18} {sources["label"]}') else: - print_standard(' {label:<18} {YELLOW}Not Specified{CLEAR}'.format( - label=label+':', - **COLORS, - )) - print_standard(' ') + std.print_colored( + [f' {label+":":<18}', 'Not Specified'], + [None, 'YELLOW'], + ) + std.print_standard(' ') # Destination - print_info('Destination') + std.print_info('Destination') cmd = [ 'lsblk', '--nodeps', '--noheadings', '--paths', '--output', 'NAME,FSTYPE,TRAN,SIZE,VENDOR,MODEL,SERIAL', ufd_dev, ] - result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') - print_standard(result.stdout.strip()) + proc = run_program(cmd, check=False) + std.print_standard(proc.stdout.strip()) cmd = [ 'lsblk', '--noheadings', '--paths', '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', ufd_dev, ] - result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') - for line in result.stdout.splitlines()[1:]: - print_standard(line) + proc = run_program(cmd, check=False) + for line in proc.stdout.splitlines()[1:]: + std.print_standard(line) # Notes if args['--update']: - print_warning('Updating kit in-place') + std.print_warning('Updating kit in-place') elif args['--use-mbr']: - print_warning('Formatting using legacy MBR') - print_standard(' ') + std.print_warning('Formatting using legacy MBR') + std.print_standard(' ') def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): @@ -243,7 +261,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): # Find config files for c_path, c_ext in boot_files.items(): - c_path = find_path('/mnt/UFD{}'.format(c_path)) + c_path = io.case_insensitive_path('/mnt/UFD{c_path}') for item in os.scandir(c_path): if item.name.lower().endswith(c_ext.lower()): configs.append(item.path) @@ -253,7 +271,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): 'sed', '--in-place', '--regexp-extended', - 's/{}/{}/'.format(iso_label, ufd_label), + f's/{iso_label}/{ufd_label}/', *configs, ] run_program(cmd) @@ -261,7 +279,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): # Uncomment extra entries if present for b_path, b_comment in boot_entries.items(): try: - find_path('/mnt/UFD{}'.format(b_path)) + io.case_insensitive_path(f'/mnt/UFD{b_path}') except (FileNotFoundError, NotADirectoryError): # Entry not found, continue to next entry continue @@ -270,7 +288,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): cmd = [ 'sed', '--in-place', - 's/#{}#//'.format(b_comment), + f's/#{b_comment}#//', *configs, ] run_program(cmd, check=False) @@ -284,13 +302,13 @@ def verify_sources(args, ufd_sources): s_path = args[data['Arg']] if s_path: try: - s_path_obj = find_path(s_path) + s_path_obj = io.case_insensitive_path(s_path) except FileNotFoundError: - print_error('ERROR: {} not found: {}'.format(label, s_path)) - abort(False) + std.print_error(f'ERROR: {label} not found: {s_path}') + std.abort() if not is_valid_path(s_path_obj, data['Type']): - print_error('ERROR: Invalid {} source: {}'.format(label, s_path)) - abort(False) + std.print_error(f'ERROR: Invalid {label} source: {s_path}') + std.abort() sources[label] = s_path_obj return sources @@ -301,14 +319,14 @@ def verify_ufd(dev_path): ufd_dev = None try: - ufd_dev = find_path(dev_path) + ufd_dev = io.case_insensitive_path(dev_path) except FileNotFoundError: - print_error('ERROR: UFD device not found: {}'.format(dev_path)) - abort(False) + std.print_error(f'ERROR: UFD device not found: {dev_path}') + std.abort() if not is_valid_path(ufd_dev, 'UFD'): - print_error('ERROR: Invalid UFD device: {}'.format(ufd_dev)) - abort(False) + std.print_error(f'ERROR: Invalid UFD device: {ufd_dev}') + std.abort() return ufd_dev From 8b9672313aeb697fec22a8b287a7668b7e2d2ef4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 7 Jan 2020 23:21:18 -0700 Subject: [PATCH 321/324] Added ufd settings --- scripts/wk/cfg/__init__.py | 1 + scripts/{wk.prev/settings => wk/cfg}/ufd.py | 52 ++++----------------- scripts/wk/kit/ufd.py | 28 +++++++++++ 3 files changed, 39 insertions(+), 42 deletions(-) rename scripts/{wk.prev/settings => wk/cfg}/ufd.py (61%) diff --git a/scripts/wk/cfg/__init__.py b/scripts/wk/cfg/__init__.py index d86f7245..23ca608f 100644 --- a/scripts/wk/cfg/__init__.py +++ b/scripts/wk/cfg/__init__.py @@ -5,3 +5,4 @@ from wk.cfg import hw from wk.cfg import log from wk.cfg import main from wk.cfg import net +from wk.cfg import ufd diff --git a/scripts/wk.prev/settings/ufd.py b/scripts/wk/cfg/ufd.py similarity index 61% rename from scripts/wk.prev/settings/ufd.py rename to scripts/wk/cfg/ufd.py index b157e392..a22f0db7 100644 --- a/scripts/wk.prev/settings/ufd.py +++ b/scripts/wk/cfg/ufd.py @@ -1,40 +1,15 @@ -'''Wizard Kit: Settings - UFD''' -# pylint: disable=C0326,E0611 +"""WizardKit: Config - UFD""" +# pylint: disable=bad-whitespace # vim: sts=2 sw=2 ts=2 from collections import OrderedDict -from settings.main import KIT_NAME_FULL,KIT_NAME_SHORT + +from wk.cfg.main import KIT_NAME_FULL + # General -DOCSTRING = '''WizardKit: Build UFD - -Usage: - build-ufd [options] --ufd-device PATH --linux PATH - [--linux-minimal PATH] - [--main-kit PATH] - [--winpe PATH] - [--extra-dir PATH] - build-ufd (-h | --help) - -Options: - -d PATH, --linux-dgpu PATH - -e PATH, --extra-dir PATH - -k PATH, --main-kit PATH - -l PATH, --linux PATH - -m PATH, --linux-minimal PATH - -u PATH, --ufd-device PATH - -w PATH, --winpe PATH - - -h --help Show this page - -M --use-mbr Use real MBR instead of GPT w/ Protective MBR - -F --force Bypass all confirmation messages. USE WITH EXTREME CAUTION! - -U --update Don't format device, just update -''' -ISO_LABEL = '{}_LINUX'.format(KIT_NAME_SHORT) -UFD_LABEL = '{}_UFD'.format(KIT_NAME_SHORT) -UFD_SOURCES = OrderedDict({ +SOURCES = OrderedDict({ 'Linux': {'Arg': '--linux', 'Type': 'ISO'}, - 'Linux (dGPU)': {'Arg': '--linux-dgpu', 'Type': 'ISO'}, 'Linux (Minimal)': {'Arg': '--linux-minimal', 'Type': 'ISO'}, 'WinPE': {'Arg': '--winpe', 'Type': 'ISO'}, 'Main Kit': {'Arg': '--main-kit', 'Type': 'KIT'}, @@ -45,7 +20,6 @@ UFD_SOURCES = OrderedDict({ BOOT_ENTRIES = { # Path to check: Comment to remove '/arch_minimal': 'UFD-MINIMAL', - '/dgpu': 'UFD-DGPU', '/sources/boot.wim': 'UFD-WINPE', } BOOT_FILES = { @@ -67,12 +41,6 @@ ITEMS = { ('/EFI/boot', '/EFI/'), ('/EFI/memtest86', '/EFI/'), ), - 'Linux (dGPU)': ( - ('/arch/boot/x86_64/archiso.img', '/dgpu/'), - ('/arch/boot/x86_64/vmlinuz', '/dgpu/'), - ('/arch/pkglist.x86_64.txt', '/dgpu/'), - ('/arch/x86_64', '/dgpu/'), - ), 'Linux (Minimal)': ( ('/arch/boot/x86_64/archiso.img', '/arch_minimal/'), ('/arch/boot/x86_64/vmlinuz', '/arch_minimal/'), @@ -80,7 +48,7 @@ ITEMS = { ('/arch/x86_64', '/arch_minimal/'), ), 'Main Kit': ( - ('/', '/{}/'.format(KIT_NAME_FULL)), + ('/', f'/{KIT_NAME_FULL}/'), ), 'WinPE': ( ('/bootmgr', '/'), @@ -99,12 +67,11 @@ ITEMS_HIDDEN = ( # Linux (all versions) 'arch', 'arch_minimal', - 'dgpu', 'EFI', 'isolinux', # Main Kit - '{}/.bin'.format(KIT_NAME_FULL), - '{}/.cbin'.format(KIT_NAME_FULL), + f'{KIT_NAME_FULL}/.bin', + f'{KIT_NAME_FULL}/.cbin', # WinPE 'boot', 'bootmgr', @@ -114,5 +81,6 @@ ITEMS_HIDDEN = ( 'sources', ) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/scripts/wk/kit/ufd.py b/scripts/wk/kit/ufd.py index 0d83b3d0..d5040299 100644 --- a/scripts/wk/kit/ufd.py +++ b/scripts/wk/kit/ufd.py @@ -1,6 +1,7 @@ """WizardKit: UFD Functions""" # vim: sts=2 sw=2 ts=2 # TODO: Replace some lsblk usage with hw_obj? +# TODO: Reduce imports if possible # TODO: Needs testing import logging @@ -10,12 +11,39 @@ import shutil from collections import OrderedDict from wk import io, std +from wk.cfg.main import KIT_NAME_SHORT +from wk.cfg.ufd import BOOT_ENTRIES, BOOT_FILES, ITEMS, ITEMS_HIDDEN, SOURCES from wk.exe import run_program from wk.os import linux # STATIC VARIABLES +DOCSTRING = '''WizardKit: Build UFD + +Usage: + build-ufd [options] --ufd-device PATH --linux PATH + [--linux-minimal PATH] + [--main-kit PATH] + [--winpe PATH] + [--extra-dir PATH] + build-ufd (-h | --help) + +Options: + -e PATH, --extra-dir PATH + -k PATH, --main-kit PATH + -l PATH, --linux PATH + -m PATH, --linux-minimal PATH + -u PATH, --ufd-device PATH + -w PATH, --winpe PATH + + -h --help Show this page + -M --use-mbr Use real MBR instead of GPT w/ Protective MBR + -F --force Bypass all confirmation messages. USE WITH EXTREME CAUTION! + -U --update Don't format device, just update +''' LOG = logging.getLogger(__name__) +ISO_LABEL = f'{KIT_NAME_SHORT}_LINUX' +UFD_LABEL = f'{KIT_NAME_SHORT}_UFD' # Functions From 7702cdcf0af4f0efa0a26dd8ff3a423f23f1a191 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 7 Jan 2020 23:53:55 -0700 Subject: [PATCH 322/324] Finished converting UFD sections, testing next --- scripts/build-ufd | 151 +++--------------------------------------- scripts/wk/kit/ufd.py | 117 ++++++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 149 deletions(-) diff --git a/scripts/build-ufd b/scripts/build-ufd index 04958a83..237c6691 100755 --- a/scripts/build-ufd +++ b/scripts/build-ufd @@ -1,149 +1,14 @@ -#!/bin/env python3 -# -# pylint: disable=no-name-in-module,wildcard-import,wrong-import-position +#!/usr/bin/env python3 +"""Wizard Kit: Build UFD Tool""" # vim: sts=2 sw=2 ts=2 -"""Wizard Kit: UFD build tool""" -import os -import sys +import wk -# Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from docopt import docopt -from functions.common import * -from functions.ufd import * -from settings.ufd import * -init_global_vars(silent=True) -# Main section if __name__ == '__main__': - # pylint: disable=invalid-name - # Set log try: - global_vars['LogDir'] = '{}/Logs'.format( - get_user_home(get_user_name())) - set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) - except: # pylint: disable=bare-except - major_exception() - - # Header - print_success(KIT_NAME_FULL) - print_standard('UFD Build Tool') - print_standard(' ') - - # Check if running as root - if not running_as_root(): - print_error('ERROR: This script is meant to be run as root.') - abort(False) - - # Docopt - try: - args = docopt(DOCSTRING) - except SystemExit as sys_exit: - # Catch docopt exits - exit_script(sys_exit.code) - except: # pylint: disable=bare-except - major_exception() - - try: - # Verify selections - ufd_dev = verify_ufd(args['--ufd-device']) - sources = verify_sources(args, UFD_SOURCES) - show_selections(args, sources, ufd_dev, UFD_SOURCES) - if not args['--force']: - confirm_selections(update=args['--update']) - - # Prep UFD - if not args['--update']: - print_info('Prep UFD') - prep_device(ufd_dev, UFD_LABEL, use_mbr=args['--use-mbr']) - - # Mount UFD - try_and_print( - indent=2, - message='Mounting UFD...', - function=mount, - mount_source=find_first_partition(ufd_dev), - mount_point='/mnt/UFD', - read_write=True, - ) - - # Remove Arch folder - if args['--update']: - try_and_print( - indent=2, - message='Removing Linux...', - function=remove_arch, - ) - - # Copy sources - print_standard(' ') - print_info('Copy Sources') - for s_label, s_path in sources.items(): - try_and_print( - indent=2, - message='Copying {}...'.format(s_label), - function=copy_source, - source=s_path, - items=ITEMS[s_label], - overwrite=True, - ) - - # Update boot entries - print_standard(' ') - print_info('Boot Setup') - try_and_print( - indent=2, - message='Updating boot entries...', - function=update_boot_entries, - boot_entries=BOOT_ENTRIES, - boot_files=BOOT_FILES, - iso_label=ISO_LABEL, - ufd_label=UFD_LABEL, - ) - - # Install syslinux (to partition) - try_and_print( - indent=2, - message='Syslinux (partition)...', - function=install_syslinux_to_partition, - partition=find_first_partition(ufd_dev), - ) - - # Unmount UFD - try_and_print( - indent=2, - message='Unmounting UFD...', - function=unmount, - mount_point='/mnt/UFD', - ) - - # Install syslinux (to device) - try_and_print( - indent=2, - message='Syslinux (device)...', - function=install_syslinux_to_dev, - ufd_dev=ufd_dev, - use_mbr=args['--use-mbr'], - ) - - # Hide items - print_standard(' ') - print_info('Final Touches') - try_and_print( - indent=2, - message='Hiding items...', - function=hide_items, - ufd_dev=ufd_dev, - items=ITEMS_HIDDEN, - ) - - # Done - if not args['--force']: - print_standard('\nDone.') - pause('Press Enter to exit...') - exit_script() - except SystemExit as sys_exit: - exit_script(sys_exit.code) - except: # pylint: disable=bare-except - major_exception() + wk.kit.ufd.build_ufd() + except SystemExit: + raise + except: #pylint: disable=bare-except + wk.std.major_exception() diff --git a/scripts/wk/kit/ufd.py b/scripts/wk/kit/ufd.py index d5040299..f432a3f3 100644 --- a/scripts/wk/kit/ufd.py +++ b/scripts/wk/kit/ufd.py @@ -9,9 +9,10 @@ import os import shutil from collections import OrderedDict +from docopt import docopt -from wk import io, std -from wk.cfg.main import KIT_NAME_SHORT +from wk import io, log, std +from wk.cfg.main import KIT_NAME_FULL, KIT_NAME_SHORT from wk.cfg.ufd import BOOT_ENTRIES, BOOT_FILES, ITEMS, ITEMS_HIDDEN, SOURCES from wk.exe import run_program from wk.os import linux @@ -47,6 +48,109 @@ UFD_LABEL = f'{KIT_NAME_SHORT}_UFD' # Functions +def build_ufd(): + """Build UFD using selected sources.""" + args = docopt(DOCSTRING) + log.update_log_path(dest_name='build-ufd', timestamp=True) + try_print = std.TryAndPrint() + try_print.indent = 2 + + # Check if running with root permissions + if not linux.running_as_root(): + std.print_error('This script is meant to be run as root') + std.abort() + + # Show header + std.print_success(KIT_NAME_FULL) + std.print_warning('UFD Build Tool') + std.print_warning(' ') + + # Verify selections + ufd_dev = verify_ufd(args['--ufd-device']) + sources = verify_sources(args, SOURCES) + show_selections(args, sources, ufd_dev, SOURCES) + if not args['--force']: + confirm_selections(update=args['--update']) + + # Prep UFD + if not args['--update']: + std.print_info('Prep UFD') + prep_device(ufd_dev, UFD_LABEL, use_mbr=args['--use-mbr']) + + # Mount UFD + try_print.run( + message='Mounting UFD...', + function=linux.mount, + mount_source=find_first_partition(ufd_dev), + mount_point='/mnt/UFD', + read_write=True, + ) + + # Remove Arch folder + if args['--update']: + try_print.run( + message='Removing Linux...', + function=remove_arch, + ) + + # Copy sources + std.print_standard(' ') + std.print_info('Copy Sources') + for s_label, s_path in sources.items(): + try_print.run( + message='Copying {}...'.format(s_label), + function=copy_source, + source=s_path, + items=ITEMS[s_label], + overwrite=True, + ) + + # Update boot entries + std.print_standard(' ') + std.print_info('Boot Setup') + try_print.run( + message='Updating boot entries...', + function=update_boot_entries, + ) + + # Install syslinux (to partition) + try_print.run( + message='Syslinux (partition)...', + function=install_syslinux_to_partition, + partition=find_first_partition(ufd_dev), + ) + + # Unmount UFD + try_print.run( + message='Unmounting UFD...', + function=linux.unmount, + mount_point='/mnt/UFD', + ) + + # Install syslinux (to device) + try_print.run( + message='Syslinux (device)...', + function=install_syslinux_to_dev, + ufd_dev=ufd_dev, + use_mbr=args['--use-mbr'], + ) + + # Hide items + std.print_standard(' ') + std.print_info('Final Touches') + try_print.run( + message='Hiding items...', + function=hide_items, + ufd_dev=ufd_dev, + items=ITEMS_HIDDEN, + ) + + # Done + std.print_standard('\nDone.') + if not args['--force']: + std.pause('Press Enter to exit...') + + def confirm_selections(update=False): """Ask tech to confirm selections, twice if necessary.""" if not std.ask('Is the above information correct?'): @@ -178,6 +282,7 @@ def prep_device(dev_path, label, use_mbr=False): 4. Format partition (FAT32, 4K aligned) """ try_print = std.TryAndPrint() + try_print.indent = 2 # Zero-out first 64MB cmd = [ @@ -283,12 +388,12 @@ def show_selections(args, sources, ufd_dev, ufd_sources): std.print_standard(' ') -def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): +def update_boot_entries(): """Update boot files for UFD usage""" configs = [] # Find config files - for c_path, c_ext in boot_files.items(): + for c_path, c_ext in BOOT_FILES.items(): c_path = io.case_insensitive_path('/mnt/UFD{c_path}') for item in os.scandir(c_path): if item.name.lower().endswith(c_ext.lower()): @@ -299,13 +404,13 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): 'sed', '--in-place', '--regexp-extended', - f's/{iso_label}/{ufd_label}/', + f's/{ISO_LABEL}/{UFD_LABEL}/', *configs, ] run_program(cmd) # Uncomment extra entries if present - for b_path, b_comment in boot_entries.items(): + for b_path, b_comment in BOOT_ENTRIES.items(): try: io.case_insensitive_path(f'/mnt/UFD{b_path}') except (FileNotFoundError, NotADirectoryError): From da6f818d909d342ccbde9956411424dd5cfc7890 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 8 Jan 2020 14:59:54 -0700 Subject: [PATCH 323/324] Updated build_linux for use under new organization --- setup/build_linux | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/setup/build_linux b/setup/build_linux index f9acd897..50f84190 100755 --- a/setup/build_linux +++ b/setup/build_linux @@ -11,10 +11,10 @@ set -o pipefail DATE="$(date +%F)" DATETIME="$(date +%F_%H%M)" ROOT_DIR="$(realpath $(dirname "$0"))" -BUILD_DIR="$ROOT_DIR/BUILD_LINUX" +BUILD_DIR="$ROOT_DIR/setup/BUILD_LINUX" LIVE_DIR="$BUILD_DIR/live" LOG_DIR="$BUILD_DIR/logs" -OUT_DIR="$ROOT_DIR/OUT_LINUX" +OUT_DIR="$ROOT_DIR/setup/OUT_LINUX" REPO_DIR="$BUILD_DIR/repo" SKEL_DIR="$LIVE_DIR/airootfs/etc/skel" TEMP_DIR="$BUILD_DIR/temp" @@ -57,7 +57,7 @@ function cleanup() { function fix_kit_permissions() { # GitHub zip archives don't preserve the correct permissions - for d in .bin .cbin .kit_items .linux_items .pe_items Images; do + for d in docs images scripts setup; do find "$ROOT_DIR/$d" -type d -exec chmod 755 "{}" \; done } @@ -80,7 +80,7 @@ function load_settings() { if [[ "${1:-}" == "--edit" ]]; then # Copy settings if [[ ! -e "$BUILD_DIR/main.py" ]] || ask "Overwrite main.py?"; then - cp -bv "$ROOT_DIR/.bin/Scripts/settings/main.py" "$BUILD_DIR/main.py" + cp -bv "$ROOT_DIR/scripts/wk/cfg/main.py" "$BUILD_DIR/main.py" dos2unix "$BUILD_DIR/main.py" fi @@ -89,7 +89,7 @@ function load_settings() { "$EDITOR" "$BUILD_DIR/main.py" else # Load settings from $LIVE_DIR - _main_path="$LIVE_DIR/airootfs/usr/local/bin/settings/main.py" + _main_path="$LIVE_DIR/airootfs/usr/local/bin/wk/cfg/main.py" fi # Load settings @@ -118,13 +118,13 @@ function copy_live_env() { rm "$LIVE_DIR/syslinux"/*.cfg "$LIVE_DIR/syslinux"/*.png # Add items - rsync -aI "$ROOT_DIR/.linux_items/include/" "$LIVE_DIR/" + rsync -aI "$ROOT_DIR/setup/linux/include/" "$LIVE_DIR/" if [[ "${1:-}" != "--minimal" ]]; then - rsync -aI "$ROOT_DIR/.linux_items/include_x/" "$LIVE_DIR/" + rsync -aI "$ROOT_DIR/setup/linux/include_x/" "$LIVE_DIR/" fi mkdir -p "$LIVE_DIR/airootfs/usr/local/bin" - rsync -aI "$ROOT_DIR/.bin/Scripts/" "$LIVE_DIR/airootfs/usr/local/bin/" - cp -a "$BUILD_DIR/main.py" "$LIVE_DIR/airootfs/usr/local/bin/settings/" + rsync -aI "$ROOT_DIR/scripts/" "$LIVE_DIR/airootfs/usr/local/bin/" + cp -a "$BUILD_DIR/main.py" "$LIVE_DIR/airootfs/usr/local/bin/wk/cfg/" } function run_elevated() { @@ -155,8 +155,8 @@ function update_live_env() { # Boot config (legacy) mkdir -p "$LIVE_DIR/arch" - cp "$ROOT_DIR/Images/Pxelinux.png" "$LIVE_DIR/arch/pxelinux.png" - cp "$ROOT_DIR/Images/Syslinux.png" "$LIVE_DIR/arch/syslinux.png" + cp "$ROOT_DIR/images/Pxelinux.png" "$LIVE_DIR/arch/pxelinux.png" + cp "$ROOT_DIR/images/Syslinux.png" "$LIVE_DIR/arch/syslinux.png" sed -i -r "s/_+/$KIT_NAME_FULL/" "$LIVE_DIR/syslinux/wk_head.cfg" mkdir -p "$TEMP_DIR" 2>/dev/null curl -Lo "$TEMP_DIR/wimboot.zip" "http://git.ipxe.org/releases/wimboot/wimboot-latest.zip" @@ -165,7 +165,7 @@ function update_live_env() { # Boot config (UEFI) mkdir -p "$LIVE_DIR/EFI/boot" cp "/usr/share/refind/refind_x64.efi" "$LIVE_DIR/EFI/boot/bootx64.efi" - cp "$ROOT_DIR/Images/rEFInd.png" "$LIVE_DIR/EFI/boot/rEFInd.png" + cp "$ROOT_DIR/images/rEFInd.png" "$LIVE_DIR/EFI/boot/rEFInd.png" rsync -aI "/usr/share/refind/drivers_x64/" "$LIVE_DIR/EFI/boot/drivers_x64/" rsync -aI "/usr/share/refind/icons/" "$LIVE_DIR/EFI/boot/icons/" --exclude "/usr/share/refind/icons/svg" sed -i "s/%ARCHISO_LABEL%/${label}/" "$LIVE_DIR/EFI/boot/refind.conf" @@ -199,12 +199,12 @@ function update_live_env() { # Live packages while read -r p; do sed -i "/$p/d" "$LIVE_DIR/packages.x86_64" - done < "$ROOT_DIR/.linux_items/packages/live_remove" - cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.x86_64" + done < "$ROOT_DIR/setup/linux/packages/live_remove" + cat "$ROOT_DIR/setup/linux/packages/live_add" >> "$LIVE_DIR/packages.x86_64" if [[ "${1:-}" == "--minimal" ]]; then - cat "$ROOT_DIR/.linux_items/packages/live_add_min" >> "$LIVE_DIR/packages.x86_64" + cat "$ROOT_DIR/setup/linux/packages/live_add_min" >> "$LIVE_DIR/packages.x86_64" else - cat "$ROOT_DIR/.linux_items/packages/live_add_x" >> "$LIVE_DIR/packages.x86_64" + cat "$ROOT_DIR/setup/linux/packages/live_add_x" >> "$LIVE_DIR/packages.x86_64" fi echo "[custom]" >> "$LIVE_DIR/pacman.conf" echo "SigLevel = Optional TrustAll" >> "$LIVE_DIR/pacman.conf" @@ -246,7 +246,7 @@ function update_live_env() { echo 'rm /root/.zlogin' >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" sed -i -r '/.*PermitRootLogin.*/d' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" echo "sed -i -r '/.*PermitRootLogin.*/d' /etc/ssh/sshd_config" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" - cp "$ROOT_DIR/.linux_items/authorized_keys" "$SKEL_DIR/.ssh/authorized_keys" + cp "$ROOT_DIR/setup/linux/authorized_keys" "$SKEL_DIR/.ssh/authorized_keys" # Root user echo "echo 'root:$ROOT_PASSWORD' | chpasswd" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" @@ -279,11 +279,11 @@ function update_live_env() { # Wallpaper mkdir -p "$LIVE_DIR/airootfs/usr/share/wallpaper" - cp "$ROOT_DIR/Images/Linux.png" "$LIVE_DIR/airootfs/usr/share/wallpaper/burned.in" + cp "$ROOT_DIR/images/Linux.png" "$LIVE_DIR/airootfs/usr/share/wallpaper/burned.in" fi # WiFi - cp "$ROOT_DIR/.linux_items/known_networks" "$LIVE_DIR/airootfs/root/known_networks" + cp "$ROOT_DIR/setup/linux/known_networks" "$LIVE_DIR/airootfs/root/known_networks" echo "add-known-networks --user=$username" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" } @@ -315,7 +315,7 @@ function update_repo() { makepkg -d popd >/dev/null mv -n $p/*xz "$REPO_DIR"/ - done < "$ROOT_DIR/.linux_items/packages/aur" + done < "$ROOT_DIR/setup/linux/packages/aur" popd >/dev/null # Build custom repo database @@ -329,7 +329,7 @@ function install_deps() { packages= while read -r line; do packages="$packages $line" - done < "$ROOT_DIR/.linux_items/packages/dependencies" + done < "$ROOT_DIR/setup/linux/packages/dependencies" run_elevated pacman -Syu --needed --noconfirm $packages } @@ -347,7 +347,7 @@ function build_iso() { chmod 600 "$LIVE_DIR/airootfs/etc/skel/.ssh/id_rsa" # Removing cached (and possibly outdated) custom repo packages - for package in $(cat "$ROOT_DIR/.linux_items/packages/aur"); do + for package in $(cat "$ROOT_DIR/setup/linux/packages/aur"); do for p in /var/cache/pacman/pkg/*${package}*; do if [[ -f "${p}" ]]; then rm "${p}" From a7bb7e1e23d30fca032034f572a253765d40c543 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 8 Jan 2020 15:04:40 -0700 Subject: [PATCH 324/324] Updated .gitignore --- .gitignore | 48 +++++++----------------------------------------- 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index bcaa6c80..420330cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,41 +1,7 @@ -**/__pycache__/* -*.bak -*.exe -*.swp -.bin/7-Zip/ -.bin/AIDA64/ -.bin/BleachBit/ -.bin/ClassicStartSkin/ -.bin/ConEmu/ -.bin/Erunt/ -.bin/Everything/ -.bin/FastCopy/ -.bin/HWiNFO/HWiNFO*.ini -.bin/NotepadPlusPlus/ -.bin/ProcessKiller/ -.bin/ProduKey/ -.bin/Python/ -.bin/Tmp/ -.bin/XMPlay/ -.bin/_Drivers/SDIO/ -.cbin/*.7z -.cbin/AIDA64/ -.cbin/Autoruns/ -.cbin/BleachBit-Portable/ -.cbin/BlueScreenView/ -.cbin/Caffeine/ -.cbin/Du/ -.cbin/Everything/ -.cbin/FirefoxExtensions/ -.cbin/IObitUninstallerPortable/ -.cbin/ProduKey/ -.cbin/TestDisk/ -.cbin/TreeSizeFree-Portable/ -.cbin/XMPlay/ -.cbin/XYplorerFree/ -.cbin/_Drivers/ -.cbin/_Office/ -.cbin/_vcredists/ -.cbin/wimlib/ -BUILD*/ -OUT*/ +**/__pycache__ +**/*.7z +**/*.bak +**/*.exe +**/*.swp +setup/BUILD* +setup/OUT*