From 5380d133e42298c8e41192ec824c6894145030f0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 17:07:13 -0700 Subject: [PATCH 001/121] Adjusted Linux boot options --- .linux_items/include/EFI/boot/refind.conf | 4 ++-- .linux_items/include/syslinux/wk_iso_linux.cfg | 6 +++--- .linux_items/include/syslinux/wk_pxe_linux.cfg | 6 +++--- .linux_items/include/syslinux/wk_sys_linux.cfg | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.linux_items/include/EFI/boot/refind.conf b/.linux_items/include/EFI/boot/refind.conf index f4a128e2..1406ac73 100644 --- a/.linux_items/include/EFI/boot/refind.conf +++ b/.linux_items/include/EFI/boot/refind.conf @@ -25,12 +25,12 @@ menuentry "Linux" { initrd /arch/boot/intel_ucode.img initrd /arch/boot/amd_ucode.img initrd /arch/boot/x86_64/archiso.img - options "archisobasedir=arch archisolabel=%ARCHISO_LABEL% quiet copytoram loglevel=3" + options "archisobasedir=arch archisolabel=%ARCHISO_LABEL% copytoram loglevel=3" submenuentry "Linux (i3)" { add_options "i3" } submenuentry "Linux (CLI)" { - add_options "nox" + add_options "loglevel=4 nomodeset nox" } } #UFD#menuentry "WindowsPE" { diff --git a/.linux_items/include/syslinux/wk_iso_linux.cfg b/.linux_items/include/syslinux/wk_iso_linux.cfg index 3f2c3556..fef9a9e1 100644 --- a/.linux_items/include/syslinux/wk_iso_linux.cfg +++ b/.linux_items/include/syslinux/wk_iso_linux.cfg @@ -6,7 +6,7 @@ ENDTEXT MENU LABEL Linux LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% quiet loglevel=3 +APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% loglevel=3 LABEL wk_iso_linux_i3 TEXT HELP @@ -16,7 +16,7 @@ ENDTEXT MENU LABEL Linux (i3) LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% quiet loglevel=3 i3 +APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% loglevel=3 i3 SYSAPPEND 3 LABEL wk_iso_linux_cli @@ -27,5 +27,5 @@ ENDTEXT MENU LABEL Linux (CLI) LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% nox nomodeset +APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% loglevel=4 nomodeset nox SYSAPPEND 3 diff --git a/.linux_items/include/syslinux/wk_pxe_linux.cfg b/.linux_items/include/syslinux/wk_pxe_linux.cfg index d2468e03..caa6a1cc 100644 --- a/.linux_items/include/syslinux/wk_pxe_linux.cfg +++ b/.linux_items/include/syslinux/wk_pxe_linux.cfg @@ -6,7 +6,7 @@ ENDTEXT MENU LABEL Linux (PXE) LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archiso_http_srv=http://${pxeserver}/ quiet loglevel=3 +APPEND archisobasedir=%INSTALL_DIR% archiso_http_srv=http://${pxeserver}/ loglevel=3 SYSAPPEND 3 LABEL wk_http_linux_i3 @@ -17,7 +17,7 @@ ENDTEXT MENU LABEL Linux (PXE) (i3) LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archiso_http_srv=http://${pxeserver}/ quiet loglevel=3 i3 +APPEND archisobasedir=%INSTALL_DIR% archiso_http_srv=http://${pxeserver}/ loglevel=3 i3 SYSAPPEND 3 LABEL wk_http_linux_cli @@ -28,5 +28,5 @@ ENDTEXT MENU LABEL Linux (PXE) (CLI) LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archiso_http_srv=http://${pxeserver}/ nox nomodeset +APPEND archisobasedir=%INSTALL_DIR% archiso_http_srv=http://${pxeserver}/ loglevel=4 nomodeset nox SYSAPPEND 3 diff --git a/.linux_items/include/syslinux/wk_sys_linux.cfg b/.linux_items/include/syslinux/wk_sys_linux.cfg index 55b5f239..d4319c00 100644 --- a/.linux_items/include/syslinux/wk_sys_linux.cfg +++ b/.linux_items/include/syslinux/wk_sys_linux.cfg @@ -6,7 +6,7 @@ ENDTEXT MENU LABEL Linux LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% quiet copytoram loglevel=3 +APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% copytoram loglevel=3 LABEL wk_linux_i3 TEXT HELP @@ -16,7 +16,7 @@ ENDTEXT MENU LABEL Linux (i3) LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% quiet copytoram loglevel=3 i3 +APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% copytoram loglevel=3 i3 SYSAPPEND 3 LABEL wk_linux_cli From 5e1daf36dfae16d5e5ed680da6fbbc1a32d80373 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 17:08:32 -0700 Subject: [PATCH 002/121] Updated Notepad++ config --- .cbin/_include/NotepadPlusPlus/config.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.cbin/_include/NotepadPlusPlus/config.xml b/.cbin/_include/NotepadPlusPlus/config.xml index c2dd100c..1975e9a7 100644 --- a/.cbin/_include/NotepadPlusPlus/config.xml +++ b/.cbin/_include/NotepadPlusPlus/config.xml @@ -4,8 +4,8 @@ standard - hide - + show + vertical hide @@ -18,7 +18,7 @@ no yes - + yes @@ -29,7 +29,7 @@ - hide + show @@ -37,10 +37,10 @@ - + yes - + From 63fcaed8cd172d25a8b8ccc7314257b140682abd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 17:54:46 -0700 Subject: [PATCH 003/121] Updated Launchers --- .bin/Scripts/Launch.cmd | 10 +++++----- .bin/Scripts/Launcher_Template.cmd | 4 ++-- .bin/Scripts/init_client_dir.cmd | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/Launch.cmd b/.bin/Scripts/Launch.cmd index f7080adb..0a0330c6 100644 --- a/.bin/Scripts/Launch.cmd +++ b/.bin/Scripts/Launch.cmd @@ -279,9 +279,9 @@ rem Create VB script mkdir "%bin%\tmp" 2>nul echo Set UAC = CreateObject^("Shell.Application"^) > "%bin%\tmp\Elevate.vbs" if defined L_NCMD ( - echo UAC.ShellExecute "%PYTHON%", """%script%""", "", "runas", 3 >> "%bin%\tmp\Elevate.vbs" + echo UAC.ShellExecute "%PYTHON%", """%script%"" %L_ARGS%", "", "runas", 3 >> "%bin%\tmp\Elevate.vbs" ) else ( - echo UAC.ShellExecute "%CON%", "-run ""%PYTHON%"" ""%script%"" -new_console:n", "", "runas", 1 >> "%bin%\tmp\Elevate.vbs" + echo UAC.ShellExecute "%CON%", "-run ""%PYTHON%"" ""%script%"" %L_ARGS% -new_console:n", "", "runas", 1 >> "%bin%\tmp\Elevate.vbs" ) rem Run @@ -290,9 +290,9 @@ goto Exit :LaunchPyScriptUser if defined L_NCMD ( - start "" "%PYTHON%" "%script%" || goto ErrorUnknown + start "" "%PYTHON%" "%script%" %L_ARGS% || goto ErrorUnknown ) else ( - start "" "%CON%" -run "%PYTHON%" "%script%" -new_console:n || goto ErrorUnknown + start "" "%CON%" -run "%PYTHON%" "%script%" %L_ARGS% -new_console:n || goto ErrorUnknown ) goto Exit @@ -332,7 +332,7 @@ echo. Executable Working Dir Program Args [L_7ZIP] [L_ELEV] [L__CLI] echo. Folder Folder '.' [L_7ZIP] echo. Office Year Product [L_7ZIP] echo. PSScript Scripts Script [L_7ZIP] [L_ELEV] [L_NCMD] -echo. PyScript Scripts Script [L_7ZIP] [L_ELEV] [L_NCMD] +echo. PyScript Scripts Script Args [L_7ZIP] [L_ELEV] [L_NCMD] echo. QuickBooks Year Product [L_7ZIP] echo. echo.L_7ZIP: Extra arguments for 7-Zip (in the :ExtractCBin label) diff --git a/.bin/Scripts/Launcher_Template.cmd b/.bin/Scripts/Launcher_Template.cmd index 90b15482..01d1758d 100644 --- a/.bin/Scripts/Launcher_Template.cmd +++ b/.bin/Scripts/Launcher_Template.cmd @@ -17,7 +17,7 @@ call :SetTitle Launcher rem EXTRA_CODE :DefineLaunch -:: See %bin%\SCripts\Launch.cmd for details under :Usage label +:: See %bin%\Scripts\Launch.cmd for details under :Usage label set L_TYPE= set L_PATH= set L_ITEM= @@ -110,4 +110,4 @@ goto Exit :: Cleanup and exit :: :Exit endlocal -exit /b %errorlevel% \ No newline at end of file +exit /b %errorlevel% diff --git a/.bin/Scripts/init_client_dir.cmd b/.bin/Scripts/init_client_dir.cmd index 4d73e7ab..302b27d1 100644 --- a/.bin/Scripts/init_client_dir.cmd +++ b/.bin/Scripts/init_client_dir.cmd @@ -33,7 +33,7 @@ for /f "tokens=* usebackq" %%f in (`findstr KIT_NAME_SHORT "%SETTINGS%"`) do ( set "KIT_NAME_SHORT=!_v:~0,-1!" ) set "client_dir=%systemdrive%\%KIT_NAME_SHORT%" -set "log_dir=%client_dir%\Info\%iso_date%" +set "log_dir=%client_dir%\Logs\%iso_date%" :Flags set _backups= @@ -45,7 +45,7 @@ set _transfer= for %%f in (%*) do ( if /i "%%f" == "/DEBUG" (@echo on) if /i "%%f" == "/Backups" set _backups=True - if /i "%%f" == "/Info" set _info=True + if /i "%%f" == "/Logs" set _logs=True if /i "%%f" == "/Office" set _office=True if /i "%%f" == "/Quarantine" set _quarantine=True if /i "%%f" == "/QuickBooks" set _quickbooks=True @@ -54,7 +54,9 @@ for %%f in (%*) do ( :CreateDirs if defined _backups mkdir "%client_dir%\Backups">nul 2>&1 -if defined _info mkdir "%client_dir%\Info">nul 2>&1 +if defined _logs ( + mkdir "%log_dir%\%KIT_NAME_FULL%">nul 2>&1 + mkdir "%log_dir%\Tools">nul 2>&1) if defined _office mkdir "%client_dir%\Office">nul 2>&1 if defined _quarantine mkdir "%client_dir%\Quarantine">nul 2>&1 if defined _quickbooks mkdir "%client_dir%\QuickBooks">nul 2>&1 From 8064fc4a1721d2fcd27edcc15efabf42769871cc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:01:03 -0700 Subject: [PATCH 004/121] Removed whitespace from empty lines --- .bin/Scripts/activate.py | 2 +- .bin/Scripts/borrowed/sensors.py | 54 +++++++++++----------- .bin/Scripts/cbs_fix.py | 8 ++-- .bin/Scripts/check_disk.py | 4 +- .bin/Scripts/dism.py | 4 +- .bin/Scripts/functions/activation.py | 2 +- .bin/Scripts/functions/backup.py | 26 +++++------ .bin/Scripts/functions/disk.py | 60 ++++++++++++------------- .bin/Scripts/functions/network.py | 4 +- .bin/Scripts/functions/product_keys.py | 2 +- .bin/Scripts/functions/windows_setup.py | 16 +++---- .bin/Scripts/functions/winpe_menus.py | 30 ++++++------- .bin/Scripts/safemode_enter.py | 6 +-- .bin/Scripts/safemode_exit.py | 6 +-- .bin/Scripts/sfc_scan.py | 4 +- .bin/Scripts/transferred_keys.py | 4 +- .bin/Scripts/user_data_transfer.py | 16 +++---- .bin/Scripts/winpe_root_menu.py | 2 +- 18 files changed, 125 insertions(+), 125 deletions(-) diff --git a/.bin/Scripts/activate.py b/.bin/Scripts/activate.py index 3555c46a..642e5edd 100644 --- a/.bin/Scripts/activate.py +++ b/.bin/Scripts/activate.py @@ -39,7 +39,7 @@ if __name__ == '__main__': selection = menu_select( '{}: Windows Activation Menu'.format(KIT_NAME_FULL), main_entries=activation_methods, action_entries=actions) - + if (selection.isnumeric()): result = try_and_print( message = activation_methods[int(selection)-1]['Name'], diff --git a/.bin/Scripts/borrowed/sensors.py b/.bin/Scripts/borrowed/sensors.py index 847f2619..39b00a4f 100644 --- a/.bin/Scripts/borrowed/sensors.py +++ b/.bin/Scripts/borrowed/sensors.py @@ -35,7 +35,7 @@ class feature(Structure): _fields_ = [("name", c_char_p), ("number", c_int), ("type", c_int)] - + # sensors_feature_type IN = 0x00 FAN = 0x01 @@ -71,10 +71,10 @@ COMPUTE_MAPPING = 4 def init(cfg_file = None): file = _libc.fopen(cfg_file.encode("utf-8"), "r") if cfg_file is not None else None - + if _hdl.sensors_init(file) != 0: raise Exception("sensors_init failed") - + if file is not None: _libc.fclose(file) @@ -84,10 +84,10 @@ def cleanup(): def parse_chip_name(orig_name): ret = chip_name() err= _hdl.sensors_parse_chip_name(orig_name.encode("utf-8"), byref(ret)) - + if err < 0: raise Exception(strerror(err)) - + return ret def strerror(errnum): @@ -101,10 +101,10 @@ def get_detected_chips(match, nr): @return: (chip, next nr to query) """ _nr = c_int(nr) - + if match is not None: match = byref(match) - + chip = _hdl.sensors_get_detected_chips(match, byref(_nr)) chip = chip.contents if bool(chip) else None return chip, _nr.value @@ -115,10 +115,10 @@ def chip_snprintf_name(chip, buffer_size=200): """ ret = create_string_buffer(buffer_size) err = _hdl.sensors_snprintf_chip_name(ret, buffer_size, byref(chip)) - + if err < 0: raise Exception(strerror(err)) - + return ret.value.decode("utf-8") def do_chip_sets(chip): @@ -128,7 +128,7 @@ def do_chip_sets(chip): err = _hdl.sensors_do_chip_sets(byref(chip)) if err < 0: raise Exception(strerror(err)) - + def get_adapter_name(bus): return _hdl.sensors_get_adapter_name(byref(bus)).decode("utf-8") @@ -177,60 +177,60 @@ class ChipIterator: def __init__(self, match = None): self.match = parse_chip_name(match) if match is not None else None self.nr = 0 - + def __iter__(self): return self - + def __next__(self): chip, self.nr = get_detected_chips(self.match, self.nr) - + if chip is None: raise StopIteration - + return chip - + def __del__(self): if self.match is not None: free_chip_name(self.match) - + def next(self): # python2 compability return self.__next__() - + class FeatureIterator: def __init__(self, chip): self.chip = chip self.nr = 0 - + def __iter__(self): return self - + def __next__(self): feature, self.nr = get_features(self.chip, self.nr) - + if feature is None: raise StopIteration - + return feature def next(self): # python2 compability return self.__next__() - + class SubFeatureIterator: def __init__(self, chip, feature): self.chip = chip self.feature = feature self.nr = 0 - + def __iter__(self): return self - + def __next__(self): subfeature, self.nr = get_all_subfeatures(self.chip, self.feature, self.nr) - + if subfeature is None: raise StopIteration - + return subfeature - + def next(self): # python2 compability return self.__next__() diff --git a/.bin/Scripts/cbs_fix.py b/.bin/Scripts/cbs_fix.py index a3b40a8d..9a8ff10c 100644 --- a/.bin/Scripts/cbs_fix.py +++ b/.bin/Scripts/cbs_fix.py @@ -10,7 +10,7 @@ from functions.cleanup import * from functions.data import * init_global_vars() os.system('title {}: CBS Cleanup'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\CBS Cleanup.log'.format(**global_vars) +set_log_file('CBS Cleanup.log') if __name__ == '__main__': try: @@ -20,18 +20,18 @@ if __name__ == '__main__': folder_path = r'{}\Backups'.format(KIT_NAME_SHORT) dest = select_destination(folder_path=folder_path, prompt='Which disk are we using for temp data and backup?') - + # Show details print_info('{}: CBS Cleanup Tool\n'.format(KIT_NAME_FULL)) show_data('Backup / Temp path:', dest) print_standard('\n') if (not ask('Proceed with CBS cleanup?')): abort() - + # Run Cleanup try_and_print(message='Running cleanup...', function=cleanup_cbs, cs='Done', dest_folder=dest) - + # Done print_standard('\nDone.') pause("Press Enter to exit...") diff --git a/.bin/Scripts/check_disk.py b/.bin/Scripts/check_disk.py index 734319f0..7e59fb2b 100644 --- a/.bin/Scripts/check_disk.py +++ b/.bin/Scripts/check_disk.py @@ -9,7 +9,7 @@ sys.path.append(os.getcwd()) from functions.repairs import * init_global_vars() os.system('title {}: Check Disk Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\Check Disk.log'.format(**global_vars) +set_log_file('Check Disk.log') if __name__ == '__main__': try: @@ -45,7 +45,7 @@ if __name__ == '__main__': cs=cs, other_results=other_results, repair=repair) else: abort() - + # Done print_success('Done.') pause("Press Enter to exit...") diff --git a/.bin/Scripts/dism.py b/.bin/Scripts/dism.py index 1c88e51b..e49a9512 100644 --- a/.bin/Scripts/dism.py +++ b/.bin/Scripts/dism.py @@ -9,7 +9,7 @@ sys.path.append(os.getcwd()) from functions.repairs import * init_global_vars() os.system('title {}: DISM helper Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\DISM helper tool.log'.format(**global_vars) +set_log_file('DISM Helper.log') if __name__ == '__main__': try: @@ -46,7 +46,7 @@ if __name__ == '__main__': other_results=other_results, repair=repair) else: abort() - + # Done print_success('Done.') pause("Press Enter to exit...") diff --git a/.bin/Scripts/functions/activation.py b/.bin/Scripts/functions/activation.py index f54d1dca..24436418 100644 --- a/.bin/Scripts/functions/activation.py +++ b/.bin/Scripts/functions/activation.py @@ -59,7 +59,7 @@ def windows_is_activated(): ['cscript', '//nologo', SLMGR, '/xpr'], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) activation_string = activation_string.stdout.decode() - + return bool(activation_string and 'permanent' in activation_string) if __name__ == '__main__': diff --git a/.bin/Scripts/functions/backup.py b/.bin/Scripts/functions/backup.py index e33cea2c..d872c4d0 100644 --- a/.bin/Scripts/functions/backup.py +++ b/.bin/Scripts/functions/backup.py @@ -16,7 +16,7 @@ def backup_partition(disk, par): """Create a backup image of a partition.""" if par.get('Image Exists', False) or par['Number'] in disk['Bad Partitions']: raise GenericAbort - + cmd = [ global_vars['Tools']['wimlib-imagex'], 'capture', @@ -48,7 +48,7 @@ def get_volume_display_name(mountpoint): serial_number = None max_component_length = None file_system_flags = None - + vol_info = kernel32.GetVolumeInformationW( ctypes.c_wchar_p(mountpoint), vol_name_buffer, @@ -59,16 +59,16 @@ def get_volume_display_name(mountpoint): fs_name_buffer, ctypes.sizeof(fs_name_buffer) ) - + name = '{} "{}"'.format(name, vol_name_buffer.value) except: pass - + return name def prep_disk_for_backup(destination, disk, backup_prefix): """Gather details about the disk and its partitions. - + This includes partitions that can't be backed up, whether backups already exist on the BACKUP_SERVER, partition names/sizes/used space, etc.""" @@ -83,7 +83,7 @@ def prep_disk_for_backup(destination, disk, backup_prefix): if disk['Valid Partitions'] <= 0: print_error('ERROR: No partitions can be backed up for this disk') raise GenericAbort - + # Prep partitions for par in disk['Partitions']: display = '{size} {fs}'.format( @@ -91,7 +91,7 @@ def prep_disk_for_backup(destination, disk, backup_prefix): width = width, size = par['Size'], fs = par['FileSystem']) - + if par['Number'] in disk['Bad Partitions']: # Set display string using partition description & OS type display = '* {display}\t\t{q}{name}{q}\t{desc} ({os})'.format( @@ -120,7 +120,7 @@ def prep_disk_for_backup(destination, disk, backup_prefix): display = '+ {}'.format(display) else: display = ' {}'.format(display) - + # Append rest of Display String for valid/clobber partitions display += ' (Used: {used})\t{q}{name}{q}'.format( used = par['Used Space'], @@ -128,7 +128,7 @@ def prep_disk_for_backup(destination, disk, backup_prefix): name = par['Name']) # For all partitions par['Display String'] = display - + # Set description for bad partitions warnings = '\n' if disk['Bad Partitions']: @@ -148,7 +148,7 @@ def select_backup_destination(auto_select=True): actions = [ {'Name': 'Main Menu', 'Letter': 'M'}, ] - + # Add local disks for d in psutil.disk_partitions(): if re.search(r'^{}'.format(global_vars['Env']['SYSTEMDRIVE']), d.mountpoint, re.IGNORECASE): @@ -161,7 +161,7 @@ def select_backup_destination(auto_select=True): get_volume_display_name(d.mountpoint)), 'Letter': re.sub(r'^(\w):\\.*$', r'\1', d.mountpoint), }) - + # Size check for dest in destinations: if 'IP' in dest: @@ -175,11 +175,11 @@ def select_backup_destination(auto_select=True): if not destinations: print_warning('No backup destinations found.') raise GenericAbort - + # Skip menu? if len(destinations) == 1 and auto_select: return destinations[0] - + selection = menu_select( title = 'Where are we backing up to?', main_entries = destinations, diff --git a/.bin/Scripts/functions/disk.py b/.bin/Scripts/functions/disk.py index 2d7cf0bb..75879ff8 100644 --- a/.bin/Scripts/functions/disk.py +++ b/.bin/Scripts/functions/disk.py @@ -14,13 +14,13 @@ REGEX_DISK_RAW = re.compile(r'Disk ID: 00000000', re.IGNORECASE) def assign_volume_letters(): """Assign a volume letter to all available volumes.""" remove_volume_letters() - + # Write script script = [] for vol in get_volumes(): script.append('select volume {}'.format(vol['Number'])) script.append('assign') - + # Run run_diskpart(script) @@ -35,7 +35,7 @@ def get_boot_mode(): boot_mode = 'UEFI' except: boot_mode = 'Unknown' - + return boot_mode def get_disk_details(disk): @@ -44,7 +44,7 @@ def get_disk_details(disk): script = [ 'select disk {}'.format(disk['Number']), 'detail disk'] - + # Run try: result = run_diskpart(script) @@ -60,13 +60,13 @@ def get_disk_details(disk): tmp = [s.split(':') for s in tmp if ':' in s] # Add key/value pairs to the details variable and return dict details.update({key.strip(): value.strip() for (key, value) in tmp}) - + return details - + def get_disks(): """Get list of attached disks using DiskPart.""" disks = [] - + try: # Run script result = run_diskpart(['list disk']) @@ -79,7 +79,7 @@ def get_disks(): num = tmp[0] size = human_readable_size(tmp[1]) disks.append({'Number': num, 'Size': size}) - + return disks def get_partition_details(disk, partition): @@ -89,7 +89,7 @@ def get_partition_details(disk, partition): 'select disk {}'.format(disk['Number']), 'select partition {}'.format(partition['Number']), 'detail partition'] - + # Diskpart details try: # Run script @@ -111,14 +111,14 @@ def get_partition_details(disk, partition): tmp = [s.split(':') for s in tmp if ':' in s] # Add key/value pairs to the details variable and return dict details.update({key.strip(): value.strip() for (key, value) in tmp}) - + # Get MBR type / GPT GUID for extra details on "Unknown" partitions guid = partition_uids.lookup_guid(details.get('Type')) if guid: details.update({ 'Description': guid.get('Description', '')[:29], 'OS': guid.get('OS', 'Unknown')[:27]}) - + if 'Letter' in details: # Disk usage try: @@ -128,7 +128,7 @@ def get_partition_details(disk, partition): details['Error'] = err.strerror else: details['Used Space'] = human_readable_size(tmp.used) - + # fsutil details cmd = [ 'fsutil', @@ -151,14 +151,14 @@ def get_partition_details(disk, partition): tmp = [s.split(':') for s in tmp if ':' in s] # Add key/value pairs to the details variable and return dict details.update({key.strip(): value.strip() for (key, value) in tmp}) - + # Set Volume Name details['Name'] = details.get('Volume Name', '') - + # Set FileSystem Type if details.get('FileSystem', '') not in ['RAW', 'Unknown']: details['FileSystem'] = details.get('File System Name', 'Unknown') - + return details def get_partitions(disk): @@ -167,7 +167,7 @@ def get_partitions(disk): script = [ 'select disk {}'.format(disk['Number']), 'list partition'] - + try: # Run script result = run_diskpart(script) @@ -181,7 +181,7 @@ def get_partitions(disk): num = tmp[0] size = human_readable_size(tmp[1]) partitions.append({'Number': num, 'Size': size}) - + return partitions def get_table_type(disk): @@ -190,7 +190,7 @@ def get_table_type(disk): script = [ 'select disk {}'.format(disk['Number']), 'uniqueid disk'] - + try: result = run_diskpart(script) except subprocess.CalledProcessError: @@ -203,7 +203,7 @@ def get_table_type(disk): part_type = 'MBR' elif REGEX_DISK_RAW.search(output): part_type = 'RAW' - + return part_type def get_volumes(): @@ -218,7 +218,7 @@ def get_volumes(): output = result.stdout.decode().strip() for tmp in re.findall(r'Volume (\d+)\s+([A-Za-z]?)\s+', output): vols.append({'Number': tmp[0], 'Letter': tmp[1]}) - + return vols def is_bad_partition(par): @@ -229,7 +229,7 @@ def prep_disk_for_formatting(disk=None): """Gather details about the disk and its partitions.""" disk['Format Warnings'] = '\n' width = len(str(len(disk['Partitions']))) - + # Bail early if disk is None: raise Exception('Disk not provided.') @@ -242,7 +242,7 @@ def prep_disk_for_formatting(disk=None): else: if (ask("Setup Windows to use BIOS/Legacy booting?")): disk['Use GPT'] = False - + # Set Display and Warning Strings if len(disk['Partitions']) == 0: disk['Format Warnings'] += 'No partitions found\n' @@ -252,7 +252,7 @@ def prep_disk_for_formatting(disk=None): width = width, size = partition['Size'], fs = partition['FileSystem']) - + if is_bad_partition(partition): # Set display string using partition description & OS type display += '\t\t{q}{name}{q}\t{desc} ({os})'.format( @@ -290,13 +290,13 @@ def remove_volume_letters(keep=None): """Remove all assigned volume letters using DiskPart.""" if not keep: keep = '' - + script = [] for vol in get_volumes(): if vol['Letter'].upper() != keep.upper(): script.append('select volume {}'.format(vol['Number'])) script.append('remove noerr') - + # Run script try: run_diskpart(script) @@ -306,12 +306,12 @@ def remove_volume_letters(keep=None): def run_diskpart(script): """Run DiskPart script.""" tempfile = r'{}\diskpart.script'.format(global_vars['Env']['TMP']) - + # Write script with open(tempfile, 'w') as f: for line in script: f.write('{}\n'.format(line)) - + # Run script cmd = [ r'{}\Windows\System32\diskpart.exe'.format( @@ -335,7 +335,7 @@ def scan_disks(): # Get partition info for disk disk['Partitions'] = get_partitions(disk) - + for partition in disk['Partitions']: # Get partition details partition.update(get_partition_details(disk, partition)) @@ -364,12 +364,12 @@ def select_disk(title='Which disk?', disks=[]): fs = partition['FileSystem']) if partition['Name']: p_name += '\t"{}"'.format(partition['Name']) - + # Show unsupported partition(s) if is_bad_partition(partition): p_name = '{YELLOW}{p_name}{CLEAR}'.format( p_name=p_name, **COLORS) - + display_name += '\n\t\t\t{}'.format(p_name) if not disk['Partitions']: display_name += '\n\t\t\t{}No partitions found.{}'.format( diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index d040e343..5735c486 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -26,7 +26,7 @@ def connect_to_network(): # Bail if currently connected if is_connected(): return - + # WiFi if 'wl' in net_ifs: cmd = [ @@ -37,7 +37,7 @@ def connect_to_network(): message = 'Connecting to {}...'.format(WIFI_SSID), function = run_program, cmd = cmd) - + def is_connected(): """Check for a valid private IP.""" devs = psutil.net_if_addrs() diff --git a/.bin/Scripts/functions/product_keys.py b/.bin/Scripts/functions/product_keys.py index 705f46e9..988d36fd 100644 --- a/.bin/Scripts/functions/product_keys.py +++ b/.bin/Scripts/functions/product_keys.py @@ -39,7 +39,7 @@ def extract_keys(): keys[product] = [] if key not in keys[product]: keys[product].append(key) - + # Done return keys diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index cd7f8444..0952e64d 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -17,7 +17,7 @@ WINDOWS_VERSIONS = [ {'Name': 'Windows 7 Ultimate', 'Image File': 'Win7', 'Image Name': 'Windows 7 ULTIMATE'}, - + {'Name': 'Windows 8.1', 'Image File': 'Win8', 'Image Name': 'Windows 8.1', @@ -25,7 +25,7 @@ WINDOWS_VERSIONS = [ {'Name': 'Windows 8.1 Pro', 'Image File': 'Win8', 'Image Name': 'Windows 8.1 Pro'}, - + {'Name': 'Windows 10 Home', 'Image File': 'Win10', 'Image Name': 'Windows 10 Home', @@ -75,7 +75,7 @@ def find_windows_image(windows_version): image['Glob'] = '--ref="{}*.swm"'.format( image['Path'][:-4]) break - + # Display image to be used (if any) and return if image: print_info('Using image: {}'.format(image['Path'])) @@ -122,7 +122,7 @@ def format_gpt(disk): 'set id="de94bba4-06d1-4d40-a16a-bfd50179d6ac"', 'gpt attributes=0x8000000000000001', ] - + # Run run_diskpart(script) @@ -151,7 +151,7 @@ def format_mbr(disk): 'assign letter="T"', 'set id=27', ] - + # Run run_diskpart(script) @@ -197,11 +197,11 @@ def setup_windows_re(windows_version, windows_letter='W', tools_letter='T'): win = r'{}:\Windows'.format(windows_letter) winre = r'{}\System32\Recovery\WinRE.wim'.format(win) dest = r'{}:\Recovery\WindowsRE'.format(tools_letter) - + # Copy WinRE.wim os.makedirs(dest, exist_ok=True) shutil.copy(winre, r'{}\WinRE.wim'.format(dest)) - + # Set location cmd = [ r'{}\System32\ReAgentc.exe'.format(win), @@ -231,7 +231,7 @@ def wim_contains_image(filename, imagename): run_program(cmd) except subprocess.CalledProcessError: return False - + return True if __name__ == '__main__': diff --git a/.bin/Scripts/functions/winpe_menus.py b/.bin/Scripts/functions/winpe_menus.py index 64ffb666..1c732eca 100644 --- a/.bin/Scripts/functions/winpe_menus.py +++ b/.bin/Scripts/functions/winpe_menus.py @@ -90,7 +90,7 @@ def menu_backup(): message = 'Assigning letters...', function = assign_volume_letters, other_results = other_results) - + # Mount backup shares mount_backup_shares(read_write=True) @@ -107,12 +107,12 @@ def menu_backup(): else: print_error('ERROR: No disks found.') raise GenericAbort - + # Select disk to backup disk = select_disk('For which disk are we creating backups?', disks) if not disk: raise GenericAbort - + # "Prep" disk prep_disk_for_backup(destination, disk, backup_prefix) @@ -150,7 +150,7 @@ def menu_backup(): # Ask to proceed if (not ask('Proceed with backup?')): raise GenericAbort - + # Backup partition(s) print_info('\n\nStarting task.\n') for par in disk['Partitions']: @@ -163,7 +163,7 @@ def menu_backup(): if not result['CS'] and not isinstance(result['Error'], GenericAbort): errors = True par['Error'] = result['Error'] - + # Verify backup(s) if disk['Valid Partitions']: print_info('\n\nVerifying backup images(s)\n') @@ -270,7 +270,7 @@ def menu_setup(): # Select the version of Windows to apply windows_version = select_windows_version() - + # Find Windows image # NOTE: Reassign volume letters to ensure all devices are scanned try_and_print( @@ -289,12 +289,12 @@ def menu_setup(): else: print_error('ERROR: No disks found.') raise GenericAbort - + # Select disk to use as the OS disk dest_disk = select_disk('To which disk are we installing Windows?', disks) if not dest_disk: raise GenericAbort - + # "Prep" disk prep_disk_for_formatting(dest_disk) @@ -323,10 +323,10 @@ def menu_setup(): data = par['Display String'], warning = True) print_warning(dest_disk['Format Warnings']) - + if (not ask('Is this correct?')): raise GenericAbort - + # Safety check print_standard('\nSAFETY CHECK') print_warning('All data will be DELETED from the ' @@ -342,7 +342,7 @@ def menu_setup(): function = remove_volume_letters, other_results = other_results, keep=windows_image['Letter']) - + # Assign new letter for local source if necessary if windows_image['Local'] and windows_image['Letter'] in ['S', 'T', 'W']: new_letter = try_and_print( @@ -377,13 +377,13 @@ def menu_setup(): # We need to crash as the disk is in an unknown state print_error('ERROR: Failed to apply image.') raise GenericAbort - + # Create Boot files try_and_print( message = 'Updating boot files...', function = update_boot_partition, other_results = other_results) - + # Setup WinRE try_and_print( message = 'Updating recovery tools...', @@ -392,8 +392,8 @@ def menu_setup(): windows_version = windows_version) # Copy WinPE log(s) - source = r'{}\Info'.format(global_vars['ClientDir']) - dest = r'W:\{}\Info'.format(KIT_NAME_SHORT) + source = r'{}\Logs'.format(global_vars['ClientDir']) + dest = r'W:\{}\Logs\WinPE'.format(KIT_NAME_SHORT) shutil.copytree(source, dest) # Print summary diff --git a/.bin/Scripts/safemode_enter.py b/.bin/Scripts/safemode_enter.py index c3213cb9..cce7e28a 100644 --- a/.bin/Scripts/safemode_enter.py +++ b/.bin/Scripts/safemode_enter.py @@ -17,16 +17,16 @@ if __name__ == '__main__': other_results = { 'Error': {'CalledProcessError': 'Unknown Error'}, 'Warning': {}} - + if not ask('Enable booting to SafeMode (with Networking)?'): abort() - + # Configure SafeMode try_and_print(message='Set BCD option...', function=enable_safemode, other_results=other_results) try_and_print(message='Enable MSI in SafeMode...', function=enable_safemode_msi, other_results=other_results) - + # Done print_standard('\nDone.') pause('Press Enter to reboot...') diff --git a/.bin/Scripts/safemode_exit.py b/.bin/Scripts/safemode_exit.py index 1449cff5..af66222e 100644 --- a/.bin/Scripts/safemode_exit.py +++ b/.bin/Scripts/safemode_exit.py @@ -17,16 +17,16 @@ if __name__ == '__main__': other_results = { 'Error': {'CalledProcessError': 'Unknown Error'}, 'Warning': {}} - + if not ask('Disable booting to SafeMode?'): abort() - + # Configure SafeMode try_and_print(message='Remove BCD option...', function=disable_safemode, other_results=other_results) try_and_print(message='Disable MSI in SafeMode...', function=disable_safemode_msi, other_results=other_results) - + # Done print_standard('\nDone.') pause('Press Enter to reboot...') diff --git a/.bin/Scripts/sfc_scan.py b/.bin/Scripts/sfc_scan.py index 81211747..d7a3d3fc 100644 --- a/.bin/Scripts/sfc_scan.py +++ b/.bin/Scripts/sfc_scan.py @@ -9,7 +9,7 @@ sys.path.append(os.getcwd()) from functions.repairs import * init_global_vars() os.system('title {}: SFC Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\SFC Tool.log'.format(**global_vars) +set_log_file('SFC Tool.log') if __name__ == '__main__': try: @@ -28,7 +28,7 @@ if __name__ == '__main__': function=run_sfc_scan, other_results=other_results) else: abort() - + # Done print_standard('\nDone.') pause('Press Enter to exit...') diff --git a/.bin/Scripts/transferred_keys.py b/.bin/Scripts/transferred_keys.py index 9829207e..b95ff3c9 100644 --- a/.bin/Scripts/transferred_keys.py +++ b/.bin/Scripts/transferred_keys.py @@ -9,7 +9,7 @@ sys.path.append(os.getcwd()) from functions.product_keys import * init_global_vars() os.system('title {}: Transferred Key Finder'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\Transferred Keys.log'.format(**global_vars) +set_log_file('Transferred Keys.log') if __name__ == '__main__': try: @@ -18,7 +18,7 @@ if __name__ == '__main__': print_info('{}: Transferred Key Finder\n'.format(KIT_NAME_FULL)) try_and_print(message='Searching for keys...', function=list_clientdir_keys, print_return=True) - + # Done print_standard('\nDone.') exit_script() diff --git a/.bin/Scripts/user_data_transfer.py b/.bin/Scripts/user_data_transfer.py index ce572f69..981e235a 100644 --- a/.bin/Scripts/user_data_transfer.py +++ b/.bin/Scripts/user_data_transfer.py @@ -10,7 +10,7 @@ from functions.data import * from functions.repairs import * init_global_vars() os.system('title {}: User Data Transfer Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\User Data Transfer.log'.format(**global_vars) +set_log_file('User Data Transfer.log') if __name__ == '__main__': try: @@ -18,7 +18,7 @@ if __name__ == '__main__': stay_awake() clear_screen() print_info('{}: User Data Transfer Tool\n'.format(KIT_NAME_FULL)) - + # Get backup name prefix ticket_number = get_ticket_number() if ENABLED_TICKET_NUMBERS: @@ -26,16 +26,16 @@ if __name__ == '__main__': else: backup_prefix = get_simple_string(prompt='Enter backup name prefix') backup_prefix = backup_prefix.replace(' ', '_') - + # Set destination folder_path = r'{}\Transfer'.format(KIT_NAME_SHORT) dest = select_destination(folder_path=folder_path, prompt='Which disk are we transferring to?') - + # Set source items source = select_source(backup_prefix) items = scan_source(source, dest) - + # Transfer clear_screen() print_info('Transfer Details:\n') @@ -43,17 +43,17 @@ if __name__ == '__main__': show_data('Ticket:', ticket_number) show_data('Source:', source.path) show_data('Destination:', dest) - + if (not ask('Proceed with transfer?')): umount_backup_shares() abort() - + print_info('Transferring Data') transfer_source(source, dest, items) try_and_print(message='Removing extra files...', function=cleanup_transfer, cs='Done', dest_path=dest) umount_backup_shares() - + # Done try_and_print(message='Running KVRT...', function=run_kvrt, cs='Started') diff --git a/.bin/Scripts/winpe_root_menu.py b/.bin/Scripts/winpe_root_menu.py index 03c763af..743d987b 100644 --- a/.bin/Scripts/winpe_root_menu.py +++ b/.bin/Scripts/winpe_root_menu.py @@ -11,7 +11,7 @@ from functions.winpe_menus import * TOOLS['SevenZip'].pop('64') init_global_vars() set_title('{}: Root Menu'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\WinPE.log'.format(**global_vars) +set_log_file('WinPE.log') if __name__ == '__main__': try: From 923ddd7cd388c97450a7e234dc844fbc82a81b5e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:04:12 -0700 Subject: [PATCH 005/121] Removed whitespace from more empty lines --- .bin/Scripts/install_sw_bundle.py | 6 ++++-- .bin/Scripts/install_vcredists.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/install_sw_bundle.py b/.bin/Scripts/install_sw_bundle.py index d98eb8d2..5ee1a8f3 100644 --- a/.bin/Scripts/install_sw_bundle.py +++ b/.bin/Scripts/install_sw_bundle.py @@ -9,7 +9,7 @@ sys.path.append(os.getcwd()) from functions.setup import * init_global_vars() os.system('title {}: SW Bundle Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\Install SW Bundle.log'.format(**global_vars) +set_log_file('Install SW Bundle.log') if __name__ == '__main__': try: @@ -34,7 +34,7 @@ if __name__ == '__main__': answer_mse = ask('Install MSE?') else: answer_mse = False - + print_info('Installing Programs') if answer_adobe_reader: try_and_print(message='Adobe Reader DC...', @@ -62,3 +62,5 @@ if __name__ == '__main__': pass except: major_exception() + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/install_vcredists.py b/.bin/Scripts/install_vcredists.py index 4a1f53ea..fd953551 100644 --- a/.bin/Scripts/install_vcredists.py +++ b/.bin/Scripts/install_vcredists.py @@ -9,7 +9,7 @@ sys.path.append(os.getcwd()) from functions.setup import * init_global_vars() os.system('title {}: Install Visual C++ Runtimes'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\Install Visual C++ Runtimes.log'.format(**global_vars) +set_log_file('Install Visual C++ Runtimes.log') if __name__ == '__main__': try: @@ -20,12 +20,12 @@ if __name__ == '__main__': 'Error': { 'CalledProcessError': 'Unknown Error', }} - + if ask('Install Visual C++ Runtimes?'): install_vcredists() else: abort() - + print_standard('\nDone.') exit_script() except SystemExit: From 18b13cf506a89d2f8f4a93853950601c1067f6f9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:06:57 -0700 Subject: [PATCH 006/121] Updated common.py --- .bin/Scripts/functions/common.py | 35 +++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 9bc4732d..ae958645 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -32,7 +32,8 @@ COLORS = { 'BLUE': '\033[34m' } try: - HKU = winreg.HKEY_USERS + HKU = winreg.HKEY_USERS + HKCR = winreg.HKEY_CLASSES_ROOT HKCU = winreg.HKEY_CURRENT_USER HKLM = winreg.HKEY_LOCAL_MACHINE except NameError: @@ -167,14 +168,13 @@ def exit_script(return_value=0): # Remove dirs (if empty) for dir in ['BackupDir', 'LogDir', 'TmpDir']: try: - dir = global_vars[dir] - os.rmdir(dir) + os.rmdir(global_vars[dir]) except Exception: pass # Open Log (if it exists) log = global_vars.get('LogFile', '') - if log and os.path.exists(log) and psutil.WINDOWS: + if log and os.path.exists(log) and psutil.WINDOWS and ENABLED_OPEN_LOGS: try: extract_item('NotepadPlusPlus', silent=True) popen_program( @@ -499,6 +499,8 @@ def sleep(seconds=2): def stay_awake(): """Prevent the system from sleeping or hibernating.""" + # DISABLED due to VCR2008 dependency + return # Bail if caffeine is already running for proc in psutil.process_iter(): if proc.name() == 'caffeine.exe': @@ -506,7 +508,7 @@ def stay_awake(): # Extract and run extract_item('Caffeine', silent=True) try: - popen_program(global_vars['Tools']['Caffeine']) + popen_program([global_vars['Tools']['Caffeine']]) except Exception: print_error('ERROR: No caffeine available.') print_warning('Please set the power setting to High Performance.') @@ -601,9 +603,10 @@ global_vars: {}'''.format(f.read(), sys.argv, global_vars) CRASH_SERVER['Url'], global_vars.get('Date-Time', 'Unknown Date-Time'), filename) - r = requests.put(url, data=data, - headers = {'X-Requested-With': 'XMLHttpRequest'}, - auth = (CRASH_SERVER['User'], CRASH_SERVER['Pass'])) + r = requests.put( + url, data=data, + headers={'X-Requested-With': 'XMLHttpRequest'}, + auth=(CRASH_SERVER['User'], CRASH_SERVER['Pass'])) # Raise exception if upload NS if not r.ok: raise Exception @@ -752,6 +755,9 @@ def make_tmp_dirs(): """Make temp directories.""" os.makedirs(global_vars['BackupDir'], exist_ok=True) os.makedirs(global_vars['LogDir'], exist_ok=True) + os.makedirs(r'{}\{}'.format( + global_vars['LogDir'], KIT_NAME_FULL), exist_ok=True) + os.makedirs(r'{}\Tools'.format(global_vars['LogDir']), exist_ok=True) os.makedirs(global_vars['TmpDir'], exist_ok=True) def set_common_vars(): @@ -767,11 +773,9 @@ def set_common_vars(): **global_vars) global_vars['ClientDir'] = r'{SYSTEMDRIVE}\{prefix}'.format( prefix=KIT_NAME_SHORT, **global_vars['Env']) - global_vars['BackupDir'] = r'{ClientDir}\Backups\{Date}'.format( + global_vars['BackupDir'] = r'{ClientDir}\Backups'.format( **global_vars) - global_vars['LogDir'] = r'{ClientDir}\Info\{Date}'.format( - **global_vars) - global_vars['ProgBackupDir'] = r'{ClientDir}\Backups'.format( + global_vars['LogDir'] = r'{ClientDir}\Logs\{Date}'.format( **global_vars) global_vars['QuarantineDir'] = r'{ClientDir}\Quarantine'.format( **global_vars) @@ -794,5 +798,12 @@ def set_linux_vars(): 'SevenZip': '7z', } +def set_log_file(log_name): + """Sets global var LogFile and creates path as needed.""" + folder_path = r'{}\{}'.format(global_vars['LogDir'], KIT_NAME_FULL) + log_file = r'{}\{}'.format(folder_path, log_name) + os.makedirs(folder_path, exist_ok=True) + global_vars['LogFile'] = log_file + if __name__ == '__main__': print("This file is not meant to be called directly.") From 7f7c220073ea57c2a4fd061e27fd3d70a92facf3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:08:41 -0700 Subject: [PATCH 007/121] Updated browsers.py --- .bin/Scripts/functions/browsers.py | 91 +++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index 99086f18..143d018d 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -2,6 +2,8 @@ from functions.common import * +from operator import itemgetter + # Define other_results for later try_and_print browser_data = {} other_results = { @@ -101,16 +103,63 @@ SUPPORTED_BROWSERS = { }, } +def archive_all_users(): + """Create backups for all browsers for all users.""" + users_root = r'{}\Users'.format(global_vars['Env']['SYSTEMDRIVE']) + user_envs = [] + + # Build list of valid users + for user_name in os.listdir(users_root): + valid_user = True + if user_name in ('Default', 'Default User'): + # Skip default users + continue + user_path = os.path.join(users_root, user_name) + appdata_local = os.path.join(user_path, r'AppData\Local') + appdata_roaming = os.path.join(user_path, r'AppData\Roaming') + valid_user &= os.path.exists(appdata_local) + valid_user &= os.path.exists(appdata_roaming) + if valid_user: + user_envs.append({ + 'USERNAME': user_name, + 'USERPROFILE': user_path, + 'APPDATA': appdata_roaming, + 'LOCALAPPDATA': appdata_local}) + + # Backup browsers for all valid users + print_info('Backing up browsers') + for fake_env in sorted(user_envs, key=itemgetter('USERPROFILE')): + print_standard(' {}'.format(fake_env['USERNAME'])) + for b_k, b_v in sorted(SUPPORTED_BROWSERS.items()): + if b_k == 'Mozilla Firefox Dev': + continue + source_path = b_v['user_data_path'].format(**fake_env) + if not os.path.exists(source_path): + continue + source_items = source_path + '*' + archive_path = r'{BackupDir}\Browsers ({USERNAME})\{Date}'.format( + **global_vars, **fake_env) + os.makedirs(archive_path, exist_ok=True) + archive_path += r'\{}.7z'.format(b_k) + cmd = [ + global_vars['Tools']['SevenZip'], + 'a', '-aoa', '-bso0', '-bse0', '-mx=1', + archive_path, source_items] + try_and_print(message='{}...'.format(b_k), + function=run_program, cmd=cmd) + print_standard(' ') + def archive_browser(name): """Create backup of Browser saved in the BackupDir.""" source = '{}*'.format(browser_data[name]['user_data_path']) - dest = r'{BackupDir}\Browsers ({USERNAME})'.format( + dest = r'{BackupDir}\Browsers ({USERNAME})\{Date}'.format( **global_vars, **global_vars['Env']) archive = r'{}\{}.7z'.format(dest, name) os.makedirs(dest, exist_ok=True) cmd = [ global_vars['Tools']['SevenZip'], 'a', '-aoa', '-bso0', '-bse0', '-mx=1', + '-mhe=on', '-p{}'.format(ARCHIVE_PASSWORD), archive, source] run_program(cmd) @@ -138,7 +187,7 @@ def clean_chromium_profile(profile): def clean_internet_explorer(**kwargs): """Uses the built-in function to reset IE and sets the homepage. - + NOTE: kwargs set but unused as a workaround.""" kill_process('iexplore.exe') run_program(['rundll32.exe', 'inetcpl.cpl,ResetIEtoDefaults'], check=False) @@ -182,11 +231,11 @@ def clean_mozilla_profile(profile): def get_browser_details(name): """Get installation status and profile details for all supported browsers.""" browser = SUPPORTED_BROWSERS[name].copy() - + # Update user_data_path browser['user_data_path'] = browser['user_data_path'].format( **global_vars['Env']) - + # Find executable (if multiple files are found, the last one is used) exe_path = None num_installs = 0 @@ -197,7 +246,7 @@ def get_browser_details(name): if os.path.exists(test_path): num_installs += 1 exe_path = test_path - + # Find profile(s) profiles = [] if browser['base'] == 'ie': @@ -225,12 +274,12 @@ def get_browser_details(name): profiles.extend( get_mozilla_profiles( search_path=browser['user_data_path'], dev=dev)) - + elif 'Opera' in name: if os.path.exists(browser['user_data_path']): profiles.append( {'name': 'Default', 'path': browser['user_data_path']}) - + # Get homepages if browser['base'] == 'ie': # IE is set to only have one profile above @@ -239,14 +288,14 @@ def get_browser_details(name): for profile in profiles: prefs_path = r'{path}\prefs.js'.format(**profile) profile['homepages'] = get_mozilla_homepages(prefs_path=prefs_path) - + # Add to browser_data browser_data[name] = browser browser_data[name].update({ 'exe_path': exe_path, 'profiles': profiles, }) - + # Raise installation warnings (if any) if num_installs == 0: raise NotInstalledError @@ -305,7 +354,7 @@ def get_mozilla_homepages(prefs_path): homepages = search.group(1).split('|') except Exception: pass - + return homepages def get_mozilla_profiles(search_path, dev=False): @@ -332,9 +381,11 @@ def get_mozilla_profiles(search_path, dev=False): return profiles -def install_adblock(indent=8, width=32): +def install_adblock(indent=8, width=32, just_firefox=False): """Install adblock for all supported browsers.""" for browser in sorted(browser_data): + if just_firefox and browser_data[browser]['base'] != 'mozilla': + continue exe_path = browser_data[browser].get('exe_path', None) function=run_program if not exe_path: @@ -362,7 +413,7 @@ def install_adblock(indent=8, width=32): winreg.QueryValue(HKLM, UBO_EXTRA_CHROME_REG) except FileNotFoundError: urls.append(UBO_EXTRA_CHROME) - + if len(urls) == 0: urls = ['chrome://extensions'] elif 'Opera' in browser: @@ -370,7 +421,7 @@ def install_adblock(indent=8, width=32): else: urls.append(UBO_CHROME) urls.append(UBO_EXTRA_CHROME) - + elif browser_data[browser]['base'] == 'mozilla': # Check for system extensions try: @@ -383,11 +434,11 @@ def install_adblock(indent=8, width=32): urls = ['about:addons'] else: urls = [UBO_MOZILLA] - + elif browser_data[browser]['base'] == 'ie': urls.append(IE_GALLERY) function=popen_program - + # By using check=False we're skipping any return codes so # it should only fail if the program can't be run # (or can't be found). @@ -400,7 +451,7 @@ def install_adblock(indent=8, width=32): def list_homepages(indent=8, width=32): """List current homepages for reference.""" - + for browser in [k for k, v in sorted(browser_data.items()) if v['exe_path']]: # Skip Chromium-based browsers if browser_data[browser]['base'] == 'chromium': @@ -410,7 +461,7 @@ def list_homepages(indent=8, width=32): end='', flush=True) print_warning('Not implemented', timestamp=False) continue - + # All other browsers print_info('{indent}{browser:<{width}}'.format( indent=' '*indent, width=width, browser=browser+'...')) @@ -444,9 +495,11 @@ def reset_browsers(indent=8, width=32): indent=indent, width=width, function=function, other_results=other_results, profile=profile) -def scan_for_browsers(): +def scan_for_browsers(just_firefox=False): """Scan system for any supported browsers.""" - for name in sorted(SUPPORTED_BROWSERS): + for name, details in sorted(SUPPORTED_BROWSERS.items()): + if just_firefox and details['base'] != 'mozilla': + continue try_and_print(message='{}...'.format(name), function=get_browser_details, cs='Detected', other_results=other_results, name=name) From 1839edf84d84623dc140acba49bebef67c14d34a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:09:31 -0700 Subject: [PATCH 008/121] Updated cleanup.py --- .bin/Scripts/functions/cleanup.py | 72 +++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/.bin/Scripts/functions/cleanup.py b/.bin/Scripts/functions/cleanup.py index 1bac4c6c..d401b555 100644 --- a/.bin/Scripts/functions/cleanup.py +++ b/.bin/Scripts/functions/cleanup.py @@ -6,7 +6,7 @@ def cleanup_adwcleaner(): """Move AdwCleaner folders into the ClientDir.""" source_path = r'{SYSTEMDRIVE}\AdwCleaner'.format(**global_vars['Env']) source_quarantine = r'{}\Quarantine'.format(source_path) - + # Quarantine if os.path.exists(source_quarantine): os.makedirs(global_vars['QuarantineDir'], exist_ok=True) @@ -14,27 +14,24 @@ def cleanup_adwcleaner(): **global_vars) dest_name = non_clobber_rename(dest_name) shutil.move(source_quarantine, dest_name) - + # Delete source folder if empty - try: - os.rmdir(source_path) - except OSError: - pass - + delete_empty_folders(source_path) + # Main folder if os.path.exists(source_path): - os.makedirs(global_vars['ProgBackupDir'], exist_ok=True) - dest_name = r'{ProgBackupDir}\AdwCleaner_{Date-Time}'.format( + os.makedirs(global_vars['LogDir'], exist_ok=True) + dest_name = r'{LogDir}\Tools\AdwCleaner'.format( **global_vars) dest_name = non_clobber_rename(dest_name) shutil.move(source_path, dest_name) def cleanup_cbs(dest_folder): """Safely cleanup a known CBS archive bug under Windows 7. - + If a CbsPersist file is larger than 2 Gb then the auto archive feature continually fails and will fill up the system drive with temp files. - + This function moves the temp files and CbsPersist file to a temp folder, compresses the CbsPersist files with 7-Zip, and then opens the temp folder for the user to manually save the backup files and delete the temp files. @@ -43,7 +40,7 @@ def cleanup_cbs(dest_folder): temp_folder = r'{backup_folder}\Temp'.format(backup_folder=backup_folder) os.makedirs(backup_folder, exist_ok=True) os.makedirs(temp_folder, exist_ok=True) - + # Move files into temp folder cbs_path = r'{SYSTEMROOT}\Logs\CBS'.format(**global_vars['Env']) for entry in os.scandir(cbs_path): @@ -59,7 +56,7 @@ def cleanup_cbs(dest_folder): dest_name = r'{}\{}'.format(temp_folder, entry.name) dest_name = non_clobber_rename(dest_name) shutil.move(entry.path, dest_name) - + # Compress CbsPersist files with 7-Zip cmd = [ global_vars['Tools']['SevenZip'], @@ -70,9 +67,9 @@ def cleanup_cbs(dest_folder): def cleanup_desktop(): """Move known backup files and reports into the ClientDir.""" - dest_folder = r'{ProgBackupDir}\Desktop_{Date-Time}'.format(**global_vars) + dest_folder = r'{LogDir}\Tools'.format(**global_vars) os.makedirs(dest_folder, exist_ok=True) - + desktop_path = r'{USERPROFILE}\Desktop'.format(**global_vars['Env']) for entry in os.scandir(desktop_path): # JRT, RKill, Shortcut cleaner @@ -80,12 +77,53 @@ def cleanup_desktop(): dest_name = r'{}\{}'.format(dest_folder, entry.name) dest_name = non_clobber_rename(dest_name) shutil.move(entry.path, dest_name) - + # Remove dir if empty + delete_empty_folders(dest_folder) + +def delete_empty_folders(folder_path): + """Delete all empty folders in path (depth first).""" + if not os.path.exists(folder_path) or not os.path.isdir(folder_path): + # Bail early (silently) + return + + # Delete empty subfolders first + for item in os.scandir(folder_path): + if item.is_dir(): + delete_empty_folders(item.path) + + # Remove top folder try: - os.rmdir(dest_folder) + os.rmdir(folder_path) except OSError: pass +def delete_registry_key(hive, key, recurse=False): + """Delete a registry key and all it's subkeys.""" + access = winreg.KEY_ALL_ACCESS + + try: + if recurse: + # Delete all subkeys first + with winreg.OpenKeyEx(hive, key, 0, access) as k: + key_info = winreg.QueryInfoKey(k) + for x in range(key_info[0]): + subkey = r'{}\{}'.format(key, winreg.EnumKey(k, 0)) + delete_registry_key(hive, subkey) + + # Delete key + winreg.DeleteKey(hive, key) + except FileNotFoundError: + # Ignore + pass + +def delete_registry_value(hive, key, value): + """Delete a registry value.""" + access = winreg.KEY_ALL_ACCESS + with winreg.OpenKeyEx(hive, key, 0, access) as k: + winreg.DeleteValue(k, value) + if __name__ == '__main__': print("This file is not meant to be called directly.") + +# vim: sts=4 sw=4 ts=4 From 7f37bc88021458f8f70e7cec3b0610b2266b4d3e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:10:50 -0700 Subject: [PATCH 009/121] Updated diags.py --- .bin/Scripts/functions/diags.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/diags.py b/.bin/Scripts/functions/diags.py index f66d59e0..e55f5b12 100644 --- a/.bin/Scripts/functions/diags.py +++ b/.bin/Scripts/functions/diags.py @@ -38,7 +38,7 @@ def check_connection(): else: abort() -def check_secure_boot_status(): +def check_secure_boot_status(show_alert=False): """Checks UEFI Secure Boot status via PowerShell.""" boot_mode = get_boot_mode() cmd = ['PowerShell', '-Command', 'Confirm-SecureBootUEFI'] @@ -51,18 +51,30 @@ def check_secure_boot_status(): # It's on, do nothing return elif 'False' in out: + if show_alert: + show_alert_box('Secure Boot DISABLED') raise SecureBootDisabledError else: + if show_alert: + show_alert_box('Secure Boot status UNKNOWN') raise SecureBootUnknownError else: if boot_mode != 'UEFI': + if (show_alert and + global_vars['OS']['Version'] in ('8', '8.1', '10')): + # OS supports Secure Boot + show_alert_box('Secure Boot DISABLED\n\nOS installed LEGACY') raise OSInstalledLegacyError else: # Check error message err = result.stderr.decode() if 'Cmdlet not supported' in err: + if show_alert: + show_alert_box('Secure Boot UNAVAILABLE?') raise SecureBootNotAvailError else: + if show_alert: + show_alert_box('Secure Boot ERROR') raise GenericError def get_boot_mode(): @@ -123,7 +135,8 @@ def run_xmplay(): r'{BinDir}\XMPlay\music.7z'.format(**global_vars)] # Unmute audio first - run_nircmd(['mutesysvolume', '0']) + extract_item('NirCmd', silent=True) + run_nircmd('mutesysvolume', '0') # Open XMPlay popen_program(cmd) @@ -134,7 +147,7 @@ def run_hitmanpro(): cmd = [ global_vars['Tools']['HitmanPro'], '/quiet', '/noinstall', '/noupload', - r'/log={LogDir}\hitman.xml'.format(**global_vars)] + r'/log={LogDir}\Tools\HitmanPro.txt'.format(**global_vars)] popen_program(cmd) def run_process_killer(): @@ -152,23 +165,25 @@ def run_rkill(): extract_item('RKill', silent=True) cmd = [ global_vars['Tools']['RKill'], - '-l', r'{LogDir}\RKill.log'.format(**global_vars), + '-s', '-l', r'{LogDir}\Tools\RKill.log'.format(**global_vars), '-new_console:n', '-new_console:s33V'] run_program(cmd, check=False) wait_for_process('RKill') - kill_process('notepad.exe') # RKill cleanup desktop_path = r'{USERPROFILE}\Desktop'.format(**global_vars['Env']) if os.path.exists(desktop_path): for item in os.scandir(desktop_path): if re.search(r'^RKill', item.name, re.IGNORECASE): - dest = re.sub(r'^(.*)\.', '\1_{Date-Time}.'.format( - **global_vars), item.name) - dest = r'{ClientDir}\Info\{name}'.format( + dest = r'{LogDir}\Tools\{name}'.format( name=dest, **global_vars) dest = non_clobber_rename(dest) shutil.move(item.path, dest) +def show_alert_box(message, title='Wizard Kit Warning'): + """Show Windows alert box with message.""" + message_box = ctypes.windll.user32.MessageBoxW + message_box(None, message, title, 0x00001030) + if __name__ == '__main__': print("This file is not meant to be called directly.") From 1cac083916867d31082e608ca62b50db87d2ba97 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:12:36 -0700 Subject: [PATCH 010/121] Updated info.py --- .bin/Scripts/functions/info.py | 89 ++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/.bin/Scripts/functions/info.py b/.bin/Scripts/functions/info.py index b81a4922..7630a766 100644 --- a/.bin/Scripts/functions/info.py +++ b/.bin/Scripts/functions/info.py @@ -68,7 +68,8 @@ def backup_file_list(): def backup_power_plans(): """Export current power plans.""" - os.makedirs(r'{BackupDir}\Power Plans'.format(**global_vars), exist_ok=True) + os.makedirs(r'{BackupDir}\Power Plans\{Date}'.format( + **global_vars), exist_ok=True) plans = run_program(['powercfg', '/L']) plans = plans.stdout.decode().splitlines() plans = [p for p in plans if re.search(r'^Power Scheme', p)] @@ -76,22 +77,24 @@ def backup_power_plans(): guid = re.sub(r'Power Scheme GUID:\s+([0-9a-f\-]+).*', r'\1', p) name = re.sub( r'Power Scheme GUID:\s+[0-9a-f\-]+\s+\(([^\)]+)\).*', r'\1', p) - out = r'{BackupDir}\Power Plans\{name}.pow'.format( + out = r'{BackupDir}\Power Plans\{Date}\{name}.pow'.format( name=name, **global_vars) if not os.path.exists(out): cmd = ['powercfg', '-export', out, guid] run_program(cmd, check=False) -def backup_registry(): +def backup_registry(overwrite=False): """Backup registry including user hives.""" extract_item('erunt', silent=True) cmd = [ global_vars['Tools']['ERUNT'], - r'{BackupDir}\Registry'.format(**global_vars), + r'{BackupDir}\Registry\{Date}'.format(**global_vars), 'sysreg', 'curuser', 'otherusers', '/noprogresswindow'] + if overwrite: + cmd.append('/noconfirmdelete') run_program(cmd) def get_folder_size(path): @@ -162,7 +165,7 @@ def get_installed_office(): def get_shell_path(folder, user='current'): """Get shell path using SHGetKnownFolderPath via knownpaths, returns str. - + NOTE: Only works for the current user. Code based on https://gist.github.com/mkropat/7550097 """ @@ -175,14 +178,14 @@ def get_shell_path(folder, user='current'): except AttributeError: # Unknown folder ID, ignore and return None pass - + if folderid: try: path = knownpaths.get_path(folderid, getattr(knownpaths.UserHandle, user)) except PathNotFoundError: # Folder not found, ignore and return None pass - + return path def get_user_data_paths(user): @@ -196,7 +199,7 @@ def get_user_data_paths(user): 'Extra Folders': {}, } unload_hive = False - + if user['Name'] == global_vars['Env']['USERNAME']: # We can use SHGetKnownFolderPath for the current user paths['Profile']['Path'] = get_shell_path('Profile') @@ -212,7 +215,7 @@ def get_user_data_paths(user): except Exception: # Profile path not found, leaving as None. pass - + # Shell folders (Prep) if not reg_path_exists(HKU, hive_path) and paths['Profile']['Path']: # User not logged-in, loading hive @@ -226,7 +229,7 @@ def get_user_data_paths(user): except subprocess.CalledProcessError: # Failed to load user hive pass - + # Shell folders shell_folders = r'{}\{}'.format(hive_path, REG_SHELL_FOLDERS) if (reg_path_exists(HKU, hive_path) @@ -252,7 +255,7 @@ def get_user_data_paths(user): if (folder not in paths['Shell Folders'] and os.path.exists(folder_path)): paths['Shell Folders'][folder] = {'Path': folder_path} - + # Extra folders if paths['Profile']['Path']: for folder in EXTRA_FOLDERS: @@ -260,12 +263,12 @@ def get_user_data_paths(user): folder=folder, **paths['Profile']) if os.path.exists(folder_path): paths['Extra Folders'][folder] = {'Path': folder_path} - + # Shell folders (cleanup) if unload_hive: cmd = ['reg', 'unload', r'HKU\{}'.format(TMP_HIVE_PATH)] run_program(cmd, check=False) - + # Done return paths @@ -277,7 +280,7 @@ def get_user_folder_sizes(users): with winreg.OpenKey(HKCU, r'Software\Sysinternals\Du', access=winreg.KEY_WRITE) as key: winreg.SetValueEx(key, 'EulaAccepted', 0, winreg.REG_DWORD, 1) - + for u in users: u.update(get_user_data_paths(u)) if u['Profile']['Path']: @@ -292,7 +295,7 @@ def get_user_folder_sizes(users): def get_user_list(): """Get user list via WMIC, returns list of dicts.""" users = [] - + # Get user info from WMI cmd = ['wmic', 'useraccount', 'get', '/format:csv'] try: @@ -300,10 +303,10 @@ def get_user_list(): except subprocess.CalledProcessError: # Meh, return empty list to avoid a full crash return users - + entries = out.stdout.decode().splitlines() entries = [e.strip().split(',') for e in entries if e.strip()] - + # Add user(s) to dict keys = entries[0] for e in entries[1:]: @@ -314,10 +317,10 @@ def get_user_list(): # Assume SIDs ending with 1000+ are "Standard" and others are "System" e['Type'] = 'Standard' if re.search(r'-1\d+$', e['SID']) else 'System' users.append(e) - + # Sort list users.sort(key=itemgetter('Name')) - + # Done return users @@ -368,26 +371,38 @@ def run_aida64(): '/TEXT', '/SILENT', '/SAFEST'] run_program(cmd, check=False) -def run_bleachbit(): +def run_bleachbit(cleaners=None, preview=True): """Run BleachBit preview and save log. - - This is a preview so no files should be deleted.""" - if not os.path.exists(global_vars['LogDir']+r'\BleachBit.log'): - extract_item('BleachBit', silent=True) - cmd = [global_vars['Tools']['BleachBit'], '--preview', '--preset'] - out = run_program(cmd, check=False) - # Save stderr - if out.stderr.decode().splitlines(): - with open(global_vars['LogDir']+r'\BleachBit.err', 'a', - encoding='utf-8') as f: - for line in out.stderr.decode().splitlines(): - f.write(line.strip() + '\n') - # Save stdout - with open(global_vars['LogDir']+r'\BleachBit.log', 'a', - encoding='utf-8') as f: - for line in out.stdout.decode().splitlines(): + + If preview is True then no files should be deleted.""" + error_path = r'{}\Tools\BleachBit.err'.format(global_vars['LogDir']) + log_path = error_path.replace('err', 'log') + extract_item('BleachBit', silent=True) + + # Safety check + if not cleaners: + # Disable cleaning and use preset config + cleaners = ['--preset'] + preview = True + + # Run + cmd = [ + global_vars['Tools']['BleachBit'], + '--preview' if preview else '--clean'] + cmd.extend(cleaners) + out = run_program(cmd, check=False) + + # Save stderr + if out.stderr.decode().splitlines(): + with open(error_path, 'a', encoding='utf-8') as f: + for line in out.stderr.decode().splitlines(): f.write(line.strip() + '\n') + # Save stdout + with open(log_path, 'a', encoding='utf-8') as f: + for line in out.stdout.decode().splitlines(): + f.write(line.strip() + '\n') + def show_disk_usage(disk): """Show free and used space for a specified disk.""" print_standard('{:5}'.format(disk.device.replace('/', ' ')), @@ -459,7 +474,7 @@ def show_os_name(): def show_temp_files_size(): """Show total size of temp files identified by BleachBit.""" size = None - with open(r'{LogDir}\BleachBit.log'.format(**global_vars), 'r') as f: + with open(r'{LogDir}\Tools\BleachBit.log'.format(**global_vars), 'r') as f: for line in f.readlines(): if re.search(r'^disk space to be recovered:', line, re.IGNORECASE): size = re.sub(r'.*: ', '', line.strip()) From 832ef993e69c625c239ade1c64825545cae47e20 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:13:11 -0700 Subject: [PATCH 011/121] Updated repairs.py --- .bin/Scripts/functions/repairs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/repairs.py b/.bin/Scripts/functions/repairs.py index e4d5e74f..589dccc3 100644 --- a/.bin/Scripts/functions/repairs.py +++ b/.bin/Scripts/functions/repairs.py @@ -24,11 +24,11 @@ def run_chkdsk_scan(): raise GenericError # Save stderr - with open(r'{LogDir}\CHKDSK.err'.format(**global_vars), 'a') as f: + with open(r'{LogDir}\Tools\CHKDSK.err'.format(**global_vars), 'a') as f: for line in out.stderr.decode().splitlines(): f.write(line.strip() + '\n') # Save stdout - with open(r'{LogDir}\CHKDSK.log'.format(**global_vars), 'a') as f: + with open(r'{LogDir}\Tools\CHKDSK.log'.format(**global_vars), 'a') as f: for line in out.stdout.decode().splitlines(): f.write(line.strip() + '\n') @@ -50,7 +50,7 @@ def run_dism(repair=False): cmd = [ 'DISM', '/Online', '/Cleanup-Image', '/RestoreHealth', - r'/LogPath:"{LogDir}\DISM_RestoreHealth.log"'.format( + r'/LogPath:"{LogDir}\Tools\DISM_RestoreHealth.log"'.format( **global_vars), '-new_console:n', '-new_console:s33V'] else: @@ -58,7 +58,7 @@ def run_dism(repair=False): cmd = [ 'DISM', '/Online', '/Cleanup-Image', '/ScanHealth', - r'/LogPath:"{LogDir}\DISM_ScanHealth.log"'.format( + r'/LogPath:"{LogDir}\Tools\DISM_ScanHealth.log"'.format( **global_vars), '-new_console:n', '-new_console:s33V'] run_program(cmd, pipe=False, check=False, shell=True) @@ -67,7 +67,7 @@ def run_dism(repair=False): cmd = [ 'DISM', '/Online', '/Cleanup-Image', '/CheckHealth', - r'/LogPath:"{LogDir}\DISM_CheckHealth.log"'.format(**global_vars)] + r'/LogPath:"{LogDir}\Tools\DISM_CheckHealth.log"'.format(**global_vars)] result = run_program(cmd, shell=True).stdout.decode() # Check result if 'no component store corruption detected' not in result.lower(): @@ -93,11 +93,11 @@ def run_sfc_scan(): '/scannow'] out = run_program(cmd, check=False) # Save stderr - with open(r'{LogDir}\SFC.err'.format(**global_vars), 'a') as f: + with open(r'{LogDir}\Tools\SFC.err'.format(**global_vars), 'a') as f: for line in out.stderr.decode('utf-8', 'ignore').splitlines(): f.write(line.strip() + '\n') # Save stdout - with open(r'{LogDir}\SFC.log'.format(**global_vars), 'a') as f: + with open(r'{LogDir}\Tools\SFC.log'.format(**global_vars), 'a') as f: for line in out.stdout.decode('utf-8', 'ignore').splitlines(): f.write(line.strip() + '\n') # Check result @@ -116,7 +116,7 @@ def run_tdsskiller(): **global_vars), exist_ok=True) cmd = [ global_vars['Tools']['TDSSKiller'], - '-l', r'{LogDir}\TDSSKiller.log'.format(**global_vars), + '-l', r'{LogDir}\Tools\TDSSKiller.log'.format(**global_vars), '-qpath', r'{QuarantineDir}\TDSSKiller'.format(**global_vars), '-accepteula', '-accepteulaksn', '-dcexact', '-tdlfs'] From 7e960df2d6353c608137ca5848a2aadb9ac50df0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:18:00 -0700 Subject: [PATCH 012/121] Updated setup.py --- .bin/Scripts/functions/setup.py | 91 ++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index c5c10a48..4e1e2ee7 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -1,8 +1,11 @@ # Wizard Kit: Functions - Setup from functions.common import * +from functions.update import * # STATIC VARIABLES +HKU = winreg.HKEY_USERS +HKCR = winreg.HKEY_CLASSES_ROOT HKCU = winreg.HKEY_CURRENT_USER HKLM = winreg.HKEY_LOCAL_MACHINE MOZILLA_FIREFOX_UBO_PATH = r'{}\{}\ublock_origin.xpi'.format( @@ -29,15 +32,22 @@ SETTINGS_CLASSIC_START = { }, } SETTINGS_EXPLORER_SYSTEM = { + # Disable Location Tracking + r'Software\Microsoft\Windows NT\CurrentVersion\Sensor\Overrides\{BFA794E4-F964-4FDB-90F6-51056BFE4B44}': { + 'DWORD Items': {'SensorPermissionState': 0}, + }, + r'System\CurrentControlSet\Services\lfsvc\Service\Configuration': { + 'Status': {'Value': 0}, + }, # Disable Telemetry - r'SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { + r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { 'DWORD Items': {'AllowTelemetry': 0}, }, - r'SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { + r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { 'DWORD Items': {'AllowTelemetry': 0}, 'WOW64_32': True, }, - r'SOFTWARE\Policies\Microsoft\Windows\DataCollection': { + r'Software\Policies\Microsoft\Windows\DataCollection': { 'DWORD Items': {'AllowTelemetry': 0}, }, # Disable Wi-Fi Sense @@ -47,27 +57,19 @@ SETTINGS_EXPLORER_SYSTEM = { r'Software\Microsoft\PolicyManager\default\WiFi\AllowAutoConnectToWiFiSenseHotspots': { 'DWORD Items': {'Value': 0}, }, - # Disable Location Tracking - r'SOFTWARE\Microsoft\Windows NT\CurrentVersion\Sensor\Overrides\{BFA794E4-F964-4FDB-90F6-51056BFE4B44}': { - 'DWORD Items': {'SensorPermissionState': 0}, - }, - r'System\CurrentControlSet\Services\lfsvc\Service\Configuration': { - 'Status': {'Value': 0}, - }, } SETTINGS_EXPLORER_USER = { - # Disable Cortana - r'Software\Microsoft\Personalization\Settings': { - 'DWORD Items': {'AcceptedPrivacyPolicy': 0}, + # Disable silently installed apps + r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { + 'DWORD Items': {'SilentInstalledAppsEnabled': 0}, }, - r'Software\Microsoft\InputPersonalization': { - 'DWORD Items': { - 'RestrictImplicitTextCollection': 1, - 'RestrictImplicitInkCollection': 1 - }, + # Disable Tips and Tricks + r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { + 'DWORD Items': {'SoftLandingEnabled ': 0}, }, - r'Software\Microsoft\InputPersonalization\TrainedDataStore': { - 'DWORD Items': {'HarvestContacts': 1}, + # Hide People bar + r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced\People': { + 'DWORD Items': {'PeopleBand': 0}, }, # Hide Search button / box r'Software\Microsoft\Windows\CurrentVersion\Search': { @@ -178,25 +180,6 @@ def config_classicstart(): sleep(1) popen_program(cs_exe) -def write_registry_settings(settings, all_users=False): - """Write registry values from custom dict of dicts.""" - hive = HKCU - if all_users: - hive = HKLM - for k, v in settings.items(): - # CreateKey - access = winreg.KEY_WRITE - if 'WOW64_32' in v: - access = access | winreg.KEY_WOW64_32KEY - winreg.CreateKeyEx(hive, k, 0, access) - - # Create values - with winreg.OpenKeyEx(hive, k, 0, access) as key: - for name, value in v.get('DWORD Items', {}).items(): - winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) - for name, value in v.get('SZ Items', {}).items(): - winreg.SetValueEx(key, name, 0, winreg.REG_SZ, value) - def config_explorer_system(): """Configure Windows Explorer for all users via Registry settings.""" write_registry_settings(SETTINGS_EXPLORER_SYSTEM, all_users=True) @@ -205,6 +188,15 @@ def config_explorer_user(): """Configure Windows Explorer for current user via Registry settings.""" write_registry_settings(SETTINGS_EXPLORER_USER, all_users=False) +def disable_windows_telemetry(): + """Disable Windows 10 telemetry settings with O&O ShutUp10.""" + extract_item('ShutUp10', silent=True) + cmd = [ + r'{BinDir}\ShutUp10\OOSU10.exe'.format(**global_vars), + r'{BinDir}\ShutUp10\1201.cfg'.format(**global_vars), + '/quiet'] + run_program(cmd) + def update_clock(): """Set Timezone and sync clock.""" run_program(['tzutil' ,'/s', WINDOWS_TIME_ZONE], check=False) @@ -217,6 +209,25 @@ def update_clock(): run_program(['net', 'start', 'w32ime'], check=False) run_program(['w32tm', '/resync', '/nowait'], check=False) +def write_registry_settings(settings, all_users=False): + """Write registry values from custom dict of dicts.""" + hive = HKCU + if all_users: + hive = HKLM + for k, v in settings.items(): + # CreateKey + access = winreg.KEY_WRITE + if 'WOW64_32' in v: + access = access | winreg.KEY_WOW64_32KEY + winreg.CreateKeyEx(hive, k, 0, access) + + # Create values + with winreg.OpenKeyEx(hive, k, 0, access) as key: + for name, value in v.get('DWORD Items', {}).items(): + winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) + for name, value in v.get('SZ Items', {}).items(): + winreg.SetValueEx(key, name, 0, winreg.REG_SZ, value) + # Installations def install_adobe_reader(): """Install Adobe Reader.""" @@ -257,7 +268,7 @@ def install_firefox_extensions(): # Update registry write_registry_settings(SETTINGS_MOZILLA_FIREFOX_32, all_users=True) write_registry_settings(SETTINGS_MOZILLA_FIREFOX_64, all_users=True) - + # Extract extension(s) to distribution folder cmd = [ global_vars['Tools']['SevenZip'], 'e', '-aos', '-bso0', '-bse0', From 5da9d4e36bc77c8adc002fe46119580c985e5d26 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:21:33 -0700 Subject: [PATCH 013/121] Updated update.py and update_kit.py --- .bin/Scripts/functions/update.py | 17 +++++++++++++++++ .bin/Scripts/update_kit.py | 9 +++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index ca48ff3f..b4068c22 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -138,7 +138,9 @@ def remove_from_kit(item): item_locations = [] for p in [global_vars['BinDir'], global_vars['CBinDir']]: item_locations.append(r'{}\{}'.format(p, item)) + item_locations.append(r'{}\{}.7z'.format(p, item)) item_locations.append(r'{}\_Drivers\{}'.format(p, item)) + item_locations.append(r'{}\_Drivers\{}.7z'.format(p, item)) for item_path in item_locations: remove_item(item_path) @@ -597,6 +599,21 @@ def update_adobe_reader_dc(): download_generic( dest, 'Adobe Reader DC.exe', SOURCE_URLS['Adobe Reader DC']) +def update_macs_fan_control(): + # Prep + dest = r'{}\Installers'.format( + global_vars['BaseDir']) + + # Remove existing installer + try: + os.remove(r'{}\Macs Fan Control.exe'.format(dest)) + except FileNotFoundError: + pass + + # Download + download_generic( + dest, 'Macs Fan Control.exe', SOURCE_URLS['Macs Fan Control']) + def update_office(): # Remove existing folders remove_from_kit('_Office') diff --git a/.bin/Scripts/update_kit.py b/.bin/Scripts/update_kit.py index 075c699c..67949d93 100644 --- a/.bin/Scripts/update_kit.py +++ b/.bin/Scripts/update_kit.py @@ -40,9 +40,9 @@ if __name__ == '__main__': try_and_print(message='AIDA64...', function=update_aida64, other_results=other_results, width=40) try_and_print(message='Autoruns...', function=update_autoruns, other_results=other_results, width=40) try_and_print(message='BleachBit...', function=update_bleachbit, other_results=other_results, width=40) - try_and_print(message='BlueScreenView...', function=update_bluescreenview, other_results=other_results, width=40) + try_and_print(message='Blue Screen View...', function=update_bluescreenview, other_results=other_results, width=40) try_and_print(message='ERUNT...', function=update_erunt, other_results=other_results, width=40) - try_and_print(message='HitmanPro...', function=update_hitmanpro, other_results=other_results, width=40) + try_and_print(message='Hitman Pro...', function=update_hitmanpro, other_results=other_results, width=40) try_and_print(message='HWiNFO...', function=update_hwinfo, other_results=other_results, width=40) try_and_print(message='NirCmd...', function=update_nircmd, other_results=other_results, width=40) try_and_print(message='ProduKey...', function=update_produkey, other_results=other_results, width=40) @@ -58,6 +58,7 @@ if __name__ == '__main__': # Installers print_info(' Installers') try_and_print(message='Adobe Reader DC...', function=update_adobe_reader_dc, other_results=other_results, width=40) + try_and_print(message='Macs Fan Control...', function=update_macs_fan_control, other_results=other_results, width=40) try_and_print(message='MS Office...', function=update_office, other_results=other_results, width=40) try_and_print(message='Visual C++ Runtimes...', function=update_vcredists, other_results=other_results, width=40) update_all_ninite(other_results=other_results, width=40) @@ -68,7 +69,7 @@ if __name__ == '__main__': try_and_print(message='Classic Start Skin...', function=update_classic_start_skin, other_results=other_results, width=40) try_and_print(message='Du...', function=update_du, other_results=other_results, width=40) try_and_print(message='Everything...', function=update_everything, other_results=other_results, width=40) - try_and_print(message='FirefoxExtensions...', function=update_firefox_ublock_origin, other_results=other_results, width=40) + try_and_print(message='Firefox Extensions...', function=update_firefox_ublock_origin, other_results=other_results, width=40) try_and_print(message='PuTTY...', function=update_putty, other_results=other_results, width=40) try_and_print(message='Notepad++...', function=update_notepadplusplus, other_results=other_results, width=40) try_and_print(message='WizTree...', function=update_wiztree, other_results=other_results, width=40) @@ -79,7 +80,7 @@ if __name__ == '__main__': try_and_print(message='AdwCleaner...', function=update_adwcleaner, other_results=other_results, width=40) try_and_print(message='KVRT...', function=update_kvrt, other_results=other_results, width=40) try_and_print(message='RKill...', function=update_rkill, other_results=other_results, width=40) - try_and_print(message='TDSSKiller...', function=update_tdsskiller, other_results=other_results, width=40) + try_and_print(message='TDSS Killer...', function=update_tdsskiller, other_results=other_results, width=40) # Uninstallers print_info(' Uninstallers') From 2d9de7b69ea0f26986f0092b33dc2a5bc947037e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:22:56 -0700 Subject: [PATCH 014/121] Updated system diags/checklist and user checklist --- .bin/Scripts/system_checklist.py | 16 ++++-- .bin/Scripts/system_diagnostics.py | 82 ++++++++++++++++++++++++------ .bin/Scripts/user_checklist.py | 19 +++---- 3 files changed, 90 insertions(+), 27 deletions(-) diff --git a/.bin/Scripts/system_checklist.py b/.bin/Scripts/system_checklist.py index 72bb82bf..a76433ee 100644 --- a/.bin/Scripts/system_checklist.py +++ b/.bin/Scripts/system_checklist.py @@ -14,7 +14,7 @@ from functions.product_keys import * from functions.setup import * init_global_vars() os.system('title {}: System Checklist Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\System Checklist.log'.format(**global_vars) +set_log_file('System Checklist.log') if __name__ == '__main__': try: @@ -49,10 +49,13 @@ if __name__ == '__main__': # Cleanup print_info('Cleanup') - try_and_print(message='Desktop...', - function=cleanup_desktop, cs='Done') try_and_print(message='AdwCleaner...', function=cleanup_adwcleaner, cs='Done', other_results=other_results) + try_and_print(message='Desktop...', + function=cleanup_desktop, cs='Done') + try_and_print(message='{}...'.format(KIT_NAME_FULL), + function=delete_empty_folders, cs='Done', + folder_path=global_vars['ClientDir']) # Export system info print_info('Backup System Information') @@ -107,6 +110,11 @@ if __name__ == '__main__': sleep(3) try_and_print(message='Running XMPlay...', function=run_xmplay, cs='Started', other_results=other_results) + try: + check_secure_boot_status(show_alert=True) + except: + # Only trying to open alert message boxes + pass # Done print_standard('\nDone.') @@ -116,3 +124,5 @@ if __name__ == '__main__': pass except: major_exception() + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/system_diagnostics.py b/.bin/Scripts/system_diagnostics.py index 9a6e1c0b..bbeb2d11 100644 --- a/.bin/Scripts/system_diagnostics.py +++ b/.bin/Scripts/system_diagnostics.py @@ -13,8 +13,54 @@ from functions.product_keys import * from functions.repairs import * init_global_vars() os.system('title {}: System Diagnostics Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\System Diagnostics.log'.format( - **global_vars) +set_log_file('System Diagnostics.log') + +# Static Variables +BLEACH_BIT_CLEANERS = { + 'Applications': ( + 'adobe_reader.cache', + 'adobe_reader.tmp', + 'amule.tmp', + 'flash.cache', + 'gimp.tmp', + 'hippo_opensim_viewer.cache', + 'java.cache', + 'libreoffice.cache', + 'liferea.cache', + 'miro.cache', + 'openofficeorg.cache', + 'pidgin.cache', + 'secondlife_viewer.Cache', + 'thunderbird.cache', + 'vuze.backup_files', + 'vuze.cache', + 'vuze.tmp', + 'yahoo_messenger.cache', + ), + 'Browsers': ( + 'chromium.cache', + 'chromium.current_session', + 'firefox.cache', + 'firefox.session_restore', + 'google_chrome.cache', + 'google_chrome.session', + 'google_earth.temporary_files', + 'internet_explorer.temporary_files', + 'opera.cache', + 'opera.current_session', + 'safari.cache', + 'seamonkey.cache', + ), + 'System': ( + 'system.clipboard', + 'system.tmp', + 'winapp2_windows.jump_lists', + 'winapp2_windows.ms_search', + 'windows_explorer.run', + 'windows_explorer.search_history', + 'windows_explorer.thumbnails', + ), +} if __name__ == '__main__': try: @@ -34,19 +80,17 @@ if __name__ == '__main__': if ENABLED_TICKET_NUMBERS: print_info('Starting System Diagnostics for Ticket #{}\n'.format( ticket_number)) - + # Sanitize Environment print_info('Sanitizing Environment') - # try_and_print(message='Killing processes...', - # function=run_process_killer, cs='Done') try_and_print(message='Running RKill...', function=run_rkill, cs='Done', other_results=other_results) try_and_print(message='Running TDSSKiller...', function=run_tdsskiller, cs='Done', other_results=other_results) - + # Re-run if earlier process was stopped. stay_awake() - + # Start diags print_info('Starting Background Scans') check_connection() @@ -54,7 +98,7 @@ if __name__ == '__main__': function=run_hitmanpro, cs='Started', other_results=other_results) try_and_print(message='Running Autoruns...', function=run_autoruns, cs='Started', other_results=other_results) - + # OS Health Checks print_info('OS Health Checks') try_and_print( @@ -64,17 +108,23 @@ if __name__ == '__main__': function=run_sfc_scan, other_results=other_results) try_and_print(message='DISM CheckHealth...', function=run_dism, other_results=other_results, repair=False) - + # Scan for supported browsers print_info('Scanning for browsers') scan_for_browsers() - + + # Run BleachBit cleaners + print_info('BleachBit Cleanup') + for k, v in sorted(BLEACH_BIT_CLEANERS.items()): + try_and_print(message=' {}...'.format(k), + function=run_bleachbit, + cs='Done', other_results=other_results, + cleaners=v, preview=True) + # Export system info print_info('Backup System Information') try_and_print(message='AIDA64 reports...', function=run_aida64, cs='Done', other_results=other_results) - try_and_print(message='BleachBit report...', - function=run_bleachbit, cs='Done', other_results=other_results) backup_browsers() try_and_print(message='File listing...', function=backup_file_list, cs='Done', other_results=other_results) @@ -84,7 +134,7 @@ if __name__ == '__main__': function=run_produkey, cs='Done', other_results=other_results) try_and_print(message='Registry...', function=backup_registry, cs='Done', other_results=other_results) - + # Summary print_info('Summary') try_and_print(message='Operating System:', @@ -104,14 +154,14 @@ if __name__ == '__main__': other_results=other_results, print_return=True) try_and_print(message='Product Keys:', function=get_product_keys, ns='Unknown', print_return=True) - + # User data print_info('User Data') try: show_user_data_summary() except Exception: print_error(' Unknown error.') - + # Done print_standard('\nDone.') pause('Press Enter to exit...') @@ -120,3 +170,5 @@ if __name__ == '__main__': pass except: major_exception() + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/user_checklist.py b/.bin/Scripts/user_checklist.py index 72697af7..fd348dec 100644 --- a/.bin/Scripts/user_checklist.py +++ b/.bin/Scripts/user_checklist.py @@ -11,8 +11,7 @@ from functions.cleanup import * from functions.setup import * init_global_vars() os.system('title {}: User Checklist Tool'.format(KIT_NAME_FULL)) -global_vars['LogFile'] = r'{LogDir}\User Checklist ({USERNAME}).log'.format( - **global_vars, **global_vars['Env']) +set_log_file('User Checklist ({USERNAME}).log'.format(**global_vars['Env'])) if __name__ == '__main__': try: @@ -31,29 +30,29 @@ if __name__ == '__main__': if global_vars['OS']['Version'] == '10': answer_config_classicshell = ask('Configure ClassicShell?') answer_config_explorer_user = ask('Configure Explorer?') - + # Cleanup print_info('Cleanup') try_and_print(message='Desktop...', function=cleanup_desktop, cs='Done') - + # Scan for supported browsers print_info('Scanning for browsers') scan_for_browsers() - + # Homepages print_info('Current homepages') list_homepages() - + # Backup print_info('Backing up browsers') backup_browsers() - + # Reset if answer_config_browsers and answer_reset_browsers: print_info('Resetting browsers') reset_browsers() - + # Configure print_info('Configuring programs') if answer_config_browsers: @@ -75,7 +74,7 @@ if __name__ == '__main__': # Run speedtest popen_program(['start', '', 'https://fast.com'], shell=True) - + # Done print_standard('\nDone.') pause('Press Enter to exit...') @@ -84,3 +83,5 @@ if __name__ == '__main__': pass except: major_exception() + +# vim: sts=4 sw=4 ts=4 From 566e1474a59c688672477e09fc36e9d832e7f886 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:24:05 -0700 Subject: [PATCH 015/121] Updated launchers.py --- .bin/Scripts/settings/launchers.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index a125a1f8..ea77d983 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -54,7 +54,7 @@ LAUNCHERS = { 'L_PATH': 'FastCopy', 'L_ITEM': 'FastCopy.exe', 'L_ARGS': ( - r' /logfile=%log_dir%\FastCopy.log' + r' /logfile=%log_dir%\Tools\FastCopy.log' r' /cmd=noexist_only' r' /utf8' r' /skip_empty_dir' @@ -94,7 +94,7 @@ LAUNCHERS = { ), 'L_ELEV': 'True', 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Info /Transfer', + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs /Transfer', ], }, 'FastCopy': { @@ -102,7 +102,7 @@ LAUNCHERS = { 'L_PATH': 'FastCopy', 'L_ITEM': 'FastCopy.exe', 'L_ARGS': ( - r' /logfile=%log_dir%\FastCopy.log' + r' /logfile=%log_dir%\Tools\FastCopy.log' r' /cmd=noexist_only' r' /utf8' r' /skip_empty_dir' @@ -141,7 +141,7 @@ LAUNCHERS = { r' /to=%client_dir%\Transfer_%iso_date%\ ' ), 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Info /Transfer', + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs /Transfer', ], }, 'KVRT': { @@ -212,7 +212,6 @@ LAUNCHERS = { r')', ], }, - }, r'Diagnostics\Extras': { 'AIDA64': { 'L_TYPE': 'Executable', @@ -251,10 +250,10 @@ LAUNCHERS = { 'L_TYPE': 'Executable', 'L_PATH': 'erunt', 'L_ITEM': 'ERUNT.EXE', - 'L_ARGS': '%client_dir%\Backups\%iso_date%\Registry sysreg curuser otherusers', + 'L_ARGS': '%client_dir%\Backups\Registry\%iso_date% sysreg curuser otherusers', 'L_ELEV': 'True', 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Info', + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', ], }, 'HitmanPro': { @@ -262,7 +261,7 @@ LAUNCHERS = { 'L_PATH': 'HitmanPro', 'L_ITEM': 'HitmanPro.exe', 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Info', + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', ], }, 'HWiNFO (Sensors)': { @@ -455,11 +454,6 @@ LAUNCHERS = { 'L_ITEM': 'WizTree.exe', 'L_ELEV': 'True', }, - 'Update Kit': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'update_kit.py', - }, 'XMPlay': { 'L_TYPE': 'Executable', 'L_PATH': 'XMPlay', @@ -524,8 +518,10 @@ LAUNCHERS = { 'L_TYPE': 'Executable', 'L_PATH': 'RKill', 'L_ITEM': 'RKill.exe', + 'L_ARGS': '-s -l %log_dir%\Tools\RKill.log', + 'L_ELEV': 'True', 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Info', + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', ], }, 'SFC Scan': { @@ -539,7 +535,7 @@ LAUNCHERS = { 'L_PATH': 'TDSSKiller', 'L_ITEM': 'TDSSKiller.exe', 'L_ARGS': ( - r' -l %log_dir%\TDSSKiller.log' + r' -l %log_dir%\Tools\TDSSKiller.log' r' -qpath %q_dir%' r' -accepteula' r' -accepteulaksn' From e3076152ba8d5137b4007cc0c8081abf0225fcdf Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:31:36 -0700 Subject: [PATCH 016/121] Updated main.py --- .bin/Scripts/settings/main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index c49da96e..75fef0fd 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -1,14 +1,15 @@ # Wizard Kit: Settings - Main / Branding # Features -ENABLED_UPLOAD_DATA = False +ENABLED_OPEN_LOGS = False ENABLED_TICKET_NUMBERS = False +ENABLED_UPLOAD_DATA = False # STATIC VARIABLES (also used by BASH and BATCH files) ## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH # Main Kit ARCHIVE_PASSWORD='Abracadabra' -KIT_NAME_FULL='Wizard Kit' +KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux @@ -19,10 +20,10 @@ TECH_PASSWORD='Abracadabra' OFFICE_SERVER_IP='10.0.0.10' QUICKBOOKS_SERVER_IP='10.0.0.10' # Time Zones -LINUX_TIME_ZONE='America/Los_Angeles' # See 'timedatectl list-timezones' for valid values -WINDOWS_TIME_ZONE='Pacific Standard Time' # See 'tzutil /l' for valid values +LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values +WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values # WiFi -WIFI_SSID='SomeWifi' +WIFI_SSID='SomeWiFi' WIFI_PASSWORD='Abracadabra' # SERVER VARIABLES From 972d91a4b869fd563a0c70f16721f636bf59e376 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:32:52 -0700 Subject: [PATCH 017/121] Updated sources.py --- .bin/Scripts/settings/sources.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 5f4947b4..9f4350db 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -23,6 +23,7 @@ SOURCE_URLS = { 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/27656/eng/Intel%20SSD%20Toolbox%20-%20v3.5.2.exe', 'IOBit_Uninstaller': 'https://portableapps.duckduckgo.com/IObitUninstallerPortable_7.5.0.7.paf.exe', 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', + 'Macs Fan Control': 'https://www.crystalidea.com/downloads/macsfancontrol_setup.exe', 'NirCmd32': 'https://www.nirsoft.net/utils/nircmd.zip', 'NirCmd64': 'https://www.nirsoft.net/utils/nircmd-x64.zip', 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z', @@ -197,3 +198,5 @@ RST_SOURCES = { if __name__ == '__main__': print("This file is not meant to be called directly.") + +# vim: sts=4 sw=4 ts=4 tw=0 nowrap From 8a86edb5bb7293292581aecbca0d287c1503eb21 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 23 Nov 2018 18:33:18 -0700 Subject: [PATCH 018/121] Updated windows_builds.py --- .bin/Scripts/settings/windows_builds.py | 36 ++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/settings/windows_builds.py b/.bin/Scripts/settings/windows_builds.py index bdac8bf4..2db18b6f 100644 --- a/.bin/Scripts/settings/windows_builds.py +++ b/.bin/Scripts/settings/windows_builds.py @@ -6,16 +6,16 @@ WINDOWS_BUILDS = { '6000': ( 'Vista', 'RTM', 'Longhorn', None, 'unsupported'), '6001': ( 'Vista', 'SP1', 'Longhorn', None, 'unsupported'), '6002': ( 'Vista', 'SP2', 'Longhorn', None, 'unsupported'), - + '7600': ( '7', 'RTM', 'Vienna', None, 'unsupported'), '7601': ( '7', 'SP1', 'Vienna', None, 'outdated'), - + #9199 is a fake build since Win 8 is 6.2.9200 but that collides with Win 8.1 (6.3.9200) '9199': ( '8', 'RTM', None, None, 'unsupported'), '9200': ( '8.1', None, 'Blue', None, 'outdated'), '9600': ( '8.1', None, 'Update', None, None), - + '9841': ( '10', None, 'Threshold 1', None, 'preview build'), '9860': ( '10', None, 'Threshold 1', None, 'preview build'), '9879': ( '10', None, 'Threshold 1', None, 'preview build'), @@ -149,6 +149,36 @@ WINDOWS_BUILDS = { '17655': ( '10', None, 'Redstone 5', None, 'preview build'), '17661': ( '10', None, 'Redstone 5', None, 'preview build'), '17666': ( '10', None, 'Redstone 5', None, 'preview build'), + '17677': ( '10', None, 'Redstone 5', None, 'preview build'), + '17682': ( '10', None, 'Redstone 5', None, 'preview build'), + '17686': ( '10', None, 'Redstone 5', None, 'preview build'), + '17692': ( '10', None, 'Redstone 5', None, 'preview build'), + '17704': ( '10', None, 'Redstone 5', None, 'preview build'), + '17711': ( '10', None, 'Redstone 5', None, 'preview build'), + '17713': ( '10', None, 'Redstone 5', None, 'preview build'), + '17723': ( '10', None, 'Redstone 5', None, 'preview build'), + '17728': ( '10', None, 'Redstone 5', None, 'preview build'), + '17730': ( '10', None, 'Redstone 5', None, 'preview build'), + '17733': ( '10', None, 'Redstone 5', None, 'preview build'), + '17735': ( '10', None, 'Redstone 5', None, 'preview build'), + '17738': ( '10', None, 'Redstone 5', None, 'preview build'), + '17741': ( '10', None, 'Redstone 5', None, 'preview build'), + '17744': ( '10', None, 'Redstone 5', None, 'preview build'), + '17746': ( '10', None, 'Redstone 5', None, 'preview build'), + '17751': ( '10', None, 'Redstone 5', None, 'preview build'), + '17754': ( '10', None, 'Redstone 5', None, 'preview build'), + '17755': ( '10', None, 'Redstone 5', None, 'preview build'), + '17758': ( '10', None, 'Redstone 5', None, 'preview build'), + '17760': ( '10', None, 'Redstone 5', None, 'preview build'), + '17763': ( '10', 'v1809', 'Redstone 5', 'October 2018 Update', 'preview build'), + '18204': ( '10', None, '19H1', None, 'preview build'), + '18214': ( '10', None, '19H1', None, 'preview build'), + '18219': ( '10', None, '19H1', None, 'preview build'), + '18234': ( '10', None, '19H1', None, 'preview build'), + '18237': ( '10', None, '19H1', None, 'preview build'), + '18242': ( '10', None, '19H1', None, 'preview build'), + '18247': ( '10', None, '19H1', None, 'preview build'), + '18252': ( '10', None, '19H1', None, 'preview build'), } if __name__ == '__main__': From b3b821a8689849e540350df722678431e7f01db5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 26 Nov 2018 18:12:24 -0700 Subject: [PATCH 019/121] Allow mounting all volumes per device --- .bin/Scripts/functions/data.py | 56 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index c8eec384..029c674b 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -126,11 +126,11 @@ def cleanup_transfer(dest_path): if not os.path.exists(dest_path): # Bail if dest_path was empty and removed raise Exception - + # Fix attributes cmd = ['attrib', '-a', '-h', '-r', '-s', dest_path] run_program(cmd, check=False) - + for root, dirs, files in os.walk(dest_path, topdown=False): for name in dirs: # Remove empty directories and junction points @@ -153,7 +153,7 @@ def cleanup_transfer(dest_path): except Exception: pass -def find_core_storage_volumes(): +def find_core_storage_volumes(device_path=None): """Try to create block devices for any Apple CoreStorage volumes.""" corestorage_uuid = '53746f72-6167-11aa-aa11-00306543ecac' dmsetup_cmd_file = '{TmpDir}/dmsetup_command'.format(**global_vars) @@ -162,6 +162,8 @@ def find_core_storage_volumes(): cmd = [ 'lsblk', '--json', '--list', '--paths', '--output', 'NAME,PARTTYPE'] + if device_path: + cmd.append(device_path) result = run_program(cmd) json_data = json.loads(result.stdout.decode()) devs = json_data.get('blockdevices', []) @@ -248,7 +250,9 @@ def get_mounted_volumes(): mounted_volumes.extend(item.get('children', [])) return {item['source']: item for item in mounted_volumes} -def mount_volumes(all_devices=True, device_path=None, read_write=False): +def mount_volumes( + all_devices=True, device_path=None, + read_write=False, core_storage=True): """Mount all detected filesystems.""" report = {} cmd = [ @@ -257,9 +261,10 @@ def mount_volumes(all_devices=True, device_path=None, read_write=False): if not all_devices and device_path: # Only mount volumes for specific device cmd.append(device_path) - else: - # Check for Apple CoreStorage volumes first - find_core_storage_volumes() + + # Check for Apple CoreStorage volumes first + if core_storage: + find_core_storage_volumes(device_path) # Get list of block devices result = run_program(cmd) @@ -269,11 +274,14 @@ def mount_volumes(all_devices=True, device_path=None, read_write=False): # Get list of volumes volumes = {} for dev in devs: + if not dev.get('children', []): + volumes.update({dev['name']: dev}) for child in dev.get('children', []): - volumes.update({child['name']: child}) + if not child.get('children', []): + volumes.update({child['name']: child}) for grandchild in child.get('children', []): volumes.update({grandchild['name']: grandchild}) - + # Get list of mounted volumes mounted_volumes = get_mounted_volumes() @@ -352,7 +360,7 @@ def mount_backup_shares(read_write=False): if server['Mounted']: print_warning(mounted_str) continue - + mount_network_share(server, read_write) def mount_network_share(server, read_write=False): @@ -414,12 +422,12 @@ def run_fast_copy(items, dest): """Copy items to dest using FastCopy.""" if not items: raise Exception - + cmd = [global_vars['Tools']['FastCopy'], *FAST_COPY_ARGS] - cmd.append(r'/logfile={}\FastCopy.log'.format(global_vars['LogDir'])) + cmd.append(r'/logfile={LogDir}\Tools\FastCopy.log'.format(**global_vars)) cmd.extend(items) cmd.append('/to={}\\'.format(dest)) - + run_program(cmd) def run_wimextract(source, items, dest): @@ -486,7 +494,7 @@ def list_source_items(source_obj, rel_path=None): def scan_source(source_obj, dest_path, rel_path='', interactive=True): """Scan source for files/folders to transfer, returns list. - + This will scan the root and (recursively) any Windows.old folders.""" selected_items = [] win_olds = [] @@ -563,7 +571,7 @@ def scan_source(source_obj, dest_path, rel_path='', interactive=True): '{}{}{}'.format(dest_path, os.sep, old.name), rel_path=old.name, interactive=False)) - + # Done return selected_items @@ -707,7 +715,7 @@ def select_source(backup_prefix): item.name, # Image file ), 'Source': item}) - + # Check for local sources print_standard('Scanning for local sources...') set_thread_error_mode(silent=True) # Prevents "No disk" popups @@ -747,7 +755,7 @@ def select_source(backup_prefix): ' Local', d.mountpoint, item.name), 'Sort': r'{}{}'.format(d.mountpoint, item.name), 'Source': item}) - + set_thread_error_mode(silent=False) # Return to normal # Build Menu @@ -775,7 +783,7 @@ def select_source(backup_prefix): umount_backup_shares() pause("Press Enter to exit...") exit_script() - + # Sanity check if selected_source.is_file(): # Image-Based @@ -783,7 +791,7 @@ def select_source(backup_prefix): print_error('ERROR: Unsupported image: {}'.format( selected_source.path)) raise GenericError - + # Done return selected_source @@ -791,7 +799,7 @@ def select_volume(title='Select disk', auto_select=True): """Select disk from attached disks. returns dict.""" actions = [{'Name': 'Quit', 'Letter': 'Q'}] disks = [] - + # Build list of disks set_thread_error_mode(silent=True) # Prevents "No disk" popups for d in psutil.disk_partitions(): @@ -812,11 +820,11 @@ def select_volume(title='Select disk', auto_select=True): info['Display Name'] = '{} ({})'.format(info['Name'], free) disks.append(info) set_thread_error_mode(silent=False) # Return to normal - + # Skip menu? if len(disks) == 1 and auto_select: return disks[0] - + # Show menu selection = menu_select(title, main_entries=disks, action_entries=actions) if selection == 'Q': @@ -826,12 +834,12 @@ def select_volume(title='Select disk', auto_select=True): def set_thread_error_mode(silent=True): """Disable or Enable Windows error message dialogs. - + Disable when scanning for disks to avoid popups for empty cardreaders, etc """ # Code borrowed from: https://stackoverflow.com/a/29075319 kernel32 = ctypes.WinDLL('kernel32') - + if silent: kernel32.SetThreadErrorMode(SEM_FAIL, ctypes.byref(SEM_NORMAL)) else: From bb23d49833e90c58dbc852cf709925d175b0b8ca Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 26 Nov 2018 18:14:06 -0700 Subject: [PATCH 020/121] Adjusted mounting shares/volumes --- .bin/Scripts/functions/data.py | 9 +++------ .bin/Scripts/mount-backup-shares | 2 +- .bin/Scripts/remount-rw | 2 -- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 029c674b..9557ebdb 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -372,12 +372,9 @@ def mount_network_share(server, read_write=False): username = server['User'] password = server['Pass'] if psutil.WINDOWS: - cmd = r'net use \\{ip}\{share} /user:{username} {password}'.format( - ip = server['IP'], - share = server['Share'], - username = username, - password = password) - cmd = cmd.split(' ') + cmd = [ + 'net', 'use', r'\\{IP}\{Share}'.format(**server), + '/user:{}'.format(username), password] warning = r'Failed to mount \\{Name}\{Share}, {IP} unreachable.'.format( **server) error = r'Failed to mount \\{Name}\{Share} ({IP})'.format(**server) diff --git a/.bin/Scripts/mount-backup-shares b/.bin/Scripts/mount-backup-shares index 9706a0a5..9f3612b6 100755 --- a/.bin/Scripts/mount-backup-shares +++ b/.bin/Scripts/mount-backup-shares @@ -22,7 +22,7 @@ if __name__ == '__main__': # Mount if is_connected(): - mount_backup_shares() + mount_backup_shares(read_write=True) else: # Couldn't connect print_error('ERROR: No network connectivity.') diff --git a/.bin/Scripts/remount-rw b/.bin/Scripts/remount-rw index 4a0b833e..1ba4b41d 100755 --- a/.bin/Scripts/remount-rw +++ b/.bin/Scripts/remount-rw @@ -18,6 +18,4 @@ if udevil mount $DEVICE; then else echo "Failed" fi - -sleep 2s exit 0 From f802ea860d20ac2f99d51515fd8ea949a3d41958 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 26 Nov 2018 18:15:41 -0700 Subject: [PATCH 021/121] Updated ddrescue.py --- .bin/Scripts/functions/ddrescue.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index eba04452..7f1fc7aa 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -323,7 +323,7 @@ class RecoveryState(): elif not is_writable_filesystem(dest): raise GenericError( 'Destination is mounted read-only, refusing to continue.') - + # Safety checks passed self.block_pairs.append(BlockPair(self.mode, source, dest)) @@ -392,7 +392,7 @@ class RecoveryState(): self.status_percent = get_formatted_status( label='Recovered:', data=self.rescued_percent) self.status_amount = get_formatted_status( - label='', data=human_readable_size(self.rescued)) + label='', data=human_readable_size(self.rescued, decimals=2)) # Functions @@ -1084,7 +1084,7 @@ def select_path(skip_device=None): except OSError: raise GenericError( 'Failed to create folder "{}"'.format(s_path)) - + # Create DirObj selected_path = DirObj(s_path) From c42c7647896ff36eab988d2e58c704a2eda1bd78 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 26 Nov 2018 18:35:33 -0700 Subject: [PATCH 022/121] Updated hw_diags.py --- .bin/Scripts/functions/hw_diags.py | 172 ++++++++++++++++++----------- 1 file changed, 110 insertions(+), 62 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d2282692..169eb337 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -8,23 +8,23 @@ from functions.common import * # STATIC VARIABLES ATTRIBUTES = { 'NVMe': { - 'critical_warning': {'Error': 1}, - 'media_errors': {'Error': 1}, - 'power_on_hours': {'Warning': 12000, 'Error': 18000, 'Ignore': True}, + 'critical_warning': {'Error': 1}, + 'media_errors': {'Error': 1}, + 'power_on_hours': {'Warning': 12000, 'Error': 18000, 'Ignore': True}, 'unsafe_shutdowns': {'Warning': 1}, }, 'SMART': { - 5: {'Error': 1}, - 9: {'Warning': 12000, 'Error': 18000, 'Ignore': True}, - 10: {'Warning': 1}, - 184: {'Error': 1}, - 187: {'Warning': 1}, - 188: {'Warning': 1}, - 196: {'Warning': 1, 'Error': 10, 'Ignore': True}, - 197: {'Error': 1}, - 198: {'Error': 1}, - 199: {'Error': 1, 'Ignore': True}, - 201: {'Warning': 1}, + 5: {'Hex': '05', 'Error': 1}, + 9: {'Hex': '09', 'Warning': 12000, 'Error': 18000, 'Ignore': True}, + 10: {'Hex': '0A', 'Error': 1}, + 184: {'Hex': 'B8', 'Error': 1}, + 187: {'Hex': 'BB', 'Error': 1}, + 188: {'Hex': 'BC', 'Error': 1}, + 196: {'Hex': 'C4', 'Error': 1}, + 197: {'Hex': 'C5', 'Error': 1}, + 198: {'Hex': 'C6', 'Error': 1}, + 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + 201: {'Hex': 'C9', 'Error': 1}, }, } IO_VARS = { @@ -37,9 +37,15 @@ IO_VARS = { 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], - 'Threshold Fail': 65*1024**2, - 'Threshold Warn': 135*1024**2, - 'Threshold Great': 750*1024**2, + 'Threshold Graph Fail': 65*1024**2, + 'Threshold Graph Warn': 135*1024**2, + 'Threshold Graph Great': 750*1024**2, + 'Threshold HDD Min': 50*1024**2, + 'Threshold HDD High Avg': 75*1024**2, + 'Threshold HDD Low Avg': 65*1024**2, + 'Threshold SSD Min': 90*1024**2, + 'Threshold SSD High Avg': 135*1024**2, + 'Threshold SSD Low Avg': 100*1024**2, 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), 'Graph Horizontal Width': 40, 'Graph Vertical': ( @@ -60,6 +66,7 @@ TESTS = { 'NVMe/SMART': { 'Enabled': False, 'Quick': False, + 'Short Test': {}, 'Status': {}, }, 'badblocks': { @@ -88,11 +95,11 @@ def generate_horizontal_graph(rates, oneline=False): # Set color r_color = COLORS['CLEAR'] - if r < IO_VARS['Threshold Fail']: + if r < IO_VARS['Threshold Graph Fail']: r_color = COLORS['RED'] - elif r < IO_VARS['Threshold Warn']: + elif r < IO_VARS['Threshold Graph Warn']: r_color = COLORS['YELLOW'] - elif r > IO_VARS['Threshold Great']: + elif r > IO_VARS['Threshold Graph Great']: r_color = COLORS['GREEN'] # Build graph @@ -225,16 +232,21 @@ def menu_diags(*args): action_entries = actions, spacer = '──────────────────────────') if selection.isnumeric(): + ticket_number = None if diag_modes[int(selection)-1]['Name'] != 'Quick drive test': - # Save log for non-quick tests + clear_screen() + print_standard(' ') ticket_number = get_ticket_number() - global_vars['LogDir'] = '{}/Logs/{}'.format( + # Save log for non-quick tests + global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") + global_vars['LogDir'] = '{}/Logs/{}_{}'.format( global_vars['Env']['HOME'], - ticket_number if ticket_number else global_vars['Date-Time']) + ticket_number, + global_vars['Date-Time']) os.makedirs(global_vars['LogDir'], exist_ok=True) global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( global_vars['LogDir']) - run_tests(diag_modes[int(selection)-1]['Tests']) + run_tests(diag_modes[int(selection)-1]['Tests'], ticket_number) elif selection == 'A': run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') @@ -256,7 +268,7 @@ def menu_diags(*args): elif selection == 'Q': break -def run_badblocks(): +def run_badblocks(ticket_number): """Run a read-only test for all detected disks.""" aborted = False clear_screen() @@ -318,7 +330,7 @@ def run_badblocks(): run_program('tmux kill-pane -a'.split(), check=False) pass -def run_iobenchmark(): +def run_iobenchmark(ticket_number): """Run a read-only test for all detected disks.""" aborted = False clear_screen() @@ -469,10 +481,25 @@ def run_iobenchmark(): TESTS['iobenchmark']['Results'][name] = report # Set CS/NS - if min(TESTS['iobenchmark']['Data'][name]['Read Rates']) <= IO_VARS['Threshold Fail']: + min_read = min(TESTS['iobenchmark']['Data'][name]['Read Rates']) + avg_read = sum( + TESTS['iobenchmark']['Data'][name]['Read Rates'])/len( + TESTS['iobenchmark']['Data'][name]['Read Rates']) + dev_rotational = dev['lsblk'].get('rota', None) + if dev_rotational == "0": + # Use SSD scale + thresh_min = IO_VARS['Threshold SSD Min'] + thresh_high_avg = IO_VARS['Threshold SSD High Avg'] + thresh_low_avg = IO_VARS['Threshold SSD Low Avg'] + else: + # Use HDD scale + thresh_min = IO_VARS['Threshold HDD Min'] + thresh_high_avg = IO_VARS['Threshold HDD High Avg'] + thresh_low_avg = IO_VARS['Threshold HDD Low Avg'] + if min_read <= thresh_min and avg_read <= thresh_high_avg: + TESTS['iobenchmark']['Status'][name] = 'NS' + elif avg_read <= thresh_low_avg: TESTS['iobenchmark']['Status'][name] = 'NS' - elif min(TESTS['iobenchmark']['Data'][name]['Read Rates']) <= IO_VARS['Threshold Warn']: - TESTS['iobenchmark']['Status'][name] = 'Unknown' else: TESTS['iobenchmark']['Status'][name] = 'CS' @@ -487,7 +514,7 @@ def run_iobenchmark(): run_program('tmux kill-pane -a'.split(), check=False) pass -def run_mprime(): +def run_mprime(ticket_number): """Run Prime95 for MPRIME_LIMIT minutes while showing the temps.""" aborted = False print_log('\nStart Prime95 test') @@ -501,7 +528,7 @@ def run_mprime(): TESTS['Progress Out']).split()) run_program('tmux split-window -bd watch -c -n1 -t hw-sensors'.split()) run_program('tmux resize-pane -y 3'.split()) - + # Start test run_program(['apple-fans', 'max']) try: @@ -516,6 +543,9 @@ def run_mprime(): except KeyboardInterrupt: # Catch CTRL+C aborted = True + TESTS['Prime95']['Status'] = 'Aborted' + print_warning('\nAborted.') + update_progress() # Save "final" temps run_program( @@ -563,15 +593,7 @@ def run_mprime(): TESTS['Prime95']['CS'] = bool(r) # Update status - if aborted: - TESTS['Prime95']['Status'] = 'Aborted' - print_warning('\nAborted.') - update_progress() - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled']: - if not ask('Proceed to next test?'): - run_program('tmux kill-pane -a'.split()) - raise GenericError - else: + if not aborted: if TESTS['Prime95']['NS']: TESTS['Prime95']['Status'] = 'NS' elif TESTS['Prime95']['CS']: @@ -580,10 +602,21 @@ def run_mprime(): TESTS['Prime95']['Status'] = 'Unknown' update_progress() + if aborted: + if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled']: + if not ask('Proceed to next test?'): + for name in TESTS['NVMe/SMART']['Devices'].keys(): + for t in ['NVMe/SMART', 'badblocks', 'iobenchmark']: + cur_status = TESTS[t]['Status'][name] + if cur_status not in ['CS', 'Denied', 'NS']: + TESTS[t]['Status'][name] = 'Aborted' + run_program('tmux kill-pane -a'.split()) + raise GenericError + # Done run_program('tmux kill-pane -a'.split()) -def run_nvme_smart(): +def run_nvme_smart(ticket_number): """Run the built-in NVMe or SMART test for all detected disks.""" aborted = False clear_screen() @@ -605,6 +638,7 @@ def run_nvme_smart(): # Run for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): + TESTS['NVMe/SMART']['Short Test'][name] = None cur_status = TESTS['NVMe/SMART']['Status'][name] if cur_status == 'OVERRIDE': # Skipping test per user request @@ -635,7 +669,7 @@ def run_nvme_smart(): run_program( 'sudo smartctl -t short /dev/{}'.format(name).split(), check=False) - + # Wait and show progress (in 10 second increments) for iteration in range(int(test_length*60/10)): # Update SMART data @@ -670,18 +704,24 @@ def run_nvme_smart(): 'passed', False) if test_passed: TESTS['NVMe/SMART']['Status'][name] = 'CS' + TESTS['NVMe/SMART']['Short Test'][name] = 'CS' else: TESTS['NVMe/SMART']['Status'][name] = 'NS' + TESTS['NVMe/SMART']['Short Test'][name] = 'NS' update_progress() print_standard('Done', timestamp=False) # Done run_program('tmux kill-pane -a'.split(), check=False) -def run_tests(tests): +def run_tests(tests, ticket_number=None): """Run selected hardware test(s).""" - print_log('Starting Hardware Diagnostics') - print_log('\nRunning tests: {}'.format(', '.join(tests))) + clear_screen() + print_standard('Starting Hardware Diagnostics') + if ticket_number: + print_standard(' For Ticket #{}'.format(ticket_number)) + print_standard(' ') + print_standard('Running tests: {}'.format(', '.join(tests))) # Enable selected tests for t in ['Prime95', 'NVMe/SMART', 'badblocks', 'iobenchmark']: TESTS[t]['Enabled'] = t in tests @@ -690,7 +730,6 @@ def run_tests(tests): # Initialize if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: print_standard(' ') - print_standard('Scanning disks...') scan_disks() update_progress() @@ -698,22 +737,22 @@ def run_tests(tests): mprime_aborted = False if TESTS['Prime95']['Enabled']: try: - run_mprime() + run_mprime(ticket_number) except GenericError: mprime_aborted = True if not mprime_aborted: if TESTS['NVMe/SMART']['Enabled']: - run_nvme_smart() + run_nvme_smart(ticket_number) if TESTS['badblocks']['Enabled']: - run_badblocks() + run_badblocks(ticket_number) if TESTS['iobenchmark']['Enabled']: - run_iobenchmark() - + run_iobenchmark(ticket_number) + # Show results show_results() # Open log - if not TESTS['NVMe/SMART']['Quick']: + if not TESTS['NVMe/SMART']['Quick'] and ENABLED_OPEN_LOGS: try: popen_program(['nohup', 'leafpad', global_vars['LogFile']], pipe=True) except Exception: @@ -723,7 +762,6 @@ def run_tests(tests): def scan_disks(full_paths=False, only_path=None): """Scan for disks eligible for hardware testing.""" - clear_screen() # Get eligible disk list cmd = ['lsblk', '-J', '-O'] @@ -743,13 +781,18 @@ def scan_disks(full_paths=False, only_path=None): TESTS['iobenchmark']['Status'][d['name']] = 'Pending' else: # Skip WizardKit devices - wk_label = '{}_LINUX'.format(KIT_NAME_SHORT) - if wk_label not in [c.get('label', '') for c in d.get('children', [])]: + skip_dev=False + wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) + for c in d.get('children', []): + r = re.search( + wk_label_regex, c.get('label', ''), re.IGNORECASE) + skip_dev = bool(r) + if not skip_dev: devs[d['name']] = {'lsblk': d} TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' TESTS['badblocks']['Status'][d['name']] = 'Pending' TESTS['iobenchmark']['Status'][d['name']] = 'Pending' - + for dev, data in devs.items(): # Get SMART attributes run_program( @@ -758,7 +801,7 @@ def scan_disks(full_paths=False, only_path=None): dev).split(), check = False) data['smartctl'] = get_smart_details(dev) - + # Get NVMe attributes if data['lsblk']['tran'] == 'nvme': cmd = 'sudo nvme smart-log /dev/{} -o json'.format(dev).split() @@ -782,7 +825,12 @@ def scan_disks(full_paths=False, only_path=None): ] if data.get('NVMe Disk', False): crit_warn = data['nvme-cli'].get('critical_warning', 1) - data['Quick Health OK'] = True if crit_warn == 0 else False + if crit_warn == 0: + dev_name = data['lsblk']['name'] + data['Quick Health OK'] = True + TESTS['NVMe/SMART']['Status'][dev_name] = 'CS' + else: + data['Quick Health OK'] = False elif set(wanted_smart_list).issubset(data['smartctl'].keys()): data['SMART Pass'] = data['smartctl'].get('smart_status', {}).get( 'passed', False) @@ -791,7 +839,7 @@ def scan_disks(full_paths=False, only_path=None): else: data['Quick Health OK'] = False data['SMART Support'] = False - + # Ask for manual overrides if necessary if TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: show_disk_details(data) @@ -989,13 +1037,13 @@ def update_io_progress(percent, rate, progress_file): bar_color = COLORS['CLEAR'] rate_color = COLORS['CLEAR'] step = get_graph_step(rate, scale=32) - if rate < IO_VARS['Threshold Fail']: + if rate < IO_VARS['Threshold Graph Fail']: bar_color = COLORS['RED'] rate_color = COLORS['YELLOW'] - elif rate < IO_VARS['Threshold Warn']: + elif rate < IO_VARS['Threshold Graph Warn']: bar_color = COLORS['YELLOW'] rate_color = COLORS['YELLOW'] - elif rate > IO_VARS['Threshold Great']: + elif rate > IO_VARS['Threshold Graph Great']: bar_color = COLORS['GREEN'] rate_color = COLORS['GREEN'] line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( From 83dda97ff62ace7e445816543ac72778e802243c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 26 Nov 2018 19:13:00 -0700 Subject: [PATCH 023/121] Bugfix: launchers.py --- .bin/Scripts/settings/launchers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index ea77d983..43e8e158 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -556,7 +556,8 @@ LAUNCHERS = { 'L_ITEM': 'IObitUninstallerPortable.exe', }, }, - } + }, +} if __name__ == '__main__': print("This file is not meant to be called directly.") From 10f2fca2bfa7e24d52e238e1d3ed611e5e9200fe Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 17:52:07 -0700 Subject: [PATCH 024/121] Added classes DevObj and State --- .bin/Scripts/functions/hw_diags.py | 1310 +++++----------------------- 1 file changed, 221 insertions(+), 1089 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 169eb337..830d948a 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1,1114 +1,246 @@ # Wizard Kit: Functions - HW Diagnostics import json +import re import time from functions.common import * # STATIC VARIABLES ATTRIBUTES = { - 'NVMe': { - 'critical_warning': {'Error': 1}, - 'media_errors': {'Error': 1}, - 'power_on_hours': {'Warning': 12000, 'Error': 18000, 'Ignore': True}, - 'unsafe_shutdowns': {'Warning': 1}, - }, - 'SMART': { - 5: {'Hex': '05', 'Error': 1}, - 9: {'Hex': '09', 'Warning': 12000, 'Error': 18000, 'Ignore': True}, - 10: {'Hex': '0A', 'Error': 1}, - 184: {'Hex': 'B8', 'Error': 1}, - 187: {'Hex': 'BB', 'Error': 1}, - 188: {'Hex': 'BC', 'Error': 1}, - 196: {'Hex': 'C4', 'Error': 1}, - 197: {'Hex': 'C5', 'Error': 1}, - 198: {'Hex': 'C6', 'Error': 1}, - 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - 201: {'Hex': 'C9', 'Error': 1}, - }, - } + 'NVMe': { + 'critical_warning': {'Error': 1}, + 'media_errors': {'Error': 1}, + 'power_on_hours': {'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 'unsafe_shutdowns': {'Warning': 1}, + }, + 'SMART': { + 5: {'Hex': '05', 'Error': 1}, + 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 10: {'Hex': '0A', 'Error': 1}, + 184: {'Hex': 'B8', 'Error': 1}, + 187: {'Hex': 'BB', 'Error': 1}, + 188: {'Hex': 'BC', 'Error': 1}, + 196: {'Hex': 'C4', 'Error': 1}, + 197: {'Hex': 'C5', 'Error': 1}, + 198: {'Hex': 'C6', 'Error': 1}, + 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + 201: {'Hex': 'C9', 'Error': 1}, + }, + } IO_VARS = { - 'Block Size': 512*1024, - 'Chunk Size': 32*1024**2, - 'Minimum Dev Size': 8*1024**3, - 'Minimum Test Size': 10*1024**3, - 'Alt Test Size Factor': 0.01, - 'Progress Refresh Rate': 5, - 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], - 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], - 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], - 'Threshold Graph Fail': 65*1024**2, - 'Threshold Graph Warn': 135*1024**2, - 'Threshold Graph Great': 750*1024**2, - 'Threshold HDD Min': 50*1024**2, - 'Threshold HDD High Avg': 75*1024**2, - 'Threshold HDD Low Avg': 65*1024**2, - 'Threshold SSD Min': 90*1024**2, - 'Threshold SSD High Avg': 135*1024**2, - 'Threshold SSD Low Avg': 100*1024**2, - 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), - 'Graph Horizontal Width': 40, - 'Graph Vertical': ( - '▏', '▎', '▍', '▌', - '▋', '▊', '▉', '█', - '█▏', '█▎', '█▍', '█▌', - '█▋', '█▊', '█▉', '██', - '██▏', '██▎', '██▍', '██▌', - '██▋', '██▊', '██▉', '███', - '███▏', '███▎', '███▍', '███▌', - '███▋', '███▊', '███▉', '████'), - } -TESTS = { - 'Prime95': { - 'Enabled': False, - 'Status': 'Pending', - }, - 'NVMe/SMART': { - 'Enabled': False, - 'Quick': False, - 'Short Test': {}, - 'Status': {}, - }, - 'badblocks': { - 'Enabled': False, - 'Results': {}, - 'Status': {}, - }, - 'iobenchmark': { - 'Data': {}, - 'Enabled': False, - 'Results': {}, - 'Status': {}, - }, - } + 'Block Size': 512*1024, + 'Chunk Size': 32*1024**2, + 'Minimum Dev Size': 8*1024**3, + 'Minimum Test Size': 10*1024**3, + 'Alt Test Size Factor': 0.01, + 'Progress Refresh Rate': 5, + 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], + 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], + 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], + 'Threshold Graph Fail': 65*1024**2, + 'Threshold Graph Warn': 135*1024**2, + 'Threshold Graph Great': 750*1024**2, + 'Threshold HDD Min': 50*1024**2, + 'Threshold HDD High Avg': 75*1024**2, + 'Threshold HDD Low Avg': 65*1024**2, + 'Threshold SSD Min': 90*1024**2, + 'Threshold SSD High Avg': 135*1024**2, + 'Threshold SSD Low Avg': 100*1024**2, + 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), + 'Graph Horizontal Width': 40, + 'Graph Vertical': ( + '▏', '▎', '▍', '▌', + '▋', '▊', '▉', '█', + '█▏', '█▎', '█▍', '█▌', + '█▋', '█▊', '█▉', '██', + '██▏', '██▎', '██▍', '██▌', + '██▋', '██▊', '██▉', '███', + '███▏', '███▎', '███▍', '███▌', + '███▋', '███▊', '███▉', '████'), + } +KEY_NVME = 'nvme_smart_health_information_log' +KEY_SMART = 'ata_smart_attributes' +SIDE_PATH_WIDTH = 21 +# Classes +class DevObj(): + """Device object for tracking device specific data.""" + def __init__(self, dev_path): + self.failing = False + self.nvme_attributes = {} + self.override = False + self.path = dev_path + self.smart_attributes = {} + self.tests = { + 'NVMe / SMART': {'Result': None, 'Status': None}, + 'badblocks': {'Result': None, 'Status': None}, + 'I/O Benchmark': { + 'Result': None, + 'Status': None, + 'Read Rates': [], + 'Graph Data': []}, + } + self.get_details() + + def get_details(self): + """Get data from smartctl.""" + cmd = ['sudo', 'smartctl', '--all', '--json', self.path] + result = run_program(cmd, check=False) + self.data = json.loads(result.stdout.decode()) + + # Check for attributes + if KEY_NVME in self.data: + self.nvme_attributes.update(self.data[KEY_NVME]) + elif KEY_SMART in self.data: + for a in self.data[KEY_SMART].get('table', {}): + _id = str(a.get('id', 'UNKNOWN')) + _name = str(a.get('name', 'UNKNOWN')) + _raw = a.get('raw', {}).get('value', -1) + _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') + + # Fix power-on time + _r = re.match(r'^(\d+)[Hh].*', _raw_str) + if _id == '9' and _r: + try: + _raw = int(_r.group(1)) + except ValueError: + # That's fine + pass + self.smart_attributes[_id] = { + 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + +class State(): + """Object to track device objects and overall state.""" + def __init__(self): + self.devs = [] + self.finished = False + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.started = False + self.tests = { + 'Prime95': {'Enabled': False, 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False}, + 'badblocks': {'Enabled': False}, + 'I/O Benchmark': {'Enabled': False}, + } + self.add_devs() + + def add_devs(self): + """Add all block devices listed by lsblk.""" + cmd = ['lsblk', '--json', '--nodeps', '--paths'] + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + for dev in json_data['blockdevices']: + self.devs.append(DevObj(dev['name'])) + +# Functions def generate_horizontal_graph(rates, oneline=False): - """Generate two-line horizontal graph from rates, returns str.""" - line_1 = '' - line_2 = '' - line_3 = '' - line_4 = '' - for r in rates: - step = get_graph_step(r, scale=32) - if oneline: - step = get_graph_step(r, scale=8) - - # Set color - r_color = COLORS['CLEAR'] - if r < IO_VARS['Threshold Graph Fail']: - r_color = COLORS['RED'] - elif r < IO_VARS['Threshold Graph Warn']: - r_color = COLORS['YELLOW'] - elif r > IO_VARS['Threshold Graph Great']: - r_color = COLORS['GREEN'] - - # Build graph - full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) - if step >= 24: - line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) - line_2 += full_block - line_3 += full_block - line_4 += full_block - elif step >= 16: - line_1 += ' ' - line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) - line_3 += full_block - line_4 += full_block - elif step >= 8: - line_1 += ' ' - line_2 += ' ' - line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) - line_4 += full_block - else: - line_1 += ' ' - line_2 += ' ' - line_3 += ' ' - line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) - line_1 += COLORS['CLEAR'] - line_2 += COLORS['CLEAR'] - line_3 += COLORS['CLEAR'] - line_4 += COLORS['CLEAR'] + """Generate two-line horizontal graph from rates, returns str.""" + line_1 = '' + line_2 = '' + line_3 = '' + line_4 = '' + for r in rates: + step = get_graph_step(r, scale=32) if oneline: - return line_4 + step = get_graph_step(r, scale=8) + + # Set color + r_color = COLORS['CLEAR'] + if r < IO_VARS['Threshold Graph Fail']: + r_color = COLORS['RED'] + elif r < IO_VARS['Threshold Graph Warn']: + r_color = COLORS['YELLOW'] + elif r > IO_VARS['Threshold Graph Great']: + r_color = COLORS['GREEN'] + + # Build graph + full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) + if step >= 24: + line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) + line_2 += full_block + line_3 += full_block + line_4 += full_block + elif step >= 16: + line_1 += ' ' + line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) + line_3 += full_block + line_4 += full_block + elif step >= 8: + line_1 += ' ' + line_2 += ' ' + line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + line_4 += full_block else: - return '\n'.join([line_1, line_2, line_3, line_4]) + line_1 += ' ' + line_2 += ' ' + line_3 += ' ' + line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + line_1 += COLORS['CLEAR'] + line_2 += COLORS['CLEAR'] + line_3 += COLORS['CLEAR'] + line_4 += COLORS['CLEAR'] + if oneline: + return line_4 + else: + return '\n'.join([line_1, line_2, line_3, line_4]) def get_graph_step(rate, scale=16): - """Get graph step based on rate and scale, returns int.""" - m_rate = rate / (1024**2) - step = 0 - scale_name = 'Scale {}'.format(scale) - for x in range(scale-1, -1, -1): - # Iterate over scale backwards - if m_rate >= IO_VARS[scale_name][x]: - step = x - break - return step + """Get graph step based on rate and scale, returns int.""" + m_rate = rate / (1024**2) + step = 0 + scale_name = 'Scale {}'.format(scale) + for x in range(scale-1, -1, -1): + # Iterate over scale backwards + if m_rate >= IO_VARS[scale_name][x]: + step = x + break + return step def get_read_rate(s): - """Get read rate in bytes/s from dd progress output.""" - real_rate = None - if re.search(r'[KMGT]B/s', s): - human_rate = re.sub(r'^.*\s+(\d+\.?\d*)\s+(.B)/s\s*$', r'\1 \2', s) - real_rate = convert_to_bytes(human_rate) - return real_rate - -def get_smart_details(dev): - """Get SMART data for dev if possible, returns dict.""" - cmd = 'sudo smartctl --all --json {}{}'.format( - '' if '/dev/' in dev else '/dev/', - dev).split() - result = run_program(cmd, check=False) - try: - return json.loads(result.stdout.decode()) - except Exception: - # Let other sections deal with the missing data - return {} - -def get_smart_value(smart_data, smart_id): - """Get SMART value from table, returns int or None.""" - value = None - table = smart_data.get('ata_smart_attributes', {}).get('table', []) - for row in table: - if str(row.get('id', '?')) == str(smart_id): - value = row.get('raw', {}).get('value', None) - return value + """Get read rate in bytes/s from dd progress output.""" + real_rate = None + if re.search(r'[KMGT]B/s', s): + human_rate = re.sub(r'^.*\s+(\d+\.?\d*)\s+(.B)/s\s*$', r'\1 \2', s) + real_rate = convert_to_bytes(human_rate) + return real_rate def get_status_color(s): - """Get color based on status, returns str.""" - color = COLORS['CLEAR'] - if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: - color = COLORS['RED'] - elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: - color = COLORS['YELLOW'] - elif s in ['CS']: - color = COLORS['GREEN'] - return color - -def menu_diags(*args): - """Main HW-Diagnostic menu.""" - diag_modes = [ - {'Name': 'All tests', - 'Tests': ['Prime95', 'NVMe/SMART', 'badblocks', 'iobenchmark']}, - {'Name': 'Prime95', - 'Tests': ['Prime95']}, - {'Name': 'All drive tests', - 'Tests': ['NVMe/SMART', 'badblocks', 'iobenchmark']}, - {'Name': 'NVMe/SMART', - 'Tests': ['NVMe/SMART']}, - {'Name': 'badblocks', - 'Tests': ['badblocks']}, - {'Name': 'I/O Benchmark', - 'Tests': ['iobenchmark']}, - {'Name': 'Quick drive test', - 'Tests': ['Quick', 'NVMe/SMART']}, - ] - actions = [ - {'Letter': 'A', 'Name': 'Audio test'}, - {'Letter': 'K', 'Name': 'Keyboard test'}, - {'Letter': 'N', 'Name': 'Network test'}, - {'Letter': 'M', 'Name': 'Screen Saver - Matrix', 'CRLF': True}, - {'Letter': 'P', 'Name': 'Screen Saver - Pipes'}, - {'Letter': 'Q', 'Name': 'Quit', 'CRLF': True}, - ] - - # CLI-mode actions - if 'DISPLAY' not in global_vars['Env']: - actions.extend([ - {'Letter': 'R', 'Name': 'Reboot', 'CRLF': True}, - {'Letter': 'S', 'Name': 'Shutdown'}, - ]) - - # Quick disk check - if 'quick' in args: - run_tests(['Quick', 'NVMe/SMART']) - exit_script() - - # Show menu - while True: - selection = menu_select( - title = 'Hardware Diagnostics: Menu', - main_entries = diag_modes, - action_entries = actions, - spacer = '──────────────────────────') - if selection.isnumeric(): - ticket_number = None - if diag_modes[int(selection)-1]['Name'] != 'Quick drive test': - clear_screen() - print_standard(' ') - ticket_number = get_ticket_number() - # Save log for non-quick tests - global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") - global_vars['LogDir'] = '{}/Logs/{}_{}'.format( - global_vars['Env']['HOME'], - ticket_number, - global_vars['Date-Time']) - os.makedirs(global_vars['LogDir'], exist_ok=True) - global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( - global_vars['LogDir']) - run_tests(diag_modes[int(selection)-1]['Tests'], ticket_number) - elif selection == 'A': - run_program(['hw-diags-audio'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') - elif selection == 'K': - run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) - elif selection == 'N': - run_program(['hw-diags-network'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') - elif selection == 'M': - run_program(['cmatrix', '-abs'], check=False, pipe=False) - elif selection == 'P': - run_program( - 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split(), - check=False, pipe=False) - elif selection == 'R': - run_program(['systemctl', 'reboot']) - elif selection == 'S': - run_program(['systemctl', 'poweroff']) - elif selection == 'Q': - break - -def run_badblocks(ticket_number): - """Run a read-only test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart badblocks test(s)\n') - progress_file = '{}/badblocks_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['badblocks']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - print_standard('Running badblock test(s):') - for name, dev in sorted(TESTS['badblocks']['Devices'].items()): - cur_status = TESTS['badblocks']['Status'][name] - nvme_smart_status = TESTS['NVMe/SMART']['Status'].get(name, None) - if cur_status == 'Denied': - # Skip denied disks - continue - if nvme_smart_status == 'NS': - TESTS['badblocks']['Status'][name] = 'Skipped' - else: - # Not testing SMART, SMART CS, or SMART OVERRIDE - TESTS['badblocks']['Status'][name] = 'Working' - update_progress() - print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - run_program('tmux split-window -dl 5 {} {} {}'.format( - 'hw-diags-badblocks', - '/dev/{}'.format(name), - progress_file).split()) - wait_for_process('badblocks') - print_standard('Done', timestamp=False) - - # Check results - if os.path.exists(progress_file): - with open(progress_file, 'r') as f: - text = f.read() - TESTS['badblocks']['Results'][name] = text - r = re.search(r'Pass completed.*0/0/0 errors', text) - if r: - TESTS['badblocks']['Status'][name] = 'CS' - else: - TESTS['badblocks']['Status'][name] = 'NS' - - # Move temp file - shutil.move(progress_file, '{}/badblocks-{}.log'.format( - global_vars['LogDir'], name)) - else: - TESTS['badblocks']['Status'][name] = 'NS' - update_progress() - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - pass - -def run_iobenchmark(ticket_number): - """Run a read-only test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart I/O Benchmark test(s)\n') - progress_file = '{}/iobenchmark_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['iobenchmark']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - print_standard('Running benchmark test(s):') - for name, dev in sorted(TESTS['iobenchmark']['Devices'].items()): - cur_status = TESTS['iobenchmark']['Status'][name] - nvme_smart_status = TESTS['NVMe/SMART']['Status'].get(name, None) - bb_status = TESTS['badblocks']['Status'].get(name, None) - if cur_status == 'Denied': - # Skip denied disks - continue - if nvme_smart_status == 'NS': - TESTS['iobenchmark']['Status'][name] = 'Skipped' - elif bb_status in ['NS', 'Skipped']: - TESTS['iobenchmark']['Status'][name] = 'Skipped' - else: - # (SMART tests not run or CS/OVERRIDE) - # AND (BADBLOCKS tests not run or CS) - TESTS['iobenchmark']['Status'][name] = 'Working' - update_progress() - print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - - # Get dev size - cmd = 'sudo lsblk -bdno size /dev/{}'.format(name) - try: - result = run_program(cmd.split()) - dev_size = result.stdout.decode().strip() - dev_size = int(dev_size) - except: - # Failed to get dev size, requires manual testing instead - TESTS['iobenchmark']['Status'][name] = 'ERROR' - continue - if dev_size < IO_VARS['Minimum Dev Size']: - TESTS['iobenchmark']['Status'][name] = 'ERROR' - continue - - # Calculate dd values - ## test_size is the area to be read in bytes - ## If the dev is < 10Gb then it's the whole dev - ## Otherwise it's the larger of 10Gb or 1% of the dev - ## - ## test_chunks is the number of groups of "Chunk Size" in test_size - ## This number is reduced to a multiple of the graph width in - ## order to allow for the data to be condensed cleanly - ## - ## skip_blocks is the number of "Block Size" groups not tested - ## skip_count is the number of blocks to skip per test_chunk - ## skip_extra is how often to add an additional skip block - ## This is needed to ensure an even testing across the dev - ## This is calculated by using the fractional amount left off - ## of the skip_count variable - test_size = min(IO_VARS['Minimum Test Size'], dev_size) - test_size = max( - test_size, dev_size*IO_VARS['Alt Test Size Factor']) - test_chunks = int(test_size // IO_VARS['Chunk Size']) - test_chunks -= test_chunks % IO_VARS['Graph Horizontal Width'] - test_size = test_chunks * IO_VARS['Chunk Size'] - skip_blocks = int((dev_size - test_size) // IO_VARS['Block Size']) - skip_count = int((skip_blocks / test_chunks) // 1) - skip_extra = 0 - try: - skip_extra = 1 + int(1 / ((skip_blocks / test_chunks) % 1)) - except ZeroDivisionError: - # skip_extra == 0 is fine - pass - - # Open dd progress pane after initializing file - with open(progress_file, 'w') as f: - f.write('') - sleep(1) - cmd = 'tmux split-window -dp 75 -PF #D tail -f {}'.format( - progress_file) - result = run_program(cmd.split()) - bottom_pane = result.stdout.decode().strip() - - # Run dd read tests - offset = 0 - TESTS['iobenchmark']['Data'][name] = { - 'Graph': [], - 'Read Rates': []} - for i in range(test_chunks): - i += 1 - s = skip_count - c = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) - if skip_extra and i % skip_extra == 0: - s += 1 - cmd = 'sudo dd bs={b} skip={s} count={c} if=/dev/{n} of={o} iflag=direct'.format( - b=IO_VARS['Block Size'], - s=offset+s, - c=c, - n=name, - o='/dev/null') - result = run_program(cmd.split()) - result_str = result.stderr.decode().replace('\n', '') - cur_rate = get_read_rate(result_str) - TESTS['iobenchmark']['Data'][name]['Read Rates'].append( - cur_rate) - TESTS['iobenchmark']['Data'][name]['Graph'].append( - '{percent:0.1f} {rate}'.format( - percent=i/test_chunks*100, - rate=int(cur_rate/(1024**2)))) - if i % IO_VARS['Progress Refresh Rate'] == 0: - # Update vertical graph - update_io_progress( - percent=i/test_chunks*100, - rate=cur_rate, - progress_file=progress_file) - # Update offset - offset += s + c - print_standard('Done', timestamp=False) - - # Close bottom pane - run_program(['tmux', 'kill-pane', '-t', bottom_pane]) - - # Build report - avg_min_max = 'Average read speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( - sum(TESTS['iobenchmark']['Data'][name]['Read Rates'])/len( - TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2), - min(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2), - max(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2)) - TESTS['iobenchmark']['Data'][name]['Avg/Min/Max'] = avg_min_max - TESTS['iobenchmark']['Data'][name]['Merged Rates'] = [] - pos = 0 - width = int(test_chunks / IO_VARS['Graph Horizontal Width']) - for i in range(IO_VARS['Graph Horizontal Width']): - # Append average rate for WIDTH number of rates to new array - TESTS['iobenchmark']['Data'][name]['Merged Rates'].append(sum( - TESTS['iobenchmark']['Data'][name]['Read Rates'][pos:pos+width])/width) - pos += width - report = generate_horizontal_graph( - TESTS['iobenchmark']['Data'][name]['Merged Rates']) - report += '\n{}'.format(avg_min_max) - TESTS['iobenchmark']['Results'][name] = report - - # Set CS/NS - min_read = min(TESTS['iobenchmark']['Data'][name]['Read Rates']) - avg_read = sum( - TESTS['iobenchmark']['Data'][name]['Read Rates'])/len( - TESTS['iobenchmark']['Data'][name]['Read Rates']) - dev_rotational = dev['lsblk'].get('rota', None) - if dev_rotational == "0": - # Use SSD scale - thresh_min = IO_VARS['Threshold SSD Min'] - thresh_high_avg = IO_VARS['Threshold SSD High Avg'] - thresh_low_avg = IO_VARS['Threshold SSD Low Avg'] - else: - # Use HDD scale - thresh_min = IO_VARS['Threshold HDD Min'] - thresh_high_avg = IO_VARS['Threshold HDD High Avg'] - thresh_low_avg = IO_VARS['Threshold HDD Low Avg'] - if min_read <= thresh_min and avg_read <= thresh_high_avg: - TESTS['iobenchmark']['Status'][name] = 'NS' - elif avg_read <= thresh_low_avg: - TESTS['iobenchmark']['Status'][name] = 'NS' - else: - TESTS['iobenchmark']['Status'][name] = 'CS' - - # Save logs - dest_filename = '{}/iobenchmark-{}.log'.format(global_vars['LogDir'], name) - shutil.move(progress_file, dest_filename) - with open(dest_filename.replace('.', '-raw.'), 'a') as f: - f.write('\n'.join(TESTS['iobenchmark']['Data'][name]['Graph'])) - update_progress() - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - pass - -def run_mprime(ticket_number): - """Run Prime95 for MPRIME_LIMIT minutes while showing the temps.""" - aborted = False - print_log('\nStart Prime95 test') - TESTS['Prime95']['Status'] = 'Working' - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dl 10 -c {wd} {cmd} {wd}'.format( - wd=global_vars['TmpDir'], cmd='hw-diags-prime95').split()) - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - run_program('tmux split-window -bd watch -c -n1 -t hw-sensors'.split()) - run_program('tmux resize-pane -y 3'.split()) - - # Start test - run_program(['apple-fans', 'max']) - try: - for i in range(int(MPRIME_LIMIT)): - clear_screen() - min_left = int(MPRIME_LIMIT) - i - print_standard('Running Prime95 ({} minute{} left)'.format( - min_left, - 's' if min_left != 1 else '')) - print_warning('If running too hot, press CTRL+c to abort the test') - sleep(60) - except KeyboardInterrupt: - # Catch CTRL+C - aborted = True - TESTS['Prime95']['Status'] = 'Aborted' - print_warning('\nAborted.') - update_progress() - - # Save "final" temps - run_program( - cmd = 'hw-sensors >> "{}/Final Temps.out"'.format( - global_vars['LogDir']).split(), - check = False, - pipe = False, - shell = True) - run_program( - cmd = 'hw-sensors --nocolor >> "{}/Final Temps.log"'.format( - global_vars['LogDir']).split(), - check = False, - pipe = False, - shell = True) - - # Stop test - run_program('killall -s INT mprime'.split(), check=False) - run_program(['apple-fans', 'auto']) - - # Move logs to Ticket folder - for item in os.scandir(global_vars['TmpDir']): - try: - shutil.move(item.path, global_vars['LogDir']) - except Exception: - print_error('ERROR: Failed to move "{}" to "{}"'.format( - item.path, - global_vars['LogDir'])) - - # Check logs - TESTS['Prime95']['NS'] = False - TESTS['Prime95']['CS'] = False - log = '{}/results.txt'.format(global_vars['LogDir']) - if os.path.exists(log): - with open(log, 'r') as f: - text = f.read() - TESTS['Prime95']['results.txt'] = text - r = re.search(r'(error|fail)', text) - TESTS['Prime95']['NS'] = bool(r) - log = '{}/prime.log'.format(global_vars['LogDir']) - if os.path.exists(log): - with open(log, 'r') as f: - text = f.read() - TESTS['Prime95']['prime.log'] = text - r = re.search(r'completed.*0 errors, 0 warnings', text) - TESTS['Prime95']['CS'] = bool(r) - - # Update status - if not aborted: - if TESTS['Prime95']['NS']: - TESTS['Prime95']['Status'] = 'NS' - elif TESTS['Prime95']['CS']: - TESTS['Prime95']['Status'] = 'CS' - else: - TESTS['Prime95']['Status'] = 'Unknown' - update_progress() - - if aborted: - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled']: - if not ask('Proceed to next test?'): - for name in TESTS['NVMe/SMART']['Devices'].keys(): - for t in ['NVMe/SMART', 'badblocks', 'iobenchmark']: - cur_status = TESTS[t]['Status'][name] - if cur_status not in ['CS', 'Denied', 'NS']: - TESTS[t]['Status'][name] = 'Aborted' - run_program('tmux kill-pane -a'.split()) - raise GenericError - - # Done - run_program('tmux kill-pane -a'.split()) - -def run_nvme_smart(ticket_number): - """Run the built-in NVMe or SMART test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart NVMe/SMART test(s)\n') - progress_file = '{}/selftest_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dl 3 watch -c -n1 -t cat {}'.format( - progress_file).split()) - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - TESTS['NVMe/SMART']['Short Test'][name] = None - cur_status = TESTS['NVMe/SMART']['Status'][name] - if cur_status == 'OVERRIDE': - # Skipping test per user request - continue - if TESTS['NVMe/SMART']['Quick'] or dev.get('NVMe Disk', False): - # Skip SMART self-tests for quick checks and NVMe disks - if dev['Quick Health OK']: - TESTS['NVMe/SMART']['Status'][name] = 'CS' - else: - TESTS['NVMe/SMART']['Status'][name] = 'NS' - elif not dev['Quick Health OK']: - # SMART overall == Failed or attributes bad, avoid self-test - TESTS['NVMe/SMART']['Status'][name] = 'NS' - else: - # Start SMART short self-test - test_length = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'polling_minutes', {}).get( - 'short', 5) - test_length = int(test_length) + 5 - TESTS['NVMe/SMART']['Status'][name] = 'Working' - update_progress() - print_standard('Running SMART short self-test(s):') - print_standard( - ' /dev/{:8}({} minutes)... '.format(name, test_length), - end='', flush=True) - run_program( - 'sudo smartctl -t short /dev/{}'.format(name).split(), - check=False) - - # Wait and show progress (in 10 second increments) - for iteration in range(int(test_length*60/10)): - # Update SMART data - dev['smartctl'] = get_smart_details(name) - - # Check if test is complete - if iteration >= 6: - done = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'passed', False) - if done: - break - - # Update progress_file - with open(progress_file, 'w') as f: - f.write('SMART self-test status:\n {}'.format( - dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'string', 'unknown'))) - sleep(10) - os.remove(progress_file) - - # Check result - test_passed = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'passed', False) - if test_passed: - TESTS['NVMe/SMART']['Status'][name] = 'CS' - TESTS['NVMe/SMART']['Short Test'][name] = 'CS' - else: - TESTS['NVMe/SMART']['Status'][name] = 'NS' - TESTS['NVMe/SMART']['Short Test'][name] = 'NS' - update_progress() - print_standard('Done', timestamp=False) - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - -def run_tests(tests, ticket_number=None): - """Run selected hardware test(s).""" - clear_screen() - print_standard('Starting Hardware Diagnostics') - if ticket_number: - print_standard(' For Ticket #{}'.format(ticket_number)) - print_standard(' ') - print_standard('Running tests: {}'.format(', '.join(tests))) - # Enable selected tests - for t in ['Prime95', 'NVMe/SMART', 'badblocks', 'iobenchmark']: - TESTS[t]['Enabled'] = t in tests - TESTS['NVMe/SMART']['Quick'] = 'Quick' in tests - - # Initialize - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - print_standard(' ') - scan_disks() - update_progress() - - # Run - mprime_aborted = False - if TESTS['Prime95']['Enabled']: - try: - run_mprime(ticket_number) - except GenericError: - mprime_aborted = True - if not mprime_aborted: - if TESTS['NVMe/SMART']['Enabled']: - run_nvme_smart(ticket_number) - if TESTS['badblocks']['Enabled']: - run_badblocks(ticket_number) - if TESTS['iobenchmark']['Enabled']: - run_iobenchmark(ticket_number) - - # Show results - show_results() - - # Open log - if not TESTS['NVMe/SMART']['Quick'] and ENABLED_OPEN_LOGS: - try: - popen_program(['nohup', 'leafpad', global_vars['LogFile']], pipe=True) - except Exception: - print_error('ERROR: Failed to open log: {}'.format( - global_vars['LogFile'])) - pause('Press Enter to exit...') - -def scan_disks(full_paths=False, only_path=None): - """Scan for disks eligible for hardware testing.""" - - # Get eligible disk list - cmd = ['lsblk', '-J', '-O'] - if full_paths: - cmd.append('-p') - if only_path: - cmd.append(only_path) - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - devs = {} - for d in json_data.get('blockdevices', []): - if d['type'] == 'disk': - if d['hotplug'] == '0': - devs[d['name']] = {'lsblk': d} - TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' - TESTS['badblocks']['Status'][d['name']] = 'Pending' - TESTS['iobenchmark']['Status'][d['name']] = 'Pending' - else: - # Skip WizardKit devices - skip_dev=False - wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) - for c in d.get('children', []): - r = re.search( - wk_label_regex, c.get('label', ''), re.IGNORECASE) - skip_dev = bool(r) - if not skip_dev: - devs[d['name']] = {'lsblk': d} - TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' - TESTS['badblocks']['Status'][d['name']] = 'Pending' - TESTS['iobenchmark']['Status'][d['name']] = 'Pending' - - for dev, data in devs.items(): - # Get SMART attributes - run_program( - cmd = 'sudo smartctl -s on {}{}'.format( - '' if full_paths else '/dev/', - dev).split(), - check = False) - data['smartctl'] = get_smart_details(dev) - - # Get NVMe attributes - if data['lsblk']['tran'] == 'nvme': - cmd = 'sudo nvme smart-log /dev/{} -o json'.format(dev).split() - cmd = 'sudo nvme smart-log {}{} -o json'.format( - '' if full_paths else '/dev/', - dev).split() - result = run_program(cmd, check=False) - try: - data['nvme-cli'] = json.loads(result.stdout.decode()) - except Exception: - # Let other sections deal with the missing data - data['nvme-cli'] = {} - data['NVMe Disk'] = True - - # Set "Quick Health OK" value - ## NOTE: If False then require override for badblocks test - wanted_smart_list = [ - 'ata_smart_attributes', - 'ata_smart_data', - 'smart_status', - ] - if data.get('NVMe Disk', False): - crit_warn = data['nvme-cli'].get('critical_warning', 1) - if crit_warn == 0: - dev_name = data['lsblk']['name'] - data['Quick Health OK'] = True - TESTS['NVMe/SMART']['Status'][dev_name] = 'CS' - else: - data['Quick Health OK'] = False - elif set(wanted_smart_list).issubset(data['smartctl'].keys()): - data['SMART Pass'] = data['smartctl'].get('smart_status', {}).get( - 'passed', False) - data['Quick Health OK'] = data['SMART Pass'] - data['SMART Support'] = True - else: - data['Quick Health OK'] = False - data['SMART Support'] = False - - # Ask for manual overrides if necessary - if TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - show_disk_details(data) - needs_override = False - if not data['Quick Health OK']: - needs_override = True - print_warning( - "WARNING: Health can't be confirmed for: /dev/{}".format(dev)) - if get_smart_value(data['smartctl'], '199'): - # SMART attribute present and it's value is non-zero - needs_override = True - print_warning( - 'WARNING: SMART 199/C7 error detected on /dev/{}'.format(dev)) - print_standard(' (Have you tried swapping the drive cable?)') - if needs_override: - dev_name = data['lsblk']['name'] - print_standard(' ') - if ask('Run tests on this device anyway?'): - TESTS['NVMe/SMART']['Status'][dev_name] = 'OVERRIDE' - else: - TESTS['NVMe/SMART']['Status'][dev_name] = 'Skipped' - TESTS['badblocks']['Status'][dev_name] = 'Denied' - TESTS['iobenchmark']['Status'][dev_name] = 'Denied' - print_standard(' ') # In case there's more than one "OVERRIDE" disk - - TESTS['NVMe/SMART']['Devices'] = devs - TESTS['badblocks']['Devices'] = devs - TESTS['iobenchmark']['Devices'] = devs - return devs - -def show_disk_details(dev, only_attributes=False): - """Display disk details.""" - dev_name = dev['lsblk']['name'] - if not only_attributes: - # Device description - print_info('Device: {}{}'.format( - '' if '/dev/' in dev['lsblk']['name'] else '/dev/', - dev['lsblk']['name'])) - print_standard(' {:>4} ({}) {} {}'.format( - str(dev['lsblk'].get('size', '???b')).strip(), - str(dev['lsblk'].get('tran', '???')).strip().upper().replace( - 'NVME', 'NVMe'), - str(dev['lsblk'].get('model', 'Unknown Model')).strip(), - str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(), - )) - - # Warnings - if dev.get('NVMe Disk', False): - if dev['Quick Health OK']: - print_warning('WARNING: NVMe support is still experimental') - else: - print_error('ERROR: NVMe disk is reporting critical warnings') - elif not dev['SMART Support']: - print_error('ERROR: Unable to retrieve SMART data') - elif not dev['SMART Pass']: - print_error('ERROR: SMART overall-health assessment result: FAILED') - - # Attributes - if dev.get('NVMe Disk', False): - if only_attributes: - print_info('SMART Attributes:', end='') - print_warning(' Updated: {}'.format( - time.strftime('%Y-%m-%d %H:%M %Z'))) - else: - print_info('Attributes:') - for attrib, threshold in sorted(ATTRIBUTES['NVMe'].items()): - if attrib in dev['nvme-cli']: - print_standard( - ' {:37}'.format(attrib.replace('_', ' ').title()), - end='', flush=True) - raw_num = dev['nvme-cli'][attrib] - raw_str = str(raw_num) - if (threshold.get('Error', False) and - raw_num >= threshold.get('Error', -1)): - print_error(raw_str, timestamp=False) - if not threshold.get('Ignore', False): - dev['Quick Health OK'] = False - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - elif (threshold.get('Warning', False) and - raw_num >= threshold.get('Warning', -1)): - print_warning(raw_str, timestamp=False) - else: - print_success(raw_str, timestamp=False) - elif dev['smartctl'].get('ata_smart_attributes', None): - # SMART attributes - if only_attributes: - print_info('SMART Attributes:', end='') - print_warning(' Updated: {}'.format( - time.strftime('%Y-%m-%d %H:%M %Z'))) - else: - print_info('Attributes:') - s_table = dev['smartctl'].get('ata_smart_attributes', {}).get( - 'table', {}) - s_table = {a.get('id', 'Unknown'): a for a in s_table} - for attrib, threshold in sorted(ATTRIBUTES['SMART'].items()): - if attrib in s_table: - print_standard( - ' {:>3} {:32}'.format( - attrib, - s_table[attrib]['name']).replace('_', ' ').title(), - end='', flush=True) - raw_str = s_table[attrib]['raw']['string'] - raw_num = re.sub(r'^(\d+).*$', r'\1', raw_str) - try: - raw_num = float(raw_num) - except ValueError: - # Not sure about this one, print raw_str without color? - print_standard(raw_str, timestamp=False) - continue - if (threshold.get('Error', False) and - raw_num >= threshold.get('Error', -1)): - print_error(raw_str, timestamp=False) - if not threshold.get('Ignore', False): - dev['Quick Health OK'] = False - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - elif (threshold.get('Warning', False) and - raw_num >= threshold.get('Warning', -1)): - print_warning(raw_str, timestamp=False) - else: - print_success(raw_str, timestamp=False) - -def show_results(): - """Show results for selected test(s).""" - clear_screen() - print_log('\n───────────────────────────') - print_standard('Hardware Diagnostic Results') - update_progress() - - # Set Window layout and show progress - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Prime95 - if TESTS['Prime95']['Enabled']: - print_success('\nPrime95:') - for log, regex in [ - ['results.txt', r'(error|fail)'], - ['prime.log', r'completed.*0 errors, 0 warnings']]: - if log in TESTS['Prime95']: - print_info('Log: {}'.format(log)) - lines = [line.strip() for line - in TESTS['Prime95'][log].splitlines() - if re.search(regex, line, re.IGNORECASE)] - for line in lines[-4:]: - line = re.sub(r'^.*Worker #\d.*Torture Test (.*)', r'\1', - line, re.IGNORECASE) - if TESTS['Prime95'].get('NS', False): - print_error(' {}'.format(line)) - else: - print_standard(' {}'.format(line)) - print_info('Final temps') - print_log(' See Final Temps.log') - with open('{}/Final Temps.out'.format(global_vars['LogDir']), 'r') as f: - for line in f.readlines(): - if re.search(r'^\s*$', line.strip()): - # Stop after coretemps (which should be first) - break - print(' {}'.format(line.strip())) - print_standard(' ') - - # NVMe/SMART / badblocks / iobenchmark - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - print_success('Disks:') - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - show_disk_details(dev) - bb_status = TESTS['badblocks']['Status'].get(name, None) - if (TESTS['badblocks']['Enabled'] - and bb_status not in ['Denied', 'OVERRIDE', 'Skipped']): - print_info('badblocks:') - result = TESTS['badblocks']['Results'].get(name, '') - for line in result.splitlines(): - if re.search(r'Pass completed', line, re.IGNORECASE): - line = re.sub( - r'Pass completed,?\s+', r'', - line.strip(), re.IGNORECASE) - if TESTS['badblocks']['Status'][name] == 'CS': - print_standard(' {}'.format(line)) - else: - print_error(' {}'.format(line)) - io_status = TESTS['iobenchmark']['Status'].get(name, None) - if (TESTS['iobenchmark']['Enabled'] - and io_status not in ['Denied', 'OVERRIDE', 'Skipped']): - print_info('Benchmark:') - result = TESTS['iobenchmark']['Results'].get(name, '') - for line in result.split('\n'): - print_standard(' {}'.format(line)) - print_standard(' ') - - # Done - pause('Press Enter to return to main menu... ') - run_program('tmux kill-pane -a'.split()) + """Get color based on status, returns str.""" + color = COLORS['CLEAR'] + if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + color = COLORS['RED'] + elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: + color = COLORS['YELLOW'] + elif s in ['CS']: + color = COLORS['GREEN'] + return color def update_io_progress(percent, rate, progress_file): - """Update I/O progress file.""" - bar_color = COLORS['CLEAR'] - rate_color = COLORS['CLEAR'] - step = get_graph_step(rate, scale=32) - if rate < IO_VARS['Threshold Graph Fail']: - bar_color = COLORS['RED'] - rate_color = COLORS['YELLOW'] - elif rate < IO_VARS['Threshold Graph Warn']: - bar_color = COLORS['YELLOW'] - rate_color = COLORS['YELLOW'] - elif rate > IO_VARS['Threshold Graph Great']: - bar_color = COLORS['GREEN'] - rate_color = COLORS['GREEN'] - line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( - p=percent, - b_color=bar_color, - b=IO_VARS['Graph Vertical'][step], - r_color=rate_color, - r=rate/(1024**2), - c=COLORS['CLEAR']) - with open(progress_file, 'a') as f: - f.write(line) - -def update_progress(): - """Update progress file.""" - if 'Progress Out' not in TESTS: - TESTS['Progress Out'] = '{}/progress.out'.format(global_vars['LogDir']) - output = [] - output.append('{BLUE}HW Diagnostics{CLEAR}'.format(**COLORS)) - output.append('───────────────') - if TESTS['Prime95']['Enabled']: - output.append(' ') - output.append('{BLUE}Prime95{s_color}{status:>8}{CLEAR}'.format( - s_color = get_status_color(TESTS['Prime95']['Status']), - status = TESTS['Prime95']['Status'], - **COLORS)) - if TESTS['NVMe/SMART']['Enabled']: - output.append(' ') - output.append('{BLUE}NVMe / SMART{CLEAR}'.format(**COLORS)) - if TESTS['NVMe/SMART']['Quick']: - output.append('{YELLOW} (Quick Check){CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['NVMe/SMART']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - if TESTS['badblocks']['Enabled']: - output.append(' ') - output.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['badblocks']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - if TESTS['iobenchmark']['Enabled']: - output.append(' ') - output.append('{BLUE}I/O Benchmark{CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['iobenchmark']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - - # Add line-endings - output = ['{}\n'.format(line) for line in output] - - with open(TESTS['Progress Out'], 'w') as f: - f.writelines(output) + """Update I/O progress file.""" + bar_color = COLORS['CLEAR'] + rate_color = COLORS['CLEAR'] + step = get_graph_step(rate, scale=32) + if rate < IO_VARS['Threshold Graph Fail']: + bar_color = COLORS['RED'] + rate_color = COLORS['YELLOW'] + elif rate < IO_VARS['Threshold Graph Warn']: + bar_color = COLORS['YELLOW'] + rate_color = COLORS['YELLOW'] + elif rate > IO_VARS['Threshold Graph Great']: + bar_color = COLORS['GREEN'] + rate_color = COLORS['GREEN'] + line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( + p=percent, + b_color=bar_color, + b=IO_VARS['Graph Vertical'][step], + r_color=rate_color, + r=rate/(1024**2), + c=COLORS['CLEAR']) + with open(progress_file, 'a') as f: + f.write(line) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 From 3fdd8c629c8afd6c2ded1a52f821e63d548145ab Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:47:44 -0700 Subject: [PATCH 025/121] Rewrote main menu * First options are presets followed by individual tests * Selecting presets will toggle the selections * Screensavers are hidden but still present --- .bin/Scripts/functions/common.py | 6 +- .bin/Scripts/functions/hw_diags.py | 168 +++++++++++++++++++++++++++++ .bin/Scripts/hw-diags-menu | 3 +- 3 files changed, 174 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index ae958645..d3f04ef6 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -318,7 +318,7 @@ def major_exception(): exit_script(1) def menu_select(title='~ Untitled Menu ~', - prompt='Please make a selection', secret_exit=False, + prompt='Please make a selection', secret_actions=[], secret_exit=False, main_entries=[], action_entries=[], disabled_label='DISABLED', spacer=''): """Display options in a menu and return selected option as a str.""" @@ -334,8 +334,10 @@ def menu_select(title='~ Untitled Menu ~', menu_splash = '{}\n{}\n'.format(title, spacer) width = len(str(len(main_entries))) valid_answers = [] - if (secret_exit): + if secret_exit: valid_answers.append('Q') + if secret_actions: + valid_answers.extend(secret_actions) # Add main entries for i in range(len(main_entries)): diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 830d948a..d32fb499 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -216,6 +216,174 @@ def get_status_color(s): color = COLORS['GREEN'] return color +def menu_diags(state, args): + """Main menu to select and run HW tests.""" + args = [a.lower() for a in args] + quick_label = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) + title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( + **COLORS) + # NOTE: Changing the order of main_options will break everything + main_options = [ + {'Base Name': 'Full Diagnostic', 'Enabled': True}, + {'Base Name': 'Drive Diagnostic', 'Enabled': False}, + {'Base Name': 'Drive Diagnostic (Quick)', 'Enabled': False}, + {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, + {'Base Name': 'NVMe / SMART', 'Enabled': True}, + {'Base Name': 'badblocks', 'Enabled': True}, + {'Base Name': 'I/O Benchmark', 'Enabled': True}, + ] + actions = [ + {'Letter': 'A', 'Name': 'Audio Test'}, + {'Letter': 'K', 'Name': 'Keyboard Test'}, + {'Letter': 'N', 'Name': 'Network Test'}, + {'Letter': 'S', 'Name': 'Start', 'CRLF': True}, + {'Letter': 'Q', 'Name': 'Quit'}, + ] + secret_actions = ['M', 'T'] + + # CLI mode check + if '--cli' in args or 'DISPLAY' not in global_vars['Env']: + actions.append({'Letter': 'R', 'Name': 'Reboot'}) + actions.append({'Letter': 'P', 'Name': 'Power Off'}) + + while True: + # Set quick mode as necessary + if main_options[2]['Enabled'] and main_options[4]['Enabled']: + # Check if only Drive Diags (Quick) and NVMe/SMART are enabled + # If so, verify no other tests are enabled and set quick_mode + state.quick_mode = True + for opt in main_options[3:4] + main_options[5:]: + state.quick_mode &= not opt['Enabled'] + else: + state.quick_mode = False + + # Deselect presets + slice_end = 3 + if state.quick_mode: + slice_end = 2 + for opt in main_options[:slice_end]: + opt['Enabled'] = False + + # Verify preset selections + num_tests_selected = 0 + for opt in main_options[3:]: + if opt['Enabled']: + num_tests_selected += 1 + if num_tests_selected == 4: + # Full + main_options[0]['Enabled'] = True + elif num_tests_selected == 3 and not main_options[3]['Enabled']: + # Drive + main_options[1]['Enabled'] = True + + # Update checkboxes + for opt in main_options: + _nvme_smart = opt['Base Name'] == 'NVMe / SMART' + opt['Name'] = '{} {} {}'.format( + '[✓]' if opt['Enabled'] else '[ ]', + opt['Base Name'], + quick_label if state.quick_mode and _nvme_smart else '') + + # Show menu + selection = menu_select( + title=title, + main_entries=main_options, + action_entries=actions, + secret_actions=secret_actions, + spacer='───────────────────────────────') + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + + # Handle presets + if index == 0: + # Full + if main_options[index]['Enabled']: + for opt in main_options[1:3]: + opt['Enabled'] = False + for opt in main_options[3:]: + opt['Enabled'] = True + else: + for opt in main_options[3:]: + opt['Enabled'] = False + elif index == 1: + # Drive + if main_options[index]['Enabled']: + main_options[0]['Enabled'] = False + for opt in main_options[2:4]: + opt['Enabled'] = False + for opt in main_options[4:]: + opt['Enabled'] = True + else: + for opt in main_options[4:]: + opt['Enabled'] = False + elif index == 2: + # Drive (Quick) + if main_options[index]['Enabled']: + for opt in main_options[:2] + main_options[3:]: + opt['Enabled'] = False + main_options[4]['Enabled'] = True + else: + main_options[4]['Enabled'] = False + elif selection == 'A': + run_audio_test() + elif selection == 'K': + run_keyboard_test() + elif selection == 'N': + run_network_test() + elif selection == 'M': + secret_screensaver('matrix') + elif selection == 'T': + # Tubes is close to pipes + secret_screensaver('pipes') + elif selection == 'R': + run_program(['systemctl', 'reboot']) + elif selection == 'P': + run_program(['systemctl', 'poweroff']) + elif selection == 'Q': + break + elif selection == 'S': + # Run test(s) + clear_screen() + print('Fake test(s) placeholder for now...') + pause('Press Enter to return to main menu... ') + +def run_audio_test(): + """Run audio test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake audio placeholder for now...') + #run_program(['hw-diags-audio'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def run_keyboard_test(): + """Run keyboard test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake keyboard placeholder for now...') + #run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def run_network_test(): + """Run network test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake network placeholder for now...') + #run_program(['hw-diags-network'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def secret_screensaver(screensaver=None): + """Show screensaver.""" + if screensaver == 'matrix': + cmd = 'cmatrix -abs'.split() + elif screensaver == 'pipes': + cmd = 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split() + else: + raise Exception('Invalid screensaver') + run_program(cmd, check=False, pipe=False) + def update_io_progress(percent, rate, progress_file): """Update I/O progress file.""" bar_color = COLORS['CLEAR'] diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index f7c1739c..c67cc5f4 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -17,7 +17,8 @@ if __name__ == '__main__': clear_screen() # Show menu - menu_diags(*sys.argv) + state = State() + menu_diags(state, sys.argv) # Done #print_standard('\nDone.') From 18fc97293e546d50e2606e055a8d1ec57be04913 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:50:55 -0700 Subject: [PATCH 026/121] Renamed Drive to Disk to align options in menu --- .bin/Scripts/functions/hw_diags.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d32fb499..b5d553de 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -225,8 +225,8 @@ def menu_diags(state, args): # NOTE: Changing the order of main_options will break everything main_options = [ {'Base Name': 'Full Diagnostic', 'Enabled': True}, - {'Base Name': 'Drive Diagnostic', 'Enabled': False}, - {'Base Name': 'Drive Diagnostic (Quick)', 'Enabled': False}, + {'Base Name': 'Disk Diagnostic', 'Enabled': False}, + {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, {'Base Name': 'NVMe / SMART', 'Enabled': True}, {'Base Name': 'badblocks', 'Enabled': True}, @@ -249,7 +249,7 @@ def menu_diags(state, args): while True: # Set quick mode as necessary if main_options[2]['Enabled'] and main_options[4]['Enabled']: - # Check if only Drive Diags (Quick) and NVMe/SMART are enabled + # Check if only Disk Diags (Quick) and NVMe/SMART are enabled # If so, verify no other tests are enabled and set quick_mode state.quick_mode = True for opt in main_options[3:4] + main_options[5:]: @@ -273,7 +273,7 @@ def menu_diags(state, args): # Full main_options[0]['Enabled'] = True elif num_tests_selected == 3 and not main_options[3]['Enabled']: - # Drive + # Disk main_options[1]['Enabled'] = True # Update checkboxes @@ -309,7 +309,7 @@ def menu_diags(state, args): for opt in main_options[3:]: opt['Enabled'] = False elif index == 1: - # Drive + # Disk if main_options[index]['Enabled']: main_options[0]['Enabled'] = False for opt in main_options[2:4]: @@ -320,7 +320,7 @@ def menu_diags(state, args): for opt in main_options[4:]: opt['Enabled'] = False elif index == 2: - # Drive (Quick) + # Disk (Quick) if main_options[index]['Enabled']: for opt in main_options[:2] + main_options[3:]: opt['Enabled'] = False From 560929e2fa74ba35301758d17bfbfbfc526f1ad9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:54:06 -0700 Subject: [PATCH 027/121] Removed extra line break in menu_select --- .bin/Scripts/functions/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index d3f04ef6..b5896966 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -369,7 +369,6 @@ def menu_select(title='~ Untitled Menu ~', letter = entry['Letter'].upper(), width = len(str(len(action_entries))), name = entry['Name']) - menu_splash += '\n' answer = '' From 2df4d48bb3af0f424fd1248a5e89af24035d588f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 20:15:56 -0700 Subject: [PATCH 028/121] Show selected tests on run --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index b5d553de..3e08c9ed 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -118,10 +118,10 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.started = False self.tests = { - 'Prime95': {'Enabled': False, 'Result': None, 'Status': None}, - 'NVMe / SMART': {'Enabled': False}, - 'badblocks': {'Enabled': False}, - 'I/O Benchmark': {'Enabled': False}, + 'Prime95 & Temps': {'Enabled': False, 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False}, + 'badblocks': {'Enabled': False}, + 'I/O Benchmark': {'Enabled': False}, } self.add_devs() @@ -347,7 +347,18 @@ def menu_diags(state, args): elif selection == 'S': # Run test(s) clear_screen() - print('Fake test(s) placeholder for now...') + print('Tests:') + for opt in main_options[3:]: + _nvme_smart = opt['Base Name'] == 'NVMe / SMART' + # Update state + state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] + print(' {:<15} {}{}{} {}'.format( + opt['Base Name'], + COLORS['GREEN'] if opt['Enabled'] else COLORS['RED'], + 'Enabled' if opt['Enabled'] else 'Disabled', + COLORS['CLEAR'], + quick_label if state.quick_mode and _nvme_smart else '')) + print('\nFake test(s) placeholder for now...') pause('Press Enter to return to main menu... ') def run_audio_test(): From 70a742e69c832484df6c5c1b6f70e5e6c1abde74 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:10:58 -0700 Subject: [PATCH 029/121] Add device details from lsblk * Also ensure sane types for some attributes --- .bin/Scripts/functions/hw_diags.py | 52 ++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3e08c9ed..5bb03c9b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -68,10 +68,14 @@ class DevObj(): """Device object for tracking device specific data.""" def __init__(self, dev_path): self.failing = False + self.labels = [] + self.lsblk = {} + self.name = re.sub(r'^.*/(.*)', r'\1', dev_path) self.nvme_attributes = {} self.override = False self.path = dev_path self.smart_attributes = {} + self.smartctl = {} self.tests = { 'NVMe / SMART': {'Result': None, 'Status': None}, 'badblocks': {'Result': None, 'Status': None}, @@ -82,18 +86,54 @@ class DevObj(): 'Graph Data': []}, } self.get_details() + self.get_smart_details() def get_details(self): + """Get data from lsblk.""" + cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + self.lsblk = json_data['blockdevices'][0] + except Exception: + # Leave self.lsblk empty + pass + + # Set necessary details + self.lsblk['model'] = self.lsblk.get('model', 'Unknown Model') + self.lsblk['name'] = self.lsblk.get('name', self.path) + self.lsblk['rota'] = self.lsblk.get('rota', True) + self.lsblk['serial'] = self.lsblk.get('serial', 'Unknown Serial') + self.lsblk['size'] = self.lsblk.get('size', '???b') + self.lsblk['tran'] = self.lsblk.get('tran', '???') + + # Ensure certain attributes are strings + for attr in ['model', 'name', 'rota', 'serial', 'size', 'tran']: + if not isinstance(self.lsblk[attr], str): + self.lsblk[attr] = str(self.lsblk[attr]) + self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') + + # Build list of labels + for dev in [self.lsblk, *self.lsblk.get('children', [])]: + self.labels.append(dev.get('label', '')) + self.labels.append(dev.get('partlabel', '')) + self.labels = [str(label) for label in self.labels if label] + + def get_smart_details(self): """Get data from smartctl.""" cmd = ['sudo', 'smartctl', '--all', '--json', self.path] - result = run_program(cmd, check=False) - self.data = json.loads(result.stdout.decode()) + try: + result = run_program(cmd, check=False) + self.smartctl = json.loads(result.stdout.decode()) + except Exception: + # Leave self.smartctl empty + pass # Check for attributes - if KEY_NVME in self.data: - self.nvme_attributes.update(self.data[KEY_NVME]) - elif KEY_SMART in self.data: - for a in self.data[KEY_SMART].get('table', {}): + if KEY_NVME in self.smartctl: + self.nvme_attributes.update(self.smartctl[KEY_NVME]) + elif KEY_SMART in self.smartctl: + for a in self.smartctl[KEY_SMART].get('table', {}): _id = str(a.get('id', 'UNKNOWN')) _name = str(a.get('name', 'UNKNOWN')) _raw = a.get('raw', {}).get('value', -1) From 6014a8fb70275d7be2b8b6a7f6c548b7ae718161 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:18:45 -0700 Subject: [PATCH 030/121] Don't add WK or loopback devices --- .bin/Scripts/functions/hw_diags.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 5bb03c9b..8f543aa5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -171,7 +171,22 @@ class State(): result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) for dev in json_data['blockdevices']: - self.devs.append(DevObj(dev['name'])) + skip_dev = False + dev_obj = DevObj(dev['name']) + + # Skip loopback devices + if dev_obj.lsblk['tran'] == 'NONE': + skip_dev = True + + # Skip WK devices + wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) + for label in dev_obj.labels: + if re.search(wk_label_regex, label, re.IGNORECASE): + skip_dev = True + + # Add device + if not skip_dev: + self.devs.append(DevObj(dev['name'])) # Functions def generate_horizontal_graph(rates, oneline=False): From 5701b53026aa758c879db61454fcff9326a67e73 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:55:17 -0700 Subject: [PATCH 031/121] Added --quick argument to skip menu --- .bin/Scripts/functions/hw_diags.py | 147 +++++++++++++++++------------ 1 file changed, 86 insertions(+), 61 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8f543aa5..3de0806f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -61,6 +61,7 @@ IO_VARS = { } KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' +QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PATH_WIDTH = 21 # Classes @@ -156,12 +157,14 @@ class State(): self.devs = [] self.finished = False self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.quick_mode = False self.started = False self.tests = { - 'Prime95 & Temps': {'Enabled': False, 'Result': None, 'Status': None}, - 'NVMe / SMART': {'Enabled': False}, - 'badblocks': {'Enabled': False}, - 'I/O Benchmark': {'Enabled': False}, + 'Prime95 & Temps': {'Enabled': False, 'Order': 1, + 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False, 'Order': 2}, + 'badblocks': {'Enabled': False, 'Order': 3}, + 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } self.add_devs() @@ -274,18 +277,17 @@ def get_status_color(s): def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] - quick_label = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( **COLORS) # NOTE: Changing the order of main_options will break everything main_options = [ - {'Base Name': 'Full Diagnostic', 'Enabled': True}, + {'Base Name': 'Full Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, - {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, - {'Base Name': 'NVMe / SMART', 'Enabled': True}, - {'Base Name': 'badblocks', 'Enabled': True}, - {'Base Name': 'I/O Benchmark', 'Enabled': True}, + {'Base Name': 'Prime95 & Temps', 'Enabled': False, 'CRLF': True}, + {'Base Name': 'NVMe / SMART', 'Enabled': False}, + {'Base Name': 'badblocks', 'Enabled': False}, + {'Base Name': 'I/O Benchmark', 'Enabled': False}, ] actions = [ {'Letter': 'A', 'Name': 'Audio Test'}, @@ -295,12 +297,22 @@ def menu_diags(state, args): {'Letter': 'Q', 'Name': 'Quit'}, ] secret_actions = ['M', 'T'] + + # Set initial selections + update_main_options(state, '1', main_options) # CLI mode check if '--cli' in args or 'DISPLAY' not in global_vars['Env']: actions.append({'Letter': 'R', 'Name': 'Reboot'}) actions.append({'Letter': 'P', 'Name': 'Power Off'}) + # Skip menu if running quick check + if '--quick' in args: + update_main_options(state, '3', main_options) + state.quick_mode = True + run_hw_tests(state) + return True + while True: # Set quick mode as necessary if main_options[2]['Enabled'] and main_options[4]['Enabled']: @@ -337,7 +349,7 @@ def menu_diags(state, args): opt['Name'] = '{} {} {}'.format( '[✓]' if opt['Enabled'] else '[ ]', opt['Base Name'], - quick_label if state.quick_mode and _nvme_smart else '') + QUICK_LABEL if state.quick_mode and _nvme_smart else '') # Show menu selection = menu_select( @@ -348,40 +360,7 @@ def menu_diags(state, args): spacer='───────────────────────────────') if selection.isnumeric(): - # Toggle selection - index = int(selection) - 1 - main_options[index]['Enabled'] = not main_options[index]['Enabled'] - - # Handle presets - if index == 0: - # Full - if main_options[index]['Enabled']: - for opt in main_options[1:3]: - opt['Enabled'] = False - for opt in main_options[3:]: - opt['Enabled'] = True - else: - for opt in main_options[3:]: - opt['Enabled'] = False - elif index == 1: - # Disk - if main_options[index]['Enabled']: - main_options[0]['Enabled'] = False - for opt in main_options[2:4]: - opt['Enabled'] = False - for opt in main_options[4:]: - opt['Enabled'] = True - else: - for opt in main_options[4:]: - opt['Enabled'] = False - elif index == 2: - # Disk (Quick) - if main_options[index]['Enabled']: - for opt in main_options[:2] + main_options[3:]: - opt['Enabled'] = False - main_options[4]['Enabled'] = True - else: - main_options[4]['Enabled'] = False + update_main_options(state, selection, main_options) elif selection == 'A': run_audio_test() elif selection == 'K': @@ -391,7 +370,7 @@ def menu_diags(state, args): elif selection == 'M': secret_screensaver('matrix') elif selection == 'T': - # Tubes is close to pipes + # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': run_program(['systemctl', 'reboot']) @@ -400,21 +379,7 @@ def menu_diags(state, args): elif selection == 'Q': break elif selection == 'S': - # Run test(s) - clear_screen() - print('Tests:') - for opt in main_options[3:]: - _nvme_smart = opt['Base Name'] == 'NVMe / SMART' - # Update state - state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] - print(' {:<15} {}{}{} {}'.format( - opt['Base Name'], - COLORS['GREEN'] if opt['Enabled'] else COLORS['RED'], - 'Enabled' if opt['Enabled'] else 'Disabled', - COLORS['CLEAR'], - quick_label if state.quick_mode and _nvme_smart else '')) - print('\nFake test(s) placeholder for now...') - pause('Press Enter to return to main menu... ') + run_hw_tests(state) def run_audio_test(): """Run audio test.""" @@ -424,6 +389,23 @@ def run_audio_test(): #run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_hw_tests(state): + """Run enabled hardware tests.""" + # Run test(s) + clear_screen() + print('Tests:') + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + print_standard(' {:<15} {}{}{} {}'.format( + k, + COLORS['GREEN'] if v['Enabled'] else COLORS['RED'], + 'Enabled' if v['Enabled'] else 'Disabled', + COLORS['CLEAR'], + QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) + print('\nFake test(s) placeholder for now...') + pause('Press Enter to return to main menu... ') + def run_keyboard_test(): """Run keyboard test.""" # TODO: Enable real test and remove placeholder @@ -450,6 +432,49 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def update_main_options(state, selection, main_options): + """Update menu and state based on selection.""" + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + + # Handle presets + if index == 0: + # Full + if main_options[index]['Enabled']: + for opt in main_options[1:3]: + opt['Enabled'] = False + for opt in main_options[3:]: + opt['Enabled'] = True + else: + for opt in main_options[3:]: + opt['Enabled'] = False + elif index == 1: + # Disk + if main_options[index]['Enabled']: + main_options[0]['Enabled'] = False + for opt in main_options[2:4]: + opt['Enabled'] = False + for opt in main_options[4:]: + opt['Enabled'] = True + else: + for opt in main_options[4:]: + opt['Enabled'] = False + elif index == 2: + # Disk (Quick) + if main_options[index]['Enabled']: + for opt in main_options[:2] + main_options[3:]: + opt['Enabled'] = False + main_options[4]['Enabled'] = True + else: + main_options[4]['Enabled'] = False + + # Update state + for opt in main_options[3:]: + state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] + + # Done + return main_options + def update_io_progress(percent, rate, progress_file): """Update I/O progress file.""" bar_color = COLORS['CLEAR'] From 62c9d82fd2cba7166c7c572e8de9f64387c2a417 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 17:05:53 -0700 Subject: [PATCH 032/121] Adjusted placeholders --- .bin/Scripts/functions/hw_diags.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3de0806f..4593392f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -373,9 +373,15 @@ def menu_diags(state, args): # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': - run_program(['systemctl', 'reboot']) + print('(FAKE) reboot...') + sleep(1) + # TODO uncomment below + #run_program(['systemctl', 'reboot']) elif selection == 'P': - run_program(['systemctl', 'poweroff']) + print('(FAKE) poweroff...') + sleep(1) + # TODO uncomment below + #run_program(['systemctl', 'poweroff']) elif selection == 'Q': break elif selection == 'S': @@ -383,10 +389,8 @@ def menu_diags(state, args): def run_audio_test(): """Run audio test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake audio placeholder for now...') - #run_program(['hw-diags-audio'], check=False, pipe=False) + run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') def run_hw_tests(state): @@ -403,23 +407,17 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) - print('\nFake test(s) placeholder for now...') - pause('Press Enter to return to main menu... ') + pause('\nPress Enter to return to main menu... ') def run_keyboard_test(): """Run keyboard test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake keyboard placeholder for now...') - #run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') + run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) def run_network_test(): """Run network test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake network placeholder for now...') - #run_program(['hw-diags-network'], check=False, pipe=False) + run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') def secret_screensaver(screensaver=None): From 1489ad4237e3bc20d22fc0f89e7d0bdc0c378790 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 18:43:50 -0700 Subject: [PATCH 033/121] Added safety check for devices --- .bin/Scripts/functions/hw_diags.py | 61 ++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4593392f..67195d79 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -176,22 +176,53 @@ class State(): for dev in json_data['blockdevices']: skip_dev = False dev_obj = DevObj(dev['name']) - + # Skip loopback devices if dev_obj.lsblk['tran'] == 'NONE': skip_dev = True - + # Skip WK devices wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) for label in dev_obj.labels: if re.search(wk_label_regex, label, re.IGNORECASE): skip_dev = True - + # Add device if not skip_dev: self.devs.append(DevObj(dev['name'])) # Functions +def check_dev_attributes(dev): + """Check if device should be tested and allow overrides.""" + needs_override = False + print_standard(' {size:>6} ({tran}) {model} {serial}'.format( + **dev.lsblk)) + + # General checks + if not dev.nvme_attributes and not dev.smart_attributes: + needs_override = True + print_warning( + ' WARNING: No NVMe or SMART attributes available for: {}'.format( + dev.path)) + + # NVMe checks + # TODO check all tracked attributes and set dev.failing if needed + + # SMART checks + # TODO check all tracked attributes and set dev.failing if needed + + # Ask for override if necessary + if needs_override: + if ask(' Run tests on this device anyway?'): + # TODO Set override for this dev + pass + else: + for v in dev.tests.values(): + v['Enabled'] = False + v['Result'] = 'Skipped' + v['Status'] = 'Skipped' + print_standard('') + def generate_horizontal_graph(rates, oneline=False): """Generate two-line horizontal graph from rates, returns str.""" line_1 = '' @@ -297,7 +328,7 @@ def menu_diags(state, args): {'Letter': 'Q', 'Name': 'Quit'}, ] secret_actions = ['M', 'T'] - + # Set initial selections update_main_options(state, '1', main_options) @@ -397,7 +428,7 @@ def run_hw_tests(state): """Run enabled hardware tests.""" # Run test(s) clear_screen() - print('Tests:') + print_info('Selected Tests:') for k, v in sorted( state.tests.items(), key=lambda kv: kv[1]['Order']): @@ -407,7 +438,21 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) - pause('\nPress Enter to return to main menu... ') + print_standard('') + + # Check devices if necessary + if (state.tests['badblocks']['Enabled'] + or state.tests['I/O Benchmark']['Enabled']): + print_info('Selected Disks:') + for dev in state.devs: + check_dev_attributes(dev) + print_standard('') + + # Run tests + # TODO + + # Done + pause('Press Enter to return to main menu... ') def run_keyboard_test(): """Run keyboard test.""" @@ -465,11 +510,11 @@ def update_main_options(state, selection, main_options): main_options[4]['Enabled'] = True else: main_options[4]['Enabled'] = False - + # Update state for opt in main_options[3:]: state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] - + # Done return main_options From 597a23608951f38dfc3b987eed3ebff053081c98 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 18:44:52 -0700 Subject: [PATCH 034/121] Don't clear screen twice at startup * Combined init_global_vars and add_devs output --- .bin/Scripts/functions/hw_diags.py | 5 ++++- .bin/Scripts/hw-diags-menu | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 67195d79..6c43fe28 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -166,7 +166,10 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - self.add_devs() + try_and_print( + message='Scanning devices...', + function=self.add_devs, + cs='Done') def add_devs(self): """Add all block devices listed by lsblk.""" diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index c67cc5f4..e60f8fc4 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -13,9 +13,6 @@ init_global_vars() if __name__ == '__main__': try: - # Prep - clear_screen() - # Show menu state = State() menu_diags(state, sys.argv) From 8fb1620c940ac2d337cb7cfb36998b3d1fac7c18 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 19:23:35 -0700 Subject: [PATCH 035/121] Added placeholder functions for HW tests --- .bin/Scripts/functions/hw_diags.py | 45 ++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6c43fe28..15bea407 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -302,7 +302,7 @@ def get_status_color(s): color = COLORS['CLEAR'] if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: color = COLORS['RED'] - elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: + elif s in ['Aborted', 'N/A', 'Unknown', 'Working', 'Skipped']: color = COLORS['YELLOW'] elif s in ['CS']: color = COLORS['GREEN'] @@ -427,6 +427,10 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_badblocks_test(state): + """TODO""" + print_standard('TODO: run_badblocks_test()') + def run_hw_tests(state): """Run enabled hardware tests.""" # Run test(s) @@ -452,22 +456,55 @@ def run_hw_tests(state): print_standard('') # Run tests - # TODO + if state.tests['Prime95 & Temps']['Enabled']: + run_mprime_test(state) + if state.tests['NVMe / SMART']['Enabled']: + run_nvme_smart(state) + if state.tests['badblocks']['Enabled']: + run_badblocks_test(state) + if state.tests['I/O Benchmark']['Enabled']: + run_io_benchmark(state) # Done pause('Press Enter to return to main menu... ') +def run_io_benchmark(state): + """TODO""" + print_standard('TODO: run_io_benchmark()') + def run_keyboard_test(): """Run keyboard test.""" clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) +def run_mprime_test(state): + """TODO""" + print_standard('TODO: run_mprime_test()') + def run_network_test(): """Run network test.""" clear_screen() run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_nvme_smart(state): + """TODO""" + for dev in state.devs: + if dev.nvme_attributes: + run_nvme_tests(dev) + elif dev.smart_attributes: + run_smart_tests(dev) + else: + print_standard('TODO: run_nvme_smart({})'.format( + dev.path)) + print_warning( + " WARNING: Device {} doesn't support NVMe or SMART test".format( + dev.path)) + +def run_nvme_tests(dev): + """TODO""" + print_standard('TODO: run_nvme_test({})'.format(dev.path)) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': @@ -478,6 +515,10 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def run_smart_tests(dev): + """TODO""" + print_standard('TODO: run_smart_tests({})'.format(dev.path)) + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 4bb1402ac5a992c2e5f3f030a7a9131f743d00fb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 20:50:47 -0700 Subject: [PATCH 036/121] Added tmux functions * Going to try and replace the send-keys sections next --- .bin/Scripts/functions/hw_diags.py | 80 +++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 15bea407..a446036d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -62,7 +62,7 @@ IO_VARS = { KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) -SIDE_PATH_WIDTH = 21 +SIDE_PANE_WIDTH = 21 # Classes class DevObj(): @@ -156,6 +156,7 @@ class State(): def __init__(self): self.devs = [] self.finished = False + self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.started = False @@ -195,6 +196,31 @@ class State(): self.devs.append(DevObj(dev['name'])) # Functions +def build_outer_panes(state): + """Build top and side panes.""" + clear_screen() + + # Create panes + state.panes['Top'] = tmux_split_window( + behind=True, lines=2, vertical=True) + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top']) + state.panes['Progress'] = tmux_split_window(lines=SIDE_PANE_WIDTH) + + # Set text + tmux_update_pane_text( + state.panes['Top'], + text='{GREEN}Hardware Diagnostics{CLEAR}'.format( + **COLORS)) + tmux_update_pane_text( + state.panes['Started'], + text='{BLUE}Started{CLEAR}\n{text}'.format( + text=time.strftime("%Y-%m-%d %H:%M %Z"), + **COLORS)) + tmux_update_pane_text( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here'.format(**COLORS)) + def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" needs_override = False @@ -433,8 +459,10 @@ def run_badblocks_test(state): def run_hw_tests(state): """Run enabled hardware tests.""" + # Build Panes + build_outer_panes(state) + # Run test(s) - clear_screen() print_info('Selected Tests:') for k, v in sorted( state.tests.items(), @@ -468,6 +496,9 @@ def run_hw_tests(state): # Done pause('Press Enter to return to main menu... ') + # Cleanup + tmux_kill_pane(*state.panes.values()) + def run_io_benchmark(state): """TODO""" print_standard('TODO: run_io_benchmark()') @@ -519,6 +550,51 @@ def run_smart_tests(dev): """TODO""" print_standard('TODO: run_smart_tests({})'.format(dev.path)) +def tmux_kill_pane(*panes): + """Kill tmux pane by id.""" + cmd = ['tmux', 'kill-pane', '-t'] + for pane_id in panes: + print(pane_id) + run_program(cmd+[pane_id], check=False) + +def tmux_split_window( + lines=None, percent=None, + behind=False, vertical=False, + follow=False, target_pane=None): + """Run tmux split-window command and return pane_id as str.""" + # Bail early + if not lines and not percent: + raise Exception('Neither lines nor percent specified.') + + # Build cmd + cmd = ['tmux', 'split-window', '-PF', '#D'] + if behind: + cmd.append('-b') + if vertical: + cmd.append('-v') + else: + cmd.append('-h') + if not follow: + cmd.append('-d') + if lines is not None: + cmd.extend(['-l', str(lines)]) + elif percent is not None: + cmd.extend(['-p', str(percent)]) + if target_pane: + cmd.extend(['-t', str(target_pane)]) + + # Run and return pane_id + result = run_program(cmd) + return result.stdout.decode().strip() + +def tmux_update_pane_text(pane_id, text): + """Print text to tmux pane.""" + text = text.replace('\033', r'\e') + cmd = ['tmux', 'send-keys', '-t', pane_id] + run_program(cmd+['Enter']) + run_program(cmd+['clear; echo-and-hold "{}"'.format(text)]) + run_program(cmd+['Enter']) + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 43b9645c69fee66d5154013abe5d67b58bf54a77 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 23:39:15 -0700 Subject: [PATCH 037/121] Update tmux panes via respawn-pane Instead of send-keys * Avoids flooding zsh history * Less flickering --- .bin/Scripts/functions/hw_diags.py | 121 +++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 34 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a446036d..743db0a2 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -63,6 +63,7 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 21 +TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes class DevObj(): @@ -200,26 +201,24 @@ def build_outer_panes(state): """Build top and side panes.""" clear_screen() - # Create panes + # Top state.panes['Top'] = tmux_split_window( - behind=True, lines=2, vertical=True) - state.panes['Started'] = tmux_split_window( - lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top']) - state.panes['Progress'] = tmux_split_window(lines=SIDE_PANE_WIDTH) + behind=True, lines=2, vertical=True, + text='{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS)) - # Set text - tmux_update_pane_text( - state.panes['Top'], - text='{GREEN}Hardware Diagnostics{CLEAR}'.format( - **COLORS)) - tmux_update_pane_text( - state.panes['Started'], + # Started + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top'], text='{BLUE}Started{CLEAR}\n{text}'.format( text=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) - tmux_update_pane_text( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here'.format(**COLORS)) + + # Progress + state.panes['Progress'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" @@ -337,8 +336,7 @@ def get_status_color(s): def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] - title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( - **COLORS) + title = '{}\nMain Menu'.format(TOP_PANE_TEXT) # NOTE: Changing the order of main_options will break everything main_options = [ {'Base Name': 'Full Diagnostic', 'Enabled': False}, @@ -455,7 +453,16 @@ def run_audio_test(): def run_badblocks_test(state): """TODO""" + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'badblocks')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) print_standard('TODO: run_badblocks_test()') + sleep(3) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -501,7 +508,16 @@ def run_hw_tests(state): def run_io_benchmark(state): """TODO""" + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'I/O Benchmark')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) print_standard('TODO: run_io_benchmark()') + sleep(3) def run_keyboard_test(): """Run keyboard test.""" @@ -509,8 +525,21 @@ def run_keyboard_test(): run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) def run_mprime_test(state): - """TODO""" - print_standard('TODO: run_mprime_test()') + """Test CPU with Prime95 and track temps.""" + # Prep + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'Prime95 & Temps')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) + # Get idle temps + # Stress CPU + # Get max temp + # Get cooldown temp + sleep(3) def run_network_test(): """Run network test.""" @@ -521,21 +550,35 @@ def run_network_test(): def run_nvme_smart(state): """TODO""" for dev in state.devs: + tmux_update_pane( + state.panes['Top'], + text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( + t=TOP_PANE_TEXT, **dev.lsblk)) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) if dev.nvme_attributes: - run_nvme_tests(dev) + run_nvme_tests(state, dev) elif dev.smart_attributes: - run_smart_tests(dev) + run_smart_tests(state, dev) else: print_standard('TODO: run_nvme_smart({})'.format( dev.path)) print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( dev.path)) + sleep(3) -def run_nvme_tests(dev): +def run_nvme_tests(state, dev): """TODO""" print_standard('TODO: run_nvme_test({})'.format(dev.path)) +def run_smart_tests(state, dev): + """TODO""" + print_standard('TODO: run_smart_tests({})'.format(dev.path)) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': @@ -546,10 +589,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def run_smart_tests(dev): - """TODO""" - print_standard('TODO: run_smart_tests({})'.format(dev.path)) - def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] @@ -560,11 +599,14 @@ def tmux_kill_pane(*panes): def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, - follow=False, target_pane=None): + follow=False, target_pane=None, + command=None, text=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: raise Exception('Neither lines nor percent specified.') + if not command and not text: + raise Exception('Neither command nor text specified.') # Build cmd cmd = ['tmux', 'split-window', '-PF', '#D'] @@ -583,17 +625,28 @@ def tmux_split_window( if target_pane: cmd.extend(['-t', str(target_pane)]) + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold', text]) + # Run and return pane_id result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane_text(pane_id, text): - """Print text to tmux pane.""" - text = text.replace('\033', r'\e') - cmd = ['tmux', 'send-keys', '-t', pane_id] - run_program(cmd+['Enter']) - run_program(cmd+['clear; echo-and-hold "{}"'.format(text)]) - run_program(cmd+['Enter']) +def tmux_update_pane(pane_id, command=None, text=None): + """Respawn with either a new command or new text.""" + # Bail early + if not command and not text: + raise Exception('Neither command nor text specified.') + + cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold', text]) + + run_program(cmd) def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" From 2d69d93154f6fc2882b3cb6d85a6b8c55bed3a36 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:41:27 -0700 Subject: [PATCH 038/121] Added watch option for tmux_split_window() --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 743db0a2..440ec1c5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -204,7 +204,7 @@ def build_outer_panes(state): # Top state.panes['Top'] = tmux_split_window( behind=True, lines=2, vertical=True, - text='{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS)) + text=TOP_PANE_TEXT) # Started state.panes['Started'] = tmux_split_window( @@ -216,9 +216,7 @@ def build_outer_panes(state): # Progress state.panes['Progress'] = tmux_split_window( lines=SIDE_PANE_WIDTH, - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + watch=state.progress_out) def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" @@ -600,13 +598,13 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - command=None, text=None): + command=None, text=None, watch=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: raise Exception('Neither lines nor percent specified.') - if not command and not text: - raise Exception('Neither command nor text specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') # Build cmd cmd = ['tmux', 'split-window', '-PF', '#D'] @@ -628,7 +626,12 @@ def tmux_split_window( if command: cmd.extend(command) elif text: - cmd.extend(['echo-and-hold', text]) + cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) # Run and return pane_id result = run_program(cmd) @@ -644,7 +647,7 @@ def tmux_update_pane(pane_id, command=None, text=None): if command: cmd.extend(command) elif text: - cmd.extend(['echo-and-hold', text]) + cmd.extend(['echo-and-hold "{}"'.format(text)]) run_program(cmd) From d025b8dc9e91555b9cf123580a49ab8f09eeb3ea Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:49:25 -0700 Subject: [PATCH 039/121] Adjusted how devices are added to the state obj * The change allows for devices to be (dis)connected while the script is running * Devices are scanned and added during run_hw_diags() * Fixes bug that prevented any devices from being added as well --- .bin/Scripts/functions/hw_diags.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 440ec1c5..9e720d92 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -78,6 +78,7 @@ class DevObj(): self.path = dev_path self.smart_attributes = {} self.smartctl = {} + self.state = state self.tests = { 'NVMe / SMART': {'Result': None, 'Status': None}, 'badblocks': {'Result': None, 'Status': None}, @@ -158,7 +159,8 @@ class State(): self.devs = [] self.finished = False self.panes = {} - self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + # TODO Switch to LogDir + self.progress_out = '{}/progress.out'.format(global_vars['TmpDir']) self.quick_mode = False self.started = False self.tests = { @@ -168,19 +170,20 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - try_and_print( - message='Scanning devices...', - function=self.add_devs, - cs='Done') - def add_devs(self): - """Add all block devices listed by lsblk.""" + def init(self): + """Scan for block devices and reset all tests.""" + self.devs = [] + for k in ['Result', 'Started', 'Status']: + self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) for dev in json_data['blockdevices']: skip_dev = False - dev_obj = DevObj(dev['name']) + dev_obj = DevObj(self, dev['name']) # Skip loopback devices if dev_obj.lsblk['tran'] == 'NONE': @@ -194,7 +197,7 @@ class State(): # Add device if not skip_dev: - self.devs.append(DevObj(dev['name'])) + self.devs.append(dev_obj) # Functions def build_outer_panes(state): @@ -464,6 +467,9 @@ def run_badblocks_test(state): def run_hw_tests(state): """Run enabled hardware tests.""" + print_standard('Scanning devices...') + state.init() + # Build Panes build_outer_panes(state) From 7c163a8110b8474a8d75dee1e091b7b109d43eea Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:52:24 -0700 Subject: [PATCH 040/121] Added update progress sections --- .bin/Scripts/functions/hw_diags.py | 158 +++++++++++++++++++++++------ 1 file changed, 126 insertions(+), 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9e720d92..ac128ea1 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -62,13 +62,13 @@ IO_VARS = { KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) -SIDE_PANE_WIDTH = 21 +SIDE_PANE_WIDTH = 20 TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes class DevObj(): """Device object for tracking device specific data.""" - def __init__(self, dev_path): + def __init__(self, state, dev_path): self.failing = False self.labels = [] self.lsblk = {} @@ -80,13 +80,17 @@ class DevObj(): self.smartctl = {} self.state = state self.tests = { - 'NVMe / SMART': {'Result': None, 'Status': None}, - 'badblocks': {'Result': None, 'Status': None}, + 'NVMe / SMART': { + 'Result': '', 'Started': False, 'Status': '', 'Order': 1}, + 'badblocks': { + 'Result': '', 'Started': False, 'Status': '', 'Order': 2}, 'I/O Benchmark': { - 'Result': None, - 'Status': None, + 'Result': '', + 'Started': False, + 'Status': '', 'Read Rates': [], - 'Graph Data': []}, + 'Graph Data': [], + 'Order': 3}, } self.get_details() self.get_smart_details() @@ -153,6 +157,21 @@ class DevObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + def update_progress(self): + """Update status strings.""" + for k, v in self.tests.items(): + if self.state.tests[k]['Enabled']: + _status = '' + if not v['Status']: + _status = 'Pending' + if v['Started']: + if v['Result']: + _status = v['Result'] + else: + _status = 'Working' + if _status: + v['Status'] = build_status_string(self.name, _status) + class State(): """Object to track device objects and overall state.""" def __init__(self): @@ -165,7 +184,7 @@ class State(): self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': None, 'Status': None}, + 'Result': '', 'Started': False, 'Status': ''}, 'NVMe / SMART': {'Enabled': False, 'Order': 2}, 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, @@ -199,6 +218,27 @@ class State(): if not skip_dev: self.devs.append(dev_obj) + def update_progress(self): + """Update status strings.""" + # Prime95 + p = self.tests['Prime95 & Temps'] + if p['Enabled']: + _status = '' + if not p['Status']: + _status = 'Pending' + if p['Started']: + if p['Result']: + _status = p['Result'] + else: + _status = 'Working' + if _status: + p['Status'] = build_status_string( + 'Prime95', _status, info_label=True) + + # Disks + for dev in self.devs: + dev.update_progress() + # Functions def build_outer_panes(state): """Build top and side panes.""" @@ -221,6 +261,24 @@ def build_outer_panes(state): lines=SIDE_PANE_WIDTH, watch=state.progress_out) +def build_status_string(label, status, info_label=False): + """Build status string with appropriate colors.""" + status_color = COLORS['CLEAR'] + if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + status_color = COLORS['RED'] + elif status in ['Aborted', 'Unknown', 'Working', 'Skipped']: + status_color = COLORS['YELLOW'] + elif status in ['CS']: + status_color = COLORS['GREEN'] + + return '{l_c}{l}{CLEAR}{s_c}{s:>{s_w}}{CLEAR}'.format( + l_c=COLORS['BLUE'] if info_label else '', + l=label, + s_c=status_color, + s=status, + s_w=SIDE_PANE_WIDTH-len(label), + **COLORS) + def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" needs_override = False @@ -247,8 +305,9 @@ def check_dev_attributes(dev): pass else: for v in dev.tests.values(): - v['Enabled'] = False + # Started is set to True to fix the status string v['Result'] = 'Skipped' + v['Started'] = True v['Status'] = 'Skipped' print_standard('') @@ -457,13 +516,13 @@ def run_badblocks_test(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) print_standard('TODO: run_badblocks_test()') - sleep(3) + for dev in state.devs: + dev.tests['badblocks']['Started'] = True + update_progress_pane(state) + sleep(3) + dev.tests['badblocks']['Result'] = 'OVERRIDE' + update_progress_pane(state) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -471,6 +530,7 @@ def run_hw_tests(state): state.init() # Build Panes + update_progress_pane(state) build_outer_panes(state) # Run test(s) @@ -515,13 +575,13 @@ def run_io_benchmark(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) print_standard('TODO: run_io_benchmark()') - sleep(3) + for dev in state.devs: + dev.tests['I/O Benchmark']['Started'] = True + update_progress_pane(state) + sleep(3) + dev.tests['I/O Benchmark']['Result'] = 'Unknown' + update_progress_pane(state) def run_keyboard_test(): """Run keyboard test.""" @@ -534,16 +594,18 @@ def run_mprime_test(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + state.tests['Prime95 & Temps']['Started'] = True + update_progress_pane(state) + # Get idle temps # Stress CPU # Get max temp # Get cooldown temp + + # Done sleep(3) + state.tests['Prime95 & Temps']['Result'] = 'Unknown' + update_progress_pane(state) def run_network_test(): """Run network test.""" @@ -558,11 +620,8 @@ def run_nvme_smart(state): state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **dev.lsblk)) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + dev.tests['NVMe / SMART']['Started'] = True + update_progress_pane(state) if dev.nvme_attributes: run_nvme_tests(state, dev) elif dev.smart_attributes: @@ -573,15 +632,24 @@ def run_nvme_smart(state): print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( dev.path)) - sleep(3) + dev.tests['NVMe / SMART']['Status'] = 'N/A' + dev.tests['NVMe / SMART']['Result'] = 'N/A' + update_progress_pane(state) + sleep(3) def run_nvme_tests(state, dev): """TODO""" print_standard('TODO: run_nvme_test({})'.format(dev.path)) + sleep(3) + dev.tests['NVMe / SMART']['Result'] = 'CS' + update_progress_pane(state) def run_smart_tests(state, dev): """TODO""" print_standard('TODO: run_smart_tests({})'.format(dev.path)) + sleep(3) + dev.tests['NVMe / SMART']['Result'] = 'CS' + update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" @@ -724,6 +792,32 @@ def update_io_progress(percent, rate, progress_file): with open(progress_file, 'a') as f: f.write(line) +def update_progress_pane(state): + """Update progress file for side pane.""" + output = [] + state.update_progress() + + # Prime95 + output.append(state.tests['Prime95 & Temps']['Status']) + output.append(' ') + + # Disks + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + if 'Prime95' not in k and v['Enabled']: + output.append('{BLUE}{test_name}{CLEAR}'.format( + test_name=k, **COLORS)) + for dev in state.devs: + output.append(dev.tests[k]['Status']) + output.append(' ') + + # Add line-endings + output = ['{}\n'.format(line) for line in output] + + with open(state.progress_out, 'w') as f: + f.writelines(output) + if __name__ == '__main__': print("This file is not meant to be called directly.") From 372f80bf38d56dcb58a4ac76cf7b04555f3f66f5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 04:08:59 -0700 Subject: [PATCH 041/121] Skip optical drives --- .bin/Scripts/functions/hw_diags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index ac128ea1..d122f6c6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -204,8 +204,8 @@ class State(): skip_dev = False dev_obj = DevObj(self, dev['name']) - # Skip loopback devices - if dev_obj.lsblk['tran'] == 'NONE': + # Skip loopback and optical devices + if dev_obj.lsblk['type'] in ['loop', 'rom']: skip_dev = True # Skip WK devices From 163f64dda7921847a5601aaa3d2b0d7825b5a5b0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 04:10:20 -0700 Subject: [PATCH 042/121] Reduced timeout for major exceptions --- .bin/Scripts/functions/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index b5896966..cc3d98b8 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -305,13 +305,13 @@ def major_exception(): except GenericAbort: # User declined upload print_warning('Upload: Aborted') - sleep(30) + sleep(10) except GenericError: # No log file or uploading disabled - sleep(30) + sleep(10) except: print_error('Upload: NS') - sleep(30) + sleep(10) else: print_success('Upload: CS') pause('Press Enter to exit...') From 5dd8fa84164876952a85b4cb97b94eb0a691ea4c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 17:48:30 -0700 Subject: [PATCH 043/121] Get CPU details from lscpu --- .bin/Scripts/functions/hw_diags.py | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d122f6c6..892dff90 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -175,6 +175,7 @@ class DevObj(): class State(): """Object to track device objects and overall state.""" def __init__(self): + self.lscpu = {} self.devs = [] self.finished = False self.panes = {} @@ -189,6 +190,28 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } + self.get_cpu_details() + + def get_cpu_details(self): + """Get CPU details from lscpu.""" + cmd = ['lscpu', '--json'] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + except Exception as err: + # Ignore and leave self.cpu empty + print_error(err) + pause() + return + for line in json_data.get('lscpu', []): + _field = line.get('field', None).replace(':', '') + _data = line.get('data', None) + if not _field and not _data: + # Skip + print_warning(_field, _data) + pause() + continue + self.lscpu[_field] = _data def init(self): """Scan for block devices and reset all tests.""" @@ -591,9 +614,11 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" # Prep - tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'Prime95 & Temps')) + _title = '{}\n{}{}{}'.format( + TOP_PANE_TEXT, 'Prime95 & Temps', + ': ' if 'Model name' in state.lscpu else '', + state.lscpu.get('Model name', '')) + tmux_update_pane(state.panes['Top'], text=_title) state.tests['Prime95 & Temps']['Started'] = True update_progress_pane(state) From cb67f7e3c3ce26c7475b739ae937849b970d198a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 19:59:41 -0700 Subject: [PATCH 044/121] Added new sensors.py and dropped borrowed sensors --- .bin/Scripts/borrowed/sensors-README.md | 35 ---- .bin/Scripts/borrowed/sensors.py | 236 ------------------------ .bin/Scripts/functions/common.py | 11 +- .bin/Scripts/functions/sensors.py | 111 +++++++++++ 4 files changed, 117 insertions(+), 276 deletions(-) delete mode 100644 .bin/Scripts/borrowed/sensors-README.md delete mode 100644 .bin/Scripts/borrowed/sensors.py create mode 100644 .bin/Scripts/functions/sensors.py diff --git a/.bin/Scripts/borrowed/sensors-README.md b/.bin/Scripts/borrowed/sensors-README.md deleted file mode 100644 index 11858382..00000000 --- a/.bin/Scripts/borrowed/sensors-README.md +++ /dev/null @@ -1,35 +0,0 @@ -sensors.py -========== -python bindings using ctypes for libsensors3 of the [lm-sensors project](https://github.com/groeck/lm-sensors). The code was written against libsensors 3.3.4. - -For documentation of the low level API see [sensors.h](https://github.com/groeck/lm-sensors/blob/master/lib/sensors.h). For an example of the high level API see [example.py](example.py). - -For a GUI application that displays the sensor readings and is based on this library, take a look at [sensors-unity](https://launchpad.net/sensors-unity). - -Features --------- -* Full access to low level libsensors3 API -* High level iterator API -* unicode handling -* Python2 and Python3 compatible - -Licensing ---------- -LGPLv2 (same as libsensors3) - -Usage Notes ------------ -As Python does not support call by reference for primitive types some of the libsensors API had to be adapted: - -```python -# nr is changed by refrence in the C API -chip_name, nr = sensors.get_detected_chips(None, nr) - -# returns the value. throws on error -val = sensors.get_value(chip, subfeature_nr) -``` - -Missing Features (pull requests are welcome): -* `sensors_subfeature_type` enum -* `sensors_get_subfeature` -* Error handlers diff --git a/.bin/Scripts/borrowed/sensors.py b/.bin/Scripts/borrowed/sensors.py deleted file mode 100644 index 39b00a4f..00000000 --- a/.bin/Scripts/borrowed/sensors.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -@package sensors.py -Python Bindings for libsensors3 - -use the documentation of libsensors for the low level API. -see example.py for high level API usage. - -@author: Pavel Rojtberg (http://www.rojtberg.net) -@see: https://github.com/paroj/sensors.py -@copyright: LGPLv2 (same as libsensors) -""" - -from ctypes import * -import ctypes.util - -_libc = cdll.LoadLibrary(ctypes.util.find_library("c")) -# see https://github.com/paroj/sensors.py/issues/1 -_libc.free.argtypes = [c_void_p] - -_hdl = cdll.LoadLibrary(ctypes.util.find_library("sensors")) - -version = c_char_p.in_dll(_hdl, "libsensors_version").value.decode("ascii") - -class bus_id(Structure): - _fields_ = [("type", c_short), - ("nr", c_short)] - -class chip_name(Structure): - _fields_ = [("prefix", c_char_p), - ("bus", bus_id), - ("addr", c_int), - ("path", c_char_p)] - -class feature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int)] - - # sensors_feature_type - IN = 0x00 - FAN = 0x01 - TEMP = 0x02 - POWER = 0x03 - ENERGY = 0x04 - CURR = 0x05 - HUMIDITY = 0x06 - MAX_MAIN = 0x7 - VID = 0x10 - INTRUSION = 0x11 - MAX_OTHER = 0x12 - BEEP_ENABLE = 0x18 - -class subfeature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int), - ("mapping", c_int), - ("flags", c_uint)] - -_hdl.sensors_get_detected_chips.restype = POINTER(chip_name) -_hdl.sensors_get_features.restype = POINTER(feature) -_hdl.sensors_get_all_subfeatures.restype = POINTER(subfeature) -_hdl.sensors_get_label.restype = c_void_p # return pointer instead of str so we can free it -_hdl.sensors_get_adapter_name.restype = c_char_p # docs do not say whether to free this or not -_hdl.sensors_strerror.restype = c_char_p - -### RAW API ### -MODE_R = 1 -MODE_W = 2 -COMPUTE_MAPPING = 4 - -def init(cfg_file = None): - file = _libc.fopen(cfg_file.encode("utf-8"), "r") if cfg_file is not None else None - - if _hdl.sensors_init(file) != 0: - raise Exception("sensors_init failed") - - if file is not None: - _libc.fclose(file) - -def cleanup(): - _hdl.sensors_cleanup() - -def parse_chip_name(orig_name): - ret = chip_name() - err= _hdl.sensors_parse_chip_name(orig_name.encode("utf-8"), byref(ret)) - - if err < 0: - raise Exception(strerror(err)) - - return ret - -def strerror(errnum): - return _hdl.sensors_strerror(errnum).decode("utf-8") - -def free_chip_name(chip): - _hdl.sensors_free_chip_name(byref(chip)) - -def get_detected_chips(match, nr): - """ - @return: (chip, next nr to query) - """ - _nr = c_int(nr) - - if match is not None: - match = byref(match) - - chip = _hdl.sensors_get_detected_chips(match, byref(_nr)) - chip = chip.contents if bool(chip) else None - return chip, _nr.value - -def chip_snprintf_name(chip, buffer_size=200): - """ - @param buffer_size defaults to the size used in the sensors utility - """ - ret = create_string_buffer(buffer_size) - err = _hdl.sensors_snprintf_chip_name(ret, buffer_size, byref(chip)) - - if err < 0: - raise Exception(strerror(err)) - - return ret.value.decode("utf-8") - -def do_chip_sets(chip): - """ - @attention this function was not tested - """ - err = _hdl.sensors_do_chip_sets(byref(chip)) - if err < 0: - raise Exception(strerror(err)) - -def get_adapter_name(bus): - return _hdl.sensors_get_adapter_name(byref(bus)).decode("utf-8") - -def get_features(chip, nr): - """ - @return: (feature, next nr to query) - """ - _nr = c_int(nr) - feature = _hdl.sensors_get_features(byref(chip), byref(_nr)) - feature = feature.contents if bool(feature) else None - return feature, _nr.value - -def get_label(chip, feature): - ptr = _hdl.sensors_get_label(byref(chip), byref(feature)) - val = cast(ptr, c_char_p).value.decode("utf-8") - _libc.free(ptr) - return val - -def get_all_subfeatures(chip, feature, nr): - """ - @return: (subfeature, next nr to query) - """ - _nr = c_int(nr) - subfeature = _hdl.sensors_get_all_subfeatures(byref(chip), byref(feature), byref(_nr)) - subfeature = subfeature.contents if bool(subfeature) else None - return subfeature, _nr.value - -def get_value(chip, subfeature_nr): - val = c_double() - err = _hdl.sensors_get_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - return val.value - -def set_value(chip, subfeature_nr, value): - """ - @attention this function was not tested - """ - val = c_double(value) - err = _hdl.sensors_set_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - -### Convenience API ### -class ChipIterator: - def __init__(self, match = None): - self.match = parse_chip_name(match) if match is not None else None - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - chip, self.nr = get_detected_chips(self.match, self.nr) - - if chip is None: - raise StopIteration - - return chip - - def __del__(self): - if self.match is not None: - free_chip_name(self.match) - - def next(self): # python2 compability - return self.__next__() - -class FeatureIterator: - def __init__(self, chip): - self.chip = chip - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - feature, self.nr = get_features(self.chip, self.nr) - - if feature is None: - raise StopIteration - - return feature - - def next(self): # python2 compability - return self.__next__() - -class SubFeatureIterator: - def __init__(self, chip, feature): - self.chip = chip - self.feature = feature - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - subfeature, self.nr = get_all_subfeatures(self.chip, self.feature, self.nr) - - if subfeature is None: - raise StopIteration - - return subfeature - - def next(self): # python2 compability - return self.__next__() diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index cc3d98b8..2bf52a85 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -25,12 +25,13 @@ global_vars = {} # STATIC VARIABLES COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'GREEN': '\033[32m', + 'CLEAR': '\033[0m', + 'RED': '\033[31m', + 'GREEN': '\033[32m', 'YELLOW': '\033[33m', - 'BLUE': '\033[34m' -} + 'ORANGE': '\033[31;1m', + 'BLUE': '\033[34m' + } try: HKU = winreg.HKEY_USERS HKCR = winreg.HKEY_CLASSES_ROOT diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py new file mode 100644 index 00000000..0cb65acd --- /dev/null +++ b/.bin/Scripts/functions/sensors.py @@ -0,0 +1,111 @@ +# Wizard Kit: Functions - Sensors + +import itertools +import json +import re + +from functions.common import * + +# STATIC VARIABLES +TEMP_LIMITS = { + 'GREEN': 60, + 'YELLOW': 70, + 'ORANGE': 80, + 'RED': 90, + } + +# REGEX +REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') + +def fix_sensor_str(s): + """Cleanup string and return str.""" + s = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', s, re.IGNORECASE) + s = s.title() + s = s.replace('Coretemp', 'CoreTemp') + s = s.replace('Acpi', 'ACPI') + s = s.replace('Isa ', 'ISA ') + s = s.replace('Id ', 'ID ') + s = re.sub(r'(\D+)(\d+)', r'\1 \2', s, re.IGNORECASE) + s = s.replace(' ', ' ') + return s + +def get_color_temp(temp): + """Get colored temp string, returns str.""" + try: + temp = float(temp) + except ValueError: + return '{YELLOW}{temp}{CLEAR}'.format(temp=temp, **COLORS) + if temp > TEMP_LIMITS['RED']: + color = COLORS['RED'] + elif temp > TEMP_LIMITS['ORANGE']: + color = COLORS['ORANGE'] + elif temp > TEMP_LIMITS['YELLOW']: + color = COLORS['YELLOW'] + elif temp > TEMP_LIMITS['GREEN']: + color = COLORS['GREEN'] + elif temp > 0: + color = COLORS['BLUE'] + else: + color = COLORS['CLEAR'] + return '{color}{prefix}{temp:2.0f}°C{CLEAR}'.format( + color = color, + prefix = '-' if temp < 0 else '', + temp = temp, + **COLORS) + +def get_raw_sensor_data(): + """Read sensor data and return dict.""" + cmd = ['sensors', '-j'] + result = run_program(cmd) + return json.loads(result.stdout.decode()) + +def get_sensor_data(): + """Parse raw sensor data and return new dict.""" + json_data = get_raw_sensor_data() + sensor_data = {'CoreTemps': {}, 'Other': {}} + for _adapter, _sources in json_data.items(): + if 'coretemp' in _adapter: + _section = 'CoreTemps' + else: + _section = 'Other' + sensor_data[_section][_adapter] = {} + _sources.pop('Adapter', None) + + # Find current temp and add to dict + ## current temp is labeled xxxx_input + for _source, _labels in _sources.items(): + for _label, _temp in _labels.items(): + if 'input' in _label: + sensor_data[_section][_adapter][_source] = { + 'Temps': [_temp], + 'Label': _label, + } + + # Remove empty sections + for k, v in sensor_data.items(): + v = {k2: v2 for k2, v2 in v.items() if v2} + + # Done + return sensor_data + +def update_sensor_data(sensor_data): + """Read sensors and update existing sensor_data, returns dict.""" + json_data = get_raw_sensor_data() + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _label = _ddata['Label'] + _temp = json_data[_adapter][_source][_label] + _data['Temps'].append(_temp) + return sensor_data + +def join_columns(column1, column2, width=55): + return '{:<{}}{}'.format( + column1, + 55+len(column1)-len(REGEX_COLORS.sub('', column1)), + column2) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 7140f38ba4fefdfb571405b97a80e2eb7ee6f783 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:11:10 -0700 Subject: [PATCH 045/121] Added average, clear, and max temps sections --- .bin/Scripts/functions/sensors.py | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 0cb65acd..10a2e664 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -17,6 +17,14 @@ TEMP_LIMITS = { # REGEX REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') +def clear_temps(sensor_data): + """Clear saved temps but keep structure, returns dict.""" + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data['Temps'] = [] + return sensor_data + def fix_sensor_str(s): """Cleanup string and return str.""" s = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', s, re.IGNORECASE) @@ -88,13 +96,41 @@ def get_sensor_data(): # Done return sensor_data +def save_max_temp(sensor_data): + """Record max temps seen this session, returns dict.""" + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data['Max'] = max(_data['Temps']) + + # Done + return sensor_data + +def save_average_temp(sensor_data, save_label, seconds=10): + """Calculate average temps and record under save_label, returns dict.""" + clear_temps(sensor_data) + + # Get temps + for i in range(seconds): + sensor_data = update_sensor_data(sensor_data) + sleep(1) + + # Calculate averages + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data[save_label] = sum(_data['Temps']) / len(_data['Temps']) + + # Done + return sensor_data + def update_sensor_data(sensor_data): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() for _section, _adapters in sensor_data.items(): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): - _label = _ddata['Label'] + _label = _data['Label'] _temp = json_data[_adapter][_source][_label] _data['Temps'].append(_temp) return sensor_data From 2eccc236a960235df41ec864116affae0efd281f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:40:25 -0700 Subject: [PATCH 046/121] Added generate_report() * Also merged save_max_temp() with update_sensor_data() * Max doesn't need resetting so just calc max everytime --- .bin/Scripts/functions/sensors.py | 59 +++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 10a2e664..30df47de 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -37,8 +37,30 @@ def fix_sensor_str(s): s = s.replace(' ', ' ') return s -def get_color_temp(temp): - """Get colored temp string, returns str.""" +def generate_report(sensor_data, *temp_labels, colors=True): + """Build report based on temp_labels, returns list if str.""" + report = [] + for _section, _adapters in sorted(sensor_data.items()): + # CoreTemps then Other temps + for _adapter, _sources in sorted(_adapters.items()): + # Adapter + report.append(fix_sensor_str(_adapter)) + for _source, _data in sorted(_sources.items()): + # Source + _line = '{:18} '.format(fix_sensor_str(_source)) + _temps = [] + for _label in temp_labels: + _temps.append('{}{}{}'.format( + _label.lower() if _label != 'Current' else '', + ': ' if _label != 'Current' else '', + get_temp_str(_data[_label], colors=colors))) + _line += ', '.join(_temps) + report.append(_line) + report.append(' ') + return sensor_data + +def get_colored_temp_str(temp): + """Get colored string based on temp, returns str.""" try: temp = float(temp) except ValueError: @@ -85,8 +107,10 @@ def get_sensor_data(): for _label, _temp in _labels.items(): if 'input' in _label: sensor_data[_section][_adapter][_source] = { - 'Temps': [_temp], + 'Current': _temp, 'Label': _label, + 'Max': _temp, + 'Temps': [_temp], } # Remove empty sections @@ -96,18 +120,21 @@ def get_sensor_data(): # Done return sensor_data -def save_max_temp(sensor_data): - """Record max temps seen this session, returns dict.""" - for _section, _adapters in sensor_data.items(): - for _adapter, _sources in _adapters.items(): - for _source, _data in _sources.items(): - _data['Max'] = max(_data['Temps']) +def get_temp_str(temp, colors=True): + """Get temp string, returns str.""" + if colors: + return get_colored_temp_str(temp) + try: + temp = float(temp) + except ValueError: + return '{}°C'.format(temp) + else: + return '{}{:2.0f}°C'.format( + '-' if temp < 0 else '', + temp) - # Done - return sensor_data - -def save_average_temp(sensor_data, save_label, seconds=10): - """Calculate average temps and record under save_label, returns dict.""" +def save_average_temp(sensor_data, temp_label, seconds=10): + """Calculate average temps and record under temp_label, returns dict.""" clear_temps(sensor_data) # Get temps @@ -119,7 +146,7 @@ def save_average_temp(sensor_data, save_label, seconds=10): for _section, _adapters in sensor_data.items(): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): - _data[save_label] = sum(_data['Temps']) / len(_data['Temps']) + _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) # Done return sensor_data @@ -132,6 +159,8 @@ def update_sensor_data(sensor_data): for _source, _data in _sources.items(): _label = _data['Label'] _temp = json_data[_adapter][_source][_label] + _data['Current'] = _temp + _data['Max'] = max(_temp, _data['Max']) _data['Temps'].append(_temp) return sensor_data From 328d6eb29484711774a5386c79a9415d25d2c3a4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:47:40 -0700 Subject: [PATCH 047/121] Modify sensor_data in place --- .bin/Scripts/functions/sensors.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 30df47de..bead3bfa 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -23,7 +23,6 @@ def clear_temps(sensor_data): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): _data['Temps'] = [] - return sensor_data def fix_sensor_str(s): """Cleanup string and return str.""" @@ -57,7 +56,6 @@ def generate_report(sensor_data, *temp_labels, colors=True): _line += ', '.join(_temps) report.append(_line) report.append(' ') - return sensor_data def get_colored_temp_str(temp): """Get colored string based on temp, returns str.""" @@ -139,7 +137,7 @@ def save_average_temp(sensor_data, temp_label, seconds=10): # Get temps for i in range(seconds): - sensor_data = update_sensor_data(sensor_data) + update_sensor_data(sensor_data) sleep(1) # Calculate averages @@ -148,9 +146,6 @@ def save_average_temp(sensor_data, temp_label, seconds=10): for _source, _data in _sources.items(): _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) - # Done - return sensor_data - def update_sensor_data(sensor_data): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() @@ -162,7 +157,6 @@ def update_sensor_data(sensor_data): _data['Current'] = _temp _data['Max'] = max(_temp, _data['Max']) _data['Temps'].append(_temp) - return sensor_data def join_columns(column1, column2, width=55): return '{:<{}}{}'.format( From 95b0d1e3f4823d31bcbf5bc0c2ee20753b20379e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 21:54:41 -0700 Subject: [PATCH 048/121] Wrap reports if necessary --- .bin/Scripts/functions/sensors.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index bead3bfa..5e33e1cb 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -3,6 +3,7 @@ import itertools import json import re +import shutil from functions.common import * @@ -37,7 +38,7 @@ def fix_sensor_str(s): return s def generate_report(sensor_data, *temp_labels, colors=True): - """Build report based on temp_labels, returns list if str.""" + """Generate report based on temp_labels, returns list if str.""" report = [] for _section, _adapters in sorted(sensor_data.items()): # CoreTemps then Other temps @@ -48,6 +49,7 @@ def generate_report(sensor_data, *temp_labels, colors=True): # Source _line = '{:18} '.format(fix_sensor_str(_source)) _temps = [] + # Temps (skip label for Current) for _label in temp_labels: _temps.append('{}{}{}'.format( _label.lower() if _label != 'Current' else '', @@ -57,6 +59,26 @@ def generate_report(sensor_data, *temp_labels, colors=True): report.append(_line) report.append(' ') + # Wrap lines if necessary + screen_size = shutil.get_terminal_size() + rows = screen_size.lines - 1 + if len(report) > rows and screen_size.columns > 55*2: + report = list(itertools.zip_longest( + report[:rows], report[rows:], fillvalue='')) + report = [join_columns(a, b) for a, b in report] + + # Handle empty reports (i.e. no sensors detected) + if not report: + report = [ + '{}WARNING: No sensors found{}'.format( + COLORS['YELLOW'] if colors else '', + COLORS['CLEAR'] if colors else ''), + ' ', + 'Please monitor temps manually'] + + # Done + return report + def get_colored_temp_str(temp): """Get colored string based on temp, returns str.""" try: From 0e5fab01047c69fd875d43e5ced54377f70a5ec5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 21:57:55 -0700 Subject: [PATCH 049/121] Handle missing labels in generate_report() --- .bin/Scripts/functions/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 5e33e1cb..b47f9afd 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -54,7 +54,7 @@ def generate_report(sensor_data, *temp_labels, colors=True): _temps.append('{}{}{}'.format( _label.lower() if _label != 'Current' else '', ': ' if _label != 'Current' else '', - get_temp_str(_data[_label], colors=colors))) + get_temp_str(_data.get(_label, '???'), colors=colors))) _line += ', '.join(_temps) report.append(_line) report.append(' ') @@ -147,7 +147,7 @@ def get_temp_str(temp, colors=True): try: temp = float(temp) except ValueError: - return '{}°C'.format(temp) + return '{}'.format(temp) else: return '{}{:2.0f}°C'.format( '-' if temp < 0 else '', From 46080b43634f0317b78581c28a86493afc8acf06 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 22:25:44 -0700 Subject: [PATCH 050/121] Moved tmux sections to separate file --- .bin/Scripts/functions/hw_diags.py | 66 +-------------------------- .bin/Scripts/functions/tmux.py | 72 ++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 65 deletions(-) create mode 100644 .bin/Scripts/functions/tmux.py diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 892dff90..cc9a2c40 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,7 +4,7 @@ import json import re import time -from functions.common import * +from functions.tmux import * # STATIC VARIABLES ATTRIBUTES = { @@ -686,70 +686,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def tmux_kill_pane(*panes): - """Kill tmux pane by id.""" - cmd = ['tmux', 'kill-pane', '-t'] - for pane_id in panes: - print(pane_id) - run_program(cmd+[pane_id], check=False) - -def tmux_split_window( - lines=None, percent=None, - behind=False, vertical=False, - follow=False, target_pane=None, - command=None, text=None, watch=None): - """Run tmux split-window command and return pane_id as str.""" - # Bail early - if not lines and not percent: - raise Exception('Neither lines nor percent specified.') - if not command and not text and not watch: - raise Exception('No command, text, or watch file specified.') - - # Build cmd - cmd = ['tmux', 'split-window', '-PF', '#D'] - if behind: - cmd.append('-b') - if vertical: - cmd.append('-v') - else: - cmd.append('-h') - if not follow: - cmd.append('-d') - if lines is not None: - cmd.extend(['-l', str(lines)]) - elif percent is not None: - cmd.extend(['-p', str(percent)]) - if target_pane: - cmd.extend(['-t', str(target_pane)]) - - if command: - cmd.extend(command) - elif text: - cmd.extend(['echo-and-hold "{}"'.format(text)]) - elif watch: - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) - - # Run and return pane_id - result = run_program(cmd) - return result.stdout.decode().strip() - -def tmux_update_pane(pane_id, command=None, text=None): - """Respawn with either a new command or new text.""" - # Bail early - if not command and not text: - raise Exception('Neither command nor text specified.') - - cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] - if command: - cmd.extend(command) - elif text: - cmd.extend(['echo-and-hold "{}"'.format(text)]) - - run_program(cmd) - def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py new file mode 100644 index 00000000..ba00220f --- /dev/null +++ b/.bin/Scripts/functions/tmux.py @@ -0,0 +1,72 @@ +# Wizard Kit: Functions - tmux + +from functions.common import * + +def tmux_kill_pane(*panes): + """Kill tmux pane by id.""" + cmd = ['tmux', 'kill-pane', '-t'] + for pane_id in panes: + print(pane_id) + run_program(cmd+[pane_id], check=False) + +def tmux_split_window( + lines=None, percent=None, + behind=False, vertical=False, + follow=False, target_pane=None, + command=None, text=None, watch=None): + """Run tmux split-window command and return pane_id as str.""" + # Bail early + if not lines and not percent: + raise Exception('Neither lines nor percent specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') + + # Build cmd + cmd = ['tmux', 'split-window', '-PF', '#D'] + if behind: + cmd.append('-b') + if vertical: + cmd.append('-v') + else: + cmd.append('-h') + if not follow: + cmd.append('-d') + if lines is not None: + cmd.extend(['-l', str(lines)]) + elif percent is not None: + cmd.extend(['-p', str(percent)]) + if target_pane: + cmd.extend(['-t', str(target_pane)]) + + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + + # Run and return pane_id + result = run_program(cmd) + return result.stdout.decode().strip() + +def tmux_update_pane(pane_id, command=None, text=None): + """Respawn with either a new command or new text.""" + # Bail early + if not command and not text: + raise Exception('Neither command nor text specified.') + + cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold "{}"'.format(text)]) + + run_program(cmd) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 5405b97eb1cb6b6747ea44e9f64a848e5977f175 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:09:42 -0700 Subject: [PATCH 051/121] Standalone sensor monitor working again --- .bin/Scripts/functions/sensors.py | 14 ++- .bin/Scripts/functions/tmux.py | 7 ++ .bin/Scripts/hw-sensors | 168 +----------------------------- .bin/Scripts/hw-sensors-monitor | 31 ++++++ 4 files changed, 56 insertions(+), 164 deletions(-) create mode 100755 .bin/Scripts/hw-sensors-monitor diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index b47f9afd..ec53e548 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -5,7 +5,7 @@ import json import re import shutil -from functions.common import * +from functions.tmux import * # STATIC VARIABLES TEMP_LIMITS = { @@ -153,6 +153,18 @@ def get_temp_str(temp, colors=True): '-' if temp < 0 else '', temp) +def monitor_sensors(monitor_pane, monitor_file): + """Continually update sensor data and report to screen.""" + sensor_data = get_sensor_data() + while True: + update_sensor_data(sensor_data) + with open(monitor_file, 'w') as f: + report = generate_report(sensor_data, 'Current', 'Max') + f.write('\n'.join(report)) + sleep(1) + if not tmux_poll_pane(monitor_pane): + break + def save_average_temp(sensor_data, temp_label, seconds=10): """Calculate average temps and record under temp_label, returns dict.""" clear_temps(sensor_data) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index ba00220f..10412a8c 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -9,6 +9,13 @@ def tmux_kill_pane(*panes): print(pane_id) run_program(cmd+[pane_id], check=False) +def tmux_poll_pane(pane_id): + """Check if pane exists, returns bool.""" + cmd = ['tmux', 'list-panes', '-F', '#D'] + result = run_program(cmd, check=False) + panes = result.stdout.decode().splitlines() + return pane_id in panes + def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, diff --git a/.bin/Scripts/hw-sensors b/.bin/Scripts/hw-sensors index f251c589..39ca7147 100755 --- a/.bin/Scripts/hw-sensors +++ b/.bin/Scripts/hw-sensors @@ -1,168 +1,10 @@ -#!/bin/python3 +#!/bin/bash # ## Wizard Kit: Sensor monitoring tool -import itertools -import os -import shutil -import sys +WINDOW_NAME="Hardware Sensors" +MONITOR="hw-sensors-monitor" -# Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) -from functions.common import * -from borrowed import sensors - -# STATIC VARIABLES -COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'GREEN': '\033[32m', - 'YELLOW': '\033[33m', - 'ORANGE': '\033[31;1m', - 'BLUE': '\033[34m' - } -TEMP_LIMITS = { - 'GREEN': 60, - 'YELLOW': 70, - 'ORANGE': 80, - 'RED': 90, - } - -# REGEX -REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') - -def color_temp(temp): - try: - temp = float(temp) - except ValueError: - return '{YELLOW}{temp}{CLEAR}'.format(temp=temp, **COLORS) - if temp > TEMP_LIMITS['RED']: - color = COLORS['RED'] - elif temp > TEMP_LIMITS['ORANGE']: - color = COLORS['ORANGE'] - elif temp > TEMP_LIMITS['YELLOW']: - color = COLORS['YELLOW'] - elif temp > TEMP_LIMITS['GREEN']: - color = COLORS['GREEN'] - elif temp > 0: - color = COLORS['BLUE'] - else: - color = COLORS['CLEAR'] - return '{color}{prefix}{temp:2.0f}°C{CLEAR}'.format( - color = color, - prefix = '-' if temp < 0 else '', - temp = temp, - **COLORS) - -def get_feature_string(chip, feature): - sfs = list(sensors.SubFeatureIterator(chip, feature)) # get a list of all subfeatures - label = sensors.get_label(chip, feature) - skipname = len(feature.name)+1 # skip common prefix - data = {} - - if feature.type != sensors.feature.TEMP: - # Skip non-temperature sensors - return None - - for sf in sfs: - name = sf.name[skipname:].decode("utf-8").strip() - try: - val = sensors.get_value(chip, sf.number) - except Exception: - # Ignore upstream sensor bugs and lie instead - val = -123456789 - if 'alarm' in name: - # Skip - continue - if '--nocolor' in sys.argv: - try: - temp = float(val) - except ValueError: - data[name] = ' {}°C'.format(val) - else: - data[name] = '{}{:2.0f}°C'.format( - '-' if temp < 0 else '', - temp) - else: - data[name] = color_temp(val) - - main_temp = data.pop('input', None) - if main_temp: - list_data = [] - for item in ['max', 'crit']: - if item in data: - list_data.append('{}: {}'.format(item, data.pop(item))) - list_data.extend( - ['{}: {}'.format(k, v) for k, v in sorted(data.items())]) - data_str = '{:18} {} {}'.format( - label, main_temp, ', '.join(list_data)) - else: - list_data.extend(sorted(data.items())) - list_data = ['{}: {}'.format(item[0], item[1]) for item in list_data] - data_str = '{:18} {}'.format(label, ', '.join(list_data)) - return data_str - -def join_columns(column1, column2, width=55): - return '{:<{}}{}'.format( - column1, - 55+len(column1)-len(REGEX_COLORS.sub('', column1)), - column2) - -if __name__ == '__main__': - try: - # Prep - sensors.init() - - # Get sensor data - chip_temps = {} - for chip in sensors.ChipIterator(): - chip_name = '{} ({})'.format( - sensors.chip_snprintf_name(chip), - sensors.get_adapter_name(chip.bus)) - chip_feats = [get_feature_string(chip, feature) - for feature in sensors.FeatureIterator(chip)] - # Strip empty/None items - chip_feats = [f for f in chip_feats if f] - - if chip_feats: - chip_temps[chip_name] = [chip_name, *chip_feats, ''] - - # Sort chips - sensor_temps = [] - for chip in [k for k in sorted(chip_temps.keys()) if 'coretemp' in k]: - sensor_temps.extend(chip_temps[chip]) - for chip in sorted(chip_temps.keys()): - if 'coretemp' not in chip: - sensor_temps.extend(chip_temps[chip]) - - # Wrap columns as needed - screen_size = shutil.get_terminal_size() - rows = screen_size.lines - 1 - if len(sensor_temps) > rows and screen_size.columns > 55*2: - sensor_temps = list(itertools.zip_longest( - sensor_temps[:rows], sensor_temps[rows:], fillvalue='')) - sensor_temps = [join_columns(a, b) for a, b in sensor_temps] - - # Print data - if sensor_temps: - for line in sensor_temps: - print_standard(line) - else: - if '--nocolor' in sys.argv: - print_standard('WARNING: No sensors found') - print_standard('\nPlease monitor temps manually') - else: - print_warning('WARNING: No sensors found') - print_standard('\nPlease monitor temps manually') - - # Done - sensors.cleanup() - exit_script() - except SystemExit: - sensors.cleanup() - pass - except: - sensors.cleanup() - major_exception() +# Start session +tmux new-session -n "$WINDOW_NAME" "$MONITOR" diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor new file mode 100755 index 00000000..91651f32 --- /dev/null +++ b/.bin/Scripts/hw-sensors-monitor @@ -0,0 +1,31 @@ +#!/bin/python3 +# +## Wizard Kit: Sensor monitoring tool + +import os +import sys + +# Init +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) +from functions.sensors import * +from functions.tmux import * +init_global_vars() + +if __name__ == '__main__': + try: + result = run_program(['mktemp']) + monitor_file = result.stdout.decode().strip() + print(monitor_file) + monitor_pane = tmux_split_window( + percent=1, vertical=True, watch=monitor_file) + cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] + run_program(cmd, check=False) + monitor_sensors(monitor_pane, monitor_file) + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 From c777d490913d14666c958f7015996170e6d32c49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:54:37 -0700 Subject: [PATCH 052/121] Added tmux_resize_pane() --- .bin/Scripts/functions/tmux.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 10412a8c..6726e3ff 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -16,6 +16,19 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes +def tmux_resize_pane(pane_id, x=None, y=None): + """Resize pane to specific hieght or width.""" + if not x and not y: + raise Exception('Neither height nor width specified.') + + cmd = ['tmux', 'resize-pane', '-t', pane_id] + if x: + cmd.extend(['-x', str(x)]) + elif y: + cmd.extend(['-y', str(y)]) + + run_program(cmd, check=False) + def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, From 5550cce8dbf4c77d67f15213b2e016a321b6bf87 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:55:15 -0700 Subject: [PATCH 053/121] Add background mode for monitoring sensors * This will be called by hw_diags.py to update a file in the background * NOTE: This uses a naive check before attempting to write data --- .bin/Scripts/functions/sensors.py | 2 +- .bin/Scripts/hw-sensors-monitor | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index ec53e548..41508927 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -162,7 +162,7 @@ def monitor_sensors(monitor_pane, monitor_file): report = generate_report(sensor_data, 'Current', 'Max') f.write('\n'.join(report)) sleep(1) - if not tmux_poll_pane(monitor_pane): + if monitor_pane and not tmux_poll_pane(monitor_pane): break def save_average_temp(sensor_data, temp_label, seconds=10): diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 91651f32..22067b91 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -13,14 +13,20 @@ from functions.tmux import * init_global_vars() if __name__ == '__main__': + background = False try: - result = run_program(['mktemp']) - monitor_file = result.stdout.decode().strip() - print(monitor_file) - monitor_pane = tmux_split_window( - percent=1, vertical=True, watch=monitor_file) - cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] - run_program(cmd, check=False) + if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): + background = True + monitor_file = sys.argv[1] + monitor_pane=None + else: + result = run_program(['mktemp']) + monitor_file = result.stdout.decode().strip() + if not background: + monitor_pane = tmux_split_window( + percent=1, vertical=True, watch=monitor_file) + cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] + run_program(cmd, check=False) monitor_sensors(monitor_pane, monitor_file) exit_script() except SystemExit: From 74bb31e795c41b2466d33ef9fa76f169e3b8ad89 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:57:38 -0700 Subject: [PATCH 054/121] Open temps monitor during run_mprime --- .bin/Scripts/functions/hw_diags.py | 17 +++++++++++++++++ .bin/Scripts/hw-sensors-monitor | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cc9a2c40..594f807a 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,6 +4,7 @@ import json import re import time +from functions.sensors import * from functions.tmux import * # STATIC VARIABLES @@ -614,6 +615,13 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" # Prep + _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(_sensors_out, 'w') as f: + f.write(' ') + sleep(1) + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) _title = '{}\n{}{}{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps', ': ' if 'Model name' in state.lscpu else '', @@ -621,6 +629,15 @@ def run_mprime_test(state): tmux_update_pane(state.panes['Top'], text=_title) state.tests['Prime95 & Temps']['Started'] = True update_progress_pane(state) + state.panes['mprime'] = tmux_split_window( + lines=10, vertical=True, text='Prime95 output goes here...') + state.panes['Temps'] = tmux_split_window( + behind=True, percent=80, vertical=True, watch=_sensors_out) + tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) + + # Start live monitor + pause() + monitor_proc.kill() # Get idle temps # Stress CPU diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 22067b91..42757748 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -18,7 +18,7 @@ if __name__ == '__main__': if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): background = True monitor_file = sys.argv[1] - monitor_pane=None + monitor_pane = None else: result = run_program(['mktemp']) monitor_file = result.stdout.decode().strip() From 30ba6516745347f6a979c7f0763284938c0c62e8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 00:10:51 -0700 Subject: [PATCH 055/121] Removing report wrapping section * Doesn't work properly with background processes --- .bin/Scripts/functions/sensors.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 41508927..29ceaa45 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -1,9 +1,7 @@ # Wizard Kit: Functions - Sensors -import itertools import json import re -import shutil from functions.tmux import * @@ -59,14 +57,6 @@ def generate_report(sensor_data, *temp_labels, colors=True): report.append(_line) report.append(' ') - # Wrap lines if necessary - screen_size = shutil.get_terminal_size() - rows = screen_size.lines - 1 - if len(report) > rows and screen_size.columns > 55*2: - report = list(itertools.zip_longest( - report[:rows], report[rows:], fillvalue='')) - report = [join_columns(a, b) for a, b in report] - # Handle empty reports (i.e. no sensors detected) if not report: report = [ From dc606a87806661a994d8472b188bdf5199fe90d0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 01:06:21 -0700 Subject: [PATCH 056/121] Main Prime95 sections working * Still need check results and update progress sections --- .bin/Scripts/functions/hw_diags.py | 102 ++++++++++++++++++++++++----- .bin/Scripts/functions/sensors.py | 6 +- .bin/Scripts/functions/tmux.py | 1 - .bin/Scripts/hw-diags-prime95 | 3 +- 4 files changed, 88 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 594f807a..a2aca6c8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -186,7 +186,8 @@ class State(): self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': '', 'Started': False, 'Status': ''}, + 'Result': '', 'Sensor Data': get_sensor_data(), + 'Started': False, 'Status': ''}, 'NVMe / SMART': {'Enabled': False, 'Order': 2}, 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, @@ -614,41 +615,106 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" - # Prep - _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(_sensors_out, 'w') as f: - f.write(' ') - sleep(1) - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], - pipe=True) + state.tests['Prime95 & Temps']['Started'] = True + update_progress_pane(state) + _sensor_data = state.tests['Prime95 & Temps']['Sensor Data'] + + # Update top pane _title = '{}\n{}{}{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps', ': ' if 'Model name' in state.lscpu else '', state.lscpu.get('Model name', '')) tmux_update_pane(state.panes['Top'], text=_title) - state.tests['Prime95 & Temps']['Started'] = True - update_progress_pane(state) + + # Start live sensor monitor + _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(_sensors_out, 'w') as f: + f.write(' ') + f.flush() + sleep(0.5) + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) + + # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( - lines=10, vertical=True, text='Prime95 output goes here...') + lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( behind=True, percent=80, vertical=True, watch=_sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) - # Start live monitor - pause() - monitor_proc.kill() - # Get idle temps + clear_screen() + try_and_print( + message='Getting idle temps...', indent=0, + function=save_average_temp, cs='Done', + sensor_data=_sensor_data, temp_label='Idle') + # Stress CPU - # Get max temp + run_program(['apple-fans', 'max']) + tmux_update_pane( + state.panes['mprime'], + command=['hw-diags-prime95', global_vars['TmpDir']]) + time_limit = int(MPRIME_LIMIT) * 60 + try: + for i in range(time_limit): + clear_screen() + sec_left = time_limit - i + min_left = int(sec_left / 60) + if min_left > 0: + print_standard( + 'Running Prime95 ({} minute{} left)'.format( + min_left, + 's' if min_left != 1 else '')) + else: + print_standard( + 'Running Prime95 ({} second{} left)'.format( + sec_left, + 's' if sec_left != 1 else '')) + print_warning('If running too hot, press CTRL+c to abort the test') + update_sensor_data(_sensor_data) + sleep(1) + except KeyboardInterrupt: + # Catch CTRL+C + aborted = True + state.tests['Prime95 & Temps']['Result'] = 'Aborted' + print_warning('\nAborted.') + update_progress_pane(state) + + # Restart live monitor + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) + + # Stop Prime95 (twice for good measure) + tmux_kill_pane(state.panes['mprime']) + run_program(['killall', '-s', 'INT', 'mprime'], check=False) + # Get cooldown temp + run_program(['apple-fans', 'auto']) + clear_screen() + try_and_print( + message='Letting CPU cooldown for bit...', indent=0, + function=sleep, cs='Done', seconds=10) + try_and_print( + message='Getting cooldown temps...', indent=0, + function=save_average_temp, cs='Done', + sensor_data=_sensor_data, temp_label='Cooldown') + + # Check results + # TODO # Done - sleep(3) state.tests['Prime95 & Temps']['Result'] = 'Unknown' update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) + monitor_proc.kill() + + # TODO Testing + print('\n'.join(generate_report(_sensor_data, 'Idle', 'Max', 'Cooldown'))) + def run_network_test(): """Run network test.""" clear_screen() diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 29ceaa45..066dc446 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -46,14 +46,12 @@ def generate_report(sensor_data, *temp_labels, colors=True): for _source, _data in sorted(_sources.items()): # Source _line = '{:18} '.format(fix_sensor_str(_source)) - _temps = [] # Temps (skip label for Current) for _label in temp_labels: - _temps.append('{}{}{}'.format( + _line += '{}{}{} '.format( _label.lower() if _label != 'Current' else '', ': ' if _label != 'Current' else '', - get_temp_str(_data.get(_label, '???'), colors=colors))) - _line += ', '.join(_temps) + get_temp_str(_data.get(_label, '???'), colors=colors)) report.append(_line) report.append(' ') diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 6726e3ff..c9a92194 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -6,7 +6,6 @@ def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] for pane_id in panes: - print(pane_id) run_program(cmd+[pane_id], check=False) def tmux_poll_pane(pane_id): diff --git a/.bin/Scripts/hw-diags-prime95 b/.bin/Scripts/hw-diags-prime95 index 30c6994d..4927da76 100755 --- a/.bin/Scripts/hw-diags-prime95 +++ b/.bin/Scripts/hw-diags-prime95 @@ -14,5 +14,6 @@ if [ ! -d "$1" ]; then fi # Run Prime95 -mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "$1/prime.log" +cd "$1" +mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "prime.log" From ca4234b1c36ace85e82d98ff895885eda76b346d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 15:29:06 -0700 Subject: [PATCH 057/121] Added working_dir arg for tmux command sections --- .bin/Scripts/functions/hw_diags.py | 3 ++- .bin/Scripts/functions/tmux.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a2aca6c8..3c1fc5b3 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -654,7 +654,8 @@ def run_mprime_test(state): run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], - command=['hw-diags-prime95', global_vars['TmpDir']]) + command=['hw-diags-prime95', global_vars['TmpDir']], + working_dir=global_vars['TmpDir']) time_limit = int(MPRIME_LIMIT) * 60 try: for i in range(time_limit): diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index c9a92194..84ab96cc 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -32,7 +32,8 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - command=None, text=None, watch=None): + working_dir=None, command=None, + text=None, watch=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: @@ -57,6 +58,8 @@ def tmux_split_window( if target_pane: cmd.extend(['-t', str(target_pane)]) + if working_dir: + cmd.extend(['-c', working_dir]) if command: cmd.extend(command) elif text: @@ -71,13 +74,15 @@ def tmux_split_window( result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane(pane_id, command=None, text=None): +def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): """Respawn with either a new command or new text.""" # Bail early if not command and not text: raise Exception('Neither command nor text specified.') cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if working_dir: + cmd.extend(['-c', working_dir]) if command: cmd.extend(command) elif text: From a910f2cb033066438af1030a1cc34d1af6c07f6f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 18:27:19 -0700 Subject: [PATCH 058/121] Adjusted Prime95 countdown --- .bin/Scripts/functions/hw_diags.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3c1fc5b3..a76666ab 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -651,6 +651,8 @@ def run_mprime_test(state): sensor_data=_sensor_data, temp_label='Idle') # Stress CPU + print_log('Starting Prime95') + _abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -660,24 +662,23 @@ def run_mprime_test(state): try: for i in range(time_limit): clear_screen() - sec_left = time_limit - i - min_left = int(sec_left / 60) + sec_left = (time_limit - i) % 60 + min_left = int( (time_limit - i) / 60) + _status_str = 'Running Prime95 (' if min_left > 0: - print_standard( - 'Running Prime95 ({} minute{} left)'.format( - min_left, - 's' if min_left != 1 else '')) - else: - print_standard( - 'Running Prime95 ({} second{} left)'.format( - sec_left, - 's' if sec_left != 1 else '')) - print_warning('If running too hot, press CTRL+c to abort the test') + _status_str += '{} minute{}, '.format( + min_left, + 's' if min_left != 1 else '') + _status_str += '{} second{} left)'.format( + sec_left, + 's' if sec_left != 1 else '') + # Not using print wrappers to avoid flooding the log + print(_status_str) + print('{YELLOW}{msg}{CLEAR}'.format(msg=_abort_msg, **COLORS)) update_sensor_data(_sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C - aborted = True state.tests['Prime95 & Temps']['Result'] = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) From 12ff99eb3213569a776363f630c3767efd596da6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 18:27:43 -0700 Subject: [PATCH 059/121] Set LogDir for non-quick tests --- .bin/Scripts/functions/hw_diags.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a76666ab..e1b97df9 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -180,8 +180,7 @@ class State(): self.devs = [] self.finished = False self.panes = {} - # TODO Switch to LogDir - self.progress_out = '{}/progress.out'.format(global_vars['TmpDir']) + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.started = False self.tests = { @@ -221,6 +220,16 @@ class State(): for k in ['Result', 'Started', 'Status']: self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + # Update LogDir + if not self.quick_mode: + global_vars['LogDir'] = '{}/Logs/{}_{}'.format( + global_vars['Env']['HOME'], + get_ticket_number(), + time.strftime('%Y-%m-%d_%H%M_%z')) + os.makedirs(global_vars['LogDir'], exist_ok=True) + global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( + global_vars['LogDir']) + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) @@ -703,6 +712,15 @@ def run_mprime_test(state): function=save_average_temp, cs='Done', sensor_data=_sensor_data, temp_label='Cooldown') + # Move logs to Ticket folder + for item in os.scandir(global_vars['TmpDir']): + try: + shutil.move(item.path, global_vars['LogDir']) + except Exception: + print_error('ERROR: Failed to move "{}" to "{}"'.format( + item.path, + global_vars['LogDir'])) + # Check results # TODO From 6a3ef60881ed43800605b4692e79836a2d978c03 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:41:29 -0700 Subject: [PATCH 060/121] Added CpuObj and renamed dev names to disk * This should make the code more clear * The CpuObj is similar to DiskObj to abstract the device/tests calls * New calls will be like: run_test(state, dev) --- .bin/Scripts/functions/hw_diags.py | 204 +++++++++++++---------------- 1 file changed, 90 insertions(+), 114 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index e1b97df9..a2c8293d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -67,32 +67,43 @@ SIDE_PANE_WIDTH = 20 TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes -class DevObj(): - """Device object for tracking device specific data.""" - def __init__(self, state, dev_path): - self.failing = False +class CpuObj(): + """Object for tracking CPU specific data.""" + def __init__(self): + self.lscpu = {} + self.tests = {} + self.get_details() + + def get_details(self): + """Get CPU details from lscpu.""" + cmd = ['lscpu', '--json'] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + except Exception: + # Ignore and leave self.lscpu empty + return + for line in json_data.get('lscpu', []): + _field = line.get('field', None).replace(':', '') + _data = line.get('data', None) + if not _field and not _data: + # Skip + print_warning(_field, _data) + pause() + continue + self.lscpu[_field] = _data + +class DiskObj(): + """Object for tracking disk specific data.""" + def __init__(self, disk_path): self.labels = [] self.lsblk = {} - self.name = re.sub(r'^.*/(.*)', r'\1', dev_path) + self.name = re.sub(r'^.*/(.*)', r'\1', disk_path) self.nvme_attributes = {} - self.override = False - self.path = dev_path + self.path = disk_path self.smart_attributes = {} self.smartctl = {} - self.state = state - self.tests = { - 'NVMe / SMART': { - 'Result': '', 'Started': False, 'Status': '', 'Order': 1}, - 'badblocks': { - 'Result': '', 'Started': False, 'Status': '', 'Order': 2}, - 'I/O Benchmark': { - 'Result': '', - 'Started': False, - 'Status': '', - 'Read Rates': [], - 'Graph Data': [], - 'Order': 3}, - } + self.tests = {} self.get_details() self.get_smart_details() @@ -122,9 +133,9 @@ class DevObj(): self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') # Build list of labels - for dev in [self.lsblk, *self.lsblk.get('children', [])]: - self.labels.append(dev.get('label', '')) - self.labels.append(dev.get('partlabel', '')) + for disk in [self.lsblk, *self.lsblk.get('children', [])]: + self.labels.append(disk.get('label', '')) + self.labels.append(disk.get('partlabel', '')) self.labels = [str(label) for label in self.labels if label] def get_smart_details(self): @@ -158,31 +169,15 @@ class DevObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def update_progress(self): - """Update status strings.""" - for k, v in self.tests.items(): - if self.state.tests[k]['Enabled']: - _status = '' - if not v['Status']: - _status = 'Pending' - if v['Started']: - if v['Result']: - _status = v['Result'] - else: - _status = 'Working' - if _status: - v['Status'] = build_status_string(self.name, _status) class State(): """Object to track device objects and overall state.""" def __init__(self): - self.lscpu = {} - self.devs = [] - self.finished = False + self.cpu = None + self.disks = [] self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False - self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, 'Result': '', 'Sensor Data': get_sensor_data(), @@ -191,32 +186,10 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - self.get_cpu_details() - - def get_cpu_details(self): - """Get CPU details from lscpu.""" - cmd = ['lscpu', '--json'] - try: - result = run_program(cmd, check=False) - json_data = json.loads(result.stdout.decode()) - except Exception as err: - # Ignore and leave self.cpu empty - print_error(err) - pause() - return - for line in json_data.get('lscpu', []): - _field = line.get('field', None).replace(':', '') - _data = line.get('data', None) - if not _field and not _data: - # Skip - print_warning(_field, _data) - pause() - continue - self.lscpu[_field] = _data def init(self): - """Scan for block devices and reset all tests.""" - self.devs = [] + """Set log and add devices.""" + self.disks = [] for k in ['Result', 'Started', 'Status']: self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' @@ -230,27 +203,30 @@ class State(): global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( global_vars['LogDir']) + # Add CPU + self.cpu = CpuObj() + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) - for dev in json_data['blockdevices']: - skip_dev = False - dev_obj = DevObj(self, dev['name']) + for disk in json_data['blockdevices']: + skip_disk = False + disk_obj = DiskObj(disk['name']) - # Skip loopback and optical devices - if dev_obj.lsblk['type'] in ['loop', 'rom']: - skip_dev = True + # Skip loopback devices, optical devices, etc + if disk_obj.lsblk['type'] != 'disk': + skip_disk = True - # Skip WK devices + # Skip WK disks wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) - for label in dev_obj.labels: + for label in disk_obj.labels: if re.search(wk_label_regex, label, re.IGNORECASE): - skip_dev = True + skip_disk = True - # Add device - if not skip_dev: - self.devs.append(dev_obj) + # Add disk + if not skip_disk: + self.disks.append(disk_obj) def update_progress(self): """Update status strings.""" @@ -313,32 +289,32 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) -def check_dev_attributes(dev): - """Check if device should be tested and allow overrides.""" +def check_disk_attributes(disk): + """Check if disk should be tested and allow overrides.""" needs_override = False print_standard(' {size:>6} ({tran}) {model} {serial}'.format( - **dev.lsblk)) + **disk.lsblk)) # General checks - if not dev.nvme_attributes and not dev.smart_attributes: + if not disk.nvme_attributes and not disk.smart_attributes: needs_override = True print_warning( ' WARNING: No NVMe or SMART attributes available for: {}'.format( - dev.path)) + disk.path)) # NVMe checks - # TODO check all tracked attributes and set dev.failing if needed + # TODO check all tracked attributes and set disk.failing if needed # SMART checks - # TODO check all tracked attributes and set dev.failing if needed + # TODO check all tracked attributes and set disk.failing if needed # Ask for override if necessary if needs_override: if ask(' Run tests on this device anyway?'): - # TODO Set override for this dev + # TODO Set override for this disk pass else: - for v in dev.tests.values(): + for v in disk.tests.values(): # Started is set to True to fix the status string v['Result'] = 'Skipped' v['Started'] = True @@ -551,11 +527,11 @@ def run_badblocks_test(state): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test()') - for dev in state.devs: - dev.tests['badblocks']['Started'] = True + for disk in state.disks: + disk.tests['badblocks']['Started'] = True update_progress_pane(state) sleep(3) - dev.tests['badblocks']['Result'] = 'OVERRIDE' + disk.tests['badblocks']['Result'] = 'OVERRIDE' update_progress_pane(state) def run_hw_tests(state): @@ -610,11 +586,11 @@ def run_io_benchmark(state): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark()') - for dev in state.devs: - dev.tests['I/O Benchmark']['Started'] = True + for disk in state.disks: + disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) sleep(3) - dev.tests['I/O Benchmark']['Result'] = 'Unknown' + disk.tests['I/O Benchmark']['Result'] = 'Unknown' update_progress_pane(state) def run_keyboard_test(): @@ -741,42 +717,42 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart(state): +def run_nvme_smart_tests(state): """TODO""" - for dev in state.devs: + for disk in state.disks: tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **dev.lsblk)) - dev.tests['NVMe / SMART']['Started'] = True + t=TOP_PANE_TEXT, **disk.lsblk)) + disk.tests['NVMe / SMART']['Started'] = True update_progress_pane(state) - if dev.nvme_attributes: - run_nvme_tests(state, dev) - elif dev.smart_attributes: - run_smart_tests(state, dev) + if disk.nvme_attributes: + run_nvme_tests(state, disk) + elif disk.smart_attributes: + run_smart_tests(state, disk) else: - print_standard('TODO: run_nvme_smart({})'.format( - dev.path)) + print_standard('TODO: run_nvme_smart_tests({})'.format( + disk.path)) print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( - dev.path)) - dev.tests['NVMe / SMART']['Status'] = 'N/A' - dev.tests['NVMe / SMART']['Result'] = 'N/A' + disk.path)) + disk.tests['NVMe / SMART']['Status'] = 'N/A' + disk.tests['NVMe / SMART']['Result'] = 'N/A' update_progress_pane(state) sleep(3) -def run_nvme_tests(state, dev): +def run_nvme_tests(state, disk): """TODO""" - print_standard('TODO: run_nvme_test({})'.format(dev.path)) + print_standard('TODO: run_nvme_test({})'.format(disk.path)) sleep(3) - dev.tests['NVMe / SMART']['Result'] = 'CS' + disk.tests['NVMe / SMART']['Result'] = 'CS' update_progress_pane(state) -def run_smart_tests(state, dev): +def run_smart_tests(state, disk): """TODO""" - print_standard('TODO: run_smart_tests({})'.format(dev.path)) + print_standard('TODO: run_smart_tests({})'.format(disk.path)) sleep(3) - dev.tests['NVMe / SMART']['Result'] = 'CS' + disk.tests['NVMe / SMART']['Result'] = 'CS' update_progress_pane(state) def secret_screensaver(screensaver=None): @@ -872,8 +848,8 @@ def update_progress_pane(state): if 'Prime95' not in k and v['Enabled']: output.append('{BLUE}{test_name}{CLEAR}'.format( test_name=k, **COLORS)) - for dev in state.devs: - output.append(dev.tests[k]['Status']) + for disk in state.disks: + output.append(disk.tests[k]['Status']) output.append(' ') # Add line-endings From 0390290f10d44ec9ab8b41d3bfe374b61744795b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:46:17 -0700 Subject: [PATCH 061/121] Added TestObj() * This object will track test specific vars and results * Moved status code into TestObj * Test calls will now be: run_test(state, dev, test_obj) * NOTE: Code is not done and is quite broken --- .bin/Scripts/functions/hw_diags.py | 120 ++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a2c8293d..861305df 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -64,6 +64,12 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 +TESTS_CPU = ['Prime95 & Temps'] +TESTS_DISK = [ + 'I/O Benchmark', + 'NVMe / SMART', + 'badblocks', + ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes @@ -169,6 +175,10 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + def safety_check(self): + """Check enabled tests and verify it's safe to run them.""" + # TODO + pass class State(): """Object to track device objects and overall state.""" @@ -179,13 +189,31 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = { - 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': '', 'Sensor Data': get_sensor_data(), - 'Started': False, 'Status': ''}, - 'NVMe / SMART': {'Enabled': False, 'Order': 2}, - 'badblocks': {'Enabled': False, 'Order': 3}, - 'I/O Benchmark': {'Enabled': False, 'Order': 4}, - } + 'Prime95 & Temps': { + 'Enabled': False, + 'Function': run_mprime_test, + 'Objects': [], + 'Order': 1, + }, + 'NVMe / SMART': { + 'Enabled': False, + 'Function': run_nvme_smart_tests, + 'Objects': [], + 'Order': 2, + }, + 'badblocks': { + 'Enabled': False, + 'Function': run_badblocks_test, + 'Objects': [], + 'Order': 3, + }, + 'I/O Benchmark': { + 'Enabled': False, + 'Function': run_io_benchmark, + 'Objects': [], + 'Order': 4, + }, + } def init(self): """Set log and add devices.""" @@ -228,26 +256,32 @@ class State(): if not skip_disk: self.disks.append(disk_obj) - def update_progress(self): - """Update status strings.""" - # Prime95 - p = self.tests['Prime95 & Temps'] - if p['Enabled']: - _status = '' - if not p['Status']: - _status = 'Pending' - if p['Started']: - if p['Result']: - _status = p['Result'] - else: - _status = 'Working' - if _status: - p['Status'] = build_status_string( - 'Prime95', _status, info_label=True) +class TestObj(): + """Object to track test data.""" + def __init__(self, label, info_label=False): + self.started = False + self.passed = False + self.failed = False + self.report = '' + self.status = '' + self.label = label + self.info_label = info_label + self.disabled = False + self.update_status() - # Disks - for dev in self.devs: - dev.update_progress() + def update_status(self, new_status=None): + """Update status strings.""" + if self.disabled: + return + if new_status: + self.status = build_status_string( + self.label, new_status, self.info_label) + elif not self.status: + self.status = build_status_string( + self.label, 'Pending', self.info_label) + elif self.started and 'Pending' in self.status: + self.status = build_status_string( + self.label, 'Working', self.info_label) # Functions def build_outer_panes(state): @@ -543,7 +577,7 @@ def run_hw_tests(state): update_progress_pane(state) build_outer_panes(state) - # Run test(s) + # Show selected tests and create TestObj()s print_info('Selected Tests:') for k, v in sorted( state.tests.items(), @@ -554,21 +588,36 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) + if v['Enabled']: + # Create TestObj and track under both CpuObj/DiskObj and State + if k in TESTS_CPU: + test_obj = TestObj(info_label=True) + state.cpu.tests[k] = test_obj + v['Objects'].append(test_obj) + elif k in TESTS_DISK: + for disk in state.disks: + test_obj = TestObj() + disk.tests[k] = test_obj + v['Objects'].append(test_obj) print_standard('') - # Check devices if necessary - if (state.tests['badblocks']['Enabled'] - or state.tests['I/O Benchmark']['Enabled']): - print_info('Selected Disks:') - for dev in state.devs: - check_dev_attributes(dev) - print_standard('') + # Run safety checks + for disk in state.disks: + disk.safety_check() + + # Run tests + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + if v['Enabled']: + # TODO + pass # Run tests if state.tests['Prime95 & Temps']['Enabled']: run_mprime_test(state) if state.tests['NVMe / SMART']['Enabled']: - run_nvme_smart(state) + run_nvme_smart_tests(state) if state.tests['badblocks']['Enabled']: run_badblocks_test(state) if state.tests['I/O Benchmark']['Enabled']: @@ -835,7 +884,6 @@ def update_io_progress(percent, rate, progress_file): def update_progress_pane(state): """Update progress file for side pane.""" output = [] - state.update_progress() # Prime95 output.append(state.tests['Prime95 & Temps']['Status']) From 49471663f50ac3fe239a0417063b8ee814422976 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:50:11 -0700 Subject: [PATCH 062/121] Use OrderedDicts to avoid lambda sorting --- .bin/Scripts/functions/hw_diags.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 861305df..05c7ebb7 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,6 +4,7 @@ import json import re import time +from collections import OrderedDict from functions.sensors import * from functions.tmux import * @@ -109,7 +110,7 @@ class DiskObj(): self.path = disk_path self.smart_attributes = {} self.smartctl = {} - self.tests = {} + self.tests = OrderedDict() self.get_details() self.get_smart_details() @@ -188,32 +189,28 @@ class State(): self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False - self.tests = { + self.tests = OrderedDict({ 'Prime95 & Temps': { 'Enabled': False, 'Function': run_mprime_test, 'Objects': [], - 'Order': 1, }, 'NVMe / SMART': { 'Enabled': False, 'Function': run_nvme_smart_tests, 'Objects': [], - 'Order': 2, }, 'badblocks': { 'Enabled': False, 'Function': run_badblocks_test, 'Objects': [], - 'Order': 3, }, 'I/O Benchmark': { 'Enabled': False, 'Function': run_io_benchmark, 'Objects': [], - 'Order': 4, }, - } + }) def init(self): """Set log and add devices.""" @@ -579,9 +576,7 @@ def run_hw_tests(state): # Show selected tests and create TestObj()s print_info('Selected Tests:') - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): print_standard(' {:<15} {}{}{} {}'.format( k, COLORS['GREEN'] if v['Enabled'] else COLORS['RED'], @@ -606,9 +601,7 @@ def run_hw_tests(state): disk.safety_check() # Run tests - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): if v['Enabled']: # TODO pass @@ -890,9 +883,7 @@ def update_progress_pane(state): output.append(' ') # Disks - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): if 'Prime95' not in k and v['Enabled']: output.append('{BLUE}{test_name}{CLEAR}'.format( test_name=k, **COLORS)) From 941a5537669a0c19f9c37f5f139211e1154b5c91 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:16:31 -0700 Subject: [PATCH 063/121] Renamed "Prime95 & Temps" to "Prime95" for brevity --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 05c7ebb7..fd8b0ebb 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -65,7 +65,7 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 -TESTS_CPU = ['Prime95 & Temps'] +TESTS_CPU = ['Prime95'] TESTS_DISK = [ 'I/O Benchmark', 'NVMe / SMART', @@ -190,7 +190,7 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = OrderedDict({ - 'Prime95 & Temps': { + 'Prime95': { 'Enabled': False, 'Function': run_mprime_test, 'Objects': [], @@ -216,7 +216,7 @@ class State(): """Set log and add devices.""" self.disks = [] for k in ['Result', 'Started', 'Status']: - self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + self.tests['Prime95'][k] = False if k == 'Started' else '' # Update LogDir if not self.quick_mode: @@ -443,7 +443,7 @@ def menu_diags(state, args): {'Base Name': 'Full Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, - {'Base Name': 'Prime95 & Temps', 'Enabled': False, 'CRLF': True}, + {'Base Name': 'Prime95', 'Enabled': False, 'CRLF': True}, {'Base Name': 'NVMe / SMART', 'Enabled': False}, {'Base Name': 'badblocks', 'Enabled': False}, {'Base Name': 'I/O Benchmark', 'Enabled': False}, @@ -642,13 +642,13 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" - state.tests['Prime95 & Temps']['Started'] = True + state.tests['Prime95']['Started'] = True update_progress_pane(state) - _sensor_data = state.tests['Prime95 & Temps']['Sensor Data'] + _sensor_data = state.tests['Prime95']['Sensor Data'] # Update top pane _title = '{}\n{}{}{}'.format( - TOP_PANE_TEXT, 'Prime95 & Temps', + TOP_PANE_TEXT, 'Prime95', ': ' if 'Model name' in state.lscpu else '', state.lscpu.get('Model name', '')) tmux_update_pane(state.panes['Top'], text=_title) @@ -706,7 +706,7 @@ def run_mprime_test(state): sleep(1) except KeyboardInterrupt: # Catch CTRL+C - state.tests['Prime95 & Temps']['Result'] = 'Aborted' + state.tests['Prime95']['Result'] = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) @@ -743,7 +743,7 @@ def run_mprime_test(state): # TODO # Done - state.tests['Prime95 & Temps']['Result'] = 'Unknown' + state.tests['Prime95']['Result'] = 'Unknown' update_progress_pane(state) # Cleanup @@ -879,7 +879,7 @@ def update_progress_pane(state): output = [] # Prime95 - output.append(state.tests['Prime95 & Temps']['Status']) + output.append(state.tests['Prime95']['Status']) output.append(' ') # Disks From 668c7c4c6a440fdcee3fa281d0590d728b634545 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:32:03 -0700 Subject: [PATCH 064/121] Updated run_mprime_test to use test_obj --- .bin/Scripts/functions/hw_diags.py | 95 +++++++++++++++--------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index fd8b0ebb..cd64e2e8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -80,6 +80,7 @@ class CpuObj(): self.lscpu = {} self.tests = {} self.get_details() + self.name = self.lscpu.get('Model name', 'Unknown CPU') def get_details(self): """Get CPU details from lscpu.""" @@ -255,15 +256,16 @@ class State(): class TestObj(): """Object to track test data.""" - def __init__(self, label, info_label=False): - self.started = False - self.passed = False - self.failed = False - self.report = '' - self.status = '' + def __init__(self, dev, label=None, info_label=False): + self.dev = dev self.label = label self.info_label = info_label self.disabled = False + self.failed = False + self.passed = False + self.report = '' + self.started = False + self.status = '' self.update_status() def update_status(self, new_status=None): @@ -552,12 +554,13 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_badblocks_test(state): +def run_badblocks_test(state, test_obj): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) - print_standard('TODO: run_badblocks_test()') + print_standard('TODO: run_badblocks_test({})'.format( + test_obj.dev.path)) for disk in state.disks: disk.tests['badblocks']['Started'] = True update_progress_pane(state) @@ -586,12 +589,13 @@ def run_hw_tests(state): if v['Enabled']: # Create TestObj and track under both CpuObj/DiskObj and State if k in TESTS_CPU: - test_obj = TestObj(info_label=True) + test_obj = TestObj( + dev=state.cpu, label='Prime95', info_label=True) state.cpu.tests[k] = test_obj v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj() + test_obj = TestObj(dev=k) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -601,20 +605,13 @@ def run_hw_tests(state): disk.safety_check() # Run tests + ## Because state.tests is an OrderedDict and the disks were added + ## in order, the tests will be run in order. for k, v in state.tests.items(): if v['Enabled']: - # TODO - pass - - # Run tests - if state.tests['Prime95 & Temps']['Enabled']: - run_mprime_test(state) - if state.tests['NVMe / SMART']['Enabled']: - run_nvme_smart_tests(state) - if state.tests['badblocks']['Enabled']: - run_badblocks_test(state) - if state.tests['I/O Benchmark']['Enabled']: - run_io_benchmark(state) + f = v['Function'] + for test_obj in v['Objects']: + f(state, test_obj) # Done pause('Press Enter to return to main menu... ') @@ -622,12 +619,13 @@ def run_hw_tests(state): # Cleanup tmux_kill_pane(*state.panes.values()) -def run_io_benchmark(state): +def run_io_benchmark(state, test_obj): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) - print_standard('TODO: run_io_benchmark()') + print_standard('TODO: run_io_benchmark({})'.format( + test_obj.dev.path)) for disk in state.disks: disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) @@ -640,34 +638,32 @@ def run_keyboard_test(): clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) -def run_mprime_test(state): +def run_mprime_test(state, test_obj): """Test CPU with Prime95 and track temps.""" state.tests['Prime95']['Started'] = True update_progress_pane(state) - _sensor_data = state.tests['Prime95']['Sensor Data'] + test_obj.sensor_data = get_sensor_data() # Update top pane - _title = '{}\n{}{}{}'.format( - TOP_PANE_TEXT, 'Prime95', - ': ' if 'Model name' in state.lscpu else '', - state.lscpu.get('Model name', '')) - tmux_update_pane(state.panes['Top'], text=_title) + test_obj.title = '{}\nPrime95: {}'.format( + TOP_PANE_TEXT, test_obj.dev.name) + tmux_update_pane(state.panes['Top'], text=test_obj.title) # Start live sensor monitor - _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(_sensors_out, 'w') as f: + test_obj.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(test_obj.sensors_out, 'w') as f: f.write(' ') f.flush() sleep(0.5) - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], + test_obj.monitor_proc = popen_program( + ['hw-sensors-monitor', test_obj.sensors_out], pipe=True) # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( - behind=True, percent=80, vertical=True, watch=_sensors_out) + behind=True, percent=80, vertical=True, watch=test_obj.sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) # Get idle temps @@ -675,11 +671,11 @@ def run_mprime_test(state): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=_sensor_data, temp_label='Idle') + sensor_data=test_obj.sensor_data, temp_label='Idle') # Stress CPU print_log('Starting Prime95') - _abort_msg = 'If running too hot, press CTRL+c to abort the test' + test_obj.abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -701,8 +697,8 @@ def run_mprime_test(state): 's' if sec_left != 1 else '') # Not using print wrappers to avoid flooding the log print(_status_str) - print('{YELLOW}{msg}{CLEAR}'.format(msg=_abort_msg, **COLORS)) - update_sensor_data(_sensor_data) + print('{YELLOW}{msg}{CLEAR}'.format(msg=test_obj.abort_msg, **COLORS)) + update_sensor_data(test_obj.sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C @@ -711,8 +707,8 @@ def run_mprime_test(state): update_progress_pane(state) # Restart live monitor - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], + test_obj.monitor_proc = popen_program( + ['hw-sensors-monitor', test_obj.sensors_out], pipe=True) # Stop Prime95 (twice for good measure) @@ -728,7 +724,7 @@ def run_mprime_test(state): try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=_sensor_data, temp_label='Cooldown') + sensor_data=test_obj.sensor_data, temp_label='Cooldown') # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -741,6 +737,12 @@ def run_mprime_test(state): # Check results # TODO + _log = '{}/results.txt'.format(global_vars['LogDir']) + if os.path.exists(_log): + with open(_log, 'r') as f: + for line in f.readlines(): + if re.search(r'(error|fail)', line, re.IGNORECASE): + state.tests['Prime95']['Result'] = 'NS' # Done state.tests['Prime95']['Result'] = 'Unknown' @@ -748,10 +750,11 @@ def run_mprime_test(state): # Cleanup tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) - monitor_proc.kill() + test_obj.monitor_proc.kill() # TODO Testing - print('\n'.join(generate_report(_sensor_data, 'Idle', 'Max', 'Cooldown'))) + print('\n'.join( + generate_report(test_obj.sensor_data, 'Idle', 'Max', 'Cooldown'))) def run_network_test(): """Run network test.""" @@ -759,7 +762,7 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart_tests(state): +def run_nvme_smart_tests(state, test_obj): """TODO""" for disk in state.disks: tmux_update_pane( From d88a9f39f2a0a85ce4e3fe9dde99a39a64c39046 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:36:24 -0700 Subject: [PATCH 065/121] Added tmux_kill_all_panes() --- .bin/Scripts/functions/tmux.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 84ab96cc..e1066e18 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -2,6 +2,13 @@ from functions.common import * +def tmux_kill_all_panes(pane_id=None): + """Kill all tmux panes except the active pane or pane_id if specified.""" + cmd = ['tmux', 'kill-pane', '-a'] + if pane_id: + cmd.extend(['-t', pane_id]) + run_program(cmd, check=False) + def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] From 465a3b42fb0cc92c03600b820bca0007fd9c3227 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:36:50 -0700 Subject: [PATCH 066/121] Kill all tmux panes before exiting --- .bin/Scripts/hw-diags-menu | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index e60f8fc4..6f0247cd 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -9,20 +9,24 @@ import sys os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.hw_diags import * +from functions.tmux import * init_global_vars() if __name__ == '__main__': - try: - # Show menu - state = State() - menu_diags(state, sys.argv) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + try: + # Show menu + state = State() + menu_diags(state, sys.argv) + # Done + #print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + tmux_kill_all_panes() + pass + except: + tmux_kill_all_panes() + major_exception() + +# vim: sts=2 sw=2 ts=2 From bb93386fa0b7dc9dec3f21529d9279cbbec62bc8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 16:32:00 -0700 Subject: [PATCH 067/121] Updated Prime95 checks --- .bin/Scripts/functions/hw_diags.py | 98 +++++++++++++++++++----------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cd64e2e8..8cebd636 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -257,6 +257,7 @@ class State(): class TestObj(): """Object to track test data.""" def __init__(self, dev, label=None, info_label=False): + self.aborted = False self.dev = dev self.label = label self.info_label = info_label @@ -554,13 +555,13 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_badblocks_test(state, test_obj): +def run_badblocks_test(state, test): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test({})'.format( - test_obj.dev.path)) + test.dev.path)) for disk in state.disks: disk.tests['badblocks']['Started'] = True update_progress_pane(state) @@ -619,13 +620,13 @@ def run_hw_tests(state): # Cleanup tmux_kill_pane(*state.panes.values()) -def run_io_benchmark(state, test_obj): +def run_io_benchmark(state, test): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark({})'.format( - test_obj.dev.path)) + test.dev.path)) for disk in state.disks: disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) @@ -638,32 +639,33 @@ def run_keyboard_test(): clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) -def run_mprime_test(state, test_obj): +def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" - state.tests['Prime95']['Started'] = True + test.started = True + test.update_status() update_progress_pane(state) - test_obj.sensor_data = get_sensor_data() + test.sensor_data = get_sensor_data() # Update top pane - test_obj.title = '{}\nPrime95: {}'.format( - TOP_PANE_TEXT, test_obj.dev.name) - tmux_update_pane(state.panes['Top'], text=test_obj.title) + test.title = '{}\nPrime95: {}'.format( + TOP_PANE_TEXT, test.dev.name) + tmux_update_pane(state.panes['Top'], text=test.title) # Start live sensor monitor - test_obj.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(test_obj.sensors_out, 'w') as f: + test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(test.sensors_out, 'w') as f: f.write(' ') f.flush() sleep(0.5) - test_obj.monitor_proc = popen_program( - ['hw-sensors-monitor', test_obj.sensors_out], + test.monitor_proc = popen_program( + ['hw-sensors-monitor', test.sensors_out], pipe=True) # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( - behind=True, percent=80, vertical=True, watch=test_obj.sensors_out) + behind=True, percent=80, vertical=True, watch=test.sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) # Get idle temps @@ -671,11 +673,11 @@ def run_mprime_test(state, test_obj): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test_obj.sensor_data, temp_label='Idle') + sensor_data=test.sensor_data, temp_label='Idle') # Stress CPU print_log('Starting Prime95') - test_obj.abort_msg = 'If running too hot, press CTRL+c to abort the test' + test.abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -697,18 +699,19 @@ def run_mprime_test(state, test_obj): 's' if sec_left != 1 else '') # Not using print wrappers to avoid flooding the log print(_status_str) - print('{YELLOW}{msg}{CLEAR}'.format(msg=test_obj.abort_msg, **COLORS)) - update_sensor_data(test_obj.sensor_data) + print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) + update_sensor_data(test.sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C - state.tests['Prime95']['Result'] = 'Aborted' + test.aborted = True + test.status = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) # Restart live monitor - test_obj.monitor_proc = popen_program( - ['hw-sensors-monitor', test_obj.sensors_out], + test.monitor_proc = popen_program( + ['hw-sensors-monitor', test.sensors_out], pipe=True) # Stop Prime95 (twice for good measure) @@ -724,7 +727,7 @@ def run_mprime_test(state, test_obj): try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test_obj.sensor_data, temp_label='Cooldown') + sensor_data=test.sensor_data, temp_label='Cooldown') # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -736,25 +739,52 @@ def run_mprime_test(state, test_obj): global_vars['LogDir'])) # Check results - # TODO - _log = '{}/results.txt'.format(global_vars['LogDir']) - if os.path.exists(_log): - with open(_log, 'r') as f: - for line in f.readlines(): - if re.search(r'(error|fail)', line, re.IGNORECASE): - state.tests['Prime95']['Result'] = 'NS' + test.logs = {} + for log in ['results.txt', 'prime.log']: + _data = '' + log_path = '{}/{}'.format(global_vars['LogDir'], log) + + # Read and save log + try: + with open(log_path, 'r') as f: + _data = f.read() + test.logs[log] = _data.splitlines() + except FileNotFoundError: + # Ignore since files may be missing for slower CPUs + pass + + # results.txt: NS check + if log == 'results.txt': + if re.search(r'(error|fail)', _data, re.IGNORECASE): + test.failed = True + test.status = 'NS' + + # prime.log: CS check + if log == 'prime.log': + if re.search( + r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): + test.passed = True + test.status = 'CS' + elif re.search( + r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): + # If the first re.search does not match and this one does then + # that means that either errors or warnings, or both, are non-zero + test.failed = True + test.passed = False + test.status = 'NS' + if not (test.aborted or test.failed or test.passed): + test.status = 'Unknown' # Done - state.tests['Prime95']['Result'] = 'Unknown' update_progress_pane(state) # Cleanup tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) - test_obj.monitor_proc.kill() + test.monitor_proc.kill() # TODO Testing print('\n'.join( - generate_report(test_obj.sensor_data, 'Idle', 'Max', 'Cooldown'))) + generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'))) def run_network_test(): """Run network test.""" @@ -762,7 +792,7 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart_tests(state, test_obj): +def run_nvme_smart_tests(state, test): """TODO""" for disk in state.disks: tmux_update_pane( From a00105f71818bce77842be25ce36173c6cb5d000 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 16:57:43 -0700 Subject: [PATCH 068/121] Fixed status updates --- .bin/Scripts/functions/hw_diags.py | 46 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8cebd636..6d6b1a2c 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -214,10 +214,10 @@ class State(): }) def init(self): - """Set log and add devices.""" + """Remove test objects, set log, and add devices.""" self.disks = [] - for k in ['Result', 'Started', 'Status']: - self.tests['Prime95'][k] = False if k == 'Started' else '' + for k, v in self.tests.items(): + v['Objects'] = [] # Update LogDir if not self.quick_mode: @@ -596,7 +596,7 @@ def run_hw_tests(state): v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj(dev=k) + test_obj = TestObj(dev=k, label=disk.name) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -705,7 +705,7 @@ def run_mprime_test(state, test): except KeyboardInterrupt: # Catch CTRL+C test.aborted = True - test.status = 'Aborted' + test.update_status('Aborted') print_warning('\nAborted.') update_progress_pane(state) @@ -757,23 +757,23 @@ def run_mprime_test(state, test): if log == 'results.txt': if re.search(r'(error|fail)', _data, re.IGNORECASE): test.failed = True - test.status = 'NS' + test.update_status('NS') # prime.log: CS check if log == 'prime.log': if re.search( r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): test.passed = True - test.status = 'CS' + test.update_status('CS') elif re.search( r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): # If the first re.search does not match and this one does then # that means that either errors or warnings, or both, are non-zero test.failed = True test.passed = False - test.status = 'NS' + test.update_status('NS') if not (test.aborted or test.failed or test.passed): - test.status = 'Unknown' + test.update_status('Unknown') # Done update_progress_pane(state) @@ -910,19 +910,23 @@ def update_io_progress(percent, rate, progress_file): def update_progress_pane(state): """Update progress file for side pane.""" output = [] - - # Prime95 - output.append(state.tests['Prime95']['Status']) - output.append(' ') - - # Disks for k, v in state.tests.items(): - if 'Prime95' not in k and v['Enabled']: - output.append('{BLUE}{test_name}{CLEAR}'.format( - test_name=k, **COLORS)) - for disk in state.disks: - output.append(disk.tests[k]['Status']) - output.append(' ') + # Skip disabled sections + if not v['Enabled']: + continue + + # Add section name + if k != 'Prime95': + output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) + if 'SMART' in k and state.quick_mode: + output.append(' {YELLOW}(Quick Check){CLEAR}'.format(**COLORS)) + + # Add status from test object(s) + for test in v['Objects']: + output.append(test.status) + + # Add spacer before next section + output.append(' ') # Add line-endings output = ['{}\n'.format(line) for line in output] From 8a8a63eb66c412b428ddb0bb9ccd1ae8241dbd49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:16:43 -0700 Subject: [PATCH 069/121] Build Prime95 report --- .bin/Scripts/functions/hw_diags.py | 83 +++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6d6b1a2c..181c3a4d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -264,7 +264,7 @@ class TestObj(): self.disabled = False self.failed = False self.passed = False - self.report = '' + self.report = [] self.started = False self.status = '' self.update_status() @@ -615,6 +615,7 @@ def run_hw_tests(state): f(state, test_obj) # Done + show_results(state) pause('Press Enter to return to main menu... ') # Cleanup @@ -738,42 +739,67 @@ def run_mprime_test(state, test): item.path, global_vars['LogDir'])) - # Check results + # Check results and build report test.logs = {} for log in ['results.txt', 'prime.log']: - _data = '' + lines = [] log_path = '{}/{}'.format(global_vars['LogDir'], log) # Read and save log try: with open(log_path, 'r') as f: - _data = f.read() - test.logs[log] = _data.splitlines() + lines = f.read().splitlines() + test.logs[log] = lines except FileNotFoundError: # Ignore since files may be missing for slower CPUs pass - # results.txt: NS check + # results.txt (NS check) if log == 'results.txt': - if re.search(r'(error|fail)', _data, re.IGNORECASE): - test.failed = True - test.update_status('NS') + _tmp = [] + for line in lines: + if re.search(r'(error|fail)', line, re.IGNORECASE): + test.failed = True + test.update_status('NS') + _tmp.append(' {YELLOW}{line}{CLEAR}'.format(**COLORS)) + if _tmp: + test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) + test.report.extend(_tmp) - # prime.log: CS check + # prime.log (CS check) if log == 'prime.log': - if re.search( - r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): - test.passed = True - test.update_status('CS') - elif re.search( - r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): - # If the first re.search does not match and this one does then - # that means that either errors or warnings, or both, are non-zero - test.failed = True - test.passed = False - test.update_status('NS') + _tmp_pass = [] + _tmp_warn = [] + for line in lines: + if re.search( + r'completed.*0 errors, 0 warnings', line, re.IGNORECASE): + _tmp_pass.append(line) + elif re.search( + r'completed.*\d+ errors, \d+ warnings', line, re.IGNORECASE): + # If the first re.search does not match and this one does then + # that means that either errors or warnings, or both, are non-zero + _tmp_warn.append(line) + if len(_tmp_warn) > 0: + test.failed = True + test.passed = False + test.update_status('NS') + elif len(_tmp_pass) > 0: + test.passed = True + test.update_status('CS') + if len(_tmp_pass) + len(_tmp_warn) > 0: + test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) + for line in _tmp_pass: + test.report.append(' {}'.format(line)) + for line in _tmp_warn: + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line, **COLORS)) + test.report.append(' ') + + # Finalize report if not (test.aborted or test.failed or test.passed): test.update_status('Unknown') + test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) + for line in generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'): + test.report.append(' {}'.format(line)) # Done update_progress_pane(state) @@ -782,10 +808,6 @@ def run_mprime_test(state, test): tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) test.monitor_proc.kill() - # TODO Testing - print('\n'.join( - generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'))) - def run_network_test(): """Run network test.""" clear_screen() @@ -840,6 +862,17 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_results(state): + """Show results for all tests.""" + for k, v in state.tests.items(): + print_success('{}:'.format(k)) + for obj in v['Objects']: + for line in obj.report: + print(line) + print_log(strip_colors(line)) + print_standard(' ') + print_standard(' ') + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 30d4acd9861672340443ecbbfc5614d434245a01 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:18:16 -0700 Subject: [PATCH 070/121] Added watch mode to respawn-pane --- .bin/Scripts/functions/tmux.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index e1066e18..e1892417 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -81,11 +81,14 @@ def tmux_split_window( result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): +def tmux_update_pane( + pane_id, command=None, + text=None, watch=None, + working_dir=None): """Respawn with either a new command or new text.""" # Bail early - if not command and not text: - raise Exception('Neither command nor text specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] if working_dir: @@ -94,6 +97,11 @@ def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): cmd.extend(command) elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) run_program(cmd) From 2b43cdf9e27d861406b20e6c35512471450c5ae2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:19:11 -0700 Subject: [PATCH 071/121] Create watch file if it doesn't exist yet --- .bin/Scripts/functions/tmux.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index e1892417..69b906d1 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -2,6 +2,12 @@ from functions.common import * +def create_file(filepath): + """Create file if it doesn't exist.""" + if not os.path.exists(filepath): + with open(filepath, 'w') as f: + f.write('') + def tmux_kill_all_panes(pane_id=None): """Kill all tmux panes except the active pane or pane_id if specified.""" cmd = ['tmux', 'kill-pane', '-a'] @@ -72,6 +78,7 @@ def tmux_split_window( elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: + create_file(watch) cmd.extend([ 'watch', '--color', '--no-title', '--interval', '1', @@ -98,6 +105,7 @@ def tmux_update_pane( elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: + create_file(watch) cmd.extend([ 'watch', '--color', '--no-title', '--interval', '1', From a2ef06e6db0004bb9cb23af0a351cffd33326e2d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:19:35 -0700 Subject: [PATCH 072/121] Added strip_colors() function --- .bin/Scripts/functions/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 2bf52a85..d87f772e 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -515,6 +515,12 @@ def stay_awake(): print_error('ERROR: No caffeine available.') print_warning('Please set the power setting to High Performance.') +def strip_colors(s): + """Remove all ASCII color escapes from string, returns str.""" + for c in COLORS.values(): + s = s.replace(c, '') + return s + def get_exception(s): """Get exception by name, returns Exception object.""" try: From d9554314d55ef845670887e3b019b39de449c343 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:42:10 -0700 Subject: [PATCH 073/121] Updated run_program() and popen_program() * Use dicts for clarity * Support cwd flag --- .bin/Scripts/functions/common.py | 36 ++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index d87f772e..7f14bdbd 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -405,19 +405,24 @@ def ping(addr='google.com'): def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): """Run program and return a subprocess.Popen object.""" - startupinfo=None + cmd_kwargs = {'args': cmd, 'shell': shell} + if minimized: startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = 6 + cmd_kwargs['startupinfo'] = startupinfo if pipe: - popen_obj = subprocess.Popen(cmd, shell=shell, startupinfo=startupinfo, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - popen_obj = subprocess.Popen(cmd, shell=shell, startupinfo=startupinfo) + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) - return popen_obj + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] + + return subprocess.Popen(**cmd_kwargs) def print_error(*args, **kwargs): """Prints message to screen in RED.""" @@ -456,7 +461,7 @@ def print_log(message='', end='\n', timestamp=True): line = line, end = end)) -def run_program(cmd, args=[], check=True, pipe=True, shell=False): +def run_program(cmd, args=[], check=True, pipe=True, shell=False, **kwargs): """Run program and return a subprocess.CompletedProcess object.""" if args: # Deprecated so let's raise an exception to find & fix all occurances @@ -466,13 +471,18 @@ def run_program(cmd, args=[], check=True, pipe=True, shell=False): if shell: cmd = ' '.join(cmd) - if pipe: - process_return = subprocess.run(cmd, check=check, shell=shell, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - process_return = subprocess.run(cmd, check=check, shell=shell) + cmd_kwargs = {'args': cmd, 'check': check, 'shell': shell} - return process_return + if pipe: + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) + + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] + + return subprocess.run(**cmd_kwargs) def set_title(title='~Some Title~'): """Set title. From 6c06a67fdf0dafb0b792fdbf48d0608e08a73651 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 22:54:56 -0700 Subject: [PATCH 074/121] Prime95 section complete --- .bin/Scripts/functions/hw_diags.py | 73 ++++++++++++++++++------------ .bin/Scripts/functions/sensors.py | 10 +++- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 181c3a4d..9cdde459 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -674,7 +674,9 @@ def run_mprime_test(state, test): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test.sensor_data, temp_label='Idle') + sensor_data=test.sensor_data, temp_label='Idle', + seconds=3) + # TODO: Remove seconds kwarg above # Stress CPU print_log('Starting Prime95') @@ -684,7 +686,9 @@ def run_mprime_test(state, test): state.panes['mprime'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) - time_limit = int(MPRIME_LIMIT) * 60 + #time_limit = int(MPRIME_LIMIT) * 60 + # TODO: restore above line + time_limit = 10 try: for i in range(time_limit): clear_screen() @@ -716,19 +720,23 @@ def run_mprime_test(state, test): pipe=True) # Stop Prime95 (twice for good measure) - tmux_kill_pane(state.panes['mprime']) run_program(['killall', '-s', 'INT', 'mprime'], check=False) + sleep(1) + tmux_kill_pane(state.panes['mprime']) # Get cooldown temp run_program(['apple-fans', 'auto']) clear_screen() try_and_print( message='Letting CPU cooldown for bit...', indent=0, - function=sleep, cs='Done', seconds=10) + function=sleep, cs='Done', seconds=3) + # TODO: Above seconds should be 10 try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test.sensor_data, temp_label='Cooldown') + sensor_data=test.sensor_data, temp_label='Cooldown', + seconds=3) + # TODO: Remove seconds kwarg above # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -761,44 +769,47 @@ def run_mprime_test(state, test): if re.search(r'(error|fail)', line, re.IGNORECASE): test.failed = True test.update_status('NS') - _tmp.append(' {YELLOW}{line}{CLEAR}'.format(**COLORS)) + _tmp.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) if _tmp: test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) test.report.extend(_tmp) # prime.log (CS check) if log == 'prime.log': - _tmp_pass = [] - _tmp_warn = [] + _tmp = {'Pass': {}, 'Warn': {}} for line in lines: - if re.search( - r'completed.*0 errors, 0 warnings', line, re.IGNORECASE): - _tmp_pass.append(line) - elif re.search( - r'completed.*\d+ errors, \d+ warnings', line, re.IGNORECASE): - # If the first re.search does not match and this one does then - # that means that either errors or warnings, or both, are non-zero - _tmp_warn.append(line) - if len(_tmp_warn) > 0: - test.failed = True - test.passed = False - test.update_status('NS') - elif len(_tmp_pass) > 0: - test.passed = True - test.update_status('CS') - if len(_tmp_pass) + len(_tmp_warn) > 0: + _r = re.search( + r'(completed.*(\d+) errors, (\d+) warnings)', + line, + re.IGNORECASE) + if _r: + if int(_r.group(2)) + int(_r.group(3)) > 0: + # Encountered errors and/or warnings + _tmp['Warn'][_r.group(1)] = None + else: + # No errors + _tmp['Pass'][_r.group(1)] = None + if len(_tmp['Warn']) > 0: + # NS + test.failed = True + test.passed = False + test.update_status('NS') + elif len(_tmp['Pass']) > 0: + test.passed = True + test.update_status('CS') + if len(_tmp['Pass']) + len(_tmp['Warn']) > 0: test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) - for line in _tmp_pass: + for line in sorted(_tmp['Pass'].keys()): test.report.append(' {}'.format(line)) - for line in _tmp_warn: - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line, **COLORS)) - test.report.append(' ') + for line in sorted(_tmp['Warn'].keys()): + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # Finalize report if not (test.aborted or test.failed or test.passed): test.update_status('Unknown') test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) - for line in generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'): + for line in generate_report( + test.sensor_data, 'Idle', 'Max', 'Cooldown', core_only=True): test.report.append(' {}'.format(line)) # Done @@ -864,6 +875,7 @@ def secret_screensaver(screensaver=None): def show_results(state): """Show results for all tests.""" + clear_screen() for k, v in state.tests.items(): print_success('{}:'.format(k)) for obj in v['Objects']: @@ -871,7 +883,8 @@ def show_results(state): print(line) print_log(strip_colors(line)) print_standard(' ') - print_standard(' ') + if 'Prime95' not in k: + print_standard(' ') def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 066dc446..b6319744 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -29,17 +29,22 @@ def fix_sensor_str(s): s = s.title() s = s.replace('Coretemp', 'CoreTemp') s = s.replace('Acpi', 'ACPI') + s = s.replace('ACPItz', 'ACPI TZ') s = s.replace('Isa ', 'ISA ') s = s.replace('Id ', 'ID ') s = re.sub(r'(\D+)(\d+)', r'\1 \2', s, re.IGNORECASE) s = s.replace(' ', ' ') return s -def generate_report(sensor_data, *temp_labels, colors=True): +def generate_report( + sensor_data, *temp_labels, + colors=True, core_only=False): """Generate report based on temp_labels, returns list if str.""" report = [] for _section, _adapters in sorted(sensor_data.items()): # CoreTemps then Other temps + if core_only and 'Core' not in _section: + continue for _adapter, _sources in sorted(_adapters.items()): # Adapter report.append(fix_sensor_str(_adapter)) @@ -53,7 +58,8 @@ def generate_report(sensor_data, *temp_labels, colors=True): ': ' if _label != 'Current' else '', get_temp_str(_data.get(_label, '???'), colors=colors)) report.append(_line) - report.append(' ') + if not core_only: + report.append(' ') # Handle empty reports (i.e. no sensors detected) if not report: From a3f7e5ad89372c922917662626a5d1421686cba7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 00:54:16 -0700 Subject: [PATCH 075/121] Disk quick check almost done --- .bin/Scripts/functions/hw_diags.py | 193 +++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 51 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9cdde459..da4d24e7 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -11,23 +11,23 @@ from functions.tmux import * # STATIC VARIABLES ATTRIBUTES = { 'NVMe': { - 'critical_warning': {'Error': 1}, - 'media_errors': {'Error': 1}, + 'critical_warning': {'Error': 1, 'Critical': True}, + 'media_errors': {'Error': 1, 'Critical': True}, 'power_on_hours': {'Warning': 12000, 'Error': 26298, 'Ignore': True}, 'unsafe_shutdowns': {'Warning': 1}, }, 'SMART': { - 5: {'Hex': '05', 'Error': 1}, - 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, - 10: {'Hex': '0A', 'Error': 1}, - 184: {'Hex': 'B8', 'Error': 1}, - 187: {'Hex': 'BB', 'Error': 1}, - 188: {'Hex': 'BC', 'Error': 1}, - 196: {'Hex': 'C4', 'Error': 1}, - 197: {'Hex': 'C5', 'Error': 1}, - 198: {'Hex': 'C6', 'Error': 1}, - 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - 201: {'Hex': 'C9', 'Error': 1}, + '5': {'Hex': '05', 'Error': 1, 'Critical': True}, + '9': {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + '10': {'Hex': '0A', 'Error': 1}, + '184': {'Hex': 'B8', 'Error': 1}, + '187': {'Hex': 'BB', 'Error': 1}, + '188': {'Hex': 'BC', 'Error': 1}, + '196': {'Hex': 'C4', 'Error': 1}, + '197': {'Hex': 'C5', 'Error': 1, 'Critical': True}, + '198': {'Hex': 'C6', 'Error': 1, 'Critical': True}, + '199': {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + '201': {'Hex': 'C9', 'Error': 1}, }, } IO_VARS = { @@ -284,6 +284,82 @@ class TestObj(): self.label, 'Working', self.info_label) # Functions +def attributes_ok_nvme(disk): + """Check NVMe attributes for errors, returns bool.""" + disk_ok = True + override_disabled = False + for k, v in disk.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + if 'Error' not in ATTRIBUTES['NVMe'][k]: + # Only worried about error thresholds + continue + if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: + disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['NVMe'][k].get( + 'Critical', False) + + # Print errors + if not disk_ok: + show_disk_attributes(disk) + if override_disabled: + print_error('NVMe error(s) detected.') + print_standard('Tests disabled for this device') + pause() + else: + print_warning('NVMe error(s) detected.') + disk_ok = ask('Run tests on this device anyway?') + + return disk_ok + +def attributes_ok_smart(disk): + """Check SMART attributes for errors, returns bool.""" + disk_ok = True + override_disabled = False + smart_overall_pass = True + for k, v in disk.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + if 'Error' not in ATTRIBUTES['SMART'][k]: + # Only worried about error thresholds + continue + if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: + disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['SMART'][k].get( + 'Critical', False) + + # SMART overall assessment + if not disk.smartctl.get('smart_status', {}).get('passed', False): + smart_overall_pass = False + disk_ok = False + override_disabled = True + + # Print errors + if not disk_ok: + show_disk_attributes(disk) + + # 199/C7 warning + if disk.smart_attributes.get('199', {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the drive cable?)') + + # Override? + if not smart_overall_pass: + print_error('SMART overall self-assessment: Failed') + print_standard('Tests disabled for this device') + pause() + elif override_disabled: + print_error('SMART error(s) detected.') + print_standard('Tests disabled for this device') + pause() + else: + print_warning('SMART error(s) detected.') + disk_ok = ask('Run tests on this device anyway?') + + return disk_ok + def build_outer_panes(state): """Build top and side panes.""" clear_screen() @@ -310,7 +386,7 @@ def build_status_string(label, status, info_label=False): status_color = COLORS['CLEAR'] if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: status_color = COLORS['RED'] - elif status in ['Aborted', 'Unknown', 'Working', 'Skipped']: + elif status in ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working']: status_color = COLORS['YELLOW'] elif status in ['CS']: status_color = COLORS['GREEN'] @@ -596,7 +672,7 @@ def run_hw_tests(state): v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj(dev=k, label=disk.name) + test_obj = TestObj(dev=disk, label=disk.name) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -616,7 +692,10 @@ def run_hw_tests(state): # Done show_results(state) - pause('Press Enter to return to main menu... ') + if '--quick' in sys.argv: + pause('Press Enter to exit...') + else: + pause('Press Enter to return to main menu... ') # Cleanup tmux_kill_pane(*state.panes.values()) @@ -826,42 +905,32 @@ def run_network_test(): pause('Press Enter to return to main menu... ') def run_nvme_smart_tests(state, test): - """TODO""" - for disk in state.disks: - tmux_update_pane( - state.panes['Top'], - text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **disk.lsblk)) - disk.tests['NVMe / SMART']['Started'] = True - update_progress_pane(state) - if disk.nvme_attributes: - run_nvme_tests(state, disk) - elif disk.smart_attributes: - run_smart_tests(state, disk) + """Run NVMe or SMART test for test.dev.""" + tmux_update_pane( + state.panes['Top'], + text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( + t=TOP_PANE_TEXT, **test.dev.lsblk)) + if test.dev.nvme_attributes: + if attributes_ok_nvme(test.dev): + test.passed = True + test.update_status('CS') else: - print_standard('TODO: run_nvme_smart_tests({})'.format( - disk.path)) - print_warning( - " WARNING: Device {} doesn't support NVMe or SMART test".format( - disk.path)) - disk.tests['NVMe / SMART']['Status'] = 'N/A' - disk.tests['NVMe / SMART']['Result'] = 'N/A' - update_progress_pane(state) - sleep(3) - -def run_nvme_tests(state, disk): - """TODO""" - print_standard('TODO: run_nvme_test({})'.format(disk.path)) + test.failed = True + test.update_status('NS') + elif test.dev.smart_attributes: + if attributes_ok_smart(test.dev): + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') + else: + print_standard('Tests disabled for this device') + test.update_status('N/A') + if not ask('Run tests on this device anyway?'): + test.failed = True + update_progress_pane(state) sleep(3) - disk.tests['NVMe / SMART']['Result'] = 'CS' - update_progress_pane(state) - -def run_smart_tests(state, disk): - """TODO""" - print_standard('TODO: run_smart_tests({})'.format(disk.path)) - sleep(3) - disk.tests['NVMe / SMART']['Result'] = 'CS' - update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" @@ -873,10 +942,32 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_disk_attributes(disk): + """Show NVMe/SMART attributes for disk.""" + print_info('Device: {}'.format(disk.path)) + print_standard(' {size:6} ({tran}) {model} {serial}'.format(**disk.lsblk)) + print_info('Attributes') + if disk.nvme_attributes: + for k, v in disk.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + print('TODO: {} {}'.format(k, v)) + elif disk.smart_attributes: + for k, v in disk.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + print('TODO: {} {}'.format(k, v)) + else: + print_warning(' No NVMe or SMART data available') + def show_results(state): """Show results for all tests.""" clear_screen() + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'Results')) for k, v in state.tests.items(): + # Skip disabled tests + if not v['Enabled']: + continue print_success('{}:'.format(k)) for obj in v['Objects']: for line in obj.report: @@ -965,7 +1056,7 @@ def update_progress_pane(state): if k != 'Prime95': output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) if 'SMART' in k and state.quick_mode: - output.append(' {YELLOW}(Quick Check){CLEAR}'.format(**COLORS)) + output[-1] += ' {YELLOW}(Quick){CLEAR}'.format(**COLORS) # Add status from test object(s) for test in v['Objects']: From a967a5c425122f7d2bf1998f662e035e55b57865 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 20:40:57 -0700 Subject: [PATCH 076/121] Switched back to int keys for SMART attributes * Allows for easier sorting --- .bin/Scripts/functions/hw_diags.py | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index da4d24e7..2c2d63fa 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -17,17 +17,19 @@ ATTRIBUTES = { 'unsafe_shutdowns': {'Warning': 1}, }, 'SMART': { - '5': {'Hex': '05', 'Error': 1, 'Critical': True}, - '9': {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, - '10': {'Hex': '0A', 'Error': 1}, - '184': {'Hex': 'B8', 'Error': 1}, - '187': {'Hex': 'BB', 'Error': 1}, - '188': {'Hex': 'BC', 'Error': 1}, - '196': {'Hex': 'C4', 'Error': 1}, - '197': {'Hex': 'C5', 'Error': 1, 'Critical': True}, - '198': {'Hex': 'C6', 'Error': 1, 'Critical': True}, - '199': {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - '201': {'Hex': 'C9', 'Error': 1}, + 5: {'Hex': '05', 'Error': 1, 'Critical': True}, + 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 10: {'Hex': '0A', 'Error': 1}, + 184: {'Hex': 'B8', 'Error': 1}, + 187: {'Hex': 'BB', 'Error': 1}, + 188: {'Hex': 'BC', 'Error': 1}, + 196: {'Hex': 'C4', 'Error': 1}, + 197: {'Hex': 'C5', 'Error': 1, 'Critical': True}, + 198: {'Hex': 'C6', 'Error': 1, 'Critical': True}, + 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + 201: {'Hex': 'C9', 'Error': 1}, + # TODO: Delete below + 177: {'Hex': 'FF', 'Error': 1}, }, } IO_VARS = { @@ -161,19 +163,21 @@ class DiskObj(): self.nvme_attributes.update(self.smartctl[KEY_NVME]) elif KEY_SMART in self.smartctl: for a in self.smartctl[KEY_SMART].get('table', {}): - _id = str(a.get('id', 'UNKNOWN')) + try: + _id = int(a.get('id', -1)) + except ValueError: + # Ignoring invalid attribute + continue _name = str(a.get('name', 'UNKNOWN')) - _raw = a.get('raw', {}).get('value', -1) + _raw = int(a.get('raw', {}).get('value', -1)) _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') # Fix power-on time _r = re.match(r'^(\d+)[Hh].*', _raw_str) - if _id == '9' and _r: - try: - _raw = int(_r.group(1)) - except ValueError: - # That's fine - pass + if _id == 9 and _r: + _raw = int(_r.group(1)) + + # Add to dict self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} @@ -1056,7 +1060,7 @@ def update_progress_pane(state): if k != 'Prime95': output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) if 'SMART' in k and state.quick_mode: - output[-1] += ' {YELLOW}(Quick){CLEAR}'.format(**COLORS) + output[-1] += ' {}'.format(QUICK_LABEL) # Add status from test object(s) for test in v['Objects']: From 62a60ff3fd0ff1e24181ebb60d66b9cbfc38f02e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 22:56:09 -0700 Subject: [PATCH 077/121] Reworked disk safety checks * Moved several functions into DiskObj * Added HW_OVERRIDES_FORCED and HW_OVERRIDES_LIMITED to main.py * These adjust when overrides are requested * Disable badblocks and/or io_benchmark if disk fails safety check --- .bin/Scripts/functions/hw_diags.py | 220 ++++++++++++++++------------- .bin/Scripts/settings/main.py | 2 + 2 files changed, 122 insertions(+), 100 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 2c2d63fa..63003e8e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -28,10 +28,9 @@ ATTRIBUTES = { 198: {'Hex': 'C6', 'Error': 1, 'Critical': True}, 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, 201: {'Hex': 'C9', 'Error': 1}, - # TODO: Delete below - 177: {'Hex': 'FF', 'Error': 1}, }, } +HW_OVERRIDES_FORCED = HW_OVERRIDES_FORCED and not HW_OVERRIDES_LIMITED IO_VARS = { 'Block Size': 512*1024, 'Chunk Size': 32*1024**2, @@ -106,6 +105,7 @@ class CpuObj(): class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): + self.disk_ok = True self.labels = [] self.lsblk = {} self.name = re.sub(r'^.*/(.*)', r'\1', disk_path) @@ -181,10 +181,121 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def safety_check(self): - """Check enabled tests and verify it's safe to run them.""" - # TODO - pass + def nvme_check(self, silent=False): + """Check NVMe attributes for errors.""" + override_disabled = False + for k, v in self.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + if 'Error' not in ATTRIBUTES['NVMe'][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES['NVMe'][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['NVMe'][k].get( + 'Critical', False) + + # Print errors + if not self.disk_ok and not silent: + self.show_attributes() + print_warning('NVMe error(s) detected.') + + # Override? + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + + def safety_check(self, silent=False): + """Check attributes and disable tests if necessary.""" + if self.nvme_attributes: + self.nvme_check(silent) + elif self.smart_attributes: + self.smart_check(silent) + else: + # No NVMe/SMART details + if silent: + self.disk_ok = HW_OVERRIDES_FORCED + else: + print_warning( + ' WARNING: No NVMe or SMART attributes available for: {}'.format( + self.path)) + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + + if not self.disk_ok: + for t in ['badblocks', 'I/O Benchmark']: + if t in self.tests: + self.tests[t].disabled = True + self.tests[t].update_status('Denied') + + def show_attributes(self): + """Show NVMe/SMART attributes.""" + print_info('Device: {}'.format(self.path)) + print_standard( + ' {size:>6} ({tran}) {model} {serial}'.format(**self.lsblk)) + print_info('Attributes') + if self.nvme_attributes: + for k, v in self.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + print('TODO: {} {}'.format(k, v)) + elif self.smart_attributes: + for k, v in self.smart_attributes.items(): + # TODO: If k == 199/C7 then append ' (bad cable?)' to line + if k in ATTRIBUTES['SMART']: + print('TODO: {} {}'.format(k, v)) + if not self.smartctl.get('smart_status', {}).get('passed', True): + print_error('SMART overall self-assessment: Failed') + else: + print_warning(' No NVMe or SMART data available') + + def smart_check(self, silent=False): + """Check SMART attributes for errors.""" + override_disabled = False + for k, v in self.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + if 'Error' not in ATTRIBUTES['SMART'][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES['SMART'][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['SMART'][k].get( + 'Critical', False) + + # SMART overall assessment + ## NOTE: Only fail drives if the overall value exists and reports failed + if not self.smartctl.get('smart_status', {}).get('passed', True): + self.disk_ok = False + override_disabled = True + + # Print errors + if not silent: + if self.disk_ok: + # 199/C7 warning + if self.smart_attributes.get(199, {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the disk cable?)') + else: + # Override? + self.show_attributes() + print_warning('SMART error(s) detected.') + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') class State(): """Object to track device objects and overall state.""" @@ -288,82 +399,6 @@ class TestObj(): self.label, 'Working', self.info_label) # Functions -def attributes_ok_nvme(disk): - """Check NVMe attributes for errors, returns bool.""" - disk_ok = True - override_disabled = False - for k, v in disk.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - if 'Error' not in ATTRIBUTES['NVMe'][k]: - # Only worried about error thresholds - continue - if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: - disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['NVMe'][k].get( - 'Critical', False) - - # Print errors - if not disk_ok: - show_disk_attributes(disk) - if override_disabled: - print_error('NVMe error(s) detected.') - print_standard('Tests disabled for this device') - pause() - else: - print_warning('NVMe error(s) detected.') - disk_ok = ask('Run tests on this device anyway?') - - return disk_ok - -def attributes_ok_smart(disk): - """Check SMART attributes for errors, returns bool.""" - disk_ok = True - override_disabled = False - smart_overall_pass = True - for k, v in disk.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - if 'Error' not in ATTRIBUTES['SMART'][k]: - # Only worried about error thresholds - continue - if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: - disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['SMART'][k].get( - 'Critical', False) - - # SMART overall assessment - if not disk.smartctl.get('smart_status', {}).get('passed', False): - smart_overall_pass = False - disk_ok = False - override_disabled = True - - # Print errors - if not disk_ok: - show_disk_attributes(disk) - - # 199/C7 warning - if disk.smart_attributes.get('199', {}).get('raw', 0) > 0: - print_warning('199/C7 error detected') - print_standard(' (Have you tried swapping the drive cable?)') - - # Override? - if not smart_overall_pass: - print_error('SMART overall self-assessment: Failed') - print_standard('Tests disabled for this device') - pause() - elif override_disabled: - print_error('SMART error(s) detected.') - print_standard('Tests disabled for this device') - pause() - else: - print_warning('SMART error(s) detected.') - disk_ok = ask('Run tests on this device anyway?') - - return disk_ok - def build_outer_panes(state): """Build top and side panes.""" clear_screen() @@ -915,14 +950,15 @@ def run_nvme_smart_tests(state, test): text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **test.dev.lsblk)) if test.dev.nvme_attributes: - if attributes_ok_nvme(test.dev): + # NOTE: Pass/Fail is just the attribute check + if test.dev.disk_ok: test.passed = True test.update_status('CS') else: test.failed = True test.update_status('NS') elif test.dev.smart_attributes: - if attributes_ok_smart(test.dev): + if test.dev.disk_ok: test.passed = True test.update_status('CS') else: @@ -946,22 +982,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def show_disk_attributes(disk): - """Show NVMe/SMART attributes for disk.""" - print_info('Device: {}'.format(disk.path)) - print_standard(' {size:6} ({tran}) {model} {serial}'.format(**disk.lsblk)) - print_info('Attributes') - if disk.nvme_attributes: - for k, v in disk.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - print('TODO: {} {}'.format(k, v)) - elif disk.smart_attributes: - for k, v in disk.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - print('TODO: {} {}'.format(k, v)) - else: - print_warning(' No NVMe or SMART data available') - def show_results(state): """Show results for all tests.""" clear_screen() diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 75fef0fd..7b915bdb 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -4,6 +4,8 @@ ENABLED_OPEN_LOGS = False ENABLED_TICKET_NUMBERS = False ENABLED_UPLOAD_DATA = False +HW_OVERRIDES_FORCED = False +HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED # STATIC VARIABLES (also used by BASH and BATCH files) ## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH From 47084efe1725aae5cdd302735d3f84097097d28d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 23:18:51 -0700 Subject: [PATCH 078/121] Combined nvme_check() and smart_check() --- .bin/Scripts/functions/hw_diags.py | 129 ++++++++++++----------------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 63003e8e..dd2bfa4f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -117,6 +117,54 @@ class DiskObj(): self.get_details() self.get_smart_details() + def check_attributes(self, silent=False): + """Check NVMe / SMART attributes for errors.""" + override_disabled = False + if self.nvme_attributes: + attr_type = 'NVMe' + items = self.nvme_attributes.items() + elif self.smart_attributes: + attr_type = 'SMART' + items = self.smar_attributes.items() + for k, v in items: + if k in ATTRIBUTES[attr_type]: + if 'Error' not in ATTRIBUTES[attr_type][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES[attr_type][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES[attr_type][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES[attr_type][k].get( + 'Critical', False) + + # SMART overall assessment + ## NOTE: Only fail drives if the overall value exists and reports failed + if not self.smartctl.get('smart_status', {}).get('passed', True): + self.disk_ok = False + override_disabled = True + + # Print errors + if not silent: + if self.disk_ok: + # 199/C7 warning + if self.smart_attributes.get(199, {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the disk cable?)') + else: + # Override? + self.show_attributes() + print_warning('{} error(s) detected.'.format(attr_type)) + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -181,43 +229,10 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def nvme_check(self, silent=False): - """Check NVMe attributes for errors.""" - override_disabled = False - for k, v in self.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - if 'Error' not in ATTRIBUTES['NVMe'][k]: - # Only worried about error thresholds - continue - if ATTRIBUTES['NVMe'][k].get('Ignore', False): - # Attribute is non-failing, skip - continue - if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: - self.disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['NVMe'][k].get( - 'Critical', False) - - # Print errors - if not self.disk_ok and not silent: - self.show_attributes() - print_warning('NVMe error(s) detected.') - - # Override? - if override_disabled: - print_standard('Tests disabled for this device') - pause() - elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') - def safety_check(self, silent=False): - """Check attributes and disable tests if necessary.""" - if self.nvme_attributes: - self.nvme_check(silent) - elif self.smart_attributes: - self.smart_check(silent) + """Run safety checks and disable tests if necessary.""" + if self.nvme_attributes or self.smart_attributes: + self.check_attributes(silent) else: # No NVMe/SMART details if silent: @@ -255,48 +270,6 @@ class DiskObj(): else: print_warning(' No NVMe or SMART data available') - def smart_check(self, silent=False): - """Check SMART attributes for errors.""" - override_disabled = False - for k, v in self.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - if 'Error' not in ATTRIBUTES['SMART'][k]: - # Only worried about error thresholds - continue - if ATTRIBUTES['SMART'][k].get('Ignore', False): - # Attribute is non-failing, skip - continue - if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: - self.disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['SMART'][k].get( - 'Critical', False) - - # SMART overall assessment - ## NOTE: Only fail drives if the overall value exists and reports failed - if not self.smartctl.get('smart_status', {}).get('passed', True): - self.disk_ok = False - override_disabled = True - - # Print errors - if not silent: - if self.disk_ok: - # 199/C7 warning - if self.smart_attributes.get(199, {}).get('raw', 0) > 0: - print_warning('199/C7 error detected') - print_standard(' (Have you tried swapping the disk cable?)') - else: - # Override? - self.show_attributes() - print_warning('SMART error(s) detected.') - if override_disabled: - print_standard('Tests disabled for this device') - pause() - elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') - class State(): """Object to track device objects and overall state.""" def __init__(self): From b5c93317dc33716c813927097da0838a2a8ecc1b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 23:54:02 -0700 Subject: [PATCH 079/121] Override sections working --- .bin/Scripts/functions/hw_diags.py | 66 +++++++++++++++++++----------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index dd2bfa4f..345cb279 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -125,7 +125,7 @@ class DiskObj(): items = self.nvme_attributes.items() elif self.smart_attributes: attr_type = 'SMART' - items = self.smar_attributes.items() + items = self.smart_attributes.items() for k, v in items: if k in ATTRIBUTES[attr_type]: if 'Error' not in ATTRIBUTES[attr_type][k]: @@ -359,8 +359,6 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled: - return if new_status: self.status = build_status_string( self.label, new_status, self.info_label) @@ -650,12 +648,12 @@ def run_badblocks_test(state, test): TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test({})'.format( test.dev.path)) - for disk in state.disks: - disk.tests['badblocks']['Started'] = True - update_progress_pane(state) - sleep(3) - disk.tests['badblocks']['Result'] = 'OVERRIDE' - update_progress_pane(state) + test.started = True + test.update_status() + update_progress_pane(state) + sleep(3) + test.update_status('Unknown') + update_progress_pane(state) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -691,7 +689,7 @@ def run_hw_tests(state): # Run safety checks for disk in state.disks: - disk.safety_check() + disk.safety_check(silent=state.quick_mode) # Run tests ## Because state.tests is an OrderedDict and the disks were added @@ -704,7 +702,7 @@ def run_hw_tests(state): # Done show_results(state) - if '--quick' in sys.argv: + if state.quick_mode: pause('Press Enter to exit...') else: pause('Press Enter to return to main menu... ') @@ -719,12 +717,12 @@ def run_io_benchmark(state, test): TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark({})'.format( test.dev.path)) - for disk in state.disks: - disk.tests['I/O Benchmark']['Started'] = True - update_progress_pane(state) - sleep(3) - disk.tests['I/O Benchmark']['Result'] = 'Unknown' - update_progress_pane(state) + test.started = True + test.update_status() + update_progress_pane(state) + sleep(3) + test.update_status('Unknown') + update_progress_pane(state) def run_keyboard_test(): """Run keyboard test.""" @@ -928,22 +926,42 @@ def run_nvme_smart_tests(state, test): test.passed = True test.update_status('CS') else: + # NOTE: Other test(s) should've been disabled by DiskObj.safety_check() test.failed = True test.update_status('NS') elif test.dev.smart_attributes: + # NOTE: Pass/Fail based on both attributes and SMART short self-test if test.dev.disk_ok: - test.passed = True - test.update_status('CS') + # Run short test + pause('TODO: Run SMART short self-test') + + # Check result + # TODO + short_test_passed = True + if short_test_passed: + test.passed = True + test.update_status('CS') + else: + for t in ['badblocks', 'I/O Benchmark']: + if t in test.dev.tests: + test.dev.tests[t].disabled = True + test.dev.tests[t].update_status('Denied') + # TODO + if no_logs: + test.update_status('Unknown') + else: + test.failed = True + test.update_status('NS') else: test.failed = True test.update_status('NS') else: - print_standard('Tests disabled for this device') + # NOTE: Pass/Fail not applicable without NVMe/SMART data + # Override request earlier disabled other test(s) as appropriate test.update_status('N/A') - if not ask('Run tests on this device anyway?'): - test.failed = True - update_progress_pane(state) - sleep(3) + + # Done + update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" From 5b748798053a97c200998286d1b67fcd64e70017 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 13 Dec 2018 19:02:28 -0700 Subject: [PATCH 080/121] Fixed OVERRIDE and N/A NVMe/SMART status handling --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 345cb279..d967ae15 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -162,8 +162,11 @@ class DiskObj(): print_standard('Tests disabled for this device') pause() elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') + if HW_OVERRIDES_FORCED or ask('Run tests on this device anyway?'): + self.disk_ok = True + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].update_status('OVERRIDE') + self.tests['NVMe / SMART'].disabled = True def get_details(self): """Get data from lsblk.""" @@ -235,6 +238,9 @@ class DiskObj(): self.check_attributes(silent) else: # No NVMe/SMART details + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].update_status('N/A') + self.tests['NVMe / SMART'].disabled = True if silent: self.disk_ok = HW_OVERRIDES_FORCED else: @@ -247,8 +253,8 @@ class DiskObj(): if not self.disk_ok: for t in ['badblocks', 'I/O Benchmark']: if t in self.tests: - self.tests[t].disabled = True self.tests[t].update_status('Denied') + self.tests[t].disabled = True def show_attributes(self): """Show NVMe/SMART attributes.""" @@ -359,6 +365,8 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" + if self.disabled: + return if new_status: self.status = build_status_string( self.label, new_status, self.info_label) @@ -944,8 +952,8 @@ def run_nvme_smart_tests(state, test): else: for t in ['badblocks', 'I/O Benchmark']: if t in test.dev.tests: - test.dev.tests[t].disabled = True test.dev.tests[t].update_status('Denied') + test.dev.tests[t].disabled = True # TODO if no_logs: test.update_status('Unknown') @@ -955,10 +963,6 @@ def run_nvme_smart_tests(state, test): else: test.failed = True test.update_status('NS') - else: - # NOTE: Pass/Fail not applicable without NVMe/SMART data - # Override request earlier disabled other test(s) as appropriate - test.update_status('N/A') # Done update_progress_pane(state) From 81f05fa79f3217cc37e000f4a9998278907508d4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 16:37:14 -0700 Subject: [PATCH 081/121] Replaced show_attributes() with generate_report() * Returns list of colored strings * Optionally includes short-test results * Optionally excludes disk info --- .bin/Scripts/functions/hw_diags.py | 157 ++++++++++++++++++++++++----- 1 file changed, 133 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d967ae15..7c40c741 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -112,6 +112,7 @@ class DiskObj(): self.nvme_attributes = {} self.path = disk_path self.smart_attributes = {} + self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() self.get_details() @@ -156,7 +157,9 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - self.show_attributes() + for line in self.generate_report(): + print(line) + print_log(strip_colors(line)) print_warning('{} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') @@ -168,6 +171,93 @@ class DiskObj(): self.tests['NVMe / SMART'].update_status('OVERRIDE') self.tests['NVMe / SMART'].disabled = True + def generate_report(self, brief=False, short_test=False): + """Generate NVMe / SMART report, returns list.""" + report = [] + if not brief: + report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( + dev_path=self.path, **COLORS)) + report.append(' {size:>6} ({tran}) {model} {serial}'.format( + **self.lsblk)) + + # Warnings + if self.nvme_attributes: + attr_type = 'NVMe' + report.append( + ' {YELLOW}NVMe disk support is still experimental{CLEAR}'.format( + **COLORS)) + elif self.smart_attributes: + attr_type = 'SMART' + else: + # No attribute data available, return short report + report.append( + ' {YELLOW}No NVMe or SMART data available{CLEAR}'.format( + **COLORS)) + return report + if not self.smartctl.get('smart_status', {}).get('passed', True): + report.append( + ' {RED}SMART overall self-assessment: Failed{CLEAR}'.format( + **COLORS)) + + # Attributes + report.append('{BLUE}{a} Attributes{YELLOW}{u:>23} {t}{CLEAR}'.format( + a=attr_type, + u='Updated:' if brief else '', + t=time.strftime('%Y-%m-%d %H:%M %Z') if brief else '', + **COLORS)) + if self.nvme_attributes: + attr_type = 'NVMe' + items = self.nvme_attributes.items() + elif self.smart_attributes: + attr_type = 'SMART' + items = self.smart_attributes.items() + for k, v in items: + if k in ATTRIBUTES[attr_type]: + _note = '' + _color = COLORS['GREEN'] + + # Attribute ID & Name + if attr_type == 'NVMe': + _line = ' {:38}'.format(k.replace('_', ' ').title()) + else: + _line = ' {i:>3} / {h}: {n:28}'.format( + i=k, + h=ATTRIBUTES[attr_type][k]['Hex'], + n=v['name'][:28]) + + # Set color + for _t, _c in [['Warning', 'YELLOW'], ['Error', 'RED']]: + if _t in ATTRIBUTES[attr_type][k]: + if v['raw'] >= ATTRIBUTES[attr_type][k][_t]: + _color = COLORS[_c] + + # 199/C7 warning + if str(k) == '199': + _note = '(bad cable?)' + + # Attribute value + _line += '{}{} {}{}'.format( + _color, + v['raw_str'], + _note, + COLORS['CLEAR']) + + # Add line to report + report.append(_line) + + # SMART short-test + if short_test: + report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) + if 'TimedOut' in self.tests['NVMe / SMART'].status: + report.append(' {YELLOW}UNKNOWN{CLEAR}: Timed out'.format(**COLORS)) + else: + report.append(' {}'.format( + self.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) + + # Done + return report + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -219,7 +309,7 @@ class DiskObj(): except ValueError: # Ignoring invalid attribute continue - _name = str(a.get('name', 'UNKNOWN')) + _name = str(a.get('name', 'UNKNOWN')).replace('_', ' ').title() _raw = int(a.get('raw', {}).get('value', -1)) _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') @@ -232,6 +322,13 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + # Self-test data + for k in ['polling_minutes', 'status']: + self.smart_self_test[k] = self.smartctl.get( + 'ata_smart_data', {}).get( + 'self_test', {}).get( + k, {}) + def safety_check(self, silent=False): """Run safety checks and disable tests if necessary.""" if self.nvme_attributes or self.smart_attributes: @@ -251,31 +348,15 @@ class DiskObj(): 'Run tests on this device anyway?') if not self.disk_ok: + if 'NVMe / SMART' in self.tests: + # NOTE: This will not overwrite the existing status if set + self.tests['NVMe / SMART'].update_status('NS') + self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: if t in self.tests: self.tests[t].update_status('Denied') self.tests[t].disabled = True - def show_attributes(self): - """Show NVMe/SMART attributes.""" - print_info('Device: {}'.format(self.path)) - print_standard( - ' {size:>6} ({tran}) {model} {serial}'.format(**self.lsblk)) - print_info('Attributes') - if self.nvme_attributes: - for k, v in self.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - print('TODO: {} {}'.format(k, v)) - elif self.smart_attributes: - for k, v in self.smart_attributes.items(): - # TODO: If k == 199/C7 then append ' (bad cable?)' to line - if k in ATTRIBUTES['SMART']: - print('TODO: {} {}'.format(k, v)) - if not self.smartctl.get('smart_status', {}).get('passed', True): - print_error('SMART overall self-assessment: Failed') - else: - print_warning(' No NVMe or SMART data available') - class State(): """Object to track device objects and overall state.""" def __init__(self): @@ -402,7 +483,7 @@ def build_outer_panes(state): def build_status_string(label, status, info_label=False): """Build status string with appropriate colors.""" status_color = COLORS['CLEAR'] - if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE', 'TimedOut']: status_color = COLORS['RED'] elif status in ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working']: status_color = COLORS['YELLOW'] @@ -651,6 +732,9 @@ def run_audio_test(): def run_badblocks_test(state, test): """TODO""" + # Bail early + if test.disabled: + return tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) @@ -699,6 +783,15 @@ def run_hw_tests(state): for disk in state.disks: disk.safety_check(silent=state.quick_mode) + # TODO Remove + clear_screen() + print_info('Running tests:') + for k, v in state.tests.items(): + if v['Enabled']: + print_standard(' {}'.format(k)) + update_progress_pane(state) + pause() + # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. @@ -720,6 +813,9 @@ def run_hw_tests(state): def run_io_benchmark(state, test): """TODO""" + # Bail early + if test.disabled: + return tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) @@ -739,6 +835,9 @@ def run_keyboard_test(): def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" + # Bail early + if test.disabled: + return test.started = True test.update_status() update_progress_pane(state) @@ -924,6 +1023,7 @@ def run_network_test(): def run_nvme_smart_tests(state, test): """Run NVMe or SMART test for test.dev.""" + _include_short_test = False tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( @@ -941,10 +1041,14 @@ def run_nvme_smart_tests(state, test): # NOTE: Pass/Fail based on both attributes and SMART short self-test if test.dev.disk_ok: # Run short test - pause('TODO: Run SMART short self-test') + # TODO + _include_short_test = True + _timeout = test.dev.smart_self_test['polling_minutes'].get('short', 5) + _timeout = int(_timeout) + 5 # Check result # TODO + # if 'remaining_percent' in 'status' then we've started. short_test_passed = True if short_test_passed: test.passed = True @@ -960,10 +1064,15 @@ def run_nvme_smart_tests(state, test): else: test.failed = True test.update_status('NS') + else: test.failed = True test.update_status('NS') + # Save report + test.report = test.dev.generate_report( + short_test=_include_short_test) + # Done update_progress_pane(state) From cee825245505376a757a3de5eda45cc7eb1deb95 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:03:00 -0700 Subject: [PATCH 082/121] Added CYAN to COLORS --- .bin/Scripts/functions/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 7f14bdbd..5327d895 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -27,10 +27,11 @@ global_vars = {} COLORS = { 'CLEAR': '\033[0m', 'RED': '\033[31m', + 'ORANGE': '\033[31;1m', 'GREEN': '\033[32m', 'YELLOW': '\033[33m', - 'ORANGE': '\033[31;1m', - 'BLUE': '\033[34m' + 'BLUE': '\033[34m', + 'CYAN': '\033[36m', } try: HKU = winreg.HKEY_USERS From 99984603ed601e1af78e688b2258d20ab9a7b7ef Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:32:17 -0700 Subject: [PATCH 083/121] NVMe/SMART sections working * Added timout status for clarity * Added short-test result to report --- .bin/Scripts/functions/hw_diags.py | 123 +++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 7c40c741..141d324e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -169,7 +169,9 @@ class DiskObj(): self.disk_ok = True if 'NVMe / SMART' in self.tests: self.tests['NVMe / SMART'].update_status('OVERRIDE') - self.tests['NVMe / SMART'].disabled = True + if self.nvme_attributes or not self.smart_attributes: + # i.e. only leave enabled for SMART short-tests + self.tests['NVMe / SMART'].disabled = True def generate_report(self, brief=False, short_test=False): """Generate NVMe / SMART report, returns list.""" @@ -323,6 +325,7 @@ class DiskObj(): 'name': _name, 'raw': _raw, 'raw_str': _raw_str} # Self-test data + self.smart_self_test = {} for k in ['polling_minutes', 'status']: self.smart_self_test[k] = self.smartctl.get( 'ata_smart_data', {}).get( @@ -333,6 +336,22 @@ class DiskObj(): """Run safety checks and disable tests if necessary.""" if self.nvme_attributes or self.smart_attributes: self.check_attributes(silent) + + # Check if a self-test is currently running + if 'remaining_percent' in self.smart_self_test['status']: + _msg='SMART self-test in progress, all tests disabled' + if not silent: + print_warning('WARNING: {}'.format(_msg)) + print_standard(' ') + if ask('Abort HW Diagnostics?'): + exit_script() + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report.append( + '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) + for t in self.tests.values(): + t.update_status('Denied') + t.disabled = True else: # No NVMe/SMART details if 'NVMe / SMART' in self.tests: @@ -350,6 +369,8 @@ class DiskObj(): if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set + if not self.tests['NVMe / SMART'].report: + self.tests['NVMe / SMART'].report = self.generate_report() self.tests['NVMe / SMART'].update_status('NS') self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: @@ -446,7 +467,7 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled: + if self.disabled or 'OVERRIDE' in self.status: return if new_status: self.status = build_status_string( @@ -1023,11 +1044,19 @@ def run_network_test(): def run_nvme_smart_tests(state, test): """Run NVMe or SMART test for test.dev.""" + # Bail early + if test.disabled: + return _include_short_test = False + test.started = True + test.update_status() tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **test.dev.lsblk)) + update_progress_pane(state) + + # NVMe if test.dev.nvme_attributes: # NOTE: Pass/Fail is just the attribute check if test.dev.disk_ok: @@ -1037,37 +1066,80 @@ def run_nvme_smart_tests(state, test): # NOTE: Other test(s) should've been disabled by DiskObj.safety_check() test.failed = True test.update_status('NS') + + # SMART elif test.dev.smart_attributes: # NOTE: Pass/Fail based on both attributes and SMART short self-test - if test.dev.disk_ok: - # Run short test - # TODO + if not (test.dev.disk_ok or 'OVERRIDE' in test.status): + test.failed = True + test.update_status('NS') + else: + # Prep + test.timeout = test.dev.smart_self_test['polling_minutes'].get( + 'short', 5) + # TODO: fix timeout, set to polling + 5 + test.timeout = int(test.timeout) + 1 _include_short_test = True - _timeout = test.dev.smart_self_test['polling_minutes'].get('short', 5) - _timeout = int(_timeout) + 5 + _self_test_started = False - # Check result - # TODO - # if 'remaining_percent' in 'status' then we've started. - short_test_passed = True - if short_test_passed: - test.passed = True - test.update_status('CS') + # Create monitor pane + test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n Pending') + state.panes['smart'] = tmux_split_window( + lines=3, vertical=True, watch=test.smart_out) + + # Show attributes + clear_screen() + for line in test.dev.generate_report(): + # Not saving to log; that will happen after all tests have been run + print(line) + print(' ') + + # Start short test + print_standard('Running self-test...') + cmd = ['sudo', 'smartctl', '--test=short', test.dev.path] + run_program(cmd, check=False) + + # Monitor progress (in 5 second increments) + for iteration in range(int(test.timeout*60/5)): + sleep(5) + + # Update SMART data + test.dev.get_smart_details() + + if _self_test_started: + # Update progress file + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n {}'.format( + test.dev.smart_self_test['status'].get('string', 'UNKNOWN'))) + + # Check if test has finished + if 'remaining_percent' not in test.dev.smart_self_test['status']: + break + + else: + # Check if test has started + if 'remaining_percent' in test.dev.smart_self_test['status']: + _self_test_started = True + + # Check if timed out + if test.dev.smart_self_test['status'].get('passed', False): + if 'OVERRIDE' not in test.status: + test.passed = True + test.update_status('CS') else: + test.failed = True + test.update_status('NS') + if not (test.failed or test.passed): + test.update_status('TimedOut') + + # Disable other drive tests if necessary + if not test.passed: for t in ['badblocks', 'I/O Benchmark']: if t in test.dev.tests: test.dev.tests[t].update_status('Denied') test.dev.tests[t].disabled = True - # TODO - if no_logs: - test.update_status('Unknown') - else: - test.failed = True - test.update_status('NS') - - else: - test.failed = True - test.update_status('NS') # Save report test.report = test.dev.generate_report( @@ -1076,6 +1148,9 @@ def run_nvme_smart_tests(state, test): # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['smart']) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': From 37b8676b9c436968ab7d68032b8deb3bc164f445 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:57:30 -0700 Subject: [PATCH 084/121] Fixed quick check --- .bin/Scripts/functions/hw_diags.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 141d324e..a4bc4f5f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1073,6 +1073,13 @@ def run_nvme_smart_tests(state, test): if not (test.dev.disk_ok or 'OVERRIDE' in test.status): test.failed = True test.update_status('NS') + elif state.quick_mode: + if test.dev.disk_ok: + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') else: # Prep test.timeout = test.dev.smart_self_test['polling_minutes'].get( @@ -1141,6 +1148,9 @@ def run_nvme_smart_tests(state, test): test.dev.tests[t].update_status('Denied') test.dev.tests[t].disabled = True + # Cleanup + tmux_kill_pane(state.panes['smart']) + # Save report test.report = test.dev.generate_report( short_test=_include_short_test) @@ -1148,9 +1158,6 @@ def run_nvme_smart_tests(state, test): # Done update_progress_pane(state) - # Cleanup - tmux_kill_pane(state.panes['smart']) - def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': From f2a519b7ec7ed3738ad3370c350e51ade8fd1d54 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:58:32 -0700 Subject: [PATCH 085/121] Adjusted log and results screen --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a4bc4f5f..8fa6d743 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -157,9 +157,7 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - for line in self.generate_report(): - print(line) - print_log(strip_colors(line)) + show_report(self.generate_report()) print_warning('{} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') @@ -756,6 +754,7 @@ def run_badblocks_test(state, test): # Bail early if test.disabled: return + print_log('Starting badblocks test for {}'.format(test.dev.path)) tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) @@ -837,6 +836,7 @@ def run_io_benchmark(state, test): # Bail early if test.disabled: return + print_log('Starting I/O benchmark test for {}'.format(test.dev.path)) tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) @@ -859,6 +859,7 @@ def run_mprime_test(state, test): # Bail early if test.disabled: return + print_log('Starting Prime95 test') test.started = True test.update_status() update_progress_pane(state) @@ -1047,6 +1048,7 @@ def run_nvme_smart_tests(state, test): # Bail early if test.disabled: return + print_log('Starting NVMe/SMART test for {}'.format(test.dev.path)) _include_short_test = False test.started = True test.update_status() @@ -1168,6 +1170,12 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_report(report): + """Show report on screen and save to log w/out color.""" + for line in report: + print(line) + print_log(strip_colors(line)) + def show_results(state): """Show results for all tests.""" clear_screen() @@ -1180,11 +1188,7 @@ def show_results(state): continue print_success('{}:'.format(k)) for obj in v['Objects']: - for line in obj.report: - print(line) - print_log(strip_colors(line)) - print_standard(' ') - if 'Prime95' not in k: + show_report(obj.report) print_standard(' ') def update_main_options(state, selection, main_options): From a5d92537f54c8af6d96db97036b721289c317290 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 16:54:48 -0700 Subject: [PATCH 086/121] Removed unused function --- .bin/Scripts/functions/hw_diags.py | 32 ------------------------------ 1 file changed, 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8fa6d743..8409b4c8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -517,38 +517,6 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) -def check_disk_attributes(disk): - """Check if disk should be tested and allow overrides.""" - needs_override = False - print_standard(' {size:>6} ({tran}) {model} {serial}'.format( - **disk.lsblk)) - - # General checks - if not disk.nvme_attributes and not disk.smart_attributes: - needs_override = True - print_warning( - ' WARNING: No NVMe or SMART attributes available for: {}'.format( - disk.path)) - - # NVMe checks - # TODO check all tracked attributes and set disk.failing if needed - - # SMART checks - # TODO check all tracked attributes and set disk.failing if needed - - # Ask for override if necessary - if needs_override: - if ask(' Run tests on this device anyway?'): - # TODO Set override for this disk - pass - else: - for v in disk.tests.values(): - # Started is set to True to fix the status string - v['Result'] = 'Skipped' - v['Started'] = True - v['Status'] = 'Skipped' - print_standard('') - def generate_horizontal_graph(rates, oneline=False): """Generate two-line horizontal graph from rates, returns str.""" line_1 = '' From dc8416b5f71fb708a971fe6cde9a9bec9f2dff2a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 16:55:32 -0700 Subject: [PATCH 087/121] Adjusted formatting --- .bin/Scripts/functions/hw_diags.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8409b4c8..0b88001b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -117,6 +117,8 @@ class DiskObj(): self.tests = OrderedDict() self.get_details() self.get_smart_details() + self.description = '{size:>6} ({tran}) {model} {serial}'.format( + **self.lsblk) def check_attributes(self, silent=False): """Check NVMe / SMART attributes for errors.""" @@ -177,8 +179,7 @@ class DiskObj(): if not brief: report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( dev_path=self.path, **COLORS)) - report.append(' {size:>6} ({tran}) {model} {serial}'.format( - **self.lsblk)) + report.append(' {}'.format(self.description)) # Warnings if self.nvme_attributes: @@ -718,14 +719,14 @@ def run_audio_test(): pause('Press Enter to return to main menu... ') def run_badblocks_test(state, test): - """TODO""" + """Run a read-only surface scan with badblocks.""" # Bail early if test.disabled: return print_log('Starting badblocks test for {}'.format(test.dev.path)) tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'badblocks')) + state.panes['Top'], + text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) print_standard('TODO: run_badblocks_test({})'.format( test.dev.path)) test.started = True @@ -1022,8 +1023,7 @@ def run_nvme_smart_tests(state, test): test.update_status() tmux_update_pane( state.panes['Top'], - text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **test.dev.lsblk)) + text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) update_progress_pane(state) # NVMe @@ -1062,7 +1062,7 @@ def run_nvme_smart_tests(state, test): # Create monitor pane test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n Pending') + f.write('SMART self-test status:\n Starting...') state.panes['smart'] = tmux_split_window( lines=3, vertical=True, watch=test.smart_out) @@ -1089,7 +1089,8 @@ def run_nvme_smart_tests(state, test): # Update progress file with open(test.smart_out, 'w') as f: f.write('SMART self-test status:\n {}'.format( - test.dev.smart_self_test['status'].get('string', 'UNKNOWN'))) + test.dev.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) # Check if test has finished if 'remaining_percent' not in test.dev.smart_self_test['status']: From e96ac5c156c1eca05e14dda41a95cff0997ae492 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:09:54 -0700 Subject: [PATCH 088/121] Added watch option to use tail instead of cat * tail -f acurately prints backspace (^H) characters * badblocks output uses them and wouldn't work with watch/cat --- .bin/Scripts/functions/tmux.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 69b906d1..5fbca65d 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -46,7 +46,7 @@ def tmux_split_window( behind=False, vertical=False, follow=False, target_pane=None, working_dir=None, command=None, - text=None, watch=None): + text=None, watch=None, watch_cmd='cat'): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: @@ -79,19 +79,21 @@ def tmux_split_window( cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: create_file(watch) - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) + if watch_cmd == 'cat': + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + elif watch_cmd == 'tail': + cmd.extend(['tail', '-f', watch]) # Run and return pane_id result = run_program(cmd) return result.stdout.decode().strip() def tmux_update_pane( - pane_id, command=None, - text=None, watch=None, - working_dir=None): + pane_id, command=None, working_dir=None, + text=None, watch=None, watch_cmd='cat'): """Respawn with either a new command or new text.""" # Bail early if not command and not text and not watch: @@ -106,10 +108,13 @@ def tmux_update_pane( cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: create_file(watch) - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) + if watch_cmd == 'cat': + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + elif watch_cmd == 'tail': + cmd.extend(['tail', '-f', watch]) run_program(cmd) From 8b936f54137d0275fbdf398fa5722749aad8903f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:45:43 -0700 Subject: [PATCH 089/121] badblocks section working --- .bin/Scripts/functions/hw_diags.py | 73 +++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 0b88001b..54ce8ba5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -383,7 +383,6 @@ class State(): self.cpu = None self.disks = [] self.panes = {} - self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = OrderedDict({ 'Prime95': { @@ -423,6 +422,7 @@ class State(): os.makedirs(global_vars['LogDir'], exist_ok=True) global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( global_vars['LogDir']) + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) # Add CPU self.cpu = CpuObj() @@ -723,19 +723,68 @@ def run_badblocks_test(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting badblocks test for {}'.format(test.dev.path)) - tmux_update_pane( - state.panes['Top'], - text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) - print_standard('TODO: run_badblocks_test({})'.format( - test.dev.path)) test.started = True test.update_status() update_progress_pane(state) - sleep(3) - test.update_status('Unknown') + + # Update top pane + tmux_update_pane( + state.panes['Top'], + text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + + # Create monitor pane + test.badblocks_out = '{}/badblocks.out'.format(global_vars['LogDir']) + state.panes['badblocks'] = tmux_split_window( + lines=5, vertical=True, watch=test.badblocks_out, watch_cmd='tail') + + # Show disk details + clear_screen() + show_report(test.dev.generate_report()) + print_standard(' ') + + # Start badblocks + print_standard('Running badblocks test...') + test.badblocks_proc = popen_program( + ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], + pipe=True) + test.badblocks_proc.wait() + + # Check result and create report + try: + test.badblocks_out = test.badblocks_proc.stdout.read().decode() + except Exception as err: + test.badblocks_out = 'Error: {}'.format(err) + for line in test.badblocks_out.splitlines(): + line = line.strip() + if not line or re.search(r'^Checking', line, re.IGNORECASE): + # Skip empty and progress lines + continue + if re.search(r'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): + test.report.append(' {}'.format(line)) + test.passed = True + else: + test.report.append(' {YELLOW}{line}{CLEAR}'.format( + line=line, **COLORS)) + test.failed = True + + # Update status + if test.failed: + test.update_status('NS') + elif test.passed: + test.update_status('CS') + else: + test.update_status('Unknown') + + # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['badblocks']) + pause() + def run_hw_tests(state): """Run enabled hardware tests.""" print_standard('Scanning devices...') @@ -828,6 +877,8 @@ def run_mprime_test(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting Prime95 test') test.started = True test.update_status() @@ -835,9 +886,9 @@ def run_mprime_test(state, test): test.sensor_data = get_sensor_data() # Update top pane - test.title = '{}\nPrime95: {}'.format( - TOP_PANE_TEXT, test.dev.name) - tmux_update_pane(state.panes['Top'], text=test.title) + tmux_update_pane( + state.panes['Top'], + text='{}\nPrime95: {}'.format(TOP_PANE_TEXT, test.dev.name)) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) From ef42b596d95199503380828c101eeb2d01554439 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:56:41 -0700 Subject: [PATCH 090/121] Catch CTRL+c aborts and show results --- .bin/Scripts/functions/hw_diags.py | 78 ++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 54ce8ba5..2ed8e686 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -747,10 +747,13 @@ def run_badblocks_test(state, test): # Start badblocks print_standard('Running badblocks test...') - test.badblocks_proc = popen_program( - ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], - pipe=True) - test.badblocks_proc.wait() + try: + test.badblocks_proc = popen_program( + ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], + pipe=True) + test.badblocks_proc.wait() + except KeyboardInterrupt: + raise GenericAbort('Aborted') # Check result and create report try: @@ -833,11 +836,31 @@ def run_hw_tests(state): # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. - for k, v in state.tests.items(): - if v['Enabled']: - f = v['Function'] - for test_obj in v['Objects']: - f(state, test_obj) + try: + for k, v in state.tests.items(): + if v['Enabled']: + f = v['Function'] + for test_obj in v['Objects']: + f(state, test_obj) + except GenericAbort: + # Cleanup + tmux_kill_pane(*state.panes.values()) + + # Rebuild panes + update_progress_pane(state) + build_outer_panes(state) + + # Mark unfinished tests as aborted + for k, v in state.tests.items(): + if v['Enabled']: + for test_obj in v['Objects']: + if re.search(r'(Pending|Working)', test_obj.status): + test_obj.update_status('Aborted') + test_obj.report.append(' {YELLOW}Aborted{CLEAR}'.format( + **COLORS)) + + # Update side pane + update_progress_pane(state) # Done show_results(state) @@ -1130,27 +1153,30 @@ def run_nvme_smart_tests(state, test): run_program(cmd, check=False) # Monitor progress (in 5 second increments) - for iteration in range(int(test.timeout*60/5)): - sleep(5) + try: + for iteration in range(int(test.timeout*60/5)): + sleep(5) - # Update SMART data - test.dev.get_smart_details() + # Update SMART data + test.dev.get_smart_details() - if _self_test_started: - # Update progress file - with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n {}'.format( - test.dev.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) + if _self_test_started: + # Update progress file + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n {}'.format( + test.dev.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) - # Check if test has finished - if 'remaining_percent' not in test.dev.smart_self_test['status']: - break + # Check if test has finished + if 'remaining_percent' not in test.dev.smart_self_test['status']: + break - else: - # Check if test has started - if 'remaining_percent' in test.dev.smart_self_test['status']: - _self_test_started = True + else: + # Check if test has started + if 'remaining_percent' in test.dev.smart_self_test['status']: + _self_test_started = True + except KeyboardInterrupt: + raise GenericAbort('Aborted') # Check if timed out if test.dev.smart_self_test['status'].get('passed', False): From 8993b483a633884bc6634e38c619701e4518dd3f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:30:46 -0700 Subject: [PATCH 091/121] Fix bad cable note --- .bin/Scripts/functions/hw_diags.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 2ed8e686..f0fd09b4 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -233,15 +233,15 @@ class DiskObj(): _color = COLORS[_c] # 199/C7 warning - if str(k) == '199': + if str(k) == '199' and v['raw'] > 0: _note = '(bad cable?)' # Attribute value - _line += '{}{} {}{}'.format( - _color, - v['raw_str'], - _note, - COLORS['CLEAR']) + _line += '{c}{v} {YELLOW}{n}{CLEAR}'.format( + c=_color, + v=v['raw_str'], + n=_note, + **COLORS) # Add line to report report.append(_line) From a4896a55f619ed7709a7bfa9a490bcac0445fb04 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:31:34 -0700 Subject: [PATCH 092/121] Adjust log names --- .bin/Scripts/functions/hw_diags.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index f0fd09b4..f2fa7469 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -736,7 +736,8 @@ def run_badblocks_test(state, test): text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) # Create monitor pane - test.badblocks_out = '{}/badblocks.out'.format(global_vars['LogDir']) + test.badblocks_out = '{}/badblocks_{}.out'.format( + global_vars['LogDir'], test.dev.name) state.panes['badblocks'] = tmux_split_window( lines=5, vertical=True, watch=test.badblocks_out, watch_cmd='tail') @@ -1134,7 +1135,8 @@ def run_nvme_smart_tests(state, test): _self_test_started = False # Create monitor pane - test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) + test.smart_out = '{}/smart_{}.out'.format( + global_vars['LogDir'], test.dev.name) with open(test.smart_out, 'w') as f: f.write('SMART self-test status:\n Starting...') state.panes['smart'] = tmux_split_window( From 503e6f2b4251fbf7355d413fb3df680441a45abb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:45:25 -0700 Subject: [PATCH 093/121] Fix SMART short-test timeout detection --- .bin/Scripts/functions/hw_diags.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index f2fa7469..361aad2d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1181,13 +1181,14 @@ def run_nvme_smart_tests(state, test): raise GenericAbort('Aborted') # Check if timed out - if test.dev.smart_self_test['status'].get('passed', False): - if 'OVERRIDE' not in test.status: - test.passed = True - test.update_status('CS') - else: - test.failed = True - test.update_status('NS') + if 'passed' in test.dev.smart_self_test['status']: + if test.dev.smart_self_test['status']['passed']: + if 'OVERRIDE' not in test.status: + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') if not (test.failed or test.passed): test.update_status('TimedOut') From 4c0bb1c9b7d954319f249422c1b41f2191fa0658 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:06:03 -0700 Subject: [PATCH 094/121] Group results by device instead of test --- .bin/Scripts/functions/hw_diags.py | 139 ++++++++++++++++++----------- 1 file changed, 87 insertions(+), 52 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 361aad2d..4b544303 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -79,7 +79,7 @@ class CpuObj(): """Object for tracking CPU specific data.""" def __init__(self): self.lscpu = {} - self.tests = {} + self.tests = OrderedDict() self.get_details() self.name = self.lscpu.get('Model name', 'Unknown CPU') @@ -102,6 +102,18 @@ class CpuObj(): continue self.lscpu[_field] = _data + def generate_cpu_report(self): + """Generate CPU report with data from all tests.""" + report = [] + report.append('{BLUE}Device{CLEAR}'.format(**COLORS)) + report.append(' {}'.format(self.name)) + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): @@ -115,9 +127,10 @@ class DiskObj(): self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() + self.warnings = [] self.get_details() self.get_smart_details() - self.description = '{size:>6} ({tran}) {model} {serial}'.format( + self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) def check_attributes(self, silent=False): @@ -159,8 +172,8 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - show_report(self.generate_report()) - print_warning('{} error(s) detected.'.format(attr_type)) + show_report(self.generate_attribute_report(description=True)) + print_warning(' {} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') pause() @@ -172,13 +185,15 @@ class DiskObj(): if self.nvme_attributes or not self.smart_attributes: # i.e. only leave enabled for SMART short-tests self.tests['NVMe / SMART'].disabled = True + print_standard(' ') - def generate_report(self, brief=False, short_test=False): + def generate_attribute_report( + self, description=False, short_test=False, timestamp=False): """Generate NVMe / SMART report, returns list.""" report = [] - if not brief: - report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( - dev_path=self.path, **COLORS)) + if description: + report.append('{BLUE}Device ({name}){CLEAR}'.format( + name=self.name, **COLORS)) report.append(' {}'.format(self.description)) # Warnings @@ -203,8 +218,8 @@ class DiskObj(): # Attributes report.append('{BLUE}{a} Attributes{YELLOW}{u:>23} {t}{CLEAR}'.format( a=attr_type, - u='Updated:' if brief else '', - t=time.strftime('%Y-%m-%d %H:%M %Z') if brief else '', + u='Updated:' if timestamp else '', + t=time.strftime('%Y-%m-%d %H:%M %Z') if timestamp else '', **COLORS)) if self.nvme_attributes: attr_type = 'NVMe' @@ -259,6 +274,21 @@ class DiskObj(): # Done return report + def generate_disk_report(self): + """Generate disk report with data from all tests.""" + report = [] + report.append('{BLUE}Device ({name}){CLEAR}'.format( + name=self.name, **COLORS)) + report.append(' {}'.format(self.description)) + for w in self.warnings: + report.append(' {YELLOW}{w}{CLEAR}'.format(w=w, **COLORS)) + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -338,14 +368,14 @@ class DiskObj(): # Check if a self-test is currently running if 'remaining_percent' in self.smart_self_test['status']: - _msg='SMART self-test in progress, all tests disabled' + _msg = 'SMART self-test in progress, all tests disabled' if not silent: print_warning('WARNING: {}'.format(_msg)) print_standard(' ') if ask('Abort HW Diagnostics?'): exit_script() if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].report.append( '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) for t in self.tests.values(): @@ -359,17 +389,20 @@ class DiskObj(): if silent: self.disk_ok = HW_OVERRIDES_FORCED else: - print_warning( - ' WARNING: No NVMe or SMART attributes available for: {}'.format( - self.path)) + _msg = 'No NVMe or SMART data available' + self.warnings.append(_msg) + print_info('Device ({})'.format(self.name)) + print_standard(' {}'.format(self.description)) + print_warning(' {}'.format(_msg)) self.disk_ok = HW_OVERRIDES_FORCED or ask( 'Run tests on this device anyway?') + print_standard(' ') if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set if not self.tests['NVMe / SMART'].report: - self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].update_status('NS') self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: @@ -743,7 +776,7 @@ def run_badblocks_test(state, test): # Show disk details clear_screen() - show_report(test.dev.generate_report()) + show_report(test.dev.generate_attribute_report()) print_standard(' ') # Start badblocks @@ -757,6 +790,7 @@ def run_badblocks_test(state, test): raise GenericAbort('Aborted') # Check result and create report + test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) try: test.badblocks_out = test.badblocks_proc.stdout.read().decode() except Exception as err: @@ -787,7 +821,6 @@ def run_badblocks_test(state, test): # Cleanup tmux_kill_pane(state.panes['badblocks']) - pause() def run_hw_tests(state): """Run enabled hardware tests.""" @@ -825,15 +858,6 @@ def run_hw_tests(state): for disk in state.disks: disk.safety_check(silent=state.quick_mode) - # TODO Remove - clear_screen() - print_info('Running tests:') - for k, v in state.tests.items(): - if v['Enabled']: - print_standard(' {}'.format(k)) - update_progress_pane(state) - pause() - # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. @@ -1010,6 +1034,7 @@ def run_mprime_test(state, test): global_vars['LogDir'])) # Check results and build report + test.report.append('{BLUE}Prime95{CLEAR}'.format(**COLORS)) test.logs = {} for log in ['results.txt', 'prime.log']: lines = [] @@ -1026,20 +1051,18 @@ def run_mprime_test(state, test): # results.txt (NS check) if log == 'results.txt': - _tmp = [] for line in lines: + line = line.strip() if re.search(r'(error|fail)', line, re.IGNORECASE): test.failed = True test.update_status('NS') - _tmp.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) - if _tmp: - test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) - test.report.extend(_tmp) + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # prime.log (CS check) if log == 'prime.log': _tmp = {'Pass': {}, 'Warn': {}} for line in lines: + line = line.strip() _r = re.search( r'(completed.*(\d+) errors, (\d+) warnings)', line, @@ -1059,18 +1082,19 @@ def run_mprime_test(state, test): elif len(_tmp['Pass']) > 0: test.passed = True test.update_status('CS') - if len(_tmp['Pass']) + len(_tmp['Warn']) > 0: - test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) - for line in sorted(_tmp['Pass'].keys()): - test.report.append(' {}'.format(line)) - for line in sorted(_tmp['Warn'].keys()): - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) + for line in sorted(_tmp['Pass'].keys()): + test.report.append(' {}'.format(line)) + for line in sorted(_tmp['Warn'].keys()): + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) - # Finalize report + # Unknown result if not (test.aborted or test.failed or test.passed): + test.report.append(' {YELLOW}Unknown result{CLEAR}'.format(**COLORS)) test.update_status('Unknown') + + # Add temps to report test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) - for line in generate_report( + for line in generate_sensor_report( test.sensor_data, 'Idle', 'Max', 'Cooldown', core_only=True): test.report.append(' {}'.format(line)) @@ -1144,10 +1168,10 @@ def run_nvme_smart_tests(state, test): # Show attributes clear_screen() - for line in test.dev.generate_report(): - # Not saving to log; that will happen after all tests have been run - print(line) - print(' ') + print_info('Device ({})'.format(test.dev.name)) + print_standard(' {}'.format(test.dev.description)) + show_report(test.dev.generate_attribute_report()) + print_standard(' ') # Start short test print_standard('Running self-test...') @@ -1203,7 +1227,7 @@ def run_nvme_smart_tests(state, test): tmux_kill_pane(state.panes['smart']) # Save report - test.report = test.dev.generate_report( + test.report = test.dev.generate_attribute_report( short_test=_include_short_test) # Done @@ -1231,13 +1255,24 @@ def show_results(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'Results')) - for k, v in state.tests.items(): - # Skip disabled tests - if not v['Enabled']: - continue - print_success('{}:'.format(k)) - for obj in v['Objects']: - show_report(obj.report) + + # CPU tests + _enabled = False + for k in TESTS_CPU: + _enabled |= state.tests[k]['Enabled'] + if _enabled: + print_success('CPU:'.format(k)) + show_report(state.cpu.generate_cpu_report()) + print_standard(' ') + + # Disk tests + for k in TESTS_DISK: + _enabled |= state.tests[k]['Enabled'] + if _enabled: + print_success('Disk{}:'.format( + '' if len(state.disks) == 1 else 's')) + for disk in state.disks: + show_report(disk.generate_disk_report()) print_standard(' ') def update_main_options(state, selection, main_options): From d8123a71ec5ca887115b76d07d197e54282bcd07 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:07:34 -0700 Subject: [PATCH 095/121] Renamed generate_report to generate_sensor_report --- .bin/Scripts/functions/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index b6319744..99e7999a 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -36,7 +36,7 @@ def fix_sensor_str(s): s = s.replace(' ', ' ') return s -def generate_report( +def generate_sensor_report( sensor_data, *temp_labels, colors=True, core_only=False): """Generate report based on temp_labels, returns list if str.""" @@ -153,7 +153,7 @@ def monitor_sensors(monitor_pane, monitor_file): while True: update_sensor_data(sensor_data) with open(monitor_file, 'w') as f: - report = generate_report(sensor_data, 'Current', 'Max') + report = generate_sensor_report(sensor_data, 'Current', 'Max') f.write('\n'.join(report)) sleep(1) if monitor_pane and not tmux_poll_pane(monitor_pane): From e0a2993c362469acaa08674bc85d28f3ef2920a2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:18:34 -0700 Subject: [PATCH 096/121] Skip disk safety checks if only testing the CPU --- .bin/Scripts/functions/hw_diags.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4b544303..6fd384a6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -854,9 +854,13 @@ def run_hw_tests(state): v['Objects'].append(test_obj) print_standard('') - # Run safety checks - for disk in state.disks: - disk.safety_check(silent=state.quick_mode) + # Run disk safety checks (if necessary) + _disk_tests_enabled = False + for k in TESTS_DISK: + _disk_tests_enabled |= state.tests[k]['Enabled'] + if _disk_tests_enabled: + for disk in state.disks: + disk.safety_check(silent=state.quick_mode) # Run tests ## Because state.tests is an OrderedDict and the disks were added @@ -1266,6 +1270,7 @@ def show_results(state): print_standard(' ') # Disk tests + _enabled = False for k in TESTS_DISK: _enabled |= state.tests[k]['Enabled'] if _enabled: From baaf1994e3312df5f39ee4bdd8c3009dacd50b9f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:44:46 -0700 Subject: [PATCH 097/121] Catch keyboard interrupt and gracefully abort --- .bin/Scripts/hw-diags-menu | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index 6f0247cd..5b6f4f76 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -13,20 +13,25 @@ from functions.tmux import * init_global_vars() if __name__ == '__main__': + # Show menu try: - # Show menu state = State() menu_diags(state, sys.argv) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() + except KeyboardInterrupt: + print_standard(' ') + print_warning('Aborted') + print_standard(' ') + sleep(1) + pause('Press Enter to exit...') except SystemExit: - tmux_kill_all_panes() + # Normal exit pass except: tmux_kill_all_panes() major_exception() + # Done + tmux_kill_all_panes() + exit_script() + # vim: sts=2 sw=2 ts=2 From c820d2ac6de623a4d6f1ef3400bca890b8f2965a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 13:20:39 -0700 Subject: [PATCH 098/121] Fixed Prime95 abort handling --- .bin/Scripts/functions/hw_diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6fd384a6..ef411220 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1083,7 +1083,7 @@ def run_mprime_test(state, test): test.failed = True test.passed = False test.update_status('NS') - elif len(_tmp['Pass']) > 0: + elif len(_tmp['Pass']) > 0 and not test.aborted: test.passed = True test.update_status('CS') for line in sorted(_tmp['Pass'].keys()): From a25a10e616cbeaaec796bc0c0c3cf22f6e9c35f7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 14:07:19 -0700 Subject: [PATCH 099/121] More abort logic updates --- .bin/Scripts/functions/hw_diags.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index ef411220..c9a84940 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -127,7 +127,6 @@ class DiskObj(): self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() - self.warnings = [] self.get_details() self.get_smart_details() self.description = '{size} ({tran}) {model} {serial}'.format( @@ -280,8 +279,10 @@ class DiskObj(): report.append('{BLUE}Device ({name}){CLEAR}'.format( name=self.name, **COLORS)) report.append(' {}'.format(self.description)) - for w in self.warnings: - report.append(' {YELLOW}{w}{CLEAR}'.format(w=w, **COLORS)) + + # Attributes + if 'NVMe / SMART' not in self.tests: + report.extend(self.generate_attribute_report()) # Tests for test in self.tests.values(): @@ -389,11 +390,9 @@ class DiskObj(): if silent: self.disk_ok = HW_OVERRIDES_FORCED else: - _msg = 'No NVMe or SMART data available' - self.warnings.append(_msg) print_info('Device ({})'.format(self.name)) print_standard(' {}'.format(self.description)) - print_warning(' {}'.format(_msg)) + print_warning(' No NVMe or SMART data available') self.disk_ok = HW_OVERRIDES_FORCED or ask( 'Run tests on this device anyway?') print_standard(' ') @@ -787,9 +786,9 @@ def run_badblocks_test(state, test): pipe=True) test.badblocks_proc.wait() except KeyboardInterrupt: - raise GenericAbort('Aborted') + test.aborted = True - # Check result and create report + # Check result and build report test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) try: test.badblocks_out = test.badblocks_proc.stdout.read().decode() @@ -802,11 +801,16 @@ def run_badblocks_test(state, test): continue if re.search(r'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): test.report.append(' {}'.format(line)) - test.passed = True + if not test.aborted: + test.passed = True else: test.report.append(' {YELLOW}{line}{CLEAR}'.format( line=line, **COLORS)) test.failed = True + if test.aborted: + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + test.update_status('Aborted') + raise GenericAbort('Aborted') # Update status if test.failed: @@ -885,8 +889,6 @@ def run_hw_tests(state): for test_obj in v['Objects']: if re.search(r'(Pending|Working)', test_obj.status): test_obj.update_status('Aborted') - test_obj.report.append(' {YELLOW}Aborted{CLEAR}'.format( - **COLORS)) # Update side pane update_progress_pane(state) @@ -1206,6 +1208,12 @@ def run_nvme_smart_tests(state, test): if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True except KeyboardInterrupt: + test.aborted = True + test.report = test.dev.generate_attribute_report() + test.report.append('{BLUE}SMART Short self-test{CLEAR}'.format( + **COLORS)) + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + test.update_status('Aborted') raise GenericAbort('Aborted') # Check if timed out From 385bdd7dbf11f3e5f6b99029263ea9f4fc2f7879 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:10:58 -0700 Subject: [PATCH 100/121] Allow resizing current pane --- .bin/Scripts/functions/tmux.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 5fbca65d..20f35921 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -28,12 +28,15 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes -def tmux_resize_pane(pane_id, x=None, y=None): +def tmux_resize_pane(pane_id=None, x=None, y=None): """Resize pane to specific hieght or width.""" if not x and not y: raise Exception('Neither height nor width specified.') - cmd = ['tmux', 'resize-pane', '-t', pane_id] + cmd = ['tmux', 'resize-pane'] + if pane_id: + # NOTE: If pane_id not specified then the current pane will be resized + cmd.extend(['-t', pane_id]) if x: cmd.extend(['-x', str(x)]) elif y: From ec8c78197b45b3a6d812474dc0f3d0d30af55ec6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:15:40 -0700 Subject: [PATCH 101/121] I/O Benchmark test is working --- .bin/Scripts/functions/hw_diags.py | 223 +++++++++++++++++++++++++++-- 1 file changed, 212 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index c9a84940..50eca7b5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -34,7 +34,6 @@ HW_OVERRIDES_FORCED = HW_OVERRIDES_FORCED and not HW_OVERRIDES_LIMITED IO_VARS = { 'Block Size': 512*1024, 'Chunk Size': 32*1024**2, - 'Minimum Dev Size': 8*1024**3, 'Minimum Test Size': 10*1024**3, 'Alt Test Size Factor': 0.01, 'Progress Refresh Rate': 5, @@ -74,6 +73,10 @@ TESTS_DISK = [ ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) +# Error Classe +class DeviceTooSmallError(Exception): + pass + # Classes class CpuObj(): """Object for tracking CPU specific data.""" @@ -132,6 +135,60 @@ class DiskObj(): self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) + def calc_io_dd_values(self): + """Calcualte I/O benchmark dd values.""" + # Get real disk size + cmd = ['lsblk', + '--bytes', '--nodeps', '--noheadings', + '--output', 'size', self.path] + result = run_program(cmd) + self.size_bytes = int(result.stdout.decode().strip()) + + # dd calculations + ## The minimum dev size is 'Graph Horizontal Width' * 'Chunk Size' + ## (e.g. 1.25 GB for a width of 40 and a chunk size of 32MB) + ## If the device is smaller than the minimum dd_chunks would be set + ## to zero which would cause a divide by zero error. + ## If the device is below the minimum size an Exception will be raised + ## + ## dd_size is the area to be read in bytes + ## If the dev is < 10Gb then it's the whole dev + ## Otherwise it's the larger of 10Gb or 1% of the dev + ## + ## dd_chunks is the number of groups of "Chunk Size" in self.dd_size + ## This number is reduced to a multiple of the graph width in + ## order to allow for the data to be condensed cleanly + ## + ## dd_chunk_blocks is the chunk size in number of blocks + ## (e.g. 64 if block size is 512KB and chunk size is 32MB + ## + ## dd_skip_blocks is the number of "Block Size" groups not tested + ## dd_skip_count is the number of blocks to skip per self.dd_chunk + ## dd_skip_extra is how often to add an additional skip block + ## This is needed to ensure an even testing across the dev + ## This is calculated by using the fractional amount left off + ## of the dd_skip_count variable + self.dd_size = min(IO_VARS['Minimum Test Size'], self.size_bytes) + self.dd_size = max( + self.dd_size, + self.size_bytes * IO_VARS['Alt Test Size Factor']) + self.dd_chunks = int(self.dd_size // IO_VARS['Chunk Size']) + self.dd_chunks -= self.dd_chunks % IO_VARS['Graph Horizontal Width'] + if self.dd_chunks < IO_VARS['Graph Horizontal Width']: + raise DeviceTooSmallError + self.dd_chunk_blocks = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) + self.dd_size = self.dd_chunks * IO_VARS['Chunk Size'] + self.dd_skip_blocks = int( + (self.size_bytes - self.dd_size) // IO_VARS['Block Size']) + self.dd_skip_count = int((self.dd_skip_blocks / self.dd_chunks) // 1) + self.dd_skip_extra = 0 + try: + self.dd_skip_extra = 1 + int( + 1 / ((self.dd_skip_blocks / self.dd_chunks) % 1)) + except ZeroDivisionError: + # self.dd_skip_extra == 0 is fine + pass + def check_attributes(self, silent=False): """Check NVMe / SMART attributes for errors.""" override_disabled = False @@ -498,7 +555,7 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled or 'OVERRIDE' in self.status: + if self.disabled or re.search(r'ERROR|OVERRIDE', self.status): return if new_status: self.status = build_status_string( @@ -904,23 +961,163 @@ def run_hw_tests(state): tmux_kill_pane(*state.panes.values()) def run_io_benchmark(state, test): - """TODO""" + """Run a read-only I/O benchmark using dd.""" # Bail early if test.disabled: return + + # Prep print_log('Starting I/O benchmark test for {}'.format(test.dev.path)) - tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'I/O Benchmark')) - print_standard('TODO: run_io_benchmark({})'.format( - test.dev.path)) test.started = True test.update_status() update_progress_pane(state) - sleep(3) - test.update_status('Unknown') + + # Update top pane + tmux_update_pane( + state.panes['Top'], + text='{}\nI/O Benchmark: {}'.format( + TOP_PANE_TEXT, test.dev.description)) + + # Create monitor pane + test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( + global_vars['LogDir'], test.dev.name) + state.panes['io_benchmark'] = tmux_split_window( + percent=75, vertical=True, + watch=test.io_benchmark_out, watch_cmd='tail') + tmux_resize_pane(y=15) + + # Show disk details + clear_screen() + show_report(test.dev.generate_attribute_report()) + print_standard(' ') + + # Start I/O Benchmark + print_standard('Running I/O benchmark test...') + try: + test.merged_rates = [] + test.read_rates = [] + test.vertical_graph = [] + test.dev.calc_io_dd_values() + + # Run dd read tests + offset = 0 + for i in range(test.dev.dd_chunks): + # Build cmd + i += 1 + skip = test.dev.dd_skip_count + if test.dev.dd_skip_extra and i % test.dev.dd_skip_extra == 0: + skip += 1 + cmd = [ + 'sudo', 'dd', + 'bs={}'.format(IO_VARS['Block Size']), + 'skip={}'.format(offset+skip), + 'count={}'.format(test.dev.dd_chunk_blocks), + 'iflag=direct', + 'if={}'.format(test.dev.path), + 'of=/dev/null'] + + # Run cmd and get read rate + result = run_program(cmd) + result_str = result.stderr.decode().replace('\n', '') + cur_rate = get_read_rate(result_str) + + # Add rate to lists + test.read_rates.append(cur_rate) + test.vertical_graph.append( + '{percent:0.1f} {rate}'.format( + percent=(i/test.dev.dd_chunks)*100, + rate=int(cur_rate/(1024**2)))) + + # Show progress + if i % IO_VARS['Progress Refresh Rate'] == 0: + update_io_progress( + percent=(i/test.dev.dd_chunks)*100, + rate=cur_rate, + progress_file=test.io_benchmark_out) + + # Update offset + offset += test.dev.dd_chunk_blocks + skip + + except DeviceTooSmallError: + # Device too small, skipping test + test.update_status('N/A') + except KeyboardInterrupt: + test.aborted = True + except (subprocess.CalledProcessError, TypeError, ValueError): + # Something went wrong, results unknown + test.update_status('ERROR') + + # Check result and build report + test.report.append('{BLUE}I/O Benchmark{CLEAR}'.format(**COLORS)) + if test.aborted: + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + raise GenericAbort('Aborted') + elif not test.read_rates: + if 'ERROR' in test.status: + test.report.append(' {RED}Unknown error{CLEAR}'.format(**COLORS)) + elif 'N/A' in test.status: + # Device too small + test.report.append(' {YELLOW}Disk too small to test{CLEAR}'.format( + **COLORS)) + else: + # Merge rates for horizontal graph + offset = 0 + width = int(test.dev.dd_chunks / IO_VARS['Graph Horizontal Width']) + for i in range(IO_VARS['Graph Horizontal Width']): + test.merged_rates.append( + sum(test.read_rates[offset:offset+width])/width) + offset += width + + # Add horizontal graph to report + for line in generate_horizontal_graph(test.merged_rates): + if not re.match(r'^\s+$', line): + test.report.append(line) + + # Add read speeds to report + avg_read = sum(test.read_rates) / len(test.read_rates) + min_read = min(test.read_rates) + max_read = max(test.read_rates) + avg_min_max = 'Read speeds avg: {:3.1f}'.format(avg_read/(1024**2)) + avg_min_max += ' min: {:3.1f}'.format(min_read/(1024**2)) + avg_min_max += ' max: {:3.1f}'.format(max_read/(1024**2)) + test.report.append(avg_min_max) + + # Compare read speeds to thresholds + if test.dev.lsblk['rota']: + # Use HDD scale + thresh_min = IO_VARS['Threshold HDD Min'] + thresh_high_avg = IO_VARS['Threshold HDD High Avg'] + thresh_low_avg = IO_VARS['Threshold HDD Low Avg'] + else: + # Use SSD scale + thresh_min = IO_VARS['Threshold SSD Min'] + thresh_high_avg = IO_VARS['Threshold SSD High Avg'] + thresh_low_avg = IO_VARS['Threshold SSD Low Avg'] + if min_read <= thresh_min and avg_read <= thresh_high_avg: + test.failed = True + elif avg_read <= thresh_low_avg: + test.failed = True + else: + test.passed = True + + # Update status + if test.failed: + test.update_status('NS') + elif test.passed: + test.update_status('CS') + elif not 'N/A' in test.status: + test.update_status('Unknown') + + # Save log + with open(test.io_benchmark_out.replace('.', '-raw.'), 'a') as f: + f.write('\n'.join(test.vertical_graph)) + + # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['io_benchmark']) + def run_keyboard_test(): """Run keyboard test.""" clear_screen() @@ -1122,14 +1319,18 @@ def run_nvme_smart_tests(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting NVMe/SMART test for {}'.format(test.dev.path)) _include_short_test = False test.started = True test.update_status() + update_progress_pane(state) + + # Update top pane tmux_update_pane( state.panes['Top'], text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) - update_progress_pane(state) # NVMe if test.dev.nvme_attributes: From 8c5820d5aa73cbface183311655f97842854eb99 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:16:35 -0700 Subject: [PATCH 102/121] Fix horizontal graph * generate_horizontal_graph() now returns a list instead of a str --- .bin/Scripts/functions/hw_diags.py | 48 +++++++++++++----------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 50eca7b5..85c4d86e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -608,11 +608,8 @@ def build_status_string(label, status, info_label=False): **COLORS) def generate_horizontal_graph(rates, oneline=False): - """Generate two-line horizontal graph from rates, returns str.""" - line_1 = '' - line_2 = '' - line_3 = '' - line_4 = '' + """Generate horizontal graph from rates, returns list.""" + graph = ['', '', '', ''] for r in rates: step = get_graph_step(r, scale=32) if oneline: @@ -630,33 +627,30 @@ def generate_horizontal_graph(rates, oneline=False): # Build graph full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) if step >= 24: - line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) - line_2 += full_block - line_3 += full_block - line_4 += full_block + graph[0] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) + graph[1] += full_block + graph[2] += full_block + graph[3] += full_block elif step >= 16: - line_1 += ' ' - line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) - line_3 += full_block - line_4 += full_block + graph[0] += ' ' + graph[1] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) + graph[2] += full_block + graph[3] += full_block elif step >= 8: - line_1 += ' ' - line_2 += ' ' - line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) - line_4 += full_block + graph[0] += ' ' + graph[1] += ' ' + graph[2] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + graph[3] += full_block else: - line_1 += ' ' - line_2 += ' ' - line_3 += ' ' - line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) - line_1 += COLORS['CLEAR'] - line_2 += COLORS['CLEAR'] - line_3 += COLORS['CLEAR'] - line_4 += COLORS['CLEAR'] + graph[0] += ' ' + graph[1] += ' ' + graph[2] += ' ' + graph[3] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + graph = [line+COLORS['CLEAR'] for line in graph] if oneline: - return line_4 + return graph[:-1] else: - return '\n'.join([line_1, line_2, line_3, line_4]) + return graph def get_graph_step(rate, scale=16): """Get graph step based on rate and scale, returns int.""" From 41c9a4d23fa5c9df58ceef85d6a2e35934827a59 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:29:09 -0700 Subject: [PATCH 103/121] Fixed only showing non-empty graph lines --- .bin/Scripts/functions/hw_diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 85c4d86e..30220583 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1064,7 +1064,7 @@ def run_io_benchmark(state, test): # Add horizontal graph to report for line in generate_horizontal_graph(test.merged_rates): - if not re.match(r'^\s+$', line): + if not re.match(r'^\s+$', strip_colors(line)): test.report.append(line) # Add read speeds to report From 0c0f8e895021c7bfae0ae1a1d258bf86046f54ae Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:51:02 -0700 Subject: [PATCH 104/121] Added disable_test() to Disk class --- .bin/Scripts/functions/hw_diags.py | 44 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 30220583..4a855dc6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -237,12 +237,18 @@ class DiskObj(): if HW_OVERRIDES_FORCED or ask('Run tests on this device anyway?'): self.disk_ok = True if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].update_status('OVERRIDE') - if self.nvme_attributes or not self.smart_attributes: - # i.e. only leave enabled for SMART short-tests - self.tests['NVMe / SMART'].disabled = True + self.disable_test('NVMe / SMART', 'OVERRIDE') + if not self.nvme_attributes and self.smart_attributes: + # Re-enable for SMART short-tests + self.tests['NVMe / SMART'].disabled = False print_standard(' ') + def disable_test(self, name, status): + """Disable test by name and update status.""" + if name in self.tests: + self.tests[name].update_status(status) + self.tests[name].disabled = True + def generate_attribute_report( self, description=False, short_test=False, timestamp=False): """Generate NVMe / SMART report, returns list.""" @@ -427,23 +433,26 @@ class DiskObj(): # Check if a self-test is currently running if 'remaining_percent' in self.smart_self_test['status']: _msg = 'SMART self-test in progress, all tests disabled' + + # Ask to abort if not silent: print_warning('WARNING: {}'.format(_msg)) print_standard(' ') if ask('Abort HW Diagnostics?'): exit_script() + + # Add warning to report if 'NVMe / SMART' in self.tests: self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].report.append( '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) - for t in self.tests.values(): - t.update_status('Denied') - t.disabled = True + + # Disable all tests for this disk + for t in self.tests.keys(): + self.disable_test(k, 'Denied') else: # No NVMe/SMART details - if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].update_status('N/A') - self.tests['NVMe / SMART'].disabled = True + self.disable_test('NVMe / SMART', 'N/A') if silent: self.disk_ok = HW_OVERRIDES_FORCED else: @@ -457,14 +466,11 @@ class DiskObj(): if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set + self.disable_test('NVMe / SMART', 'NS') if not self.tests['NVMe / SMART'].report: self.tests['NVMe / SMART'].report = self.generate_attribute_report() - self.tests['NVMe / SMART'].update_status('NS') - self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: - if t in self.tests: - self.tests[t].update_status('Denied') - self.tests[t].disabled = True + self.disable_test(t, 'Denied') class State(): """Object to track device objects and overall state.""" @@ -863,6 +869,10 @@ def run_badblocks_test(state, test): test.update_status('Aborted') raise GenericAbort('Aborted') + # Disable other drive tests if necessary + if not test.passed: + test.dev.disable_test('I/O Benchmark', 'Denied') + # Update status if test.failed: test.update_status('NS') @@ -1426,9 +1436,7 @@ def run_nvme_smart_tests(state, test): # Disable other drive tests if necessary if not test.passed: for t in ['badblocks', 'I/O Benchmark']: - if t in test.dev.tests: - test.dev.tests[t].update_status('Denied') - test.dev.tests[t].disabled = True + test.dev.disable_test(t, 'Denied') # Cleanup tmux_kill_pane(state.panes['smart']) From 10ae59be197c431f74e64e03758cdd15b8599432 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 00:55:57 -0700 Subject: [PATCH 105/121] Update tmux layout periodically --- .bin/Scripts/functions/hw_diags.py | 106 ++++++++++++++++++++++++++--- .bin/Scripts/functions/tmux.py | 20 +++++- 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4a855dc6..56b3c501 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -72,6 +72,11 @@ TESTS_DISK = [ 'badblocks', ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) +TMUX_LAYOUT = OrderedDict({ + 'Top': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, +}) # Error Classe class DeviceTooSmallError(Exception): @@ -449,7 +454,7 @@ class DiskObj(): # Disable all tests for this disk for t in self.tests.keys(): - self.disable_test(k, 'Denied') + self.disable_test(t, 'Denied') else: # No NVMe/SMART details self.disable_test('NVMe / SMART', 'N/A') @@ -613,6 +618,49 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) +def fix_tmux_panes(state, tmux_layout): + """Fix pane sizes in case the window has been resized.""" + needs_fixed = False + + # Check layout + for k, v in tmux_layout.items(): + if not v.get('Check'): + # Not concerned with the size of this pane + continue + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Get pane size + x, y = tmux_get_pane_size(pane_id=target) + if v.get('x', False) and v.get['x'] != x: + needs_fixed = True + if v.get('y', False) and v.get['y'] != y: + needs_fixed = True + + # Bail? + if not needs_fixed: + return + + # Update layout + for k, v in tmux_layout.items(): + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Resize pane + tmux_resize_pane(pane_id=target, **v) + def generate_horizontal_graph(rates, oneline=False): """Generate horizontal graph from rates, returns list.""" graph = ['', '', '', ''] @@ -819,10 +867,14 @@ def run_badblocks_test(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'badblocks': {'y': 5, 'Check': True}, + }) # Create monitor pane test.badblocks_out = '{}/badblocks_{}.out'.format( @@ -841,7 +893,15 @@ def run_badblocks_test(state, test): test.badblocks_proc = popen_program( ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], pipe=True) - test.badblocks_proc.wait() + while True: + try: + test.badblocks_proc.wait(timeout=10) + except subprocess.TimeoutExpired: + fix_tmux_panes(state, test.tmux_layout) + else: + # badblocks finished, exit loop + break + except KeyboardInterrupt: test.aborted = True @@ -976,11 +1036,16 @@ def run_io_benchmark(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nI/O Benchmark: {}'.format( TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'io_benchmark': {'y': 1000, 'Check': False}, + 'Current': {'y': 15, 'Check': True}, + }) # Create monitor pane test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( @@ -1042,6 +1107,10 @@ def run_io_benchmark(state, test): # Update offset offset += test.dev.dd_chunk_blocks + skip + # Fix panes + if i % 5 == 0: + fix_tmux_panes(state, test.tmux_layout) + except DeviceTooSmallError: # Device too small, skipping test test.update_status('N/A') @@ -1140,10 +1209,16 @@ def run_mprime_test(state, test): update_progress_pane(state) test.sensor_data = get_sensor_data() - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nPrime95: {}'.format(TOP_PANE_TEXT, test.dev.name)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'Temps': {'y': 1000, 'Check': False}, + 'mprime': {'y': 11, 'Check': False}, + 'Current': {'y': 3, 'Check': True}, + }) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) @@ -1181,7 +1256,7 @@ def run_mprime_test(state, test): working_dir=global_vars['TmpDir']) #time_limit = int(MPRIME_LIMIT) * 60 # TODO: restore above line - time_limit = 10 + time_limit = 30 try: for i in range(time_limit): clear_screen() @@ -1199,6 +1274,12 @@ def run_mprime_test(state, test): print(_status_str) print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) update_sensor_data(test.sensor_data) + + # Fix panes + if i % 10 == 0: + fix_tmux_panes(state, test.tmux_layout) + + # Wait sleep(1) except KeyboardInterrupt: # Catch CTRL+C @@ -1331,10 +1412,14 @@ def run_nvme_smart_tests(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'smart': {'y': 3, 'Check': True}, + }) # NVMe if test.dev.nvme_attributes: @@ -1391,7 +1476,7 @@ def run_nvme_smart_tests(state, test): # Monitor progress (in 5 second increments) try: - for iteration in range(int(test.timeout*60/5)): + for i in range(int(test.timeout*60/5)): sleep(5) # Update SMART data @@ -1412,6 +1497,11 @@ def run_nvme_smart_tests(state, test): # Check if test has started if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True + + # Fix panes + if i % 2 == 0: + fix_tmux_panes(state, test.tmux_layout) + except KeyboardInterrupt: test.aborted = True test.report = test.dev.generate_attribute_report() diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 20f35921..0e585df6 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -8,6 +8,24 @@ def create_file(filepath): with open(filepath, 'w') as f: f.write('') +def tmux_get_pane_size(pane_id=None): + """Get target, or current, pane size, returns tuple.""" + x = -1 + y = -1 + cmd = ['tmux', 'display', '-p', '#{pane_width}x#{pane_height}'] + if pane_id: + cmd.extend(['-t', pane_id]) + + # Run cmd and set x & y + result = run_program(cmd, check=False) + try: + x, y = result.stdout.decode().strip().split() + except Exception: + # Ignore and return unrealistic values + pass + + return (x, y) + def tmux_kill_all_panes(pane_id=None): """Kill all tmux panes except the active pane or pane_id if specified.""" cmd = ['tmux', 'kill-pane', '-a'] @@ -28,7 +46,7 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes -def tmux_resize_pane(pane_id=None, x=None, y=None): +def tmux_resize_pane(pane_id=None, x=None, y=None, **kwargs): """Resize pane to specific hieght or width.""" if not x and not y: raise Exception('Neither height nor width specified.') From 932669844baee7a485ece26f83f7885875245641 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:13:33 -0700 Subject: [PATCH 106/121] Fixed tmux pane size handling --- .bin/Scripts/functions/hw_diags.py | 29 +++++++++++++++-------------- .bin/Scripts/functions/tmux.py | 5 ++++- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 56b3c501..dc71c54f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -638,9 +638,9 @@ def fix_tmux_panes(state, tmux_layout): # Get pane size x, y = tmux_get_pane_size(pane_id=target) - if v.get('x', False) and v.get['x'] != x: + if v.get('x', False) and v['x'] != x: needs_fixed = True - if v.get('y', False) and v.get['y'] != y: + if v.get('y', False) and v['y'] != y: needs_fixed = True # Bail? @@ -895,7 +895,7 @@ def run_badblocks_test(state, test): pipe=True) while True: try: - test.badblocks_proc.wait(timeout=10) + test.badblocks_proc.wait(timeout=1) except subprocess.TimeoutExpired: fix_tmux_panes(state, test.tmux_layout) else: @@ -1108,8 +1108,7 @@ def run_io_benchmark(state, test): offset += test.dev.dd_chunk_blocks + skip # Fix panes - if i % 5 == 0: - fix_tmux_panes(state, test.tmux_layout) + fix_tmux_panes(state, test.tmux_layout) except DeviceTooSmallError: # Device too small, skipping test @@ -1276,8 +1275,7 @@ def run_mprime_test(state, test): update_sensor_data(test.sensor_data) # Fix panes - if i % 10 == 0: - fix_tmux_panes(state, test.tmux_layout) + fix_tmux_panes(state, test.tmux_layout) # Wait sleep(1) @@ -1474,10 +1472,17 @@ def run_nvme_smart_tests(state, test): cmd = ['sudo', 'smartctl', '--test=short', test.dev.path] run_program(cmd, check=False) - # Monitor progress (in 5 second increments) + # Monitor progress try: - for i in range(int(test.timeout*60/5)): - sleep(5) + for i in range(int(test.timeout*60)): + sleep(1) + + # Fix panes + fix_tmux_panes(state, test.tmux_layout) + + # Only update SMART progress every 5 seconds + if i % 5 != 0: + continue # Update SMART data test.dev.get_smart_details() @@ -1498,10 +1503,6 @@ def run_nvme_smart_tests(state, test): if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True - # Fix panes - if i % 2 == 0: - fix_tmux_panes(state, test.tmux_layout) - except KeyboardInterrupt: test.aborted = True test.report = test.dev.generate_attribute_report() diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 0e585df6..ce35f5b0 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -12,14 +12,17 @@ def tmux_get_pane_size(pane_id=None): """Get target, or current, pane size, returns tuple.""" x = -1 y = -1 - cmd = ['tmux', 'display', '-p', '#{pane_width}x#{pane_height}'] + cmd = ['tmux', 'display', '-p'] if pane_id: cmd.extend(['-t', pane_id]) + cmd.append('#{pane_width} #{pane_height}') # Run cmd and set x & y result = run_program(cmd, check=False) try: x, y = result.stdout.decode().strip().split() + x = int(x) + y = int(y) except Exception: # Ignore and return unrealistic values pass From 7ac035c578b2fe62b308a14d783b96d6f3fbc98d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:21:05 -0700 Subject: [PATCH 107/121] Safety wheels off --- .bin/Scripts/functions/hw_diags.py | 22 +++++++--------------- .bin/Scripts/settings/main.py | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index dc71c54f..cfe68c69 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -837,13 +837,11 @@ def menu_diags(state, args): elif selection == 'R': print('(FAKE) reboot...') sleep(1) - # TODO uncomment below - #run_program(['systemctl', 'reboot']) + run_program(['systemctl', 'reboot']) elif selection == 'P': print('(FAKE) poweroff...') sleep(1) - # TODO uncomment below - #run_program(['systemctl', 'poweroff']) + run_program(['systemctl', 'poweroff']) elif selection == 'Q': break elif selection == 'S': @@ -1242,8 +1240,7 @@ def run_mprime_test(state, test): message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', sensor_data=test.sensor_data, temp_label='Idle', - seconds=3) - # TODO: Remove seconds kwarg above + seconds=5) # Stress CPU print_log('Starting Prime95') @@ -1253,9 +1250,7 @@ def run_mprime_test(state, test): state.panes['mprime'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) - #time_limit = int(MPRIME_LIMIT) * 60 - # TODO: restore above line - time_limit = 30 + time_limit = int(MPRIME_LIMIT) * 60 try: for i in range(time_limit): clear_screen() @@ -1301,14 +1296,12 @@ def run_mprime_test(state, test): clear_screen() try_and_print( message='Letting CPU cooldown for bit...', indent=0, - function=sleep, cs='Done', seconds=3) - # TODO: Above seconds should be 10 + function=sleep, cs='Done', seconds=10) try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', sensor_data=test.sensor_data, temp_label='Cooldown', - seconds=3) - # TODO: Remove seconds kwarg above + seconds=5) # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -1447,8 +1440,7 @@ def run_nvme_smart_tests(state, test): # Prep test.timeout = test.dev.smart_self_test['polling_minutes'].get( 'short', 5) - # TODO: fix timeout, set to polling + 5 - test.timeout = int(test.timeout) + 1 + test.timeout = int(test.timeout) + 5 _include_short_test = True _self_test_started = False diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 7b915bdb..9d32b3ef 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -15,7 +15,7 @@ KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux -MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags +MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags ROOT_PASSWORD='Abracadabra' TECH_PASSWORD='Abracadabra' # Server IP addresses From 91a77bb14e9bfb1f9525aaffc6390e667e32c2d9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:47:03 -0700 Subject: [PATCH 108/121] Ensure SMART timeout message is in the report --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cfe68c69..64d417da 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -132,6 +132,7 @@ class DiskObj(): self.nvme_attributes = {} self.path = disk_path self.smart_attributes = {} + self.smart_timeout = False self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() @@ -331,12 +332,11 @@ class DiskObj(): # SMART short-test if short_test: report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) - if 'TimedOut' in self.tests['NVMe / SMART'].status: - report.append(' {YELLOW}UNKNOWN{CLEAR}: Timed out'.format(**COLORS)) - else: - report.append(' {}'.format( - self.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) + report.append(' {}'.format( + self.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) + if self.smart_timeout: + report.append(' {YELLOW}Timed out{CLEAR}'.format(**COLORS)) # Done return report @@ -1443,6 +1443,7 @@ def run_nvme_smart_tests(state, test): test.timeout = int(test.timeout) + 5 _include_short_test = True _self_test_started = False + _self_test_finished = False # Create monitor pane test.smart_out = '{}/smart_{}.out'.format( @@ -1488,6 +1489,7 @@ def run_nvme_smart_tests(state, test): # Check if test has finished if 'remaining_percent' not in test.dev.smart_self_test['status']: + _self_test_finished = True break else: @@ -1505,15 +1507,16 @@ def run_nvme_smart_tests(state, test): raise GenericAbort('Aborted') # Check if timed out - if 'passed' in test.dev.smart_self_test['status']: - if test.dev.smart_self_test['status']['passed']: + if _self_test_finished: + if test.dev.smart_self_test['status'].get('passed', False): if 'OVERRIDE' not in test.status: test.passed = True test.update_status('CS') else: test.failed = True test.update_status('NS') - if not (test.failed or test.passed): + else: + test.dev.smart_timeout = True test.update_status('TimedOut') # Disable other drive tests if necessary From e5f0ccb5d51bf8d89460c3a35eff26d10e8dc333 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:57:48 -0700 Subject: [PATCH 109/121] Formatting cleanup --- .bin/Scripts/functions/hw_diags.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 64d417da..b09dbf05 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -591,8 +591,8 @@ def build_outer_panes(state): # Started state.panes['Started'] = tmux_split_window( lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top'], - text='{BLUE}Started{CLEAR}\n{text}'.format( - text=time.strftime("%Y-%m-%d %H:%M %Z"), + text='{BLUE}Started{CLEAR}\n{s}'.format( + s=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) # Progress @@ -868,7 +868,8 @@ def run_badblocks_test(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + text='{}\nbadblocks: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'badblocks': {'y': 5, 'Check': True}, @@ -1037,8 +1038,8 @@ def run_io_benchmark(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nI/O Benchmark: {}'.format( - TOP_PANE_TEXT, test.dev.description)) + text='{}\nI/O Benchmark: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'io_benchmark': {'y': 1000, 'Check': False}, @@ -1406,7 +1407,8 @@ def run_nvme_smart_tests(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) + text='{}\nDisk Health: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'smart': {'y': 3, 'Check': True}, @@ -1554,8 +1556,8 @@ def show_results(state): """Show results for all tests.""" clear_screen() tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'Results')) + state.panes['Top'], + text='{}\nResults'.format(TOP_PANE_TEXT)) # CPU tests _enabled = False From ad9662c1208f00071d6ce4e721fd996e3e320035 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 16:38:40 -0700 Subject: [PATCH 110/121] Updated to use new hw_diags.py --- .bin/Scripts/ddrescue-tui-smart-display | 52 +++++++++++++------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display index 285229d6..65b76890 100755 --- a/.bin/Scripts/ddrescue-tui-smart-display +++ b/.bin/Scripts/ddrescue-tui-smart-display @@ -10,30 +10,34 @@ import time os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.hw_diags import * -#init_global_vars() +init_global_vars() if __name__ == '__main__': - try: - # Prep - clear_screen() - dev_path = sys.argv[1] - devs = scan_disks(True, dev_path) - - # Warn if SMART unavailable - if dev_path not in devs: - print_error('SMART data not available') - exit_script() - - # Initial screen - dev = devs[dev_path] - clear_screen() - show_disk_details(dev, only_attributes=True) - - # Done - exit_script() - except SystemExit: - pass - except: - major_exception() + try: + # Prep + clear_screen() + state = State() + state.init() -# vim: sts=4 sw=4 ts=4 + # Find disk + disk = None + for d in state.disks: + if d.path == sys.argv[1]: + disk = d + + # Show details + clear_screen() + if disk: + for line in disk.generate_attribute_report(timestamp=True): + print(line) + else: + print_error('Disk "{}" not found'.format(sys.argv[1])) + + # Done + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 From e1834d517953fd6a183be960770a4c52a91a31a5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 17:26:20 -0700 Subject: [PATCH 111/121] Added silent mode to init_global_vars() --- .bin/Scripts/functions/common.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 5327d895..82c65bb1 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -652,9 +652,10 @@ def wait_for_process(name, poll_rate=3): sleep(1) # global_vars functions -def init_global_vars(): +def init_global_vars(silent=False): """Sets global variables based on system info.""" - print_info('Initializing') + if not silent: + print_info('Initializing') if psutil.WINDOWS: os.system('title Wizard Kit') if psutil.LINUX: @@ -672,10 +673,14 @@ def init_global_vars(): ['Clearing collisions...', clean_env_vars], ] try: - for f in init_functions: - try_and_print( - message=f[0], function=f[1], - cs='Done', ns='Error', catch_all=False) + if silent: + for f in init_functions: + f[1]() + else: + for f in init_functions: + try_and_print( + message=f[0], function=f[1], + cs='Done', ns='Error', catch_all=False) except: major_exception() From 04cfdff2bf7a5f2f085caf165235a5a5219f261e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 17:27:15 -0700 Subject: [PATCH 112/121] Don't show init, just disk details --- .bin/Scripts/ddrescue-tui-smart-display | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display index 65b76890..4b8afcde 100755 --- a/.bin/Scripts/ddrescue-tui-smart-display +++ b/.bin/Scripts/ddrescue-tui-smart-display @@ -10,12 +10,11 @@ import time os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.hw_diags import * -init_global_vars() +init_global_vars(silent=True) if __name__ == '__main__': try: # Prep - clear_screen() state = State() state.init() @@ -26,7 +25,6 @@ if __name__ == '__main__': disk = d # Show details - clear_screen() if disk: for line in disk.generate_attribute_report(timestamp=True): print(line) From ad15cdad5612940940ba243b4b58911924fc3018 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 17:28:06 -0700 Subject: [PATCH 113/121] Added warning if not saving map to a preferred FS * Fixes #76 --- .bin/Scripts/functions/ddrescue.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 7f1fc7aa..6d722165 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -351,7 +351,34 @@ class RecoveryState(): self.set_pass_num() def self_checks(self): - """Run self-checks for each BlockPair and update state values.""" + """Run self-checks and update state values.""" + cmd = ['findmnt', '--json', '--target', os.getcwd()] + map_allowed_fstypes = RECOMMENDED_FSTYPES.copy() + map_allowed_fstypes.extend(['cifs', 'ext2', 'vfat']) + map_allowed_fstypes.sort() + json_data = {} + + # Avoid saving map to non-persistent filesystem + try: + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + except Exception: + print_error('ERROR: Failed to verify map path') + raise GenericAbort() + fstype = json_data.get( + 'filesystems', [{}])[0].get( + 'fstype', 'unknown') + if fstype not in map_allowed_fstypes: + print_error( + "Map isn't being saved to a recommended filesystem ({})".format( + fstype.upper())) + print_info('Recommended types are: {}'.format( + ' / '.join(map_allowed_fstypes).upper())) + print_standard(' ') + if not ask('Proceed anyways? (Strongly discouraged)'): + raise GenericAbort() + + # Run BlockPair self checks and get total size self.total_size = 0 for bp in self.block_pairs: bp.self_check() From 62b8e51705367034f03635aaab1d0e1e016e4a39 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 19:45:02 -0700 Subject: [PATCH 114/121] Updated ddrescue-tui tmux pane size handling --- .bin/Scripts/functions/ddrescue.py | 190 +++++++++++++++++++---------- .bin/Scripts/functions/hw_diags.py | 4 +- .bin/Scripts/functions/tmux.py | 2 +- 3 files changed, 130 insertions(+), 66 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 6d722165..6cbc9430 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -8,8 +8,10 @@ import signal import stat import time +from collections import OrderedDict from functions.common import * from functions.data import * +from functions.tmux import * from operator import itemgetter # STATIC VARIABLES @@ -30,6 +32,11 @@ DDRESCUE_SETTINGS = { } RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] SIDE_PANE_WIDTH = 21 +TMUX_LAYOUT = OrderedDict({ + 'Source': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, +}) USAGE = """ {script_name} clone [source [destination]] {script_name} image [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) @@ -276,6 +283,7 @@ class RecoveryState(): self.current_pass_str = '0: Initializing' self.settings = DDRESCUE_SETTINGS.copy() self.finished = False + self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.rescued = 0 self.resumed = False @@ -425,46 +433,22 @@ class RecoveryState(): # Functions def build_outer_panes(state): """Build top and side panes.""" - clear_screen() - result = run_program(['tput', 'cols']) - width = int( - (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 - - # Top panes - source_str = state.source.name - if len(source_str) > width: - source_str = '{}...'.format(source_str[:width-3]) - dest_str = state.dest.name - if len(dest_str) > width: - if state.mode == 'clone': - dest_str = '{}...'.format(dest_str[:width-3]) - else: - dest_str = '...{}'.format(dest_str[-width+3:]) - source_pane = tmux_splitw( - '-bdvl', '2', - '-PF', '#D', - 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( - text=source_str, - **COLORS)) - tmux_splitw( - '-t', source_pane, - '-dhl', '{}'.format(SIDE_PANE_WIDTH), - 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( - text=time.strftime("%Y-%m-%d %H:%M %Z"), - **COLORS)) - tmux_splitw( - '-t', source_pane, - '-dhp', '50', - 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( - text=dest_str, + state.panes['Source'] = tmux_split_window( + behind=True, vertical=True, lines=2, + text='{BLUE}Source{CLEAR}'.format(**COLORS)) + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Source'], + text='{BLUE}Started{CLEAR}\n{s}'.format( + s=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) + state.panes['Destination'] = tmux_split_window( + percent=50, target_pane=state.panes['Source'], + text='{BLUE}Destination{CLEAR}'.format(**COLORS)) # Side pane update_sidepane(state) - tmux_splitw( - '-dhl', str(SIDE_PANE_WIDTH), - 'watch', '--color', '--no-title', '--interval', '1', - 'cat', state.progress_out) + state.panes['Progress'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, watch=state.progress_out) def create_path_obj(path): @@ -491,6 +475,94 @@ def double_confirm_clone(): return ask('Asking again to confirm, is this correct?') +def fix_tmux_panes(state, forced=False): + """Fix pane sizes if the winodw has been resized.""" + needs_fixed = False + + # Check layout + for k, v in TMUX_LAYOUT.items(): + if not v.get('Check'): + # Not concerned with the size of this pane + continue + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Check pane size + x, y = tmux_get_pane_size(pane_id=target) + if v.get('x', False) and v['x'] != x: + needs_fixed = True + if v.get('y', False) and v['y'] != y: + needs_fixed = True + + # Bail? + if not needs_fixed and not forced: + return + + # Remove Destination pane (temporarily) + tmux_kill_pane(state.panes['Destination']) + + # Update layout + for k, v in TMUX_LAYOUT.items(): + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Resize pane + tmux_resize_pane(pane_id=target, **v) + + # Calc Source/Destination pane sizes + width, height = tmux_get_pane_size() + width = int(width / 2) - 1 + + # Update Source string + source_str = state.source.name + if len(source_str) > width: + source_str = '{}...'.format(source_str[:width-3]) + + # Update Destination string + dest_str = state.dest.name + if len(dest_str) > width: + if state.mode == 'clone': + dest_str = '{}...'.format(dest_str[:width-3]) + else: + dest_str = '...{}'.format(dest_str[-width+3:]) + + # Rebuild Source/Destination panes + tmux_update_pane( + pane_id=state.panes['Source'], + text='{BLUE}Source{CLEAR}\n{s}'.format( + s=source_str, **COLORS)) + state.panes['Destination'] = tmux_split_window( + percent=50, target_pane=state.panes['Source'], + text='{BLUE}Destination{CLEAR}\n{s}'.format( + s=dest_str, **COLORS)) + + if 'SMART' in state.panes: + # Calc SMART/ddrescue/Journal panes sizes + ratio = [12, 22, 4] + width, height = tmux_get_pane_size(pane_id=state.panes['Progress']) + height -= 2 + total = sum(ratio) + p_ratio = [int((x/total) * height) for x in ratio] + p_ratio[1] = height - p_ratio[0] - p_ratio[2] + + # Resize SMART/Journal panes + tmux_resize_pane(state.panes['SMART'], y=ratio[0]) + tmux_resize_pane(y=ratio[1]) + tmux_resize_pane(state.panes['Journal'], y=ratio[2]) + + def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -687,7 +759,9 @@ def menu_ddrescue(source_path, dest_path, run_mode): raise GenericAbort() # Main menu + clear_screen() build_outer_panes(state) + fix_tmux_panes(state, forced=True) menu_main(state) # Done @@ -877,29 +951,24 @@ def run_ddrescue(state, pass_settings): pause('Press Enter to return to main menu...') return - # Set heights - # NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) - result = run_program(['tput', 'lines']) - height = int(result.stdout.decode().strip()) - height_smart = int(height * (8 / 33)) - height_journal = int(height * (4 / 33)) - height_ddrescue = height - height_smart - height_journal - # Show SMART status smart_dev = state.source_path if state.source.parent: smart_dev = state.source.parent - smart_pane = tmux_splitw( - '-bdvl', str(height_smart), - '-PF', '#D', - 'watch', '--color', '--no-title', '--interval', '300', - 'ddrescue-tui-smart-display', smart_dev) + smart_cmd = [ + 'watch', '--color', '--no-title', '--interval', '5', + 'ddrescue-tui-smart-display', smart_dev, + ] + state.panes['SMART'] = tmux_split_window( + behind=True, lines=12, vertical=True, command=smart_cmd) # Show systemd journal output - journal_pane = tmux_splitw( - '-dvl', str(height_journal), - '-PF', '#D', - 'journalctl', '-f') + state.panes['Journal'] = tmux_split_window( + lines=4, vertical=True, + command=['sudo', 'journalctl', '-f']) + + # Fix layout + fix_tmux_panes(state, forced=True) # Run pass for each block-pair for bp in state.block_pairs: @@ -931,8 +1000,9 @@ def run_ddrescue(state, pass_settings): while True: bp.update_progress(state.current_pass) update_sidepane(state) + fix_tmux_panes(state) try: - ddrescue_proc.wait(timeout=10) + ddrescue_proc.wait(timeout=1) sleep(2) bp.update_progress(state.current_pass) update_sidepane(state) @@ -967,8 +1037,9 @@ def run_ddrescue(state, pass_settings): if str(return_code) != '0': # Pause on errors pause('Press Enter to return to main menu... ') - run_program(['tmux', 'kill-pane', '-t', smart_pane]) - run_program(['tmux', 'kill-pane', '-t', journal_pane]) + + # Cleanup + tmux_kill_pane(state.panes['SMART'], state.panes['Journal']) def select_parts(source_device): @@ -1219,13 +1290,6 @@ def show_usage(script_name): pause() -def tmux_splitw(*args): - """Run tmux split-window command and return output as str.""" - cmd = ['tmux', 'split-window', *args] - result = run_program(cmd) - return result.stdout.decode().strip() - - def update_sidepane(state): """Update progress file for side pane.""" output = [] diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index b09dbf05..9cef1d33 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -619,7 +619,7 @@ def build_status_string(label, status, info_label=False): **COLORS) def fix_tmux_panes(state, tmux_layout): - """Fix pane sizes in case the window has been resized.""" + """Fix pane sizes if the window has been resized.""" needs_fixed = False # Check layout @@ -636,7 +636,7 @@ def fix_tmux_panes(state, tmux_layout): else: target = state.panes[k] - # Get pane size + # Check pane size x, y = tmux_get_pane_size(pane_id=target) if v.get('x', False) and v['x'] != x: needs_fixed = True diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index ce35f5b0..84046d0b 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -69,7 +69,7 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - working_dir=None, command=None, + command=None, working_dir=None, text=None, watch=None, watch_cmd='cat'): """Run tmux split-window command and return pane_id as str.""" # Bail early From 44fe8882301095ae789de3b393a19a0ee92daca0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 20:15:35 -0700 Subject: [PATCH 115/121] Replaced ddrescue-tui-smart-display * Output data to file and have tmux pane watching said file * This method handles resizing much better --- .bin/Scripts/ddrescue-tui-smart-display | 41 ----------------------- .bin/Scripts/functions/ddrescue.py | 44 +++++++++++++++++++------ 2 files changed, 34 insertions(+), 51 deletions(-) delete mode 100755 .bin/Scripts/ddrescue-tui-smart-display diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display deleted file mode 100755 index 4b8afcde..00000000 --- a/.bin/Scripts/ddrescue-tui-smart-display +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: SMART attributes display for ddrescue TUI - -import os -import sys -import time - -# Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) -from functions.hw_diags import * -init_global_vars(silent=True) - -if __name__ == '__main__': - try: - # Prep - state = State() - state.init() - - # Find disk - disk = None - for d in state.disks: - if d.path == sys.argv[1]: - disk = d - - # Show details - if disk: - for line in disk.generate_attribute_report(timestamp=True): - print(line) - else: - print_error('Disk "{}" not found'.format(sys.argv[1])) - - # Done - exit_script() - except SystemExit: - pass - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 6cbc9430..00fb35f7 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -11,6 +11,7 @@ import time from collections import OrderedDict from functions.common import * from functions.data import * +from functions.hw_diags import * from functions.tmux import * from operator import itemgetter @@ -291,6 +292,7 @@ class RecoveryState(): self.total_size = 0 if mode not in ('clone', 'image'): raise GenericError('Unsupported mode') + self.get_smart_source() def add_block_pair(self, source, dest): """Run safety checks and append new BlockPair to internal list.""" @@ -349,6 +351,14 @@ class RecoveryState(): min_percent = min(min_percent, bp.rescued_percent) return min_percent + def get_smart_source(self): + """Get source for SMART dispay.""" + disk_path = self.source.path + if self.source.parent: + disk_path = self.source.parent + + self.smart_source = DiskObj(disk_path) + def retry_all_passes(self): """Mark all passes as pending for all block-pairs.""" self.finished = False @@ -951,16 +961,13 @@ def run_ddrescue(state, pass_settings): pause('Press Enter to return to main menu...') return - # Show SMART status - smart_dev = state.source_path - if state.source.parent: - smart_dev = state.source.parent - smart_cmd = [ - 'watch', '--color', '--no-title', '--interval', '5', - 'ddrescue-tui-smart-display', smart_dev, - ] + # Create SMART monitor pane + state.smart_out = '{}/smart_{}.out'.format( + global_vars['TmpDir'], state.smart_source.name) + with open(state.smart_out, 'w') as f: + f.write('Initializing...') state.panes['SMART'] = tmux_split_window( - behind=True, lines=12, vertical=True, command=smart_cmd) + behind=True, lines=12, vertical=True, watch=state.smart_out) # Show systemd journal output state.panes['Journal'] = tmux_split_window( @@ -997,10 +1004,26 @@ def run_ddrescue(state, pass_settings): clear_screen() print_info('Current dev: {}'.format(bp.source_path)) ddrescue_proc = popen_program(cmd) + i = 0 while True: + # Update SMART display (every 30 seconds) + i += 1 + if i % 30 == 0: + state.smart_source.get_smart_details() + with open(state.smart_out, 'w') as f: + report = state.smart_source.generate_attribute_report( + timestamp=True) + for line in report: + f.write('{}\n'.format(line)) + + # Update progress bp.update_progress(state.current_pass) update_sidepane(state) + + # Fix panes fix_tmux_panes(state) + + # Check if ddrescue has finished try: ddrescue_proc.wait(timeout=1) sleep(2) @@ -1008,8 +1031,9 @@ def run_ddrescue(state, pass_settings): update_sidepane(state) break except subprocess.TimeoutExpired: - # Catch to update bp/sidepane + # Catch to update smart/bp/sidepane pass + except KeyboardInterrupt: # Catch user abort pass From 42407f0eca0f75be518ecc34b99d8f09baad40c7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 20:35:21 -0700 Subject: [PATCH 116/121] Adjusted ddrescue exit handling * Wait for ddrescue_proc after KeyboardInterrupt * ddrescue prints extra info to the screen after a CTRL+c * Explicitly mark KeyboardInterrupt events as an abort * Add 'DDRESCUE PROCESS HALTED' message in red if exiting non-zero * More clearly indicates that user interaction is required * Fixes issue #72 --- .bin/Scripts/functions/ddrescue.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 00fb35f7..6e515655 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -953,7 +953,8 @@ def read_map_file(map_path): def run_ddrescue(state, pass_settings): """Run ddrescue pass.""" - return_code = None + return_code = -1 + aborted = False if state.finished: clear_screen() @@ -1007,7 +1008,6 @@ def run_ddrescue(state, pass_settings): i = 0 while True: # Update SMART display (every 30 seconds) - i += 1 if i % 30 == 0: state.smart_source.get_smart_details() with open(state.smart_out, 'w') as f: @@ -1015,6 +1015,7 @@ def run_ddrescue(state, pass_settings): timestamp=True) for line in report: f.write('{}\n'.format(line)) + i += 1 # Update progress bp.update_progress(state.current_pass) @@ -1036,7 +1037,8 @@ def run_ddrescue(state, pass_settings): except KeyboardInterrupt: # Catch user abort - pass + aborted = True + ddrescue_proc.wait(timeout=10) # Update progress/sidepane again bp.update_progress(state.current_pass) @@ -1044,12 +1046,19 @@ def run_ddrescue(state, pass_settings): # Was ddrescue aborted? return_code = ddrescue_proc.poll() - if return_code is None or return_code is 130: - clear_screen() + if aborted: + print_standard(' ') + print_standard(' ') + print_error('DDRESCUE PROCESS HALTED') + print_standard(' ') print_warning('Aborted') break elif return_code: - # i.e. not None and not 0 + # i.e. True when non-zero + print_standard(' ') + print_standard(' ') + print_error('DDRESCUE PROCESS HALTED') + print_standard(' ') print_error('Error(s) encountered, see message above.') break else: From f022d0ca76e369aa2a7f9792c58c2e98a5eae05a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 19 Dec 2018 18:45:58 -0700 Subject: [PATCH 117/121] Fallback to HW-Diags CLI if X fails to start * Fixes issue #74 --- .linux_items/include/airootfs/etc/skel/.update_x | 3 +++ .linux_items/include/airootfs/etc/skel/.zlogin | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/.linux_items/include/airootfs/etc/skel/.update_x b/.linux_items/include/airootfs/etc/skel/.update_x index 4763a175..650d3162 100755 --- a/.linux_items/include/airootfs/etc/skel/.update_x +++ b/.linux_items/include/airootfs/etc/skel/.update_x @@ -92,3 +92,6 @@ else cbatticon --hide-notification & fi +# Prevent Xorg from being killed by .zlogin +touch "/tmp/x_ok" + diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/.linux_items/include/airootfs/etc/skel/.zlogin index bd736280..56b47242 100644 --- a/.linux_items/include/airootfs/etc/skel/.zlogin +++ b/.linux_items/include/airootfs/etc/skel/.zlogin @@ -14,7 +14,18 @@ if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then # Start X or HW-diags if ! fgrep -q "nox" /proc/cmdline; then + # Kill Xorg after 30 seconds if it doesn't fully initialize + (sleep 30s; if ! [[ -f "/tmp/x_ok" ]]; then pkill '(Xorg|startx)'; fi) & + + # Try starting X startx >/dev/null + + # Run Hw-Diags CLI if necessary + if ! [[ -f "/tmp/x_ok" ]]; then + echo "There was an issue starting Xorg, starting CLI interface..." + sleep 2s + hw-diags cli + fi else hw-diags cli fi From c6eb7cdfd6e52fd1bcd533c49c74aeb8baecc3a0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 19 Dec 2018 18:53:13 -0700 Subject: [PATCH 118/121] Use new arguments when calling hw-diags --- .linux_items/include/airootfs/etc/skel/.config/i3/config | 2 +- .linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml | 2 +- .linux_items/include/airootfs/etc/skel/.zlogin | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.linux_items/include/airootfs/etc/skel/.config/i3/config b/.linux_items/include/airootfs/etc/skel/.config/i3/config index ca1a5439..102b50f5 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/i3/config +++ b/.linux_items/include/airootfs/etc/skel/.config/i3/config @@ -72,7 +72,7 @@ bindsym $mod+d exec "urxvt -title 'Hardware Diagnostics' -e hw-diags" bindsym $mod+f exec "thunar ~" bindsym $mod+i exec "hardinfo" bindsym $mod+m exec "urxvt -title 'Mount All Volumes' -e mount-all-volumes gui" -bindsym $mod+s exec "urxvt -title 'Hardware Diagnostics' -e hw-diags quick" +bindsym $mod+s exec "urxvt -title 'Hardware Diagnostics' -e hw-diags --quick" bindsym $mod+t exec "urxvt -e zsh -c 'tmux new-session -A -t general; zsh'" bindsym $mod+v exec "urxvt -title 'Hardware Sensors' -e watch -c -n1 -t hw-sensors" bindsym $mod+w exec "firefox" diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml index 90c7b0e0..e563ec04 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml +++ b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml @@ -324,7 +324,7 @@ - urxvt -title "Hardware Diagnostics" -e hw-diags quick + urxvt -title "Hardware Diagnostics" -e hw-diags --quick diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/.linux_items/include/airootfs/etc/skel/.zlogin index 56b47242..04b1316e 100644 --- a/.linux_items/include/airootfs/etc/skel/.zlogin +++ b/.linux_items/include/airootfs/etc/skel/.zlogin @@ -24,9 +24,9 @@ if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then if ! [[ -f "/tmp/x_ok" ]]; then echo "There was an issue starting Xorg, starting CLI interface..." sleep 2s - hw-diags cli + hw-diags --cli fi else - hw-diags cli + hw-diags --cli fi fi From eed8a1e40c48b205e40c422a32cc608a1934b080 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 20 Dec 2018 15:25:39 -0700 Subject: [PATCH 119/121] Fix poweroff/reboot calls --- .bin/Scripts/functions/hw_diags.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9cef1d33..9f10f997 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -835,13 +835,9 @@ def menu_diags(state, args): # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': - print('(FAKE) reboot...') - sleep(1) - run_program(['systemctl', 'reboot']) + run_program(['/usr/local/bin/wk-power-command', 'reboot']) elif selection == 'P': - print('(FAKE) poweroff...') - sleep(1) - run_program(['systemctl', 'poweroff']) + run_program(['/usr/local/bin/wk-power-command', 'poweroff']) elif selection == 'Q': break elif selection == 'S': From d60aab958408d9eba1ddd9c38dde8bdcd7c17a77 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 20 Dec 2018 16:03:54 -0700 Subject: [PATCH 120/121] Updated MemTest86 to 8.0 * Passmark is no longer providing ISOs so the UFD image is used instead * This is an alternative solution to issue #71 --- Build Linux | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Build Linux b/Build Linux index 56233d51..41e5dea5 100755 --- a/Build Linux +++ b/Build Linux @@ -170,13 +170,15 @@ function update_live_env() { # Memtest86 mkdir -p "$LIVE_DIR/EFI/memtest86/Benchmark" mkdir -p "$TEMP_DIR/memtest86" - curl -Lo "$TEMP_DIR/memtest86/memtest86.iso.tar.gz" "https://www.memtest86.com/downloads/memtest86-iso.tar.gz" - tar xvf "$TEMP_DIR/memtest86/memtest86.iso.tar.gz" -C "$TEMP_DIR/memtest86" - 7z x "$TEMP_DIR/memtest86"/*.iso -o"$TEMP_DIR/memtest86" - mv "$TEMP_DIR/memtest86/EFI/BOOT/BLACKLIS.CFG" "$LIVE_DIR/EFI/memtest86/blacklist.cfg" + curl -Lo "$TEMP_DIR/memtest86/memtest86-usb.zip" "https://www.memtest86.com/downloads/memtest86-usb.zip" + 7z e "$TEMP_DIR/memtest86/memtest86-usb.zip" -o"$TEMP_DIR/memtest86" "memtest86-usb.img" + 7z e "$TEMP_DIR/memtest86/memtest86-usb.img" -o"$TEMP_DIR/memtest86" "MemTest86.img" + 7z x "$TEMP_DIR/memtest86/MemTest86.img" -o"$TEMP_DIR/memtest86" + rm "$TEMP_DIR/memtest86/EFI/BOOT/BOOTIA32.EFI" mv "$TEMP_DIR/memtest86/EFI/BOOT/BOOTX64.EFI" "$LIVE_DIR/EFI/memtest86/memtestx64.efi" - mv "$TEMP_DIR/memtest86/EFI/BOOT/MT86.PNG" "$LIVE_DIR/EFI/memtest86/mt86.png" - mv "$TEMP_DIR/memtest86/EFI/BOOT/UNIFONT.BIN" "$LIVE_DIR/EFI/memtest86/unifont.bin" + for f in "$TEMP_DIR/memtest86/EFI/BOOT"/* "help"/* license.rtf; do + mv "$f" "$LIVE_DIR/EFI/memtest86"/ + done # build.sh if ! grep -iq 'wizardkit additions' "$LIVE_DIR/build.sh"; then From d930bdddbda5a72eccd2bbff88a8b5442b5f6c7e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 20 Dec 2018 16:10:08 -0700 Subject: [PATCH 121/121] Zero beginning of UFD before formatting * Fixes issue #68 --- .bin/Scripts/build-ufd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index c606892b..1253472c 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -533,6 +533,9 @@ echo -e "${GREEN}Building Kit${CLEAR}" touch "${LOG_FILE}" tmux split-window -dl 10 tail -f "${LOG_FILE}" +# Zero beginning of device +dd bs=4M count=16 if=/dev/zero of="${DEST_DEV}" >> "${LOG_FILE}" 2>&1 + # Format echo "Formatting drive..." if [[ "${USE_MBR}" == "True" ]]; then