From 10f2fca2bfa7e24d52e238e1d3ed611e5e9200fe Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 17:52:07 -0700 Subject: [PATCH 01/86] Added classes DevObj and State --- .bin/Scripts/functions/hw_diags.py | 1310 +++++----------------------- 1 file changed, 221 insertions(+), 1089 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 169eb337..830d948a 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1,1114 +1,246 @@ # Wizard Kit: Functions - HW Diagnostics import json +import re import time from functions.common import * # STATIC VARIABLES ATTRIBUTES = { - 'NVMe': { - 'critical_warning': {'Error': 1}, - 'media_errors': {'Error': 1}, - 'power_on_hours': {'Warning': 12000, 'Error': 18000, 'Ignore': True}, - 'unsafe_shutdowns': {'Warning': 1}, - }, - 'SMART': { - 5: {'Hex': '05', 'Error': 1}, - 9: {'Hex': '09', 'Warning': 12000, 'Error': 18000, 'Ignore': True}, - 10: {'Hex': '0A', 'Error': 1}, - 184: {'Hex': 'B8', 'Error': 1}, - 187: {'Hex': 'BB', 'Error': 1}, - 188: {'Hex': 'BC', 'Error': 1}, - 196: {'Hex': 'C4', 'Error': 1}, - 197: {'Hex': 'C5', 'Error': 1}, - 198: {'Hex': 'C6', 'Error': 1}, - 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - 201: {'Hex': 'C9', 'Error': 1}, - }, - } + 'NVMe': { + 'critical_warning': {'Error': 1}, + 'media_errors': {'Error': 1}, + 'power_on_hours': {'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 'unsafe_shutdowns': {'Warning': 1}, + }, + 'SMART': { + 5: {'Hex': '05', 'Error': 1}, + 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 10: {'Hex': '0A', 'Error': 1}, + 184: {'Hex': 'B8', 'Error': 1}, + 187: {'Hex': 'BB', 'Error': 1}, + 188: {'Hex': 'BC', 'Error': 1}, + 196: {'Hex': 'C4', 'Error': 1}, + 197: {'Hex': 'C5', 'Error': 1}, + 198: {'Hex': 'C6', 'Error': 1}, + 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + 201: {'Hex': 'C9', 'Error': 1}, + }, + } IO_VARS = { - 'Block Size': 512*1024, - 'Chunk Size': 32*1024**2, - 'Minimum Dev Size': 8*1024**3, - 'Minimum Test Size': 10*1024**3, - 'Alt Test Size Factor': 0.01, - 'Progress Refresh Rate': 5, - 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], - 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], - 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], - 'Threshold Graph Fail': 65*1024**2, - 'Threshold Graph Warn': 135*1024**2, - 'Threshold Graph Great': 750*1024**2, - 'Threshold HDD Min': 50*1024**2, - 'Threshold HDD High Avg': 75*1024**2, - 'Threshold HDD Low Avg': 65*1024**2, - 'Threshold SSD Min': 90*1024**2, - 'Threshold SSD High Avg': 135*1024**2, - 'Threshold SSD Low Avg': 100*1024**2, - 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), - 'Graph Horizontal Width': 40, - 'Graph Vertical': ( - '▏', '▎', '▍', '▌', - '▋', '▊', '▉', '█', - '█▏', '█▎', '█▍', '█▌', - '█▋', '█▊', '█▉', '██', - '██▏', '██▎', '██▍', '██▌', - '██▋', '██▊', '██▉', '███', - '███▏', '███▎', '███▍', '███▌', - '███▋', '███▊', '███▉', '████'), - } -TESTS = { - 'Prime95': { - 'Enabled': False, - 'Status': 'Pending', - }, - 'NVMe/SMART': { - 'Enabled': False, - 'Quick': False, - 'Short Test': {}, - 'Status': {}, - }, - 'badblocks': { - 'Enabled': False, - 'Results': {}, - 'Status': {}, - }, - 'iobenchmark': { - 'Data': {}, - 'Enabled': False, - 'Results': {}, - 'Status': {}, - }, - } + 'Block Size': 512*1024, + 'Chunk Size': 32*1024**2, + 'Minimum Dev Size': 8*1024**3, + 'Minimum Test Size': 10*1024**3, + 'Alt Test Size Factor': 0.01, + 'Progress Refresh Rate': 5, + 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], + 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], + 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], + 'Threshold Graph Fail': 65*1024**2, + 'Threshold Graph Warn': 135*1024**2, + 'Threshold Graph Great': 750*1024**2, + 'Threshold HDD Min': 50*1024**2, + 'Threshold HDD High Avg': 75*1024**2, + 'Threshold HDD Low Avg': 65*1024**2, + 'Threshold SSD Min': 90*1024**2, + 'Threshold SSD High Avg': 135*1024**2, + 'Threshold SSD Low Avg': 100*1024**2, + 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), + 'Graph Horizontal Width': 40, + 'Graph Vertical': ( + '▏', '▎', '▍', '▌', + '▋', '▊', '▉', '█', + '█▏', '█▎', '█▍', '█▌', + '█▋', '█▊', '█▉', '██', + '██▏', '██▎', '██▍', '██▌', + '██▋', '██▊', '██▉', '███', + '███▏', '███▎', '███▍', '███▌', + '███▋', '███▊', '███▉', '████'), + } +KEY_NVME = 'nvme_smart_health_information_log' +KEY_SMART = 'ata_smart_attributes' +SIDE_PATH_WIDTH = 21 +# Classes +class DevObj(): + """Device object for tracking device specific data.""" + def __init__(self, dev_path): + self.failing = False + self.nvme_attributes = {} + self.override = False + self.path = dev_path + self.smart_attributes = {} + self.tests = { + 'NVMe / SMART': {'Result': None, 'Status': None}, + 'badblocks': {'Result': None, 'Status': None}, + 'I/O Benchmark': { + 'Result': None, + 'Status': None, + 'Read Rates': [], + 'Graph Data': []}, + } + self.get_details() + + def get_details(self): + """Get data from smartctl.""" + cmd = ['sudo', 'smartctl', '--all', '--json', self.path] + result = run_program(cmd, check=False) + self.data = json.loads(result.stdout.decode()) + + # Check for attributes + if KEY_NVME in self.data: + self.nvme_attributes.update(self.data[KEY_NVME]) + elif KEY_SMART in self.data: + for a in self.data[KEY_SMART].get('table', {}): + _id = str(a.get('id', 'UNKNOWN')) + _name = str(a.get('name', 'UNKNOWN')) + _raw = a.get('raw', {}).get('value', -1) + _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') + + # Fix power-on time + _r = re.match(r'^(\d+)[Hh].*', _raw_str) + if _id == '9' and _r: + try: + _raw = int(_r.group(1)) + except ValueError: + # That's fine + pass + self.smart_attributes[_id] = { + 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + +class State(): + """Object to track device objects and overall state.""" + def __init__(self): + self.devs = [] + self.finished = False + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.started = False + self.tests = { + 'Prime95': {'Enabled': False, 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False}, + 'badblocks': {'Enabled': False}, + 'I/O Benchmark': {'Enabled': False}, + } + self.add_devs() + + def add_devs(self): + """Add all block devices listed by lsblk.""" + cmd = ['lsblk', '--json', '--nodeps', '--paths'] + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + for dev in json_data['blockdevices']: + self.devs.append(DevObj(dev['name'])) + +# Functions def generate_horizontal_graph(rates, oneline=False): - """Generate two-line horizontal graph from rates, returns str.""" - line_1 = '' - line_2 = '' - line_3 = '' - line_4 = '' - for r in rates: - step = get_graph_step(r, scale=32) - if oneline: - step = get_graph_step(r, scale=8) - - # Set color - r_color = COLORS['CLEAR'] - if r < IO_VARS['Threshold Graph Fail']: - r_color = COLORS['RED'] - elif r < IO_VARS['Threshold Graph Warn']: - r_color = COLORS['YELLOW'] - elif r > IO_VARS['Threshold Graph Great']: - r_color = COLORS['GREEN'] - - # Build graph - full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) - if step >= 24: - line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) - line_2 += full_block - line_3 += full_block - line_4 += full_block - elif step >= 16: - line_1 += ' ' - line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) - line_3 += full_block - line_4 += full_block - elif step >= 8: - line_1 += ' ' - line_2 += ' ' - line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) - line_4 += full_block - else: - line_1 += ' ' - line_2 += ' ' - line_3 += ' ' - line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) - line_1 += COLORS['CLEAR'] - line_2 += COLORS['CLEAR'] - line_3 += COLORS['CLEAR'] - line_4 += COLORS['CLEAR'] + """Generate two-line horizontal graph from rates, returns str.""" + line_1 = '' + line_2 = '' + line_3 = '' + line_4 = '' + for r in rates: + step = get_graph_step(r, scale=32) if oneline: - return line_4 + step = get_graph_step(r, scale=8) + + # Set color + r_color = COLORS['CLEAR'] + if r < IO_VARS['Threshold Graph Fail']: + r_color = COLORS['RED'] + elif r < IO_VARS['Threshold Graph Warn']: + r_color = COLORS['YELLOW'] + elif r > IO_VARS['Threshold Graph Great']: + r_color = COLORS['GREEN'] + + # Build graph + full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) + if step >= 24: + line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) + line_2 += full_block + line_3 += full_block + line_4 += full_block + elif step >= 16: + line_1 += ' ' + line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) + line_3 += full_block + line_4 += full_block + elif step >= 8: + line_1 += ' ' + line_2 += ' ' + line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + line_4 += full_block else: - return '\n'.join([line_1, line_2, line_3, line_4]) + line_1 += ' ' + line_2 += ' ' + line_3 += ' ' + line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + line_1 += COLORS['CLEAR'] + line_2 += COLORS['CLEAR'] + line_3 += COLORS['CLEAR'] + line_4 += COLORS['CLEAR'] + if oneline: + return line_4 + else: + return '\n'.join([line_1, line_2, line_3, line_4]) def get_graph_step(rate, scale=16): - """Get graph step based on rate and scale, returns int.""" - m_rate = rate / (1024**2) - step = 0 - scale_name = 'Scale {}'.format(scale) - for x in range(scale-1, -1, -1): - # Iterate over scale backwards - if m_rate >= IO_VARS[scale_name][x]: - step = x - break - return step + """Get graph step based on rate and scale, returns int.""" + m_rate = rate / (1024**2) + step = 0 + scale_name = 'Scale {}'.format(scale) + for x in range(scale-1, -1, -1): + # Iterate over scale backwards + if m_rate >= IO_VARS[scale_name][x]: + step = x + break + return step def get_read_rate(s): - """Get read rate in bytes/s from dd progress output.""" - real_rate = None - if re.search(r'[KMGT]B/s', s): - human_rate = re.sub(r'^.*\s+(\d+\.?\d*)\s+(.B)/s\s*$', r'\1 \2', s) - real_rate = convert_to_bytes(human_rate) - return real_rate - -def get_smart_details(dev): - """Get SMART data for dev if possible, returns dict.""" - cmd = 'sudo smartctl --all --json {}{}'.format( - '' if '/dev/' in dev else '/dev/', - dev).split() - result = run_program(cmd, check=False) - try: - return json.loads(result.stdout.decode()) - except Exception: - # Let other sections deal with the missing data - return {} - -def get_smart_value(smart_data, smart_id): - """Get SMART value from table, returns int or None.""" - value = None - table = smart_data.get('ata_smart_attributes', {}).get('table', []) - for row in table: - if str(row.get('id', '?')) == str(smart_id): - value = row.get('raw', {}).get('value', None) - return value + """Get read rate in bytes/s from dd progress output.""" + real_rate = None + if re.search(r'[KMGT]B/s', s): + human_rate = re.sub(r'^.*\s+(\d+\.?\d*)\s+(.B)/s\s*$', r'\1 \2', s) + real_rate = convert_to_bytes(human_rate) + return real_rate def get_status_color(s): - """Get color based on status, returns str.""" - color = COLORS['CLEAR'] - if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: - color = COLORS['RED'] - elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: - color = COLORS['YELLOW'] - elif s in ['CS']: - color = COLORS['GREEN'] - return color - -def menu_diags(*args): - """Main HW-Diagnostic menu.""" - diag_modes = [ - {'Name': 'All tests', - 'Tests': ['Prime95', 'NVMe/SMART', 'badblocks', 'iobenchmark']}, - {'Name': 'Prime95', - 'Tests': ['Prime95']}, - {'Name': 'All drive tests', - 'Tests': ['NVMe/SMART', 'badblocks', 'iobenchmark']}, - {'Name': 'NVMe/SMART', - 'Tests': ['NVMe/SMART']}, - {'Name': 'badblocks', - 'Tests': ['badblocks']}, - {'Name': 'I/O Benchmark', - 'Tests': ['iobenchmark']}, - {'Name': 'Quick drive test', - 'Tests': ['Quick', 'NVMe/SMART']}, - ] - actions = [ - {'Letter': 'A', 'Name': 'Audio test'}, - {'Letter': 'K', 'Name': 'Keyboard test'}, - {'Letter': 'N', 'Name': 'Network test'}, - {'Letter': 'M', 'Name': 'Screen Saver - Matrix', 'CRLF': True}, - {'Letter': 'P', 'Name': 'Screen Saver - Pipes'}, - {'Letter': 'Q', 'Name': 'Quit', 'CRLF': True}, - ] - - # CLI-mode actions - if 'DISPLAY' not in global_vars['Env']: - actions.extend([ - {'Letter': 'R', 'Name': 'Reboot', 'CRLF': True}, - {'Letter': 'S', 'Name': 'Shutdown'}, - ]) - - # Quick disk check - if 'quick' in args: - run_tests(['Quick', 'NVMe/SMART']) - exit_script() - - # Show menu - while True: - selection = menu_select( - title = 'Hardware Diagnostics: Menu', - main_entries = diag_modes, - action_entries = actions, - spacer = '──────────────────────────') - if selection.isnumeric(): - ticket_number = None - if diag_modes[int(selection)-1]['Name'] != 'Quick drive test': - clear_screen() - print_standard(' ') - ticket_number = get_ticket_number() - # Save log for non-quick tests - global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") - global_vars['LogDir'] = '{}/Logs/{}_{}'.format( - global_vars['Env']['HOME'], - ticket_number, - global_vars['Date-Time']) - os.makedirs(global_vars['LogDir'], exist_ok=True) - global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( - global_vars['LogDir']) - run_tests(diag_modes[int(selection)-1]['Tests'], ticket_number) - elif selection == 'A': - run_program(['hw-diags-audio'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') - elif selection == 'K': - run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) - elif selection == 'N': - run_program(['hw-diags-network'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') - elif selection == 'M': - run_program(['cmatrix', '-abs'], check=False, pipe=False) - elif selection == 'P': - run_program( - 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split(), - check=False, pipe=False) - elif selection == 'R': - run_program(['systemctl', 'reboot']) - elif selection == 'S': - run_program(['systemctl', 'poweroff']) - elif selection == 'Q': - break - -def run_badblocks(ticket_number): - """Run a read-only test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart badblocks test(s)\n') - progress_file = '{}/badblocks_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['badblocks']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - print_standard('Running badblock test(s):') - for name, dev in sorted(TESTS['badblocks']['Devices'].items()): - cur_status = TESTS['badblocks']['Status'][name] - nvme_smart_status = TESTS['NVMe/SMART']['Status'].get(name, None) - if cur_status == 'Denied': - # Skip denied disks - continue - if nvme_smart_status == 'NS': - TESTS['badblocks']['Status'][name] = 'Skipped' - else: - # Not testing SMART, SMART CS, or SMART OVERRIDE - TESTS['badblocks']['Status'][name] = 'Working' - update_progress() - print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - run_program('tmux split-window -dl 5 {} {} {}'.format( - 'hw-diags-badblocks', - '/dev/{}'.format(name), - progress_file).split()) - wait_for_process('badblocks') - print_standard('Done', timestamp=False) - - # Check results - if os.path.exists(progress_file): - with open(progress_file, 'r') as f: - text = f.read() - TESTS['badblocks']['Results'][name] = text - r = re.search(r'Pass completed.*0/0/0 errors', text) - if r: - TESTS['badblocks']['Status'][name] = 'CS' - else: - TESTS['badblocks']['Status'][name] = 'NS' - - # Move temp file - shutil.move(progress_file, '{}/badblocks-{}.log'.format( - global_vars['LogDir'], name)) - else: - TESTS['badblocks']['Status'][name] = 'NS' - update_progress() - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - pass - -def run_iobenchmark(ticket_number): - """Run a read-only test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart I/O Benchmark test(s)\n') - progress_file = '{}/iobenchmark_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['iobenchmark']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - print_standard('Running benchmark test(s):') - for name, dev in sorted(TESTS['iobenchmark']['Devices'].items()): - cur_status = TESTS['iobenchmark']['Status'][name] - nvme_smart_status = TESTS['NVMe/SMART']['Status'].get(name, None) - bb_status = TESTS['badblocks']['Status'].get(name, None) - if cur_status == 'Denied': - # Skip denied disks - continue - if nvme_smart_status == 'NS': - TESTS['iobenchmark']['Status'][name] = 'Skipped' - elif bb_status in ['NS', 'Skipped']: - TESTS['iobenchmark']['Status'][name] = 'Skipped' - else: - # (SMART tests not run or CS/OVERRIDE) - # AND (BADBLOCKS tests not run or CS) - TESTS['iobenchmark']['Status'][name] = 'Working' - update_progress() - print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - - # Get dev size - cmd = 'sudo lsblk -bdno size /dev/{}'.format(name) - try: - result = run_program(cmd.split()) - dev_size = result.stdout.decode().strip() - dev_size = int(dev_size) - except: - # Failed to get dev size, requires manual testing instead - TESTS['iobenchmark']['Status'][name] = 'ERROR' - continue - if dev_size < IO_VARS['Minimum Dev Size']: - TESTS['iobenchmark']['Status'][name] = 'ERROR' - continue - - # Calculate dd values - ## test_size is the area to be read in bytes - ## If the dev is < 10Gb then it's the whole dev - ## Otherwise it's the larger of 10Gb or 1% of the dev - ## - ## test_chunks is the number of groups of "Chunk Size" in test_size - ## This number is reduced to a multiple of the graph width in - ## order to allow for the data to be condensed cleanly - ## - ## skip_blocks is the number of "Block Size" groups not tested - ## skip_count is the number of blocks to skip per test_chunk - ## skip_extra is how often to add an additional skip block - ## This is needed to ensure an even testing across the dev - ## This is calculated by using the fractional amount left off - ## of the skip_count variable - test_size = min(IO_VARS['Minimum Test Size'], dev_size) - test_size = max( - test_size, dev_size*IO_VARS['Alt Test Size Factor']) - test_chunks = int(test_size // IO_VARS['Chunk Size']) - test_chunks -= test_chunks % IO_VARS['Graph Horizontal Width'] - test_size = test_chunks * IO_VARS['Chunk Size'] - skip_blocks = int((dev_size - test_size) // IO_VARS['Block Size']) - skip_count = int((skip_blocks / test_chunks) // 1) - skip_extra = 0 - try: - skip_extra = 1 + int(1 / ((skip_blocks / test_chunks) % 1)) - except ZeroDivisionError: - # skip_extra == 0 is fine - pass - - # Open dd progress pane after initializing file - with open(progress_file, 'w') as f: - f.write('') - sleep(1) - cmd = 'tmux split-window -dp 75 -PF #D tail -f {}'.format( - progress_file) - result = run_program(cmd.split()) - bottom_pane = result.stdout.decode().strip() - - # Run dd read tests - offset = 0 - TESTS['iobenchmark']['Data'][name] = { - 'Graph': [], - 'Read Rates': []} - for i in range(test_chunks): - i += 1 - s = skip_count - c = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) - if skip_extra and i % skip_extra == 0: - s += 1 - cmd = 'sudo dd bs={b} skip={s} count={c} if=/dev/{n} of={o} iflag=direct'.format( - b=IO_VARS['Block Size'], - s=offset+s, - c=c, - n=name, - o='/dev/null') - result = run_program(cmd.split()) - result_str = result.stderr.decode().replace('\n', '') - cur_rate = get_read_rate(result_str) - TESTS['iobenchmark']['Data'][name]['Read Rates'].append( - cur_rate) - TESTS['iobenchmark']['Data'][name]['Graph'].append( - '{percent:0.1f} {rate}'.format( - percent=i/test_chunks*100, - rate=int(cur_rate/(1024**2)))) - if i % IO_VARS['Progress Refresh Rate'] == 0: - # Update vertical graph - update_io_progress( - percent=i/test_chunks*100, - rate=cur_rate, - progress_file=progress_file) - # Update offset - offset += s + c - print_standard('Done', timestamp=False) - - # Close bottom pane - run_program(['tmux', 'kill-pane', '-t', bottom_pane]) - - # Build report - avg_min_max = 'Average read speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( - sum(TESTS['iobenchmark']['Data'][name]['Read Rates'])/len( - TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2), - min(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2), - max(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2)) - TESTS['iobenchmark']['Data'][name]['Avg/Min/Max'] = avg_min_max - TESTS['iobenchmark']['Data'][name]['Merged Rates'] = [] - pos = 0 - width = int(test_chunks / IO_VARS['Graph Horizontal Width']) - for i in range(IO_VARS['Graph Horizontal Width']): - # Append average rate for WIDTH number of rates to new array - TESTS['iobenchmark']['Data'][name]['Merged Rates'].append(sum( - TESTS['iobenchmark']['Data'][name]['Read Rates'][pos:pos+width])/width) - pos += width - report = generate_horizontal_graph( - TESTS['iobenchmark']['Data'][name]['Merged Rates']) - report += '\n{}'.format(avg_min_max) - TESTS['iobenchmark']['Results'][name] = report - - # Set CS/NS - min_read = min(TESTS['iobenchmark']['Data'][name]['Read Rates']) - avg_read = sum( - TESTS['iobenchmark']['Data'][name]['Read Rates'])/len( - TESTS['iobenchmark']['Data'][name]['Read Rates']) - dev_rotational = dev['lsblk'].get('rota', None) - if dev_rotational == "0": - # Use SSD scale - thresh_min = IO_VARS['Threshold SSD Min'] - thresh_high_avg = IO_VARS['Threshold SSD High Avg'] - thresh_low_avg = IO_VARS['Threshold SSD Low Avg'] - else: - # Use HDD scale - thresh_min = IO_VARS['Threshold HDD Min'] - thresh_high_avg = IO_VARS['Threshold HDD High Avg'] - thresh_low_avg = IO_VARS['Threshold HDD Low Avg'] - if min_read <= thresh_min and avg_read <= thresh_high_avg: - TESTS['iobenchmark']['Status'][name] = 'NS' - elif avg_read <= thresh_low_avg: - TESTS['iobenchmark']['Status'][name] = 'NS' - else: - TESTS['iobenchmark']['Status'][name] = 'CS' - - # Save logs - dest_filename = '{}/iobenchmark-{}.log'.format(global_vars['LogDir'], name) - shutil.move(progress_file, dest_filename) - with open(dest_filename.replace('.', '-raw.'), 'a') as f: - f.write('\n'.join(TESTS['iobenchmark']['Data'][name]['Graph'])) - update_progress() - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - pass - -def run_mprime(ticket_number): - """Run Prime95 for MPRIME_LIMIT minutes while showing the temps.""" - aborted = False - print_log('\nStart Prime95 test') - TESTS['Prime95']['Status'] = 'Working' - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dl 10 -c {wd} {cmd} {wd}'.format( - wd=global_vars['TmpDir'], cmd='hw-diags-prime95').split()) - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - run_program('tmux split-window -bd watch -c -n1 -t hw-sensors'.split()) - run_program('tmux resize-pane -y 3'.split()) - - # Start test - run_program(['apple-fans', 'max']) - try: - for i in range(int(MPRIME_LIMIT)): - clear_screen() - min_left = int(MPRIME_LIMIT) - i - print_standard('Running Prime95 ({} minute{} left)'.format( - min_left, - 's' if min_left != 1 else '')) - print_warning('If running too hot, press CTRL+c to abort the test') - sleep(60) - except KeyboardInterrupt: - # Catch CTRL+C - aborted = True - TESTS['Prime95']['Status'] = 'Aborted' - print_warning('\nAborted.') - update_progress() - - # Save "final" temps - run_program( - cmd = 'hw-sensors >> "{}/Final Temps.out"'.format( - global_vars['LogDir']).split(), - check = False, - pipe = False, - shell = True) - run_program( - cmd = 'hw-sensors --nocolor >> "{}/Final Temps.log"'.format( - global_vars['LogDir']).split(), - check = False, - pipe = False, - shell = True) - - # Stop test - run_program('killall -s INT mprime'.split(), check=False) - run_program(['apple-fans', 'auto']) - - # Move logs to Ticket folder - for item in os.scandir(global_vars['TmpDir']): - try: - shutil.move(item.path, global_vars['LogDir']) - except Exception: - print_error('ERROR: Failed to move "{}" to "{}"'.format( - item.path, - global_vars['LogDir'])) - - # Check logs - TESTS['Prime95']['NS'] = False - TESTS['Prime95']['CS'] = False - log = '{}/results.txt'.format(global_vars['LogDir']) - if os.path.exists(log): - with open(log, 'r') as f: - text = f.read() - TESTS['Prime95']['results.txt'] = text - r = re.search(r'(error|fail)', text) - TESTS['Prime95']['NS'] = bool(r) - log = '{}/prime.log'.format(global_vars['LogDir']) - if os.path.exists(log): - with open(log, 'r') as f: - text = f.read() - TESTS['Prime95']['prime.log'] = text - r = re.search(r'completed.*0 errors, 0 warnings', text) - TESTS['Prime95']['CS'] = bool(r) - - # Update status - if not aborted: - if TESTS['Prime95']['NS']: - TESTS['Prime95']['Status'] = 'NS' - elif TESTS['Prime95']['CS']: - TESTS['Prime95']['Status'] = 'CS' - else: - TESTS['Prime95']['Status'] = 'Unknown' - update_progress() - - if aborted: - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled']: - if not ask('Proceed to next test?'): - for name in TESTS['NVMe/SMART']['Devices'].keys(): - for t in ['NVMe/SMART', 'badblocks', 'iobenchmark']: - cur_status = TESTS[t]['Status'][name] - if cur_status not in ['CS', 'Denied', 'NS']: - TESTS[t]['Status'][name] = 'Aborted' - run_program('tmux kill-pane -a'.split()) - raise GenericError - - # Done - run_program('tmux kill-pane -a'.split()) - -def run_nvme_smart(ticket_number): - """Run the built-in NVMe or SMART test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart NVMe/SMART test(s)\n') - progress_file = '{}/selftest_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dl 3 watch -c -n1 -t cat {}'.format( - progress_file).split()) - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - TESTS['NVMe/SMART']['Short Test'][name] = None - cur_status = TESTS['NVMe/SMART']['Status'][name] - if cur_status == 'OVERRIDE': - # Skipping test per user request - continue - if TESTS['NVMe/SMART']['Quick'] or dev.get('NVMe Disk', False): - # Skip SMART self-tests for quick checks and NVMe disks - if dev['Quick Health OK']: - TESTS['NVMe/SMART']['Status'][name] = 'CS' - else: - TESTS['NVMe/SMART']['Status'][name] = 'NS' - elif not dev['Quick Health OK']: - # SMART overall == Failed or attributes bad, avoid self-test - TESTS['NVMe/SMART']['Status'][name] = 'NS' - else: - # Start SMART short self-test - test_length = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'polling_minutes', {}).get( - 'short', 5) - test_length = int(test_length) + 5 - TESTS['NVMe/SMART']['Status'][name] = 'Working' - update_progress() - print_standard('Running SMART short self-test(s):') - print_standard( - ' /dev/{:8}({} minutes)... '.format(name, test_length), - end='', flush=True) - run_program( - 'sudo smartctl -t short /dev/{}'.format(name).split(), - check=False) - - # Wait and show progress (in 10 second increments) - for iteration in range(int(test_length*60/10)): - # Update SMART data - dev['smartctl'] = get_smart_details(name) - - # Check if test is complete - if iteration >= 6: - done = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'passed', False) - if done: - break - - # Update progress_file - with open(progress_file, 'w') as f: - f.write('SMART self-test status:\n {}'.format( - dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'string', 'unknown'))) - sleep(10) - os.remove(progress_file) - - # Check result - test_passed = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'passed', False) - if test_passed: - TESTS['NVMe/SMART']['Status'][name] = 'CS' - TESTS['NVMe/SMART']['Short Test'][name] = 'CS' - else: - TESTS['NVMe/SMART']['Status'][name] = 'NS' - TESTS['NVMe/SMART']['Short Test'][name] = 'NS' - update_progress() - print_standard('Done', timestamp=False) - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - -def run_tests(tests, ticket_number=None): - """Run selected hardware test(s).""" - clear_screen() - print_standard('Starting Hardware Diagnostics') - if ticket_number: - print_standard(' For Ticket #{}'.format(ticket_number)) - print_standard(' ') - print_standard('Running tests: {}'.format(', '.join(tests))) - # Enable selected tests - for t in ['Prime95', 'NVMe/SMART', 'badblocks', 'iobenchmark']: - TESTS[t]['Enabled'] = t in tests - TESTS['NVMe/SMART']['Quick'] = 'Quick' in tests - - # Initialize - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - print_standard(' ') - scan_disks() - update_progress() - - # Run - mprime_aborted = False - if TESTS['Prime95']['Enabled']: - try: - run_mprime(ticket_number) - except GenericError: - mprime_aborted = True - if not mprime_aborted: - if TESTS['NVMe/SMART']['Enabled']: - run_nvme_smart(ticket_number) - if TESTS['badblocks']['Enabled']: - run_badblocks(ticket_number) - if TESTS['iobenchmark']['Enabled']: - run_iobenchmark(ticket_number) - - # Show results - show_results() - - # Open log - if not TESTS['NVMe/SMART']['Quick'] and ENABLED_OPEN_LOGS: - try: - popen_program(['nohup', 'leafpad', global_vars['LogFile']], pipe=True) - except Exception: - print_error('ERROR: Failed to open log: {}'.format( - global_vars['LogFile'])) - pause('Press Enter to exit...') - -def scan_disks(full_paths=False, only_path=None): - """Scan for disks eligible for hardware testing.""" - - # Get eligible disk list - cmd = ['lsblk', '-J', '-O'] - if full_paths: - cmd.append('-p') - if only_path: - cmd.append(only_path) - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - devs = {} - for d in json_data.get('blockdevices', []): - if d['type'] == 'disk': - if d['hotplug'] == '0': - devs[d['name']] = {'lsblk': d} - TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' - TESTS['badblocks']['Status'][d['name']] = 'Pending' - TESTS['iobenchmark']['Status'][d['name']] = 'Pending' - else: - # Skip WizardKit devices - skip_dev=False - wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) - for c in d.get('children', []): - r = re.search( - wk_label_regex, c.get('label', ''), re.IGNORECASE) - skip_dev = bool(r) - if not skip_dev: - devs[d['name']] = {'lsblk': d} - TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' - TESTS['badblocks']['Status'][d['name']] = 'Pending' - TESTS['iobenchmark']['Status'][d['name']] = 'Pending' - - for dev, data in devs.items(): - # Get SMART attributes - run_program( - cmd = 'sudo smartctl -s on {}{}'.format( - '' if full_paths else '/dev/', - dev).split(), - check = False) - data['smartctl'] = get_smart_details(dev) - - # Get NVMe attributes - if data['lsblk']['tran'] == 'nvme': - cmd = 'sudo nvme smart-log /dev/{} -o json'.format(dev).split() - cmd = 'sudo nvme smart-log {}{} -o json'.format( - '' if full_paths else '/dev/', - dev).split() - result = run_program(cmd, check=False) - try: - data['nvme-cli'] = json.loads(result.stdout.decode()) - except Exception: - # Let other sections deal with the missing data - data['nvme-cli'] = {} - data['NVMe Disk'] = True - - # Set "Quick Health OK" value - ## NOTE: If False then require override for badblocks test - wanted_smart_list = [ - 'ata_smart_attributes', - 'ata_smart_data', - 'smart_status', - ] - if data.get('NVMe Disk', False): - crit_warn = data['nvme-cli'].get('critical_warning', 1) - if crit_warn == 0: - dev_name = data['lsblk']['name'] - data['Quick Health OK'] = True - TESTS['NVMe/SMART']['Status'][dev_name] = 'CS' - else: - data['Quick Health OK'] = False - elif set(wanted_smart_list).issubset(data['smartctl'].keys()): - data['SMART Pass'] = data['smartctl'].get('smart_status', {}).get( - 'passed', False) - data['Quick Health OK'] = data['SMART Pass'] - data['SMART Support'] = True - else: - data['Quick Health OK'] = False - data['SMART Support'] = False - - # Ask for manual overrides if necessary - if TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - show_disk_details(data) - needs_override = False - if not data['Quick Health OK']: - needs_override = True - print_warning( - "WARNING: Health can't be confirmed for: /dev/{}".format(dev)) - if get_smart_value(data['smartctl'], '199'): - # SMART attribute present and it's value is non-zero - needs_override = True - print_warning( - 'WARNING: SMART 199/C7 error detected on /dev/{}'.format(dev)) - print_standard(' (Have you tried swapping the drive cable?)') - if needs_override: - dev_name = data['lsblk']['name'] - print_standard(' ') - if ask('Run tests on this device anyway?'): - TESTS['NVMe/SMART']['Status'][dev_name] = 'OVERRIDE' - else: - TESTS['NVMe/SMART']['Status'][dev_name] = 'Skipped' - TESTS['badblocks']['Status'][dev_name] = 'Denied' - TESTS['iobenchmark']['Status'][dev_name] = 'Denied' - print_standard(' ') # In case there's more than one "OVERRIDE" disk - - TESTS['NVMe/SMART']['Devices'] = devs - TESTS['badblocks']['Devices'] = devs - TESTS['iobenchmark']['Devices'] = devs - return devs - -def show_disk_details(dev, only_attributes=False): - """Display disk details.""" - dev_name = dev['lsblk']['name'] - if not only_attributes: - # Device description - print_info('Device: {}{}'.format( - '' if '/dev/' in dev['lsblk']['name'] else '/dev/', - dev['lsblk']['name'])) - print_standard(' {:>4} ({}) {} {}'.format( - str(dev['lsblk'].get('size', '???b')).strip(), - str(dev['lsblk'].get('tran', '???')).strip().upper().replace( - 'NVME', 'NVMe'), - str(dev['lsblk'].get('model', 'Unknown Model')).strip(), - str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(), - )) - - # Warnings - if dev.get('NVMe Disk', False): - if dev['Quick Health OK']: - print_warning('WARNING: NVMe support is still experimental') - else: - print_error('ERROR: NVMe disk is reporting critical warnings') - elif not dev['SMART Support']: - print_error('ERROR: Unable to retrieve SMART data') - elif not dev['SMART Pass']: - print_error('ERROR: SMART overall-health assessment result: FAILED') - - # Attributes - if dev.get('NVMe Disk', False): - if only_attributes: - print_info('SMART Attributes:', end='') - print_warning(' Updated: {}'.format( - time.strftime('%Y-%m-%d %H:%M %Z'))) - else: - print_info('Attributes:') - for attrib, threshold in sorted(ATTRIBUTES['NVMe'].items()): - if attrib in dev['nvme-cli']: - print_standard( - ' {:37}'.format(attrib.replace('_', ' ').title()), - end='', flush=True) - raw_num = dev['nvme-cli'][attrib] - raw_str = str(raw_num) - if (threshold.get('Error', False) and - raw_num >= threshold.get('Error', -1)): - print_error(raw_str, timestamp=False) - if not threshold.get('Ignore', False): - dev['Quick Health OK'] = False - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - elif (threshold.get('Warning', False) and - raw_num >= threshold.get('Warning', -1)): - print_warning(raw_str, timestamp=False) - else: - print_success(raw_str, timestamp=False) - elif dev['smartctl'].get('ata_smart_attributes', None): - # SMART attributes - if only_attributes: - print_info('SMART Attributes:', end='') - print_warning(' Updated: {}'.format( - time.strftime('%Y-%m-%d %H:%M %Z'))) - else: - print_info('Attributes:') - s_table = dev['smartctl'].get('ata_smart_attributes', {}).get( - 'table', {}) - s_table = {a.get('id', 'Unknown'): a for a in s_table} - for attrib, threshold in sorted(ATTRIBUTES['SMART'].items()): - if attrib in s_table: - print_standard( - ' {:>3} {:32}'.format( - attrib, - s_table[attrib]['name']).replace('_', ' ').title(), - end='', flush=True) - raw_str = s_table[attrib]['raw']['string'] - raw_num = re.sub(r'^(\d+).*$', r'\1', raw_str) - try: - raw_num = float(raw_num) - except ValueError: - # Not sure about this one, print raw_str without color? - print_standard(raw_str, timestamp=False) - continue - if (threshold.get('Error', False) and - raw_num >= threshold.get('Error', -1)): - print_error(raw_str, timestamp=False) - if not threshold.get('Ignore', False): - dev['Quick Health OK'] = False - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - elif (threshold.get('Warning', False) and - raw_num >= threshold.get('Warning', -1)): - print_warning(raw_str, timestamp=False) - else: - print_success(raw_str, timestamp=False) - -def show_results(): - """Show results for selected test(s).""" - clear_screen() - print_log('\n───────────────────────────') - print_standard('Hardware Diagnostic Results') - update_progress() - - # Set Window layout and show progress - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Prime95 - if TESTS['Prime95']['Enabled']: - print_success('\nPrime95:') - for log, regex in [ - ['results.txt', r'(error|fail)'], - ['prime.log', r'completed.*0 errors, 0 warnings']]: - if log in TESTS['Prime95']: - print_info('Log: {}'.format(log)) - lines = [line.strip() for line - in TESTS['Prime95'][log].splitlines() - if re.search(regex, line, re.IGNORECASE)] - for line in lines[-4:]: - line = re.sub(r'^.*Worker #\d.*Torture Test (.*)', r'\1', - line, re.IGNORECASE) - if TESTS['Prime95'].get('NS', False): - print_error(' {}'.format(line)) - else: - print_standard(' {}'.format(line)) - print_info('Final temps') - print_log(' See Final Temps.log') - with open('{}/Final Temps.out'.format(global_vars['LogDir']), 'r') as f: - for line in f.readlines(): - if re.search(r'^\s*$', line.strip()): - # Stop after coretemps (which should be first) - break - print(' {}'.format(line.strip())) - print_standard(' ') - - # NVMe/SMART / badblocks / iobenchmark - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - print_success('Disks:') - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - show_disk_details(dev) - bb_status = TESTS['badblocks']['Status'].get(name, None) - if (TESTS['badblocks']['Enabled'] - and bb_status not in ['Denied', 'OVERRIDE', 'Skipped']): - print_info('badblocks:') - result = TESTS['badblocks']['Results'].get(name, '') - for line in result.splitlines(): - if re.search(r'Pass completed', line, re.IGNORECASE): - line = re.sub( - r'Pass completed,?\s+', r'', - line.strip(), re.IGNORECASE) - if TESTS['badblocks']['Status'][name] == 'CS': - print_standard(' {}'.format(line)) - else: - print_error(' {}'.format(line)) - io_status = TESTS['iobenchmark']['Status'].get(name, None) - if (TESTS['iobenchmark']['Enabled'] - and io_status not in ['Denied', 'OVERRIDE', 'Skipped']): - print_info('Benchmark:') - result = TESTS['iobenchmark']['Results'].get(name, '') - for line in result.split('\n'): - print_standard(' {}'.format(line)) - print_standard(' ') - - # Done - pause('Press Enter to return to main menu... ') - run_program('tmux kill-pane -a'.split()) + """Get color based on status, returns str.""" + color = COLORS['CLEAR'] + if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + color = COLORS['RED'] + elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: + color = COLORS['YELLOW'] + elif s in ['CS']: + color = COLORS['GREEN'] + return color def update_io_progress(percent, rate, progress_file): - """Update I/O progress file.""" - bar_color = COLORS['CLEAR'] - rate_color = COLORS['CLEAR'] - step = get_graph_step(rate, scale=32) - if rate < IO_VARS['Threshold Graph Fail']: - bar_color = COLORS['RED'] - rate_color = COLORS['YELLOW'] - elif rate < IO_VARS['Threshold Graph Warn']: - bar_color = COLORS['YELLOW'] - rate_color = COLORS['YELLOW'] - elif rate > IO_VARS['Threshold Graph Great']: - bar_color = COLORS['GREEN'] - rate_color = COLORS['GREEN'] - line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( - p=percent, - b_color=bar_color, - b=IO_VARS['Graph Vertical'][step], - r_color=rate_color, - r=rate/(1024**2), - c=COLORS['CLEAR']) - with open(progress_file, 'a') as f: - f.write(line) - -def update_progress(): - """Update progress file.""" - if 'Progress Out' not in TESTS: - TESTS['Progress Out'] = '{}/progress.out'.format(global_vars['LogDir']) - output = [] - output.append('{BLUE}HW Diagnostics{CLEAR}'.format(**COLORS)) - output.append('───────────────') - if TESTS['Prime95']['Enabled']: - output.append(' ') - output.append('{BLUE}Prime95{s_color}{status:>8}{CLEAR}'.format( - s_color = get_status_color(TESTS['Prime95']['Status']), - status = TESTS['Prime95']['Status'], - **COLORS)) - if TESTS['NVMe/SMART']['Enabled']: - output.append(' ') - output.append('{BLUE}NVMe / SMART{CLEAR}'.format(**COLORS)) - if TESTS['NVMe/SMART']['Quick']: - output.append('{YELLOW} (Quick Check){CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['NVMe/SMART']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - if TESTS['badblocks']['Enabled']: - output.append(' ') - output.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['badblocks']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - if TESTS['iobenchmark']['Enabled']: - output.append(' ') - output.append('{BLUE}I/O Benchmark{CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['iobenchmark']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - - # Add line-endings - output = ['{}\n'.format(line) for line in output] - - with open(TESTS['Progress Out'], 'w') as f: - f.writelines(output) + """Update I/O progress file.""" + bar_color = COLORS['CLEAR'] + rate_color = COLORS['CLEAR'] + step = get_graph_step(rate, scale=32) + if rate < IO_VARS['Threshold Graph Fail']: + bar_color = COLORS['RED'] + rate_color = COLORS['YELLOW'] + elif rate < IO_VARS['Threshold Graph Warn']: + bar_color = COLORS['YELLOW'] + rate_color = COLORS['YELLOW'] + elif rate > IO_VARS['Threshold Graph Great']: + bar_color = COLORS['GREEN'] + rate_color = COLORS['GREEN'] + line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( + p=percent, + b_color=bar_color, + b=IO_VARS['Graph Vertical'][step], + r_color=rate_color, + r=rate/(1024**2), + c=COLORS['CLEAR']) + with open(progress_file, 'a') as f: + f.write(line) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 From 3fdd8c629c8afd6c2ded1a52f821e63d548145ab Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:47:44 -0700 Subject: [PATCH 02/86] Rewrote main menu * First options are presets followed by individual tests * Selecting presets will toggle the selections * Screensavers are hidden but still present --- .bin/Scripts/functions/common.py | 6 +- .bin/Scripts/functions/hw_diags.py | 168 +++++++++++++++++++++++++++++ .bin/Scripts/hw-diags-menu | 3 +- 3 files changed, 174 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index ae958645..d3f04ef6 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -318,7 +318,7 @@ def major_exception(): exit_script(1) def menu_select(title='~ Untitled Menu ~', - prompt='Please make a selection', secret_exit=False, + prompt='Please make a selection', secret_actions=[], secret_exit=False, main_entries=[], action_entries=[], disabled_label='DISABLED', spacer=''): """Display options in a menu and return selected option as a str.""" @@ -334,8 +334,10 @@ def menu_select(title='~ Untitled Menu ~', menu_splash = '{}\n{}\n'.format(title, spacer) width = len(str(len(main_entries))) valid_answers = [] - if (secret_exit): + if secret_exit: valid_answers.append('Q') + if secret_actions: + valid_answers.extend(secret_actions) # Add main entries for i in range(len(main_entries)): diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 830d948a..d32fb499 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -216,6 +216,174 @@ def get_status_color(s): color = COLORS['GREEN'] return color +def menu_diags(state, args): + """Main menu to select and run HW tests.""" + args = [a.lower() for a in args] + quick_label = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) + title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( + **COLORS) + # NOTE: Changing the order of main_options will break everything + main_options = [ + {'Base Name': 'Full Diagnostic', 'Enabled': True}, + {'Base Name': 'Drive Diagnostic', 'Enabled': False}, + {'Base Name': 'Drive Diagnostic (Quick)', 'Enabled': False}, + {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, + {'Base Name': 'NVMe / SMART', 'Enabled': True}, + {'Base Name': 'badblocks', 'Enabled': True}, + {'Base Name': 'I/O Benchmark', 'Enabled': True}, + ] + actions = [ + {'Letter': 'A', 'Name': 'Audio Test'}, + {'Letter': 'K', 'Name': 'Keyboard Test'}, + {'Letter': 'N', 'Name': 'Network Test'}, + {'Letter': 'S', 'Name': 'Start', 'CRLF': True}, + {'Letter': 'Q', 'Name': 'Quit'}, + ] + secret_actions = ['M', 'T'] + + # CLI mode check + if '--cli' in args or 'DISPLAY' not in global_vars['Env']: + actions.append({'Letter': 'R', 'Name': 'Reboot'}) + actions.append({'Letter': 'P', 'Name': 'Power Off'}) + + while True: + # Set quick mode as necessary + if main_options[2]['Enabled'] and main_options[4]['Enabled']: + # Check if only Drive Diags (Quick) and NVMe/SMART are enabled + # If so, verify no other tests are enabled and set quick_mode + state.quick_mode = True + for opt in main_options[3:4] + main_options[5:]: + state.quick_mode &= not opt['Enabled'] + else: + state.quick_mode = False + + # Deselect presets + slice_end = 3 + if state.quick_mode: + slice_end = 2 + for opt in main_options[:slice_end]: + opt['Enabled'] = False + + # Verify preset selections + num_tests_selected = 0 + for opt in main_options[3:]: + if opt['Enabled']: + num_tests_selected += 1 + if num_tests_selected == 4: + # Full + main_options[0]['Enabled'] = True + elif num_tests_selected == 3 and not main_options[3]['Enabled']: + # Drive + main_options[1]['Enabled'] = True + + # Update checkboxes + for opt in main_options: + _nvme_smart = opt['Base Name'] == 'NVMe / SMART' + opt['Name'] = '{} {} {}'.format( + '[✓]' if opt['Enabled'] else '[ ]', + opt['Base Name'], + quick_label if state.quick_mode and _nvme_smart else '') + + # Show menu + selection = menu_select( + title=title, + main_entries=main_options, + action_entries=actions, + secret_actions=secret_actions, + spacer='───────────────────────────────') + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + + # Handle presets + if index == 0: + # Full + if main_options[index]['Enabled']: + for opt in main_options[1:3]: + opt['Enabled'] = False + for opt in main_options[3:]: + opt['Enabled'] = True + else: + for opt in main_options[3:]: + opt['Enabled'] = False + elif index == 1: + # Drive + if main_options[index]['Enabled']: + main_options[0]['Enabled'] = False + for opt in main_options[2:4]: + opt['Enabled'] = False + for opt in main_options[4:]: + opt['Enabled'] = True + else: + for opt in main_options[4:]: + opt['Enabled'] = False + elif index == 2: + # Drive (Quick) + if main_options[index]['Enabled']: + for opt in main_options[:2] + main_options[3:]: + opt['Enabled'] = False + main_options[4]['Enabled'] = True + else: + main_options[4]['Enabled'] = False + elif selection == 'A': + run_audio_test() + elif selection == 'K': + run_keyboard_test() + elif selection == 'N': + run_network_test() + elif selection == 'M': + secret_screensaver('matrix') + elif selection == 'T': + # Tubes is close to pipes + secret_screensaver('pipes') + elif selection == 'R': + run_program(['systemctl', 'reboot']) + elif selection == 'P': + run_program(['systemctl', 'poweroff']) + elif selection == 'Q': + break + elif selection == 'S': + # Run test(s) + clear_screen() + print('Fake test(s) placeholder for now...') + pause('Press Enter to return to main menu... ') + +def run_audio_test(): + """Run audio test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake audio placeholder for now...') + #run_program(['hw-diags-audio'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def run_keyboard_test(): + """Run keyboard test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake keyboard placeholder for now...') + #run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def run_network_test(): + """Run network test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake network placeholder for now...') + #run_program(['hw-diags-network'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def secret_screensaver(screensaver=None): + """Show screensaver.""" + if screensaver == 'matrix': + cmd = 'cmatrix -abs'.split() + elif screensaver == 'pipes': + cmd = 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split() + else: + raise Exception('Invalid screensaver') + run_program(cmd, check=False, pipe=False) + def update_io_progress(percent, rate, progress_file): """Update I/O progress file.""" bar_color = COLORS['CLEAR'] diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index f7c1739c..c67cc5f4 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -17,7 +17,8 @@ if __name__ == '__main__': clear_screen() # Show menu - menu_diags(*sys.argv) + state = State() + menu_diags(state, sys.argv) # Done #print_standard('\nDone.') From 18fc97293e546d50e2606e055a8d1ec57be04913 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:50:55 -0700 Subject: [PATCH 03/86] Renamed Drive to Disk to align options in menu --- .bin/Scripts/functions/hw_diags.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d32fb499..b5d553de 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -225,8 +225,8 @@ def menu_diags(state, args): # NOTE: Changing the order of main_options will break everything main_options = [ {'Base Name': 'Full Diagnostic', 'Enabled': True}, - {'Base Name': 'Drive Diagnostic', 'Enabled': False}, - {'Base Name': 'Drive Diagnostic (Quick)', 'Enabled': False}, + {'Base Name': 'Disk Diagnostic', 'Enabled': False}, + {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, {'Base Name': 'NVMe / SMART', 'Enabled': True}, {'Base Name': 'badblocks', 'Enabled': True}, @@ -249,7 +249,7 @@ def menu_diags(state, args): while True: # Set quick mode as necessary if main_options[2]['Enabled'] and main_options[4]['Enabled']: - # Check if only Drive Diags (Quick) and NVMe/SMART are enabled + # Check if only Disk Diags (Quick) and NVMe/SMART are enabled # If so, verify no other tests are enabled and set quick_mode state.quick_mode = True for opt in main_options[3:4] + main_options[5:]: @@ -273,7 +273,7 @@ def menu_diags(state, args): # Full main_options[0]['Enabled'] = True elif num_tests_selected == 3 and not main_options[3]['Enabled']: - # Drive + # Disk main_options[1]['Enabled'] = True # Update checkboxes @@ -309,7 +309,7 @@ def menu_diags(state, args): for opt in main_options[3:]: opt['Enabled'] = False elif index == 1: - # Drive + # Disk if main_options[index]['Enabled']: main_options[0]['Enabled'] = False for opt in main_options[2:4]: @@ -320,7 +320,7 @@ def menu_diags(state, args): for opt in main_options[4:]: opt['Enabled'] = False elif index == 2: - # Drive (Quick) + # Disk (Quick) if main_options[index]['Enabled']: for opt in main_options[:2] + main_options[3:]: opt['Enabled'] = False From 560929e2fa74ba35301758d17bfbfbfc526f1ad9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:54:06 -0700 Subject: [PATCH 04/86] Removed extra line break in menu_select --- .bin/Scripts/functions/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index d3f04ef6..b5896966 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -369,7 +369,6 @@ def menu_select(title='~ Untitled Menu ~', letter = entry['Letter'].upper(), width = len(str(len(action_entries))), name = entry['Name']) - menu_splash += '\n' answer = '' From 2df4d48bb3af0f424fd1248a5e89af24035d588f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 20:15:56 -0700 Subject: [PATCH 05/86] Show selected tests on run --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index b5d553de..3e08c9ed 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -118,10 +118,10 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.started = False self.tests = { - 'Prime95': {'Enabled': False, 'Result': None, 'Status': None}, - 'NVMe / SMART': {'Enabled': False}, - 'badblocks': {'Enabled': False}, - 'I/O Benchmark': {'Enabled': False}, + 'Prime95 & Temps': {'Enabled': False, 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False}, + 'badblocks': {'Enabled': False}, + 'I/O Benchmark': {'Enabled': False}, } self.add_devs() @@ -347,7 +347,18 @@ def menu_diags(state, args): elif selection == 'S': # Run test(s) clear_screen() - print('Fake test(s) placeholder for now...') + print('Tests:') + for opt in main_options[3:]: + _nvme_smart = opt['Base Name'] == 'NVMe / SMART' + # Update state + state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] + print(' {:<15} {}{}{} {}'.format( + opt['Base Name'], + COLORS['GREEN'] if opt['Enabled'] else COLORS['RED'], + 'Enabled' if opt['Enabled'] else 'Disabled', + COLORS['CLEAR'], + quick_label if state.quick_mode and _nvme_smart else '')) + print('\nFake test(s) placeholder for now...') pause('Press Enter to return to main menu... ') def run_audio_test(): From 70a742e69c832484df6c5c1b6f70e5e6c1abde74 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:10:58 -0700 Subject: [PATCH 06/86] Add device details from lsblk * Also ensure sane types for some attributes --- .bin/Scripts/functions/hw_diags.py | 52 ++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3e08c9ed..5bb03c9b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -68,10 +68,14 @@ class DevObj(): """Device object for tracking device specific data.""" def __init__(self, dev_path): self.failing = False + self.labels = [] + self.lsblk = {} + self.name = re.sub(r'^.*/(.*)', r'\1', dev_path) self.nvme_attributes = {} self.override = False self.path = dev_path self.smart_attributes = {} + self.smartctl = {} self.tests = { 'NVMe / SMART': {'Result': None, 'Status': None}, 'badblocks': {'Result': None, 'Status': None}, @@ -82,18 +86,54 @@ class DevObj(): 'Graph Data': []}, } self.get_details() + self.get_smart_details() def get_details(self): + """Get data from lsblk.""" + cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + self.lsblk = json_data['blockdevices'][0] + except Exception: + # Leave self.lsblk empty + pass + + # Set necessary details + self.lsblk['model'] = self.lsblk.get('model', 'Unknown Model') + self.lsblk['name'] = self.lsblk.get('name', self.path) + self.lsblk['rota'] = self.lsblk.get('rota', True) + self.lsblk['serial'] = self.lsblk.get('serial', 'Unknown Serial') + self.lsblk['size'] = self.lsblk.get('size', '???b') + self.lsblk['tran'] = self.lsblk.get('tran', '???') + + # Ensure certain attributes are strings + for attr in ['model', 'name', 'rota', 'serial', 'size', 'tran']: + if not isinstance(self.lsblk[attr], str): + self.lsblk[attr] = str(self.lsblk[attr]) + self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') + + # Build list of labels + for dev in [self.lsblk, *self.lsblk.get('children', [])]: + self.labels.append(dev.get('label', '')) + self.labels.append(dev.get('partlabel', '')) + self.labels = [str(label) for label in self.labels if label] + + def get_smart_details(self): """Get data from smartctl.""" cmd = ['sudo', 'smartctl', '--all', '--json', self.path] - result = run_program(cmd, check=False) - self.data = json.loads(result.stdout.decode()) + try: + result = run_program(cmd, check=False) + self.smartctl = json.loads(result.stdout.decode()) + except Exception: + # Leave self.smartctl empty + pass # Check for attributes - if KEY_NVME in self.data: - self.nvme_attributes.update(self.data[KEY_NVME]) - elif KEY_SMART in self.data: - for a in self.data[KEY_SMART].get('table', {}): + if KEY_NVME in self.smartctl: + self.nvme_attributes.update(self.smartctl[KEY_NVME]) + elif KEY_SMART in self.smartctl: + for a in self.smartctl[KEY_SMART].get('table', {}): _id = str(a.get('id', 'UNKNOWN')) _name = str(a.get('name', 'UNKNOWN')) _raw = a.get('raw', {}).get('value', -1) From 6014a8fb70275d7be2b8b6a7f6c548b7ae718161 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:18:45 -0700 Subject: [PATCH 07/86] Don't add WK or loopback devices --- .bin/Scripts/functions/hw_diags.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 5bb03c9b..8f543aa5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -171,7 +171,22 @@ class State(): result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) for dev in json_data['blockdevices']: - self.devs.append(DevObj(dev['name'])) + skip_dev = False + dev_obj = DevObj(dev['name']) + + # Skip loopback devices + if dev_obj.lsblk['tran'] == 'NONE': + skip_dev = True + + # Skip WK devices + wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) + for label in dev_obj.labels: + if re.search(wk_label_regex, label, re.IGNORECASE): + skip_dev = True + + # Add device + if not skip_dev: + self.devs.append(DevObj(dev['name'])) # Functions def generate_horizontal_graph(rates, oneline=False): From 5701b53026aa758c879db61454fcff9326a67e73 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:55:17 -0700 Subject: [PATCH 08/86] Added --quick argument to skip menu --- .bin/Scripts/functions/hw_diags.py | 147 +++++++++++++++++------------ 1 file changed, 86 insertions(+), 61 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8f543aa5..3de0806f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -61,6 +61,7 @@ IO_VARS = { } KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' +QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PATH_WIDTH = 21 # Classes @@ -156,12 +157,14 @@ class State(): self.devs = [] self.finished = False self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.quick_mode = False self.started = False self.tests = { - 'Prime95 & Temps': {'Enabled': False, 'Result': None, 'Status': None}, - 'NVMe / SMART': {'Enabled': False}, - 'badblocks': {'Enabled': False}, - 'I/O Benchmark': {'Enabled': False}, + 'Prime95 & Temps': {'Enabled': False, 'Order': 1, + 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False, 'Order': 2}, + 'badblocks': {'Enabled': False, 'Order': 3}, + 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } self.add_devs() @@ -274,18 +277,17 @@ def get_status_color(s): def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] - quick_label = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( **COLORS) # NOTE: Changing the order of main_options will break everything main_options = [ - {'Base Name': 'Full Diagnostic', 'Enabled': True}, + {'Base Name': 'Full Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, - {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, - {'Base Name': 'NVMe / SMART', 'Enabled': True}, - {'Base Name': 'badblocks', 'Enabled': True}, - {'Base Name': 'I/O Benchmark', 'Enabled': True}, + {'Base Name': 'Prime95 & Temps', 'Enabled': False, 'CRLF': True}, + {'Base Name': 'NVMe / SMART', 'Enabled': False}, + {'Base Name': 'badblocks', 'Enabled': False}, + {'Base Name': 'I/O Benchmark', 'Enabled': False}, ] actions = [ {'Letter': 'A', 'Name': 'Audio Test'}, @@ -295,12 +297,22 @@ def menu_diags(state, args): {'Letter': 'Q', 'Name': 'Quit'}, ] secret_actions = ['M', 'T'] + + # Set initial selections + update_main_options(state, '1', main_options) # CLI mode check if '--cli' in args or 'DISPLAY' not in global_vars['Env']: actions.append({'Letter': 'R', 'Name': 'Reboot'}) actions.append({'Letter': 'P', 'Name': 'Power Off'}) + # Skip menu if running quick check + if '--quick' in args: + update_main_options(state, '3', main_options) + state.quick_mode = True + run_hw_tests(state) + return True + while True: # Set quick mode as necessary if main_options[2]['Enabled'] and main_options[4]['Enabled']: @@ -337,7 +349,7 @@ def menu_diags(state, args): opt['Name'] = '{} {} {}'.format( '[✓]' if opt['Enabled'] else '[ ]', opt['Base Name'], - quick_label if state.quick_mode and _nvme_smart else '') + QUICK_LABEL if state.quick_mode and _nvme_smart else '') # Show menu selection = menu_select( @@ -348,40 +360,7 @@ def menu_diags(state, args): spacer='───────────────────────────────') if selection.isnumeric(): - # Toggle selection - index = int(selection) - 1 - main_options[index]['Enabled'] = not main_options[index]['Enabled'] - - # Handle presets - if index == 0: - # Full - if main_options[index]['Enabled']: - for opt in main_options[1:3]: - opt['Enabled'] = False - for opt in main_options[3:]: - opt['Enabled'] = True - else: - for opt in main_options[3:]: - opt['Enabled'] = False - elif index == 1: - # Disk - if main_options[index]['Enabled']: - main_options[0]['Enabled'] = False - for opt in main_options[2:4]: - opt['Enabled'] = False - for opt in main_options[4:]: - opt['Enabled'] = True - else: - for opt in main_options[4:]: - opt['Enabled'] = False - elif index == 2: - # Disk (Quick) - if main_options[index]['Enabled']: - for opt in main_options[:2] + main_options[3:]: - opt['Enabled'] = False - main_options[4]['Enabled'] = True - else: - main_options[4]['Enabled'] = False + update_main_options(state, selection, main_options) elif selection == 'A': run_audio_test() elif selection == 'K': @@ -391,7 +370,7 @@ def menu_diags(state, args): elif selection == 'M': secret_screensaver('matrix') elif selection == 'T': - # Tubes is close to pipes + # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': run_program(['systemctl', 'reboot']) @@ -400,21 +379,7 @@ def menu_diags(state, args): elif selection == 'Q': break elif selection == 'S': - # Run test(s) - clear_screen() - print('Tests:') - for opt in main_options[3:]: - _nvme_smart = opt['Base Name'] == 'NVMe / SMART' - # Update state - state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] - print(' {:<15} {}{}{} {}'.format( - opt['Base Name'], - COLORS['GREEN'] if opt['Enabled'] else COLORS['RED'], - 'Enabled' if opt['Enabled'] else 'Disabled', - COLORS['CLEAR'], - quick_label if state.quick_mode and _nvme_smart else '')) - print('\nFake test(s) placeholder for now...') - pause('Press Enter to return to main menu... ') + run_hw_tests(state) def run_audio_test(): """Run audio test.""" @@ -424,6 +389,23 @@ def run_audio_test(): #run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_hw_tests(state): + """Run enabled hardware tests.""" + # Run test(s) + clear_screen() + print('Tests:') + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + print_standard(' {:<15} {}{}{} {}'.format( + k, + COLORS['GREEN'] if v['Enabled'] else COLORS['RED'], + 'Enabled' if v['Enabled'] else 'Disabled', + COLORS['CLEAR'], + QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) + print('\nFake test(s) placeholder for now...') + pause('Press Enter to return to main menu... ') + def run_keyboard_test(): """Run keyboard test.""" # TODO: Enable real test and remove placeholder @@ -450,6 +432,49 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def update_main_options(state, selection, main_options): + """Update menu and state based on selection.""" + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + + # Handle presets + if index == 0: + # Full + if main_options[index]['Enabled']: + for opt in main_options[1:3]: + opt['Enabled'] = False + for opt in main_options[3:]: + opt['Enabled'] = True + else: + for opt in main_options[3:]: + opt['Enabled'] = False + elif index == 1: + # Disk + if main_options[index]['Enabled']: + main_options[0]['Enabled'] = False + for opt in main_options[2:4]: + opt['Enabled'] = False + for opt in main_options[4:]: + opt['Enabled'] = True + else: + for opt in main_options[4:]: + opt['Enabled'] = False + elif index == 2: + # Disk (Quick) + if main_options[index]['Enabled']: + for opt in main_options[:2] + main_options[3:]: + opt['Enabled'] = False + main_options[4]['Enabled'] = True + else: + main_options[4]['Enabled'] = False + + # Update state + for opt in main_options[3:]: + state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] + + # Done + return main_options + def update_io_progress(percent, rate, progress_file): """Update I/O progress file.""" bar_color = COLORS['CLEAR'] From 62c9d82fd2cba7166c7c572e8de9f64387c2a417 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 17:05:53 -0700 Subject: [PATCH 09/86] Adjusted placeholders --- .bin/Scripts/functions/hw_diags.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3de0806f..4593392f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -373,9 +373,15 @@ def menu_diags(state, args): # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': - run_program(['systemctl', 'reboot']) + print('(FAKE) reboot...') + sleep(1) + # TODO uncomment below + #run_program(['systemctl', 'reboot']) elif selection == 'P': - run_program(['systemctl', 'poweroff']) + print('(FAKE) poweroff...') + sleep(1) + # TODO uncomment below + #run_program(['systemctl', 'poweroff']) elif selection == 'Q': break elif selection == 'S': @@ -383,10 +389,8 @@ def menu_diags(state, args): def run_audio_test(): """Run audio test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake audio placeholder for now...') - #run_program(['hw-diags-audio'], check=False, pipe=False) + run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') def run_hw_tests(state): @@ -403,23 +407,17 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) - print('\nFake test(s) placeholder for now...') - pause('Press Enter to return to main menu... ') + pause('\nPress Enter to return to main menu... ') def run_keyboard_test(): """Run keyboard test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake keyboard placeholder for now...') - #run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') + run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) def run_network_test(): """Run network test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake network placeholder for now...') - #run_program(['hw-diags-network'], check=False, pipe=False) + run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') def secret_screensaver(screensaver=None): From 1489ad4237e3bc20d22fc0f89e7d0bdc0c378790 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 18:43:50 -0700 Subject: [PATCH 10/86] Added safety check for devices --- .bin/Scripts/functions/hw_diags.py | 61 ++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4593392f..67195d79 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -176,22 +176,53 @@ class State(): for dev in json_data['blockdevices']: skip_dev = False dev_obj = DevObj(dev['name']) - + # Skip loopback devices if dev_obj.lsblk['tran'] == 'NONE': skip_dev = True - + # Skip WK devices wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) for label in dev_obj.labels: if re.search(wk_label_regex, label, re.IGNORECASE): skip_dev = True - + # Add device if not skip_dev: self.devs.append(DevObj(dev['name'])) # Functions +def check_dev_attributes(dev): + """Check if device should be tested and allow overrides.""" + needs_override = False + print_standard(' {size:>6} ({tran}) {model} {serial}'.format( + **dev.lsblk)) + + # General checks + if not dev.nvme_attributes and not dev.smart_attributes: + needs_override = True + print_warning( + ' WARNING: No NVMe or SMART attributes available for: {}'.format( + dev.path)) + + # NVMe checks + # TODO check all tracked attributes and set dev.failing if needed + + # SMART checks + # TODO check all tracked attributes and set dev.failing if needed + + # Ask for override if necessary + if needs_override: + if ask(' Run tests on this device anyway?'): + # TODO Set override for this dev + pass + else: + for v in dev.tests.values(): + v['Enabled'] = False + v['Result'] = 'Skipped' + v['Status'] = 'Skipped' + print_standard('') + def generate_horizontal_graph(rates, oneline=False): """Generate two-line horizontal graph from rates, returns str.""" line_1 = '' @@ -297,7 +328,7 @@ def menu_diags(state, args): {'Letter': 'Q', 'Name': 'Quit'}, ] secret_actions = ['M', 'T'] - + # Set initial selections update_main_options(state, '1', main_options) @@ -397,7 +428,7 @@ def run_hw_tests(state): """Run enabled hardware tests.""" # Run test(s) clear_screen() - print('Tests:') + print_info('Selected Tests:') for k, v in sorted( state.tests.items(), key=lambda kv: kv[1]['Order']): @@ -407,7 +438,21 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) - pause('\nPress Enter to return to main menu... ') + print_standard('') + + # Check devices if necessary + if (state.tests['badblocks']['Enabled'] + or state.tests['I/O Benchmark']['Enabled']): + print_info('Selected Disks:') + for dev in state.devs: + check_dev_attributes(dev) + print_standard('') + + # Run tests + # TODO + + # Done + pause('Press Enter to return to main menu... ') def run_keyboard_test(): """Run keyboard test.""" @@ -465,11 +510,11 @@ def update_main_options(state, selection, main_options): main_options[4]['Enabled'] = True else: main_options[4]['Enabled'] = False - + # Update state for opt in main_options[3:]: state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] - + # Done return main_options From 597a23608951f38dfc3b987eed3ebff053081c98 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 18:44:52 -0700 Subject: [PATCH 11/86] Don't clear screen twice at startup * Combined init_global_vars and add_devs output --- .bin/Scripts/functions/hw_diags.py | 5 ++++- .bin/Scripts/hw-diags-menu | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 67195d79..6c43fe28 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -166,7 +166,10 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - self.add_devs() + try_and_print( + message='Scanning devices...', + function=self.add_devs, + cs='Done') def add_devs(self): """Add all block devices listed by lsblk.""" diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index c67cc5f4..e60f8fc4 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -13,9 +13,6 @@ init_global_vars() if __name__ == '__main__': try: - # Prep - clear_screen() - # Show menu state = State() menu_diags(state, sys.argv) From 8fb1620c940ac2d337cb7cfb36998b3d1fac7c18 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 19:23:35 -0700 Subject: [PATCH 12/86] Added placeholder functions for HW tests --- .bin/Scripts/functions/hw_diags.py | 45 ++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6c43fe28..15bea407 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -302,7 +302,7 @@ def get_status_color(s): color = COLORS['CLEAR'] if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: color = COLORS['RED'] - elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: + elif s in ['Aborted', 'N/A', 'Unknown', 'Working', 'Skipped']: color = COLORS['YELLOW'] elif s in ['CS']: color = COLORS['GREEN'] @@ -427,6 +427,10 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_badblocks_test(state): + """TODO""" + print_standard('TODO: run_badblocks_test()') + def run_hw_tests(state): """Run enabled hardware tests.""" # Run test(s) @@ -452,22 +456,55 @@ def run_hw_tests(state): print_standard('') # Run tests - # TODO + if state.tests['Prime95 & Temps']['Enabled']: + run_mprime_test(state) + if state.tests['NVMe / SMART']['Enabled']: + run_nvme_smart(state) + if state.tests['badblocks']['Enabled']: + run_badblocks_test(state) + if state.tests['I/O Benchmark']['Enabled']: + run_io_benchmark(state) # Done pause('Press Enter to return to main menu... ') +def run_io_benchmark(state): + """TODO""" + print_standard('TODO: run_io_benchmark()') + def run_keyboard_test(): """Run keyboard test.""" clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) +def run_mprime_test(state): + """TODO""" + print_standard('TODO: run_mprime_test()') + def run_network_test(): """Run network test.""" clear_screen() run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_nvme_smart(state): + """TODO""" + for dev in state.devs: + if dev.nvme_attributes: + run_nvme_tests(dev) + elif dev.smart_attributes: + run_smart_tests(dev) + else: + print_standard('TODO: run_nvme_smart({})'.format( + dev.path)) + print_warning( + " WARNING: Device {} doesn't support NVMe or SMART test".format( + dev.path)) + +def run_nvme_tests(dev): + """TODO""" + print_standard('TODO: run_nvme_test({})'.format(dev.path)) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': @@ -478,6 +515,10 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def run_smart_tests(dev): + """TODO""" + print_standard('TODO: run_smart_tests({})'.format(dev.path)) + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 4bb1402ac5a992c2e5f3f030a7a9131f743d00fb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 20:50:47 -0700 Subject: [PATCH 13/86] Added tmux functions * Going to try and replace the send-keys sections next --- .bin/Scripts/functions/hw_diags.py | 80 +++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 15bea407..a446036d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -62,7 +62,7 @@ IO_VARS = { KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) -SIDE_PATH_WIDTH = 21 +SIDE_PANE_WIDTH = 21 # Classes class DevObj(): @@ -156,6 +156,7 @@ class State(): def __init__(self): self.devs = [] self.finished = False + self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.started = False @@ -195,6 +196,31 @@ class State(): self.devs.append(DevObj(dev['name'])) # Functions +def build_outer_panes(state): + """Build top and side panes.""" + clear_screen() + + # Create panes + state.panes['Top'] = tmux_split_window( + behind=True, lines=2, vertical=True) + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top']) + state.panes['Progress'] = tmux_split_window(lines=SIDE_PANE_WIDTH) + + # Set text + tmux_update_pane_text( + state.panes['Top'], + text='{GREEN}Hardware Diagnostics{CLEAR}'.format( + **COLORS)) + tmux_update_pane_text( + state.panes['Started'], + text='{BLUE}Started{CLEAR}\n{text}'.format( + text=time.strftime("%Y-%m-%d %H:%M %Z"), + **COLORS)) + tmux_update_pane_text( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here'.format(**COLORS)) + def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" needs_override = False @@ -433,8 +459,10 @@ def run_badblocks_test(state): def run_hw_tests(state): """Run enabled hardware tests.""" + # Build Panes + build_outer_panes(state) + # Run test(s) - clear_screen() print_info('Selected Tests:') for k, v in sorted( state.tests.items(), @@ -468,6 +496,9 @@ def run_hw_tests(state): # Done pause('Press Enter to return to main menu... ') + # Cleanup + tmux_kill_pane(*state.panes.values()) + def run_io_benchmark(state): """TODO""" print_standard('TODO: run_io_benchmark()') @@ -519,6 +550,51 @@ def run_smart_tests(dev): """TODO""" print_standard('TODO: run_smart_tests({})'.format(dev.path)) +def tmux_kill_pane(*panes): + """Kill tmux pane by id.""" + cmd = ['tmux', 'kill-pane', '-t'] + for pane_id in panes: + print(pane_id) + run_program(cmd+[pane_id], check=False) + +def tmux_split_window( + lines=None, percent=None, + behind=False, vertical=False, + follow=False, target_pane=None): + """Run tmux split-window command and return pane_id as str.""" + # Bail early + if not lines and not percent: + raise Exception('Neither lines nor percent specified.') + + # Build cmd + cmd = ['tmux', 'split-window', '-PF', '#D'] + if behind: + cmd.append('-b') + if vertical: + cmd.append('-v') + else: + cmd.append('-h') + if not follow: + cmd.append('-d') + if lines is not None: + cmd.extend(['-l', str(lines)]) + elif percent is not None: + cmd.extend(['-p', str(percent)]) + if target_pane: + cmd.extend(['-t', str(target_pane)]) + + # Run and return pane_id + result = run_program(cmd) + return result.stdout.decode().strip() + +def tmux_update_pane_text(pane_id, text): + """Print text to tmux pane.""" + text = text.replace('\033', r'\e') + cmd = ['tmux', 'send-keys', '-t', pane_id] + run_program(cmd+['Enter']) + run_program(cmd+['clear; echo-and-hold "{}"'.format(text)]) + run_program(cmd+['Enter']) + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 43b9645c69fee66d5154013abe5d67b58bf54a77 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 23:39:15 -0700 Subject: [PATCH 14/86] Update tmux panes via respawn-pane Instead of send-keys * Avoids flooding zsh history * Less flickering --- .bin/Scripts/functions/hw_diags.py | 121 +++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 34 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a446036d..743db0a2 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -63,6 +63,7 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 21 +TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes class DevObj(): @@ -200,26 +201,24 @@ def build_outer_panes(state): """Build top and side panes.""" clear_screen() - # Create panes + # Top state.panes['Top'] = tmux_split_window( - behind=True, lines=2, vertical=True) - state.panes['Started'] = tmux_split_window( - lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top']) - state.panes['Progress'] = tmux_split_window(lines=SIDE_PANE_WIDTH) + behind=True, lines=2, vertical=True, + text='{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS)) - # Set text - tmux_update_pane_text( - state.panes['Top'], - text='{GREEN}Hardware Diagnostics{CLEAR}'.format( - **COLORS)) - tmux_update_pane_text( - state.panes['Started'], + # Started + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top'], text='{BLUE}Started{CLEAR}\n{text}'.format( text=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) - tmux_update_pane_text( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here'.format(**COLORS)) + + # Progress + state.panes['Progress'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" @@ -337,8 +336,7 @@ def get_status_color(s): def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] - title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( - **COLORS) + title = '{}\nMain Menu'.format(TOP_PANE_TEXT) # NOTE: Changing the order of main_options will break everything main_options = [ {'Base Name': 'Full Diagnostic', 'Enabled': False}, @@ -455,7 +453,16 @@ def run_audio_test(): def run_badblocks_test(state): """TODO""" + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'badblocks')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) print_standard('TODO: run_badblocks_test()') + sleep(3) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -501,7 +508,16 @@ def run_hw_tests(state): def run_io_benchmark(state): """TODO""" + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'I/O Benchmark')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) print_standard('TODO: run_io_benchmark()') + sleep(3) def run_keyboard_test(): """Run keyboard test.""" @@ -509,8 +525,21 @@ def run_keyboard_test(): run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) def run_mprime_test(state): - """TODO""" - print_standard('TODO: run_mprime_test()') + """Test CPU with Prime95 and track temps.""" + # Prep + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'Prime95 & Temps')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) + # Get idle temps + # Stress CPU + # Get max temp + # Get cooldown temp + sleep(3) def run_network_test(): """Run network test.""" @@ -521,21 +550,35 @@ def run_network_test(): def run_nvme_smart(state): """TODO""" for dev in state.devs: + tmux_update_pane( + state.panes['Top'], + text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( + t=TOP_PANE_TEXT, **dev.lsblk)) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) if dev.nvme_attributes: - run_nvme_tests(dev) + run_nvme_tests(state, dev) elif dev.smart_attributes: - run_smart_tests(dev) + run_smart_tests(state, dev) else: print_standard('TODO: run_nvme_smart({})'.format( dev.path)) print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( dev.path)) + sleep(3) -def run_nvme_tests(dev): +def run_nvme_tests(state, dev): """TODO""" print_standard('TODO: run_nvme_test({})'.format(dev.path)) +def run_smart_tests(state, dev): + """TODO""" + print_standard('TODO: run_smart_tests({})'.format(dev.path)) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': @@ -546,10 +589,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def run_smart_tests(dev): - """TODO""" - print_standard('TODO: run_smart_tests({})'.format(dev.path)) - def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] @@ -560,11 +599,14 @@ def tmux_kill_pane(*panes): def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, - follow=False, target_pane=None): + follow=False, target_pane=None, + command=None, text=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: raise Exception('Neither lines nor percent specified.') + if not command and not text: + raise Exception('Neither command nor text specified.') # Build cmd cmd = ['tmux', 'split-window', '-PF', '#D'] @@ -583,17 +625,28 @@ def tmux_split_window( if target_pane: cmd.extend(['-t', str(target_pane)]) + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold', text]) + # Run and return pane_id result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane_text(pane_id, text): - """Print text to tmux pane.""" - text = text.replace('\033', r'\e') - cmd = ['tmux', 'send-keys', '-t', pane_id] - run_program(cmd+['Enter']) - run_program(cmd+['clear; echo-and-hold "{}"'.format(text)]) - run_program(cmd+['Enter']) +def tmux_update_pane(pane_id, command=None, text=None): + """Respawn with either a new command or new text.""" + # Bail early + if not command and not text: + raise Exception('Neither command nor text specified.') + + cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold', text]) + + run_program(cmd) def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" From 2d69d93154f6fc2882b3cb6d85a6b8c55bed3a36 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:41:27 -0700 Subject: [PATCH 15/86] Added watch option for tmux_split_window() --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 743db0a2..440ec1c5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -204,7 +204,7 @@ def build_outer_panes(state): # Top state.panes['Top'] = tmux_split_window( behind=True, lines=2, vertical=True, - text='{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS)) + text=TOP_PANE_TEXT) # Started state.panes['Started'] = tmux_split_window( @@ -216,9 +216,7 @@ def build_outer_panes(state): # Progress state.panes['Progress'] = tmux_split_window( lines=SIDE_PANE_WIDTH, - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + watch=state.progress_out) def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" @@ -600,13 +598,13 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - command=None, text=None): + command=None, text=None, watch=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: raise Exception('Neither lines nor percent specified.') - if not command and not text: - raise Exception('Neither command nor text specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') # Build cmd cmd = ['tmux', 'split-window', '-PF', '#D'] @@ -628,7 +626,12 @@ def tmux_split_window( if command: cmd.extend(command) elif text: - cmd.extend(['echo-and-hold', text]) + cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) # Run and return pane_id result = run_program(cmd) @@ -644,7 +647,7 @@ def tmux_update_pane(pane_id, command=None, text=None): if command: cmd.extend(command) elif text: - cmd.extend(['echo-and-hold', text]) + cmd.extend(['echo-and-hold "{}"'.format(text)]) run_program(cmd) From d025b8dc9e91555b9cf123580a49ab8f09eeb3ea Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:49:25 -0700 Subject: [PATCH 16/86] Adjusted how devices are added to the state obj * The change allows for devices to be (dis)connected while the script is running * Devices are scanned and added during run_hw_diags() * Fixes bug that prevented any devices from being added as well --- .bin/Scripts/functions/hw_diags.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 440ec1c5..9e720d92 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -78,6 +78,7 @@ class DevObj(): self.path = dev_path self.smart_attributes = {} self.smartctl = {} + self.state = state self.tests = { 'NVMe / SMART': {'Result': None, 'Status': None}, 'badblocks': {'Result': None, 'Status': None}, @@ -158,7 +159,8 @@ class State(): self.devs = [] self.finished = False self.panes = {} - self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + # TODO Switch to LogDir + self.progress_out = '{}/progress.out'.format(global_vars['TmpDir']) self.quick_mode = False self.started = False self.tests = { @@ -168,19 +170,20 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - try_and_print( - message='Scanning devices...', - function=self.add_devs, - cs='Done') - def add_devs(self): - """Add all block devices listed by lsblk.""" + def init(self): + """Scan for block devices and reset all tests.""" + self.devs = [] + for k in ['Result', 'Started', 'Status']: + self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) for dev in json_data['blockdevices']: skip_dev = False - dev_obj = DevObj(dev['name']) + dev_obj = DevObj(self, dev['name']) # Skip loopback devices if dev_obj.lsblk['tran'] == 'NONE': @@ -194,7 +197,7 @@ class State(): # Add device if not skip_dev: - self.devs.append(DevObj(dev['name'])) + self.devs.append(dev_obj) # Functions def build_outer_panes(state): @@ -464,6 +467,9 @@ def run_badblocks_test(state): def run_hw_tests(state): """Run enabled hardware tests.""" + print_standard('Scanning devices...') + state.init() + # Build Panes build_outer_panes(state) From 7c163a8110b8474a8d75dee1e091b7b109d43eea Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:52:24 -0700 Subject: [PATCH 17/86] Added update progress sections --- .bin/Scripts/functions/hw_diags.py | 158 +++++++++++++++++++++++------ 1 file changed, 126 insertions(+), 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9e720d92..ac128ea1 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -62,13 +62,13 @@ IO_VARS = { KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) -SIDE_PANE_WIDTH = 21 +SIDE_PANE_WIDTH = 20 TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes class DevObj(): """Device object for tracking device specific data.""" - def __init__(self, dev_path): + def __init__(self, state, dev_path): self.failing = False self.labels = [] self.lsblk = {} @@ -80,13 +80,17 @@ class DevObj(): self.smartctl = {} self.state = state self.tests = { - 'NVMe / SMART': {'Result': None, 'Status': None}, - 'badblocks': {'Result': None, 'Status': None}, + 'NVMe / SMART': { + 'Result': '', 'Started': False, 'Status': '', 'Order': 1}, + 'badblocks': { + 'Result': '', 'Started': False, 'Status': '', 'Order': 2}, 'I/O Benchmark': { - 'Result': None, - 'Status': None, + 'Result': '', + 'Started': False, + 'Status': '', 'Read Rates': [], - 'Graph Data': []}, + 'Graph Data': [], + 'Order': 3}, } self.get_details() self.get_smart_details() @@ -153,6 +157,21 @@ class DevObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + def update_progress(self): + """Update status strings.""" + for k, v in self.tests.items(): + if self.state.tests[k]['Enabled']: + _status = '' + if not v['Status']: + _status = 'Pending' + if v['Started']: + if v['Result']: + _status = v['Result'] + else: + _status = 'Working' + if _status: + v['Status'] = build_status_string(self.name, _status) + class State(): """Object to track device objects and overall state.""" def __init__(self): @@ -165,7 +184,7 @@ class State(): self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': None, 'Status': None}, + 'Result': '', 'Started': False, 'Status': ''}, 'NVMe / SMART': {'Enabled': False, 'Order': 2}, 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, @@ -199,6 +218,27 @@ class State(): if not skip_dev: self.devs.append(dev_obj) + def update_progress(self): + """Update status strings.""" + # Prime95 + p = self.tests['Prime95 & Temps'] + if p['Enabled']: + _status = '' + if not p['Status']: + _status = 'Pending' + if p['Started']: + if p['Result']: + _status = p['Result'] + else: + _status = 'Working' + if _status: + p['Status'] = build_status_string( + 'Prime95', _status, info_label=True) + + # Disks + for dev in self.devs: + dev.update_progress() + # Functions def build_outer_panes(state): """Build top and side panes.""" @@ -221,6 +261,24 @@ def build_outer_panes(state): lines=SIDE_PANE_WIDTH, watch=state.progress_out) +def build_status_string(label, status, info_label=False): + """Build status string with appropriate colors.""" + status_color = COLORS['CLEAR'] + if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + status_color = COLORS['RED'] + elif status in ['Aborted', 'Unknown', 'Working', 'Skipped']: + status_color = COLORS['YELLOW'] + elif status in ['CS']: + status_color = COLORS['GREEN'] + + return '{l_c}{l}{CLEAR}{s_c}{s:>{s_w}}{CLEAR}'.format( + l_c=COLORS['BLUE'] if info_label else '', + l=label, + s_c=status_color, + s=status, + s_w=SIDE_PANE_WIDTH-len(label), + **COLORS) + def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" needs_override = False @@ -247,8 +305,9 @@ def check_dev_attributes(dev): pass else: for v in dev.tests.values(): - v['Enabled'] = False + # Started is set to True to fix the status string v['Result'] = 'Skipped' + v['Started'] = True v['Status'] = 'Skipped' print_standard('') @@ -457,13 +516,13 @@ def run_badblocks_test(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) print_standard('TODO: run_badblocks_test()') - sleep(3) + for dev in state.devs: + dev.tests['badblocks']['Started'] = True + update_progress_pane(state) + sleep(3) + dev.tests['badblocks']['Result'] = 'OVERRIDE' + update_progress_pane(state) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -471,6 +530,7 @@ def run_hw_tests(state): state.init() # Build Panes + update_progress_pane(state) build_outer_panes(state) # Run test(s) @@ -515,13 +575,13 @@ def run_io_benchmark(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) print_standard('TODO: run_io_benchmark()') - sleep(3) + for dev in state.devs: + dev.tests['I/O Benchmark']['Started'] = True + update_progress_pane(state) + sleep(3) + dev.tests['I/O Benchmark']['Result'] = 'Unknown' + update_progress_pane(state) def run_keyboard_test(): """Run keyboard test.""" @@ -534,16 +594,18 @@ def run_mprime_test(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + state.tests['Prime95 & Temps']['Started'] = True + update_progress_pane(state) + # Get idle temps # Stress CPU # Get max temp # Get cooldown temp + + # Done sleep(3) + state.tests['Prime95 & Temps']['Result'] = 'Unknown' + update_progress_pane(state) def run_network_test(): """Run network test.""" @@ -558,11 +620,8 @@ def run_nvme_smart(state): state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **dev.lsblk)) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + dev.tests['NVMe / SMART']['Started'] = True + update_progress_pane(state) if dev.nvme_attributes: run_nvme_tests(state, dev) elif dev.smart_attributes: @@ -573,15 +632,24 @@ def run_nvme_smart(state): print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( dev.path)) - sleep(3) + dev.tests['NVMe / SMART']['Status'] = 'N/A' + dev.tests['NVMe / SMART']['Result'] = 'N/A' + update_progress_pane(state) + sleep(3) def run_nvme_tests(state, dev): """TODO""" print_standard('TODO: run_nvme_test({})'.format(dev.path)) + sleep(3) + dev.tests['NVMe / SMART']['Result'] = 'CS' + update_progress_pane(state) def run_smart_tests(state, dev): """TODO""" print_standard('TODO: run_smart_tests({})'.format(dev.path)) + sleep(3) + dev.tests['NVMe / SMART']['Result'] = 'CS' + update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" @@ -724,6 +792,32 @@ def update_io_progress(percent, rate, progress_file): with open(progress_file, 'a') as f: f.write(line) +def update_progress_pane(state): + """Update progress file for side pane.""" + output = [] + state.update_progress() + + # Prime95 + output.append(state.tests['Prime95 & Temps']['Status']) + output.append(' ') + + # Disks + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + if 'Prime95' not in k and v['Enabled']: + output.append('{BLUE}{test_name}{CLEAR}'.format( + test_name=k, **COLORS)) + for dev in state.devs: + output.append(dev.tests[k]['Status']) + output.append(' ') + + # Add line-endings + output = ['{}\n'.format(line) for line in output] + + with open(state.progress_out, 'w') as f: + f.writelines(output) + if __name__ == '__main__': print("This file is not meant to be called directly.") From 372f80bf38d56dcb58a4ac76cf7b04555f3f66f5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 04:08:59 -0700 Subject: [PATCH 18/86] Skip optical drives --- .bin/Scripts/functions/hw_diags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index ac128ea1..d122f6c6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -204,8 +204,8 @@ class State(): skip_dev = False dev_obj = DevObj(self, dev['name']) - # Skip loopback devices - if dev_obj.lsblk['tran'] == 'NONE': + # Skip loopback and optical devices + if dev_obj.lsblk['type'] in ['loop', 'rom']: skip_dev = True # Skip WK devices From 163f64dda7921847a5601aaa3d2b0d7825b5a5b0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 04:10:20 -0700 Subject: [PATCH 19/86] Reduced timeout for major exceptions --- .bin/Scripts/functions/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index b5896966..cc3d98b8 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -305,13 +305,13 @@ def major_exception(): except GenericAbort: # User declined upload print_warning('Upload: Aborted') - sleep(30) + sleep(10) except GenericError: # No log file or uploading disabled - sleep(30) + sleep(10) except: print_error('Upload: NS') - sleep(30) + sleep(10) else: print_success('Upload: CS') pause('Press Enter to exit...') From 5dd8fa84164876952a85b4cb97b94eb0a691ea4c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 17:48:30 -0700 Subject: [PATCH 20/86] Get CPU details from lscpu --- .bin/Scripts/functions/hw_diags.py | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d122f6c6..892dff90 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -175,6 +175,7 @@ class DevObj(): class State(): """Object to track device objects and overall state.""" def __init__(self): + self.lscpu = {} self.devs = [] self.finished = False self.panes = {} @@ -189,6 +190,28 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } + self.get_cpu_details() + + def get_cpu_details(self): + """Get CPU details from lscpu.""" + cmd = ['lscpu', '--json'] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + except Exception as err: + # Ignore and leave self.cpu empty + print_error(err) + pause() + return + for line in json_data.get('lscpu', []): + _field = line.get('field', None).replace(':', '') + _data = line.get('data', None) + if not _field and not _data: + # Skip + print_warning(_field, _data) + pause() + continue + self.lscpu[_field] = _data def init(self): """Scan for block devices and reset all tests.""" @@ -591,9 +614,11 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" # Prep - tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'Prime95 & Temps')) + _title = '{}\n{}{}{}'.format( + TOP_PANE_TEXT, 'Prime95 & Temps', + ': ' if 'Model name' in state.lscpu else '', + state.lscpu.get('Model name', '')) + tmux_update_pane(state.panes['Top'], text=_title) state.tests['Prime95 & Temps']['Started'] = True update_progress_pane(state) From cb67f7e3c3ce26c7475b739ae937849b970d198a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 19:59:41 -0700 Subject: [PATCH 21/86] Added new sensors.py and dropped borrowed sensors --- .bin/Scripts/borrowed/sensors-README.md | 35 ---- .bin/Scripts/borrowed/sensors.py | 236 ------------------------ .bin/Scripts/functions/common.py | 11 +- .bin/Scripts/functions/sensors.py | 111 +++++++++++ 4 files changed, 117 insertions(+), 276 deletions(-) delete mode 100644 .bin/Scripts/borrowed/sensors-README.md delete mode 100644 .bin/Scripts/borrowed/sensors.py create mode 100644 .bin/Scripts/functions/sensors.py diff --git a/.bin/Scripts/borrowed/sensors-README.md b/.bin/Scripts/borrowed/sensors-README.md deleted file mode 100644 index 11858382..00000000 --- a/.bin/Scripts/borrowed/sensors-README.md +++ /dev/null @@ -1,35 +0,0 @@ -sensors.py -========== -python bindings using ctypes for libsensors3 of the [lm-sensors project](https://github.com/groeck/lm-sensors). The code was written against libsensors 3.3.4. - -For documentation of the low level API see [sensors.h](https://github.com/groeck/lm-sensors/blob/master/lib/sensors.h). For an example of the high level API see [example.py](example.py). - -For a GUI application that displays the sensor readings and is based on this library, take a look at [sensors-unity](https://launchpad.net/sensors-unity). - -Features --------- -* Full access to low level libsensors3 API -* High level iterator API -* unicode handling -* Python2 and Python3 compatible - -Licensing ---------- -LGPLv2 (same as libsensors3) - -Usage Notes ------------ -As Python does not support call by reference for primitive types some of the libsensors API had to be adapted: - -```python -# nr is changed by refrence in the C API -chip_name, nr = sensors.get_detected_chips(None, nr) - -# returns the value. throws on error -val = sensors.get_value(chip, subfeature_nr) -``` - -Missing Features (pull requests are welcome): -* `sensors_subfeature_type` enum -* `sensors_get_subfeature` -* Error handlers diff --git a/.bin/Scripts/borrowed/sensors.py b/.bin/Scripts/borrowed/sensors.py deleted file mode 100644 index 39b00a4f..00000000 --- a/.bin/Scripts/borrowed/sensors.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -@package sensors.py -Python Bindings for libsensors3 - -use the documentation of libsensors for the low level API. -see example.py for high level API usage. - -@author: Pavel Rojtberg (http://www.rojtberg.net) -@see: https://github.com/paroj/sensors.py -@copyright: LGPLv2 (same as libsensors) -""" - -from ctypes import * -import ctypes.util - -_libc = cdll.LoadLibrary(ctypes.util.find_library("c")) -# see https://github.com/paroj/sensors.py/issues/1 -_libc.free.argtypes = [c_void_p] - -_hdl = cdll.LoadLibrary(ctypes.util.find_library("sensors")) - -version = c_char_p.in_dll(_hdl, "libsensors_version").value.decode("ascii") - -class bus_id(Structure): - _fields_ = [("type", c_short), - ("nr", c_short)] - -class chip_name(Structure): - _fields_ = [("prefix", c_char_p), - ("bus", bus_id), - ("addr", c_int), - ("path", c_char_p)] - -class feature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int)] - - # sensors_feature_type - IN = 0x00 - FAN = 0x01 - TEMP = 0x02 - POWER = 0x03 - ENERGY = 0x04 - CURR = 0x05 - HUMIDITY = 0x06 - MAX_MAIN = 0x7 - VID = 0x10 - INTRUSION = 0x11 - MAX_OTHER = 0x12 - BEEP_ENABLE = 0x18 - -class subfeature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int), - ("mapping", c_int), - ("flags", c_uint)] - -_hdl.sensors_get_detected_chips.restype = POINTER(chip_name) -_hdl.sensors_get_features.restype = POINTER(feature) -_hdl.sensors_get_all_subfeatures.restype = POINTER(subfeature) -_hdl.sensors_get_label.restype = c_void_p # return pointer instead of str so we can free it -_hdl.sensors_get_adapter_name.restype = c_char_p # docs do not say whether to free this or not -_hdl.sensors_strerror.restype = c_char_p - -### RAW API ### -MODE_R = 1 -MODE_W = 2 -COMPUTE_MAPPING = 4 - -def init(cfg_file = None): - file = _libc.fopen(cfg_file.encode("utf-8"), "r") if cfg_file is not None else None - - if _hdl.sensors_init(file) != 0: - raise Exception("sensors_init failed") - - if file is not None: - _libc.fclose(file) - -def cleanup(): - _hdl.sensors_cleanup() - -def parse_chip_name(orig_name): - ret = chip_name() - err= _hdl.sensors_parse_chip_name(orig_name.encode("utf-8"), byref(ret)) - - if err < 0: - raise Exception(strerror(err)) - - return ret - -def strerror(errnum): - return _hdl.sensors_strerror(errnum).decode("utf-8") - -def free_chip_name(chip): - _hdl.sensors_free_chip_name(byref(chip)) - -def get_detected_chips(match, nr): - """ - @return: (chip, next nr to query) - """ - _nr = c_int(nr) - - if match is not None: - match = byref(match) - - chip = _hdl.sensors_get_detected_chips(match, byref(_nr)) - chip = chip.contents if bool(chip) else None - return chip, _nr.value - -def chip_snprintf_name(chip, buffer_size=200): - """ - @param buffer_size defaults to the size used in the sensors utility - """ - ret = create_string_buffer(buffer_size) - err = _hdl.sensors_snprintf_chip_name(ret, buffer_size, byref(chip)) - - if err < 0: - raise Exception(strerror(err)) - - return ret.value.decode("utf-8") - -def do_chip_sets(chip): - """ - @attention this function was not tested - """ - err = _hdl.sensors_do_chip_sets(byref(chip)) - if err < 0: - raise Exception(strerror(err)) - -def get_adapter_name(bus): - return _hdl.sensors_get_adapter_name(byref(bus)).decode("utf-8") - -def get_features(chip, nr): - """ - @return: (feature, next nr to query) - """ - _nr = c_int(nr) - feature = _hdl.sensors_get_features(byref(chip), byref(_nr)) - feature = feature.contents if bool(feature) else None - return feature, _nr.value - -def get_label(chip, feature): - ptr = _hdl.sensors_get_label(byref(chip), byref(feature)) - val = cast(ptr, c_char_p).value.decode("utf-8") - _libc.free(ptr) - return val - -def get_all_subfeatures(chip, feature, nr): - """ - @return: (subfeature, next nr to query) - """ - _nr = c_int(nr) - subfeature = _hdl.sensors_get_all_subfeatures(byref(chip), byref(feature), byref(_nr)) - subfeature = subfeature.contents if bool(subfeature) else None - return subfeature, _nr.value - -def get_value(chip, subfeature_nr): - val = c_double() - err = _hdl.sensors_get_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - return val.value - -def set_value(chip, subfeature_nr, value): - """ - @attention this function was not tested - """ - val = c_double(value) - err = _hdl.sensors_set_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - -### Convenience API ### -class ChipIterator: - def __init__(self, match = None): - self.match = parse_chip_name(match) if match is not None else None - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - chip, self.nr = get_detected_chips(self.match, self.nr) - - if chip is None: - raise StopIteration - - return chip - - def __del__(self): - if self.match is not None: - free_chip_name(self.match) - - def next(self): # python2 compability - return self.__next__() - -class FeatureIterator: - def __init__(self, chip): - self.chip = chip - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - feature, self.nr = get_features(self.chip, self.nr) - - if feature is None: - raise StopIteration - - return feature - - def next(self): # python2 compability - return self.__next__() - -class SubFeatureIterator: - def __init__(self, chip, feature): - self.chip = chip - self.feature = feature - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - subfeature, self.nr = get_all_subfeatures(self.chip, self.feature, self.nr) - - if subfeature is None: - raise StopIteration - - return subfeature - - def next(self): # python2 compability - return self.__next__() diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index cc3d98b8..2bf52a85 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -25,12 +25,13 @@ global_vars = {} # STATIC VARIABLES COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'GREEN': '\033[32m', + 'CLEAR': '\033[0m', + 'RED': '\033[31m', + 'GREEN': '\033[32m', 'YELLOW': '\033[33m', - 'BLUE': '\033[34m' -} + 'ORANGE': '\033[31;1m', + 'BLUE': '\033[34m' + } try: HKU = winreg.HKEY_USERS HKCR = winreg.HKEY_CLASSES_ROOT diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py new file mode 100644 index 00000000..0cb65acd --- /dev/null +++ b/.bin/Scripts/functions/sensors.py @@ -0,0 +1,111 @@ +# Wizard Kit: Functions - Sensors + +import itertools +import json +import re + +from functions.common import * + +# STATIC VARIABLES +TEMP_LIMITS = { + 'GREEN': 60, + 'YELLOW': 70, + 'ORANGE': 80, + 'RED': 90, + } + +# REGEX +REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') + +def fix_sensor_str(s): + """Cleanup string and return str.""" + s = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', s, re.IGNORECASE) + s = s.title() + s = s.replace('Coretemp', 'CoreTemp') + s = s.replace('Acpi', 'ACPI') + s = s.replace('Isa ', 'ISA ') + s = s.replace('Id ', 'ID ') + s = re.sub(r'(\D+)(\d+)', r'\1 \2', s, re.IGNORECASE) + s = s.replace(' ', ' ') + return s + +def get_color_temp(temp): + """Get colored temp string, returns str.""" + try: + temp = float(temp) + except ValueError: + return '{YELLOW}{temp}{CLEAR}'.format(temp=temp, **COLORS) + if temp > TEMP_LIMITS['RED']: + color = COLORS['RED'] + elif temp > TEMP_LIMITS['ORANGE']: + color = COLORS['ORANGE'] + elif temp > TEMP_LIMITS['YELLOW']: + color = COLORS['YELLOW'] + elif temp > TEMP_LIMITS['GREEN']: + color = COLORS['GREEN'] + elif temp > 0: + color = COLORS['BLUE'] + else: + color = COLORS['CLEAR'] + return '{color}{prefix}{temp:2.0f}°C{CLEAR}'.format( + color = color, + prefix = '-' if temp < 0 else '', + temp = temp, + **COLORS) + +def get_raw_sensor_data(): + """Read sensor data and return dict.""" + cmd = ['sensors', '-j'] + result = run_program(cmd) + return json.loads(result.stdout.decode()) + +def get_sensor_data(): + """Parse raw sensor data and return new dict.""" + json_data = get_raw_sensor_data() + sensor_data = {'CoreTemps': {}, 'Other': {}} + for _adapter, _sources in json_data.items(): + if 'coretemp' in _adapter: + _section = 'CoreTemps' + else: + _section = 'Other' + sensor_data[_section][_adapter] = {} + _sources.pop('Adapter', None) + + # Find current temp and add to dict + ## current temp is labeled xxxx_input + for _source, _labels in _sources.items(): + for _label, _temp in _labels.items(): + if 'input' in _label: + sensor_data[_section][_adapter][_source] = { + 'Temps': [_temp], + 'Label': _label, + } + + # Remove empty sections + for k, v in sensor_data.items(): + v = {k2: v2 for k2, v2 in v.items() if v2} + + # Done + return sensor_data + +def update_sensor_data(sensor_data): + """Read sensors and update existing sensor_data, returns dict.""" + json_data = get_raw_sensor_data() + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _label = _ddata['Label'] + _temp = json_data[_adapter][_source][_label] + _data['Temps'].append(_temp) + return sensor_data + +def join_columns(column1, column2, width=55): + return '{:<{}}{}'.format( + column1, + 55+len(column1)-len(REGEX_COLORS.sub('', column1)), + column2) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 7140f38ba4fefdfb571405b97a80e2eb7ee6f783 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:11:10 -0700 Subject: [PATCH 22/86] Added average, clear, and max temps sections --- .bin/Scripts/functions/sensors.py | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 0cb65acd..10a2e664 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -17,6 +17,14 @@ TEMP_LIMITS = { # REGEX REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') +def clear_temps(sensor_data): + """Clear saved temps but keep structure, returns dict.""" + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data['Temps'] = [] + return sensor_data + def fix_sensor_str(s): """Cleanup string and return str.""" s = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', s, re.IGNORECASE) @@ -88,13 +96,41 @@ def get_sensor_data(): # Done return sensor_data +def save_max_temp(sensor_data): + """Record max temps seen this session, returns dict.""" + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data['Max'] = max(_data['Temps']) + + # Done + return sensor_data + +def save_average_temp(sensor_data, save_label, seconds=10): + """Calculate average temps and record under save_label, returns dict.""" + clear_temps(sensor_data) + + # Get temps + for i in range(seconds): + sensor_data = update_sensor_data(sensor_data) + sleep(1) + + # Calculate averages + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data[save_label] = sum(_data['Temps']) / len(_data['Temps']) + + # Done + return sensor_data + def update_sensor_data(sensor_data): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() for _section, _adapters in sensor_data.items(): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): - _label = _ddata['Label'] + _label = _data['Label'] _temp = json_data[_adapter][_source][_label] _data['Temps'].append(_temp) return sensor_data From 2eccc236a960235df41ec864116affae0efd281f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:40:25 -0700 Subject: [PATCH 23/86] Added generate_report() * Also merged save_max_temp() with update_sensor_data() * Max doesn't need resetting so just calc max everytime --- .bin/Scripts/functions/sensors.py | 59 +++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 10a2e664..30df47de 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -37,8 +37,30 @@ def fix_sensor_str(s): s = s.replace(' ', ' ') return s -def get_color_temp(temp): - """Get colored temp string, returns str.""" +def generate_report(sensor_data, *temp_labels, colors=True): + """Build report based on temp_labels, returns list if str.""" + report = [] + for _section, _adapters in sorted(sensor_data.items()): + # CoreTemps then Other temps + for _adapter, _sources in sorted(_adapters.items()): + # Adapter + report.append(fix_sensor_str(_adapter)) + for _source, _data in sorted(_sources.items()): + # Source + _line = '{:18} '.format(fix_sensor_str(_source)) + _temps = [] + for _label in temp_labels: + _temps.append('{}{}{}'.format( + _label.lower() if _label != 'Current' else '', + ': ' if _label != 'Current' else '', + get_temp_str(_data[_label], colors=colors))) + _line += ', '.join(_temps) + report.append(_line) + report.append(' ') + return sensor_data + +def get_colored_temp_str(temp): + """Get colored string based on temp, returns str.""" try: temp = float(temp) except ValueError: @@ -85,8 +107,10 @@ def get_sensor_data(): for _label, _temp in _labels.items(): if 'input' in _label: sensor_data[_section][_adapter][_source] = { - 'Temps': [_temp], + 'Current': _temp, 'Label': _label, + 'Max': _temp, + 'Temps': [_temp], } # Remove empty sections @@ -96,18 +120,21 @@ def get_sensor_data(): # Done return sensor_data -def save_max_temp(sensor_data): - """Record max temps seen this session, returns dict.""" - for _section, _adapters in sensor_data.items(): - for _adapter, _sources in _adapters.items(): - for _source, _data in _sources.items(): - _data['Max'] = max(_data['Temps']) +def get_temp_str(temp, colors=True): + """Get temp string, returns str.""" + if colors: + return get_colored_temp_str(temp) + try: + temp = float(temp) + except ValueError: + return '{}°C'.format(temp) + else: + return '{}{:2.0f}°C'.format( + '-' if temp < 0 else '', + temp) - # Done - return sensor_data - -def save_average_temp(sensor_data, save_label, seconds=10): - """Calculate average temps and record under save_label, returns dict.""" +def save_average_temp(sensor_data, temp_label, seconds=10): + """Calculate average temps and record under temp_label, returns dict.""" clear_temps(sensor_data) # Get temps @@ -119,7 +146,7 @@ def save_average_temp(sensor_data, save_label, seconds=10): for _section, _adapters in sensor_data.items(): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): - _data[save_label] = sum(_data['Temps']) / len(_data['Temps']) + _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) # Done return sensor_data @@ -132,6 +159,8 @@ def update_sensor_data(sensor_data): for _source, _data in _sources.items(): _label = _data['Label'] _temp = json_data[_adapter][_source][_label] + _data['Current'] = _temp + _data['Max'] = max(_temp, _data['Max']) _data['Temps'].append(_temp) return sensor_data From 328d6eb29484711774a5386c79a9415d25d2c3a4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:47:40 -0700 Subject: [PATCH 24/86] Modify sensor_data in place --- .bin/Scripts/functions/sensors.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 30df47de..bead3bfa 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -23,7 +23,6 @@ def clear_temps(sensor_data): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): _data['Temps'] = [] - return sensor_data def fix_sensor_str(s): """Cleanup string and return str.""" @@ -57,7 +56,6 @@ def generate_report(sensor_data, *temp_labels, colors=True): _line += ', '.join(_temps) report.append(_line) report.append(' ') - return sensor_data def get_colored_temp_str(temp): """Get colored string based on temp, returns str.""" @@ -139,7 +137,7 @@ def save_average_temp(sensor_data, temp_label, seconds=10): # Get temps for i in range(seconds): - sensor_data = update_sensor_data(sensor_data) + update_sensor_data(sensor_data) sleep(1) # Calculate averages @@ -148,9 +146,6 @@ def save_average_temp(sensor_data, temp_label, seconds=10): for _source, _data in _sources.items(): _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) - # Done - return sensor_data - def update_sensor_data(sensor_data): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() @@ -162,7 +157,6 @@ def update_sensor_data(sensor_data): _data['Current'] = _temp _data['Max'] = max(_temp, _data['Max']) _data['Temps'].append(_temp) - return sensor_data def join_columns(column1, column2, width=55): return '{:<{}}{}'.format( From 95b0d1e3f4823d31bcbf5bc0c2ee20753b20379e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 21:54:41 -0700 Subject: [PATCH 25/86] Wrap reports if necessary --- .bin/Scripts/functions/sensors.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index bead3bfa..5e33e1cb 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -3,6 +3,7 @@ import itertools import json import re +import shutil from functions.common import * @@ -37,7 +38,7 @@ def fix_sensor_str(s): return s def generate_report(sensor_data, *temp_labels, colors=True): - """Build report based on temp_labels, returns list if str.""" + """Generate report based on temp_labels, returns list if str.""" report = [] for _section, _adapters in sorted(sensor_data.items()): # CoreTemps then Other temps @@ -48,6 +49,7 @@ def generate_report(sensor_data, *temp_labels, colors=True): # Source _line = '{:18} '.format(fix_sensor_str(_source)) _temps = [] + # Temps (skip label for Current) for _label in temp_labels: _temps.append('{}{}{}'.format( _label.lower() if _label != 'Current' else '', @@ -57,6 +59,26 @@ def generate_report(sensor_data, *temp_labels, colors=True): report.append(_line) report.append(' ') + # Wrap lines if necessary + screen_size = shutil.get_terminal_size() + rows = screen_size.lines - 1 + if len(report) > rows and screen_size.columns > 55*2: + report = list(itertools.zip_longest( + report[:rows], report[rows:], fillvalue='')) + report = [join_columns(a, b) for a, b in report] + + # Handle empty reports (i.e. no sensors detected) + if not report: + report = [ + '{}WARNING: No sensors found{}'.format( + COLORS['YELLOW'] if colors else '', + COLORS['CLEAR'] if colors else ''), + ' ', + 'Please monitor temps manually'] + + # Done + return report + def get_colored_temp_str(temp): """Get colored string based on temp, returns str.""" try: From 0e5fab01047c69fd875d43e5ced54377f70a5ec5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 21:57:55 -0700 Subject: [PATCH 26/86] Handle missing labels in generate_report() --- .bin/Scripts/functions/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 5e33e1cb..b47f9afd 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -54,7 +54,7 @@ def generate_report(sensor_data, *temp_labels, colors=True): _temps.append('{}{}{}'.format( _label.lower() if _label != 'Current' else '', ': ' if _label != 'Current' else '', - get_temp_str(_data[_label], colors=colors))) + get_temp_str(_data.get(_label, '???'), colors=colors))) _line += ', '.join(_temps) report.append(_line) report.append(' ') @@ -147,7 +147,7 @@ def get_temp_str(temp, colors=True): try: temp = float(temp) except ValueError: - return '{}°C'.format(temp) + return '{}'.format(temp) else: return '{}{:2.0f}°C'.format( '-' if temp < 0 else '', From 46080b43634f0317b78581c28a86493afc8acf06 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 22:25:44 -0700 Subject: [PATCH 27/86] Moved tmux sections to separate file --- .bin/Scripts/functions/hw_diags.py | 66 +-------------------------- .bin/Scripts/functions/tmux.py | 72 ++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 65 deletions(-) create mode 100644 .bin/Scripts/functions/tmux.py diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 892dff90..cc9a2c40 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,7 +4,7 @@ import json import re import time -from functions.common import * +from functions.tmux import * # STATIC VARIABLES ATTRIBUTES = { @@ -686,70 +686,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def tmux_kill_pane(*panes): - """Kill tmux pane by id.""" - cmd = ['tmux', 'kill-pane', '-t'] - for pane_id in panes: - print(pane_id) - run_program(cmd+[pane_id], check=False) - -def tmux_split_window( - lines=None, percent=None, - behind=False, vertical=False, - follow=False, target_pane=None, - command=None, text=None, watch=None): - """Run tmux split-window command and return pane_id as str.""" - # Bail early - if not lines and not percent: - raise Exception('Neither lines nor percent specified.') - if not command and not text and not watch: - raise Exception('No command, text, or watch file specified.') - - # Build cmd - cmd = ['tmux', 'split-window', '-PF', '#D'] - if behind: - cmd.append('-b') - if vertical: - cmd.append('-v') - else: - cmd.append('-h') - if not follow: - cmd.append('-d') - if lines is not None: - cmd.extend(['-l', str(lines)]) - elif percent is not None: - cmd.extend(['-p', str(percent)]) - if target_pane: - cmd.extend(['-t', str(target_pane)]) - - if command: - cmd.extend(command) - elif text: - cmd.extend(['echo-and-hold "{}"'.format(text)]) - elif watch: - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) - - # Run and return pane_id - result = run_program(cmd) - return result.stdout.decode().strip() - -def tmux_update_pane(pane_id, command=None, text=None): - """Respawn with either a new command or new text.""" - # Bail early - if not command and not text: - raise Exception('Neither command nor text specified.') - - cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] - if command: - cmd.extend(command) - elif text: - cmd.extend(['echo-and-hold "{}"'.format(text)]) - - run_program(cmd) - def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py new file mode 100644 index 00000000..ba00220f --- /dev/null +++ b/.bin/Scripts/functions/tmux.py @@ -0,0 +1,72 @@ +# Wizard Kit: Functions - tmux + +from functions.common import * + +def tmux_kill_pane(*panes): + """Kill tmux pane by id.""" + cmd = ['tmux', 'kill-pane', '-t'] + for pane_id in panes: + print(pane_id) + run_program(cmd+[pane_id], check=False) + +def tmux_split_window( + lines=None, percent=None, + behind=False, vertical=False, + follow=False, target_pane=None, + command=None, text=None, watch=None): + """Run tmux split-window command and return pane_id as str.""" + # Bail early + if not lines and not percent: + raise Exception('Neither lines nor percent specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') + + # Build cmd + cmd = ['tmux', 'split-window', '-PF', '#D'] + if behind: + cmd.append('-b') + if vertical: + cmd.append('-v') + else: + cmd.append('-h') + if not follow: + cmd.append('-d') + if lines is not None: + cmd.extend(['-l', str(lines)]) + elif percent is not None: + cmd.extend(['-p', str(percent)]) + if target_pane: + cmd.extend(['-t', str(target_pane)]) + + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + + # Run and return pane_id + result = run_program(cmd) + return result.stdout.decode().strip() + +def tmux_update_pane(pane_id, command=None, text=None): + """Respawn with either a new command or new text.""" + # Bail early + if not command and not text: + raise Exception('Neither command nor text specified.') + + cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold "{}"'.format(text)]) + + run_program(cmd) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 5405b97eb1cb6b6747ea44e9f64a848e5977f175 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:09:42 -0700 Subject: [PATCH 28/86] Standalone sensor monitor working again --- .bin/Scripts/functions/sensors.py | 14 ++- .bin/Scripts/functions/tmux.py | 7 ++ .bin/Scripts/hw-sensors | 168 +----------------------------- .bin/Scripts/hw-sensors-monitor | 31 ++++++ 4 files changed, 56 insertions(+), 164 deletions(-) create mode 100755 .bin/Scripts/hw-sensors-monitor diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index b47f9afd..ec53e548 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -5,7 +5,7 @@ import json import re import shutil -from functions.common import * +from functions.tmux import * # STATIC VARIABLES TEMP_LIMITS = { @@ -153,6 +153,18 @@ def get_temp_str(temp, colors=True): '-' if temp < 0 else '', temp) +def monitor_sensors(monitor_pane, monitor_file): + """Continually update sensor data and report to screen.""" + sensor_data = get_sensor_data() + while True: + update_sensor_data(sensor_data) + with open(monitor_file, 'w') as f: + report = generate_report(sensor_data, 'Current', 'Max') + f.write('\n'.join(report)) + sleep(1) + if not tmux_poll_pane(monitor_pane): + break + def save_average_temp(sensor_data, temp_label, seconds=10): """Calculate average temps and record under temp_label, returns dict.""" clear_temps(sensor_data) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index ba00220f..10412a8c 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -9,6 +9,13 @@ def tmux_kill_pane(*panes): print(pane_id) run_program(cmd+[pane_id], check=False) +def tmux_poll_pane(pane_id): + """Check if pane exists, returns bool.""" + cmd = ['tmux', 'list-panes', '-F', '#D'] + result = run_program(cmd, check=False) + panes = result.stdout.decode().splitlines() + return pane_id in panes + def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, diff --git a/.bin/Scripts/hw-sensors b/.bin/Scripts/hw-sensors index f251c589..39ca7147 100755 --- a/.bin/Scripts/hw-sensors +++ b/.bin/Scripts/hw-sensors @@ -1,168 +1,10 @@ -#!/bin/python3 +#!/bin/bash # ## Wizard Kit: Sensor monitoring tool -import itertools -import os -import shutil -import sys +WINDOW_NAME="Hardware Sensors" +MONITOR="hw-sensors-monitor" -# Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) -from functions.common import * -from borrowed import sensors - -# STATIC VARIABLES -COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'GREEN': '\033[32m', - 'YELLOW': '\033[33m', - 'ORANGE': '\033[31;1m', - 'BLUE': '\033[34m' - } -TEMP_LIMITS = { - 'GREEN': 60, - 'YELLOW': 70, - 'ORANGE': 80, - 'RED': 90, - } - -# REGEX -REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') - -def color_temp(temp): - try: - temp = float(temp) - except ValueError: - return '{YELLOW}{temp}{CLEAR}'.format(temp=temp, **COLORS) - if temp > TEMP_LIMITS['RED']: - color = COLORS['RED'] - elif temp > TEMP_LIMITS['ORANGE']: - color = COLORS['ORANGE'] - elif temp > TEMP_LIMITS['YELLOW']: - color = COLORS['YELLOW'] - elif temp > TEMP_LIMITS['GREEN']: - color = COLORS['GREEN'] - elif temp > 0: - color = COLORS['BLUE'] - else: - color = COLORS['CLEAR'] - return '{color}{prefix}{temp:2.0f}°C{CLEAR}'.format( - color = color, - prefix = '-' if temp < 0 else '', - temp = temp, - **COLORS) - -def get_feature_string(chip, feature): - sfs = list(sensors.SubFeatureIterator(chip, feature)) # get a list of all subfeatures - label = sensors.get_label(chip, feature) - skipname = len(feature.name)+1 # skip common prefix - data = {} - - if feature.type != sensors.feature.TEMP: - # Skip non-temperature sensors - return None - - for sf in sfs: - name = sf.name[skipname:].decode("utf-8").strip() - try: - val = sensors.get_value(chip, sf.number) - except Exception: - # Ignore upstream sensor bugs and lie instead - val = -123456789 - if 'alarm' in name: - # Skip - continue - if '--nocolor' in sys.argv: - try: - temp = float(val) - except ValueError: - data[name] = ' {}°C'.format(val) - else: - data[name] = '{}{:2.0f}°C'.format( - '-' if temp < 0 else '', - temp) - else: - data[name] = color_temp(val) - - main_temp = data.pop('input', None) - if main_temp: - list_data = [] - for item in ['max', 'crit']: - if item in data: - list_data.append('{}: {}'.format(item, data.pop(item))) - list_data.extend( - ['{}: {}'.format(k, v) for k, v in sorted(data.items())]) - data_str = '{:18} {} {}'.format( - label, main_temp, ', '.join(list_data)) - else: - list_data.extend(sorted(data.items())) - list_data = ['{}: {}'.format(item[0], item[1]) for item in list_data] - data_str = '{:18} {}'.format(label, ', '.join(list_data)) - return data_str - -def join_columns(column1, column2, width=55): - return '{:<{}}{}'.format( - column1, - 55+len(column1)-len(REGEX_COLORS.sub('', column1)), - column2) - -if __name__ == '__main__': - try: - # Prep - sensors.init() - - # Get sensor data - chip_temps = {} - for chip in sensors.ChipIterator(): - chip_name = '{} ({})'.format( - sensors.chip_snprintf_name(chip), - sensors.get_adapter_name(chip.bus)) - chip_feats = [get_feature_string(chip, feature) - for feature in sensors.FeatureIterator(chip)] - # Strip empty/None items - chip_feats = [f for f in chip_feats if f] - - if chip_feats: - chip_temps[chip_name] = [chip_name, *chip_feats, ''] - - # Sort chips - sensor_temps = [] - for chip in [k for k in sorted(chip_temps.keys()) if 'coretemp' in k]: - sensor_temps.extend(chip_temps[chip]) - for chip in sorted(chip_temps.keys()): - if 'coretemp' not in chip: - sensor_temps.extend(chip_temps[chip]) - - # Wrap columns as needed - screen_size = shutil.get_terminal_size() - rows = screen_size.lines - 1 - if len(sensor_temps) > rows and screen_size.columns > 55*2: - sensor_temps = list(itertools.zip_longest( - sensor_temps[:rows], sensor_temps[rows:], fillvalue='')) - sensor_temps = [join_columns(a, b) for a, b in sensor_temps] - - # Print data - if sensor_temps: - for line in sensor_temps: - print_standard(line) - else: - if '--nocolor' in sys.argv: - print_standard('WARNING: No sensors found') - print_standard('\nPlease monitor temps manually') - else: - print_warning('WARNING: No sensors found') - print_standard('\nPlease monitor temps manually') - - # Done - sensors.cleanup() - exit_script() - except SystemExit: - sensors.cleanup() - pass - except: - sensors.cleanup() - major_exception() +# Start session +tmux new-session -n "$WINDOW_NAME" "$MONITOR" diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor new file mode 100755 index 00000000..91651f32 --- /dev/null +++ b/.bin/Scripts/hw-sensors-monitor @@ -0,0 +1,31 @@ +#!/bin/python3 +# +## Wizard Kit: Sensor monitoring tool + +import os +import sys + +# Init +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) +from functions.sensors import * +from functions.tmux import * +init_global_vars() + +if __name__ == '__main__': + try: + result = run_program(['mktemp']) + monitor_file = result.stdout.decode().strip() + print(monitor_file) + monitor_pane = tmux_split_window( + percent=1, vertical=True, watch=monitor_file) + cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] + run_program(cmd, check=False) + monitor_sensors(monitor_pane, monitor_file) + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 From c777d490913d14666c958f7015996170e6d32c49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:54:37 -0700 Subject: [PATCH 29/86] Added tmux_resize_pane() --- .bin/Scripts/functions/tmux.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 10412a8c..6726e3ff 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -16,6 +16,19 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes +def tmux_resize_pane(pane_id, x=None, y=None): + """Resize pane to specific hieght or width.""" + if not x and not y: + raise Exception('Neither height nor width specified.') + + cmd = ['tmux', 'resize-pane', '-t', pane_id] + if x: + cmd.extend(['-x', str(x)]) + elif y: + cmd.extend(['-y', str(y)]) + + run_program(cmd, check=False) + def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, From 5550cce8dbf4c77d67f15213b2e016a321b6bf87 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:55:15 -0700 Subject: [PATCH 30/86] Add background mode for monitoring sensors * This will be called by hw_diags.py to update a file in the background * NOTE: This uses a naive check before attempting to write data --- .bin/Scripts/functions/sensors.py | 2 +- .bin/Scripts/hw-sensors-monitor | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index ec53e548..41508927 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -162,7 +162,7 @@ def monitor_sensors(monitor_pane, monitor_file): report = generate_report(sensor_data, 'Current', 'Max') f.write('\n'.join(report)) sleep(1) - if not tmux_poll_pane(monitor_pane): + if monitor_pane and not tmux_poll_pane(monitor_pane): break def save_average_temp(sensor_data, temp_label, seconds=10): diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 91651f32..22067b91 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -13,14 +13,20 @@ from functions.tmux import * init_global_vars() if __name__ == '__main__': + background = False try: - result = run_program(['mktemp']) - monitor_file = result.stdout.decode().strip() - print(monitor_file) - monitor_pane = tmux_split_window( - percent=1, vertical=True, watch=monitor_file) - cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] - run_program(cmd, check=False) + if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): + background = True + monitor_file = sys.argv[1] + monitor_pane=None + else: + result = run_program(['mktemp']) + monitor_file = result.stdout.decode().strip() + if not background: + monitor_pane = tmux_split_window( + percent=1, vertical=True, watch=monitor_file) + cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] + run_program(cmd, check=False) monitor_sensors(monitor_pane, monitor_file) exit_script() except SystemExit: From 74bb31e795c41b2466d33ef9fa76f169e3b8ad89 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:57:38 -0700 Subject: [PATCH 31/86] Open temps monitor during run_mprime --- .bin/Scripts/functions/hw_diags.py | 17 +++++++++++++++++ .bin/Scripts/hw-sensors-monitor | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cc9a2c40..594f807a 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,6 +4,7 @@ import json import re import time +from functions.sensors import * from functions.tmux import * # STATIC VARIABLES @@ -614,6 +615,13 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" # Prep + _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(_sensors_out, 'w') as f: + f.write(' ') + sleep(1) + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) _title = '{}\n{}{}{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps', ': ' if 'Model name' in state.lscpu else '', @@ -621,6 +629,15 @@ def run_mprime_test(state): tmux_update_pane(state.panes['Top'], text=_title) state.tests['Prime95 & Temps']['Started'] = True update_progress_pane(state) + state.panes['mprime'] = tmux_split_window( + lines=10, vertical=True, text='Prime95 output goes here...') + state.panes['Temps'] = tmux_split_window( + behind=True, percent=80, vertical=True, watch=_sensors_out) + tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) + + # Start live monitor + pause() + monitor_proc.kill() # Get idle temps # Stress CPU diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 22067b91..42757748 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -18,7 +18,7 @@ if __name__ == '__main__': if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): background = True monitor_file = sys.argv[1] - monitor_pane=None + monitor_pane = None else: result = run_program(['mktemp']) monitor_file = result.stdout.decode().strip() From 30ba6516745347f6a979c7f0763284938c0c62e8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 00:10:51 -0700 Subject: [PATCH 32/86] Removing report wrapping section * Doesn't work properly with background processes --- .bin/Scripts/functions/sensors.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 41508927..29ceaa45 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -1,9 +1,7 @@ # Wizard Kit: Functions - Sensors -import itertools import json import re -import shutil from functions.tmux import * @@ -59,14 +57,6 @@ def generate_report(sensor_data, *temp_labels, colors=True): report.append(_line) report.append(' ') - # Wrap lines if necessary - screen_size = shutil.get_terminal_size() - rows = screen_size.lines - 1 - if len(report) > rows and screen_size.columns > 55*2: - report = list(itertools.zip_longest( - report[:rows], report[rows:], fillvalue='')) - report = [join_columns(a, b) for a, b in report] - # Handle empty reports (i.e. no sensors detected) if not report: report = [ From dc606a87806661a994d8472b188bdf5199fe90d0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 01:06:21 -0700 Subject: [PATCH 33/86] Main Prime95 sections working * Still need check results and update progress sections --- .bin/Scripts/functions/hw_diags.py | 102 ++++++++++++++++++++++++----- .bin/Scripts/functions/sensors.py | 6 +- .bin/Scripts/functions/tmux.py | 1 - .bin/Scripts/hw-diags-prime95 | 3 +- 4 files changed, 88 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 594f807a..a2aca6c8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -186,7 +186,8 @@ class State(): self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': '', 'Started': False, 'Status': ''}, + 'Result': '', 'Sensor Data': get_sensor_data(), + 'Started': False, 'Status': ''}, 'NVMe / SMART': {'Enabled': False, 'Order': 2}, 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, @@ -614,41 +615,106 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" - # Prep - _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(_sensors_out, 'w') as f: - f.write(' ') - sleep(1) - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], - pipe=True) + state.tests['Prime95 & Temps']['Started'] = True + update_progress_pane(state) + _sensor_data = state.tests['Prime95 & Temps']['Sensor Data'] + + # Update top pane _title = '{}\n{}{}{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps', ': ' if 'Model name' in state.lscpu else '', state.lscpu.get('Model name', '')) tmux_update_pane(state.panes['Top'], text=_title) - state.tests['Prime95 & Temps']['Started'] = True - update_progress_pane(state) + + # Start live sensor monitor + _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(_sensors_out, 'w') as f: + f.write(' ') + f.flush() + sleep(0.5) + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) + + # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( - lines=10, vertical=True, text='Prime95 output goes here...') + lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( behind=True, percent=80, vertical=True, watch=_sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) - # Start live monitor - pause() - monitor_proc.kill() - # Get idle temps + clear_screen() + try_and_print( + message='Getting idle temps...', indent=0, + function=save_average_temp, cs='Done', + sensor_data=_sensor_data, temp_label='Idle') + # Stress CPU - # Get max temp + run_program(['apple-fans', 'max']) + tmux_update_pane( + state.panes['mprime'], + command=['hw-diags-prime95', global_vars['TmpDir']]) + time_limit = int(MPRIME_LIMIT) * 60 + try: + for i in range(time_limit): + clear_screen() + sec_left = time_limit - i + min_left = int(sec_left / 60) + if min_left > 0: + print_standard( + 'Running Prime95 ({} minute{} left)'.format( + min_left, + 's' if min_left != 1 else '')) + else: + print_standard( + 'Running Prime95 ({} second{} left)'.format( + sec_left, + 's' if sec_left != 1 else '')) + print_warning('If running too hot, press CTRL+c to abort the test') + update_sensor_data(_sensor_data) + sleep(1) + except KeyboardInterrupt: + # Catch CTRL+C + aborted = True + state.tests['Prime95 & Temps']['Result'] = 'Aborted' + print_warning('\nAborted.') + update_progress_pane(state) + + # Restart live monitor + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) + + # Stop Prime95 (twice for good measure) + tmux_kill_pane(state.panes['mprime']) + run_program(['killall', '-s', 'INT', 'mprime'], check=False) + # Get cooldown temp + run_program(['apple-fans', 'auto']) + clear_screen() + try_and_print( + message='Letting CPU cooldown for bit...', indent=0, + function=sleep, cs='Done', seconds=10) + try_and_print( + message='Getting cooldown temps...', indent=0, + function=save_average_temp, cs='Done', + sensor_data=_sensor_data, temp_label='Cooldown') + + # Check results + # TODO # Done - sleep(3) state.tests['Prime95 & Temps']['Result'] = 'Unknown' update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) + monitor_proc.kill() + + # TODO Testing + print('\n'.join(generate_report(_sensor_data, 'Idle', 'Max', 'Cooldown'))) + def run_network_test(): """Run network test.""" clear_screen() diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 29ceaa45..066dc446 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -46,14 +46,12 @@ def generate_report(sensor_data, *temp_labels, colors=True): for _source, _data in sorted(_sources.items()): # Source _line = '{:18} '.format(fix_sensor_str(_source)) - _temps = [] # Temps (skip label for Current) for _label in temp_labels: - _temps.append('{}{}{}'.format( + _line += '{}{}{} '.format( _label.lower() if _label != 'Current' else '', ': ' if _label != 'Current' else '', - get_temp_str(_data.get(_label, '???'), colors=colors))) - _line += ', '.join(_temps) + get_temp_str(_data.get(_label, '???'), colors=colors)) report.append(_line) report.append(' ') diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 6726e3ff..c9a92194 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -6,7 +6,6 @@ def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] for pane_id in panes: - print(pane_id) run_program(cmd+[pane_id], check=False) def tmux_poll_pane(pane_id): diff --git a/.bin/Scripts/hw-diags-prime95 b/.bin/Scripts/hw-diags-prime95 index 30c6994d..4927da76 100755 --- a/.bin/Scripts/hw-diags-prime95 +++ b/.bin/Scripts/hw-diags-prime95 @@ -14,5 +14,6 @@ if [ ! -d "$1" ]; then fi # Run Prime95 -mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "$1/prime.log" +cd "$1" +mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "prime.log" From ca4234b1c36ace85e82d98ff895885eda76b346d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 15:29:06 -0700 Subject: [PATCH 34/86] Added working_dir arg for tmux command sections --- .bin/Scripts/functions/hw_diags.py | 3 ++- .bin/Scripts/functions/tmux.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a2aca6c8..3c1fc5b3 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -654,7 +654,8 @@ def run_mprime_test(state): run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], - command=['hw-diags-prime95', global_vars['TmpDir']]) + command=['hw-diags-prime95', global_vars['TmpDir']], + working_dir=global_vars['TmpDir']) time_limit = int(MPRIME_LIMIT) * 60 try: for i in range(time_limit): diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index c9a92194..84ab96cc 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -32,7 +32,8 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - command=None, text=None, watch=None): + working_dir=None, command=None, + text=None, watch=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: @@ -57,6 +58,8 @@ def tmux_split_window( if target_pane: cmd.extend(['-t', str(target_pane)]) + if working_dir: + cmd.extend(['-c', working_dir]) if command: cmd.extend(command) elif text: @@ -71,13 +74,15 @@ def tmux_split_window( result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane(pane_id, command=None, text=None): +def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): """Respawn with either a new command or new text.""" # Bail early if not command and not text: raise Exception('Neither command nor text specified.') cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if working_dir: + cmd.extend(['-c', working_dir]) if command: cmd.extend(command) elif text: From a910f2cb033066438af1030a1cc34d1af6c07f6f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 18:27:19 -0700 Subject: [PATCH 35/86] Adjusted Prime95 countdown --- .bin/Scripts/functions/hw_diags.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3c1fc5b3..a76666ab 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -651,6 +651,8 @@ def run_mprime_test(state): sensor_data=_sensor_data, temp_label='Idle') # Stress CPU + print_log('Starting Prime95') + _abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -660,24 +662,23 @@ def run_mprime_test(state): try: for i in range(time_limit): clear_screen() - sec_left = time_limit - i - min_left = int(sec_left / 60) + sec_left = (time_limit - i) % 60 + min_left = int( (time_limit - i) / 60) + _status_str = 'Running Prime95 (' if min_left > 0: - print_standard( - 'Running Prime95 ({} minute{} left)'.format( - min_left, - 's' if min_left != 1 else '')) - else: - print_standard( - 'Running Prime95 ({} second{} left)'.format( - sec_left, - 's' if sec_left != 1 else '')) - print_warning('If running too hot, press CTRL+c to abort the test') + _status_str += '{} minute{}, '.format( + min_left, + 's' if min_left != 1 else '') + _status_str += '{} second{} left)'.format( + sec_left, + 's' if sec_left != 1 else '') + # Not using print wrappers to avoid flooding the log + print(_status_str) + print('{YELLOW}{msg}{CLEAR}'.format(msg=_abort_msg, **COLORS)) update_sensor_data(_sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C - aborted = True state.tests['Prime95 & Temps']['Result'] = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) From 12ff99eb3213569a776363f630c3767efd596da6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 18:27:43 -0700 Subject: [PATCH 36/86] Set LogDir for non-quick tests --- .bin/Scripts/functions/hw_diags.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a76666ab..e1b97df9 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -180,8 +180,7 @@ class State(): self.devs = [] self.finished = False self.panes = {} - # TODO Switch to LogDir - self.progress_out = '{}/progress.out'.format(global_vars['TmpDir']) + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.started = False self.tests = { @@ -221,6 +220,16 @@ class State(): for k in ['Result', 'Started', 'Status']: self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + # Update LogDir + if not self.quick_mode: + global_vars['LogDir'] = '{}/Logs/{}_{}'.format( + global_vars['Env']['HOME'], + get_ticket_number(), + time.strftime('%Y-%m-%d_%H%M_%z')) + os.makedirs(global_vars['LogDir'], exist_ok=True) + global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( + global_vars['LogDir']) + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) @@ -703,6 +712,15 @@ def run_mprime_test(state): function=save_average_temp, cs='Done', sensor_data=_sensor_data, temp_label='Cooldown') + # Move logs to Ticket folder + for item in os.scandir(global_vars['TmpDir']): + try: + shutil.move(item.path, global_vars['LogDir']) + except Exception: + print_error('ERROR: Failed to move "{}" to "{}"'.format( + item.path, + global_vars['LogDir'])) + # Check results # TODO From 6a3ef60881ed43800605b4692e79836a2d978c03 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:41:29 -0700 Subject: [PATCH 37/86] Added CpuObj and renamed dev names to disk * This should make the code more clear * The CpuObj is similar to DiskObj to abstract the device/tests calls * New calls will be like: run_test(state, dev) --- .bin/Scripts/functions/hw_diags.py | 204 +++++++++++++---------------- 1 file changed, 90 insertions(+), 114 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index e1b97df9..a2c8293d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -67,32 +67,43 @@ SIDE_PANE_WIDTH = 20 TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes -class DevObj(): - """Device object for tracking device specific data.""" - def __init__(self, state, dev_path): - self.failing = False +class CpuObj(): + """Object for tracking CPU specific data.""" + def __init__(self): + self.lscpu = {} + self.tests = {} + self.get_details() + + def get_details(self): + """Get CPU details from lscpu.""" + cmd = ['lscpu', '--json'] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + except Exception: + # Ignore and leave self.lscpu empty + return + for line in json_data.get('lscpu', []): + _field = line.get('field', None).replace(':', '') + _data = line.get('data', None) + if not _field and not _data: + # Skip + print_warning(_field, _data) + pause() + continue + self.lscpu[_field] = _data + +class DiskObj(): + """Object for tracking disk specific data.""" + def __init__(self, disk_path): self.labels = [] self.lsblk = {} - self.name = re.sub(r'^.*/(.*)', r'\1', dev_path) + self.name = re.sub(r'^.*/(.*)', r'\1', disk_path) self.nvme_attributes = {} - self.override = False - self.path = dev_path + self.path = disk_path self.smart_attributes = {} self.smartctl = {} - self.state = state - self.tests = { - 'NVMe / SMART': { - 'Result': '', 'Started': False, 'Status': '', 'Order': 1}, - 'badblocks': { - 'Result': '', 'Started': False, 'Status': '', 'Order': 2}, - 'I/O Benchmark': { - 'Result': '', - 'Started': False, - 'Status': '', - 'Read Rates': [], - 'Graph Data': [], - 'Order': 3}, - } + self.tests = {} self.get_details() self.get_smart_details() @@ -122,9 +133,9 @@ class DevObj(): self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') # Build list of labels - for dev in [self.lsblk, *self.lsblk.get('children', [])]: - self.labels.append(dev.get('label', '')) - self.labels.append(dev.get('partlabel', '')) + for disk in [self.lsblk, *self.lsblk.get('children', [])]: + self.labels.append(disk.get('label', '')) + self.labels.append(disk.get('partlabel', '')) self.labels = [str(label) for label in self.labels if label] def get_smart_details(self): @@ -158,31 +169,15 @@ class DevObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def update_progress(self): - """Update status strings.""" - for k, v in self.tests.items(): - if self.state.tests[k]['Enabled']: - _status = '' - if not v['Status']: - _status = 'Pending' - if v['Started']: - if v['Result']: - _status = v['Result'] - else: - _status = 'Working' - if _status: - v['Status'] = build_status_string(self.name, _status) class State(): """Object to track device objects and overall state.""" def __init__(self): - self.lscpu = {} - self.devs = [] - self.finished = False + self.cpu = None + self.disks = [] self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False - self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, 'Result': '', 'Sensor Data': get_sensor_data(), @@ -191,32 +186,10 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - self.get_cpu_details() - - def get_cpu_details(self): - """Get CPU details from lscpu.""" - cmd = ['lscpu', '--json'] - try: - result = run_program(cmd, check=False) - json_data = json.loads(result.stdout.decode()) - except Exception as err: - # Ignore and leave self.cpu empty - print_error(err) - pause() - return - for line in json_data.get('lscpu', []): - _field = line.get('field', None).replace(':', '') - _data = line.get('data', None) - if not _field and not _data: - # Skip - print_warning(_field, _data) - pause() - continue - self.lscpu[_field] = _data def init(self): - """Scan for block devices and reset all tests.""" - self.devs = [] + """Set log and add devices.""" + self.disks = [] for k in ['Result', 'Started', 'Status']: self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' @@ -230,27 +203,30 @@ class State(): global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( global_vars['LogDir']) + # Add CPU + self.cpu = CpuObj() + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) - for dev in json_data['blockdevices']: - skip_dev = False - dev_obj = DevObj(self, dev['name']) + for disk in json_data['blockdevices']: + skip_disk = False + disk_obj = DiskObj(disk['name']) - # Skip loopback and optical devices - if dev_obj.lsblk['type'] in ['loop', 'rom']: - skip_dev = True + # Skip loopback devices, optical devices, etc + if disk_obj.lsblk['type'] != 'disk': + skip_disk = True - # Skip WK devices + # Skip WK disks wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) - for label in dev_obj.labels: + for label in disk_obj.labels: if re.search(wk_label_regex, label, re.IGNORECASE): - skip_dev = True + skip_disk = True - # Add device - if not skip_dev: - self.devs.append(dev_obj) + # Add disk + if not skip_disk: + self.disks.append(disk_obj) def update_progress(self): """Update status strings.""" @@ -313,32 +289,32 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) -def check_dev_attributes(dev): - """Check if device should be tested and allow overrides.""" +def check_disk_attributes(disk): + """Check if disk should be tested and allow overrides.""" needs_override = False print_standard(' {size:>6} ({tran}) {model} {serial}'.format( - **dev.lsblk)) + **disk.lsblk)) # General checks - if not dev.nvme_attributes and not dev.smart_attributes: + if not disk.nvme_attributes and not disk.smart_attributes: needs_override = True print_warning( ' WARNING: No NVMe or SMART attributes available for: {}'.format( - dev.path)) + disk.path)) # NVMe checks - # TODO check all tracked attributes and set dev.failing if needed + # TODO check all tracked attributes and set disk.failing if needed # SMART checks - # TODO check all tracked attributes and set dev.failing if needed + # TODO check all tracked attributes and set disk.failing if needed # Ask for override if necessary if needs_override: if ask(' Run tests on this device anyway?'): - # TODO Set override for this dev + # TODO Set override for this disk pass else: - for v in dev.tests.values(): + for v in disk.tests.values(): # Started is set to True to fix the status string v['Result'] = 'Skipped' v['Started'] = True @@ -551,11 +527,11 @@ def run_badblocks_test(state): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test()') - for dev in state.devs: - dev.tests['badblocks']['Started'] = True + for disk in state.disks: + disk.tests['badblocks']['Started'] = True update_progress_pane(state) sleep(3) - dev.tests['badblocks']['Result'] = 'OVERRIDE' + disk.tests['badblocks']['Result'] = 'OVERRIDE' update_progress_pane(state) def run_hw_tests(state): @@ -610,11 +586,11 @@ def run_io_benchmark(state): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark()') - for dev in state.devs: - dev.tests['I/O Benchmark']['Started'] = True + for disk in state.disks: + disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) sleep(3) - dev.tests['I/O Benchmark']['Result'] = 'Unknown' + disk.tests['I/O Benchmark']['Result'] = 'Unknown' update_progress_pane(state) def run_keyboard_test(): @@ -741,42 +717,42 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart(state): +def run_nvme_smart_tests(state): """TODO""" - for dev in state.devs: + for disk in state.disks: tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **dev.lsblk)) - dev.tests['NVMe / SMART']['Started'] = True + t=TOP_PANE_TEXT, **disk.lsblk)) + disk.tests['NVMe / SMART']['Started'] = True update_progress_pane(state) - if dev.nvme_attributes: - run_nvme_tests(state, dev) - elif dev.smart_attributes: - run_smart_tests(state, dev) + if disk.nvme_attributes: + run_nvme_tests(state, disk) + elif disk.smart_attributes: + run_smart_tests(state, disk) else: - print_standard('TODO: run_nvme_smart({})'.format( - dev.path)) + print_standard('TODO: run_nvme_smart_tests({})'.format( + disk.path)) print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( - dev.path)) - dev.tests['NVMe / SMART']['Status'] = 'N/A' - dev.tests['NVMe / SMART']['Result'] = 'N/A' + disk.path)) + disk.tests['NVMe / SMART']['Status'] = 'N/A' + disk.tests['NVMe / SMART']['Result'] = 'N/A' update_progress_pane(state) sleep(3) -def run_nvme_tests(state, dev): +def run_nvme_tests(state, disk): """TODO""" - print_standard('TODO: run_nvme_test({})'.format(dev.path)) + print_standard('TODO: run_nvme_test({})'.format(disk.path)) sleep(3) - dev.tests['NVMe / SMART']['Result'] = 'CS' + disk.tests['NVMe / SMART']['Result'] = 'CS' update_progress_pane(state) -def run_smart_tests(state, dev): +def run_smart_tests(state, disk): """TODO""" - print_standard('TODO: run_smart_tests({})'.format(dev.path)) + print_standard('TODO: run_smart_tests({})'.format(disk.path)) sleep(3) - dev.tests['NVMe / SMART']['Result'] = 'CS' + disk.tests['NVMe / SMART']['Result'] = 'CS' update_progress_pane(state) def secret_screensaver(screensaver=None): @@ -872,8 +848,8 @@ def update_progress_pane(state): if 'Prime95' not in k and v['Enabled']: output.append('{BLUE}{test_name}{CLEAR}'.format( test_name=k, **COLORS)) - for dev in state.devs: - output.append(dev.tests[k]['Status']) + for disk in state.disks: + output.append(disk.tests[k]['Status']) output.append(' ') # Add line-endings From 0390290f10d44ec9ab8b41d3bfe374b61744795b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:46:17 -0700 Subject: [PATCH 38/86] Added TestObj() * This object will track test specific vars and results * Moved status code into TestObj * Test calls will now be: run_test(state, dev, test_obj) * NOTE: Code is not done and is quite broken --- .bin/Scripts/functions/hw_diags.py | 120 ++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a2c8293d..861305df 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -64,6 +64,12 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 +TESTS_CPU = ['Prime95 & Temps'] +TESTS_DISK = [ + 'I/O Benchmark', + 'NVMe / SMART', + 'badblocks', + ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes @@ -169,6 +175,10 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + def safety_check(self): + """Check enabled tests and verify it's safe to run them.""" + # TODO + pass class State(): """Object to track device objects and overall state.""" @@ -179,13 +189,31 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = { - 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': '', 'Sensor Data': get_sensor_data(), - 'Started': False, 'Status': ''}, - 'NVMe / SMART': {'Enabled': False, 'Order': 2}, - 'badblocks': {'Enabled': False, 'Order': 3}, - 'I/O Benchmark': {'Enabled': False, 'Order': 4}, - } + 'Prime95 & Temps': { + 'Enabled': False, + 'Function': run_mprime_test, + 'Objects': [], + 'Order': 1, + }, + 'NVMe / SMART': { + 'Enabled': False, + 'Function': run_nvme_smart_tests, + 'Objects': [], + 'Order': 2, + }, + 'badblocks': { + 'Enabled': False, + 'Function': run_badblocks_test, + 'Objects': [], + 'Order': 3, + }, + 'I/O Benchmark': { + 'Enabled': False, + 'Function': run_io_benchmark, + 'Objects': [], + 'Order': 4, + }, + } def init(self): """Set log and add devices.""" @@ -228,26 +256,32 @@ class State(): if not skip_disk: self.disks.append(disk_obj) - def update_progress(self): - """Update status strings.""" - # Prime95 - p = self.tests['Prime95 & Temps'] - if p['Enabled']: - _status = '' - if not p['Status']: - _status = 'Pending' - if p['Started']: - if p['Result']: - _status = p['Result'] - else: - _status = 'Working' - if _status: - p['Status'] = build_status_string( - 'Prime95', _status, info_label=True) +class TestObj(): + """Object to track test data.""" + def __init__(self, label, info_label=False): + self.started = False + self.passed = False + self.failed = False + self.report = '' + self.status = '' + self.label = label + self.info_label = info_label + self.disabled = False + self.update_status() - # Disks - for dev in self.devs: - dev.update_progress() + def update_status(self, new_status=None): + """Update status strings.""" + if self.disabled: + return + if new_status: + self.status = build_status_string( + self.label, new_status, self.info_label) + elif not self.status: + self.status = build_status_string( + self.label, 'Pending', self.info_label) + elif self.started and 'Pending' in self.status: + self.status = build_status_string( + self.label, 'Working', self.info_label) # Functions def build_outer_panes(state): @@ -543,7 +577,7 @@ def run_hw_tests(state): update_progress_pane(state) build_outer_panes(state) - # Run test(s) + # Show selected tests and create TestObj()s print_info('Selected Tests:') for k, v in sorted( state.tests.items(), @@ -554,21 +588,36 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) + if v['Enabled']: + # Create TestObj and track under both CpuObj/DiskObj and State + if k in TESTS_CPU: + test_obj = TestObj(info_label=True) + state.cpu.tests[k] = test_obj + v['Objects'].append(test_obj) + elif k in TESTS_DISK: + for disk in state.disks: + test_obj = TestObj() + disk.tests[k] = test_obj + v['Objects'].append(test_obj) print_standard('') - # Check devices if necessary - if (state.tests['badblocks']['Enabled'] - or state.tests['I/O Benchmark']['Enabled']): - print_info('Selected Disks:') - for dev in state.devs: - check_dev_attributes(dev) - print_standard('') + # Run safety checks + for disk in state.disks: + disk.safety_check() + + # Run tests + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + if v['Enabled']: + # TODO + pass # Run tests if state.tests['Prime95 & Temps']['Enabled']: run_mprime_test(state) if state.tests['NVMe / SMART']['Enabled']: - run_nvme_smart(state) + run_nvme_smart_tests(state) if state.tests['badblocks']['Enabled']: run_badblocks_test(state) if state.tests['I/O Benchmark']['Enabled']: @@ -835,7 +884,6 @@ def update_io_progress(percent, rate, progress_file): def update_progress_pane(state): """Update progress file for side pane.""" output = [] - state.update_progress() # Prime95 output.append(state.tests['Prime95 & Temps']['Status']) From 49471663f50ac3fe239a0417063b8ee814422976 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:50:11 -0700 Subject: [PATCH 39/86] Use OrderedDicts to avoid lambda sorting --- .bin/Scripts/functions/hw_diags.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 861305df..05c7ebb7 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,6 +4,7 @@ import json import re import time +from collections import OrderedDict from functions.sensors import * from functions.tmux import * @@ -109,7 +110,7 @@ class DiskObj(): self.path = disk_path self.smart_attributes = {} self.smartctl = {} - self.tests = {} + self.tests = OrderedDict() self.get_details() self.get_smart_details() @@ -188,32 +189,28 @@ class State(): self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False - self.tests = { + self.tests = OrderedDict({ 'Prime95 & Temps': { 'Enabled': False, 'Function': run_mprime_test, 'Objects': [], - 'Order': 1, }, 'NVMe / SMART': { 'Enabled': False, 'Function': run_nvme_smart_tests, 'Objects': [], - 'Order': 2, }, 'badblocks': { 'Enabled': False, 'Function': run_badblocks_test, 'Objects': [], - 'Order': 3, }, 'I/O Benchmark': { 'Enabled': False, 'Function': run_io_benchmark, 'Objects': [], - 'Order': 4, }, - } + }) def init(self): """Set log and add devices.""" @@ -579,9 +576,7 @@ def run_hw_tests(state): # Show selected tests and create TestObj()s print_info('Selected Tests:') - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): print_standard(' {:<15} {}{}{} {}'.format( k, COLORS['GREEN'] if v['Enabled'] else COLORS['RED'], @@ -606,9 +601,7 @@ def run_hw_tests(state): disk.safety_check() # Run tests - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): if v['Enabled']: # TODO pass @@ -890,9 +883,7 @@ def update_progress_pane(state): output.append(' ') # Disks - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): if 'Prime95' not in k and v['Enabled']: output.append('{BLUE}{test_name}{CLEAR}'.format( test_name=k, **COLORS)) From 941a5537669a0c19f9c37f5f139211e1154b5c91 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:16:31 -0700 Subject: [PATCH 40/86] Renamed "Prime95 & Temps" to "Prime95" for brevity --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 05c7ebb7..fd8b0ebb 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -65,7 +65,7 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 -TESTS_CPU = ['Prime95 & Temps'] +TESTS_CPU = ['Prime95'] TESTS_DISK = [ 'I/O Benchmark', 'NVMe / SMART', @@ -190,7 +190,7 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = OrderedDict({ - 'Prime95 & Temps': { + 'Prime95': { 'Enabled': False, 'Function': run_mprime_test, 'Objects': [], @@ -216,7 +216,7 @@ class State(): """Set log and add devices.""" self.disks = [] for k in ['Result', 'Started', 'Status']: - self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + self.tests['Prime95'][k] = False if k == 'Started' else '' # Update LogDir if not self.quick_mode: @@ -443,7 +443,7 @@ def menu_diags(state, args): {'Base Name': 'Full Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, - {'Base Name': 'Prime95 & Temps', 'Enabled': False, 'CRLF': True}, + {'Base Name': 'Prime95', 'Enabled': False, 'CRLF': True}, {'Base Name': 'NVMe / SMART', 'Enabled': False}, {'Base Name': 'badblocks', 'Enabled': False}, {'Base Name': 'I/O Benchmark', 'Enabled': False}, @@ -642,13 +642,13 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" - state.tests['Prime95 & Temps']['Started'] = True + state.tests['Prime95']['Started'] = True update_progress_pane(state) - _sensor_data = state.tests['Prime95 & Temps']['Sensor Data'] + _sensor_data = state.tests['Prime95']['Sensor Data'] # Update top pane _title = '{}\n{}{}{}'.format( - TOP_PANE_TEXT, 'Prime95 & Temps', + TOP_PANE_TEXT, 'Prime95', ': ' if 'Model name' in state.lscpu else '', state.lscpu.get('Model name', '')) tmux_update_pane(state.panes['Top'], text=_title) @@ -706,7 +706,7 @@ def run_mprime_test(state): sleep(1) except KeyboardInterrupt: # Catch CTRL+C - state.tests['Prime95 & Temps']['Result'] = 'Aborted' + state.tests['Prime95']['Result'] = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) @@ -743,7 +743,7 @@ def run_mprime_test(state): # TODO # Done - state.tests['Prime95 & Temps']['Result'] = 'Unknown' + state.tests['Prime95']['Result'] = 'Unknown' update_progress_pane(state) # Cleanup @@ -879,7 +879,7 @@ def update_progress_pane(state): output = [] # Prime95 - output.append(state.tests['Prime95 & Temps']['Status']) + output.append(state.tests['Prime95']['Status']) output.append(' ') # Disks From 668c7c4c6a440fdcee3fa281d0590d728b634545 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:32:03 -0700 Subject: [PATCH 41/86] Updated run_mprime_test to use test_obj --- .bin/Scripts/functions/hw_diags.py | 95 +++++++++++++++--------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index fd8b0ebb..cd64e2e8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -80,6 +80,7 @@ class CpuObj(): self.lscpu = {} self.tests = {} self.get_details() + self.name = self.lscpu.get('Model name', 'Unknown CPU') def get_details(self): """Get CPU details from lscpu.""" @@ -255,15 +256,16 @@ class State(): class TestObj(): """Object to track test data.""" - def __init__(self, label, info_label=False): - self.started = False - self.passed = False - self.failed = False - self.report = '' - self.status = '' + def __init__(self, dev, label=None, info_label=False): + self.dev = dev self.label = label self.info_label = info_label self.disabled = False + self.failed = False + self.passed = False + self.report = '' + self.started = False + self.status = '' self.update_status() def update_status(self, new_status=None): @@ -552,12 +554,13 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_badblocks_test(state): +def run_badblocks_test(state, test_obj): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) - print_standard('TODO: run_badblocks_test()') + print_standard('TODO: run_badblocks_test({})'.format( + test_obj.dev.path)) for disk in state.disks: disk.tests['badblocks']['Started'] = True update_progress_pane(state) @@ -586,12 +589,13 @@ def run_hw_tests(state): if v['Enabled']: # Create TestObj and track under both CpuObj/DiskObj and State if k in TESTS_CPU: - test_obj = TestObj(info_label=True) + test_obj = TestObj( + dev=state.cpu, label='Prime95', info_label=True) state.cpu.tests[k] = test_obj v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj() + test_obj = TestObj(dev=k) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -601,20 +605,13 @@ def run_hw_tests(state): disk.safety_check() # Run tests + ## Because state.tests is an OrderedDict and the disks were added + ## in order, the tests will be run in order. for k, v in state.tests.items(): if v['Enabled']: - # TODO - pass - - # Run tests - if state.tests['Prime95 & Temps']['Enabled']: - run_mprime_test(state) - if state.tests['NVMe / SMART']['Enabled']: - run_nvme_smart_tests(state) - if state.tests['badblocks']['Enabled']: - run_badblocks_test(state) - if state.tests['I/O Benchmark']['Enabled']: - run_io_benchmark(state) + f = v['Function'] + for test_obj in v['Objects']: + f(state, test_obj) # Done pause('Press Enter to return to main menu... ') @@ -622,12 +619,13 @@ def run_hw_tests(state): # Cleanup tmux_kill_pane(*state.panes.values()) -def run_io_benchmark(state): +def run_io_benchmark(state, test_obj): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) - print_standard('TODO: run_io_benchmark()') + print_standard('TODO: run_io_benchmark({})'.format( + test_obj.dev.path)) for disk in state.disks: disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) @@ -640,34 +638,32 @@ def run_keyboard_test(): clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) -def run_mprime_test(state): +def run_mprime_test(state, test_obj): """Test CPU with Prime95 and track temps.""" state.tests['Prime95']['Started'] = True update_progress_pane(state) - _sensor_data = state.tests['Prime95']['Sensor Data'] + test_obj.sensor_data = get_sensor_data() # Update top pane - _title = '{}\n{}{}{}'.format( - TOP_PANE_TEXT, 'Prime95', - ': ' if 'Model name' in state.lscpu else '', - state.lscpu.get('Model name', '')) - tmux_update_pane(state.panes['Top'], text=_title) + test_obj.title = '{}\nPrime95: {}'.format( + TOP_PANE_TEXT, test_obj.dev.name) + tmux_update_pane(state.panes['Top'], text=test_obj.title) # Start live sensor monitor - _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(_sensors_out, 'w') as f: + test_obj.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(test_obj.sensors_out, 'w') as f: f.write(' ') f.flush() sleep(0.5) - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], + test_obj.monitor_proc = popen_program( + ['hw-sensors-monitor', test_obj.sensors_out], pipe=True) # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( - behind=True, percent=80, vertical=True, watch=_sensors_out) + behind=True, percent=80, vertical=True, watch=test_obj.sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) # Get idle temps @@ -675,11 +671,11 @@ def run_mprime_test(state): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=_sensor_data, temp_label='Idle') + sensor_data=test_obj.sensor_data, temp_label='Idle') # Stress CPU print_log('Starting Prime95') - _abort_msg = 'If running too hot, press CTRL+c to abort the test' + test_obj.abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -701,8 +697,8 @@ def run_mprime_test(state): 's' if sec_left != 1 else '') # Not using print wrappers to avoid flooding the log print(_status_str) - print('{YELLOW}{msg}{CLEAR}'.format(msg=_abort_msg, **COLORS)) - update_sensor_data(_sensor_data) + print('{YELLOW}{msg}{CLEAR}'.format(msg=test_obj.abort_msg, **COLORS)) + update_sensor_data(test_obj.sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C @@ -711,8 +707,8 @@ def run_mprime_test(state): update_progress_pane(state) # Restart live monitor - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], + test_obj.monitor_proc = popen_program( + ['hw-sensors-monitor', test_obj.sensors_out], pipe=True) # Stop Prime95 (twice for good measure) @@ -728,7 +724,7 @@ def run_mprime_test(state): try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=_sensor_data, temp_label='Cooldown') + sensor_data=test_obj.sensor_data, temp_label='Cooldown') # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -741,6 +737,12 @@ def run_mprime_test(state): # Check results # TODO + _log = '{}/results.txt'.format(global_vars['LogDir']) + if os.path.exists(_log): + with open(_log, 'r') as f: + for line in f.readlines(): + if re.search(r'(error|fail)', line, re.IGNORECASE): + state.tests['Prime95']['Result'] = 'NS' # Done state.tests['Prime95']['Result'] = 'Unknown' @@ -748,10 +750,11 @@ def run_mprime_test(state): # Cleanup tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) - monitor_proc.kill() + test_obj.monitor_proc.kill() # TODO Testing - print('\n'.join(generate_report(_sensor_data, 'Idle', 'Max', 'Cooldown'))) + print('\n'.join( + generate_report(test_obj.sensor_data, 'Idle', 'Max', 'Cooldown'))) def run_network_test(): """Run network test.""" @@ -759,7 +762,7 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart_tests(state): +def run_nvme_smart_tests(state, test_obj): """TODO""" for disk in state.disks: tmux_update_pane( From d88a9f39f2a0a85ce4e3fe9dde99a39a64c39046 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:36:24 -0700 Subject: [PATCH 42/86] Added tmux_kill_all_panes() --- .bin/Scripts/functions/tmux.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 84ab96cc..e1066e18 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -2,6 +2,13 @@ from functions.common import * +def tmux_kill_all_panes(pane_id=None): + """Kill all tmux panes except the active pane or pane_id if specified.""" + cmd = ['tmux', 'kill-pane', '-a'] + if pane_id: + cmd.extend(['-t', pane_id]) + run_program(cmd, check=False) + def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] From 465a3b42fb0cc92c03600b820bca0007fd9c3227 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:36:50 -0700 Subject: [PATCH 43/86] Kill all tmux panes before exiting --- .bin/Scripts/hw-diags-menu | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index e60f8fc4..6f0247cd 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -9,20 +9,24 @@ import sys os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.hw_diags import * +from functions.tmux import * init_global_vars() if __name__ == '__main__': - try: - # Show menu - state = State() - menu_diags(state, sys.argv) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + try: + # Show menu + state = State() + menu_diags(state, sys.argv) + # Done + #print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + tmux_kill_all_panes() + pass + except: + tmux_kill_all_panes() + major_exception() + +# vim: sts=2 sw=2 ts=2 From bb93386fa0b7dc9dec3f21529d9279cbbec62bc8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 16:32:00 -0700 Subject: [PATCH 44/86] Updated Prime95 checks --- .bin/Scripts/functions/hw_diags.py | 98 +++++++++++++++++++----------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cd64e2e8..8cebd636 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -257,6 +257,7 @@ class State(): class TestObj(): """Object to track test data.""" def __init__(self, dev, label=None, info_label=False): + self.aborted = False self.dev = dev self.label = label self.info_label = info_label @@ -554,13 +555,13 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_badblocks_test(state, test_obj): +def run_badblocks_test(state, test): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test({})'.format( - test_obj.dev.path)) + test.dev.path)) for disk in state.disks: disk.tests['badblocks']['Started'] = True update_progress_pane(state) @@ -619,13 +620,13 @@ def run_hw_tests(state): # Cleanup tmux_kill_pane(*state.panes.values()) -def run_io_benchmark(state, test_obj): +def run_io_benchmark(state, test): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark({})'.format( - test_obj.dev.path)) + test.dev.path)) for disk in state.disks: disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) @@ -638,32 +639,33 @@ def run_keyboard_test(): clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) -def run_mprime_test(state, test_obj): +def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" - state.tests['Prime95']['Started'] = True + test.started = True + test.update_status() update_progress_pane(state) - test_obj.sensor_data = get_sensor_data() + test.sensor_data = get_sensor_data() # Update top pane - test_obj.title = '{}\nPrime95: {}'.format( - TOP_PANE_TEXT, test_obj.dev.name) - tmux_update_pane(state.panes['Top'], text=test_obj.title) + test.title = '{}\nPrime95: {}'.format( + TOP_PANE_TEXT, test.dev.name) + tmux_update_pane(state.panes['Top'], text=test.title) # Start live sensor monitor - test_obj.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(test_obj.sensors_out, 'w') as f: + test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(test.sensors_out, 'w') as f: f.write(' ') f.flush() sleep(0.5) - test_obj.monitor_proc = popen_program( - ['hw-sensors-monitor', test_obj.sensors_out], + test.monitor_proc = popen_program( + ['hw-sensors-monitor', test.sensors_out], pipe=True) # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( - behind=True, percent=80, vertical=True, watch=test_obj.sensors_out) + behind=True, percent=80, vertical=True, watch=test.sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) # Get idle temps @@ -671,11 +673,11 @@ def run_mprime_test(state, test_obj): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test_obj.sensor_data, temp_label='Idle') + sensor_data=test.sensor_data, temp_label='Idle') # Stress CPU print_log('Starting Prime95') - test_obj.abort_msg = 'If running too hot, press CTRL+c to abort the test' + test.abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -697,18 +699,19 @@ def run_mprime_test(state, test_obj): 's' if sec_left != 1 else '') # Not using print wrappers to avoid flooding the log print(_status_str) - print('{YELLOW}{msg}{CLEAR}'.format(msg=test_obj.abort_msg, **COLORS)) - update_sensor_data(test_obj.sensor_data) + print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) + update_sensor_data(test.sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C - state.tests['Prime95']['Result'] = 'Aborted' + test.aborted = True + test.status = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) # Restart live monitor - test_obj.monitor_proc = popen_program( - ['hw-sensors-monitor', test_obj.sensors_out], + test.monitor_proc = popen_program( + ['hw-sensors-monitor', test.sensors_out], pipe=True) # Stop Prime95 (twice for good measure) @@ -724,7 +727,7 @@ def run_mprime_test(state, test_obj): try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test_obj.sensor_data, temp_label='Cooldown') + sensor_data=test.sensor_data, temp_label='Cooldown') # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -736,25 +739,52 @@ def run_mprime_test(state, test_obj): global_vars['LogDir'])) # Check results - # TODO - _log = '{}/results.txt'.format(global_vars['LogDir']) - if os.path.exists(_log): - with open(_log, 'r') as f: - for line in f.readlines(): - if re.search(r'(error|fail)', line, re.IGNORECASE): - state.tests['Prime95']['Result'] = 'NS' + test.logs = {} + for log in ['results.txt', 'prime.log']: + _data = '' + log_path = '{}/{}'.format(global_vars['LogDir'], log) + + # Read and save log + try: + with open(log_path, 'r') as f: + _data = f.read() + test.logs[log] = _data.splitlines() + except FileNotFoundError: + # Ignore since files may be missing for slower CPUs + pass + + # results.txt: NS check + if log == 'results.txt': + if re.search(r'(error|fail)', _data, re.IGNORECASE): + test.failed = True + test.status = 'NS' + + # prime.log: CS check + if log == 'prime.log': + if re.search( + r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): + test.passed = True + test.status = 'CS' + elif re.search( + r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): + # If the first re.search does not match and this one does then + # that means that either errors or warnings, or both, are non-zero + test.failed = True + test.passed = False + test.status = 'NS' + if not (test.aborted or test.failed or test.passed): + test.status = 'Unknown' # Done - state.tests['Prime95']['Result'] = 'Unknown' update_progress_pane(state) # Cleanup tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) - test_obj.monitor_proc.kill() + test.monitor_proc.kill() # TODO Testing print('\n'.join( - generate_report(test_obj.sensor_data, 'Idle', 'Max', 'Cooldown'))) + generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'))) def run_network_test(): """Run network test.""" @@ -762,7 +792,7 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart_tests(state, test_obj): +def run_nvme_smart_tests(state, test): """TODO""" for disk in state.disks: tmux_update_pane( From a00105f71818bce77842be25ce36173c6cb5d000 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 16:57:43 -0700 Subject: [PATCH 45/86] Fixed status updates --- .bin/Scripts/functions/hw_diags.py | 46 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8cebd636..6d6b1a2c 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -214,10 +214,10 @@ class State(): }) def init(self): - """Set log and add devices.""" + """Remove test objects, set log, and add devices.""" self.disks = [] - for k in ['Result', 'Started', 'Status']: - self.tests['Prime95'][k] = False if k == 'Started' else '' + for k, v in self.tests.items(): + v['Objects'] = [] # Update LogDir if not self.quick_mode: @@ -596,7 +596,7 @@ def run_hw_tests(state): v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj(dev=k) + test_obj = TestObj(dev=k, label=disk.name) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -705,7 +705,7 @@ def run_mprime_test(state, test): except KeyboardInterrupt: # Catch CTRL+C test.aborted = True - test.status = 'Aborted' + test.update_status('Aborted') print_warning('\nAborted.') update_progress_pane(state) @@ -757,23 +757,23 @@ def run_mprime_test(state, test): if log == 'results.txt': if re.search(r'(error|fail)', _data, re.IGNORECASE): test.failed = True - test.status = 'NS' + test.update_status('NS') # prime.log: CS check if log == 'prime.log': if re.search( r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): test.passed = True - test.status = 'CS' + test.update_status('CS') elif re.search( r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): # If the first re.search does not match and this one does then # that means that either errors or warnings, or both, are non-zero test.failed = True test.passed = False - test.status = 'NS' + test.update_status('NS') if not (test.aborted or test.failed or test.passed): - test.status = 'Unknown' + test.update_status('Unknown') # Done update_progress_pane(state) @@ -910,19 +910,23 @@ def update_io_progress(percent, rate, progress_file): def update_progress_pane(state): """Update progress file for side pane.""" output = [] - - # Prime95 - output.append(state.tests['Prime95']['Status']) - output.append(' ') - - # Disks for k, v in state.tests.items(): - if 'Prime95' not in k and v['Enabled']: - output.append('{BLUE}{test_name}{CLEAR}'.format( - test_name=k, **COLORS)) - for disk in state.disks: - output.append(disk.tests[k]['Status']) - output.append(' ') + # Skip disabled sections + if not v['Enabled']: + continue + + # Add section name + if k != 'Prime95': + output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) + if 'SMART' in k and state.quick_mode: + output.append(' {YELLOW}(Quick Check){CLEAR}'.format(**COLORS)) + + # Add status from test object(s) + for test in v['Objects']: + output.append(test.status) + + # Add spacer before next section + output.append(' ') # Add line-endings output = ['{}\n'.format(line) for line in output] From 8a8a63eb66c412b428ddb0bb9ccd1ae8241dbd49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:16:43 -0700 Subject: [PATCH 46/86] Build Prime95 report --- .bin/Scripts/functions/hw_diags.py | 83 +++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6d6b1a2c..181c3a4d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -264,7 +264,7 @@ class TestObj(): self.disabled = False self.failed = False self.passed = False - self.report = '' + self.report = [] self.started = False self.status = '' self.update_status() @@ -615,6 +615,7 @@ def run_hw_tests(state): f(state, test_obj) # Done + show_results(state) pause('Press Enter to return to main menu... ') # Cleanup @@ -738,42 +739,67 @@ def run_mprime_test(state, test): item.path, global_vars['LogDir'])) - # Check results + # Check results and build report test.logs = {} for log in ['results.txt', 'prime.log']: - _data = '' + lines = [] log_path = '{}/{}'.format(global_vars['LogDir'], log) # Read and save log try: with open(log_path, 'r') as f: - _data = f.read() - test.logs[log] = _data.splitlines() + lines = f.read().splitlines() + test.logs[log] = lines except FileNotFoundError: # Ignore since files may be missing for slower CPUs pass - # results.txt: NS check + # results.txt (NS check) if log == 'results.txt': - if re.search(r'(error|fail)', _data, re.IGNORECASE): - test.failed = True - test.update_status('NS') + _tmp = [] + for line in lines: + if re.search(r'(error|fail)', line, re.IGNORECASE): + test.failed = True + test.update_status('NS') + _tmp.append(' {YELLOW}{line}{CLEAR}'.format(**COLORS)) + if _tmp: + test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) + test.report.extend(_tmp) - # prime.log: CS check + # prime.log (CS check) if log == 'prime.log': - if re.search( - r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): - test.passed = True - test.update_status('CS') - elif re.search( - r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): - # If the first re.search does not match and this one does then - # that means that either errors or warnings, or both, are non-zero - test.failed = True - test.passed = False - test.update_status('NS') + _tmp_pass = [] + _tmp_warn = [] + for line in lines: + if re.search( + r'completed.*0 errors, 0 warnings', line, re.IGNORECASE): + _tmp_pass.append(line) + elif re.search( + r'completed.*\d+ errors, \d+ warnings', line, re.IGNORECASE): + # If the first re.search does not match and this one does then + # that means that either errors or warnings, or both, are non-zero + _tmp_warn.append(line) + if len(_tmp_warn) > 0: + test.failed = True + test.passed = False + test.update_status('NS') + elif len(_tmp_pass) > 0: + test.passed = True + test.update_status('CS') + if len(_tmp_pass) + len(_tmp_warn) > 0: + test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) + for line in _tmp_pass: + test.report.append(' {}'.format(line)) + for line in _tmp_warn: + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line, **COLORS)) + test.report.append(' ') + + # Finalize report if not (test.aborted or test.failed or test.passed): test.update_status('Unknown') + test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) + for line in generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'): + test.report.append(' {}'.format(line)) # Done update_progress_pane(state) @@ -782,10 +808,6 @@ def run_mprime_test(state, test): tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) test.monitor_proc.kill() - # TODO Testing - print('\n'.join( - generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'))) - def run_network_test(): """Run network test.""" clear_screen() @@ -840,6 +862,17 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_results(state): + """Show results for all tests.""" + for k, v in state.tests.items(): + print_success('{}:'.format(k)) + for obj in v['Objects']: + for line in obj.report: + print(line) + print_log(strip_colors(line)) + print_standard(' ') + print_standard(' ') + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 30d4acd9861672340443ecbbfc5614d434245a01 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:18:16 -0700 Subject: [PATCH 47/86] Added watch mode to respawn-pane --- .bin/Scripts/functions/tmux.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index e1066e18..e1892417 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -81,11 +81,14 @@ def tmux_split_window( result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): +def tmux_update_pane( + pane_id, command=None, + text=None, watch=None, + working_dir=None): """Respawn with either a new command or new text.""" # Bail early - if not command and not text: - raise Exception('Neither command nor text specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] if working_dir: @@ -94,6 +97,11 @@ def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): cmd.extend(command) elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) run_program(cmd) From 2b43cdf9e27d861406b20e6c35512471450c5ae2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:19:11 -0700 Subject: [PATCH 48/86] Create watch file if it doesn't exist yet --- .bin/Scripts/functions/tmux.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index e1892417..69b906d1 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -2,6 +2,12 @@ from functions.common import * +def create_file(filepath): + """Create file if it doesn't exist.""" + if not os.path.exists(filepath): + with open(filepath, 'w') as f: + f.write('') + def tmux_kill_all_panes(pane_id=None): """Kill all tmux panes except the active pane or pane_id if specified.""" cmd = ['tmux', 'kill-pane', '-a'] @@ -72,6 +78,7 @@ def tmux_split_window( elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: + create_file(watch) cmd.extend([ 'watch', '--color', '--no-title', '--interval', '1', @@ -98,6 +105,7 @@ def tmux_update_pane( elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: + create_file(watch) cmd.extend([ 'watch', '--color', '--no-title', '--interval', '1', From a2ef06e6db0004bb9cb23af0a351cffd33326e2d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:19:35 -0700 Subject: [PATCH 49/86] Added strip_colors() function --- .bin/Scripts/functions/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 2bf52a85..d87f772e 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -515,6 +515,12 @@ def stay_awake(): print_error('ERROR: No caffeine available.') print_warning('Please set the power setting to High Performance.') +def strip_colors(s): + """Remove all ASCII color escapes from string, returns str.""" + for c in COLORS.values(): + s = s.replace(c, '') + return s + def get_exception(s): """Get exception by name, returns Exception object.""" try: From d9554314d55ef845670887e3b019b39de449c343 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:42:10 -0700 Subject: [PATCH 50/86] Updated run_program() and popen_program() * Use dicts for clarity * Support cwd flag --- .bin/Scripts/functions/common.py | 36 ++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index d87f772e..7f14bdbd 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -405,19 +405,24 @@ def ping(addr='google.com'): def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): """Run program and return a subprocess.Popen object.""" - startupinfo=None + cmd_kwargs = {'args': cmd, 'shell': shell} + if minimized: startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = 6 + cmd_kwargs['startupinfo'] = startupinfo if pipe: - popen_obj = subprocess.Popen(cmd, shell=shell, startupinfo=startupinfo, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - popen_obj = subprocess.Popen(cmd, shell=shell, startupinfo=startupinfo) + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) - return popen_obj + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] + + return subprocess.Popen(**cmd_kwargs) def print_error(*args, **kwargs): """Prints message to screen in RED.""" @@ -456,7 +461,7 @@ def print_log(message='', end='\n', timestamp=True): line = line, end = end)) -def run_program(cmd, args=[], check=True, pipe=True, shell=False): +def run_program(cmd, args=[], check=True, pipe=True, shell=False, **kwargs): """Run program and return a subprocess.CompletedProcess object.""" if args: # Deprecated so let's raise an exception to find & fix all occurances @@ -466,13 +471,18 @@ def run_program(cmd, args=[], check=True, pipe=True, shell=False): if shell: cmd = ' '.join(cmd) - if pipe: - process_return = subprocess.run(cmd, check=check, shell=shell, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - process_return = subprocess.run(cmd, check=check, shell=shell) + cmd_kwargs = {'args': cmd, 'check': check, 'shell': shell} - return process_return + if pipe: + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) + + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] + + return subprocess.run(**cmd_kwargs) def set_title(title='~Some Title~'): """Set title. From 6c06a67fdf0dafb0b792fdbf48d0608e08a73651 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 22:54:56 -0700 Subject: [PATCH 51/86] Prime95 section complete --- .bin/Scripts/functions/hw_diags.py | 73 ++++++++++++++++++------------ .bin/Scripts/functions/sensors.py | 10 +++- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 181c3a4d..9cdde459 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -674,7 +674,9 @@ def run_mprime_test(state, test): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test.sensor_data, temp_label='Idle') + sensor_data=test.sensor_data, temp_label='Idle', + seconds=3) + # TODO: Remove seconds kwarg above # Stress CPU print_log('Starting Prime95') @@ -684,7 +686,9 @@ def run_mprime_test(state, test): state.panes['mprime'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) - time_limit = int(MPRIME_LIMIT) * 60 + #time_limit = int(MPRIME_LIMIT) * 60 + # TODO: restore above line + time_limit = 10 try: for i in range(time_limit): clear_screen() @@ -716,19 +720,23 @@ def run_mprime_test(state, test): pipe=True) # Stop Prime95 (twice for good measure) - tmux_kill_pane(state.panes['mprime']) run_program(['killall', '-s', 'INT', 'mprime'], check=False) + sleep(1) + tmux_kill_pane(state.panes['mprime']) # Get cooldown temp run_program(['apple-fans', 'auto']) clear_screen() try_and_print( message='Letting CPU cooldown for bit...', indent=0, - function=sleep, cs='Done', seconds=10) + function=sleep, cs='Done', seconds=3) + # TODO: Above seconds should be 10 try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test.sensor_data, temp_label='Cooldown') + sensor_data=test.sensor_data, temp_label='Cooldown', + seconds=3) + # TODO: Remove seconds kwarg above # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -761,44 +769,47 @@ def run_mprime_test(state, test): if re.search(r'(error|fail)', line, re.IGNORECASE): test.failed = True test.update_status('NS') - _tmp.append(' {YELLOW}{line}{CLEAR}'.format(**COLORS)) + _tmp.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) if _tmp: test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) test.report.extend(_tmp) # prime.log (CS check) if log == 'prime.log': - _tmp_pass = [] - _tmp_warn = [] + _tmp = {'Pass': {}, 'Warn': {}} for line in lines: - if re.search( - r'completed.*0 errors, 0 warnings', line, re.IGNORECASE): - _tmp_pass.append(line) - elif re.search( - r'completed.*\d+ errors, \d+ warnings', line, re.IGNORECASE): - # If the first re.search does not match and this one does then - # that means that either errors or warnings, or both, are non-zero - _tmp_warn.append(line) - if len(_tmp_warn) > 0: - test.failed = True - test.passed = False - test.update_status('NS') - elif len(_tmp_pass) > 0: - test.passed = True - test.update_status('CS') - if len(_tmp_pass) + len(_tmp_warn) > 0: + _r = re.search( + r'(completed.*(\d+) errors, (\d+) warnings)', + line, + re.IGNORECASE) + if _r: + if int(_r.group(2)) + int(_r.group(3)) > 0: + # Encountered errors and/or warnings + _tmp['Warn'][_r.group(1)] = None + else: + # No errors + _tmp['Pass'][_r.group(1)] = None + if len(_tmp['Warn']) > 0: + # NS + test.failed = True + test.passed = False + test.update_status('NS') + elif len(_tmp['Pass']) > 0: + test.passed = True + test.update_status('CS') + if len(_tmp['Pass']) + len(_tmp['Warn']) > 0: test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) - for line in _tmp_pass: + for line in sorted(_tmp['Pass'].keys()): test.report.append(' {}'.format(line)) - for line in _tmp_warn: - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line, **COLORS)) - test.report.append(' ') + for line in sorted(_tmp['Warn'].keys()): + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # Finalize report if not (test.aborted or test.failed or test.passed): test.update_status('Unknown') test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) - for line in generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'): + for line in generate_report( + test.sensor_data, 'Idle', 'Max', 'Cooldown', core_only=True): test.report.append(' {}'.format(line)) # Done @@ -864,6 +875,7 @@ def secret_screensaver(screensaver=None): def show_results(state): """Show results for all tests.""" + clear_screen() for k, v in state.tests.items(): print_success('{}:'.format(k)) for obj in v['Objects']: @@ -871,7 +883,8 @@ def show_results(state): print(line) print_log(strip_colors(line)) print_standard(' ') - print_standard(' ') + if 'Prime95' not in k: + print_standard(' ') def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 066dc446..b6319744 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -29,17 +29,22 @@ def fix_sensor_str(s): s = s.title() s = s.replace('Coretemp', 'CoreTemp') s = s.replace('Acpi', 'ACPI') + s = s.replace('ACPItz', 'ACPI TZ') s = s.replace('Isa ', 'ISA ') s = s.replace('Id ', 'ID ') s = re.sub(r'(\D+)(\d+)', r'\1 \2', s, re.IGNORECASE) s = s.replace(' ', ' ') return s -def generate_report(sensor_data, *temp_labels, colors=True): +def generate_report( + sensor_data, *temp_labels, + colors=True, core_only=False): """Generate report based on temp_labels, returns list if str.""" report = [] for _section, _adapters in sorted(sensor_data.items()): # CoreTemps then Other temps + if core_only and 'Core' not in _section: + continue for _adapter, _sources in sorted(_adapters.items()): # Adapter report.append(fix_sensor_str(_adapter)) @@ -53,7 +58,8 @@ def generate_report(sensor_data, *temp_labels, colors=True): ': ' if _label != 'Current' else '', get_temp_str(_data.get(_label, '???'), colors=colors)) report.append(_line) - report.append(' ') + if not core_only: + report.append(' ') # Handle empty reports (i.e. no sensors detected) if not report: From a3f7e5ad89372c922917662626a5d1421686cba7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 00:54:16 -0700 Subject: [PATCH 52/86] Disk quick check almost done --- .bin/Scripts/functions/hw_diags.py | 193 +++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 51 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9cdde459..da4d24e7 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -11,23 +11,23 @@ from functions.tmux import * # STATIC VARIABLES ATTRIBUTES = { 'NVMe': { - 'critical_warning': {'Error': 1}, - 'media_errors': {'Error': 1}, + 'critical_warning': {'Error': 1, 'Critical': True}, + 'media_errors': {'Error': 1, 'Critical': True}, 'power_on_hours': {'Warning': 12000, 'Error': 26298, 'Ignore': True}, 'unsafe_shutdowns': {'Warning': 1}, }, 'SMART': { - 5: {'Hex': '05', 'Error': 1}, - 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, - 10: {'Hex': '0A', 'Error': 1}, - 184: {'Hex': 'B8', 'Error': 1}, - 187: {'Hex': 'BB', 'Error': 1}, - 188: {'Hex': 'BC', 'Error': 1}, - 196: {'Hex': 'C4', 'Error': 1}, - 197: {'Hex': 'C5', 'Error': 1}, - 198: {'Hex': 'C6', 'Error': 1}, - 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - 201: {'Hex': 'C9', 'Error': 1}, + '5': {'Hex': '05', 'Error': 1, 'Critical': True}, + '9': {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + '10': {'Hex': '0A', 'Error': 1}, + '184': {'Hex': 'B8', 'Error': 1}, + '187': {'Hex': 'BB', 'Error': 1}, + '188': {'Hex': 'BC', 'Error': 1}, + '196': {'Hex': 'C4', 'Error': 1}, + '197': {'Hex': 'C5', 'Error': 1, 'Critical': True}, + '198': {'Hex': 'C6', 'Error': 1, 'Critical': True}, + '199': {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + '201': {'Hex': 'C9', 'Error': 1}, }, } IO_VARS = { @@ -284,6 +284,82 @@ class TestObj(): self.label, 'Working', self.info_label) # Functions +def attributes_ok_nvme(disk): + """Check NVMe attributes for errors, returns bool.""" + disk_ok = True + override_disabled = False + for k, v in disk.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + if 'Error' not in ATTRIBUTES['NVMe'][k]: + # Only worried about error thresholds + continue + if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: + disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['NVMe'][k].get( + 'Critical', False) + + # Print errors + if not disk_ok: + show_disk_attributes(disk) + if override_disabled: + print_error('NVMe error(s) detected.') + print_standard('Tests disabled for this device') + pause() + else: + print_warning('NVMe error(s) detected.') + disk_ok = ask('Run tests on this device anyway?') + + return disk_ok + +def attributes_ok_smart(disk): + """Check SMART attributes for errors, returns bool.""" + disk_ok = True + override_disabled = False + smart_overall_pass = True + for k, v in disk.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + if 'Error' not in ATTRIBUTES['SMART'][k]: + # Only worried about error thresholds + continue + if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: + disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['SMART'][k].get( + 'Critical', False) + + # SMART overall assessment + if not disk.smartctl.get('smart_status', {}).get('passed', False): + smart_overall_pass = False + disk_ok = False + override_disabled = True + + # Print errors + if not disk_ok: + show_disk_attributes(disk) + + # 199/C7 warning + if disk.smart_attributes.get('199', {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the drive cable?)') + + # Override? + if not smart_overall_pass: + print_error('SMART overall self-assessment: Failed') + print_standard('Tests disabled for this device') + pause() + elif override_disabled: + print_error('SMART error(s) detected.') + print_standard('Tests disabled for this device') + pause() + else: + print_warning('SMART error(s) detected.') + disk_ok = ask('Run tests on this device anyway?') + + return disk_ok + def build_outer_panes(state): """Build top and side panes.""" clear_screen() @@ -310,7 +386,7 @@ def build_status_string(label, status, info_label=False): status_color = COLORS['CLEAR'] if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: status_color = COLORS['RED'] - elif status in ['Aborted', 'Unknown', 'Working', 'Skipped']: + elif status in ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working']: status_color = COLORS['YELLOW'] elif status in ['CS']: status_color = COLORS['GREEN'] @@ -596,7 +672,7 @@ def run_hw_tests(state): v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj(dev=k, label=disk.name) + test_obj = TestObj(dev=disk, label=disk.name) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -616,7 +692,10 @@ def run_hw_tests(state): # Done show_results(state) - pause('Press Enter to return to main menu... ') + if '--quick' in sys.argv: + pause('Press Enter to exit...') + else: + pause('Press Enter to return to main menu... ') # Cleanup tmux_kill_pane(*state.panes.values()) @@ -826,42 +905,32 @@ def run_network_test(): pause('Press Enter to return to main menu... ') def run_nvme_smart_tests(state, test): - """TODO""" - for disk in state.disks: - tmux_update_pane( - state.panes['Top'], - text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **disk.lsblk)) - disk.tests['NVMe / SMART']['Started'] = True - update_progress_pane(state) - if disk.nvme_attributes: - run_nvme_tests(state, disk) - elif disk.smart_attributes: - run_smart_tests(state, disk) + """Run NVMe or SMART test for test.dev.""" + tmux_update_pane( + state.panes['Top'], + text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( + t=TOP_PANE_TEXT, **test.dev.lsblk)) + if test.dev.nvme_attributes: + if attributes_ok_nvme(test.dev): + test.passed = True + test.update_status('CS') else: - print_standard('TODO: run_nvme_smart_tests({})'.format( - disk.path)) - print_warning( - " WARNING: Device {} doesn't support NVMe or SMART test".format( - disk.path)) - disk.tests['NVMe / SMART']['Status'] = 'N/A' - disk.tests['NVMe / SMART']['Result'] = 'N/A' - update_progress_pane(state) - sleep(3) - -def run_nvme_tests(state, disk): - """TODO""" - print_standard('TODO: run_nvme_test({})'.format(disk.path)) + test.failed = True + test.update_status('NS') + elif test.dev.smart_attributes: + if attributes_ok_smart(test.dev): + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') + else: + print_standard('Tests disabled for this device') + test.update_status('N/A') + if not ask('Run tests on this device anyway?'): + test.failed = True + update_progress_pane(state) sleep(3) - disk.tests['NVMe / SMART']['Result'] = 'CS' - update_progress_pane(state) - -def run_smart_tests(state, disk): - """TODO""" - print_standard('TODO: run_smart_tests({})'.format(disk.path)) - sleep(3) - disk.tests['NVMe / SMART']['Result'] = 'CS' - update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" @@ -873,10 +942,32 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_disk_attributes(disk): + """Show NVMe/SMART attributes for disk.""" + print_info('Device: {}'.format(disk.path)) + print_standard(' {size:6} ({tran}) {model} {serial}'.format(**disk.lsblk)) + print_info('Attributes') + if disk.nvme_attributes: + for k, v in disk.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + print('TODO: {} {}'.format(k, v)) + elif disk.smart_attributes: + for k, v in disk.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + print('TODO: {} {}'.format(k, v)) + else: + print_warning(' No NVMe or SMART data available') + def show_results(state): """Show results for all tests.""" clear_screen() + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'Results')) for k, v in state.tests.items(): + # Skip disabled tests + if not v['Enabled']: + continue print_success('{}:'.format(k)) for obj in v['Objects']: for line in obj.report: @@ -965,7 +1056,7 @@ def update_progress_pane(state): if k != 'Prime95': output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) if 'SMART' in k and state.quick_mode: - output.append(' {YELLOW}(Quick Check){CLEAR}'.format(**COLORS)) + output[-1] += ' {YELLOW}(Quick){CLEAR}'.format(**COLORS) # Add status from test object(s) for test in v['Objects']: From a967a5c425122f7d2bf1998f662e035e55b57865 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 20:40:57 -0700 Subject: [PATCH 53/86] Switched back to int keys for SMART attributes * Allows for easier sorting --- .bin/Scripts/functions/hw_diags.py | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index da4d24e7..2c2d63fa 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -17,17 +17,19 @@ ATTRIBUTES = { 'unsafe_shutdowns': {'Warning': 1}, }, 'SMART': { - '5': {'Hex': '05', 'Error': 1, 'Critical': True}, - '9': {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, - '10': {'Hex': '0A', 'Error': 1}, - '184': {'Hex': 'B8', 'Error': 1}, - '187': {'Hex': 'BB', 'Error': 1}, - '188': {'Hex': 'BC', 'Error': 1}, - '196': {'Hex': 'C4', 'Error': 1}, - '197': {'Hex': 'C5', 'Error': 1, 'Critical': True}, - '198': {'Hex': 'C6', 'Error': 1, 'Critical': True}, - '199': {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - '201': {'Hex': 'C9', 'Error': 1}, + 5: {'Hex': '05', 'Error': 1, 'Critical': True}, + 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 10: {'Hex': '0A', 'Error': 1}, + 184: {'Hex': 'B8', 'Error': 1}, + 187: {'Hex': 'BB', 'Error': 1}, + 188: {'Hex': 'BC', 'Error': 1}, + 196: {'Hex': 'C4', 'Error': 1}, + 197: {'Hex': 'C5', 'Error': 1, 'Critical': True}, + 198: {'Hex': 'C6', 'Error': 1, 'Critical': True}, + 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + 201: {'Hex': 'C9', 'Error': 1}, + # TODO: Delete below + 177: {'Hex': 'FF', 'Error': 1}, }, } IO_VARS = { @@ -161,19 +163,21 @@ class DiskObj(): self.nvme_attributes.update(self.smartctl[KEY_NVME]) elif KEY_SMART in self.smartctl: for a in self.smartctl[KEY_SMART].get('table', {}): - _id = str(a.get('id', 'UNKNOWN')) + try: + _id = int(a.get('id', -1)) + except ValueError: + # Ignoring invalid attribute + continue _name = str(a.get('name', 'UNKNOWN')) - _raw = a.get('raw', {}).get('value', -1) + _raw = int(a.get('raw', {}).get('value', -1)) _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') # Fix power-on time _r = re.match(r'^(\d+)[Hh].*', _raw_str) - if _id == '9' and _r: - try: - _raw = int(_r.group(1)) - except ValueError: - # That's fine - pass + if _id == 9 and _r: + _raw = int(_r.group(1)) + + # Add to dict self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} @@ -1056,7 +1060,7 @@ def update_progress_pane(state): if k != 'Prime95': output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) if 'SMART' in k and state.quick_mode: - output[-1] += ' {YELLOW}(Quick){CLEAR}'.format(**COLORS) + output[-1] += ' {}'.format(QUICK_LABEL) # Add status from test object(s) for test in v['Objects']: From 62a60ff3fd0ff1e24181ebb60d66b9cbfc38f02e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 22:56:09 -0700 Subject: [PATCH 54/86] Reworked disk safety checks * Moved several functions into DiskObj * Added HW_OVERRIDES_FORCED and HW_OVERRIDES_LIMITED to main.py * These adjust when overrides are requested * Disable badblocks and/or io_benchmark if disk fails safety check --- .bin/Scripts/functions/hw_diags.py | 220 ++++++++++++++++------------- .bin/Scripts/settings/main.py | 2 + 2 files changed, 122 insertions(+), 100 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 2c2d63fa..63003e8e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -28,10 +28,9 @@ ATTRIBUTES = { 198: {'Hex': 'C6', 'Error': 1, 'Critical': True}, 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, 201: {'Hex': 'C9', 'Error': 1}, - # TODO: Delete below - 177: {'Hex': 'FF', 'Error': 1}, }, } +HW_OVERRIDES_FORCED = HW_OVERRIDES_FORCED and not HW_OVERRIDES_LIMITED IO_VARS = { 'Block Size': 512*1024, 'Chunk Size': 32*1024**2, @@ -106,6 +105,7 @@ class CpuObj(): class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): + self.disk_ok = True self.labels = [] self.lsblk = {} self.name = re.sub(r'^.*/(.*)', r'\1', disk_path) @@ -181,10 +181,121 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def safety_check(self): - """Check enabled tests and verify it's safe to run them.""" - # TODO - pass + def nvme_check(self, silent=False): + """Check NVMe attributes for errors.""" + override_disabled = False + for k, v in self.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + if 'Error' not in ATTRIBUTES['NVMe'][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES['NVMe'][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['NVMe'][k].get( + 'Critical', False) + + # Print errors + if not self.disk_ok and not silent: + self.show_attributes() + print_warning('NVMe error(s) detected.') + + # Override? + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + + def safety_check(self, silent=False): + """Check attributes and disable tests if necessary.""" + if self.nvme_attributes: + self.nvme_check(silent) + elif self.smart_attributes: + self.smart_check(silent) + else: + # No NVMe/SMART details + if silent: + self.disk_ok = HW_OVERRIDES_FORCED + else: + print_warning( + ' WARNING: No NVMe or SMART attributes available for: {}'.format( + self.path)) + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + + if not self.disk_ok: + for t in ['badblocks', 'I/O Benchmark']: + if t in self.tests: + self.tests[t].disabled = True + self.tests[t].update_status('Denied') + + def show_attributes(self): + """Show NVMe/SMART attributes.""" + print_info('Device: {}'.format(self.path)) + print_standard( + ' {size:>6} ({tran}) {model} {serial}'.format(**self.lsblk)) + print_info('Attributes') + if self.nvme_attributes: + for k, v in self.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + print('TODO: {} {}'.format(k, v)) + elif self.smart_attributes: + for k, v in self.smart_attributes.items(): + # TODO: If k == 199/C7 then append ' (bad cable?)' to line + if k in ATTRIBUTES['SMART']: + print('TODO: {} {}'.format(k, v)) + if not self.smartctl.get('smart_status', {}).get('passed', True): + print_error('SMART overall self-assessment: Failed') + else: + print_warning(' No NVMe or SMART data available') + + def smart_check(self, silent=False): + """Check SMART attributes for errors.""" + override_disabled = False + for k, v in self.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + if 'Error' not in ATTRIBUTES['SMART'][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES['SMART'][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['SMART'][k].get( + 'Critical', False) + + # SMART overall assessment + ## NOTE: Only fail drives if the overall value exists and reports failed + if not self.smartctl.get('smart_status', {}).get('passed', True): + self.disk_ok = False + override_disabled = True + + # Print errors + if not silent: + if self.disk_ok: + # 199/C7 warning + if self.smart_attributes.get(199, {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the disk cable?)') + else: + # Override? + self.show_attributes() + print_warning('SMART error(s) detected.') + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') class State(): """Object to track device objects and overall state.""" @@ -288,82 +399,6 @@ class TestObj(): self.label, 'Working', self.info_label) # Functions -def attributes_ok_nvme(disk): - """Check NVMe attributes for errors, returns bool.""" - disk_ok = True - override_disabled = False - for k, v in disk.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - if 'Error' not in ATTRIBUTES['NVMe'][k]: - # Only worried about error thresholds - continue - if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: - disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['NVMe'][k].get( - 'Critical', False) - - # Print errors - if not disk_ok: - show_disk_attributes(disk) - if override_disabled: - print_error('NVMe error(s) detected.') - print_standard('Tests disabled for this device') - pause() - else: - print_warning('NVMe error(s) detected.') - disk_ok = ask('Run tests on this device anyway?') - - return disk_ok - -def attributes_ok_smart(disk): - """Check SMART attributes for errors, returns bool.""" - disk_ok = True - override_disabled = False - smart_overall_pass = True - for k, v in disk.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - if 'Error' not in ATTRIBUTES['SMART'][k]: - # Only worried about error thresholds - continue - if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: - disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['SMART'][k].get( - 'Critical', False) - - # SMART overall assessment - if not disk.smartctl.get('smart_status', {}).get('passed', False): - smart_overall_pass = False - disk_ok = False - override_disabled = True - - # Print errors - if not disk_ok: - show_disk_attributes(disk) - - # 199/C7 warning - if disk.smart_attributes.get('199', {}).get('raw', 0) > 0: - print_warning('199/C7 error detected') - print_standard(' (Have you tried swapping the drive cable?)') - - # Override? - if not smart_overall_pass: - print_error('SMART overall self-assessment: Failed') - print_standard('Tests disabled for this device') - pause() - elif override_disabled: - print_error('SMART error(s) detected.') - print_standard('Tests disabled for this device') - pause() - else: - print_warning('SMART error(s) detected.') - disk_ok = ask('Run tests on this device anyway?') - - return disk_ok - def build_outer_panes(state): """Build top and side panes.""" clear_screen() @@ -915,14 +950,15 @@ def run_nvme_smart_tests(state, test): text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **test.dev.lsblk)) if test.dev.nvme_attributes: - if attributes_ok_nvme(test.dev): + # NOTE: Pass/Fail is just the attribute check + if test.dev.disk_ok: test.passed = True test.update_status('CS') else: test.failed = True test.update_status('NS') elif test.dev.smart_attributes: - if attributes_ok_smart(test.dev): + if test.dev.disk_ok: test.passed = True test.update_status('CS') else: @@ -946,22 +982,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def show_disk_attributes(disk): - """Show NVMe/SMART attributes for disk.""" - print_info('Device: {}'.format(disk.path)) - print_standard(' {size:6} ({tran}) {model} {serial}'.format(**disk.lsblk)) - print_info('Attributes') - if disk.nvme_attributes: - for k, v in disk.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - print('TODO: {} {}'.format(k, v)) - elif disk.smart_attributes: - for k, v in disk.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - print('TODO: {} {}'.format(k, v)) - else: - print_warning(' No NVMe or SMART data available') - def show_results(state): """Show results for all tests.""" clear_screen() diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 75fef0fd..7b915bdb 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -4,6 +4,8 @@ ENABLED_OPEN_LOGS = False ENABLED_TICKET_NUMBERS = False ENABLED_UPLOAD_DATA = False +HW_OVERRIDES_FORCED = False +HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED # STATIC VARIABLES (also used by BASH and BATCH files) ## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH From 47084efe1725aae5cdd302735d3f84097097d28d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 23:18:51 -0700 Subject: [PATCH 55/86] Combined nvme_check() and smart_check() --- .bin/Scripts/functions/hw_diags.py | 129 ++++++++++++----------------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 63003e8e..dd2bfa4f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -117,6 +117,54 @@ class DiskObj(): self.get_details() self.get_smart_details() + def check_attributes(self, silent=False): + """Check NVMe / SMART attributes for errors.""" + override_disabled = False + if self.nvme_attributes: + attr_type = 'NVMe' + items = self.nvme_attributes.items() + elif self.smart_attributes: + attr_type = 'SMART' + items = self.smar_attributes.items() + for k, v in items: + if k in ATTRIBUTES[attr_type]: + if 'Error' not in ATTRIBUTES[attr_type][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES[attr_type][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES[attr_type][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES[attr_type][k].get( + 'Critical', False) + + # SMART overall assessment + ## NOTE: Only fail drives if the overall value exists and reports failed + if not self.smartctl.get('smart_status', {}).get('passed', True): + self.disk_ok = False + override_disabled = True + + # Print errors + if not silent: + if self.disk_ok: + # 199/C7 warning + if self.smart_attributes.get(199, {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the disk cable?)') + else: + # Override? + self.show_attributes() + print_warning('{} error(s) detected.'.format(attr_type)) + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -181,43 +229,10 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def nvme_check(self, silent=False): - """Check NVMe attributes for errors.""" - override_disabled = False - for k, v in self.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - if 'Error' not in ATTRIBUTES['NVMe'][k]: - # Only worried about error thresholds - continue - if ATTRIBUTES['NVMe'][k].get('Ignore', False): - # Attribute is non-failing, skip - continue - if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: - self.disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['NVMe'][k].get( - 'Critical', False) - - # Print errors - if not self.disk_ok and not silent: - self.show_attributes() - print_warning('NVMe error(s) detected.') - - # Override? - if override_disabled: - print_standard('Tests disabled for this device') - pause() - elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') - def safety_check(self, silent=False): - """Check attributes and disable tests if necessary.""" - if self.nvme_attributes: - self.nvme_check(silent) - elif self.smart_attributes: - self.smart_check(silent) + """Run safety checks and disable tests if necessary.""" + if self.nvme_attributes or self.smart_attributes: + self.check_attributes(silent) else: # No NVMe/SMART details if silent: @@ -255,48 +270,6 @@ class DiskObj(): else: print_warning(' No NVMe or SMART data available') - def smart_check(self, silent=False): - """Check SMART attributes for errors.""" - override_disabled = False - for k, v in self.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - if 'Error' not in ATTRIBUTES['SMART'][k]: - # Only worried about error thresholds - continue - if ATTRIBUTES['SMART'][k].get('Ignore', False): - # Attribute is non-failing, skip - continue - if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: - self.disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['SMART'][k].get( - 'Critical', False) - - # SMART overall assessment - ## NOTE: Only fail drives if the overall value exists and reports failed - if not self.smartctl.get('smart_status', {}).get('passed', True): - self.disk_ok = False - override_disabled = True - - # Print errors - if not silent: - if self.disk_ok: - # 199/C7 warning - if self.smart_attributes.get(199, {}).get('raw', 0) > 0: - print_warning('199/C7 error detected') - print_standard(' (Have you tried swapping the disk cable?)') - else: - # Override? - self.show_attributes() - print_warning('SMART error(s) detected.') - if override_disabled: - print_standard('Tests disabled for this device') - pause() - elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') - class State(): """Object to track device objects and overall state.""" def __init__(self): From b5c93317dc33716c813927097da0838a2a8ecc1b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 23:54:02 -0700 Subject: [PATCH 56/86] Override sections working --- .bin/Scripts/functions/hw_diags.py | 66 +++++++++++++++++++----------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index dd2bfa4f..345cb279 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -125,7 +125,7 @@ class DiskObj(): items = self.nvme_attributes.items() elif self.smart_attributes: attr_type = 'SMART' - items = self.smar_attributes.items() + items = self.smart_attributes.items() for k, v in items: if k in ATTRIBUTES[attr_type]: if 'Error' not in ATTRIBUTES[attr_type][k]: @@ -359,8 +359,6 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled: - return if new_status: self.status = build_status_string( self.label, new_status, self.info_label) @@ -650,12 +648,12 @@ def run_badblocks_test(state, test): TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test({})'.format( test.dev.path)) - for disk in state.disks: - disk.tests['badblocks']['Started'] = True - update_progress_pane(state) - sleep(3) - disk.tests['badblocks']['Result'] = 'OVERRIDE' - update_progress_pane(state) + test.started = True + test.update_status() + update_progress_pane(state) + sleep(3) + test.update_status('Unknown') + update_progress_pane(state) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -691,7 +689,7 @@ def run_hw_tests(state): # Run safety checks for disk in state.disks: - disk.safety_check() + disk.safety_check(silent=state.quick_mode) # Run tests ## Because state.tests is an OrderedDict and the disks were added @@ -704,7 +702,7 @@ def run_hw_tests(state): # Done show_results(state) - if '--quick' in sys.argv: + if state.quick_mode: pause('Press Enter to exit...') else: pause('Press Enter to return to main menu... ') @@ -719,12 +717,12 @@ def run_io_benchmark(state, test): TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark({})'.format( test.dev.path)) - for disk in state.disks: - disk.tests['I/O Benchmark']['Started'] = True - update_progress_pane(state) - sleep(3) - disk.tests['I/O Benchmark']['Result'] = 'Unknown' - update_progress_pane(state) + test.started = True + test.update_status() + update_progress_pane(state) + sleep(3) + test.update_status('Unknown') + update_progress_pane(state) def run_keyboard_test(): """Run keyboard test.""" @@ -928,22 +926,42 @@ def run_nvme_smart_tests(state, test): test.passed = True test.update_status('CS') else: + # NOTE: Other test(s) should've been disabled by DiskObj.safety_check() test.failed = True test.update_status('NS') elif test.dev.smart_attributes: + # NOTE: Pass/Fail based on both attributes and SMART short self-test if test.dev.disk_ok: - test.passed = True - test.update_status('CS') + # Run short test + pause('TODO: Run SMART short self-test') + + # Check result + # TODO + short_test_passed = True + if short_test_passed: + test.passed = True + test.update_status('CS') + else: + for t in ['badblocks', 'I/O Benchmark']: + if t in test.dev.tests: + test.dev.tests[t].disabled = True + test.dev.tests[t].update_status('Denied') + # TODO + if no_logs: + test.update_status('Unknown') + else: + test.failed = True + test.update_status('NS') else: test.failed = True test.update_status('NS') else: - print_standard('Tests disabled for this device') + # NOTE: Pass/Fail not applicable without NVMe/SMART data + # Override request earlier disabled other test(s) as appropriate test.update_status('N/A') - if not ask('Run tests on this device anyway?'): - test.failed = True - update_progress_pane(state) - sleep(3) + + # Done + update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" From 5b748798053a97c200998286d1b67fcd64e70017 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 13 Dec 2018 19:02:28 -0700 Subject: [PATCH 57/86] Fixed OVERRIDE and N/A NVMe/SMART status handling --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 345cb279..d967ae15 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -162,8 +162,11 @@ class DiskObj(): print_standard('Tests disabled for this device') pause() elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') + if HW_OVERRIDES_FORCED or ask('Run tests on this device anyway?'): + self.disk_ok = True + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].update_status('OVERRIDE') + self.tests['NVMe / SMART'].disabled = True def get_details(self): """Get data from lsblk.""" @@ -235,6 +238,9 @@ class DiskObj(): self.check_attributes(silent) else: # No NVMe/SMART details + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].update_status('N/A') + self.tests['NVMe / SMART'].disabled = True if silent: self.disk_ok = HW_OVERRIDES_FORCED else: @@ -247,8 +253,8 @@ class DiskObj(): if not self.disk_ok: for t in ['badblocks', 'I/O Benchmark']: if t in self.tests: - self.tests[t].disabled = True self.tests[t].update_status('Denied') + self.tests[t].disabled = True def show_attributes(self): """Show NVMe/SMART attributes.""" @@ -359,6 +365,8 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" + if self.disabled: + return if new_status: self.status = build_status_string( self.label, new_status, self.info_label) @@ -944,8 +952,8 @@ def run_nvme_smart_tests(state, test): else: for t in ['badblocks', 'I/O Benchmark']: if t in test.dev.tests: - test.dev.tests[t].disabled = True test.dev.tests[t].update_status('Denied') + test.dev.tests[t].disabled = True # TODO if no_logs: test.update_status('Unknown') @@ -955,10 +963,6 @@ def run_nvme_smart_tests(state, test): else: test.failed = True test.update_status('NS') - else: - # NOTE: Pass/Fail not applicable without NVMe/SMART data - # Override request earlier disabled other test(s) as appropriate - test.update_status('N/A') # Done update_progress_pane(state) From 81f05fa79f3217cc37e000f4a9998278907508d4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 16:37:14 -0700 Subject: [PATCH 58/86] Replaced show_attributes() with generate_report() * Returns list of colored strings * Optionally includes short-test results * Optionally excludes disk info --- .bin/Scripts/functions/hw_diags.py | 157 ++++++++++++++++++++++++----- 1 file changed, 133 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d967ae15..7c40c741 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -112,6 +112,7 @@ class DiskObj(): self.nvme_attributes = {} self.path = disk_path self.smart_attributes = {} + self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() self.get_details() @@ -156,7 +157,9 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - self.show_attributes() + for line in self.generate_report(): + print(line) + print_log(strip_colors(line)) print_warning('{} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') @@ -168,6 +171,93 @@ class DiskObj(): self.tests['NVMe / SMART'].update_status('OVERRIDE') self.tests['NVMe / SMART'].disabled = True + def generate_report(self, brief=False, short_test=False): + """Generate NVMe / SMART report, returns list.""" + report = [] + if not brief: + report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( + dev_path=self.path, **COLORS)) + report.append(' {size:>6} ({tran}) {model} {serial}'.format( + **self.lsblk)) + + # Warnings + if self.nvme_attributes: + attr_type = 'NVMe' + report.append( + ' {YELLOW}NVMe disk support is still experimental{CLEAR}'.format( + **COLORS)) + elif self.smart_attributes: + attr_type = 'SMART' + else: + # No attribute data available, return short report + report.append( + ' {YELLOW}No NVMe or SMART data available{CLEAR}'.format( + **COLORS)) + return report + if not self.smartctl.get('smart_status', {}).get('passed', True): + report.append( + ' {RED}SMART overall self-assessment: Failed{CLEAR}'.format( + **COLORS)) + + # Attributes + report.append('{BLUE}{a} Attributes{YELLOW}{u:>23} {t}{CLEAR}'.format( + a=attr_type, + u='Updated:' if brief else '', + t=time.strftime('%Y-%m-%d %H:%M %Z') if brief else '', + **COLORS)) + if self.nvme_attributes: + attr_type = 'NVMe' + items = self.nvme_attributes.items() + elif self.smart_attributes: + attr_type = 'SMART' + items = self.smart_attributes.items() + for k, v in items: + if k in ATTRIBUTES[attr_type]: + _note = '' + _color = COLORS['GREEN'] + + # Attribute ID & Name + if attr_type == 'NVMe': + _line = ' {:38}'.format(k.replace('_', ' ').title()) + else: + _line = ' {i:>3} / {h}: {n:28}'.format( + i=k, + h=ATTRIBUTES[attr_type][k]['Hex'], + n=v['name'][:28]) + + # Set color + for _t, _c in [['Warning', 'YELLOW'], ['Error', 'RED']]: + if _t in ATTRIBUTES[attr_type][k]: + if v['raw'] >= ATTRIBUTES[attr_type][k][_t]: + _color = COLORS[_c] + + # 199/C7 warning + if str(k) == '199': + _note = '(bad cable?)' + + # Attribute value + _line += '{}{} {}{}'.format( + _color, + v['raw_str'], + _note, + COLORS['CLEAR']) + + # Add line to report + report.append(_line) + + # SMART short-test + if short_test: + report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) + if 'TimedOut' in self.tests['NVMe / SMART'].status: + report.append(' {YELLOW}UNKNOWN{CLEAR}: Timed out'.format(**COLORS)) + else: + report.append(' {}'.format( + self.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) + + # Done + return report + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -219,7 +309,7 @@ class DiskObj(): except ValueError: # Ignoring invalid attribute continue - _name = str(a.get('name', 'UNKNOWN')) + _name = str(a.get('name', 'UNKNOWN')).replace('_', ' ').title() _raw = int(a.get('raw', {}).get('value', -1)) _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') @@ -232,6 +322,13 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + # Self-test data + for k in ['polling_minutes', 'status']: + self.smart_self_test[k] = self.smartctl.get( + 'ata_smart_data', {}).get( + 'self_test', {}).get( + k, {}) + def safety_check(self, silent=False): """Run safety checks and disable tests if necessary.""" if self.nvme_attributes or self.smart_attributes: @@ -251,31 +348,15 @@ class DiskObj(): 'Run tests on this device anyway?') if not self.disk_ok: + if 'NVMe / SMART' in self.tests: + # NOTE: This will not overwrite the existing status if set + self.tests['NVMe / SMART'].update_status('NS') + self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: if t in self.tests: self.tests[t].update_status('Denied') self.tests[t].disabled = True - def show_attributes(self): - """Show NVMe/SMART attributes.""" - print_info('Device: {}'.format(self.path)) - print_standard( - ' {size:>6} ({tran}) {model} {serial}'.format(**self.lsblk)) - print_info('Attributes') - if self.nvme_attributes: - for k, v in self.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - print('TODO: {} {}'.format(k, v)) - elif self.smart_attributes: - for k, v in self.smart_attributes.items(): - # TODO: If k == 199/C7 then append ' (bad cable?)' to line - if k in ATTRIBUTES['SMART']: - print('TODO: {} {}'.format(k, v)) - if not self.smartctl.get('smart_status', {}).get('passed', True): - print_error('SMART overall self-assessment: Failed') - else: - print_warning(' No NVMe or SMART data available') - class State(): """Object to track device objects and overall state.""" def __init__(self): @@ -402,7 +483,7 @@ def build_outer_panes(state): def build_status_string(label, status, info_label=False): """Build status string with appropriate colors.""" status_color = COLORS['CLEAR'] - if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE', 'TimedOut']: status_color = COLORS['RED'] elif status in ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working']: status_color = COLORS['YELLOW'] @@ -651,6 +732,9 @@ def run_audio_test(): def run_badblocks_test(state, test): """TODO""" + # Bail early + if test.disabled: + return tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) @@ -699,6 +783,15 @@ def run_hw_tests(state): for disk in state.disks: disk.safety_check(silent=state.quick_mode) + # TODO Remove + clear_screen() + print_info('Running tests:') + for k, v in state.tests.items(): + if v['Enabled']: + print_standard(' {}'.format(k)) + update_progress_pane(state) + pause() + # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. @@ -720,6 +813,9 @@ def run_hw_tests(state): def run_io_benchmark(state, test): """TODO""" + # Bail early + if test.disabled: + return tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) @@ -739,6 +835,9 @@ def run_keyboard_test(): def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" + # Bail early + if test.disabled: + return test.started = True test.update_status() update_progress_pane(state) @@ -924,6 +1023,7 @@ def run_network_test(): def run_nvme_smart_tests(state, test): """Run NVMe or SMART test for test.dev.""" + _include_short_test = False tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( @@ -941,10 +1041,14 @@ def run_nvme_smart_tests(state, test): # NOTE: Pass/Fail based on both attributes and SMART short self-test if test.dev.disk_ok: # Run short test - pause('TODO: Run SMART short self-test') + # TODO + _include_short_test = True + _timeout = test.dev.smart_self_test['polling_minutes'].get('short', 5) + _timeout = int(_timeout) + 5 # Check result # TODO + # if 'remaining_percent' in 'status' then we've started. short_test_passed = True if short_test_passed: test.passed = True @@ -960,10 +1064,15 @@ def run_nvme_smart_tests(state, test): else: test.failed = True test.update_status('NS') + else: test.failed = True test.update_status('NS') + # Save report + test.report = test.dev.generate_report( + short_test=_include_short_test) + # Done update_progress_pane(state) From cee825245505376a757a3de5eda45cc7eb1deb95 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:03:00 -0700 Subject: [PATCH 59/86] Added CYAN to COLORS --- .bin/Scripts/functions/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 7f14bdbd..5327d895 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -27,10 +27,11 @@ global_vars = {} COLORS = { 'CLEAR': '\033[0m', 'RED': '\033[31m', + 'ORANGE': '\033[31;1m', 'GREEN': '\033[32m', 'YELLOW': '\033[33m', - 'ORANGE': '\033[31;1m', - 'BLUE': '\033[34m' + 'BLUE': '\033[34m', + 'CYAN': '\033[36m', } try: HKU = winreg.HKEY_USERS From 99984603ed601e1af78e688b2258d20ab9a7b7ef Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:32:17 -0700 Subject: [PATCH 60/86] NVMe/SMART sections working * Added timout status for clarity * Added short-test result to report --- .bin/Scripts/functions/hw_diags.py | 123 +++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 7c40c741..141d324e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -169,7 +169,9 @@ class DiskObj(): self.disk_ok = True if 'NVMe / SMART' in self.tests: self.tests['NVMe / SMART'].update_status('OVERRIDE') - self.tests['NVMe / SMART'].disabled = True + if self.nvme_attributes or not self.smart_attributes: + # i.e. only leave enabled for SMART short-tests + self.tests['NVMe / SMART'].disabled = True def generate_report(self, brief=False, short_test=False): """Generate NVMe / SMART report, returns list.""" @@ -323,6 +325,7 @@ class DiskObj(): 'name': _name, 'raw': _raw, 'raw_str': _raw_str} # Self-test data + self.smart_self_test = {} for k in ['polling_minutes', 'status']: self.smart_self_test[k] = self.smartctl.get( 'ata_smart_data', {}).get( @@ -333,6 +336,22 @@ class DiskObj(): """Run safety checks and disable tests if necessary.""" if self.nvme_attributes or self.smart_attributes: self.check_attributes(silent) + + # Check if a self-test is currently running + if 'remaining_percent' in self.smart_self_test['status']: + _msg='SMART self-test in progress, all tests disabled' + if not silent: + print_warning('WARNING: {}'.format(_msg)) + print_standard(' ') + if ask('Abort HW Diagnostics?'): + exit_script() + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report.append( + '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) + for t in self.tests.values(): + t.update_status('Denied') + t.disabled = True else: # No NVMe/SMART details if 'NVMe / SMART' in self.tests: @@ -350,6 +369,8 @@ class DiskObj(): if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set + if not self.tests['NVMe / SMART'].report: + self.tests['NVMe / SMART'].report = self.generate_report() self.tests['NVMe / SMART'].update_status('NS') self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: @@ -446,7 +467,7 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled: + if self.disabled or 'OVERRIDE' in self.status: return if new_status: self.status = build_status_string( @@ -1023,11 +1044,19 @@ def run_network_test(): def run_nvme_smart_tests(state, test): """Run NVMe or SMART test for test.dev.""" + # Bail early + if test.disabled: + return _include_short_test = False + test.started = True + test.update_status() tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **test.dev.lsblk)) + update_progress_pane(state) + + # NVMe if test.dev.nvme_attributes: # NOTE: Pass/Fail is just the attribute check if test.dev.disk_ok: @@ -1037,37 +1066,80 @@ def run_nvme_smart_tests(state, test): # NOTE: Other test(s) should've been disabled by DiskObj.safety_check() test.failed = True test.update_status('NS') + + # SMART elif test.dev.smart_attributes: # NOTE: Pass/Fail based on both attributes and SMART short self-test - if test.dev.disk_ok: - # Run short test - # TODO + if not (test.dev.disk_ok or 'OVERRIDE' in test.status): + test.failed = True + test.update_status('NS') + else: + # Prep + test.timeout = test.dev.smart_self_test['polling_minutes'].get( + 'short', 5) + # TODO: fix timeout, set to polling + 5 + test.timeout = int(test.timeout) + 1 _include_short_test = True - _timeout = test.dev.smart_self_test['polling_minutes'].get('short', 5) - _timeout = int(_timeout) + 5 + _self_test_started = False - # Check result - # TODO - # if 'remaining_percent' in 'status' then we've started. - short_test_passed = True - if short_test_passed: - test.passed = True - test.update_status('CS') + # Create monitor pane + test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n Pending') + state.panes['smart'] = tmux_split_window( + lines=3, vertical=True, watch=test.smart_out) + + # Show attributes + clear_screen() + for line in test.dev.generate_report(): + # Not saving to log; that will happen after all tests have been run + print(line) + print(' ') + + # Start short test + print_standard('Running self-test...') + cmd = ['sudo', 'smartctl', '--test=short', test.dev.path] + run_program(cmd, check=False) + + # Monitor progress (in 5 second increments) + for iteration in range(int(test.timeout*60/5)): + sleep(5) + + # Update SMART data + test.dev.get_smart_details() + + if _self_test_started: + # Update progress file + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n {}'.format( + test.dev.smart_self_test['status'].get('string', 'UNKNOWN'))) + + # Check if test has finished + if 'remaining_percent' not in test.dev.smart_self_test['status']: + break + + else: + # Check if test has started + if 'remaining_percent' in test.dev.smart_self_test['status']: + _self_test_started = True + + # Check if timed out + if test.dev.smart_self_test['status'].get('passed', False): + if 'OVERRIDE' not in test.status: + test.passed = True + test.update_status('CS') else: + test.failed = True + test.update_status('NS') + if not (test.failed or test.passed): + test.update_status('TimedOut') + + # Disable other drive tests if necessary + if not test.passed: for t in ['badblocks', 'I/O Benchmark']: if t in test.dev.tests: test.dev.tests[t].update_status('Denied') test.dev.tests[t].disabled = True - # TODO - if no_logs: - test.update_status('Unknown') - else: - test.failed = True - test.update_status('NS') - - else: - test.failed = True - test.update_status('NS') # Save report test.report = test.dev.generate_report( @@ -1076,6 +1148,9 @@ def run_nvme_smart_tests(state, test): # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['smart']) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': From 37b8676b9c436968ab7d68032b8deb3bc164f445 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:57:30 -0700 Subject: [PATCH 61/86] Fixed quick check --- .bin/Scripts/functions/hw_diags.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 141d324e..a4bc4f5f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1073,6 +1073,13 @@ def run_nvme_smart_tests(state, test): if not (test.dev.disk_ok or 'OVERRIDE' in test.status): test.failed = True test.update_status('NS') + elif state.quick_mode: + if test.dev.disk_ok: + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') else: # Prep test.timeout = test.dev.smart_self_test['polling_minutes'].get( @@ -1141,6 +1148,9 @@ def run_nvme_smart_tests(state, test): test.dev.tests[t].update_status('Denied') test.dev.tests[t].disabled = True + # Cleanup + tmux_kill_pane(state.panes['smart']) + # Save report test.report = test.dev.generate_report( short_test=_include_short_test) @@ -1148,9 +1158,6 @@ def run_nvme_smart_tests(state, test): # Done update_progress_pane(state) - # Cleanup - tmux_kill_pane(state.panes['smart']) - def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': From f2a519b7ec7ed3738ad3370c350e51ade8fd1d54 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:58:32 -0700 Subject: [PATCH 62/86] Adjusted log and results screen --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a4bc4f5f..8fa6d743 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -157,9 +157,7 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - for line in self.generate_report(): - print(line) - print_log(strip_colors(line)) + show_report(self.generate_report()) print_warning('{} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') @@ -756,6 +754,7 @@ def run_badblocks_test(state, test): # Bail early if test.disabled: return + print_log('Starting badblocks test for {}'.format(test.dev.path)) tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) @@ -837,6 +836,7 @@ def run_io_benchmark(state, test): # Bail early if test.disabled: return + print_log('Starting I/O benchmark test for {}'.format(test.dev.path)) tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) @@ -859,6 +859,7 @@ def run_mprime_test(state, test): # Bail early if test.disabled: return + print_log('Starting Prime95 test') test.started = True test.update_status() update_progress_pane(state) @@ -1047,6 +1048,7 @@ def run_nvme_smart_tests(state, test): # Bail early if test.disabled: return + print_log('Starting NVMe/SMART test for {}'.format(test.dev.path)) _include_short_test = False test.started = True test.update_status() @@ -1168,6 +1170,12 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_report(report): + """Show report on screen and save to log w/out color.""" + for line in report: + print(line) + print_log(strip_colors(line)) + def show_results(state): """Show results for all tests.""" clear_screen() @@ -1180,11 +1188,7 @@ def show_results(state): continue print_success('{}:'.format(k)) for obj in v['Objects']: - for line in obj.report: - print(line) - print_log(strip_colors(line)) - print_standard(' ') - if 'Prime95' not in k: + show_report(obj.report) print_standard(' ') def update_main_options(state, selection, main_options): From a5d92537f54c8af6d96db97036b721289c317290 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 16:54:48 -0700 Subject: [PATCH 63/86] Removed unused function --- .bin/Scripts/functions/hw_diags.py | 32 ------------------------------ 1 file changed, 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8fa6d743..8409b4c8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -517,38 +517,6 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) -def check_disk_attributes(disk): - """Check if disk should be tested and allow overrides.""" - needs_override = False - print_standard(' {size:>6} ({tran}) {model} {serial}'.format( - **disk.lsblk)) - - # General checks - if not disk.nvme_attributes and not disk.smart_attributes: - needs_override = True - print_warning( - ' WARNING: No NVMe or SMART attributes available for: {}'.format( - disk.path)) - - # NVMe checks - # TODO check all tracked attributes and set disk.failing if needed - - # SMART checks - # TODO check all tracked attributes and set disk.failing if needed - - # Ask for override if necessary - if needs_override: - if ask(' Run tests on this device anyway?'): - # TODO Set override for this disk - pass - else: - for v in disk.tests.values(): - # Started is set to True to fix the status string - v['Result'] = 'Skipped' - v['Started'] = True - v['Status'] = 'Skipped' - print_standard('') - def generate_horizontal_graph(rates, oneline=False): """Generate two-line horizontal graph from rates, returns str.""" line_1 = '' From dc8416b5f71fb708a971fe6cde9a9bec9f2dff2a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 16:55:32 -0700 Subject: [PATCH 64/86] Adjusted formatting --- .bin/Scripts/functions/hw_diags.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8409b4c8..0b88001b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -117,6 +117,8 @@ class DiskObj(): self.tests = OrderedDict() self.get_details() self.get_smart_details() + self.description = '{size:>6} ({tran}) {model} {serial}'.format( + **self.lsblk) def check_attributes(self, silent=False): """Check NVMe / SMART attributes for errors.""" @@ -177,8 +179,7 @@ class DiskObj(): if not brief: report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( dev_path=self.path, **COLORS)) - report.append(' {size:>6} ({tran}) {model} {serial}'.format( - **self.lsblk)) + report.append(' {}'.format(self.description)) # Warnings if self.nvme_attributes: @@ -718,14 +719,14 @@ def run_audio_test(): pause('Press Enter to return to main menu... ') def run_badblocks_test(state, test): - """TODO""" + """Run a read-only surface scan with badblocks.""" # Bail early if test.disabled: return print_log('Starting badblocks test for {}'.format(test.dev.path)) tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'badblocks')) + state.panes['Top'], + text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) print_standard('TODO: run_badblocks_test({})'.format( test.dev.path)) test.started = True @@ -1022,8 +1023,7 @@ def run_nvme_smart_tests(state, test): test.update_status() tmux_update_pane( state.panes['Top'], - text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **test.dev.lsblk)) + text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) update_progress_pane(state) # NVMe @@ -1062,7 +1062,7 @@ def run_nvme_smart_tests(state, test): # Create monitor pane test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n Pending') + f.write('SMART self-test status:\n Starting...') state.panes['smart'] = tmux_split_window( lines=3, vertical=True, watch=test.smart_out) @@ -1089,7 +1089,8 @@ def run_nvme_smart_tests(state, test): # Update progress file with open(test.smart_out, 'w') as f: f.write('SMART self-test status:\n {}'.format( - test.dev.smart_self_test['status'].get('string', 'UNKNOWN'))) + test.dev.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) # Check if test has finished if 'remaining_percent' not in test.dev.smart_self_test['status']: From e96ac5c156c1eca05e14dda41a95cff0997ae492 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:09:54 -0700 Subject: [PATCH 65/86] Added watch option to use tail instead of cat * tail -f acurately prints backspace (^H) characters * badblocks output uses them and wouldn't work with watch/cat --- .bin/Scripts/functions/tmux.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 69b906d1..5fbca65d 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -46,7 +46,7 @@ def tmux_split_window( behind=False, vertical=False, follow=False, target_pane=None, working_dir=None, command=None, - text=None, watch=None): + text=None, watch=None, watch_cmd='cat'): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: @@ -79,19 +79,21 @@ def tmux_split_window( cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: create_file(watch) - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) + if watch_cmd == 'cat': + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + elif watch_cmd == 'tail': + cmd.extend(['tail', '-f', watch]) # Run and return pane_id result = run_program(cmd) return result.stdout.decode().strip() def tmux_update_pane( - pane_id, command=None, - text=None, watch=None, - working_dir=None): + pane_id, command=None, working_dir=None, + text=None, watch=None, watch_cmd='cat'): """Respawn with either a new command or new text.""" # Bail early if not command and not text and not watch: @@ -106,10 +108,13 @@ def tmux_update_pane( cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: create_file(watch) - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) + if watch_cmd == 'cat': + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + elif watch_cmd == 'tail': + cmd.extend(['tail', '-f', watch]) run_program(cmd) From 8b936f54137d0275fbdf398fa5722749aad8903f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:45:43 -0700 Subject: [PATCH 66/86] badblocks section working --- .bin/Scripts/functions/hw_diags.py | 73 +++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 0b88001b..54ce8ba5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -383,7 +383,6 @@ class State(): self.cpu = None self.disks = [] self.panes = {} - self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = OrderedDict({ 'Prime95': { @@ -423,6 +422,7 @@ class State(): os.makedirs(global_vars['LogDir'], exist_ok=True) global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( global_vars['LogDir']) + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) # Add CPU self.cpu = CpuObj() @@ -723,19 +723,68 @@ def run_badblocks_test(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting badblocks test for {}'.format(test.dev.path)) - tmux_update_pane( - state.panes['Top'], - text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) - print_standard('TODO: run_badblocks_test({})'.format( - test.dev.path)) test.started = True test.update_status() update_progress_pane(state) - sleep(3) - test.update_status('Unknown') + + # Update top pane + tmux_update_pane( + state.panes['Top'], + text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + + # Create monitor pane + test.badblocks_out = '{}/badblocks.out'.format(global_vars['LogDir']) + state.panes['badblocks'] = tmux_split_window( + lines=5, vertical=True, watch=test.badblocks_out, watch_cmd='tail') + + # Show disk details + clear_screen() + show_report(test.dev.generate_report()) + print_standard(' ') + + # Start badblocks + print_standard('Running badblocks test...') + test.badblocks_proc = popen_program( + ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], + pipe=True) + test.badblocks_proc.wait() + + # Check result and create report + try: + test.badblocks_out = test.badblocks_proc.stdout.read().decode() + except Exception as err: + test.badblocks_out = 'Error: {}'.format(err) + for line in test.badblocks_out.splitlines(): + line = line.strip() + if not line or re.search(r'^Checking', line, re.IGNORECASE): + # Skip empty and progress lines + continue + if re.search(r'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): + test.report.append(' {}'.format(line)) + test.passed = True + else: + test.report.append(' {YELLOW}{line}{CLEAR}'.format( + line=line, **COLORS)) + test.failed = True + + # Update status + if test.failed: + test.update_status('NS') + elif test.passed: + test.update_status('CS') + else: + test.update_status('Unknown') + + # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['badblocks']) + pause() + def run_hw_tests(state): """Run enabled hardware tests.""" print_standard('Scanning devices...') @@ -828,6 +877,8 @@ def run_mprime_test(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting Prime95 test') test.started = True test.update_status() @@ -835,9 +886,9 @@ def run_mprime_test(state, test): test.sensor_data = get_sensor_data() # Update top pane - test.title = '{}\nPrime95: {}'.format( - TOP_PANE_TEXT, test.dev.name) - tmux_update_pane(state.panes['Top'], text=test.title) + tmux_update_pane( + state.panes['Top'], + text='{}\nPrime95: {}'.format(TOP_PANE_TEXT, test.dev.name)) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) From ef42b596d95199503380828c101eeb2d01554439 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:56:41 -0700 Subject: [PATCH 67/86] Catch CTRL+c aborts and show results --- .bin/Scripts/functions/hw_diags.py | 78 ++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 54ce8ba5..2ed8e686 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -747,10 +747,13 @@ def run_badblocks_test(state, test): # Start badblocks print_standard('Running badblocks test...') - test.badblocks_proc = popen_program( - ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], - pipe=True) - test.badblocks_proc.wait() + try: + test.badblocks_proc = popen_program( + ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], + pipe=True) + test.badblocks_proc.wait() + except KeyboardInterrupt: + raise GenericAbort('Aborted') # Check result and create report try: @@ -833,11 +836,31 @@ def run_hw_tests(state): # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. - for k, v in state.tests.items(): - if v['Enabled']: - f = v['Function'] - for test_obj in v['Objects']: - f(state, test_obj) + try: + for k, v in state.tests.items(): + if v['Enabled']: + f = v['Function'] + for test_obj in v['Objects']: + f(state, test_obj) + except GenericAbort: + # Cleanup + tmux_kill_pane(*state.panes.values()) + + # Rebuild panes + update_progress_pane(state) + build_outer_panes(state) + + # Mark unfinished tests as aborted + for k, v in state.tests.items(): + if v['Enabled']: + for test_obj in v['Objects']: + if re.search(r'(Pending|Working)', test_obj.status): + test_obj.update_status('Aborted') + test_obj.report.append(' {YELLOW}Aborted{CLEAR}'.format( + **COLORS)) + + # Update side pane + update_progress_pane(state) # Done show_results(state) @@ -1130,27 +1153,30 @@ def run_nvme_smart_tests(state, test): run_program(cmd, check=False) # Monitor progress (in 5 second increments) - for iteration in range(int(test.timeout*60/5)): - sleep(5) + try: + for iteration in range(int(test.timeout*60/5)): + sleep(5) - # Update SMART data - test.dev.get_smart_details() + # Update SMART data + test.dev.get_smart_details() - if _self_test_started: - # Update progress file - with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n {}'.format( - test.dev.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) + if _self_test_started: + # Update progress file + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n {}'.format( + test.dev.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) - # Check if test has finished - if 'remaining_percent' not in test.dev.smart_self_test['status']: - break + # Check if test has finished + if 'remaining_percent' not in test.dev.smart_self_test['status']: + break - else: - # Check if test has started - if 'remaining_percent' in test.dev.smart_self_test['status']: - _self_test_started = True + else: + # Check if test has started + if 'remaining_percent' in test.dev.smart_self_test['status']: + _self_test_started = True + except KeyboardInterrupt: + raise GenericAbort('Aborted') # Check if timed out if test.dev.smart_self_test['status'].get('passed', False): From 8993b483a633884bc6634e38c619701e4518dd3f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:30:46 -0700 Subject: [PATCH 68/86] Fix bad cable note --- .bin/Scripts/functions/hw_diags.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 2ed8e686..f0fd09b4 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -233,15 +233,15 @@ class DiskObj(): _color = COLORS[_c] # 199/C7 warning - if str(k) == '199': + if str(k) == '199' and v['raw'] > 0: _note = '(bad cable?)' # Attribute value - _line += '{}{} {}{}'.format( - _color, - v['raw_str'], - _note, - COLORS['CLEAR']) + _line += '{c}{v} {YELLOW}{n}{CLEAR}'.format( + c=_color, + v=v['raw_str'], + n=_note, + **COLORS) # Add line to report report.append(_line) From a4896a55f619ed7709a7bfa9a490bcac0445fb04 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:31:34 -0700 Subject: [PATCH 69/86] Adjust log names --- .bin/Scripts/functions/hw_diags.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index f0fd09b4..f2fa7469 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -736,7 +736,8 @@ def run_badblocks_test(state, test): text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) # Create monitor pane - test.badblocks_out = '{}/badblocks.out'.format(global_vars['LogDir']) + test.badblocks_out = '{}/badblocks_{}.out'.format( + global_vars['LogDir'], test.dev.name) state.panes['badblocks'] = tmux_split_window( lines=5, vertical=True, watch=test.badblocks_out, watch_cmd='tail') @@ -1134,7 +1135,8 @@ def run_nvme_smart_tests(state, test): _self_test_started = False # Create monitor pane - test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) + test.smart_out = '{}/smart_{}.out'.format( + global_vars['LogDir'], test.dev.name) with open(test.smart_out, 'w') as f: f.write('SMART self-test status:\n Starting...') state.panes['smart'] = tmux_split_window( From 503e6f2b4251fbf7355d413fb3df680441a45abb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:45:25 -0700 Subject: [PATCH 70/86] Fix SMART short-test timeout detection --- .bin/Scripts/functions/hw_diags.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index f2fa7469..361aad2d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1181,13 +1181,14 @@ def run_nvme_smart_tests(state, test): raise GenericAbort('Aborted') # Check if timed out - if test.dev.smart_self_test['status'].get('passed', False): - if 'OVERRIDE' not in test.status: - test.passed = True - test.update_status('CS') - else: - test.failed = True - test.update_status('NS') + if 'passed' in test.dev.smart_self_test['status']: + if test.dev.smart_self_test['status']['passed']: + if 'OVERRIDE' not in test.status: + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') if not (test.failed or test.passed): test.update_status('TimedOut') From 4c0bb1c9b7d954319f249422c1b41f2191fa0658 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:06:03 -0700 Subject: [PATCH 71/86] Group results by device instead of test --- .bin/Scripts/functions/hw_diags.py | 139 ++++++++++++++++++----------- 1 file changed, 87 insertions(+), 52 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 361aad2d..4b544303 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -79,7 +79,7 @@ class CpuObj(): """Object for tracking CPU specific data.""" def __init__(self): self.lscpu = {} - self.tests = {} + self.tests = OrderedDict() self.get_details() self.name = self.lscpu.get('Model name', 'Unknown CPU') @@ -102,6 +102,18 @@ class CpuObj(): continue self.lscpu[_field] = _data + def generate_cpu_report(self): + """Generate CPU report with data from all tests.""" + report = [] + report.append('{BLUE}Device{CLEAR}'.format(**COLORS)) + report.append(' {}'.format(self.name)) + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): @@ -115,9 +127,10 @@ class DiskObj(): self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() + self.warnings = [] self.get_details() self.get_smart_details() - self.description = '{size:>6} ({tran}) {model} {serial}'.format( + self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) def check_attributes(self, silent=False): @@ -159,8 +172,8 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - show_report(self.generate_report()) - print_warning('{} error(s) detected.'.format(attr_type)) + show_report(self.generate_attribute_report(description=True)) + print_warning(' {} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') pause() @@ -172,13 +185,15 @@ class DiskObj(): if self.nvme_attributes or not self.smart_attributes: # i.e. only leave enabled for SMART short-tests self.tests['NVMe / SMART'].disabled = True + print_standard(' ') - def generate_report(self, brief=False, short_test=False): + def generate_attribute_report( + self, description=False, short_test=False, timestamp=False): """Generate NVMe / SMART report, returns list.""" report = [] - if not brief: - report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( - dev_path=self.path, **COLORS)) + if description: + report.append('{BLUE}Device ({name}){CLEAR}'.format( + name=self.name, **COLORS)) report.append(' {}'.format(self.description)) # Warnings @@ -203,8 +218,8 @@ class DiskObj(): # Attributes report.append('{BLUE}{a} Attributes{YELLOW}{u:>23} {t}{CLEAR}'.format( a=attr_type, - u='Updated:' if brief else '', - t=time.strftime('%Y-%m-%d %H:%M %Z') if brief else '', + u='Updated:' if timestamp else '', + t=time.strftime('%Y-%m-%d %H:%M %Z') if timestamp else '', **COLORS)) if self.nvme_attributes: attr_type = 'NVMe' @@ -259,6 +274,21 @@ class DiskObj(): # Done return report + def generate_disk_report(self): + """Generate disk report with data from all tests.""" + report = [] + report.append('{BLUE}Device ({name}){CLEAR}'.format( + name=self.name, **COLORS)) + report.append(' {}'.format(self.description)) + for w in self.warnings: + report.append(' {YELLOW}{w}{CLEAR}'.format(w=w, **COLORS)) + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -338,14 +368,14 @@ class DiskObj(): # Check if a self-test is currently running if 'remaining_percent' in self.smart_self_test['status']: - _msg='SMART self-test in progress, all tests disabled' + _msg = 'SMART self-test in progress, all tests disabled' if not silent: print_warning('WARNING: {}'.format(_msg)) print_standard(' ') if ask('Abort HW Diagnostics?'): exit_script() if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].report.append( '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) for t in self.tests.values(): @@ -359,17 +389,20 @@ class DiskObj(): if silent: self.disk_ok = HW_OVERRIDES_FORCED else: - print_warning( - ' WARNING: No NVMe or SMART attributes available for: {}'.format( - self.path)) + _msg = 'No NVMe or SMART data available' + self.warnings.append(_msg) + print_info('Device ({})'.format(self.name)) + print_standard(' {}'.format(self.description)) + print_warning(' {}'.format(_msg)) self.disk_ok = HW_OVERRIDES_FORCED or ask( 'Run tests on this device anyway?') + print_standard(' ') if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set if not self.tests['NVMe / SMART'].report: - self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].update_status('NS') self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: @@ -743,7 +776,7 @@ def run_badblocks_test(state, test): # Show disk details clear_screen() - show_report(test.dev.generate_report()) + show_report(test.dev.generate_attribute_report()) print_standard(' ') # Start badblocks @@ -757,6 +790,7 @@ def run_badblocks_test(state, test): raise GenericAbort('Aborted') # Check result and create report + test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) try: test.badblocks_out = test.badblocks_proc.stdout.read().decode() except Exception as err: @@ -787,7 +821,6 @@ def run_badblocks_test(state, test): # Cleanup tmux_kill_pane(state.panes['badblocks']) - pause() def run_hw_tests(state): """Run enabled hardware tests.""" @@ -825,15 +858,6 @@ def run_hw_tests(state): for disk in state.disks: disk.safety_check(silent=state.quick_mode) - # TODO Remove - clear_screen() - print_info('Running tests:') - for k, v in state.tests.items(): - if v['Enabled']: - print_standard(' {}'.format(k)) - update_progress_pane(state) - pause() - # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. @@ -1010,6 +1034,7 @@ def run_mprime_test(state, test): global_vars['LogDir'])) # Check results and build report + test.report.append('{BLUE}Prime95{CLEAR}'.format(**COLORS)) test.logs = {} for log in ['results.txt', 'prime.log']: lines = [] @@ -1026,20 +1051,18 @@ def run_mprime_test(state, test): # results.txt (NS check) if log == 'results.txt': - _tmp = [] for line in lines: + line = line.strip() if re.search(r'(error|fail)', line, re.IGNORECASE): test.failed = True test.update_status('NS') - _tmp.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) - if _tmp: - test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) - test.report.extend(_tmp) + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # prime.log (CS check) if log == 'prime.log': _tmp = {'Pass': {}, 'Warn': {}} for line in lines: + line = line.strip() _r = re.search( r'(completed.*(\d+) errors, (\d+) warnings)', line, @@ -1059,18 +1082,19 @@ def run_mprime_test(state, test): elif len(_tmp['Pass']) > 0: test.passed = True test.update_status('CS') - if len(_tmp['Pass']) + len(_tmp['Warn']) > 0: - test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) - for line in sorted(_tmp['Pass'].keys()): - test.report.append(' {}'.format(line)) - for line in sorted(_tmp['Warn'].keys()): - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) + for line in sorted(_tmp['Pass'].keys()): + test.report.append(' {}'.format(line)) + for line in sorted(_tmp['Warn'].keys()): + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) - # Finalize report + # Unknown result if not (test.aborted or test.failed or test.passed): + test.report.append(' {YELLOW}Unknown result{CLEAR}'.format(**COLORS)) test.update_status('Unknown') + + # Add temps to report test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) - for line in generate_report( + for line in generate_sensor_report( test.sensor_data, 'Idle', 'Max', 'Cooldown', core_only=True): test.report.append(' {}'.format(line)) @@ -1144,10 +1168,10 @@ def run_nvme_smart_tests(state, test): # Show attributes clear_screen() - for line in test.dev.generate_report(): - # Not saving to log; that will happen after all tests have been run - print(line) - print(' ') + print_info('Device ({})'.format(test.dev.name)) + print_standard(' {}'.format(test.dev.description)) + show_report(test.dev.generate_attribute_report()) + print_standard(' ') # Start short test print_standard('Running self-test...') @@ -1203,7 +1227,7 @@ def run_nvme_smart_tests(state, test): tmux_kill_pane(state.panes['smart']) # Save report - test.report = test.dev.generate_report( + test.report = test.dev.generate_attribute_report( short_test=_include_short_test) # Done @@ -1231,13 +1255,24 @@ def show_results(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'Results')) - for k, v in state.tests.items(): - # Skip disabled tests - if not v['Enabled']: - continue - print_success('{}:'.format(k)) - for obj in v['Objects']: - show_report(obj.report) + + # CPU tests + _enabled = False + for k in TESTS_CPU: + _enabled |= state.tests[k]['Enabled'] + if _enabled: + print_success('CPU:'.format(k)) + show_report(state.cpu.generate_cpu_report()) + print_standard(' ') + + # Disk tests + for k in TESTS_DISK: + _enabled |= state.tests[k]['Enabled'] + if _enabled: + print_success('Disk{}:'.format( + '' if len(state.disks) == 1 else 's')) + for disk in state.disks: + show_report(disk.generate_disk_report()) print_standard(' ') def update_main_options(state, selection, main_options): From d8123a71ec5ca887115b76d07d197e54282bcd07 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:07:34 -0700 Subject: [PATCH 72/86] Renamed generate_report to generate_sensor_report --- .bin/Scripts/functions/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index b6319744..99e7999a 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -36,7 +36,7 @@ def fix_sensor_str(s): s = s.replace(' ', ' ') return s -def generate_report( +def generate_sensor_report( sensor_data, *temp_labels, colors=True, core_only=False): """Generate report based on temp_labels, returns list if str.""" @@ -153,7 +153,7 @@ def monitor_sensors(monitor_pane, monitor_file): while True: update_sensor_data(sensor_data) with open(monitor_file, 'w') as f: - report = generate_report(sensor_data, 'Current', 'Max') + report = generate_sensor_report(sensor_data, 'Current', 'Max') f.write('\n'.join(report)) sleep(1) if monitor_pane and not tmux_poll_pane(monitor_pane): From e0a2993c362469acaa08674bc85d28f3ef2920a2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:18:34 -0700 Subject: [PATCH 73/86] Skip disk safety checks if only testing the CPU --- .bin/Scripts/functions/hw_diags.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4b544303..6fd384a6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -854,9 +854,13 @@ def run_hw_tests(state): v['Objects'].append(test_obj) print_standard('') - # Run safety checks - for disk in state.disks: - disk.safety_check(silent=state.quick_mode) + # Run disk safety checks (if necessary) + _disk_tests_enabled = False + for k in TESTS_DISK: + _disk_tests_enabled |= state.tests[k]['Enabled'] + if _disk_tests_enabled: + for disk in state.disks: + disk.safety_check(silent=state.quick_mode) # Run tests ## Because state.tests is an OrderedDict and the disks were added @@ -1266,6 +1270,7 @@ def show_results(state): print_standard(' ') # Disk tests + _enabled = False for k in TESTS_DISK: _enabled |= state.tests[k]['Enabled'] if _enabled: From baaf1994e3312df5f39ee4bdd8c3009dacd50b9f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:44:46 -0700 Subject: [PATCH 74/86] Catch keyboard interrupt and gracefully abort --- .bin/Scripts/hw-diags-menu | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index 6f0247cd..5b6f4f76 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -13,20 +13,25 @@ from functions.tmux import * init_global_vars() if __name__ == '__main__': + # Show menu try: - # Show menu state = State() menu_diags(state, sys.argv) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() + except KeyboardInterrupt: + print_standard(' ') + print_warning('Aborted') + print_standard(' ') + sleep(1) + pause('Press Enter to exit...') except SystemExit: - tmux_kill_all_panes() + # Normal exit pass except: tmux_kill_all_panes() major_exception() + # Done + tmux_kill_all_panes() + exit_script() + # vim: sts=2 sw=2 ts=2 From c820d2ac6de623a4d6f1ef3400bca890b8f2965a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 13:20:39 -0700 Subject: [PATCH 75/86] Fixed Prime95 abort handling --- .bin/Scripts/functions/hw_diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6fd384a6..ef411220 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1083,7 +1083,7 @@ def run_mprime_test(state, test): test.failed = True test.passed = False test.update_status('NS') - elif len(_tmp['Pass']) > 0: + elif len(_tmp['Pass']) > 0 and not test.aborted: test.passed = True test.update_status('CS') for line in sorted(_tmp['Pass'].keys()): From a25a10e616cbeaaec796bc0c0c3cf22f6e9c35f7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 14:07:19 -0700 Subject: [PATCH 76/86] More abort logic updates --- .bin/Scripts/functions/hw_diags.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index ef411220..c9a84940 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -127,7 +127,6 @@ class DiskObj(): self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() - self.warnings = [] self.get_details() self.get_smart_details() self.description = '{size} ({tran}) {model} {serial}'.format( @@ -280,8 +279,10 @@ class DiskObj(): report.append('{BLUE}Device ({name}){CLEAR}'.format( name=self.name, **COLORS)) report.append(' {}'.format(self.description)) - for w in self.warnings: - report.append(' {YELLOW}{w}{CLEAR}'.format(w=w, **COLORS)) + + # Attributes + if 'NVMe / SMART' not in self.tests: + report.extend(self.generate_attribute_report()) # Tests for test in self.tests.values(): @@ -389,11 +390,9 @@ class DiskObj(): if silent: self.disk_ok = HW_OVERRIDES_FORCED else: - _msg = 'No NVMe or SMART data available' - self.warnings.append(_msg) print_info('Device ({})'.format(self.name)) print_standard(' {}'.format(self.description)) - print_warning(' {}'.format(_msg)) + print_warning(' No NVMe or SMART data available') self.disk_ok = HW_OVERRIDES_FORCED or ask( 'Run tests on this device anyway?') print_standard(' ') @@ -787,9 +786,9 @@ def run_badblocks_test(state, test): pipe=True) test.badblocks_proc.wait() except KeyboardInterrupt: - raise GenericAbort('Aborted') + test.aborted = True - # Check result and create report + # Check result and build report test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) try: test.badblocks_out = test.badblocks_proc.stdout.read().decode() @@ -802,11 +801,16 @@ def run_badblocks_test(state, test): continue if re.search(r'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): test.report.append(' {}'.format(line)) - test.passed = True + if not test.aborted: + test.passed = True else: test.report.append(' {YELLOW}{line}{CLEAR}'.format( line=line, **COLORS)) test.failed = True + if test.aborted: + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + test.update_status('Aborted') + raise GenericAbort('Aborted') # Update status if test.failed: @@ -885,8 +889,6 @@ def run_hw_tests(state): for test_obj in v['Objects']: if re.search(r'(Pending|Working)', test_obj.status): test_obj.update_status('Aborted') - test_obj.report.append(' {YELLOW}Aborted{CLEAR}'.format( - **COLORS)) # Update side pane update_progress_pane(state) @@ -1206,6 +1208,12 @@ def run_nvme_smart_tests(state, test): if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True except KeyboardInterrupt: + test.aborted = True + test.report = test.dev.generate_attribute_report() + test.report.append('{BLUE}SMART Short self-test{CLEAR}'.format( + **COLORS)) + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + test.update_status('Aborted') raise GenericAbort('Aborted') # Check if timed out From 385bdd7dbf11f3e5f6b99029263ea9f4fc2f7879 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:10:58 -0700 Subject: [PATCH 77/86] Allow resizing current pane --- .bin/Scripts/functions/tmux.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 5fbca65d..20f35921 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -28,12 +28,15 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes -def tmux_resize_pane(pane_id, x=None, y=None): +def tmux_resize_pane(pane_id=None, x=None, y=None): """Resize pane to specific hieght or width.""" if not x and not y: raise Exception('Neither height nor width specified.') - cmd = ['tmux', 'resize-pane', '-t', pane_id] + cmd = ['tmux', 'resize-pane'] + if pane_id: + # NOTE: If pane_id not specified then the current pane will be resized + cmd.extend(['-t', pane_id]) if x: cmd.extend(['-x', str(x)]) elif y: From ec8c78197b45b3a6d812474dc0f3d0d30af55ec6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:15:40 -0700 Subject: [PATCH 78/86] I/O Benchmark test is working --- .bin/Scripts/functions/hw_diags.py | 223 +++++++++++++++++++++++++++-- 1 file changed, 212 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index c9a84940..50eca7b5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -34,7 +34,6 @@ HW_OVERRIDES_FORCED = HW_OVERRIDES_FORCED and not HW_OVERRIDES_LIMITED IO_VARS = { 'Block Size': 512*1024, 'Chunk Size': 32*1024**2, - 'Minimum Dev Size': 8*1024**3, 'Minimum Test Size': 10*1024**3, 'Alt Test Size Factor': 0.01, 'Progress Refresh Rate': 5, @@ -74,6 +73,10 @@ TESTS_DISK = [ ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) +# Error Classe +class DeviceTooSmallError(Exception): + pass + # Classes class CpuObj(): """Object for tracking CPU specific data.""" @@ -132,6 +135,60 @@ class DiskObj(): self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) + def calc_io_dd_values(self): + """Calcualte I/O benchmark dd values.""" + # Get real disk size + cmd = ['lsblk', + '--bytes', '--nodeps', '--noheadings', + '--output', 'size', self.path] + result = run_program(cmd) + self.size_bytes = int(result.stdout.decode().strip()) + + # dd calculations + ## The minimum dev size is 'Graph Horizontal Width' * 'Chunk Size' + ## (e.g. 1.25 GB for a width of 40 and a chunk size of 32MB) + ## If the device is smaller than the minimum dd_chunks would be set + ## to zero which would cause a divide by zero error. + ## If the device is below the minimum size an Exception will be raised + ## + ## dd_size is the area to be read in bytes + ## If the dev is < 10Gb then it's the whole dev + ## Otherwise it's the larger of 10Gb or 1% of the dev + ## + ## dd_chunks is the number of groups of "Chunk Size" in self.dd_size + ## This number is reduced to a multiple of the graph width in + ## order to allow for the data to be condensed cleanly + ## + ## dd_chunk_blocks is the chunk size in number of blocks + ## (e.g. 64 if block size is 512KB and chunk size is 32MB + ## + ## dd_skip_blocks is the number of "Block Size" groups not tested + ## dd_skip_count is the number of blocks to skip per self.dd_chunk + ## dd_skip_extra is how often to add an additional skip block + ## This is needed to ensure an even testing across the dev + ## This is calculated by using the fractional amount left off + ## of the dd_skip_count variable + self.dd_size = min(IO_VARS['Minimum Test Size'], self.size_bytes) + self.dd_size = max( + self.dd_size, + self.size_bytes * IO_VARS['Alt Test Size Factor']) + self.dd_chunks = int(self.dd_size // IO_VARS['Chunk Size']) + self.dd_chunks -= self.dd_chunks % IO_VARS['Graph Horizontal Width'] + if self.dd_chunks < IO_VARS['Graph Horizontal Width']: + raise DeviceTooSmallError + self.dd_chunk_blocks = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) + self.dd_size = self.dd_chunks * IO_VARS['Chunk Size'] + self.dd_skip_blocks = int( + (self.size_bytes - self.dd_size) // IO_VARS['Block Size']) + self.dd_skip_count = int((self.dd_skip_blocks / self.dd_chunks) // 1) + self.dd_skip_extra = 0 + try: + self.dd_skip_extra = 1 + int( + 1 / ((self.dd_skip_blocks / self.dd_chunks) % 1)) + except ZeroDivisionError: + # self.dd_skip_extra == 0 is fine + pass + def check_attributes(self, silent=False): """Check NVMe / SMART attributes for errors.""" override_disabled = False @@ -498,7 +555,7 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled or 'OVERRIDE' in self.status: + if self.disabled or re.search(r'ERROR|OVERRIDE', self.status): return if new_status: self.status = build_status_string( @@ -904,23 +961,163 @@ def run_hw_tests(state): tmux_kill_pane(*state.panes.values()) def run_io_benchmark(state, test): - """TODO""" + """Run a read-only I/O benchmark using dd.""" # Bail early if test.disabled: return + + # Prep print_log('Starting I/O benchmark test for {}'.format(test.dev.path)) - tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'I/O Benchmark')) - print_standard('TODO: run_io_benchmark({})'.format( - test.dev.path)) test.started = True test.update_status() update_progress_pane(state) - sleep(3) - test.update_status('Unknown') + + # Update top pane + tmux_update_pane( + state.panes['Top'], + text='{}\nI/O Benchmark: {}'.format( + TOP_PANE_TEXT, test.dev.description)) + + # Create monitor pane + test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( + global_vars['LogDir'], test.dev.name) + state.panes['io_benchmark'] = tmux_split_window( + percent=75, vertical=True, + watch=test.io_benchmark_out, watch_cmd='tail') + tmux_resize_pane(y=15) + + # Show disk details + clear_screen() + show_report(test.dev.generate_attribute_report()) + print_standard(' ') + + # Start I/O Benchmark + print_standard('Running I/O benchmark test...') + try: + test.merged_rates = [] + test.read_rates = [] + test.vertical_graph = [] + test.dev.calc_io_dd_values() + + # Run dd read tests + offset = 0 + for i in range(test.dev.dd_chunks): + # Build cmd + i += 1 + skip = test.dev.dd_skip_count + if test.dev.dd_skip_extra and i % test.dev.dd_skip_extra == 0: + skip += 1 + cmd = [ + 'sudo', 'dd', + 'bs={}'.format(IO_VARS['Block Size']), + 'skip={}'.format(offset+skip), + 'count={}'.format(test.dev.dd_chunk_blocks), + 'iflag=direct', + 'if={}'.format(test.dev.path), + 'of=/dev/null'] + + # Run cmd and get read rate + result = run_program(cmd) + result_str = result.stderr.decode().replace('\n', '') + cur_rate = get_read_rate(result_str) + + # Add rate to lists + test.read_rates.append(cur_rate) + test.vertical_graph.append( + '{percent:0.1f} {rate}'.format( + percent=(i/test.dev.dd_chunks)*100, + rate=int(cur_rate/(1024**2)))) + + # Show progress + if i % IO_VARS['Progress Refresh Rate'] == 0: + update_io_progress( + percent=(i/test.dev.dd_chunks)*100, + rate=cur_rate, + progress_file=test.io_benchmark_out) + + # Update offset + offset += test.dev.dd_chunk_blocks + skip + + except DeviceTooSmallError: + # Device too small, skipping test + test.update_status('N/A') + except KeyboardInterrupt: + test.aborted = True + except (subprocess.CalledProcessError, TypeError, ValueError): + # Something went wrong, results unknown + test.update_status('ERROR') + + # Check result and build report + test.report.append('{BLUE}I/O Benchmark{CLEAR}'.format(**COLORS)) + if test.aborted: + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + raise GenericAbort('Aborted') + elif not test.read_rates: + if 'ERROR' in test.status: + test.report.append(' {RED}Unknown error{CLEAR}'.format(**COLORS)) + elif 'N/A' in test.status: + # Device too small + test.report.append(' {YELLOW}Disk too small to test{CLEAR}'.format( + **COLORS)) + else: + # Merge rates for horizontal graph + offset = 0 + width = int(test.dev.dd_chunks / IO_VARS['Graph Horizontal Width']) + for i in range(IO_VARS['Graph Horizontal Width']): + test.merged_rates.append( + sum(test.read_rates[offset:offset+width])/width) + offset += width + + # Add horizontal graph to report + for line in generate_horizontal_graph(test.merged_rates): + if not re.match(r'^\s+$', line): + test.report.append(line) + + # Add read speeds to report + avg_read = sum(test.read_rates) / len(test.read_rates) + min_read = min(test.read_rates) + max_read = max(test.read_rates) + avg_min_max = 'Read speeds avg: {:3.1f}'.format(avg_read/(1024**2)) + avg_min_max += ' min: {:3.1f}'.format(min_read/(1024**2)) + avg_min_max += ' max: {:3.1f}'.format(max_read/(1024**2)) + test.report.append(avg_min_max) + + # Compare read speeds to thresholds + if test.dev.lsblk['rota']: + # Use HDD scale + thresh_min = IO_VARS['Threshold HDD Min'] + thresh_high_avg = IO_VARS['Threshold HDD High Avg'] + thresh_low_avg = IO_VARS['Threshold HDD Low Avg'] + else: + # Use SSD scale + thresh_min = IO_VARS['Threshold SSD Min'] + thresh_high_avg = IO_VARS['Threshold SSD High Avg'] + thresh_low_avg = IO_VARS['Threshold SSD Low Avg'] + if min_read <= thresh_min and avg_read <= thresh_high_avg: + test.failed = True + elif avg_read <= thresh_low_avg: + test.failed = True + else: + test.passed = True + + # Update status + if test.failed: + test.update_status('NS') + elif test.passed: + test.update_status('CS') + elif not 'N/A' in test.status: + test.update_status('Unknown') + + # Save log + with open(test.io_benchmark_out.replace('.', '-raw.'), 'a') as f: + f.write('\n'.join(test.vertical_graph)) + + # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['io_benchmark']) + def run_keyboard_test(): """Run keyboard test.""" clear_screen() @@ -1122,14 +1319,18 @@ def run_nvme_smart_tests(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting NVMe/SMART test for {}'.format(test.dev.path)) _include_short_test = False test.started = True test.update_status() + update_progress_pane(state) + + # Update top pane tmux_update_pane( state.panes['Top'], text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) - update_progress_pane(state) # NVMe if test.dev.nvme_attributes: From 8c5820d5aa73cbface183311655f97842854eb99 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:16:35 -0700 Subject: [PATCH 79/86] Fix horizontal graph * generate_horizontal_graph() now returns a list instead of a str --- .bin/Scripts/functions/hw_diags.py | 48 +++++++++++++----------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 50eca7b5..85c4d86e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -608,11 +608,8 @@ def build_status_string(label, status, info_label=False): **COLORS) def generate_horizontal_graph(rates, oneline=False): - """Generate two-line horizontal graph from rates, returns str.""" - line_1 = '' - line_2 = '' - line_3 = '' - line_4 = '' + """Generate horizontal graph from rates, returns list.""" + graph = ['', '', '', ''] for r in rates: step = get_graph_step(r, scale=32) if oneline: @@ -630,33 +627,30 @@ def generate_horizontal_graph(rates, oneline=False): # Build graph full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) if step >= 24: - line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) - line_2 += full_block - line_3 += full_block - line_4 += full_block + graph[0] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) + graph[1] += full_block + graph[2] += full_block + graph[3] += full_block elif step >= 16: - line_1 += ' ' - line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) - line_3 += full_block - line_4 += full_block + graph[0] += ' ' + graph[1] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) + graph[2] += full_block + graph[3] += full_block elif step >= 8: - line_1 += ' ' - line_2 += ' ' - line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) - line_4 += full_block + graph[0] += ' ' + graph[1] += ' ' + graph[2] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + graph[3] += full_block else: - line_1 += ' ' - line_2 += ' ' - line_3 += ' ' - line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) - line_1 += COLORS['CLEAR'] - line_2 += COLORS['CLEAR'] - line_3 += COLORS['CLEAR'] - line_4 += COLORS['CLEAR'] + graph[0] += ' ' + graph[1] += ' ' + graph[2] += ' ' + graph[3] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + graph = [line+COLORS['CLEAR'] for line in graph] if oneline: - return line_4 + return graph[:-1] else: - return '\n'.join([line_1, line_2, line_3, line_4]) + return graph def get_graph_step(rate, scale=16): """Get graph step based on rate and scale, returns int.""" From 41c9a4d23fa5c9df58ceef85d6a2e35934827a59 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:29:09 -0700 Subject: [PATCH 80/86] Fixed only showing non-empty graph lines --- .bin/Scripts/functions/hw_diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 85c4d86e..30220583 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1064,7 +1064,7 @@ def run_io_benchmark(state, test): # Add horizontal graph to report for line in generate_horizontal_graph(test.merged_rates): - if not re.match(r'^\s+$', line): + if not re.match(r'^\s+$', strip_colors(line)): test.report.append(line) # Add read speeds to report From 0c0f8e895021c7bfae0ae1a1d258bf86046f54ae Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:51:02 -0700 Subject: [PATCH 81/86] Added disable_test() to Disk class --- .bin/Scripts/functions/hw_diags.py | 44 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 30220583..4a855dc6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -237,12 +237,18 @@ class DiskObj(): if HW_OVERRIDES_FORCED or ask('Run tests on this device anyway?'): self.disk_ok = True if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].update_status('OVERRIDE') - if self.nvme_attributes or not self.smart_attributes: - # i.e. only leave enabled for SMART short-tests - self.tests['NVMe / SMART'].disabled = True + self.disable_test('NVMe / SMART', 'OVERRIDE') + if not self.nvme_attributes and self.smart_attributes: + # Re-enable for SMART short-tests + self.tests['NVMe / SMART'].disabled = False print_standard(' ') + def disable_test(self, name, status): + """Disable test by name and update status.""" + if name in self.tests: + self.tests[name].update_status(status) + self.tests[name].disabled = True + def generate_attribute_report( self, description=False, short_test=False, timestamp=False): """Generate NVMe / SMART report, returns list.""" @@ -427,23 +433,26 @@ class DiskObj(): # Check if a self-test is currently running if 'remaining_percent' in self.smart_self_test['status']: _msg = 'SMART self-test in progress, all tests disabled' + + # Ask to abort if not silent: print_warning('WARNING: {}'.format(_msg)) print_standard(' ') if ask('Abort HW Diagnostics?'): exit_script() + + # Add warning to report if 'NVMe / SMART' in self.tests: self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].report.append( '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) - for t in self.tests.values(): - t.update_status('Denied') - t.disabled = True + + # Disable all tests for this disk + for t in self.tests.keys(): + self.disable_test(k, 'Denied') else: # No NVMe/SMART details - if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].update_status('N/A') - self.tests['NVMe / SMART'].disabled = True + self.disable_test('NVMe / SMART', 'N/A') if silent: self.disk_ok = HW_OVERRIDES_FORCED else: @@ -457,14 +466,11 @@ class DiskObj(): if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set + self.disable_test('NVMe / SMART', 'NS') if not self.tests['NVMe / SMART'].report: self.tests['NVMe / SMART'].report = self.generate_attribute_report() - self.tests['NVMe / SMART'].update_status('NS') - self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: - if t in self.tests: - self.tests[t].update_status('Denied') - self.tests[t].disabled = True + self.disable_test(t, 'Denied') class State(): """Object to track device objects and overall state.""" @@ -863,6 +869,10 @@ def run_badblocks_test(state, test): test.update_status('Aborted') raise GenericAbort('Aborted') + # Disable other drive tests if necessary + if not test.passed: + test.dev.disable_test('I/O Benchmark', 'Denied') + # Update status if test.failed: test.update_status('NS') @@ -1426,9 +1436,7 @@ def run_nvme_smart_tests(state, test): # Disable other drive tests if necessary if not test.passed: for t in ['badblocks', 'I/O Benchmark']: - if t in test.dev.tests: - test.dev.tests[t].update_status('Denied') - test.dev.tests[t].disabled = True + test.dev.disable_test(t, 'Denied') # Cleanup tmux_kill_pane(state.panes['smart']) From 10ae59be197c431f74e64e03758cdd15b8599432 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 00:55:57 -0700 Subject: [PATCH 82/86] Update tmux layout periodically --- .bin/Scripts/functions/hw_diags.py | 106 ++++++++++++++++++++++++++--- .bin/Scripts/functions/tmux.py | 20 +++++- 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4a855dc6..56b3c501 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -72,6 +72,11 @@ TESTS_DISK = [ 'badblocks', ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) +TMUX_LAYOUT = OrderedDict({ + 'Top': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, +}) # Error Classe class DeviceTooSmallError(Exception): @@ -449,7 +454,7 @@ class DiskObj(): # Disable all tests for this disk for t in self.tests.keys(): - self.disable_test(k, 'Denied') + self.disable_test(t, 'Denied') else: # No NVMe/SMART details self.disable_test('NVMe / SMART', 'N/A') @@ -613,6 +618,49 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) +def fix_tmux_panes(state, tmux_layout): + """Fix pane sizes in case the window has been resized.""" + needs_fixed = False + + # Check layout + for k, v in tmux_layout.items(): + if not v.get('Check'): + # Not concerned with the size of this pane + continue + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Get pane size + x, y = tmux_get_pane_size(pane_id=target) + if v.get('x', False) and v.get['x'] != x: + needs_fixed = True + if v.get('y', False) and v.get['y'] != y: + needs_fixed = True + + # Bail? + if not needs_fixed: + return + + # Update layout + for k, v in tmux_layout.items(): + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Resize pane + tmux_resize_pane(pane_id=target, **v) + def generate_horizontal_graph(rates, oneline=False): """Generate horizontal graph from rates, returns list.""" graph = ['', '', '', ''] @@ -819,10 +867,14 @@ def run_badblocks_test(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'badblocks': {'y': 5, 'Check': True}, + }) # Create monitor pane test.badblocks_out = '{}/badblocks_{}.out'.format( @@ -841,7 +893,15 @@ def run_badblocks_test(state, test): test.badblocks_proc = popen_program( ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], pipe=True) - test.badblocks_proc.wait() + while True: + try: + test.badblocks_proc.wait(timeout=10) + except subprocess.TimeoutExpired: + fix_tmux_panes(state, test.tmux_layout) + else: + # badblocks finished, exit loop + break + except KeyboardInterrupt: test.aborted = True @@ -976,11 +1036,16 @@ def run_io_benchmark(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nI/O Benchmark: {}'.format( TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'io_benchmark': {'y': 1000, 'Check': False}, + 'Current': {'y': 15, 'Check': True}, + }) # Create monitor pane test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( @@ -1042,6 +1107,10 @@ def run_io_benchmark(state, test): # Update offset offset += test.dev.dd_chunk_blocks + skip + # Fix panes + if i % 5 == 0: + fix_tmux_panes(state, test.tmux_layout) + except DeviceTooSmallError: # Device too small, skipping test test.update_status('N/A') @@ -1140,10 +1209,16 @@ def run_mprime_test(state, test): update_progress_pane(state) test.sensor_data = get_sensor_data() - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nPrime95: {}'.format(TOP_PANE_TEXT, test.dev.name)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'Temps': {'y': 1000, 'Check': False}, + 'mprime': {'y': 11, 'Check': False}, + 'Current': {'y': 3, 'Check': True}, + }) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) @@ -1181,7 +1256,7 @@ def run_mprime_test(state, test): working_dir=global_vars['TmpDir']) #time_limit = int(MPRIME_LIMIT) * 60 # TODO: restore above line - time_limit = 10 + time_limit = 30 try: for i in range(time_limit): clear_screen() @@ -1199,6 +1274,12 @@ def run_mprime_test(state, test): print(_status_str) print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) update_sensor_data(test.sensor_data) + + # Fix panes + if i % 10 == 0: + fix_tmux_panes(state, test.tmux_layout) + + # Wait sleep(1) except KeyboardInterrupt: # Catch CTRL+C @@ -1331,10 +1412,14 @@ def run_nvme_smart_tests(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'smart': {'y': 3, 'Check': True}, + }) # NVMe if test.dev.nvme_attributes: @@ -1391,7 +1476,7 @@ def run_nvme_smart_tests(state, test): # Monitor progress (in 5 second increments) try: - for iteration in range(int(test.timeout*60/5)): + for i in range(int(test.timeout*60/5)): sleep(5) # Update SMART data @@ -1412,6 +1497,11 @@ def run_nvme_smart_tests(state, test): # Check if test has started if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True + + # Fix panes + if i % 2 == 0: + fix_tmux_panes(state, test.tmux_layout) + except KeyboardInterrupt: test.aborted = True test.report = test.dev.generate_attribute_report() diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 20f35921..0e585df6 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -8,6 +8,24 @@ def create_file(filepath): with open(filepath, 'w') as f: f.write('') +def tmux_get_pane_size(pane_id=None): + """Get target, or current, pane size, returns tuple.""" + x = -1 + y = -1 + cmd = ['tmux', 'display', '-p', '#{pane_width}x#{pane_height}'] + if pane_id: + cmd.extend(['-t', pane_id]) + + # Run cmd and set x & y + result = run_program(cmd, check=False) + try: + x, y = result.stdout.decode().strip().split() + except Exception: + # Ignore and return unrealistic values + pass + + return (x, y) + def tmux_kill_all_panes(pane_id=None): """Kill all tmux panes except the active pane or pane_id if specified.""" cmd = ['tmux', 'kill-pane', '-a'] @@ -28,7 +46,7 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes -def tmux_resize_pane(pane_id=None, x=None, y=None): +def tmux_resize_pane(pane_id=None, x=None, y=None, **kwargs): """Resize pane to specific hieght or width.""" if not x and not y: raise Exception('Neither height nor width specified.') From 932669844baee7a485ece26f83f7885875245641 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:13:33 -0700 Subject: [PATCH 83/86] Fixed tmux pane size handling --- .bin/Scripts/functions/hw_diags.py | 29 +++++++++++++++-------------- .bin/Scripts/functions/tmux.py | 5 ++++- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 56b3c501..dc71c54f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -638,9 +638,9 @@ def fix_tmux_panes(state, tmux_layout): # Get pane size x, y = tmux_get_pane_size(pane_id=target) - if v.get('x', False) and v.get['x'] != x: + if v.get('x', False) and v['x'] != x: needs_fixed = True - if v.get('y', False) and v.get['y'] != y: + if v.get('y', False) and v['y'] != y: needs_fixed = True # Bail? @@ -895,7 +895,7 @@ def run_badblocks_test(state, test): pipe=True) while True: try: - test.badblocks_proc.wait(timeout=10) + test.badblocks_proc.wait(timeout=1) except subprocess.TimeoutExpired: fix_tmux_panes(state, test.tmux_layout) else: @@ -1108,8 +1108,7 @@ def run_io_benchmark(state, test): offset += test.dev.dd_chunk_blocks + skip # Fix panes - if i % 5 == 0: - fix_tmux_panes(state, test.tmux_layout) + fix_tmux_panes(state, test.tmux_layout) except DeviceTooSmallError: # Device too small, skipping test @@ -1276,8 +1275,7 @@ def run_mprime_test(state, test): update_sensor_data(test.sensor_data) # Fix panes - if i % 10 == 0: - fix_tmux_panes(state, test.tmux_layout) + fix_tmux_panes(state, test.tmux_layout) # Wait sleep(1) @@ -1474,10 +1472,17 @@ def run_nvme_smart_tests(state, test): cmd = ['sudo', 'smartctl', '--test=short', test.dev.path] run_program(cmd, check=False) - # Monitor progress (in 5 second increments) + # Monitor progress try: - for i in range(int(test.timeout*60/5)): - sleep(5) + for i in range(int(test.timeout*60)): + sleep(1) + + # Fix panes + fix_tmux_panes(state, test.tmux_layout) + + # Only update SMART progress every 5 seconds + if i % 5 != 0: + continue # Update SMART data test.dev.get_smart_details() @@ -1498,10 +1503,6 @@ def run_nvme_smart_tests(state, test): if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True - # Fix panes - if i % 2 == 0: - fix_tmux_panes(state, test.tmux_layout) - except KeyboardInterrupt: test.aborted = True test.report = test.dev.generate_attribute_report() diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 0e585df6..ce35f5b0 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -12,14 +12,17 @@ def tmux_get_pane_size(pane_id=None): """Get target, or current, pane size, returns tuple.""" x = -1 y = -1 - cmd = ['tmux', 'display', '-p', '#{pane_width}x#{pane_height}'] + cmd = ['tmux', 'display', '-p'] if pane_id: cmd.extend(['-t', pane_id]) + cmd.append('#{pane_width} #{pane_height}') # Run cmd and set x & y result = run_program(cmd, check=False) try: x, y = result.stdout.decode().strip().split() + x = int(x) + y = int(y) except Exception: # Ignore and return unrealistic values pass From 7ac035c578b2fe62b308a14d783b96d6f3fbc98d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:21:05 -0700 Subject: [PATCH 84/86] Safety wheels off --- .bin/Scripts/functions/hw_diags.py | 22 +++++++--------------- .bin/Scripts/settings/main.py | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index dc71c54f..cfe68c69 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -837,13 +837,11 @@ def menu_diags(state, args): elif selection == 'R': print('(FAKE) reboot...') sleep(1) - # TODO uncomment below - #run_program(['systemctl', 'reboot']) + run_program(['systemctl', 'reboot']) elif selection == 'P': print('(FAKE) poweroff...') sleep(1) - # TODO uncomment below - #run_program(['systemctl', 'poweroff']) + run_program(['systemctl', 'poweroff']) elif selection == 'Q': break elif selection == 'S': @@ -1242,8 +1240,7 @@ def run_mprime_test(state, test): message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', sensor_data=test.sensor_data, temp_label='Idle', - seconds=3) - # TODO: Remove seconds kwarg above + seconds=5) # Stress CPU print_log('Starting Prime95') @@ -1253,9 +1250,7 @@ def run_mprime_test(state, test): state.panes['mprime'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) - #time_limit = int(MPRIME_LIMIT) * 60 - # TODO: restore above line - time_limit = 30 + time_limit = int(MPRIME_LIMIT) * 60 try: for i in range(time_limit): clear_screen() @@ -1301,14 +1296,12 @@ def run_mprime_test(state, test): clear_screen() try_and_print( message='Letting CPU cooldown for bit...', indent=0, - function=sleep, cs='Done', seconds=3) - # TODO: Above seconds should be 10 + function=sleep, cs='Done', seconds=10) try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', sensor_data=test.sensor_data, temp_label='Cooldown', - seconds=3) - # TODO: Remove seconds kwarg above + seconds=5) # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -1447,8 +1440,7 @@ def run_nvme_smart_tests(state, test): # Prep test.timeout = test.dev.smart_self_test['polling_minutes'].get( 'short', 5) - # TODO: fix timeout, set to polling + 5 - test.timeout = int(test.timeout) + 1 + test.timeout = int(test.timeout) + 5 _include_short_test = True _self_test_started = False diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 7b915bdb..9d32b3ef 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -15,7 +15,7 @@ KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux -MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags +MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags ROOT_PASSWORD='Abracadabra' TECH_PASSWORD='Abracadabra' # Server IP addresses From 91a77bb14e9bfb1f9525aaffc6390e667e32c2d9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:47:03 -0700 Subject: [PATCH 85/86] Ensure SMART timeout message is in the report --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cfe68c69..64d417da 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -132,6 +132,7 @@ class DiskObj(): self.nvme_attributes = {} self.path = disk_path self.smart_attributes = {} + self.smart_timeout = False self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() @@ -331,12 +332,11 @@ class DiskObj(): # SMART short-test if short_test: report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) - if 'TimedOut' in self.tests['NVMe / SMART'].status: - report.append(' {YELLOW}UNKNOWN{CLEAR}: Timed out'.format(**COLORS)) - else: - report.append(' {}'.format( - self.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) + report.append(' {}'.format( + self.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) + if self.smart_timeout: + report.append(' {YELLOW}Timed out{CLEAR}'.format(**COLORS)) # Done return report @@ -1443,6 +1443,7 @@ def run_nvme_smart_tests(state, test): test.timeout = int(test.timeout) + 5 _include_short_test = True _self_test_started = False + _self_test_finished = False # Create monitor pane test.smart_out = '{}/smart_{}.out'.format( @@ -1488,6 +1489,7 @@ def run_nvme_smart_tests(state, test): # Check if test has finished if 'remaining_percent' not in test.dev.smart_self_test['status']: + _self_test_finished = True break else: @@ -1505,15 +1507,16 @@ def run_nvme_smart_tests(state, test): raise GenericAbort('Aborted') # Check if timed out - if 'passed' in test.dev.smart_self_test['status']: - if test.dev.smart_self_test['status']['passed']: + if _self_test_finished: + if test.dev.smart_self_test['status'].get('passed', False): if 'OVERRIDE' not in test.status: test.passed = True test.update_status('CS') else: test.failed = True test.update_status('NS') - if not (test.failed or test.passed): + else: + test.dev.smart_timeout = True test.update_status('TimedOut') # Disable other drive tests if necessary From e5f0ccb5d51bf8d89460c3a35eff26d10e8dc333 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:57:48 -0700 Subject: [PATCH 86/86] Formatting cleanup --- .bin/Scripts/functions/hw_diags.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 64d417da..b09dbf05 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -591,8 +591,8 @@ def build_outer_panes(state): # Started state.panes['Started'] = tmux_split_window( lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top'], - text='{BLUE}Started{CLEAR}\n{text}'.format( - text=time.strftime("%Y-%m-%d %H:%M %Z"), + text='{BLUE}Started{CLEAR}\n{s}'.format( + s=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) # Progress @@ -868,7 +868,8 @@ def run_badblocks_test(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + text='{}\nbadblocks: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'badblocks': {'y': 5, 'Check': True}, @@ -1037,8 +1038,8 @@ def run_io_benchmark(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nI/O Benchmark: {}'.format( - TOP_PANE_TEXT, test.dev.description)) + text='{}\nI/O Benchmark: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'io_benchmark': {'y': 1000, 'Check': False}, @@ -1406,7 +1407,8 @@ def run_nvme_smart_tests(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) + text='{}\nDisk Health: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'smart': {'y': 3, 'Check': True}, @@ -1554,8 +1556,8 @@ def show_results(state): """Show results for all tests.""" clear_screen() tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'Results')) + state.panes['Top'], + text='{}\nResults'.format(TOP_PANE_TEXT)) # CPU tests _enabled = False