From 10f2fca2bfa7e24d52e238e1d3ed611e5e9200fe Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 17:52:07 -0700 Subject: [PATCH 001/186] Added classes DevObj and State --- .bin/Scripts/functions/hw_diags.py | 1310 +++++----------------------- 1 file changed, 221 insertions(+), 1089 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 169eb337..830d948a 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1,1114 +1,246 @@ # Wizard Kit: Functions - HW Diagnostics import json +import re import time from functions.common import * # STATIC VARIABLES ATTRIBUTES = { - 'NVMe': { - 'critical_warning': {'Error': 1}, - 'media_errors': {'Error': 1}, - 'power_on_hours': {'Warning': 12000, 'Error': 18000, 'Ignore': True}, - 'unsafe_shutdowns': {'Warning': 1}, - }, - 'SMART': { - 5: {'Hex': '05', 'Error': 1}, - 9: {'Hex': '09', 'Warning': 12000, 'Error': 18000, 'Ignore': True}, - 10: {'Hex': '0A', 'Error': 1}, - 184: {'Hex': 'B8', 'Error': 1}, - 187: {'Hex': 'BB', 'Error': 1}, - 188: {'Hex': 'BC', 'Error': 1}, - 196: {'Hex': 'C4', 'Error': 1}, - 197: {'Hex': 'C5', 'Error': 1}, - 198: {'Hex': 'C6', 'Error': 1}, - 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - 201: {'Hex': 'C9', 'Error': 1}, - }, - } + 'NVMe': { + 'critical_warning': {'Error': 1}, + 'media_errors': {'Error': 1}, + 'power_on_hours': {'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 'unsafe_shutdowns': {'Warning': 1}, + }, + 'SMART': { + 5: {'Hex': '05', 'Error': 1}, + 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 10: {'Hex': '0A', 'Error': 1}, + 184: {'Hex': 'B8', 'Error': 1}, + 187: {'Hex': 'BB', 'Error': 1}, + 188: {'Hex': 'BC', 'Error': 1}, + 196: {'Hex': 'C4', 'Error': 1}, + 197: {'Hex': 'C5', 'Error': 1}, + 198: {'Hex': 'C6', 'Error': 1}, + 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + 201: {'Hex': 'C9', 'Error': 1}, + }, + } IO_VARS = { - 'Block Size': 512*1024, - 'Chunk Size': 32*1024**2, - 'Minimum Dev Size': 8*1024**3, - 'Minimum Test Size': 10*1024**3, - 'Alt Test Size Factor': 0.01, - 'Progress Refresh Rate': 5, - 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], - 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], - 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], - 'Threshold Graph Fail': 65*1024**2, - 'Threshold Graph Warn': 135*1024**2, - 'Threshold Graph Great': 750*1024**2, - 'Threshold HDD Min': 50*1024**2, - 'Threshold HDD High Avg': 75*1024**2, - 'Threshold HDD Low Avg': 65*1024**2, - 'Threshold SSD Min': 90*1024**2, - 'Threshold SSD High Avg': 135*1024**2, - 'Threshold SSD Low Avg': 100*1024**2, - 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), - 'Graph Horizontal Width': 40, - 'Graph Vertical': ( - '▏', '▎', '▍', '▌', - '▋', '▊', '▉', '█', - '█▏', '█▎', '█▍', '█▌', - '█▋', '█▊', '█▉', '██', - '██▏', '██▎', '██▍', '██▌', - '██▋', '██▊', '██▉', '███', - '███▏', '███▎', '███▍', '███▌', - '███▋', '███▊', '███▉', '████'), - } -TESTS = { - 'Prime95': { - 'Enabled': False, - 'Status': 'Pending', - }, - 'NVMe/SMART': { - 'Enabled': False, - 'Quick': False, - 'Short Test': {}, - 'Status': {}, - }, - 'badblocks': { - 'Enabled': False, - 'Results': {}, - 'Status': {}, - }, - 'iobenchmark': { - 'Data': {}, - 'Enabled': False, - 'Results': {}, - 'Status': {}, - }, - } + 'Block Size': 512*1024, + 'Chunk Size': 32*1024**2, + 'Minimum Dev Size': 8*1024**3, + 'Minimum Test Size': 10*1024**3, + 'Alt Test Size Factor': 0.01, + 'Progress Refresh Rate': 5, + 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], + 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], + 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], + 'Threshold Graph Fail': 65*1024**2, + 'Threshold Graph Warn': 135*1024**2, + 'Threshold Graph Great': 750*1024**2, + 'Threshold HDD Min': 50*1024**2, + 'Threshold HDD High Avg': 75*1024**2, + 'Threshold HDD Low Avg': 65*1024**2, + 'Threshold SSD Min': 90*1024**2, + 'Threshold SSD High Avg': 135*1024**2, + 'Threshold SSD Low Avg': 100*1024**2, + 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), + 'Graph Horizontal Width': 40, + 'Graph Vertical': ( + '▏', '▎', '▍', '▌', + '▋', '▊', '▉', '█', + '█▏', '█▎', '█▍', '█▌', + '█▋', '█▊', '█▉', '██', + '██▏', '██▎', '██▍', '██▌', + '██▋', '██▊', '██▉', '███', + '███▏', '███▎', '███▍', '███▌', + '███▋', '███▊', '███▉', '████'), + } +KEY_NVME = 'nvme_smart_health_information_log' +KEY_SMART = 'ata_smart_attributes' +SIDE_PATH_WIDTH = 21 +# Classes +class DevObj(): + """Device object for tracking device specific data.""" + def __init__(self, dev_path): + self.failing = False + self.nvme_attributes = {} + self.override = False + self.path = dev_path + self.smart_attributes = {} + self.tests = { + 'NVMe / SMART': {'Result': None, 'Status': None}, + 'badblocks': {'Result': None, 'Status': None}, + 'I/O Benchmark': { + 'Result': None, + 'Status': None, + 'Read Rates': [], + 'Graph Data': []}, + } + self.get_details() + + def get_details(self): + """Get data from smartctl.""" + cmd = ['sudo', 'smartctl', '--all', '--json', self.path] + result = run_program(cmd, check=False) + self.data = json.loads(result.stdout.decode()) + + # Check for attributes + if KEY_NVME in self.data: + self.nvme_attributes.update(self.data[KEY_NVME]) + elif KEY_SMART in self.data: + for a in self.data[KEY_SMART].get('table', {}): + _id = str(a.get('id', 'UNKNOWN')) + _name = str(a.get('name', 'UNKNOWN')) + _raw = a.get('raw', {}).get('value', -1) + _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') + + # Fix power-on time + _r = re.match(r'^(\d+)[Hh].*', _raw_str) + if _id == '9' and _r: + try: + _raw = int(_r.group(1)) + except ValueError: + # That's fine + pass + self.smart_attributes[_id] = { + 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + +class State(): + """Object to track device objects and overall state.""" + def __init__(self): + self.devs = [] + self.finished = False + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.started = False + self.tests = { + 'Prime95': {'Enabled': False, 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False}, + 'badblocks': {'Enabled': False}, + 'I/O Benchmark': {'Enabled': False}, + } + self.add_devs() + + def add_devs(self): + """Add all block devices listed by lsblk.""" + cmd = ['lsblk', '--json', '--nodeps', '--paths'] + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + for dev in json_data['blockdevices']: + self.devs.append(DevObj(dev['name'])) + +# Functions def generate_horizontal_graph(rates, oneline=False): - """Generate two-line horizontal graph from rates, returns str.""" - line_1 = '' - line_2 = '' - line_3 = '' - line_4 = '' - for r in rates: - step = get_graph_step(r, scale=32) - if oneline: - step = get_graph_step(r, scale=8) - - # Set color - r_color = COLORS['CLEAR'] - if r < IO_VARS['Threshold Graph Fail']: - r_color = COLORS['RED'] - elif r < IO_VARS['Threshold Graph Warn']: - r_color = COLORS['YELLOW'] - elif r > IO_VARS['Threshold Graph Great']: - r_color = COLORS['GREEN'] - - # Build graph - full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) - if step >= 24: - line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) - line_2 += full_block - line_3 += full_block - line_4 += full_block - elif step >= 16: - line_1 += ' ' - line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) - line_3 += full_block - line_4 += full_block - elif step >= 8: - line_1 += ' ' - line_2 += ' ' - line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) - line_4 += full_block - else: - line_1 += ' ' - line_2 += ' ' - line_3 += ' ' - line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) - line_1 += COLORS['CLEAR'] - line_2 += COLORS['CLEAR'] - line_3 += COLORS['CLEAR'] - line_4 += COLORS['CLEAR'] + """Generate two-line horizontal graph from rates, returns str.""" + line_1 = '' + line_2 = '' + line_3 = '' + line_4 = '' + for r in rates: + step = get_graph_step(r, scale=32) if oneline: - return line_4 + step = get_graph_step(r, scale=8) + + # Set color + r_color = COLORS['CLEAR'] + if r < IO_VARS['Threshold Graph Fail']: + r_color = COLORS['RED'] + elif r < IO_VARS['Threshold Graph Warn']: + r_color = COLORS['YELLOW'] + elif r > IO_VARS['Threshold Graph Great']: + r_color = COLORS['GREEN'] + + # Build graph + full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) + if step >= 24: + line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) + line_2 += full_block + line_3 += full_block + line_4 += full_block + elif step >= 16: + line_1 += ' ' + line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) + line_3 += full_block + line_4 += full_block + elif step >= 8: + line_1 += ' ' + line_2 += ' ' + line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + line_4 += full_block else: - return '\n'.join([line_1, line_2, line_3, line_4]) + line_1 += ' ' + line_2 += ' ' + line_3 += ' ' + line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + line_1 += COLORS['CLEAR'] + line_2 += COLORS['CLEAR'] + line_3 += COLORS['CLEAR'] + line_4 += COLORS['CLEAR'] + if oneline: + return line_4 + else: + return '\n'.join([line_1, line_2, line_3, line_4]) def get_graph_step(rate, scale=16): - """Get graph step based on rate and scale, returns int.""" - m_rate = rate / (1024**2) - step = 0 - scale_name = 'Scale {}'.format(scale) - for x in range(scale-1, -1, -1): - # Iterate over scale backwards - if m_rate >= IO_VARS[scale_name][x]: - step = x - break - return step + """Get graph step based on rate and scale, returns int.""" + m_rate = rate / (1024**2) + step = 0 + scale_name = 'Scale {}'.format(scale) + for x in range(scale-1, -1, -1): + # Iterate over scale backwards + if m_rate >= IO_VARS[scale_name][x]: + step = x + break + return step def get_read_rate(s): - """Get read rate in bytes/s from dd progress output.""" - real_rate = None - if re.search(r'[KMGT]B/s', s): - human_rate = re.sub(r'^.*\s+(\d+\.?\d*)\s+(.B)/s\s*$', r'\1 \2', s) - real_rate = convert_to_bytes(human_rate) - return real_rate - -def get_smart_details(dev): - """Get SMART data for dev if possible, returns dict.""" - cmd = 'sudo smartctl --all --json {}{}'.format( - '' if '/dev/' in dev else '/dev/', - dev).split() - result = run_program(cmd, check=False) - try: - return json.loads(result.stdout.decode()) - except Exception: - # Let other sections deal with the missing data - return {} - -def get_smart_value(smart_data, smart_id): - """Get SMART value from table, returns int or None.""" - value = None - table = smart_data.get('ata_smart_attributes', {}).get('table', []) - for row in table: - if str(row.get('id', '?')) == str(smart_id): - value = row.get('raw', {}).get('value', None) - return value + """Get read rate in bytes/s from dd progress output.""" + real_rate = None + if re.search(r'[KMGT]B/s', s): + human_rate = re.sub(r'^.*\s+(\d+\.?\d*)\s+(.B)/s\s*$', r'\1 \2', s) + real_rate = convert_to_bytes(human_rate) + return real_rate def get_status_color(s): - """Get color based on status, returns str.""" - color = COLORS['CLEAR'] - if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: - color = COLORS['RED'] - elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: - color = COLORS['YELLOW'] - elif s in ['CS']: - color = COLORS['GREEN'] - return color - -def menu_diags(*args): - """Main HW-Diagnostic menu.""" - diag_modes = [ - {'Name': 'All tests', - 'Tests': ['Prime95', 'NVMe/SMART', 'badblocks', 'iobenchmark']}, - {'Name': 'Prime95', - 'Tests': ['Prime95']}, - {'Name': 'All drive tests', - 'Tests': ['NVMe/SMART', 'badblocks', 'iobenchmark']}, - {'Name': 'NVMe/SMART', - 'Tests': ['NVMe/SMART']}, - {'Name': 'badblocks', - 'Tests': ['badblocks']}, - {'Name': 'I/O Benchmark', - 'Tests': ['iobenchmark']}, - {'Name': 'Quick drive test', - 'Tests': ['Quick', 'NVMe/SMART']}, - ] - actions = [ - {'Letter': 'A', 'Name': 'Audio test'}, - {'Letter': 'K', 'Name': 'Keyboard test'}, - {'Letter': 'N', 'Name': 'Network test'}, - {'Letter': 'M', 'Name': 'Screen Saver - Matrix', 'CRLF': True}, - {'Letter': 'P', 'Name': 'Screen Saver - Pipes'}, - {'Letter': 'Q', 'Name': 'Quit', 'CRLF': True}, - ] - - # CLI-mode actions - if 'DISPLAY' not in global_vars['Env']: - actions.extend([ - {'Letter': 'R', 'Name': 'Reboot', 'CRLF': True}, - {'Letter': 'S', 'Name': 'Shutdown'}, - ]) - - # Quick disk check - if 'quick' in args: - run_tests(['Quick', 'NVMe/SMART']) - exit_script() - - # Show menu - while True: - selection = menu_select( - title = 'Hardware Diagnostics: Menu', - main_entries = diag_modes, - action_entries = actions, - spacer = '──────────────────────────') - if selection.isnumeric(): - ticket_number = None - if diag_modes[int(selection)-1]['Name'] != 'Quick drive test': - clear_screen() - print_standard(' ') - ticket_number = get_ticket_number() - # Save log for non-quick tests - global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") - global_vars['LogDir'] = '{}/Logs/{}_{}'.format( - global_vars['Env']['HOME'], - ticket_number, - global_vars['Date-Time']) - os.makedirs(global_vars['LogDir'], exist_ok=True) - global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( - global_vars['LogDir']) - run_tests(diag_modes[int(selection)-1]['Tests'], ticket_number) - elif selection == 'A': - run_program(['hw-diags-audio'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') - elif selection == 'K': - run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) - elif selection == 'N': - run_program(['hw-diags-network'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') - elif selection == 'M': - run_program(['cmatrix', '-abs'], check=False, pipe=False) - elif selection == 'P': - run_program( - 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split(), - check=False, pipe=False) - elif selection == 'R': - run_program(['systemctl', 'reboot']) - elif selection == 'S': - run_program(['systemctl', 'poweroff']) - elif selection == 'Q': - break - -def run_badblocks(ticket_number): - """Run a read-only test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart badblocks test(s)\n') - progress_file = '{}/badblocks_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['badblocks']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - print_standard('Running badblock test(s):') - for name, dev in sorted(TESTS['badblocks']['Devices'].items()): - cur_status = TESTS['badblocks']['Status'][name] - nvme_smart_status = TESTS['NVMe/SMART']['Status'].get(name, None) - if cur_status == 'Denied': - # Skip denied disks - continue - if nvme_smart_status == 'NS': - TESTS['badblocks']['Status'][name] = 'Skipped' - else: - # Not testing SMART, SMART CS, or SMART OVERRIDE - TESTS['badblocks']['Status'][name] = 'Working' - update_progress() - print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - run_program('tmux split-window -dl 5 {} {} {}'.format( - 'hw-diags-badblocks', - '/dev/{}'.format(name), - progress_file).split()) - wait_for_process('badblocks') - print_standard('Done', timestamp=False) - - # Check results - if os.path.exists(progress_file): - with open(progress_file, 'r') as f: - text = f.read() - TESTS['badblocks']['Results'][name] = text - r = re.search(r'Pass completed.*0/0/0 errors', text) - if r: - TESTS['badblocks']['Status'][name] = 'CS' - else: - TESTS['badblocks']['Status'][name] = 'NS' - - # Move temp file - shutil.move(progress_file, '{}/badblocks-{}.log'.format( - global_vars['LogDir'], name)) - else: - TESTS['badblocks']['Status'][name] = 'NS' - update_progress() - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - pass - -def run_iobenchmark(ticket_number): - """Run a read-only test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart I/O Benchmark test(s)\n') - progress_file = '{}/iobenchmark_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['iobenchmark']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - print_standard('Running benchmark test(s):') - for name, dev in sorted(TESTS['iobenchmark']['Devices'].items()): - cur_status = TESTS['iobenchmark']['Status'][name] - nvme_smart_status = TESTS['NVMe/SMART']['Status'].get(name, None) - bb_status = TESTS['badblocks']['Status'].get(name, None) - if cur_status == 'Denied': - # Skip denied disks - continue - if nvme_smart_status == 'NS': - TESTS['iobenchmark']['Status'][name] = 'Skipped' - elif bb_status in ['NS', 'Skipped']: - TESTS['iobenchmark']['Status'][name] = 'Skipped' - else: - # (SMART tests not run or CS/OVERRIDE) - # AND (BADBLOCKS tests not run or CS) - TESTS['iobenchmark']['Status'][name] = 'Working' - update_progress() - print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - - # Get dev size - cmd = 'sudo lsblk -bdno size /dev/{}'.format(name) - try: - result = run_program(cmd.split()) - dev_size = result.stdout.decode().strip() - dev_size = int(dev_size) - except: - # Failed to get dev size, requires manual testing instead - TESTS['iobenchmark']['Status'][name] = 'ERROR' - continue - if dev_size < IO_VARS['Minimum Dev Size']: - TESTS['iobenchmark']['Status'][name] = 'ERROR' - continue - - # Calculate dd values - ## test_size is the area to be read in bytes - ## If the dev is < 10Gb then it's the whole dev - ## Otherwise it's the larger of 10Gb or 1% of the dev - ## - ## test_chunks is the number of groups of "Chunk Size" in test_size - ## This number is reduced to a multiple of the graph width in - ## order to allow for the data to be condensed cleanly - ## - ## skip_blocks is the number of "Block Size" groups not tested - ## skip_count is the number of blocks to skip per test_chunk - ## skip_extra is how often to add an additional skip block - ## This is needed to ensure an even testing across the dev - ## This is calculated by using the fractional amount left off - ## of the skip_count variable - test_size = min(IO_VARS['Minimum Test Size'], dev_size) - test_size = max( - test_size, dev_size*IO_VARS['Alt Test Size Factor']) - test_chunks = int(test_size // IO_VARS['Chunk Size']) - test_chunks -= test_chunks % IO_VARS['Graph Horizontal Width'] - test_size = test_chunks * IO_VARS['Chunk Size'] - skip_blocks = int((dev_size - test_size) // IO_VARS['Block Size']) - skip_count = int((skip_blocks / test_chunks) // 1) - skip_extra = 0 - try: - skip_extra = 1 + int(1 / ((skip_blocks / test_chunks) % 1)) - except ZeroDivisionError: - # skip_extra == 0 is fine - pass - - # Open dd progress pane after initializing file - with open(progress_file, 'w') as f: - f.write('') - sleep(1) - cmd = 'tmux split-window -dp 75 -PF #D tail -f {}'.format( - progress_file) - result = run_program(cmd.split()) - bottom_pane = result.stdout.decode().strip() - - # Run dd read tests - offset = 0 - TESTS['iobenchmark']['Data'][name] = { - 'Graph': [], - 'Read Rates': []} - for i in range(test_chunks): - i += 1 - s = skip_count - c = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) - if skip_extra and i % skip_extra == 0: - s += 1 - cmd = 'sudo dd bs={b} skip={s} count={c} if=/dev/{n} of={o} iflag=direct'.format( - b=IO_VARS['Block Size'], - s=offset+s, - c=c, - n=name, - o='/dev/null') - result = run_program(cmd.split()) - result_str = result.stderr.decode().replace('\n', '') - cur_rate = get_read_rate(result_str) - TESTS['iobenchmark']['Data'][name]['Read Rates'].append( - cur_rate) - TESTS['iobenchmark']['Data'][name]['Graph'].append( - '{percent:0.1f} {rate}'.format( - percent=i/test_chunks*100, - rate=int(cur_rate/(1024**2)))) - if i % IO_VARS['Progress Refresh Rate'] == 0: - # Update vertical graph - update_io_progress( - percent=i/test_chunks*100, - rate=cur_rate, - progress_file=progress_file) - # Update offset - offset += s + c - print_standard('Done', timestamp=False) - - # Close bottom pane - run_program(['tmux', 'kill-pane', '-t', bottom_pane]) - - # Build report - avg_min_max = 'Average read speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( - sum(TESTS['iobenchmark']['Data'][name]['Read Rates'])/len( - TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2), - min(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2), - max(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2)) - TESTS['iobenchmark']['Data'][name]['Avg/Min/Max'] = avg_min_max - TESTS['iobenchmark']['Data'][name]['Merged Rates'] = [] - pos = 0 - width = int(test_chunks / IO_VARS['Graph Horizontal Width']) - for i in range(IO_VARS['Graph Horizontal Width']): - # Append average rate for WIDTH number of rates to new array - TESTS['iobenchmark']['Data'][name]['Merged Rates'].append(sum( - TESTS['iobenchmark']['Data'][name]['Read Rates'][pos:pos+width])/width) - pos += width - report = generate_horizontal_graph( - TESTS['iobenchmark']['Data'][name]['Merged Rates']) - report += '\n{}'.format(avg_min_max) - TESTS['iobenchmark']['Results'][name] = report - - # Set CS/NS - min_read = min(TESTS['iobenchmark']['Data'][name]['Read Rates']) - avg_read = sum( - TESTS['iobenchmark']['Data'][name]['Read Rates'])/len( - TESTS['iobenchmark']['Data'][name]['Read Rates']) - dev_rotational = dev['lsblk'].get('rota', None) - if dev_rotational == "0": - # Use SSD scale - thresh_min = IO_VARS['Threshold SSD Min'] - thresh_high_avg = IO_VARS['Threshold SSD High Avg'] - thresh_low_avg = IO_VARS['Threshold SSD Low Avg'] - else: - # Use HDD scale - thresh_min = IO_VARS['Threshold HDD Min'] - thresh_high_avg = IO_VARS['Threshold HDD High Avg'] - thresh_low_avg = IO_VARS['Threshold HDD Low Avg'] - if min_read <= thresh_min and avg_read <= thresh_high_avg: - TESTS['iobenchmark']['Status'][name] = 'NS' - elif avg_read <= thresh_low_avg: - TESTS['iobenchmark']['Status'][name] = 'NS' - else: - TESTS['iobenchmark']['Status'][name] = 'CS' - - # Save logs - dest_filename = '{}/iobenchmark-{}.log'.format(global_vars['LogDir'], name) - shutil.move(progress_file, dest_filename) - with open(dest_filename.replace('.', '-raw.'), 'a') as f: - f.write('\n'.join(TESTS['iobenchmark']['Data'][name]['Graph'])) - update_progress() - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - pass - -def run_mprime(ticket_number): - """Run Prime95 for MPRIME_LIMIT minutes while showing the temps.""" - aborted = False - print_log('\nStart Prime95 test') - TESTS['Prime95']['Status'] = 'Working' - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dl 10 -c {wd} {cmd} {wd}'.format( - wd=global_vars['TmpDir'], cmd='hw-diags-prime95').split()) - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - run_program('tmux split-window -bd watch -c -n1 -t hw-sensors'.split()) - run_program('tmux resize-pane -y 3'.split()) - - # Start test - run_program(['apple-fans', 'max']) - try: - for i in range(int(MPRIME_LIMIT)): - clear_screen() - min_left = int(MPRIME_LIMIT) - i - print_standard('Running Prime95 ({} minute{} left)'.format( - min_left, - 's' if min_left != 1 else '')) - print_warning('If running too hot, press CTRL+c to abort the test') - sleep(60) - except KeyboardInterrupt: - # Catch CTRL+C - aborted = True - TESTS['Prime95']['Status'] = 'Aborted' - print_warning('\nAborted.') - update_progress() - - # Save "final" temps - run_program( - cmd = 'hw-sensors >> "{}/Final Temps.out"'.format( - global_vars['LogDir']).split(), - check = False, - pipe = False, - shell = True) - run_program( - cmd = 'hw-sensors --nocolor >> "{}/Final Temps.log"'.format( - global_vars['LogDir']).split(), - check = False, - pipe = False, - shell = True) - - # Stop test - run_program('killall -s INT mprime'.split(), check=False) - run_program(['apple-fans', 'auto']) - - # Move logs to Ticket folder - for item in os.scandir(global_vars['TmpDir']): - try: - shutil.move(item.path, global_vars['LogDir']) - except Exception: - print_error('ERROR: Failed to move "{}" to "{}"'.format( - item.path, - global_vars['LogDir'])) - - # Check logs - TESTS['Prime95']['NS'] = False - TESTS['Prime95']['CS'] = False - log = '{}/results.txt'.format(global_vars['LogDir']) - if os.path.exists(log): - with open(log, 'r') as f: - text = f.read() - TESTS['Prime95']['results.txt'] = text - r = re.search(r'(error|fail)', text) - TESTS['Prime95']['NS'] = bool(r) - log = '{}/prime.log'.format(global_vars['LogDir']) - if os.path.exists(log): - with open(log, 'r') as f: - text = f.read() - TESTS['Prime95']['prime.log'] = text - r = re.search(r'completed.*0 errors, 0 warnings', text) - TESTS['Prime95']['CS'] = bool(r) - - # Update status - if not aborted: - if TESTS['Prime95']['NS']: - TESTS['Prime95']['Status'] = 'NS' - elif TESTS['Prime95']['CS']: - TESTS['Prime95']['Status'] = 'CS' - else: - TESTS['Prime95']['Status'] = 'Unknown' - update_progress() - - if aborted: - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled']: - if not ask('Proceed to next test?'): - for name in TESTS['NVMe/SMART']['Devices'].keys(): - for t in ['NVMe/SMART', 'badblocks', 'iobenchmark']: - cur_status = TESTS[t]['Status'][name] - if cur_status not in ['CS', 'Denied', 'NS']: - TESTS[t]['Status'][name] = 'Aborted' - run_program('tmux kill-pane -a'.split()) - raise GenericError - - # Done - run_program('tmux kill-pane -a'.split()) - -def run_nvme_smart(ticket_number): - """Run the built-in NVMe or SMART test for all detected disks.""" - aborted = False - clear_screen() - print_log('\nStart NVMe/SMART test(s)\n') - progress_file = '{}/selftest_progress.out'.format(global_vars['LogDir']) - update_progress() - - # Set Window layout and start test - run_program('tmux split-window -dl 3 watch -c -n1 -t cat {}'.format( - progress_file).split()) - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Show disk details - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - show_disk_details(dev) - print_standard(' ') - update_progress() - - # Run - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - TESTS['NVMe/SMART']['Short Test'][name] = None - cur_status = TESTS['NVMe/SMART']['Status'][name] - if cur_status == 'OVERRIDE': - # Skipping test per user request - continue - if TESTS['NVMe/SMART']['Quick'] or dev.get('NVMe Disk', False): - # Skip SMART self-tests for quick checks and NVMe disks - if dev['Quick Health OK']: - TESTS['NVMe/SMART']['Status'][name] = 'CS' - else: - TESTS['NVMe/SMART']['Status'][name] = 'NS' - elif not dev['Quick Health OK']: - # SMART overall == Failed or attributes bad, avoid self-test - TESTS['NVMe/SMART']['Status'][name] = 'NS' - else: - # Start SMART short self-test - test_length = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'polling_minutes', {}).get( - 'short', 5) - test_length = int(test_length) + 5 - TESTS['NVMe/SMART']['Status'][name] = 'Working' - update_progress() - print_standard('Running SMART short self-test(s):') - print_standard( - ' /dev/{:8}({} minutes)... '.format(name, test_length), - end='', flush=True) - run_program( - 'sudo smartctl -t short /dev/{}'.format(name).split(), - check=False) - - # Wait and show progress (in 10 second increments) - for iteration in range(int(test_length*60/10)): - # Update SMART data - dev['smartctl'] = get_smart_details(name) - - # Check if test is complete - if iteration >= 6: - done = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'passed', False) - if done: - break - - # Update progress_file - with open(progress_file, 'w') as f: - f.write('SMART self-test status:\n {}'.format( - dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'string', 'unknown'))) - sleep(10) - os.remove(progress_file) - - # Check result - test_passed = dev['smartctl'].get( - 'ata_smart_data', {}).get( - 'self_test', {}).get( - 'status', {}).get( - 'passed', False) - if test_passed: - TESTS['NVMe/SMART']['Status'][name] = 'CS' - TESTS['NVMe/SMART']['Short Test'][name] = 'CS' - else: - TESTS['NVMe/SMART']['Status'][name] = 'NS' - TESTS['NVMe/SMART']['Short Test'][name] = 'NS' - update_progress() - print_standard('Done', timestamp=False) - - # Done - run_program('tmux kill-pane -a'.split(), check=False) - -def run_tests(tests, ticket_number=None): - """Run selected hardware test(s).""" - clear_screen() - print_standard('Starting Hardware Diagnostics') - if ticket_number: - print_standard(' For Ticket #{}'.format(ticket_number)) - print_standard(' ') - print_standard('Running tests: {}'.format(', '.join(tests))) - # Enable selected tests - for t in ['Prime95', 'NVMe/SMART', 'badblocks', 'iobenchmark']: - TESTS[t]['Enabled'] = t in tests - TESTS['NVMe/SMART']['Quick'] = 'Quick' in tests - - # Initialize - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - print_standard(' ') - scan_disks() - update_progress() - - # Run - mprime_aborted = False - if TESTS['Prime95']['Enabled']: - try: - run_mprime(ticket_number) - except GenericError: - mprime_aborted = True - if not mprime_aborted: - if TESTS['NVMe/SMART']['Enabled']: - run_nvme_smart(ticket_number) - if TESTS['badblocks']['Enabled']: - run_badblocks(ticket_number) - if TESTS['iobenchmark']['Enabled']: - run_iobenchmark(ticket_number) - - # Show results - show_results() - - # Open log - if not TESTS['NVMe/SMART']['Quick'] and ENABLED_OPEN_LOGS: - try: - popen_program(['nohup', 'leafpad', global_vars['LogFile']], pipe=True) - except Exception: - print_error('ERROR: Failed to open log: {}'.format( - global_vars['LogFile'])) - pause('Press Enter to exit...') - -def scan_disks(full_paths=False, only_path=None): - """Scan for disks eligible for hardware testing.""" - - # Get eligible disk list - cmd = ['lsblk', '-J', '-O'] - if full_paths: - cmd.append('-p') - if only_path: - cmd.append(only_path) - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - devs = {} - for d in json_data.get('blockdevices', []): - if d['type'] == 'disk': - if d['hotplug'] == '0': - devs[d['name']] = {'lsblk': d} - TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' - TESTS['badblocks']['Status'][d['name']] = 'Pending' - TESTS['iobenchmark']['Status'][d['name']] = 'Pending' - else: - # Skip WizardKit devices - skip_dev=False - wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) - for c in d.get('children', []): - r = re.search( - wk_label_regex, c.get('label', ''), re.IGNORECASE) - skip_dev = bool(r) - if not skip_dev: - devs[d['name']] = {'lsblk': d} - TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' - TESTS['badblocks']['Status'][d['name']] = 'Pending' - TESTS['iobenchmark']['Status'][d['name']] = 'Pending' - - for dev, data in devs.items(): - # Get SMART attributes - run_program( - cmd = 'sudo smartctl -s on {}{}'.format( - '' if full_paths else '/dev/', - dev).split(), - check = False) - data['smartctl'] = get_smart_details(dev) - - # Get NVMe attributes - if data['lsblk']['tran'] == 'nvme': - cmd = 'sudo nvme smart-log /dev/{} -o json'.format(dev).split() - cmd = 'sudo nvme smart-log {}{} -o json'.format( - '' if full_paths else '/dev/', - dev).split() - result = run_program(cmd, check=False) - try: - data['nvme-cli'] = json.loads(result.stdout.decode()) - except Exception: - # Let other sections deal with the missing data - data['nvme-cli'] = {} - data['NVMe Disk'] = True - - # Set "Quick Health OK" value - ## NOTE: If False then require override for badblocks test - wanted_smart_list = [ - 'ata_smart_attributes', - 'ata_smart_data', - 'smart_status', - ] - if data.get('NVMe Disk', False): - crit_warn = data['nvme-cli'].get('critical_warning', 1) - if crit_warn == 0: - dev_name = data['lsblk']['name'] - data['Quick Health OK'] = True - TESTS['NVMe/SMART']['Status'][dev_name] = 'CS' - else: - data['Quick Health OK'] = False - elif set(wanted_smart_list).issubset(data['smartctl'].keys()): - data['SMART Pass'] = data['smartctl'].get('smart_status', {}).get( - 'passed', False) - data['Quick Health OK'] = data['SMART Pass'] - data['SMART Support'] = True - else: - data['Quick Health OK'] = False - data['SMART Support'] = False - - # Ask for manual overrides if necessary - if TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - show_disk_details(data) - needs_override = False - if not data['Quick Health OK']: - needs_override = True - print_warning( - "WARNING: Health can't be confirmed for: /dev/{}".format(dev)) - if get_smart_value(data['smartctl'], '199'): - # SMART attribute present and it's value is non-zero - needs_override = True - print_warning( - 'WARNING: SMART 199/C7 error detected on /dev/{}'.format(dev)) - print_standard(' (Have you tried swapping the drive cable?)') - if needs_override: - dev_name = data['lsblk']['name'] - print_standard(' ') - if ask('Run tests on this device anyway?'): - TESTS['NVMe/SMART']['Status'][dev_name] = 'OVERRIDE' - else: - TESTS['NVMe/SMART']['Status'][dev_name] = 'Skipped' - TESTS['badblocks']['Status'][dev_name] = 'Denied' - TESTS['iobenchmark']['Status'][dev_name] = 'Denied' - print_standard(' ') # In case there's more than one "OVERRIDE" disk - - TESTS['NVMe/SMART']['Devices'] = devs - TESTS['badblocks']['Devices'] = devs - TESTS['iobenchmark']['Devices'] = devs - return devs - -def show_disk_details(dev, only_attributes=False): - """Display disk details.""" - dev_name = dev['lsblk']['name'] - if not only_attributes: - # Device description - print_info('Device: {}{}'.format( - '' if '/dev/' in dev['lsblk']['name'] else '/dev/', - dev['lsblk']['name'])) - print_standard(' {:>4} ({}) {} {}'.format( - str(dev['lsblk'].get('size', '???b')).strip(), - str(dev['lsblk'].get('tran', '???')).strip().upper().replace( - 'NVME', 'NVMe'), - str(dev['lsblk'].get('model', 'Unknown Model')).strip(), - str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(), - )) - - # Warnings - if dev.get('NVMe Disk', False): - if dev['Quick Health OK']: - print_warning('WARNING: NVMe support is still experimental') - else: - print_error('ERROR: NVMe disk is reporting critical warnings') - elif not dev['SMART Support']: - print_error('ERROR: Unable to retrieve SMART data') - elif not dev['SMART Pass']: - print_error('ERROR: SMART overall-health assessment result: FAILED') - - # Attributes - if dev.get('NVMe Disk', False): - if only_attributes: - print_info('SMART Attributes:', end='') - print_warning(' Updated: {}'.format( - time.strftime('%Y-%m-%d %H:%M %Z'))) - else: - print_info('Attributes:') - for attrib, threshold in sorted(ATTRIBUTES['NVMe'].items()): - if attrib in dev['nvme-cli']: - print_standard( - ' {:37}'.format(attrib.replace('_', ' ').title()), - end='', flush=True) - raw_num = dev['nvme-cli'][attrib] - raw_str = str(raw_num) - if (threshold.get('Error', False) and - raw_num >= threshold.get('Error', -1)): - print_error(raw_str, timestamp=False) - if not threshold.get('Ignore', False): - dev['Quick Health OK'] = False - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - elif (threshold.get('Warning', False) and - raw_num >= threshold.get('Warning', -1)): - print_warning(raw_str, timestamp=False) - else: - print_success(raw_str, timestamp=False) - elif dev['smartctl'].get('ata_smart_attributes', None): - # SMART attributes - if only_attributes: - print_info('SMART Attributes:', end='') - print_warning(' Updated: {}'.format( - time.strftime('%Y-%m-%d %H:%M %Z'))) - else: - print_info('Attributes:') - s_table = dev['smartctl'].get('ata_smart_attributes', {}).get( - 'table', {}) - s_table = {a.get('id', 'Unknown'): a for a in s_table} - for attrib, threshold in sorted(ATTRIBUTES['SMART'].items()): - if attrib in s_table: - print_standard( - ' {:>3} {:32}'.format( - attrib, - s_table[attrib]['name']).replace('_', ' ').title(), - end='', flush=True) - raw_str = s_table[attrib]['raw']['string'] - raw_num = re.sub(r'^(\d+).*$', r'\1', raw_str) - try: - raw_num = float(raw_num) - except ValueError: - # Not sure about this one, print raw_str without color? - print_standard(raw_str, timestamp=False) - continue - if (threshold.get('Error', False) and - raw_num >= threshold.get('Error', -1)): - print_error(raw_str, timestamp=False) - if not threshold.get('Ignore', False): - dev['Quick Health OK'] = False - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - elif (threshold.get('Warning', False) and - raw_num >= threshold.get('Warning', -1)): - print_warning(raw_str, timestamp=False) - else: - print_success(raw_str, timestamp=False) - -def show_results(): - """Show results for selected test(s).""" - clear_screen() - print_log('\n───────────────────────────') - print_standard('Hardware Diagnostic Results') - update_progress() - - # Set Window layout and show progress - run_program('tmux split-window -dhl 15 watch -c -n1 -t cat {}'.format( - TESTS['Progress Out']).split()) - - # Prime95 - if TESTS['Prime95']['Enabled']: - print_success('\nPrime95:') - for log, regex in [ - ['results.txt', r'(error|fail)'], - ['prime.log', r'completed.*0 errors, 0 warnings']]: - if log in TESTS['Prime95']: - print_info('Log: {}'.format(log)) - lines = [line.strip() for line - in TESTS['Prime95'][log].splitlines() - if re.search(regex, line, re.IGNORECASE)] - for line in lines[-4:]: - line = re.sub(r'^.*Worker #\d.*Torture Test (.*)', r'\1', - line, re.IGNORECASE) - if TESTS['Prime95'].get('NS', False): - print_error(' {}'.format(line)) - else: - print_standard(' {}'.format(line)) - print_info('Final temps') - print_log(' See Final Temps.log') - with open('{}/Final Temps.out'.format(global_vars['LogDir']), 'r') as f: - for line in f.readlines(): - if re.search(r'^\s*$', line.strip()): - # Stop after coretemps (which should be first) - break - print(' {}'.format(line.strip())) - print_standard(' ') - - # NVMe/SMART / badblocks / iobenchmark - if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: - print_success('Disks:') - for name, dev in sorted(TESTS['NVMe/SMART']['Devices'].items()): - show_disk_details(dev) - bb_status = TESTS['badblocks']['Status'].get(name, None) - if (TESTS['badblocks']['Enabled'] - and bb_status not in ['Denied', 'OVERRIDE', 'Skipped']): - print_info('badblocks:') - result = TESTS['badblocks']['Results'].get(name, '') - for line in result.splitlines(): - if re.search(r'Pass completed', line, re.IGNORECASE): - line = re.sub( - r'Pass completed,?\s+', r'', - line.strip(), re.IGNORECASE) - if TESTS['badblocks']['Status'][name] == 'CS': - print_standard(' {}'.format(line)) - else: - print_error(' {}'.format(line)) - io_status = TESTS['iobenchmark']['Status'].get(name, None) - if (TESTS['iobenchmark']['Enabled'] - and io_status not in ['Denied', 'OVERRIDE', 'Skipped']): - print_info('Benchmark:') - result = TESTS['iobenchmark']['Results'].get(name, '') - for line in result.split('\n'): - print_standard(' {}'.format(line)) - print_standard(' ') - - # Done - pause('Press Enter to return to main menu... ') - run_program('tmux kill-pane -a'.split()) + """Get color based on status, returns str.""" + color = COLORS['CLEAR'] + if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + color = COLORS['RED'] + elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: + color = COLORS['YELLOW'] + elif s in ['CS']: + color = COLORS['GREEN'] + return color def update_io_progress(percent, rate, progress_file): - """Update I/O progress file.""" - bar_color = COLORS['CLEAR'] - rate_color = COLORS['CLEAR'] - step = get_graph_step(rate, scale=32) - if rate < IO_VARS['Threshold Graph Fail']: - bar_color = COLORS['RED'] - rate_color = COLORS['YELLOW'] - elif rate < IO_VARS['Threshold Graph Warn']: - bar_color = COLORS['YELLOW'] - rate_color = COLORS['YELLOW'] - elif rate > IO_VARS['Threshold Graph Great']: - bar_color = COLORS['GREEN'] - rate_color = COLORS['GREEN'] - line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( - p=percent, - b_color=bar_color, - b=IO_VARS['Graph Vertical'][step], - r_color=rate_color, - r=rate/(1024**2), - c=COLORS['CLEAR']) - with open(progress_file, 'a') as f: - f.write(line) - -def update_progress(): - """Update progress file.""" - if 'Progress Out' not in TESTS: - TESTS['Progress Out'] = '{}/progress.out'.format(global_vars['LogDir']) - output = [] - output.append('{BLUE}HW Diagnostics{CLEAR}'.format(**COLORS)) - output.append('───────────────') - if TESTS['Prime95']['Enabled']: - output.append(' ') - output.append('{BLUE}Prime95{s_color}{status:>8}{CLEAR}'.format( - s_color = get_status_color(TESTS['Prime95']['Status']), - status = TESTS['Prime95']['Status'], - **COLORS)) - if TESTS['NVMe/SMART']['Enabled']: - output.append(' ') - output.append('{BLUE}NVMe / SMART{CLEAR}'.format(**COLORS)) - if TESTS['NVMe/SMART']['Quick']: - output.append('{YELLOW} (Quick Check){CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['NVMe/SMART']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - if TESTS['badblocks']['Enabled']: - output.append(' ') - output.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['badblocks']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - if TESTS['iobenchmark']['Enabled']: - output.append(' ') - output.append('{BLUE}I/O Benchmark{CLEAR}'.format(**COLORS)) - for dev, status in sorted(TESTS['iobenchmark']['Status'].items()): - output.append('{dev}{s_color}{status:>{pad}}{CLEAR}'.format( - dev = dev, - pad = 15-len(dev), - s_color = get_status_color(status), - status = status, - **COLORS)) - - # Add line-endings - output = ['{}\n'.format(line) for line in output] - - with open(TESTS['Progress Out'], 'w') as f: - f.writelines(output) + """Update I/O progress file.""" + bar_color = COLORS['CLEAR'] + rate_color = COLORS['CLEAR'] + step = get_graph_step(rate, scale=32) + if rate < IO_VARS['Threshold Graph Fail']: + bar_color = COLORS['RED'] + rate_color = COLORS['YELLOW'] + elif rate < IO_VARS['Threshold Graph Warn']: + bar_color = COLORS['YELLOW'] + rate_color = COLORS['YELLOW'] + elif rate > IO_VARS['Threshold Graph Great']: + bar_color = COLORS['GREEN'] + rate_color = COLORS['GREEN'] + line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( + p=percent, + b_color=bar_color, + b=IO_VARS['Graph Vertical'][step], + r_color=rate_color, + r=rate/(1024**2), + c=COLORS['CLEAR']) + with open(progress_file, 'a') as f: + f.write(line) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 From 3fdd8c629c8afd6c2ded1a52f821e63d548145ab Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:47:44 -0700 Subject: [PATCH 002/186] Rewrote main menu * First options are presets followed by individual tests * Selecting presets will toggle the selections * Screensavers are hidden but still present --- .bin/Scripts/functions/common.py | 6 +- .bin/Scripts/functions/hw_diags.py | 168 +++++++++++++++++++++++++++++ .bin/Scripts/hw-diags-menu | 3 +- 3 files changed, 174 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index ae958645..d3f04ef6 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -318,7 +318,7 @@ def major_exception(): exit_script(1) def menu_select(title='~ Untitled Menu ~', - prompt='Please make a selection', secret_exit=False, + prompt='Please make a selection', secret_actions=[], secret_exit=False, main_entries=[], action_entries=[], disabled_label='DISABLED', spacer=''): """Display options in a menu and return selected option as a str.""" @@ -334,8 +334,10 @@ def menu_select(title='~ Untitled Menu ~', menu_splash = '{}\n{}\n'.format(title, spacer) width = len(str(len(main_entries))) valid_answers = [] - if (secret_exit): + if secret_exit: valid_answers.append('Q') + if secret_actions: + valid_answers.extend(secret_actions) # Add main entries for i in range(len(main_entries)): diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 830d948a..d32fb499 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -216,6 +216,174 @@ def get_status_color(s): color = COLORS['GREEN'] return color +def menu_diags(state, args): + """Main menu to select and run HW tests.""" + args = [a.lower() for a in args] + quick_label = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) + title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( + **COLORS) + # NOTE: Changing the order of main_options will break everything + main_options = [ + {'Base Name': 'Full Diagnostic', 'Enabled': True}, + {'Base Name': 'Drive Diagnostic', 'Enabled': False}, + {'Base Name': 'Drive Diagnostic (Quick)', 'Enabled': False}, + {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, + {'Base Name': 'NVMe / SMART', 'Enabled': True}, + {'Base Name': 'badblocks', 'Enabled': True}, + {'Base Name': 'I/O Benchmark', 'Enabled': True}, + ] + actions = [ + {'Letter': 'A', 'Name': 'Audio Test'}, + {'Letter': 'K', 'Name': 'Keyboard Test'}, + {'Letter': 'N', 'Name': 'Network Test'}, + {'Letter': 'S', 'Name': 'Start', 'CRLF': True}, + {'Letter': 'Q', 'Name': 'Quit'}, + ] + secret_actions = ['M', 'T'] + + # CLI mode check + if '--cli' in args or 'DISPLAY' not in global_vars['Env']: + actions.append({'Letter': 'R', 'Name': 'Reboot'}) + actions.append({'Letter': 'P', 'Name': 'Power Off'}) + + while True: + # Set quick mode as necessary + if main_options[2]['Enabled'] and main_options[4]['Enabled']: + # Check if only Drive Diags (Quick) and NVMe/SMART are enabled + # If so, verify no other tests are enabled and set quick_mode + state.quick_mode = True + for opt in main_options[3:4] + main_options[5:]: + state.quick_mode &= not opt['Enabled'] + else: + state.quick_mode = False + + # Deselect presets + slice_end = 3 + if state.quick_mode: + slice_end = 2 + for opt in main_options[:slice_end]: + opt['Enabled'] = False + + # Verify preset selections + num_tests_selected = 0 + for opt in main_options[3:]: + if opt['Enabled']: + num_tests_selected += 1 + if num_tests_selected == 4: + # Full + main_options[0]['Enabled'] = True + elif num_tests_selected == 3 and not main_options[3]['Enabled']: + # Drive + main_options[1]['Enabled'] = True + + # Update checkboxes + for opt in main_options: + _nvme_smart = opt['Base Name'] == 'NVMe / SMART' + opt['Name'] = '{} {} {}'.format( + '[✓]' if opt['Enabled'] else '[ ]', + opt['Base Name'], + quick_label if state.quick_mode and _nvme_smart else '') + + # Show menu + selection = menu_select( + title=title, + main_entries=main_options, + action_entries=actions, + secret_actions=secret_actions, + spacer='───────────────────────────────') + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + + # Handle presets + if index == 0: + # Full + if main_options[index]['Enabled']: + for opt in main_options[1:3]: + opt['Enabled'] = False + for opt in main_options[3:]: + opt['Enabled'] = True + else: + for opt in main_options[3:]: + opt['Enabled'] = False + elif index == 1: + # Drive + if main_options[index]['Enabled']: + main_options[0]['Enabled'] = False + for opt in main_options[2:4]: + opt['Enabled'] = False + for opt in main_options[4:]: + opt['Enabled'] = True + else: + for opt in main_options[4:]: + opt['Enabled'] = False + elif index == 2: + # Drive (Quick) + if main_options[index]['Enabled']: + for opt in main_options[:2] + main_options[3:]: + opt['Enabled'] = False + main_options[4]['Enabled'] = True + else: + main_options[4]['Enabled'] = False + elif selection == 'A': + run_audio_test() + elif selection == 'K': + run_keyboard_test() + elif selection == 'N': + run_network_test() + elif selection == 'M': + secret_screensaver('matrix') + elif selection == 'T': + # Tubes is close to pipes + secret_screensaver('pipes') + elif selection == 'R': + run_program(['systemctl', 'reboot']) + elif selection == 'P': + run_program(['systemctl', 'poweroff']) + elif selection == 'Q': + break + elif selection == 'S': + # Run test(s) + clear_screen() + print('Fake test(s) placeholder for now...') + pause('Press Enter to return to main menu... ') + +def run_audio_test(): + """Run audio test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake audio placeholder for now...') + #run_program(['hw-diags-audio'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def run_keyboard_test(): + """Run keyboard test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake keyboard placeholder for now...') + #run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def run_network_test(): + """Run network test.""" + # TODO: Enable real test and remove placeholder + clear_screen() + print('Fake network placeholder for now...') + #run_program(['hw-diags-network'], check=False, pipe=False) + pause('Press Enter to return to main menu... ') + +def secret_screensaver(screensaver=None): + """Show screensaver.""" + if screensaver == 'matrix': + cmd = 'cmatrix -abs'.split() + elif screensaver == 'pipes': + cmd = 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split() + else: + raise Exception('Invalid screensaver') + run_program(cmd, check=False, pipe=False) + def update_io_progress(percent, rate, progress_file): """Update I/O progress file.""" bar_color = COLORS['CLEAR'] diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index f7c1739c..c67cc5f4 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -17,7 +17,8 @@ if __name__ == '__main__': clear_screen() # Show menu - menu_diags(*sys.argv) + state = State() + menu_diags(state, sys.argv) # Done #print_standard('\nDone.') From 18fc97293e546d50e2606e055a8d1ec57be04913 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:50:55 -0700 Subject: [PATCH 003/186] Renamed Drive to Disk to align options in menu --- .bin/Scripts/functions/hw_diags.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d32fb499..b5d553de 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -225,8 +225,8 @@ def menu_diags(state, args): # NOTE: Changing the order of main_options will break everything main_options = [ {'Base Name': 'Full Diagnostic', 'Enabled': True}, - {'Base Name': 'Drive Diagnostic', 'Enabled': False}, - {'Base Name': 'Drive Diagnostic (Quick)', 'Enabled': False}, + {'Base Name': 'Disk Diagnostic', 'Enabled': False}, + {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, {'Base Name': 'NVMe / SMART', 'Enabled': True}, {'Base Name': 'badblocks', 'Enabled': True}, @@ -249,7 +249,7 @@ def menu_diags(state, args): while True: # Set quick mode as necessary if main_options[2]['Enabled'] and main_options[4]['Enabled']: - # Check if only Drive Diags (Quick) and NVMe/SMART are enabled + # Check if only Disk Diags (Quick) and NVMe/SMART are enabled # If so, verify no other tests are enabled and set quick_mode state.quick_mode = True for opt in main_options[3:4] + main_options[5:]: @@ -273,7 +273,7 @@ def menu_diags(state, args): # Full main_options[0]['Enabled'] = True elif num_tests_selected == 3 and not main_options[3]['Enabled']: - # Drive + # Disk main_options[1]['Enabled'] = True # Update checkboxes @@ -309,7 +309,7 @@ def menu_diags(state, args): for opt in main_options[3:]: opt['Enabled'] = False elif index == 1: - # Drive + # Disk if main_options[index]['Enabled']: main_options[0]['Enabled'] = False for opt in main_options[2:4]: @@ -320,7 +320,7 @@ def menu_diags(state, args): for opt in main_options[4:]: opt['Enabled'] = False elif index == 2: - # Drive (Quick) + # Disk (Quick) if main_options[index]['Enabled']: for opt in main_options[:2] + main_options[3:]: opt['Enabled'] = False From 560929e2fa74ba35301758d17bfbfbfc526f1ad9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 19:54:06 -0700 Subject: [PATCH 004/186] Removed extra line break in menu_select --- .bin/Scripts/functions/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index d3f04ef6..b5896966 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -369,7 +369,6 @@ def menu_select(title='~ Untitled Menu ~', letter = entry['Letter'].upper(), width = len(str(len(action_entries))), name = entry['Name']) - menu_splash += '\n' answer = '' From 2df4d48bb3af0f424fd1248a5e89af24035d588f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Dec 2018 20:15:56 -0700 Subject: [PATCH 005/186] Show selected tests on run --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index b5d553de..3e08c9ed 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -118,10 +118,10 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.started = False self.tests = { - 'Prime95': {'Enabled': False, 'Result': None, 'Status': None}, - 'NVMe / SMART': {'Enabled': False}, - 'badblocks': {'Enabled': False}, - 'I/O Benchmark': {'Enabled': False}, + 'Prime95 & Temps': {'Enabled': False, 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False}, + 'badblocks': {'Enabled': False}, + 'I/O Benchmark': {'Enabled': False}, } self.add_devs() @@ -347,7 +347,18 @@ def menu_diags(state, args): elif selection == 'S': # Run test(s) clear_screen() - print('Fake test(s) placeholder for now...') + print('Tests:') + for opt in main_options[3:]: + _nvme_smart = opt['Base Name'] == 'NVMe / SMART' + # Update state + state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] + print(' {:<15} {}{}{} {}'.format( + opt['Base Name'], + COLORS['GREEN'] if opt['Enabled'] else COLORS['RED'], + 'Enabled' if opt['Enabled'] else 'Disabled', + COLORS['CLEAR'], + quick_label if state.quick_mode and _nvme_smart else '')) + print('\nFake test(s) placeholder for now...') pause('Press Enter to return to main menu... ') def run_audio_test(): From 70a742e69c832484df6c5c1b6f70e5e6c1abde74 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:10:58 -0700 Subject: [PATCH 006/186] Add device details from lsblk * Also ensure sane types for some attributes --- .bin/Scripts/functions/hw_diags.py | 52 ++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3e08c9ed..5bb03c9b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -68,10 +68,14 @@ class DevObj(): """Device object for tracking device specific data.""" def __init__(self, dev_path): self.failing = False + self.labels = [] + self.lsblk = {} + self.name = re.sub(r'^.*/(.*)', r'\1', dev_path) self.nvme_attributes = {} self.override = False self.path = dev_path self.smart_attributes = {} + self.smartctl = {} self.tests = { 'NVMe / SMART': {'Result': None, 'Status': None}, 'badblocks': {'Result': None, 'Status': None}, @@ -82,18 +86,54 @@ class DevObj(): 'Graph Data': []}, } self.get_details() + self.get_smart_details() def get_details(self): + """Get data from lsblk.""" + cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + self.lsblk = json_data['blockdevices'][0] + except Exception: + # Leave self.lsblk empty + pass + + # Set necessary details + self.lsblk['model'] = self.lsblk.get('model', 'Unknown Model') + self.lsblk['name'] = self.lsblk.get('name', self.path) + self.lsblk['rota'] = self.lsblk.get('rota', True) + self.lsblk['serial'] = self.lsblk.get('serial', 'Unknown Serial') + self.lsblk['size'] = self.lsblk.get('size', '???b') + self.lsblk['tran'] = self.lsblk.get('tran', '???') + + # Ensure certain attributes are strings + for attr in ['model', 'name', 'rota', 'serial', 'size', 'tran']: + if not isinstance(self.lsblk[attr], str): + self.lsblk[attr] = str(self.lsblk[attr]) + self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') + + # Build list of labels + for dev in [self.lsblk, *self.lsblk.get('children', [])]: + self.labels.append(dev.get('label', '')) + self.labels.append(dev.get('partlabel', '')) + self.labels = [str(label) for label in self.labels if label] + + def get_smart_details(self): """Get data from smartctl.""" cmd = ['sudo', 'smartctl', '--all', '--json', self.path] - result = run_program(cmd, check=False) - self.data = json.loads(result.stdout.decode()) + try: + result = run_program(cmd, check=False) + self.smartctl = json.loads(result.stdout.decode()) + except Exception: + # Leave self.smartctl empty + pass # Check for attributes - if KEY_NVME in self.data: - self.nvme_attributes.update(self.data[KEY_NVME]) - elif KEY_SMART in self.data: - for a in self.data[KEY_SMART].get('table', {}): + if KEY_NVME in self.smartctl: + self.nvme_attributes.update(self.smartctl[KEY_NVME]) + elif KEY_SMART in self.smartctl: + for a in self.smartctl[KEY_SMART].get('table', {}): _id = str(a.get('id', 'UNKNOWN')) _name = str(a.get('name', 'UNKNOWN')) _raw = a.get('raw', {}).get('value', -1) From 6014a8fb70275d7be2b8b6a7f6c548b7ae718161 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:18:45 -0700 Subject: [PATCH 007/186] Don't add WK or loopback devices --- .bin/Scripts/functions/hw_diags.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 5bb03c9b..8f543aa5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -171,7 +171,22 @@ class State(): result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) for dev in json_data['blockdevices']: - self.devs.append(DevObj(dev['name'])) + skip_dev = False + dev_obj = DevObj(dev['name']) + + # Skip loopback devices + if dev_obj.lsblk['tran'] == 'NONE': + skip_dev = True + + # Skip WK devices + wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) + for label in dev_obj.labels: + if re.search(wk_label_regex, label, re.IGNORECASE): + skip_dev = True + + # Add device + if not skip_dev: + self.devs.append(DevObj(dev['name'])) # Functions def generate_horizontal_graph(rates, oneline=False): From 5701b53026aa758c879db61454fcff9326a67e73 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 16:55:17 -0700 Subject: [PATCH 008/186] Added --quick argument to skip menu --- .bin/Scripts/functions/hw_diags.py | 147 +++++++++++++++++------------ 1 file changed, 86 insertions(+), 61 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8f543aa5..3de0806f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -61,6 +61,7 @@ IO_VARS = { } KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' +QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PATH_WIDTH = 21 # Classes @@ -156,12 +157,14 @@ class State(): self.devs = [] self.finished = False self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.quick_mode = False self.started = False self.tests = { - 'Prime95 & Temps': {'Enabled': False, 'Result': None, 'Status': None}, - 'NVMe / SMART': {'Enabled': False}, - 'badblocks': {'Enabled': False}, - 'I/O Benchmark': {'Enabled': False}, + 'Prime95 & Temps': {'Enabled': False, 'Order': 1, + 'Result': None, 'Status': None}, + 'NVMe / SMART': {'Enabled': False, 'Order': 2}, + 'badblocks': {'Enabled': False, 'Order': 3}, + 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } self.add_devs() @@ -274,18 +277,17 @@ def get_status_color(s): def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] - quick_label = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( **COLORS) # NOTE: Changing the order of main_options will break everything main_options = [ - {'Base Name': 'Full Diagnostic', 'Enabled': True}, + {'Base Name': 'Full Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, - {'Base Name': 'Prime95 & Temps', 'Enabled': True, 'CRLF': True}, - {'Base Name': 'NVMe / SMART', 'Enabled': True}, - {'Base Name': 'badblocks', 'Enabled': True}, - {'Base Name': 'I/O Benchmark', 'Enabled': True}, + {'Base Name': 'Prime95 & Temps', 'Enabled': False, 'CRLF': True}, + {'Base Name': 'NVMe / SMART', 'Enabled': False}, + {'Base Name': 'badblocks', 'Enabled': False}, + {'Base Name': 'I/O Benchmark', 'Enabled': False}, ] actions = [ {'Letter': 'A', 'Name': 'Audio Test'}, @@ -295,12 +297,22 @@ def menu_diags(state, args): {'Letter': 'Q', 'Name': 'Quit'}, ] secret_actions = ['M', 'T'] + + # Set initial selections + update_main_options(state, '1', main_options) # CLI mode check if '--cli' in args or 'DISPLAY' not in global_vars['Env']: actions.append({'Letter': 'R', 'Name': 'Reboot'}) actions.append({'Letter': 'P', 'Name': 'Power Off'}) + # Skip menu if running quick check + if '--quick' in args: + update_main_options(state, '3', main_options) + state.quick_mode = True + run_hw_tests(state) + return True + while True: # Set quick mode as necessary if main_options[2]['Enabled'] and main_options[4]['Enabled']: @@ -337,7 +349,7 @@ def menu_diags(state, args): opt['Name'] = '{} {} {}'.format( '[✓]' if opt['Enabled'] else '[ ]', opt['Base Name'], - quick_label if state.quick_mode and _nvme_smart else '') + QUICK_LABEL if state.quick_mode and _nvme_smart else '') # Show menu selection = menu_select( @@ -348,40 +360,7 @@ def menu_diags(state, args): spacer='───────────────────────────────') if selection.isnumeric(): - # Toggle selection - index = int(selection) - 1 - main_options[index]['Enabled'] = not main_options[index]['Enabled'] - - # Handle presets - if index == 0: - # Full - if main_options[index]['Enabled']: - for opt in main_options[1:3]: - opt['Enabled'] = False - for opt in main_options[3:]: - opt['Enabled'] = True - else: - for opt in main_options[3:]: - opt['Enabled'] = False - elif index == 1: - # Disk - if main_options[index]['Enabled']: - main_options[0]['Enabled'] = False - for opt in main_options[2:4]: - opt['Enabled'] = False - for opt in main_options[4:]: - opt['Enabled'] = True - else: - for opt in main_options[4:]: - opt['Enabled'] = False - elif index == 2: - # Disk (Quick) - if main_options[index]['Enabled']: - for opt in main_options[:2] + main_options[3:]: - opt['Enabled'] = False - main_options[4]['Enabled'] = True - else: - main_options[4]['Enabled'] = False + update_main_options(state, selection, main_options) elif selection == 'A': run_audio_test() elif selection == 'K': @@ -391,7 +370,7 @@ def menu_diags(state, args): elif selection == 'M': secret_screensaver('matrix') elif selection == 'T': - # Tubes is close to pipes + # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': run_program(['systemctl', 'reboot']) @@ -400,21 +379,7 @@ def menu_diags(state, args): elif selection == 'Q': break elif selection == 'S': - # Run test(s) - clear_screen() - print('Tests:') - for opt in main_options[3:]: - _nvme_smart = opt['Base Name'] == 'NVMe / SMART' - # Update state - state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] - print(' {:<15} {}{}{} {}'.format( - opt['Base Name'], - COLORS['GREEN'] if opt['Enabled'] else COLORS['RED'], - 'Enabled' if opt['Enabled'] else 'Disabled', - COLORS['CLEAR'], - quick_label if state.quick_mode and _nvme_smart else '')) - print('\nFake test(s) placeholder for now...') - pause('Press Enter to return to main menu... ') + run_hw_tests(state) def run_audio_test(): """Run audio test.""" @@ -424,6 +389,23 @@ def run_audio_test(): #run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_hw_tests(state): + """Run enabled hardware tests.""" + # Run test(s) + clear_screen() + print('Tests:') + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + print_standard(' {:<15} {}{}{} {}'.format( + k, + COLORS['GREEN'] if v['Enabled'] else COLORS['RED'], + 'Enabled' if v['Enabled'] else 'Disabled', + COLORS['CLEAR'], + QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) + print('\nFake test(s) placeholder for now...') + pause('Press Enter to return to main menu... ') + def run_keyboard_test(): """Run keyboard test.""" # TODO: Enable real test and remove placeholder @@ -450,6 +432,49 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def update_main_options(state, selection, main_options): + """Update menu and state based on selection.""" + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + + # Handle presets + if index == 0: + # Full + if main_options[index]['Enabled']: + for opt in main_options[1:3]: + opt['Enabled'] = False + for opt in main_options[3:]: + opt['Enabled'] = True + else: + for opt in main_options[3:]: + opt['Enabled'] = False + elif index == 1: + # Disk + if main_options[index]['Enabled']: + main_options[0]['Enabled'] = False + for opt in main_options[2:4]: + opt['Enabled'] = False + for opt in main_options[4:]: + opt['Enabled'] = True + else: + for opt in main_options[4:]: + opt['Enabled'] = False + elif index == 2: + # Disk (Quick) + if main_options[index]['Enabled']: + for opt in main_options[:2] + main_options[3:]: + opt['Enabled'] = False + main_options[4]['Enabled'] = True + else: + main_options[4]['Enabled'] = False + + # Update state + for opt in main_options[3:]: + state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] + + # Done + return main_options + def update_io_progress(percent, rate, progress_file): """Update I/O progress file.""" bar_color = COLORS['CLEAR'] From 62c9d82fd2cba7166c7c572e8de9f64387c2a417 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 17:05:53 -0700 Subject: [PATCH 009/186] Adjusted placeholders --- .bin/Scripts/functions/hw_diags.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3de0806f..4593392f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -373,9 +373,15 @@ def menu_diags(state, args): # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': - run_program(['systemctl', 'reboot']) + print('(FAKE) reboot...') + sleep(1) + # TODO uncomment below + #run_program(['systemctl', 'reboot']) elif selection == 'P': - run_program(['systemctl', 'poweroff']) + print('(FAKE) poweroff...') + sleep(1) + # TODO uncomment below + #run_program(['systemctl', 'poweroff']) elif selection == 'Q': break elif selection == 'S': @@ -383,10 +389,8 @@ def menu_diags(state, args): def run_audio_test(): """Run audio test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake audio placeholder for now...') - #run_program(['hw-diags-audio'], check=False, pipe=False) + run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') def run_hw_tests(state): @@ -403,23 +407,17 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) - print('\nFake test(s) placeholder for now...') - pause('Press Enter to return to main menu... ') + pause('\nPress Enter to return to main menu... ') def run_keyboard_test(): """Run keyboard test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake keyboard placeholder for now...') - #run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) - pause('Press Enter to return to main menu... ') + run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) def run_network_test(): """Run network test.""" - # TODO: Enable real test and remove placeholder clear_screen() - print('Fake network placeholder for now...') - #run_program(['hw-diags-network'], check=False, pipe=False) + run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') def secret_screensaver(screensaver=None): From 1489ad4237e3bc20d22fc0f89e7d0bdc0c378790 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 18:43:50 -0700 Subject: [PATCH 010/186] Added safety check for devices --- .bin/Scripts/functions/hw_diags.py | 61 ++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4593392f..67195d79 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -176,22 +176,53 @@ class State(): for dev in json_data['blockdevices']: skip_dev = False dev_obj = DevObj(dev['name']) - + # Skip loopback devices if dev_obj.lsblk['tran'] == 'NONE': skip_dev = True - + # Skip WK devices wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) for label in dev_obj.labels: if re.search(wk_label_regex, label, re.IGNORECASE): skip_dev = True - + # Add device if not skip_dev: self.devs.append(DevObj(dev['name'])) # Functions +def check_dev_attributes(dev): + """Check if device should be tested and allow overrides.""" + needs_override = False + print_standard(' {size:>6} ({tran}) {model} {serial}'.format( + **dev.lsblk)) + + # General checks + if not dev.nvme_attributes and not dev.smart_attributes: + needs_override = True + print_warning( + ' WARNING: No NVMe or SMART attributes available for: {}'.format( + dev.path)) + + # NVMe checks + # TODO check all tracked attributes and set dev.failing if needed + + # SMART checks + # TODO check all tracked attributes and set dev.failing if needed + + # Ask for override if necessary + if needs_override: + if ask(' Run tests on this device anyway?'): + # TODO Set override for this dev + pass + else: + for v in dev.tests.values(): + v['Enabled'] = False + v['Result'] = 'Skipped' + v['Status'] = 'Skipped' + print_standard('') + def generate_horizontal_graph(rates, oneline=False): """Generate two-line horizontal graph from rates, returns str.""" line_1 = '' @@ -297,7 +328,7 @@ def menu_diags(state, args): {'Letter': 'Q', 'Name': 'Quit'}, ] secret_actions = ['M', 'T'] - + # Set initial selections update_main_options(state, '1', main_options) @@ -397,7 +428,7 @@ def run_hw_tests(state): """Run enabled hardware tests.""" # Run test(s) clear_screen() - print('Tests:') + print_info('Selected Tests:') for k, v in sorted( state.tests.items(), key=lambda kv: kv[1]['Order']): @@ -407,7 +438,21 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) - pause('\nPress Enter to return to main menu... ') + print_standard('') + + # Check devices if necessary + if (state.tests['badblocks']['Enabled'] + or state.tests['I/O Benchmark']['Enabled']): + print_info('Selected Disks:') + for dev in state.devs: + check_dev_attributes(dev) + print_standard('') + + # Run tests + # TODO + + # Done + pause('Press Enter to return to main menu... ') def run_keyboard_test(): """Run keyboard test.""" @@ -465,11 +510,11 @@ def update_main_options(state, selection, main_options): main_options[4]['Enabled'] = True else: main_options[4]['Enabled'] = False - + # Update state for opt in main_options[3:]: state.tests[opt['Base Name']]['Enabled'] = opt['Enabled'] - + # Done return main_options From 597a23608951f38dfc3b987eed3ebff053081c98 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 18:44:52 -0700 Subject: [PATCH 011/186] Don't clear screen twice at startup * Combined init_global_vars and add_devs output --- .bin/Scripts/functions/hw_diags.py | 5 ++++- .bin/Scripts/hw-diags-menu | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 67195d79..6c43fe28 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -166,7 +166,10 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - self.add_devs() + try_and_print( + message='Scanning devices...', + function=self.add_devs, + cs='Done') def add_devs(self): """Add all block devices listed by lsblk.""" diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index c67cc5f4..e60f8fc4 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -13,9 +13,6 @@ init_global_vars() if __name__ == '__main__': try: - # Prep - clear_screen() - # Show menu state = State() menu_diags(state, sys.argv) From 8fb1620c940ac2d337cb7cfb36998b3d1fac7c18 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 19:23:35 -0700 Subject: [PATCH 012/186] Added placeholder functions for HW tests --- .bin/Scripts/functions/hw_diags.py | 45 ++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6c43fe28..15bea407 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -302,7 +302,7 @@ def get_status_color(s): color = COLORS['CLEAR'] if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: color = COLORS['RED'] - elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: + elif s in ['Aborted', 'N/A', 'Unknown', 'Working', 'Skipped']: color = COLORS['YELLOW'] elif s in ['CS']: color = COLORS['GREEN'] @@ -427,6 +427,10 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_badblocks_test(state): + """TODO""" + print_standard('TODO: run_badblocks_test()') + def run_hw_tests(state): """Run enabled hardware tests.""" # Run test(s) @@ -452,22 +456,55 @@ def run_hw_tests(state): print_standard('') # Run tests - # TODO + if state.tests['Prime95 & Temps']['Enabled']: + run_mprime_test(state) + if state.tests['NVMe / SMART']['Enabled']: + run_nvme_smart(state) + if state.tests['badblocks']['Enabled']: + run_badblocks_test(state) + if state.tests['I/O Benchmark']['Enabled']: + run_io_benchmark(state) # Done pause('Press Enter to return to main menu... ') +def run_io_benchmark(state): + """TODO""" + print_standard('TODO: run_io_benchmark()') + def run_keyboard_test(): """Run keyboard test.""" clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) +def run_mprime_test(state): + """TODO""" + print_standard('TODO: run_mprime_test()') + def run_network_test(): """Run network test.""" clear_screen() run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') +def run_nvme_smart(state): + """TODO""" + for dev in state.devs: + if dev.nvme_attributes: + run_nvme_tests(dev) + elif dev.smart_attributes: + run_smart_tests(dev) + else: + print_standard('TODO: run_nvme_smart({})'.format( + dev.path)) + print_warning( + " WARNING: Device {} doesn't support NVMe or SMART test".format( + dev.path)) + +def run_nvme_tests(dev): + """TODO""" + print_standard('TODO: run_nvme_test({})'.format(dev.path)) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': @@ -478,6 +515,10 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def run_smart_tests(dev): + """TODO""" + print_standard('TODO: run_smart_tests({})'.format(dev.path)) + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 4bb1402ac5a992c2e5f3f030a7a9131f743d00fb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 20:50:47 -0700 Subject: [PATCH 013/186] Added tmux functions * Going to try and replace the send-keys sections next --- .bin/Scripts/functions/hw_diags.py | 80 +++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 15bea407..a446036d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -62,7 +62,7 @@ IO_VARS = { KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) -SIDE_PATH_WIDTH = 21 +SIDE_PANE_WIDTH = 21 # Classes class DevObj(): @@ -156,6 +156,7 @@ class State(): def __init__(self): self.devs = [] self.finished = False + self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.started = False @@ -195,6 +196,31 @@ class State(): self.devs.append(DevObj(dev['name'])) # Functions +def build_outer_panes(state): + """Build top and side panes.""" + clear_screen() + + # Create panes + state.panes['Top'] = tmux_split_window( + behind=True, lines=2, vertical=True) + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top']) + state.panes['Progress'] = tmux_split_window(lines=SIDE_PANE_WIDTH) + + # Set text + tmux_update_pane_text( + state.panes['Top'], + text='{GREEN}Hardware Diagnostics{CLEAR}'.format( + **COLORS)) + tmux_update_pane_text( + state.panes['Started'], + text='{BLUE}Started{CLEAR}\n{text}'.format( + text=time.strftime("%Y-%m-%d %H:%M %Z"), + **COLORS)) + tmux_update_pane_text( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here'.format(**COLORS)) + def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" needs_override = False @@ -433,8 +459,10 @@ def run_badblocks_test(state): def run_hw_tests(state): """Run enabled hardware tests.""" + # Build Panes + build_outer_panes(state) + # Run test(s) - clear_screen() print_info('Selected Tests:') for k, v in sorted( state.tests.items(), @@ -468,6 +496,9 @@ def run_hw_tests(state): # Done pause('Press Enter to return to main menu... ') + # Cleanup + tmux_kill_pane(*state.panes.values()) + def run_io_benchmark(state): """TODO""" print_standard('TODO: run_io_benchmark()') @@ -519,6 +550,51 @@ def run_smart_tests(dev): """TODO""" print_standard('TODO: run_smart_tests({})'.format(dev.path)) +def tmux_kill_pane(*panes): + """Kill tmux pane by id.""" + cmd = ['tmux', 'kill-pane', '-t'] + for pane_id in panes: + print(pane_id) + run_program(cmd+[pane_id], check=False) + +def tmux_split_window( + lines=None, percent=None, + behind=False, vertical=False, + follow=False, target_pane=None): + """Run tmux split-window command and return pane_id as str.""" + # Bail early + if not lines and not percent: + raise Exception('Neither lines nor percent specified.') + + # Build cmd + cmd = ['tmux', 'split-window', '-PF', '#D'] + if behind: + cmd.append('-b') + if vertical: + cmd.append('-v') + else: + cmd.append('-h') + if not follow: + cmd.append('-d') + if lines is not None: + cmd.extend(['-l', str(lines)]) + elif percent is not None: + cmd.extend(['-p', str(percent)]) + if target_pane: + cmd.extend(['-t', str(target_pane)]) + + # Run and return pane_id + result = run_program(cmd) + return result.stdout.decode().strip() + +def tmux_update_pane_text(pane_id, text): + """Print text to tmux pane.""" + text = text.replace('\033', r'\e') + cmd = ['tmux', 'send-keys', '-t', pane_id] + run_program(cmd+['Enter']) + run_program(cmd+['clear; echo-and-hold "{}"'.format(text)]) + run_program(cmd+['Enter']) + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 43b9645c69fee66d5154013abe5d67b58bf54a77 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Dec 2018 23:39:15 -0700 Subject: [PATCH 014/186] Update tmux panes via respawn-pane Instead of send-keys * Avoids flooding zsh history * Less flickering --- .bin/Scripts/functions/hw_diags.py | 121 +++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 34 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a446036d..743db0a2 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -63,6 +63,7 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 21 +TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes class DevObj(): @@ -200,26 +201,24 @@ def build_outer_panes(state): """Build top and side panes.""" clear_screen() - # Create panes + # Top state.panes['Top'] = tmux_split_window( - behind=True, lines=2, vertical=True) - state.panes['Started'] = tmux_split_window( - lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top']) - state.panes['Progress'] = tmux_split_window(lines=SIDE_PANE_WIDTH) + behind=True, lines=2, vertical=True, + text='{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS)) - # Set text - tmux_update_pane_text( - state.panes['Top'], - text='{GREEN}Hardware Diagnostics{CLEAR}'.format( - **COLORS)) - tmux_update_pane_text( - state.panes['Started'], + # Started + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top'], text='{BLUE}Started{CLEAR}\n{text}'.format( text=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) - tmux_update_pane_text( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here'.format(**COLORS)) + + # Progress + state.panes['Progress'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" @@ -337,8 +336,7 @@ def get_status_color(s): def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] - title = '{GREEN}Hardware Diagnostics: Main Menu{CLEAR}'.format( - **COLORS) + title = '{}\nMain Menu'.format(TOP_PANE_TEXT) # NOTE: Changing the order of main_options will break everything main_options = [ {'Base Name': 'Full Diagnostic', 'Enabled': False}, @@ -455,7 +453,16 @@ def run_audio_test(): def run_badblocks_test(state): """TODO""" + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'badblocks')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) print_standard('TODO: run_badblocks_test()') + sleep(3) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -501,7 +508,16 @@ def run_hw_tests(state): def run_io_benchmark(state): """TODO""" + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'I/O Benchmark')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) print_standard('TODO: run_io_benchmark()') + sleep(3) def run_keyboard_test(): """Run keyboard test.""" @@ -509,8 +525,21 @@ def run_keyboard_test(): run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) def run_mprime_test(state): - """TODO""" - print_standard('TODO: run_mprime_test()') + """Test CPU with Prime95 and track temps.""" + # Prep + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'Prime95 & Temps')) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) + # Get idle temps + # Stress CPU + # Get max temp + # Get cooldown temp + sleep(3) def run_network_test(): """Run network test.""" @@ -521,21 +550,35 @@ def run_network_test(): def run_nvme_smart(state): """TODO""" for dev in state.devs: + tmux_update_pane( + state.panes['Top'], + text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( + t=TOP_PANE_TEXT, **dev.lsblk)) + tmux_update_pane( + state.panes['Progress'], + text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( + Meh=time.strftime('%H:%M:%S %Z'), + **COLORS)) if dev.nvme_attributes: - run_nvme_tests(dev) + run_nvme_tests(state, dev) elif dev.smart_attributes: - run_smart_tests(dev) + run_smart_tests(state, dev) else: print_standard('TODO: run_nvme_smart({})'.format( dev.path)) print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( dev.path)) + sleep(3) -def run_nvme_tests(dev): +def run_nvme_tests(state, dev): """TODO""" print_standard('TODO: run_nvme_test({})'.format(dev.path)) +def run_smart_tests(state, dev): + """TODO""" + print_standard('TODO: run_smart_tests({})'.format(dev.path)) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': @@ -546,10 +589,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def run_smart_tests(dev): - """TODO""" - print_standard('TODO: run_smart_tests({})'.format(dev.path)) - def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] @@ -560,11 +599,14 @@ def tmux_kill_pane(*panes): def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, - follow=False, target_pane=None): + follow=False, target_pane=None, + command=None, text=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: raise Exception('Neither lines nor percent specified.') + if not command and not text: + raise Exception('Neither command nor text specified.') # Build cmd cmd = ['tmux', 'split-window', '-PF', '#D'] @@ -583,17 +625,28 @@ def tmux_split_window( if target_pane: cmd.extend(['-t', str(target_pane)]) + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold', text]) + # Run and return pane_id result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane_text(pane_id, text): - """Print text to tmux pane.""" - text = text.replace('\033', r'\e') - cmd = ['tmux', 'send-keys', '-t', pane_id] - run_program(cmd+['Enter']) - run_program(cmd+['clear; echo-and-hold "{}"'.format(text)]) - run_program(cmd+['Enter']) +def tmux_update_pane(pane_id, command=None, text=None): + """Respawn with either a new command or new text.""" + # Bail early + if not command and not text: + raise Exception('Neither command nor text specified.') + + cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold', text]) + + run_program(cmd) def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" From 2d69d93154f6fc2882b3cb6d85a6b8c55bed3a36 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:41:27 -0700 Subject: [PATCH 015/186] Added watch option for tmux_split_window() --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 743db0a2..440ec1c5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -204,7 +204,7 @@ def build_outer_panes(state): # Top state.panes['Top'] = tmux_split_window( behind=True, lines=2, vertical=True, - text='{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS)) + text=TOP_PANE_TEXT) # Started state.panes['Started'] = tmux_split_window( @@ -216,9 +216,7 @@ def build_outer_panes(state): # Progress state.panes['Progress'] = tmux_split_window( lines=SIDE_PANE_WIDTH, - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + watch=state.progress_out) def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" @@ -600,13 +598,13 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - command=None, text=None): + command=None, text=None, watch=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: raise Exception('Neither lines nor percent specified.') - if not command and not text: - raise Exception('Neither command nor text specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') # Build cmd cmd = ['tmux', 'split-window', '-PF', '#D'] @@ -628,7 +626,12 @@ def tmux_split_window( if command: cmd.extend(command) elif text: - cmd.extend(['echo-and-hold', text]) + cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) # Run and return pane_id result = run_program(cmd) @@ -644,7 +647,7 @@ def tmux_update_pane(pane_id, command=None, text=None): if command: cmd.extend(command) elif text: - cmd.extend(['echo-and-hold', text]) + cmd.extend(['echo-and-hold "{}"'.format(text)]) run_program(cmd) From d025b8dc9e91555b9cf123580a49ab8f09eeb3ea Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:49:25 -0700 Subject: [PATCH 016/186] Adjusted how devices are added to the state obj * The change allows for devices to be (dis)connected while the script is running * Devices are scanned and added during run_hw_diags() * Fixes bug that prevented any devices from being added as well --- .bin/Scripts/functions/hw_diags.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 440ec1c5..9e720d92 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -78,6 +78,7 @@ class DevObj(): self.path = dev_path self.smart_attributes = {} self.smartctl = {} + self.state = state self.tests = { 'NVMe / SMART': {'Result': None, 'Status': None}, 'badblocks': {'Result': None, 'Status': None}, @@ -158,7 +159,8 @@ class State(): self.devs = [] self.finished = False self.panes = {} - self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + # TODO Switch to LogDir + self.progress_out = '{}/progress.out'.format(global_vars['TmpDir']) self.quick_mode = False self.started = False self.tests = { @@ -168,19 +170,20 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - try_and_print( - message='Scanning devices...', - function=self.add_devs, - cs='Done') - def add_devs(self): - """Add all block devices listed by lsblk.""" + def init(self): + """Scan for block devices and reset all tests.""" + self.devs = [] + for k in ['Result', 'Started', 'Status']: + self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) for dev in json_data['blockdevices']: skip_dev = False - dev_obj = DevObj(dev['name']) + dev_obj = DevObj(self, dev['name']) # Skip loopback devices if dev_obj.lsblk['tran'] == 'NONE': @@ -194,7 +197,7 @@ class State(): # Add device if not skip_dev: - self.devs.append(DevObj(dev['name'])) + self.devs.append(dev_obj) # Functions def build_outer_panes(state): @@ -464,6 +467,9 @@ def run_badblocks_test(state): def run_hw_tests(state): """Run enabled hardware tests.""" + print_standard('Scanning devices...') + state.init() + # Build Panes build_outer_panes(state) From 7c163a8110b8474a8d75dee1e091b7b109d43eea Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 03:52:24 -0700 Subject: [PATCH 017/186] Added update progress sections --- .bin/Scripts/functions/hw_diags.py | 158 +++++++++++++++++++++++------ 1 file changed, 126 insertions(+), 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9e720d92..ac128ea1 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -62,13 +62,13 @@ IO_VARS = { KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) -SIDE_PANE_WIDTH = 21 +SIDE_PANE_WIDTH = 20 TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes class DevObj(): """Device object for tracking device specific data.""" - def __init__(self, dev_path): + def __init__(self, state, dev_path): self.failing = False self.labels = [] self.lsblk = {} @@ -80,13 +80,17 @@ class DevObj(): self.smartctl = {} self.state = state self.tests = { - 'NVMe / SMART': {'Result': None, 'Status': None}, - 'badblocks': {'Result': None, 'Status': None}, + 'NVMe / SMART': { + 'Result': '', 'Started': False, 'Status': '', 'Order': 1}, + 'badblocks': { + 'Result': '', 'Started': False, 'Status': '', 'Order': 2}, 'I/O Benchmark': { - 'Result': None, - 'Status': None, + 'Result': '', + 'Started': False, + 'Status': '', 'Read Rates': [], - 'Graph Data': []}, + 'Graph Data': [], + 'Order': 3}, } self.get_details() self.get_smart_details() @@ -153,6 +157,21 @@ class DevObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + def update_progress(self): + """Update status strings.""" + for k, v in self.tests.items(): + if self.state.tests[k]['Enabled']: + _status = '' + if not v['Status']: + _status = 'Pending' + if v['Started']: + if v['Result']: + _status = v['Result'] + else: + _status = 'Working' + if _status: + v['Status'] = build_status_string(self.name, _status) + class State(): """Object to track device objects and overall state.""" def __init__(self): @@ -165,7 +184,7 @@ class State(): self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': None, 'Status': None}, + 'Result': '', 'Started': False, 'Status': ''}, 'NVMe / SMART': {'Enabled': False, 'Order': 2}, 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, @@ -199,6 +218,27 @@ class State(): if not skip_dev: self.devs.append(dev_obj) + def update_progress(self): + """Update status strings.""" + # Prime95 + p = self.tests['Prime95 & Temps'] + if p['Enabled']: + _status = '' + if not p['Status']: + _status = 'Pending' + if p['Started']: + if p['Result']: + _status = p['Result'] + else: + _status = 'Working' + if _status: + p['Status'] = build_status_string( + 'Prime95', _status, info_label=True) + + # Disks + for dev in self.devs: + dev.update_progress() + # Functions def build_outer_panes(state): """Build top and side panes.""" @@ -221,6 +261,24 @@ def build_outer_panes(state): lines=SIDE_PANE_WIDTH, watch=state.progress_out) +def build_status_string(label, status, info_label=False): + """Build status string with appropriate colors.""" + status_color = COLORS['CLEAR'] + if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + status_color = COLORS['RED'] + elif status in ['Aborted', 'Unknown', 'Working', 'Skipped']: + status_color = COLORS['YELLOW'] + elif status in ['CS']: + status_color = COLORS['GREEN'] + + return '{l_c}{l}{CLEAR}{s_c}{s:>{s_w}}{CLEAR}'.format( + l_c=COLORS['BLUE'] if info_label else '', + l=label, + s_c=status_color, + s=status, + s_w=SIDE_PANE_WIDTH-len(label), + **COLORS) + def check_dev_attributes(dev): """Check if device should be tested and allow overrides.""" needs_override = False @@ -247,8 +305,9 @@ def check_dev_attributes(dev): pass else: for v in dev.tests.values(): - v['Enabled'] = False + # Started is set to True to fix the status string v['Result'] = 'Skipped' + v['Started'] = True v['Status'] = 'Skipped' print_standard('') @@ -457,13 +516,13 @@ def run_badblocks_test(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) print_standard('TODO: run_badblocks_test()') - sleep(3) + for dev in state.devs: + dev.tests['badblocks']['Started'] = True + update_progress_pane(state) + sleep(3) + dev.tests['badblocks']['Result'] = 'OVERRIDE' + update_progress_pane(state) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -471,6 +530,7 @@ def run_hw_tests(state): state.init() # Build Panes + update_progress_pane(state) build_outer_panes(state) # Run test(s) @@ -515,13 +575,13 @@ def run_io_benchmark(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) print_standard('TODO: run_io_benchmark()') - sleep(3) + for dev in state.devs: + dev.tests['I/O Benchmark']['Started'] = True + update_progress_pane(state) + sleep(3) + dev.tests['I/O Benchmark']['Result'] = 'Unknown' + update_progress_pane(state) def run_keyboard_test(): """Run keyboard test.""" @@ -534,16 +594,18 @@ def run_mprime_test(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps')) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + state.tests['Prime95 & Temps']['Started'] = True + update_progress_pane(state) + # Get idle temps # Stress CPU # Get max temp # Get cooldown temp + + # Done sleep(3) + state.tests['Prime95 & Temps']['Result'] = 'Unknown' + update_progress_pane(state) def run_network_test(): """Run network test.""" @@ -558,11 +620,8 @@ def run_nvme_smart(state): state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **dev.lsblk)) - tmux_update_pane( - state.panes['Progress'], - text='{BLUE}Progress{CLEAR}\nGoes here\n\n{Meh}'.format( - Meh=time.strftime('%H:%M:%S %Z'), - **COLORS)) + dev.tests['NVMe / SMART']['Started'] = True + update_progress_pane(state) if dev.nvme_attributes: run_nvme_tests(state, dev) elif dev.smart_attributes: @@ -573,15 +632,24 @@ def run_nvme_smart(state): print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( dev.path)) - sleep(3) + dev.tests['NVMe / SMART']['Status'] = 'N/A' + dev.tests['NVMe / SMART']['Result'] = 'N/A' + update_progress_pane(state) + sleep(3) def run_nvme_tests(state, dev): """TODO""" print_standard('TODO: run_nvme_test({})'.format(dev.path)) + sleep(3) + dev.tests['NVMe / SMART']['Result'] = 'CS' + update_progress_pane(state) def run_smart_tests(state, dev): """TODO""" print_standard('TODO: run_smart_tests({})'.format(dev.path)) + sleep(3) + dev.tests['NVMe / SMART']['Result'] = 'CS' + update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" @@ -724,6 +792,32 @@ def update_io_progress(percent, rate, progress_file): with open(progress_file, 'a') as f: f.write(line) +def update_progress_pane(state): + """Update progress file for side pane.""" + output = [] + state.update_progress() + + # Prime95 + output.append(state.tests['Prime95 & Temps']['Status']) + output.append(' ') + + # Disks + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + if 'Prime95' not in k and v['Enabled']: + output.append('{BLUE}{test_name}{CLEAR}'.format( + test_name=k, **COLORS)) + for dev in state.devs: + output.append(dev.tests[k]['Status']) + output.append(' ') + + # Add line-endings + output = ['{}\n'.format(line) for line in output] + + with open(state.progress_out, 'w') as f: + f.writelines(output) + if __name__ == '__main__': print("This file is not meant to be called directly.") From 372f80bf38d56dcb58a4ac76cf7b04555f3f66f5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 04:08:59 -0700 Subject: [PATCH 018/186] Skip optical drives --- .bin/Scripts/functions/hw_diags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index ac128ea1..d122f6c6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -204,8 +204,8 @@ class State(): skip_dev = False dev_obj = DevObj(self, dev['name']) - # Skip loopback devices - if dev_obj.lsblk['tran'] == 'NONE': + # Skip loopback and optical devices + if dev_obj.lsblk['type'] in ['loop', 'rom']: skip_dev = True # Skip WK devices From 163f64dda7921847a5601aaa3d2b0d7825b5a5b0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 04:10:20 -0700 Subject: [PATCH 019/186] Reduced timeout for major exceptions --- .bin/Scripts/functions/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index b5896966..cc3d98b8 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -305,13 +305,13 @@ def major_exception(): except GenericAbort: # User declined upload print_warning('Upload: Aborted') - sleep(30) + sleep(10) except GenericError: # No log file or uploading disabled - sleep(30) + sleep(10) except: print_error('Upload: NS') - sleep(30) + sleep(10) else: print_success('Upload: CS') pause('Press Enter to exit...') From 5dd8fa84164876952a85b4cb97b94eb0a691ea4c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 17:48:30 -0700 Subject: [PATCH 020/186] Get CPU details from lscpu --- .bin/Scripts/functions/hw_diags.py | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d122f6c6..892dff90 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -175,6 +175,7 @@ class DevObj(): class State(): """Object to track device objects and overall state.""" def __init__(self): + self.lscpu = {} self.devs = [] self.finished = False self.panes = {} @@ -189,6 +190,28 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } + self.get_cpu_details() + + def get_cpu_details(self): + """Get CPU details from lscpu.""" + cmd = ['lscpu', '--json'] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + except Exception as err: + # Ignore and leave self.cpu empty + print_error(err) + pause() + return + for line in json_data.get('lscpu', []): + _field = line.get('field', None).replace(':', '') + _data = line.get('data', None) + if not _field and not _data: + # Skip + print_warning(_field, _data) + pause() + continue + self.lscpu[_field] = _data def init(self): """Scan for block devices and reset all tests.""" @@ -591,9 +614,11 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" # Prep - tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'Prime95 & Temps')) + _title = '{}\n{}{}{}'.format( + TOP_PANE_TEXT, 'Prime95 & Temps', + ': ' if 'Model name' in state.lscpu else '', + state.lscpu.get('Model name', '')) + tmux_update_pane(state.panes['Top'], text=_title) state.tests['Prime95 & Temps']['Started'] = True update_progress_pane(state) From cb67f7e3c3ce26c7475b739ae937849b970d198a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 19:59:41 -0700 Subject: [PATCH 021/186] Added new sensors.py and dropped borrowed sensors --- .bin/Scripts/borrowed/sensors-README.md | 35 ---- .bin/Scripts/borrowed/sensors.py | 236 ------------------------ .bin/Scripts/functions/common.py | 11 +- .bin/Scripts/functions/sensors.py | 111 +++++++++++ 4 files changed, 117 insertions(+), 276 deletions(-) delete mode 100644 .bin/Scripts/borrowed/sensors-README.md delete mode 100644 .bin/Scripts/borrowed/sensors.py create mode 100644 .bin/Scripts/functions/sensors.py diff --git a/.bin/Scripts/borrowed/sensors-README.md b/.bin/Scripts/borrowed/sensors-README.md deleted file mode 100644 index 11858382..00000000 --- a/.bin/Scripts/borrowed/sensors-README.md +++ /dev/null @@ -1,35 +0,0 @@ -sensors.py -========== -python bindings using ctypes for libsensors3 of the [lm-sensors project](https://github.com/groeck/lm-sensors). The code was written against libsensors 3.3.4. - -For documentation of the low level API see [sensors.h](https://github.com/groeck/lm-sensors/blob/master/lib/sensors.h). For an example of the high level API see [example.py](example.py). - -For a GUI application that displays the sensor readings and is based on this library, take a look at [sensors-unity](https://launchpad.net/sensors-unity). - -Features --------- -* Full access to low level libsensors3 API -* High level iterator API -* unicode handling -* Python2 and Python3 compatible - -Licensing ---------- -LGPLv2 (same as libsensors3) - -Usage Notes ------------ -As Python does not support call by reference for primitive types some of the libsensors API had to be adapted: - -```python -# nr is changed by refrence in the C API -chip_name, nr = sensors.get_detected_chips(None, nr) - -# returns the value. throws on error -val = sensors.get_value(chip, subfeature_nr) -``` - -Missing Features (pull requests are welcome): -* `sensors_subfeature_type` enum -* `sensors_get_subfeature` -* Error handlers diff --git a/.bin/Scripts/borrowed/sensors.py b/.bin/Scripts/borrowed/sensors.py deleted file mode 100644 index 39b00a4f..00000000 --- a/.bin/Scripts/borrowed/sensors.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -@package sensors.py -Python Bindings for libsensors3 - -use the documentation of libsensors for the low level API. -see example.py for high level API usage. - -@author: Pavel Rojtberg (http://www.rojtberg.net) -@see: https://github.com/paroj/sensors.py -@copyright: LGPLv2 (same as libsensors) -""" - -from ctypes import * -import ctypes.util - -_libc = cdll.LoadLibrary(ctypes.util.find_library("c")) -# see https://github.com/paroj/sensors.py/issues/1 -_libc.free.argtypes = [c_void_p] - -_hdl = cdll.LoadLibrary(ctypes.util.find_library("sensors")) - -version = c_char_p.in_dll(_hdl, "libsensors_version").value.decode("ascii") - -class bus_id(Structure): - _fields_ = [("type", c_short), - ("nr", c_short)] - -class chip_name(Structure): - _fields_ = [("prefix", c_char_p), - ("bus", bus_id), - ("addr", c_int), - ("path", c_char_p)] - -class feature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int)] - - # sensors_feature_type - IN = 0x00 - FAN = 0x01 - TEMP = 0x02 - POWER = 0x03 - ENERGY = 0x04 - CURR = 0x05 - HUMIDITY = 0x06 - MAX_MAIN = 0x7 - VID = 0x10 - INTRUSION = 0x11 - MAX_OTHER = 0x12 - BEEP_ENABLE = 0x18 - -class subfeature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int), - ("mapping", c_int), - ("flags", c_uint)] - -_hdl.sensors_get_detected_chips.restype = POINTER(chip_name) -_hdl.sensors_get_features.restype = POINTER(feature) -_hdl.sensors_get_all_subfeatures.restype = POINTER(subfeature) -_hdl.sensors_get_label.restype = c_void_p # return pointer instead of str so we can free it -_hdl.sensors_get_adapter_name.restype = c_char_p # docs do not say whether to free this or not -_hdl.sensors_strerror.restype = c_char_p - -### RAW API ### -MODE_R = 1 -MODE_W = 2 -COMPUTE_MAPPING = 4 - -def init(cfg_file = None): - file = _libc.fopen(cfg_file.encode("utf-8"), "r") if cfg_file is not None else None - - if _hdl.sensors_init(file) != 0: - raise Exception("sensors_init failed") - - if file is not None: - _libc.fclose(file) - -def cleanup(): - _hdl.sensors_cleanup() - -def parse_chip_name(orig_name): - ret = chip_name() - err= _hdl.sensors_parse_chip_name(orig_name.encode("utf-8"), byref(ret)) - - if err < 0: - raise Exception(strerror(err)) - - return ret - -def strerror(errnum): - return _hdl.sensors_strerror(errnum).decode("utf-8") - -def free_chip_name(chip): - _hdl.sensors_free_chip_name(byref(chip)) - -def get_detected_chips(match, nr): - """ - @return: (chip, next nr to query) - """ - _nr = c_int(nr) - - if match is not None: - match = byref(match) - - chip = _hdl.sensors_get_detected_chips(match, byref(_nr)) - chip = chip.contents if bool(chip) else None - return chip, _nr.value - -def chip_snprintf_name(chip, buffer_size=200): - """ - @param buffer_size defaults to the size used in the sensors utility - """ - ret = create_string_buffer(buffer_size) - err = _hdl.sensors_snprintf_chip_name(ret, buffer_size, byref(chip)) - - if err < 0: - raise Exception(strerror(err)) - - return ret.value.decode("utf-8") - -def do_chip_sets(chip): - """ - @attention this function was not tested - """ - err = _hdl.sensors_do_chip_sets(byref(chip)) - if err < 0: - raise Exception(strerror(err)) - -def get_adapter_name(bus): - return _hdl.sensors_get_adapter_name(byref(bus)).decode("utf-8") - -def get_features(chip, nr): - """ - @return: (feature, next nr to query) - """ - _nr = c_int(nr) - feature = _hdl.sensors_get_features(byref(chip), byref(_nr)) - feature = feature.contents if bool(feature) else None - return feature, _nr.value - -def get_label(chip, feature): - ptr = _hdl.sensors_get_label(byref(chip), byref(feature)) - val = cast(ptr, c_char_p).value.decode("utf-8") - _libc.free(ptr) - return val - -def get_all_subfeatures(chip, feature, nr): - """ - @return: (subfeature, next nr to query) - """ - _nr = c_int(nr) - subfeature = _hdl.sensors_get_all_subfeatures(byref(chip), byref(feature), byref(_nr)) - subfeature = subfeature.contents if bool(subfeature) else None - return subfeature, _nr.value - -def get_value(chip, subfeature_nr): - val = c_double() - err = _hdl.sensors_get_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - return val.value - -def set_value(chip, subfeature_nr, value): - """ - @attention this function was not tested - """ - val = c_double(value) - err = _hdl.sensors_set_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - -### Convenience API ### -class ChipIterator: - def __init__(self, match = None): - self.match = parse_chip_name(match) if match is not None else None - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - chip, self.nr = get_detected_chips(self.match, self.nr) - - if chip is None: - raise StopIteration - - return chip - - def __del__(self): - if self.match is not None: - free_chip_name(self.match) - - def next(self): # python2 compability - return self.__next__() - -class FeatureIterator: - def __init__(self, chip): - self.chip = chip - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - feature, self.nr = get_features(self.chip, self.nr) - - if feature is None: - raise StopIteration - - return feature - - def next(self): # python2 compability - return self.__next__() - -class SubFeatureIterator: - def __init__(self, chip, feature): - self.chip = chip - self.feature = feature - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - subfeature, self.nr = get_all_subfeatures(self.chip, self.feature, self.nr) - - if subfeature is None: - raise StopIteration - - return subfeature - - def next(self): # python2 compability - return self.__next__() diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index cc3d98b8..2bf52a85 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -25,12 +25,13 @@ global_vars = {} # STATIC VARIABLES COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'GREEN': '\033[32m', + 'CLEAR': '\033[0m', + 'RED': '\033[31m', + 'GREEN': '\033[32m', 'YELLOW': '\033[33m', - 'BLUE': '\033[34m' -} + 'ORANGE': '\033[31;1m', + 'BLUE': '\033[34m' + } try: HKU = winreg.HKEY_USERS HKCR = winreg.HKEY_CLASSES_ROOT diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py new file mode 100644 index 00000000..0cb65acd --- /dev/null +++ b/.bin/Scripts/functions/sensors.py @@ -0,0 +1,111 @@ +# Wizard Kit: Functions - Sensors + +import itertools +import json +import re + +from functions.common import * + +# STATIC VARIABLES +TEMP_LIMITS = { + 'GREEN': 60, + 'YELLOW': 70, + 'ORANGE': 80, + 'RED': 90, + } + +# REGEX +REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') + +def fix_sensor_str(s): + """Cleanup string and return str.""" + s = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', s, re.IGNORECASE) + s = s.title() + s = s.replace('Coretemp', 'CoreTemp') + s = s.replace('Acpi', 'ACPI') + s = s.replace('Isa ', 'ISA ') + s = s.replace('Id ', 'ID ') + s = re.sub(r'(\D+)(\d+)', r'\1 \2', s, re.IGNORECASE) + s = s.replace(' ', ' ') + return s + +def get_color_temp(temp): + """Get colored temp string, returns str.""" + try: + temp = float(temp) + except ValueError: + return '{YELLOW}{temp}{CLEAR}'.format(temp=temp, **COLORS) + if temp > TEMP_LIMITS['RED']: + color = COLORS['RED'] + elif temp > TEMP_LIMITS['ORANGE']: + color = COLORS['ORANGE'] + elif temp > TEMP_LIMITS['YELLOW']: + color = COLORS['YELLOW'] + elif temp > TEMP_LIMITS['GREEN']: + color = COLORS['GREEN'] + elif temp > 0: + color = COLORS['BLUE'] + else: + color = COLORS['CLEAR'] + return '{color}{prefix}{temp:2.0f}°C{CLEAR}'.format( + color = color, + prefix = '-' if temp < 0 else '', + temp = temp, + **COLORS) + +def get_raw_sensor_data(): + """Read sensor data and return dict.""" + cmd = ['sensors', '-j'] + result = run_program(cmd) + return json.loads(result.stdout.decode()) + +def get_sensor_data(): + """Parse raw sensor data and return new dict.""" + json_data = get_raw_sensor_data() + sensor_data = {'CoreTemps': {}, 'Other': {}} + for _adapter, _sources in json_data.items(): + if 'coretemp' in _adapter: + _section = 'CoreTemps' + else: + _section = 'Other' + sensor_data[_section][_adapter] = {} + _sources.pop('Adapter', None) + + # Find current temp and add to dict + ## current temp is labeled xxxx_input + for _source, _labels in _sources.items(): + for _label, _temp in _labels.items(): + if 'input' in _label: + sensor_data[_section][_adapter][_source] = { + 'Temps': [_temp], + 'Label': _label, + } + + # Remove empty sections + for k, v in sensor_data.items(): + v = {k2: v2 for k2, v2 in v.items() if v2} + + # Done + return sensor_data + +def update_sensor_data(sensor_data): + """Read sensors and update existing sensor_data, returns dict.""" + json_data = get_raw_sensor_data() + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _label = _ddata['Label'] + _temp = json_data[_adapter][_source][_label] + _data['Temps'].append(_temp) + return sensor_data + +def join_columns(column1, column2, width=55): + return '{:<{}}{}'.format( + column1, + 55+len(column1)-len(REGEX_COLORS.sub('', column1)), + column2) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 7140f38ba4fefdfb571405b97a80e2eb7ee6f783 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:11:10 -0700 Subject: [PATCH 022/186] Added average, clear, and max temps sections --- .bin/Scripts/functions/sensors.py | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 0cb65acd..10a2e664 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -17,6 +17,14 @@ TEMP_LIMITS = { # REGEX REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') +def clear_temps(sensor_data): + """Clear saved temps but keep structure, returns dict.""" + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data['Temps'] = [] + return sensor_data + def fix_sensor_str(s): """Cleanup string and return str.""" s = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', s, re.IGNORECASE) @@ -88,13 +96,41 @@ def get_sensor_data(): # Done return sensor_data +def save_max_temp(sensor_data): + """Record max temps seen this session, returns dict.""" + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data['Max'] = max(_data['Temps']) + + # Done + return sensor_data + +def save_average_temp(sensor_data, save_label, seconds=10): + """Calculate average temps and record under save_label, returns dict.""" + clear_temps(sensor_data) + + # Get temps + for i in range(seconds): + sensor_data = update_sensor_data(sensor_data) + sleep(1) + + # Calculate averages + for _section, _adapters in sensor_data.items(): + for _adapter, _sources in _adapters.items(): + for _source, _data in _sources.items(): + _data[save_label] = sum(_data['Temps']) / len(_data['Temps']) + + # Done + return sensor_data + def update_sensor_data(sensor_data): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() for _section, _adapters in sensor_data.items(): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): - _label = _ddata['Label'] + _label = _data['Label'] _temp = json_data[_adapter][_source][_label] _data['Temps'].append(_temp) return sensor_data From 2eccc236a960235df41ec864116affae0efd281f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:40:25 -0700 Subject: [PATCH 023/186] Added generate_report() * Also merged save_max_temp() with update_sensor_data() * Max doesn't need resetting so just calc max everytime --- .bin/Scripts/functions/sensors.py | 59 +++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 10a2e664..30df47de 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -37,8 +37,30 @@ def fix_sensor_str(s): s = s.replace(' ', ' ') return s -def get_color_temp(temp): - """Get colored temp string, returns str.""" +def generate_report(sensor_data, *temp_labels, colors=True): + """Build report based on temp_labels, returns list if str.""" + report = [] + for _section, _adapters in sorted(sensor_data.items()): + # CoreTemps then Other temps + for _adapter, _sources in sorted(_adapters.items()): + # Adapter + report.append(fix_sensor_str(_adapter)) + for _source, _data in sorted(_sources.items()): + # Source + _line = '{:18} '.format(fix_sensor_str(_source)) + _temps = [] + for _label in temp_labels: + _temps.append('{}{}{}'.format( + _label.lower() if _label != 'Current' else '', + ': ' if _label != 'Current' else '', + get_temp_str(_data[_label], colors=colors))) + _line += ', '.join(_temps) + report.append(_line) + report.append(' ') + return sensor_data + +def get_colored_temp_str(temp): + """Get colored string based on temp, returns str.""" try: temp = float(temp) except ValueError: @@ -85,8 +107,10 @@ def get_sensor_data(): for _label, _temp in _labels.items(): if 'input' in _label: sensor_data[_section][_adapter][_source] = { - 'Temps': [_temp], + 'Current': _temp, 'Label': _label, + 'Max': _temp, + 'Temps': [_temp], } # Remove empty sections @@ -96,18 +120,21 @@ def get_sensor_data(): # Done return sensor_data -def save_max_temp(sensor_data): - """Record max temps seen this session, returns dict.""" - for _section, _adapters in sensor_data.items(): - for _adapter, _sources in _adapters.items(): - for _source, _data in _sources.items(): - _data['Max'] = max(_data['Temps']) +def get_temp_str(temp, colors=True): + """Get temp string, returns str.""" + if colors: + return get_colored_temp_str(temp) + try: + temp = float(temp) + except ValueError: + return '{}°C'.format(temp) + else: + return '{}{:2.0f}°C'.format( + '-' if temp < 0 else '', + temp) - # Done - return sensor_data - -def save_average_temp(sensor_data, save_label, seconds=10): - """Calculate average temps and record under save_label, returns dict.""" +def save_average_temp(sensor_data, temp_label, seconds=10): + """Calculate average temps and record under temp_label, returns dict.""" clear_temps(sensor_data) # Get temps @@ -119,7 +146,7 @@ def save_average_temp(sensor_data, save_label, seconds=10): for _section, _adapters in sensor_data.items(): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): - _data[save_label] = sum(_data['Temps']) / len(_data['Temps']) + _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) # Done return sensor_data @@ -132,6 +159,8 @@ def update_sensor_data(sensor_data): for _source, _data in _sources.items(): _label = _data['Label'] _temp = json_data[_adapter][_source][_label] + _data['Current'] = _temp + _data['Max'] = max(_temp, _data['Max']) _data['Temps'].append(_temp) return sensor_data From 328d6eb29484711774a5386c79a9415d25d2c3a4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 20:47:40 -0700 Subject: [PATCH 024/186] Modify sensor_data in place --- .bin/Scripts/functions/sensors.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 30df47de..bead3bfa 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -23,7 +23,6 @@ def clear_temps(sensor_data): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): _data['Temps'] = [] - return sensor_data def fix_sensor_str(s): """Cleanup string and return str.""" @@ -57,7 +56,6 @@ def generate_report(sensor_data, *temp_labels, colors=True): _line += ', '.join(_temps) report.append(_line) report.append(' ') - return sensor_data def get_colored_temp_str(temp): """Get colored string based on temp, returns str.""" @@ -139,7 +137,7 @@ def save_average_temp(sensor_data, temp_label, seconds=10): # Get temps for i in range(seconds): - sensor_data = update_sensor_data(sensor_data) + update_sensor_data(sensor_data) sleep(1) # Calculate averages @@ -148,9 +146,6 @@ def save_average_temp(sensor_data, temp_label, seconds=10): for _source, _data in _sources.items(): _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) - # Done - return sensor_data - def update_sensor_data(sensor_data): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() @@ -162,7 +157,6 @@ def update_sensor_data(sensor_data): _data['Current'] = _temp _data['Max'] = max(_temp, _data['Max']) _data['Temps'].append(_temp) - return sensor_data def join_columns(column1, column2, width=55): return '{:<{}}{}'.format( From 95b0d1e3f4823d31bcbf5bc0c2ee20753b20379e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 21:54:41 -0700 Subject: [PATCH 025/186] Wrap reports if necessary --- .bin/Scripts/functions/sensors.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index bead3bfa..5e33e1cb 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -3,6 +3,7 @@ import itertools import json import re +import shutil from functions.common import * @@ -37,7 +38,7 @@ def fix_sensor_str(s): return s def generate_report(sensor_data, *temp_labels, colors=True): - """Build report based on temp_labels, returns list if str.""" + """Generate report based on temp_labels, returns list if str.""" report = [] for _section, _adapters in sorted(sensor_data.items()): # CoreTemps then Other temps @@ -48,6 +49,7 @@ def generate_report(sensor_data, *temp_labels, colors=True): # Source _line = '{:18} '.format(fix_sensor_str(_source)) _temps = [] + # Temps (skip label for Current) for _label in temp_labels: _temps.append('{}{}{}'.format( _label.lower() if _label != 'Current' else '', @@ -57,6 +59,26 @@ def generate_report(sensor_data, *temp_labels, colors=True): report.append(_line) report.append(' ') + # Wrap lines if necessary + screen_size = shutil.get_terminal_size() + rows = screen_size.lines - 1 + if len(report) > rows and screen_size.columns > 55*2: + report = list(itertools.zip_longest( + report[:rows], report[rows:], fillvalue='')) + report = [join_columns(a, b) for a, b in report] + + # Handle empty reports (i.e. no sensors detected) + if not report: + report = [ + '{}WARNING: No sensors found{}'.format( + COLORS['YELLOW'] if colors else '', + COLORS['CLEAR'] if colors else ''), + ' ', + 'Please monitor temps manually'] + + # Done + return report + def get_colored_temp_str(temp): """Get colored string based on temp, returns str.""" try: From 0e5fab01047c69fd875d43e5ced54377f70a5ec5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 21:57:55 -0700 Subject: [PATCH 026/186] Handle missing labels in generate_report() --- .bin/Scripts/functions/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 5e33e1cb..b47f9afd 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -54,7 +54,7 @@ def generate_report(sensor_data, *temp_labels, colors=True): _temps.append('{}{}{}'.format( _label.lower() if _label != 'Current' else '', ': ' if _label != 'Current' else '', - get_temp_str(_data[_label], colors=colors))) + get_temp_str(_data.get(_label, '???'), colors=colors))) _line += ', '.join(_temps) report.append(_line) report.append(' ') @@ -147,7 +147,7 @@ def get_temp_str(temp, colors=True): try: temp = float(temp) except ValueError: - return '{}°C'.format(temp) + return '{}'.format(temp) else: return '{}{:2.0f}°C'.format( '-' if temp < 0 else '', From 46080b43634f0317b78581c28a86493afc8acf06 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 22:25:44 -0700 Subject: [PATCH 027/186] Moved tmux sections to separate file --- .bin/Scripts/functions/hw_diags.py | 66 +-------------------------- .bin/Scripts/functions/tmux.py | 72 ++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 65 deletions(-) create mode 100644 .bin/Scripts/functions/tmux.py diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 892dff90..cc9a2c40 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,7 +4,7 @@ import json import re import time -from functions.common import * +from functions.tmux import * # STATIC VARIABLES ATTRIBUTES = { @@ -686,70 +686,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def tmux_kill_pane(*panes): - """Kill tmux pane by id.""" - cmd = ['tmux', 'kill-pane', '-t'] - for pane_id in panes: - print(pane_id) - run_program(cmd+[pane_id], check=False) - -def tmux_split_window( - lines=None, percent=None, - behind=False, vertical=False, - follow=False, target_pane=None, - command=None, text=None, watch=None): - """Run tmux split-window command and return pane_id as str.""" - # Bail early - if not lines and not percent: - raise Exception('Neither lines nor percent specified.') - if not command and not text and not watch: - raise Exception('No command, text, or watch file specified.') - - # Build cmd - cmd = ['tmux', 'split-window', '-PF', '#D'] - if behind: - cmd.append('-b') - if vertical: - cmd.append('-v') - else: - cmd.append('-h') - if not follow: - cmd.append('-d') - if lines is not None: - cmd.extend(['-l', str(lines)]) - elif percent is not None: - cmd.extend(['-p', str(percent)]) - if target_pane: - cmd.extend(['-t', str(target_pane)]) - - if command: - cmd.extend(command) - elif text: - cmd.extend(['echo-and-hold "{}"'.format(text)]) - elif watch: - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) - - # Run and return pane_id - result = run_program(cmd) - return result.stdout.decode().strip() - -def tmux_update_pane(pane_id, command=None, text=None): - """Respawn with either a new command or new text.""" - # Bail early - if not command and not text: - raise Exception('Neither command nor text specified.') - - cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] - if command: - cmd.extend(command) - elif text: - cmd.extend(['echo-and-hold "{}"'.format(text)]) - - run_program(cmd) - def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py new file mode 100644 index 00000000..ba00220f --- /dev/null +++ b/.bin/Scripts/functions/tmux.py @@ -0,0 +1,72 @@ +# Wizard Kit: Functions - tmux + +from functions.common import * + +def tmux_kill_pane(*panes): + """Kill tmux pane by id.""" + cmd = ['tmux', 'kill-pane', '-t'] + for pane_id in panes: + print(pane_id) + run_program(cmd+[pane_id], check=False) + +def tmux_split_window( + lines=None, percent=None, + behind=False, vertical=False, + follow=False, target_pane=None, + command=None, text=None, watch=None): + """Run tmux split-window command and return pane_id as str.""" + # Bail early + if not lines and not percent: + raise Exception('Neither lines nor percent specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') + + # Build cmd + cmd = ['tmux', 'split-window', '-PF', '#D'] + if behind: + cmd.append('-b') + if vertical: + cmd.append('-v') + else: + cmd.append('-h') + if not follow: + cmd.append('-d') + if lines is not None: + cmd.extend(['-l', str(lines)]) + elif percent is not None: + cmd.extend(['-p', str(percent)]) + if target_pane: + cmd.extend(['-t', str(target_pane)]) + + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + + # Run and return pane_id + result = run_program(cmd) + return result.stdout.decode().strip() + +def tmux_update_pane(pane_id, command=None, text=None): + """Respawn with either a new command or new text.""" + # Bail early + if not command and not text: + raise Exception('Neither command nor text specified.') + + cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if command: + cmd.extend(command) + elif text: + cmd.extend(['echo-and-hold "{}"'.format(text)]) + + run_program(cmd) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 5405b97eb1cb6b6747ea44e9f64a848e5977f175 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:09:42 -0700 Subject: [PATCH 028/186] Standalone sensor monitor working again --- .bin/Scripts/functions/sensors.py | 14 ++- .bin/Scripts/functions/tmux.py | 7 ++ .bin/Scripts/hw-sensors | 168 +----------------------------- .bin/Scripts/hw-sensors-monitor | 31 ++++++ 4 files changed, 56 insertions(+), 164 deletions(-) create mode 100755 .bin/Scripts/hw-sensors-monitor diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index b47f9afd..ec53e548 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -5,7 +5,7 @@ import json import re import shutil -from functions.common import * +from functions.tmux import * # STATIC VARIABLES TEMP_LIMITS = { @@ -153,6 +153,18 @@ def get_temp_str(temp, colors=True): '-' if temp < 0 else '', temp) +def monitor_sensors(monitor_pane, monitor_file): + """Continually update sensor data and report to screen.""" + sensor_data = get_sensor_data() + while True: + update_sensor_data(sensor_data) + with open(monitor_file, 'w') as f: + report = generate_report(sensor_data, 'Current', 'Max') + f.write('\n'.join(report)) + sleep(1) + if not tmux_poll_pane(monitor_pane): + break + def save_average_temp(sensor_data, temp_label, seconds=10): """Calculate average temps and record under temp_label, returns dict.""" clear_temps(sensor_data) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index ba00220f..10412a8c 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -9,6 +9,13 @@ def tmux_kill_pane(*panes): print(pane_id) run_program(cmd+[pane_id], check=False) +def tmux_poll_pane(pane_id): + """Check if pane exists, returns bool.""" + cmd = ['tmux', 'list-panes', '-F', '#D'] + result = run_program(cmd, check=False) + panes = result.stdout.decode().splitlines() + return pane_id in panes + def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, diff --git a/.bin/Scripts/hw-sensors b/.bin/Scripts/hw-sensors index f251c589..39ca7147 100755 --- a/.bin/Scripts/hw-sensors +++ b/.bin/Scripts/hw-sensors @@ -1,168 +1,10 @@ -#!/bin/python3 +#!/bin/bash # ## Wizard Kit: Sensor monitoring tool -import itertools -import os -import shutil -import sys +WINDOW_NAME="Hardware Sensors" +MONITOR="hw-sensors-monitor" -# Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) -from functions.common import * -from borrowed import sensors - -# STATIC VARIABLES -COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'GREEN': '\033[32m', - 'YELLOW': '\033[33m', - 'ORANGE': '\033[31;1m', - 'BLUE': '\033[34m' - } -TEMP_LIMITS = { - 'GREEN': 60, - 'YELLOW': 70, - 'ORANGE': 80, - 'RED': 90, - } - -# REGEX -REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') - -def color_temp(temp): - try: - temp = float(temp) - except ValueError: - return '{YELLOW}{temp}{CLEAR}'.format(temp=temp, **COLORS) - if temp > TEMP_LIMITS['RED']: - color = COLORS['RED'] - elif temp > TEMP_LIMITS['ORANGE']: - color = COLORS['ORANGE'] - elif temp > TEMP_LIMITS['YELLOW']: - color = COLORS['YELLOW'] - elif temp > TEMP_LIMITS['GREEN']: - color = COLORS['GREEN'] - elif temp > 0: - color = COLORS['BLUE'] - else: - color = COLORS['CLEAR'] - return '{color}{prefix}{temp:2.0f}°C{CLEAR}'.format( - color = color, - prefix = '-' if temp < 0 else '', - temp = temp, - **COLORS) - -def get_feature_string(chip, feature): - sfs = list(sensors.SubFeatureIterator(chip, feature)) # get a list of all subfeatures - label = sensors.get_label(chip, feature) - skipname = len(feature.name)+1 # skip common prefix - data = {} - - if feature.type != sensors.feature.TEMP: - # Skip non-temperature sensors - return None - - for sf in sfs: - name = sf.name[skipname:].decode("utf-8").strip() - try: - val = sensors.get_value(chip, sf.number) - except Exception: - # Ignore upstream sensor bugs and lie instead - val = -123456789 - if 'alarm' in name: - # Skip - continue - if '--nocolor' in sys.argv: - try: - temp = float(val) - except ValueError: - data[name] = ' {}°C'.format(val) - else: - data[name] = '{}{:2.0f}°C'.format( - '-' if temp < 0 else '', - temp) - else: - data[name] = color_temp(val) - - main_temp = data.pop('input', None) - if main_temp: - list_data = [] - for item in ['max', 'crit']: - if item in data: - list_data.append('{}: {}'.format(item, data.pop(item))) - list_data.extend( - ['{}: {}'.format(k, v) for k, v in sorted(data.items())]) - data_str = '{:18} {} {}'.format( - label, main_temp, ', '.join(list_data)) - else: - list_data.extend(sorted(data.items())) - list_data = ['{}: {}'.format(item[0], item[1]) for item in list_data] - data_str = '{:18} {}'.format(label, ', '.join(list_data)) - return data_str - -def join_columns(column1, column2, width=55): - return '{:<{}}{}'.format( - column1, - 55+len(column1)-len(REGEX_COLORS.sub('', column1)), - column2) - -if __name__ == '__main__': - try: - # Prep - sensors.init() - - # Get sensor data - chip_temps = {} - for chip in sensors.ChipIterator(): - chip_name = '{} ({})'.format( - sensors.chip_snprintf_name(chip), - sensors.get_adapter_name(chip.bus)) - chip_feats = [get_feature_string(chip, feature) - for feature in sensors.FeatureIterator(chip)] - # Strip empty/None items - chip_feats = [f for f in chip_feats if f] - - if chip_feats: - chip_temps[chip_name] = [chip_name, *chip_feats, ''] - - # Sort chips - sensor_temps = [] - for chip in [k for k in sorted(chip_temps.keys()) if 'coretemp' in k]: - sensor_temps.extend(chip_temps[chip]) - for chip in sorted(chip_temps.keys()): - if 'coretemp' not in chip: - sensor_temps.extend(chip_temps[chip]) - - # Wrap columns as needed - screen_size = shutil.get_terminal_size() - rows = screen_size.lines - 1 - if len(sensor_temps) > rows and screen_size.columns > 55*2: - sensor_temps = list(itertools.zip_longest( - sensor_temps[:rows], sensor_temps[rows:], fillvalue='')) - sensor_temps = [join_columns(a, b) for a, b in sensor_temps] - - # Print data - if sensor_temps: - for line in sensor_temps: - print_standard(line) - else: - if '--nocolor' in sys.argv: - print_standard('WARNING: No sensors found') - print_standard('\nPlease monitor temps manually') - else: - print_warning('WARNING: No sensors found') - print_standard('\nPlease monitor temps manually') - - # Done - sensors.cleanup() - exit_script() - except SystemExit: - sensors.cleanup() - pass - except: - sensors.cleanup() - major_exception() +# Start session +tmux new-session -n "$WINDOW_NAME" "$MONITOR" diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor new file mode 100755 index 00000000..91651f32 --- /dev/null +++ b/.bin/Scripts/hw-sensors-monitor @@ -0,0 +1,31 @@ +#!/bin/python3 +# +## Wizard Kit: Sensor monitoring tool + +import os +import sys + +# Init +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) +from functions.sensors import * +from functions.tmux import * +init_global_vars() + +if __name__ == '__main__': + try: + result = run_program(['mktemp']) + monitor_file = result.stdout.decode().strip() + print(monitor_file) + monitor_pane = tmux_split_window( + percent=1, vertical=True, watch=monitor_file) + cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] + run_program(cmd, check=False) + monitor_sensors(monitor_pane, monitor_file) + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 From c777d490913d14666c958f7015996170e6d32c49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:54:37 -0700 Subject: [PATCH 029/186] Added tmux_resize_pane() --- .bin/Scripts/functions/tmux.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 10412a8c..6726e3ff 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -16,6 +16,19 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes +def tmux_resize_pane(pane_id, x=None, y=None): + """Resize pane to specific hieght or width.""" + if not x and not y: + raise Exception('Neither height nor width specified.') + + cmd = ['tmux', 'resize-pane', '-t', pane_id] + if x: + cmd.extend(['-x', str(x)]) + elif y: + cmd.extend(['-y', str(y)]) + + run_program(cmd, check=False) + def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, From 5550cce8dbf4c77d67f15213b2e016a321b6bf87 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:55:15 -0700 Subject: [PATCH 030/186] Add background mode for monitoring sensors * This will be called by hw_diags.py to update a file in the background * NOTE: This uses a naive check before attempting to write data --- .bin/Scripts/functions/sensors.py | 2 +- .bin/Scripts/hw-sensors-monitor | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index ec53e548..41508927 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -162,7 +162,7 @@ def monitor_sensors(monitor_pane, monitor_file): report = generate_report(sensor_data, 'Current', 'Max') f.write('\n'.join(report)) sleep(1) - if not tmux_poll_pane(monitor_pane): + if monitor_pane and not tmux_poll_pane(monitor_pane): break def save_average_temp(sensor_data, temp_label, seconds=10): diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 91651f32..22067b91 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -13,14 +13,20 @@ from functions.tmux import * init_global_vars() if __name__ == '__main__': + background = False try: - result = run_program(['mktemp']) - monitor_file = result.stdout.decode().strip() - print(monitor_file) - monitor_pane = tmux_split_window( - percent=1, vertical=True, watch=monitor_file) - cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] - run_program(cmd, check=False) + if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): + background = True + monitor_file = sys.argv[1] + monitor_pane=None + else: + result = run_program(['mktemp']) + monitor_file = result.stdout.decode().strip() + if not background: + monitor_pane = tmux_split_window( + percent=1, vertical=True, watch=monitor_file) + cmd = ['tmux', 'resize-pane', '-Z', '-t', monitor_pane] + run_program(cmd, check=False) monitor_sensors(monitor_pane, monitor_file) exit_script() except SystemExit: From 74bb31e795c41b2466d33ef9fa76f169e3b8ad89 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 5 Dec 2018 23:57:38 -0700 Subject: [PATCH 031/186] Open temps monitor during run_mprime --- .bin/Scripts/functions/hw_diags.py | 17 +++++++++++++++++ .bin/Scripts/hw-sensors-monitor | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cc9a2c40..594f807a 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,6 +4,7 @@ import json import re import time +from functions.sensors import * from functions.tmux import * # STATIC VARIABLES @@ -614,6 +615,13 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" # Prep + _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(_sensors_out, 'w') as f: + f.write(' ') + sleep(1) + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) _title = '{}\n{}{}{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps', ': ' if 'Model name' in state.lscpu else '', @@ -621,6 +629,15 @@ def run_mprime_test(state): tmux_update_pane(state.panes['Top'], text=_title) state.tests['Prime95 & Temps']['Started'] = True update_progress_pane(state) + state.panes['mprime'] = tmux_split_window( + lines=10, vertical=True, text='Prime95 output goes here...') + state.panes['Temps'] = tmux_split_window( + behind=True, percent=80, vertical=True, watch=_sensors_out) + tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) + + # Start live monitor + pause() + monitor_proc.kill() # Get idle temps # Stress CPU diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 22067b91..42757748 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -18,7 +18,7 @@ if __name__ == '__main__': if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): background = True monitor_file = sys.argv[1] - monitor_pane=None + monitor_pane = None else: result = run_program(['mktemp']) monitor_file = result.stdout.decode().strip() From 30ba6516745347f6a979c7f0763284938c0c62e8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 00:10:51 -0700 Subject: [PATCH 032/186] Removing report wrapping section * Doesn't work properly with background processes --- .bin/Scripts/functions/sensors.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 41508927..29ceaa45 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -1,9 +1,7 @@ # Wizard Kit: Functions - Sensors -import itertools import json import re -import shutil from functions.tmux import * @@ -59,14 +57,6 @@ def generate_report(sensor_data, *temp_labels, colors=True): report.append(_line) report.append(' ') - # Wrap lines if necessary - screen_size = shutil.get_terminal_size() - rows = screen_size.lines - 1 - if len(report) > rows and screen_size.columns > 55*2: - report = list(itertools.zip_longest( - report[:rows], report[rows:], fillvalue='')) - report = [join_columns(a, b) for a, b in report] - # Handle empty reports (i.e. no sensors detected) if not report: report = [ From dc606a87806661a994d8472b188bdf5199fe90d0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 01:06:21 -0700 Subject: [PATCH 033/186] Main Prime95 sections working * Still need check results and update progress sections --- .bin/Scripts/functions/hw_diags.py | 102 ++++++++++++++++++++++++----- .bin/Scripts/functions/sensors.py | 6 +- .bin/Scripts/functions/tmux.py | 1 - .bin/Scripts/hw-diags-prime95 | 3 +- 4 files changed, 88 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 594f807a..a2aca6c8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -186,7 +186,8 @@ class State(): self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': '', 'Started': False, 'Status': ''}, + 'Result': '', 'Sensor Data': get_sensor_data(), + 'Started': False, 'Status': ''}, 'NVMe / SMART': {'Enabled': False, 'Order': 2}, 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, @@ -614,41 +615,106 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" - # Prep - _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(_sensors_out, 'w') as f: - f.write(' ') - sleep(1) - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], - pipe=True) + state.tests['Prime95 & Temps']['Started'] = True + update_progress_pane(state) + _sensor_data = state.tests['Prime95 & Temps']['Sensor Data'] + + # Update top pane _title = '{}\n{}{}{}'.format( TOP_PANE_TEXT, 'Prime95 & Temps', ': ' if 'Model name' in state.lscpu else '', state.lscpu.get('Model name', '')) tmux_update_pane(state.panes['Top'], text=_title) - state.tests['Prime95 & Temps']['Started'] = True - update_progress_pane(state) + + # Start live sensor monitor + _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(_sensors_out, 'w') as f: + f.write(' ') + f.flush() + sleep(0.5) + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) + + # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( - lines=10, vertical=True, text='Prime95 output goes here...') + lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( behind=True, percent=80, vertical=True, watch=_sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) - # Start live monitor - pause() - monitor_proc.kill() - # Get idle temps + clear_screen() + try_and_print( + message='Getting idle temps...', indent=0, + function=save_average_temp, cs='Done', + sensor_data=_sensor_data, temp_label='Idle') + # Stress CPU - # Get max temp + run_program(['apple-fans', 'max']) + tmux_update_pane( + state.panes['mprime'], + command=['hw-diags-prime95', global_vars['TmpDir']]) + time_limit = int(MPRIME_LIMIT) * 60 + try: + for i in range(time_limit): + clear_screen() + sec_left = time_limit - i + min_left = int(sec_left / 60) + if min_left > 0: + print_standard( + 'Running Prime95 ({} minute{} left)'.format( + min_left, + 's' if min_left != 1 else '')) + else: + print_standard( + 'Running Prime95 ({} second{} left)'.format( + sec_left, + 's' if sec_left != 1 else '')) + print_warning('If running too hot, press CTRL+c to abort the test') + update_sensor_data(_sensor_data) + sleep(1) + except KeyboardInterrupt: + # Catch CTRL+C + aborted = True + state.tests['Prime95 & Temps']['Result'] = 'Aborted' + print_warning('\nAborted.') + update_progress_pane(state) + + # Restart live monitor + monitor_proc = popen_program( + ['hw-sensors-monitor', _sensors_out], + pipe=True) + + # Stop Prime95 (twice for good measure) + tmux_kill_pane(state.panes['mprime']) + run_program(['killall', '-s', 'INT', 'mprime'], check=False) + # Get cooldown temp + run_program(['apple-fans', 'auto']) + clear_screen() + try_and_print( + message='Letting CPU cooldown for bit...', indent=0, + function=sleep, cs='Done', seconds=10) + try_and_print( + message='Getting cooldown temps...', indent=0, + function=save_average_temp, cs='Done', + sensor_data=_sensor_data, temp_label='Cooldown') + + # Check results + # TODO # Done - sleep(3) state.tests['Prime95 & Temps']['Result'] = 'Unknown' update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) + monitor_proc.kill() + + # TODO Testing + print('\n'.join(generate_report(_sensor_data, 'Idle', 'Max', 'Cooldown'))) + def run_network_test(): """Run network test.""" clear_screen() diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 29ceaa45..066dc446 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -46,14 +46,12 @@ def generate_report(sensor_data, *temp_labels, colors=True): for _source, _data in sorted(_sources.items()): # Source _line = '{:18} '.format(fix_sensor_str(_source)) - _temps = [] # Temps (skip label for Current) for _label in temp_labels: - _temps.append('{}{}{}'.format( + _line += '{}{}{} '.format( _label.lower() if _label != 'Current' else '', ': ' if _label != 'Current' else '', - get_temp_str(_data.get(_label, '???'), colors=colors))) - _line += ', '.join(_temps) + get_temp_str(_data.get(_label, '???'), colors=colors)) report.append(_line) report.append(' ') diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 6726e3ff..c9a92194 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -6,7 +6,6 @@ def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] for pane_id in panes: - print(pane_id) run_program(cmd+[pane_id], check=False) def tmux_poll_pane(pane_id): diff --git a/.bin/Scripts/hw-diags-prime95 b/.bin/Scripts/hw-diags-prime95 index 30c6994d..4927da76 100755 --- a/.bin/Scripts/hw-diags-prime95 +++ b/.bin/Scripts/hw-diags-prime95 @@ -14,5 +14,6 @@ if [ ! -d "$1" ]; then fi # Run Prime95 -mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "$1/prime.log" +cd "$1" +mprime -t | grep -iv --line-buffered 'stress.txt' | tee -a "prime.log" From ca4234b1c36ace85e82d98ff895885eda76b346d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 15:29:06 -0700 Subject: [PATCH 034/186] Added working_dir arg for tmux command sections --- .bin/Scripts/functions/hw_diags.py | 3 ++- .bin/Scripts/functions/tmux.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a2aca6c8..3c1fc5b3 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -654,7 +654,8 @@ def run_mprime_test(state): run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], - command=['hw-diags-prime95', global_vars['TmpDir']]) + command=['hw-diags-prime95', global_vars['TmpDir']], + working_dir=global_vars['TmpDir']) time_limit = int(MPRIME_LIMIT) * 60 try: for i in range(time_limit): diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index c9a92194..84ab96cc 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -32,7 +32,8 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - command=None, text=None, watch=None): + working_dir=None, command=None, + text=None, watch=None): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: @@ -57,6 +58,8 @@ def tmux_split_window( if target_pane: cmd.extend(['-t', str(target_pane)]) + if working_dir: + cmd.extend(['-c', working_dir]) if command: cmd.extend(command) elif text: @@ -71,13 +74,15 @@ def tmux_split_window( result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane(pane_id, command=None, text=None): +def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): """Respawn with either a new command or new text.""" # Bail early if not command and not text: raise Exception('Neither command nor text specified.') cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] + if working_dir: + cmd.extend(['-c', working_dir]) if command: cmd.extend(command) elif text: From a910f2cb033066438af1030a1cc34d1af6c07f6f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 18:27:19 -0700 Subject: [PATCH 035/186] Adjusted Prime95 countdown --- .bin/Scripts/functions/hw_diags.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3c1fc5b3..a76666ab 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -651,6 +651,8 @@ def run_mprime_test(state): sensor_data=_sensor_data, temp_label='Idle') # Stress CPU + print_log('Starting Prime95') + _abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -660,24 +662,23 @@ def run_mprime_test(state): try: for i in range(time_limit): clear_screen() - sec_left = time_limit - i - min_left = int(sec_left / 60) + sec_left = (time_limit - i) % 60 + min_left = int( (time_limit - i) / 60) + _status_str = 'Running Prime95 (' if min_left > 0: - print_standard( - 'Running Prime95 ({} minute{} left)'.format( - min_left, - 's' if min_left != 1 else '')) - else: - print_standard( - 'Running Prime95 ({} second{} left)'.format( - sec_left, - 's' if sec_left != 1 else '')) - print_warning('If running too hot, press CTRL+c to abort the test') + _status_str += '{} minute{}, '.format( + min_left, + 's' if min_left != 1 else '') + _status_str += '{} second{} left)'.format( + sec_left, + 's' if sec_left != 1 else '') + # Not using print wrappers to avoid flooding the log + print(_status_str) + print('{YELLOW}{msg}{CLEAR}'.format(msg=_abort_msg, **COLORS)) update_sensor_data(_sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C - aborted = True state.tests['Prime95 & Temps']['Result'] = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) From 12ff99eb3213569a776363f630c3767efd596da6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 6 Dec 2018 18:27:43 -0700 Subject: [PATCH 036/186] Set LogDir for non-quick tests --- .bin/Scripts/functions/hw_diags.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a76666ab..e1b97df9 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -180,8 +180,7 @@ class State(): self.devs = [] self.finished = False self.panes = {} - # TODO Switch to LogDir - self.progress_out = '{}/progress.out'.format(global_vars['TmpDir']) + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.started = False self.tests = { @@ -221,6 +220,16 @@ class State(): for k in ['Result', 'Started', 'Status']: self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + # Update LogDir + if not self.quick_mode: + global_vars['LogDir'] = '{}/Logs/{}_{}'.format( + global_vars['Env']['HOME'], + get_ticket_number(), + time.strftime('%Y-%m-%d_%H%M_%z')) + os.makedirs(global_vars['LogDir'], exist_ok=True) + global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( + global_vars['LogDir']) + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) @@ -703,6 +712,15 @@ def run_mprime_test(state): function=save_average_temp, cs='Done', sensor_data=_sensor_data, temp_label='Cooldown') + # Move logs to Ticket folder + for item in os.scandir(global_vars['TmpDir']): + try: + shutil.move(item.path, global_vars['LogDir']) + except Exception: + print_error('ERROR: Failed to move "{}" to "{}"'.format( + item.path, + global_vars['LogDir'])) + # Check results # TODO From 6a3ef60881ed43800605b4692e79836a2d978c03 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:41:29 -0700 Subject: [PATCH 037/186] Added CpuObj and renamed dev names to disk * This should make the code more clear * The CpuObj is similar to DiskObj to abstract the device/tests calls * New calls will be like: run_test(state, dev) --- .bin/Scripts/functions/hw_diags.py | 204 +++++++++++++---------------- 1 file changed, 90 insertions(+), 114 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index e1b97df9..a2c8293d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -67,32 +67,43 @@ SIDE_PANE_WIDTH = 20 TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes -class DevObj(): - """Device object for tracking device specific data.""" - def __init__(self, state, dev_path): - self.failing = False +class CpuObj(): + """Object for tracking CPU specific data.""" + def __init__(self): + self.lscpu = {} + self.tests = {} + self.get_details() + + def get_details(self): + """Get CPU details from lscpu.""" + cmd = ['lscpu', '--json'] + try: + result = run_program(cmd, check=False) + json_data = json.loads(result.stdout.decode()) + except Exception: + # Ignore and leave self.lscpu empty + return + for line in json_data.get('lscpu', []): + _field = line.get('field', None).replace(':', '') + _data = line.get('data', None) + if not _field and not _data: + # Skip + print_warning(_field, _data) + pause() + continue + self.lscpu[_field] = _data + +class DiskObj(): + """Object for tracking disk specific data.""" + def __init__(self, disk_path): self.labels = [] self.lsblk = {} - self.name = re.sub(r'^.*/(.*)', r'\1', dev_path) + self.name = re.sub(r'^.*/(.*)', r'\1', disk_path) self.nvme_attributes = {} - self.override = False - self.path = dev_path + self.path = disk_path self.smart_attributes = {} self.smartctl = {} - self.state = state - self.tests = { - 'NVMe / SMART': { - 'Result': '', 'Started': False, 'Status': '', 'Order': 1}, - 'badblocks': { - 'Result': '', 'Started': False, 'Status': '', 'Order': 2}, - 'I/O Benchmark': { - 'Result': '', - 'Started': False, - 'Status': '', - 'Read Rates': [], - 'Graph Data': [], - 'Order': 3}, - } + self.tests = {} self.get_details() self.get_smart_details() @@ -122,9 +133,9 @@ class DevObj(): self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') # Build list of labels - for dev in [self.lsblk, *self.lsblk.get('children', [])]: - self.labels.append(dev.get('label', '')) - self.labels.append(dev.get('partlabel', '')) + for disk in [self.lsblk, *self.lsblk.get('children', [])]: + self.labels.append(disk.get('label', '')) + self.labels.append(disk.get('partlabel', '')) self.labels = [str(label) for label in self.labels if label] def get_smart_details(self): @@ -158,31 +169,15 @@ class DevObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def update_progress(self): - """Update status strings.""" - for k, v in self.tests.items(): - if self.state.tests[k]['Enabled']: - _status = '' - if not v['Status']: - _status = 'Pending' - if v['Started']: - if v['Result']: - _status = v['Result'] - else: - _status = 'Working' - if _status: - v['Status'] = build_status_string(self.name, _status) class State(): """Object to track device objects and overall state.""" def __init__(self): - self.lscpu = {} - self.devs = [] - self.finished = False + self.cpu = None + self.disks = [] self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False - self.started = False self.tests = { 'Prime95 & Temps': {'Enabled': False, 'Order': 1, 'Result': '', 'Sensor Data': get_sensor_data(), @@ -191,32 +186,10 @@ class State(): 'badblocks': {'Enabled': False, 'Order': 3}, 'I/O Benchmark': {'Enabled': False, 'Order': 4}, } - self.get_cpu_details() - - def get_cpu_details(self): - """Get CPU details from lscpu.""" - cmd = ['lscpu', '--json'] - try: - result = run_program(cmd, check=False) - json_data = json.loads(result.stdout.decode()) - except Exception as err: - # Ignore and leave self.cpu empty - print_error(err) - pause() - return - for line in json_data.get('lscpu', []): - _field = line.get('field', None).replace(':', '') - _data = line.get('data', None) - if not _field and not _data: - # Skip - print_warning(_field, _data) - pause() - continue - self.lscpu[_field] = _data def init(self): - """Scan for block devices and reset all tests.""" - self.devs = [] + """Set log and add devices.""" + self.disks = [] for k in ['Result', 'Started', 'Status']: self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' @@ -230,27 +203,30 @@ class State(): global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( global_vars['LogDir']) + # Add CPU + self.cpu = CpuObj() + # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] result = run_program(cmd, check=False) json_data = json.loads(result.stdout.decode()) - for dev in json_data['blockdevices']: - skip_dev = False - dev_obj = DevObj(self, dev['name']) + for disk in json_data['blockdevices']: + skip_disk = False + disk_obj = DiskObj(disk['name']) - # Skip loopback and optical devices - if dev_obj.lsblk['type'] in ['loop', 'rom']: - skip_dev = True + # Skip loopback devices, optical devices, etc + if disk_obj.lsblk['type'] != 'disk': + skip_disk = True - # Skip WK devices + # Skip WK disks wk_label_regex = r'{}_(LINUX|UFD)'.format(KIT_NAME_SHORT) - for label in dev_obj.labels: + for label in disk_obj.labels: if re.search(wk_label_regex, label, re.IGNORECASE): - skip_dev = True + skip_disk = True - # Add device - if not skip_dev: - self.devs.append(dev_obj) + # Add disk + if not skip_disk: + self.disks.append(disk_obj) def update_progress(self): """Update status strings.""" @@ -313,32 +289,32 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) -def check_dev_attributes(dev): - """Check if device should be tested and allow overrides.""" +def check_disk_attributes(disk): + """Check if disk should be tested and allow overrides.""" needs_override = False print_standard(' {size:>6} ({tran}) {model} {serial}'.format( - **dev.lsblk)) + **disk.lsblk)) # General checks - if not dev.nvme_attributes and not dev.smart_attributes: + if not disk.nvme_attributes and not disk.smart_attributes: needs_override = True print_warning( ' WARNING: No NVMe or SMART attributes available for: {}'.format( - dev.path)) + disk.path)) # NVMe checks - # TODO check all tracked attributes and set dev.failing if needed + # TODO check all tracked attributes and set disk.failing if needed # SMART checks - # TODO check all tracked attributes and set dev.failing if needed + # TODO check all tracked attributes and set disk.failing if needed # Ask for override if necessary if needs_override: if ask(' Run tests on this device anyway?'): - # TODO Set override for this dev + # TODO Set override for this disk pass else: - for v in dev.tests.values(): + for v in disk.tests.values(): # Started is set to True to fix the status string v['Result'] = 'Skipped' v['Started'] = True @@ -551,11 +527,11 @@ def run_badblocks_test(state): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test()') - for dev in state.devs: - dev.tests['badblocks']['Started'] = True + for disk in state.disks: + disk.tests['badblocks']['Started'] = True update_progress_pane(state) sleep(3) - dev.tests['badblocks']['Result'] = 'OVERRIDE' + disk.tests['badblocks']['Result'] = 'OVERRIDE' update_progress_pane(state) def run_hw_tests(state): @@ -610,11 +586,11 @@ def run_io_benchmark(state): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark()') - for dev in state.devs: - dev.tests['I/O Benchmark']['Started'] = True + for disk in state.disks: + disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) sleep(3) - dev.tests['I/O Benchmark']['Result'] = 'Unknown' + disk.tests['I/O Benchmark']['Result'] = 'Unknown' update_progress_pane(state) def run_keyboard_test(): @@ -741,42 +717,42 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart(state): +def run_nvme_smart_tests(state): """TODO""" - for dev in state.devs: + for disk in state.disks: tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **dev.lsblk)) - dev.tests['NVMe / SMART']['Started'] = True + t=TOP_PANE_TEXT, **disk.lsblk)) + disk.tests['NVMe / SMART']['Started'] = True update_progress_pane(state) - if dev.nvme_attributes: - run_nvme_tests(state, dev) - elif dev.smart_attributes: - run_smart_tests(state, dev) + if disk.nvme_attributes: + run_nvme_tests(state, disk) + elif disk.smart_attributes: + run_smart_tests(state, disk) else: - print_standard('TODO: run_nvme_smart({})'.format( - dev.path)) + print_standard('TODO: run_nvme_smart_tests({})'.format( + disk.path)) print_warning( " WARNING: Device {} doesn't support NVMe or SMART test".format( - dev.path)) - dev.tests['NVMe / SMART']['Status'] = 'N/A' - dev.tests['NVMe / SMART']['Result'] = 'N/A' + disk.path)) + disk.tests['NVMe / SMART']['Status'] = 'N/A' + disk.tests['NVMe / SMART']['Result'] = 'N/A' update_progress_pane(state) sleep(3) -def run_nvme_tests(state, dev): +def run_nvme_tests(state, disk): """TODO""" - print_standard('TODO: run_nvme_test({})'.format(dev.path)) + print_standard('TODO: run_nvme_test({})'.format(disk.path)) sleep(3) - dev.tests['NVMe / SMART']['Result'] = 'CS' + disk.tests['NVMe / SMART']['Result'] = 'CS' update_progress_pane(state) -def run_smart_tests(state, dev): +def run_smart_tests(state, disk): """TODO""" - print_standard('TODO: run_smart_tests({})'.format(dev.path)) + print_standard('TODO: run_smart_tests({})'.format(disk.path)) sleep(3) - dev.tests['NVMe / SMART']['Result'] = 'CS' + disk.tests['NVMe / SMART']['Result'] = 'CS' update_progress_pane(state) def secret_screensaver(screensaver=None): @@ -872,8 +848,8 @@ def update_progress_pane(state): if 'Prime95' not in k and v['Enabled']: output.append('{BLUE}{test_name}{CLEAR}'.format( test_name=k, **COLORS)) - for dev in state.devs: - output.append(dev.tests[k]['Status']) + for disk in state.disks: + output.append(disk.tests[k]['Status']) output.append(' ') # Add line-endings From 0390290f10d44ec9ab8b41d3bfe374b61744795b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:46:17 -0700 Subject: [PATCH 038/186] Added TestObj() * This object will track test specific vars and results * Moved status code into TestObj * Test calls will now be: run_test(state, dev, test_obj) * NOTE: Code is not done and is quite broken --- .bin/Scripts/functions/hw_diags.py | 120 ++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a2c8293d..861305df 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -64,6 +64,12 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 +TESTS_CPU = ['Prime95 & Temps'] +TESTS_DISK = [ + 'I/O Benchmark', + 'NVMe / SMART', + 'badblocks', + ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) # Classes @@ -169,6 +175,10 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + def safety_check(self): + """Check enabled tests and verify it's safe to run them.""" + # TODO + pass class State(): """Object to track device objects and overall state.""" @@ -179,13 +189,31 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = { - 'Prime95 & Temps': {'Enabled': False, 'Order': 1, - 'Result': '', 'Sensor Data': get_sensor_data(), - 'Started': False, 'Status': ''}, - 'NVMe / SMART': {'Enabled': False, 'Order': 2}, - 'badblocks': {'Enabled': False, 'Order': 3}, - 'I/O Benchmark': {'Enabled': False, 'Order': 4}, - } + 'Prime95 & Temps': { + 'Enabled': False, + 'Function': run_mprime_test, + 'Objects': [], + 'Order': 1, + }, + 'NVMe / SMART': { + 'Enabled': False, + 'Function': run_nvme_smart_tests, + 'Objects': [], + 'Order': 2, + }, + 'badblocks': { + 'Enabled': False, + 'Function': run_badblocks_test, + 'Objects': [], + 'Order': 3, + }, + 'I/O Benchmark': { + 'Enabled': False, + 'Function': run_io_benchmark, + 'Objects': [], + 'Order': 4, + }, + } def init(self): """Set log and add devices.""" @@ -228,26 +256,32 @@ class State(): if not skip_disk: self.disks.append(disk_obj) - def update_progress(self): - """Update status strings.""" - # Prime95 - p = self.tests['Prime95 & Temps'] - if p['Enabled']: - _status = '' - if not p['Status']: - _status = 'Pending' - if p['Started']: - if p['Result']: - _status = p['Result'] - else: - _status = 'Working' - if _status: - p['Status'] = build_status_string( - 'Prime95', _status, info_label=True) +class TestObj(): + """Object to track test data.""" + def __init__(self, label, info_label=False): + self.started = False + self.passed = False + self.failed = False + self.report = '' + self.status = '' + self.label = label + self.info_label = info_label + self.disabled = False + self.update_status() - # Disks - for dev in self.devs: - dev.update_progress() + def update_status(self, new_status=None): + """Update status strings.""" + if self.disabled: + return + if new_status: + self.status = build_status_string( + self.label, new_status, self.info_label) + elif not self.status: + self.status = build_status_string( + self.label, 'Pending', self.info_label) + elif self.started and 'Pending' in self.status: + self.status = build_status_string( + self.label, 'Working', self.info_label) # Functions def build_outer_panes(state): @@ -543,7 +577,7 @@ def run_hw_tests(state): update_progress_pane(state) build_outer_panes(state) - # Run test(s) + # Show selected tests and create TestObj()s print_info('Selected Tests:') for k, v in sorted( state.tests.items(), @@ -554,21 +588,36 @@ def run_hw_tests(state): 'Enabled' if v['Enabled'] else 'Disabled', COLORS['CLEAR'], QUICK_LABEL if state.quick_mode and 'NVMe' in k else '')) + if v['Enabled']: + # Create TestObj and track under both CpuObj/DiskObj and State + if k in TESTS_CPU: + test_obj = TestObj(info_label=True) + state.cpu.tests[k] = test_obj + v['Objects'].append(test_obj) + elif k in TESTS_DISK: + for disk in state.disks: + test_obj = TestObj() + disk.tests[k] = test_obj + v['Objects'].append(test_obj) print_standard('') - # Check devices if necessary - if (state.tests['badblocks']['Enabled'] - or state.tests['I/O Benchmark']['Enabled']): - print_info('Selected Disks:') - for dev in state.devs: - check_dev_attributes(dev) - print_standard('') + # Run safety checks + for disk in state.disks: + disk.safety_check() + + # Run tests + for k, v in sorted( + state.tests.items(), + key=lambda kv: kv[1]['Order']): + if v['Enabled']: + # TODO + pass # Run tests if state.tests['Prime95 & Temps']['Enabled']: run_mprime_test(state) if state.tests['NVMe / SMART']['Enabled']: - run_nvme_smart(state) + run_nvme_smart_tests(state) if state.tests['badblocks']['Enabled']: run_badblocks_test(state) if state.tests['I/O Benchmark']['Enabled']: @@ -835,7 +884,6 @@ def update_io_progress(percent, rate, progress_file): def update_progress_pane(state): """Update progress file for side pane.""" output = [] - state.update_progress() # Prime95 output.append(state.tests['Prime95 & Temps']['Status']) From 49471663f50ac3fe239a0417063b8ee814422976 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 17:50:11 -0700 Subject: [PATCH 039/186] Use OrderedDicts to avoid lambda sorting --- .bin/Scripts/functions/hw_diags.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 861305df..05c7ebb7 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -4,6 +4,7 @@ import json import re import time +from collections import OrderedDict from functions.sensors import * from functions.tmux import * @@ -109,7 +110,7 @@ class DiskObj(): self.path = disk_path self.smart_attributes = {} self.smartctl = {} - self.tests = {} + self.tests = OrderedDict() self.get_details() self.get_smart_details() @@ -188,32 +189,28 @@ class State(): self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False - self.tests = { + self.tests = OrderedDict({ 'Prime95 & Temps': { 'Enabled': False, 'Function': run_mprime_test, 'Objects': [], - 'Order': 1, }, 'NVMe / SMART': { 'Enabled': False, 'Function': run_nvme_smart_tests, 'Objects': [], - 'Order': 2, }, 'badblocks': { 'Enabled': False, 'Function': run_badblocks_test, 'Objects': [], - 'Order': 3, }, 'I/O Benchmark': { 'Enabled': False, 'Function': run_io_benchmark, 'Objects': [], - 'Order': 4, }, - } + }) def init(self): """Set log and add devices.""" @@ -579,9 +576,7 @@ def run_hw_tests(state): # Show selected tests and create TestObj()s print_info('Selected Tests:') - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): print_standard(' {:<15} {}{}{} {}'.format( k, COLORS['GREEN'] if v['Enabled'] else COLORS['RED'], @@ -606,9 +601,7 @@ def run_hw_tests(state): disk.safety_check() # Run tests - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): if v['Enabled']: # TODO pass @@ -890,9 +883,7 @@ def update_progress_pane(state): output.append(' ') # Disks - for k, v in sorted( - state.tests.items(), - key=lambda kv: kv[1]['Order']): + for k, v in state.tests.items(): if 'Prime95' not in k and v['Enabled']: output.append('{BLUE}{test_name}{CLEAR}'.format( test_name=k, **COLORS)) From 941a5537669a0c19f9c37f5f139211e1154b5c91 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:16:31 -0700 Subject: [PATCH 040/186] Renamed "Prime95 & Temps" to "Prime95" for brevity --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 05c7ebb7..fd8b0ebb 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -65,7 +65,7 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 -TESTS_CPU = ['Prime95 & Temps'] +TESTS_CPU = ['Prime95'] TESTS_DISK = [ 'I/O Benchmark', 'NVMe / SMART', @@ -190,7 +190,7 @@ class State(): self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = OrderedDict({ - 'Prime95 & Temps': { + 'Prime95': { 'Enabled': False, 'Function': run_mprime_test, 'Objects': [], @@ -216,7 +216,7 @@ class State(): """Set log and add devices.""" self.disks = [] for k in ['Result', 'Started', 'Status']: - self.tests['Prime95 & Temps'][k] = False if k == 'Started' else '' + self.tests['Prime95'][k] = False if k == 'Started' else '' # Update LogDir if not self.quick_mode: @@ -443,7 +443,7 @@ def menu_diags(state, args): {'Base Name': 'Full Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic', 'Enabled': False}, {'Base Name': 'Disk Diagnostic (Quick)', 'Enabled': False}, - {'Base Name': 'Prime95 & Temps', 'Enabled': False, 'CRLF': True}, + {'Base Name': 'Prime95', 'Enabled': False, 'CRLF': True}, {'Base Name': 'NVMe / SMART', 'Enabled': False}, {'Base Name': 'badblocks', 'Enabled': False}, {'Base Name': 'I/O Benchmark', 'Enabled': False}, @@ -642,13 +642,13 @@ def run_keyboard_test(): def run_mprime_test(state): """Test CPU with Prime95 and track temps.""" - state.tests['Prime95 & Temps']['Started'] = True + state.tests['Prime95']['Started'] = True update_progress_pane(state) - _sensor_data = state.tests['Prime95 & Temps']['Sensor Data'] + _sensor_data = state.tests['Prime95']['Sensor Data'] # Update top pane _title = '{}\n{}{}{}'.format( - TOP_PANE_TEXT, 'Prime95 & Temps', + TOP_PANE_TEXT, 'Prime95', ': ' if 'Model name' in state.lscpu else '', state.lscpu.get('Model name', '')) tmux_update_pane(state.panes['Top'], text=_title) @@ -706,7 +706,7 @@ def run_mprime_test(state): sleep(1) except KeyboardInterrupt: # Catch CTRL+C - state.tests['Prime95 & Temps']['Result'] = 'Aborted' + state.tests['Prime95']['Result'] = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) @@ -743,7 +743,7 @@ def run_mprime_test(state): # TODO # Done - state.tests['Prime95 & Temps']['Result'] = 'Unknown' + state.tests['Prime95']['Result'] = 'Unknown' update_progress_pane(state) # Cleanup @@ -879,7 +879,7 @@ def update_progress_pane(state): output = [] # Prime95 - output.append(state.tests['Prime95 & Temps']['Status']) + output.append(state.tests['Prime95']['Status']) output.append(' ') # Disks From 668c7c4c6a440fdcee3fa281d0590d728b634545 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:32:03 -0700 Subject: [PATCH 041/186] Updated run_mprime_test to use test_obj --- .bin/Scripts/functions/hw_diags.py | 95 +++++++++++++++--------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index fd8b0ebb..cd64e2e8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -80,6 +80,7 @@ class CpuObj(): self.lscpu = {} self.tests = {} self.get_details() + self.name = self.lscpu.get('Model name', 'Unknown CPU') def get_details(self): """Get CPU details from lscpu.""" @@ -255,15 +256,16 @@ class State(): class TestObj(): """Object to track test data.""" - def __init__(self, label, info_label=False): - self.started = False - self.passed = False - self.failed = False - self.report = '' - self.status = '' + def __init__(self, dev, label=None, info_label=False): + self.dev = dev self.label = label self.info_label = info_label self.disabled = False + self.failed = False + self.passed = False + self.report = '' + self.started = False + self.status = '' self.update_status() def update_status(self, new_status=None): @@ -552,12 +554,13 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_badblocks_test(state): +def run_badblocks_test(state, test_obj): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) - print_standard('TODO: run_badblocks_test()') + print_standard('TODO: run_badblocks_test({})'.format( + test_obj.dev.path)) for disk in state.disks: disk.tests['badblocks']['Started'] = True update_progress_pane(state) @@ -586,12 +589,13 @@ def run_hw_tests(state): if v['Enabled']: # Create TestObj and track under both CpuObj/DiskObj and State if k in TESTS_CPU: - test_obj = TestObj(info_label=True) + test_obj = TestObj( + dev=state.cpu, label='Prime95', info_label=True) state.cpu.tests[k] = test_obj v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj() + test_obj = TestObj(dev=k) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -601,20 +605,13 @@ def run_hw_tests(state): disk.safety_check() # Run tests + ## Because state.tests is an OrderedDict and the disks were added + ## in order, the tests will be run in order. for k, v in state.tests.items(): if v['Enabled']: - # TODO - pass - - # Run tests - if state.tests['Prime95 & Temps']['Enabled']: - run_mprime_test(state) - if state.tests['NVMe / SMART']['Enabled']: - run_nvme_smart_tests(state) - if state.tests['badblocks']['Enabled']: - run_badblocks_test(state) - if state.tests['I/O Benchmark']['Enabled']: - run_io_benchmark(state) + f = v['Function'] + for test_obj in v['Objects']: + f(state, test_obj) # Done pause('Press Enter to return to main menu... ') @@ -622,12 +619,13 @@ def run_hw_tests(state): # Cleanup tmux_kill_pane(*state.panes.values()) -def run_io_benchmark(state): +def run_io_benchmark(state, test_obj): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) - print_standard('TODO: run_io_benchmark()') + print_standard('TODO: run_io_benchmark({})'.format( + test_obj.dev.path)) for disk in state.disks: disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) @@ -640,34 +638,32 @@ def run_keyboard_test(): clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) -def run_mprime_test(state): +def run_mprime_test(state, test_obj): """Test CPU with Prime95 and track temps.""" state.tests['Prime95']['Started'] = True update_progress_pane(state) - _sensor_data = state.tests['Prime95']['Sensor Data'] + test_obj.sensor_data = get_sensor_data() # Update top pane - _title = '{}\n{}{}{}'.format( - TOP_PANE_TEXT, 'Prime95', - ': ' if 'Model name' in state.lscpu else '', - state.lscpu.get('Model name', '')) - tmux_update_pane(state.panes['Top'], text=_title) + test_obj.title = '{}\nPrime95: {}'.format( + TOP_PANE_TEXT, test_obj.dev.name) + tmux_update_pane(state.panes['Top'], text=test_obj.title) # Start live sensor monitor - _sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(_sensors_out, 'w') as f: + test_obj.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(test_obj.sensors_out, 'w') as f: f.write(' ') f.flush() sleep(0.5) - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], + test_obj.monitor_proc = popen_program( + ['hw-sensors-monitor', test_obj.sensors_out], pipe=True) # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( - behind=True, percent=80, vertical=True, watch=_sensors_out) + behind=True, percent=80, vertical=True, watch=test_obj.sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) # Get idle temps @@ -675,11 +671,11 @@ def run_mprime_test(state): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=_sensor_data, temp_label='Idle') + sensor_data=test_obj.sensor_data, temp_label='Idle') # Stress CPU print_log('Starting Prime95') - _abort_msg = 'If running too hot, press CTRL+c to abort the test' + test_obj.abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -701,8 +697,8 @@ def run_mprime_test(state): 's' if sec_left != 1 else '') # Not using print wrappers to avoid flooding the log print(_status_str) - print('{YELLOW}{msg}{CLEAR}'.format(msg=_abort_msg, **COLORS)) - update_sensor_data(_sensor_data) + print('{YELLOW}{msg}{CLEAR}'.format(msg=test_obj.abort_msg, **COLORS)) + update_sensor_data(test_obj.sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C @@ -711,8 +707,8 @@ def run_mprime_test(state): update_progress_pane(state) # Restart live monitor - monitor_proc = popen_program( - ['hw-sensors-monitor', _sensors_out], + test_obj.monitor_proc = popen_program( + ['hw-sensors-monitor', test_obj.sensors_out], pipe=True) # Stop Prime95 (twice for good measure) @@ -728,7 +724,7 @@ def run_mprime_test(state): try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=_sensor_data, temp_label='Cooldown') + sensor_data=test_obj.sensor_data, temp_label='Cooldown') # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -741,6 +737,12 @@ def run_mprime_test(state): # Check results # TODO + _log = '{}/results.txt'.format(global_vars['LogDir']) + if os.path.exists(_log): + with open(_log, 'r') as f: + for line in f.readlines(): + if re.search(r'(error|fail)', line, re.IGNORECASE): + state.tests['Prime95']['Result'] = 'NS' # Done state.tests['Prime95']['Result'] = 'Unknown' @@ -748,10 +750,11 @@ def run_mprime_test(state): # Cleanup tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) - monitor_proc.kill() + test_obj.monitor_proc.kill() # TODO Testing - print('\n'.join(generate_report(_sensor_data, 'Idle', 'Max', 'Cooldown'))) + print('\n'.join( + generate_report(test_obj.sensor_data, 'Idle', 'Max', 'Cooldown'))) def run_network_test(): """Run network test.""" @@ -759,7 +762,7 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart_tests(state): +def run_nvme_smart_tests(state, test_obj): """TODO""" for disk in state.disks: tmux_update_pane( From d88a9f39f2a0a85ce4e3fe9dde99a39a64c39046 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:36:24 -0700 Subject: [PATCH 042/186] Added tmux_kill_all_panes() --- .bin/Scripts/functions/tmux.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 84ab96cc..e1066e18 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -2,6 +2,13 @@ from functions.common import * +def tmux_kill_all_panes(pane_id=None): + """Kill all tmux panes except the active pane or pane_id if specified.""" + cmd = ['tmux', 'kill-pane', '-a'] + if pane_id: + cmd.extend(['-t', pane_id]) + run_program(cmd, check=False) + def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] From 465a3b42fb0cc92c03600b820bca0007fd9c3227 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 8 Dec 2018 18:36:50 -0700 Subject: [PATCH 043/186] Kill all tmux panes before exiting --- .bin/Scripts/hw-diags-menu | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index e60f8fc4..6f0247cd 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -9,20 +9,24 @@ import sys os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.hw_diags import * +from functions.tmux import * init_global_vars() if __name__ == '__main__': - try: - # Show menu - state = State() - menu_diags(state, sys.argv) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + try: + # Show menu + state = State() + menu_diags(state, sys.argv) + # Done + #print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + tmux_kill_all_panes() + pass + except: + tmux_kill_all_panes() + major_exception() + +# vim: sts=2 sw=2 ts=2 From bb93386fa0b7dc9dec3f21529d9279cbbec62bc8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 16:32:00 -0700 Subject: [PATCH 044/186] Updated Prime95 checks --- .bin/Scripts/functions/hw_diags.py | 98 +++++++++++++++++++----------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cd64e2e8..8cebd636 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -257,6 +257,7 @@ class State(): class TestObj(): """Object to track test data.""" def __init__(self, dev, label=None, info_label=False): + self.aborted = False self.dev = dev self.label = label self.info_label = info_label @@ -554,13 +555,13 @@ def run_audio_test(): run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_badblocks_test(state, test_obj): +def run_badblocks_test(state, test): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test({})'.format( - test_obj.dev.path)) + test.dev.path)) for disk in state.disks: disk.tests['badblocks']['Started'] = True update_progress_pane(state) @@ -619,13 +620,13 @@ def run_hw_tests(state): # Cleanup tmux_kill_pane(*state.panes.values()) -def run_io_benchmark(state, test_obj): +def run_io_benchmark(state, test): """TODO""" tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark({})'.format( - test_obj.dev.path)) + test.dev.path)) for disk in state.disks: disk.tests['I/O Benchmark']['Started'] = True update_progress_pane(state) @@ -638,32 +639,33 @@ def run_keyboard_test(): clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) -def run_mprime_test(state, test_obj): +def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" - state.tests['Prime95']['Started'] = True + test.started = True + test.update_status() update_progress_pane(state) - test_obj.sensor_data = get_sensor_data() + test.sensor_data = get_sensor_data() # Update top pane - test_obj.title = '{}\nPrime95: {}'.format( - TOP_PANE_TEXT, test_obj.dev.name) - tmux_update_pane(state.panes['Top'], text=test_obj.title) + test.title = '{}\nPrime95: {}'.format( + TOP_PANE_TEXT, test.dev.name) + tmux_update_pane(state.panes['Top'], text=test.title) # Start live sensor monitor - test_obj.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) - with open(test_obj.sensors_out, 'w') as f: + test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) + with open(test.sensors_out, 'w') as f: f.write(' ') f.flush() sleep(0.5) - test_obj.monitor_proc = popen_program( - ['hw-sensors-monitor', test_obj.sensors_out], + test.monitor_proc = popen_program( + ['hw-sensors-monitor', test.sensors_out], pipe=True) # Create monitor and worker panes state.panes['mprime'] = tmux_split_window( lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( - behind=True, percent=80, vertical=True, watch=test_obj.sensors_out) + behind=True, percent=80, vertical=True, watch=test.sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) # Get idle temps @@ -671,11 +673,11 @@ def run_mprime_test(state, test_obj): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test_obj.sensor_data, temp_label='Idle') + sensor_data=test.sensor_data, temp_label='Idle') # Stress CPU print_log('Starting Prime95') - test_obj.abort_msg = 'If running too hot, press CTRL+c to abort the test' + test.abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( state.panes['mprime'], @@ -697,18 +699,19 @@ def run_mprime_test(state, test_obj): 's' if sec_left != 1 else '') # Not using print wrappers to avoid flooding the log print(_status_str) - print('{YELLOW}{msg}{CLEAR}'.format(msg=test_obj.abort_msg, **COLORS)) - update_sensor_data(test_obj.sensor_data) + print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) + update_sensor_data(test.sensor_data) sleep(1) except KeyboardInterrupt: # Catch CTRL+C - state.tests['Prime95']['Result'] = 'Aborted' + test.aborted = True + test.status = 'Aborted' print_warning('\nAborted.') update_progress_pane(state) # Restart live monitor - test_obj.monitor_proc = popen_program( - ['hw-sensors-monitor', test_obj.sensors_out], + test.monitor_proc = popen_program( + ['hw-sensors-monitor', test.sensors_out], pipe=True) # Stop Prime95 (twice for good measure) @@ -724,7 +727,7 @@ def run_mprime_test(state, test_obj): try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test_obj.sensor_data, temp_label='Cooldown') + sensor_data=test.sensor_data, temp_label='Cooldown') # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -736,25 +739,52 @@ def run_mprime_test(state, test_obj): global_vars['LogDir'])) # Check results - # TODO - _log = '{}/results.txt'.format(global_vars['LogDir']) - if os.path.exists(_log): - with open(_log, 'r') as f: - for line in f.readlines(): - if re.search(r'(error|fail)', line, re.IGNORECASE): - state.tests['Prime95']['Result'] = 'NS' + test.logs = {} + for log in ['results.txt', 'prime.log']: + _data = '' + log_path = '{}/{}'.format(global_vars['LogDir'], log) + + # Read and save log + try: + with open(log_path, 'r') as f: + _data = f.read() + test.logs[log] = _data.splitlines() + except FileNotFoundError: + # Ignore since files may be missing for slower CPUs + pass + + # results.txt: NS check + if log == 'results.txt': + if re.search(r'(error|fail)', _data, re.IGNORECASE): + test.failed = True + test.status = 'NS' + + # prime.log: CS check + if log == 'prime.log': + if re.search( + r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): + test.passed = True + test.status = 'CS' + elif re.search( + r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): + # If the first re.search does not match and this one does then + # that means that either errors or warnings, or both, are non-zero + test.failed = True + test.passed = False + test.status = 'NS' + if not (test.aborted or test.failed or test.passed): + test.status = 'Unknown' # Done - state.tests['Prime95']['Result'] = 'Unknown' update_progress_pane(state) # Cleanup tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) - test_obj.monitor_proc.kill() + test.monitor_proc.kill() # TODO Testing print('\n'.join( - generate_report(test_obj.sensor_data, 'Idle', 'Max', 'Cooldown'))) + generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'))) def run_network_test(): """Run network test.""" @@ -762,7 +792,7 @@ def run_network_test(): run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') -def run_nvme_smart_tests(state, test_obj): +def run_nvme_smart_tests(state, test): """TODO""" for disk in state.disks: tmux_update_pane( From a00105f71818bce77842be25ce36173c6cb5d000 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 16:57:43 -0700 Subject: [PATCH 045/186] Fixed status updates --- .bin/Scripts/functions/hw_diags.py | 46 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8cebd636..6d6b1a2c 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -214,10 +214,10 @@ class State(): }) def init(self): - """Set log and add devices.""" + """Remove test objects, set log, and add devices.""" self.disks = [] - for k in ['Result', 'Started', 'Status']: - self.tests['Prime95'][k] = False if k == 'Started' else '' + for k, v in self.tests.items(): + v['Objects'] = [] # Update LogDir if not self.quick_mode: @@ -596,7 +596,7 @@ def run_hw_tests(state): v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj(dev=k) + test_obj = TestObj(dev=k, label=disk.name) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -705,7 +705,7 @@ def run_mprime_test(state, test): except KeyboardInterrupt: # Catch CTRL+C test.aborted = True - test.status = 'Aborted' + test.update_status('Aborted') print_warning('\nAborted.') update_progress_pane(state) @@ -757,23 +757,23 @@ def run_mprime_test(state, test): if log == 'results.txt': if re.search(r'(error|fail)', _data, re.IGNORECASE): test.failed = True - test.status = 'NS' + test.update_status('NS') # prime.log: CS check if log == 'prime.log': if re.search( r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): test.passed = True - test.status = 'CS' + test.update_status('CS') elif re.search( r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): # If the first re.search does not match and this one does then # that means that either errors or warnings, or both, are non-zero test.failed = True test.passed = False - test.status = 'NS' + test.update_status('NS') if not (test.aborted or test.failed or test.passed): - test.status = 'Unknown' + test.update_status('Unknown') # Done update_progress_pane(state) @@ -910,19 +910,23 @@ def update_io_progress(percent, rate, progress_file): def update_progress_pane(state): """Update progress file for side pane.""" output = [] - - # Prime95 - output.append(state.tests['Prime95']['Status']) - output.append(' ') - - # Disks for k, v in state.tests.items(): - if 'Prime95' not in k and v['Enabled']: - output.append('{BLUE}{test_name}{CLEAR}'.format( - test_name=k, **COLORS)) - for disk in state.disks: - output.append(disk.tests[k]['Status']) - output.append(' ') + # Skip disabled sections + if not v['Enabled']: + continue + + # Add section name + if k != 'Prime95': + output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) + if 'SMART' in k and state.quick_mode: + output.append(' {YELLOW}(Quick Check){CLEAR}'.format(**COLORS)) + + # Add status from test object(s) + for test in v['Objects']: + output.append(test.status) + + # Add spacer before next section + output.append(' ') # Add line-endings output = ['{}\n'.format(line) for line in output] From 8a8a63eb66c412b428ddb0bb9ccd1ae8241dbd49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:16:43 -0700 Subject: [PATCH 046/186] Build Prime95 report --- .bin/Scripts/functions/hw_diags.py | 83 +++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6d6b1a2c..181c3a4d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -264,7 +264,7 @@ class TestObj(): self.disabled = False self.failed = False self.passed = False - self.report = '' + self.report = [] self.started = False self.status = '' self.update_status() @@ -615,6 +615,7 @@ def run_hw_tests(state): f(state, test_obj) # Done + show_results(state) pause('Press Enter to return to main menu... ') # Cleanup @@ -738,42 +739,67 @@ def run_mprime_test(state, test): item.path, global_vars['LogDir'])) - # Check results + # Check results and build report test.logs = {} for log in ['results.txt', 'prime.log']: - _data = '' + lines = [] log_path = '{}/{}'.format(global_vars['LogDir'], log) # Read and save log try: with open(log_path, 'r') as f: - _data = f.read() - test.logs[log] = _data.splitlines() + lines = f.read().splitlines() + test.logs[log] = lines except FileNotFoundError: # Ignore since files may be missing for slower CPUs pass - # results.txt: NS check + # results.txt (NS check) if log == 'results.txt': - if re.search(r'(error|fail)', _data, re.IGNORECASE): - test.failed = True - test.update_status('NS') + _tmp = [] + for line in lines: + if re.search(r'(error|fail)', line, re.IGNORECASE): + test.failed = True + test.update_status('NS') + _tmp.append(' {YELLOW}{line}{CLEAR}'.format(**COLORS)) + if _tmp: + test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) + test.report.extend(_tmp) - # prime.log: CS check + # prime.log (CS check) if log == 'prime.log': - if re.search( - r'completed.*0 errors, 0 warnings', _data, re.IGNORECASE): - test.passed = True - test.update_status('CS') - elif re.search( - r'completed.*\d+ errors, \d+ warnings', _data, re.IGNORECASE): - # If the first re.search does not match and this one does then - # that means that either errors or warnings, or both, are non-zero - test.failed = True - test.passed = False - test.update_status('NS') + _tmp_pass = [] + _tmp_warn = [] + for line in lines: + if re.search( + r'completed.*0 errors, 0 warnings', line, re.IGNORECASE): + _tmp_pass.append(line) + elif re.search( + r'completed.*\d+ errors, \d+ warnings', line, re.IGNORECASE): + # If the first re.search does not match and this one does then + # that means that either errors or warnings, or both, are non-zero + _tmp_warn.append(line) + if len(_tmp_warn) > 0: + test.failed = True + test.passed = False + test.update_status('NS') + elif len(_tmp_pass) > 0: + test.passed = True + test.update_status('CS') + if len(_tmp_pass) + len(_tmp_warn) > 0: + test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) + for line in _tmp_pass: + test.report.append(' {}'.format(line)) + for line in _tmp_warn: + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line, **COLORS)) + test.report.append(' ') + + # Finalize report if not (test.aborted or test.failed or test.passed): test.update_status('Unknown') + test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) + for line in generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'): + test.report.append(' {}'.format(line)) # Done update_progress_pane(state) @@ -782,10 +808,6 @@ def run_mprime_test(state, test): tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) test.monitor_proc.kill() - # TODO Testing - print('\n'.join( - generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'))) - def run_network_test(): """Run network test.""" clear_screen() @@ -840,6 +862,17 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_results(state): + """Show results for all tests.""" + for k, v in state.tests.items(): + print_success('{}:'.format(k)) + for obj in v['Objects']: + for line in obj.report: + print(line) + print_log(strip_colors(line)) + print_standard(' ') + print_standard(' ') + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 30d4acd9861672340443ecbbfc5614d434245a01 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:18:16 -0700 Subject: [PATCH 047/186] Added watch mode to respawn-pane --- .bin/Scripts/functions/tmux.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index e1066e18..e1892417 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -81,11 +81,14 @@ def tmux_split_window( result = run_program(cmd) return result.stdout.decode().strip() -def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): +def tmux_update_pane( + pane_id, command=None, + text=None, watch=None, + working_dir=None): """Respawn with either a new command or new text.""" # Bail early - if not command and not text: - raise Exception('Neither command nor text specified.') + if not command and not text and not watch: + raise Exception('No command, text, or watch file specified.') cmd = ['tmux', 'respawn-pane', '-k', '-t', pane_id] if working_dir: @@ -94,6 +97,11 @@ def tmux_update_pane(pane_id, command=None, text=None, working_dir=None): cmd.extend(command) elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) + elif watch: + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) run_program(cmd) From 2b43cdf9e27d861406b20e6c35512471450c5ae2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:19:11 -0700 Subject: [PATCH 048/186] Create watch file if it doesn't exist yet --- .bin/Scripts/functions/tmux.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index e1892417..69b906d1 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -2,6 +2,12 @@ from functions.common import * +def create_file(filepath): + """Create file if it doesn't exist.""" + if not os.path.exists(filepath): + with open(filepath, 'w') as f: + f.write('') + def tmux_kill_all_panes(pane_id=None): """Kill all tmux panes except the active pane or pane_id if specified.""" cmd = ['tmux', 'kill-pane', '-a'] @@ -72,6 +78,7 @@ def tmux_split_window( elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: + create_file(watch) cmd.extend([ 'watch', '--color', '--no-title', '--interval', '1', @@ -98,6 +105,7 @@ def tmux_update_pane( elif text: cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: + create_file(watch) cmd.extend([ 'watch', '--color', '--no-title', '--interval', '1', From a2ef06e6db0004bb9cb23af0a351cffd33326e2d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:19:35 -0700 Subject: [PATCH 049/186] Added strip_colors() function --- .bin/Scripts/functions/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 2bf52a85..d87f772e 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -515,6 +515,12 @@ def stay_awake(): print_error('ERROR: No caffeine available.') print_warning('Please set the power setting to High Performance.') +def strip_colors(s): + """Remove all ASCII color escapes from string, returns str.""" + for c in COLORS.values(): + s = s.replace(c, '') + return s + def get_exception(s): """Get exception by name, returns Exception object.""" try: From d9554314d55ef845670887e3b019b39de449c343 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 19:42:10 -0700 Subject: [PATCH 050/186] Updated run_program() and popen_program() * Use dicts for clarity * Support cwd flag --- .bin/Scripts/functions/common.py | 36 ++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index d87f772e..7f14bdbd 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -405,19 +405,24 @@ def ping(addr='google.com'): def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): """Run program and return a subprocess.Popen object.""" - startupinfo=None + cmd_kwargs = {'args': cmd, 'shell': shell} + if minimized: startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = 6 + cmd_kwargs['startupinfo'] = startupinfo if pipe: - popen_obj = subprocess.Popen(cmd, shell=shell, startupinfo=startupinfo, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - popen_obj = subprocess.Popen(cmd, shell=shell, startupinfo=startupinfo) + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) - return popen_obj + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] + + return subprocess.Popen(**cmd_kwargs) def print_error(*args, **kwargs): """Prints message to screen in RED.""" @@ -456,7 +461,7 @@ def print_log(message='', end='\n', timestamp=True): line = line, end = end)) -def run_program(cmd, args=[], check=True, pipe=True, shell=False): +def run_program(cmd, args=[], check=True, pipe=True, shell=False, **kwargs): """Run program and return a subprocess.CompletedProcess object.""" if args: # Deprecated so let's raise an exception to find & fix all occurances @@ -466,13 +471,18 @@ def run_program(cmd, args=[], check=True, pipe=True, shell=False): if shell: cmd = ' '.join(cmd) - if pipe: - process_return = subprocess.run(cmd, check=check, shell=shell, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - process_return = subprocess.run(cmd, check=check, shell=shell) + cmd_kwargs = {'args': cmd, 'check': check, 'shell': shell} - return process_return + if pipe: + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) + + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] + + return subprocess.run(**cmd_kwargs) def set_title(title='~Some Title~'): """Set title. From 6c06a67fdf0dafb0b792fdbf48d0608e08a73651 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 10 Dec 2018 22:54:56 -0700 Subject: [PATCH 051/186] Prime95 section complete --- .bin/Scripts/functions/hw_diags.py | 73 ++++++++++++++++++------------ .bin/Scripts/functions/sensors.py | 10 +++- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 181c3a4d..9cdde459 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -674,7 +674,9 @@ def run_mprime_test(state, test): try_and_print( message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test.sensor_data, temp_label='Idle') + sensor_data=test.sensor_data, temp_label='Idle', + seconds=3) + # TODO: Remove seconds kwarg above # Stress CPU print_log('Starting Prime95') @@ -684,7 +686,9 @@ def run_mprime_test(state, test): state.panes['mprime'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) - time_limit = int(MPRIME_LIMIT) * 60 + #time_limit = int(MPRIME_LIMIT) * 60 + # TODO: restore above line + time_limit = 10 try: for i in range(time_limit): clear_screen() @@ -716,19 +720,23 @@ def run_mprime_test(state, test): pipe=True) # Stop Prime95 (twice for good measure) - tmux_kill_pane(state.panes['mprime']) run_program(['killall', '-s', 'INT', 'mprime'], check=False) + sleep(1) + tmux_kill_pane(state.panes['mprime']) # Get cooldown temp run_program(['apple-fans', 'auto']) clear_screen() try_and_print( message='Letting CPU cooldown for bit...', indent=0, - function=sleep, cs='Done', seconds=10) + function=sleep, cs='Done', seconds=3) + # TODO: Above seconds should be 10 try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', - sensor_data=test.sensor_data, temp_label='Cooldown') + sensor_data=test.sensor_data, temp_label='Cooldown', + seconds=3) + # TODO: Remove seconds kwarg above # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -761,44 +769,47 @@ def run_mprime_test(state, test): if re.search(r'(error|fail)', line, re.IGNORECASE): test.failed = True test.update_status('NS') - _tmp.append(' {YELLOW}{line}{CLEAR}'.format(**COLORS)) + _tmp.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) if _tmp: test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) test.report.extend(_tmp) # prime.log (CS check) if log == 'prime.log': - _tmp_pass = [] - _tmp_warn = [] + _tmp = {'Pass': {}, 'Warn': {}} for line in lines: - if re.search( - r'completed.*0 errors, 0 warnings', line, re.IGNORECASE): - _tmp_pass.append(line) - elif re.search( - r'completed.*\d+ errors, \d+ warnings', line, re.IGNORECASE): - # If the first re.search does not match and this one does then - # that means that either errors or warnings, or both, are non-zero - _tmp_warn.append(line) - if len(_tmp_warn) > 0: - test.failed = True - test.passed = False - test.update_status('NS') - elif len(_tmp_pass) > 0: - test.passed = True - test.update_status('CS') - if len(_tmp_pass) + len(_tmp_warn) > 0: + _r = re.search( + r'(completed.*(\d+) errors, (\d+) warnings)', + line, + re.IGNORECASE) + if _r: + if int(_r.group(2)) + int(_r.group(3)) > 0: + # Encountered errors and/or warnings + _tmp['Warn'][_r.group(1)] = None + else: + # No errors + _tmp['Pass'][_r.group(1)] = None + if len(_tmp['Warn']) > 0: + # NS + test.failed = True + test.passed = False + test.update_status('NS') + elif len(_tmp['Pass']) > 0: + test.passed = True + test.update_status('CS') + if len(_tmp['Pass']) + len(_tmp['Warn']) > 0: test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) - for line in _tmp_pass: + for line in sorted(_tmp['Pass'].keys()): test.report.append(' {}'.format(line)) - for line in _tmp_warn: - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line, **COLORS)) - test.report.append(' ') + for line in sorted(_tmp['Warn'].keys()): + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # Finalize report if not (test.aborted or test.failed or test.passed): test.update_status('Unknown') test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) - for line in generate_report(test.sensor_data, 'Idle', 'Max', 'Cooldown'): + for line in generate_report( + test.sensor_data, 'Idle', 'Max', 'Cooldown', core_only=True): test.report.append(' {}'.format(line)) # Done @@ -864,6 +875,7 @@ def secret_screensaver(screensaver=None): def show_results(state): """Show results for all tests.""" + clear_screen() for k, v in state.tests.items(): print_success('{}:'.format(k)) for obj in v['Objects']: @@ -871,7 +883,8 @@ def show_results(state): print(line) print_log(strip_colors(line)) print_standard(' ') - print_standard(' ') + if 'Prime95' not in k: + print_standard(' ') def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 066dc446..b6319744 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -29,17 +29,22 @@ def fix_sensor_str(s): s = s.title() s = s.replace('Coretemp', 'CoreTemp') s = s.replace('Acpi', 'ACPI') + s = s.replace('ACPItz', 'ACPI TZ') s = s.replace('Isa ', 'ISA ') s = s.replace('Id ', 'ID ') s = re.sub(r'(\D+)(\d+)', r'\1 \2', s, re.IGNORECASE) s = s.replace(' ', ' ') return s -def generate_report(sensor_data, *temp_labels, colors=True): +def generate_report( + sensor_data, *temp_labels, + colors=True, core_only=False): """Generate report based on temp_labels, returns list if str.""" report = [] for _section, _adapters in sorted(sensor_data.items()): # CoreTemps then Other temps + if core_only and 'Core' not in _section: + continue for _adapter, _sources in sorted(_adapters.items()): # Adapter report.append(fix_sensor_str(_adapter)) @@ -53,7 +58,8 @@ def generate_report(sensor_data, *temp_labels, colors=True): ': ' if _label != 'Current' else '', get_temp_str(_data.get(_label, '???'), colors=colors)) report.append(_line) - report.append(' ') + if not core_only: + report.append(' ') # Handle empty reports (i.e. no sensors detected) if not report: From a3f7e5ad89372c922917662626a5d1421686cba7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 00:54:16 -0700 Subject: [PATCH 052/186] Disk quick check almost done --- .bin/Scripts/functions/hw_diags.py | 193 +++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 51 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9cdde459..da4d24e7 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -11,23 +11,23 @@ from functions.tmux import * # STATIC VARIABLES ATTRIBUTES = { 'NVMe': { - 'critical_warning': {'Error': 1}, - 'media_errors': {'Error': 1}, + 'critical_warning': {'Error': 1, 'Critical': True}, + 'media_errors': {'Error': 1, 'Critical': True}, 'power_on_hours': {'Warning': 12000, 'Error': 26298, 'Ignore': True}, 'unsafe_shutdowns': {'Warning': 1}, }, 'SMART': { - 5: {'Hex': '05', 'Error': 1}, - 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, - 10: {'Hex': '0A', 'Error': 1}, - 184: {'Hex': 'B8', 'Error': 1}, - 187: {'Hex': 'BB', 'Error': 1}, - 188: {'Hex': 'BC', 'Error': 1}, - 196: {'Hex': 'C4', 'Error': 1}, - 197: {'Hex': 'C5', 'Error': 1}, - 198: {'Hex': 'C6', 'Error': 1}, - 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - 201: {'Hex': 'C9', 'Error': 1}, + '5': {'Hex': '05', 'Error': 1, 'Critical': True}, + '9': {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + '10': {'Hex': '0A', 'Error': 1}, + '184': {'Hex': 'B8', 'Error': 1}, + '187': {'Hex': 'BB', 'Error': 1}, + '188': {'Hex': 'BC', 'Error': 1}, + '196': {'Hex': 'C4', 'Error': 1}, + '197': {'Hex': 'C5', 'Error': 1, 'Critical': True}, + '198': {'Hex': 'C6', 'Error': 1, 'Critical': True}, + '199': {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + '201': {'Hex': 'C9', 'Error': 1}, }, } IO_VARS = { @@ -284,6 +284,82 @@ class TestObj(): self.label, 'Working', self.info_label) # Functions +def attributes_ok_nvme(disk): + """Check NVMe attributes for errors, returns bool.""" + disk_ok = True + override_disabled = False + for k, v in disk.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + if 'Error' not in ATTRIBUTES['NVMe'][k]: + # Only worried about error thresholds + continue + if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: + disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['NVMe'][k].get( + 'Critical', False) + + # Print errors + if not disk_ok: + show_disk_attributes(disk) + if override_disabled: + print_error('NVMe error(s) detected.') + print_standard('Tests disabled for this device') + pause() + else: + print_warning('NVMe error(s) detected.') + disk_ok = ask('Run tests on this device anyway?') + + return disk_ok + +def attributes_ok_smart(disk): + """Check SMART attributes for errors, returns bool.""" + disk_ok = True + override_disabled = False + smart_overall_pass = True + for k, v in disk.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + if 'Error' not in ATTRIBUTES['SMART'][k]: + # Only worried about error thresholds + continue + if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: + disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['SMART'][k].get( + 'Critical', False) + + # SMART overall assessment + if not disk.smartctl.get('smart_status', {}).get('passed', False): + smart_overall_pass = False + disk_ok = False + override_disabled = True + + # Print errors + if not disk_ok: + show_disk_attributes(disk) + + # 199/C7 warning + if disk.smart_attributes.get('199', {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the drive cable?)') + + # Override? + if not smart_overall_pass: + print_error('SMART overall self-assessment: Failed') + print_standard('Tests disabled for this device') + pause() + elif override_disabled: + print_error('SMART error(s) detected.') + print_standard('Tests disabled for this device') + pause() + else: + print_warning('SMART error(s) detected.') + disk_ok = ask('Run tests on this device anyway?') + + return disk_ok + def build_outer_panes(state): """Build top and side panes.""" clear_screen() @@ -310,7 +386,7 @@ def build_status_string(label, status, info_label=False): status_color = COLORS['CLEAR'] if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: status_color = COLORS['RED'] - elif status in ['Aborted', 'Unknown', 'Working', 'Skipped']: + elif status in ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working']: status_color = COLORS['YELLOW'] elif status in ['CS']: status_color = COLORS['GREEN'] @@ -596,7 +672,7 @@ def run_hw_tests(state): v['Objects'].append(test_obj) elif k in TESTS_DISK: for disk in state.disks: - test_obj = TestObj(dev=k, label=disk.name) + test_obj = TestObj(dev=disk, label=disk.name) disk.tests[k] = test_obj v['Objects'].append(test_obj) print_standard('') @@ -616,7 +692,10 @@ def run_hw_tests(state): # Done show_results(state) - pause('Press Enter to return to main menu... ') + if '--quick' in sys.argv: + pause('Press Enter to exit...') + else: + pause('Press Enter to return to main menu... ') # Cleanup tmux_kill_pane(*state.panes.values()) @@ -826,42 +905,32 @@ def run_network_test(): pause('Press Enter to return to main menu... ') def run_nvme_smart_tests(state, test): - """TODO""" - for disk in state.disks: - tmux_update_pane( - state.panes['Top'], - text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **disk.lsblk)) - disk.tests['NVMe / SMART']['Started'] = True - update_progress_pane(state) - if disk.nvme_attributes: - run_nvme_tests(state, disk) - elif disk.smart_attributes: - run_smart_tests(state, disk) + """Run NVMe or SMART test for test.dev.""" + tmux_update_pane( + state.panes['Top'], + text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( + t=TOP_PANE_TEXT, **test.dev.lsblk)) + if test.dev.nvme_attributes: + if attributes_ok_nvme(test.dev): + test.passed = True + test.update_status('CS') else: - print_standard('TODO: run_nvme_smart_tests({})'.format( - disk.path)) - print_warning( - " WARNING: Device {} doesn't support NVMe or SMART test".format( - disk.path)) - disk.tests['NVMe / SMART']['Status'] = 'N/A' - disk.tests['NVMe / SMART']['Result'] = 'N/A' - update_progress_pane(state) - sleep(3) - -def run_nvme_tests(state, disk): - """TODO""" - print_standard('TODO: run_nvme_test({})'.format(disk.path)) + test.failed = True + test.update_status('NS') + elif test.dev.smart_attributes: + if attributes_ok_smart(test.dev): + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') + else: + print_standard('Tests disabled for this device') + test.update_status('N/A') + if not ask('Run tests on this device anyway?'): + test.failed = True + update_progress_pane(state) sleep(3) - disk.tests['NVMe / SMART']['Result'] = 'CS' - update_progress_pane(state) - -def run_smart_tests(state, disk): - """TODO""" - print_standard('TODO: run_smart_tests({})'.format(disk.path)) - sleep(3) - disk.tests['NVMe / SMART']['Result'] = 'CS' - update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" @@ -873,10 +942,32 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_disk_attributes(disk): + """Show NVMe/SMART attributes for disk.""" + print_info('Device: {}'.format(disk.path)) + print_standard(' {size:6} ({tran}) {model} {serial}'.format(**disk.lsblk)) + print_info('Attributes') + if disk.nvme_attributes: + for k, v in disk.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + print('TODO: {} {}'.format(k, v)) + elif disk.smart_attributes: + for k, v in disk.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + print('TODO: {} {}'.format(k, v)) + else: + print_warning(' No NVMe or SMART data available') + def show_results(state): """Show results for all tests.""" clear_screen() + tmux_update_pane( + state.panes['Top'], text='{}\n{}'.format( + TOP_PANE_TEXT, 'Results')) for k, v in state.tests.items(): + # Skip disabled tests + if not v['Enabled']: + continue print_success('{}:'.format(k)) for obj in v['Objects']: for line in obj.report: @@ -965,7 +1056,7 @@ def update_progress_pane(state): if k != 'Prime95': output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) if 'SMART' in k and state.quick_mode: - output.append(' {YELLOW}(Quick Check){CLEAR}'.format(**COLORS)) + output[-1] += ' {YELLOW}(Quick){CLEAR}'.format(**COLORS) # Add status from test object(s) for test in v['Objects']: From a967a5c425122f7d2bf1998f662e035e55b57865 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 20:40:57 -0700 Subject: [PATCH 053/186] Switched back to int keys for SMART attributes * Allows for easier sorting --- .bin/Scripts/functions/hw_diags.py | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index da4d24e7..2c2d63fa 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -17,17 +17,19 @@ ATTRIBUTES = { 'unsafe_shutdowns': {'Warning': 1}, }, 'SMART': { - '5': {'Hex': '05', 'Error': 1, 'Critical': True}, - '9': {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, - '10': {'Hex': '0A', 'Error': 1}, - '184': {'Hex': 'B8', 'Error': 1}, - '187': {'Hex': 'BB', 'Error': 1}, - '188': {'Hex': 'BC', 'Error': 1}, - '196': {'Hex': 'C4', 'Error': 1}, - '197': {'Hex': 'C5', 'Error': 1, 'Critical': True}, - '198': {'Hex': 'C6', 'Error': 1, 'Critical': True}, - '199': {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - '201': {'Hex': 'C9', 'Error': 1}, + 5: {'Hex': '05', 'Error': 1, 'Critical': True}, + 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, + 10: {'Hex': '0A', 'Error': 1}, + 184: {'Hex': 'B8', 'Error': 1}, + 187: {'Hex': 'BB', 'Error': 1}, + 188: {'Hex': 'BC', 'Error': 1}, + 196: {'Hex': 'C4', 'Error': 1}, + 197: {'Hex': 'C5', 'Error': 1, 'Critical': True}, + 198: {'Hex': 'C6', 'Error': 1, 'Critical': True}, + 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, + 201: {'Hex': 'C9', 'Error': 1}, + # TODO: Delete below + 177: {'Hex': 'FF', 'Error': 1}, }, } IO_VARS = { @@ -161,19 +163,21 @@ class DiskObj(): self.nvme_attributes.update(self.smartctl[KEY_NVME]) elif KEY_SMART in self.smartctl: for a in self.smartctl[KEY_SMART].get('table', {}): - _id = str(a.get('id', 'UNKNOWN')) + try: + _id = int(a.get('id', -1)) + except ValueError: + # Ignoring invalid attribute + continue _name = str(a.get('name', 'UNKNOWN')) - _raw = a.get('raw', {}).get('value', -1) + _raw = int(a.get('raw', {}).get('value', -1)) _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') # Fix power-on time _r = re.match(r'^(\d+)[Hh].*', _raw_str) - if _id == '9' and _r: - try: - _raw = int(_r.group(1)) - except ValueError: - # That's fine - pass + if _id == 9 and _r: + _raw = int(_r.group(1)) + + # Add to dict self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} @@ -1056,7 +1060,7 @@ def update_progress_pane(state): if k != 'Prime95': output.append('{BLUE}{name}{CLEAR}'.format(name=k, **COLORS)) if 'SMART' in k and state.quick_mode: - output[-1] += ' {YELLOW}(Quick){CLEAR}'.format(**COLORS) + output[-1] += ' {}'.format(QUICK_LABEL) # Add status from test object(s) for test in v['Objects']: From 62a60ff3fd0ff1e24181ebb60d66b9cbfc38f02e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 22:56:09 -0700 Subject: [PATCH 054/186] Reworked disk safety checks * Moved several functions into DiskObj * Added HW_OVERRIDES_FORCED and HW_OVERRIDES_LIMITED to main.py * These adjust when overrides are requested * Disable badblocks and/or io_benchmark if disk fails safety check --- .bin/Scripts/functions/hw_diags.py | 220 ++++++++++++++++------------- .bin/Scripts/settings/main.py | 2 + 2 files changed, 122 insertions(+), 100 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 2c2d63fa..63003e8e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -28,10 +28,9 @@ ATTRIBUTES = { 198: {'Hex': 'C6', 'Error': 1, 'Critical': True}, 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, 201: {'Hex': 'C9', 'Error': 1}, - # TODO: Delete below - 177: {'Hex': 'FF', 'Error': 1}, }, } +HW_OVERRIDES_FORCED = HW_OVERRIDES_FORCED and not HW_OVERRIDES_LIMITED IO_VARS = { 'Block Size': 512*1024, 'Chunk Size': 32*1024**2, @@ -106,6 +105,7 @@ class CpuObj(): class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): + self.disk_ok = True self.labels = [] self.lsblk = {} self.name = re.sub(r'^.*/(.*)', r'\1', disk_path) @@ -181,10 +181,121 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def safety_check(self): - """Check enabled tests and verify it's safe to run them.""" - # TODO - pass + def nvme_check(self, silent=False): + """Check NVMe attributes for errors.""" + override_disabled = False + for k, v in self.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + if 'Error' not in ATTRIBUTES['NVMe'][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES['NVMe'][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['NVMe'][k].get( + 'Critical', False) + + # Print errors + if not self.disk_ok and not silent: + self.show_attributes() + print_warning('NVMe error(s) detected.') + + # Override? + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + + def safety_check(self, silent=False): + """Check attributes and disable tests if necessary.""" + if self.nvme_attributes: + self.nvme_check(silent) + elif self.smart_attributes: + self.smart_check(silent) + else: + # No NVMe/SMART details + if silent: + self.disk_ok = HW_OVERRIDES_FORCED + else: + print_warning( + ' WARNING: No NVMe or SMART attributes available for: {}'.format( + self.path)) + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + + if not self.disk_ok: + for t in ['badblocks', 'I/O Benchmark']: + if t in self.tests: + self.tests[t].disabled = True + self.tests[t].update_status('Denied') + + def show_attributes(self): + """Show NVMe/SMART attributes.""" + print_info('Device: {}'.format(self.path)) + print_standard( + ' {size:>6} ({tran}) {model} {serial}'.format(**self.lsblk)) + print_info('Attributes') + if self.nvme_attributes: + for k, v in self.nvme_attributes.items(): + if k in ATTRIBUTES['NVMe']: + print('TODO: {} {}'.format(k, v)) + elif self.smart_attributes: + for k, v in self.smart_attributes.items(): + # TODO: If k == 199/C7 then append ' (bad cable?)' to line + if k in ATTRIBUTES['SMART']: + print('TODO: {} {}'.format(k, v)) + if not self.smartctl.get('smart_status', {}).get('passed', True): + print_error('SMART overall self-assessment: Failed') + else: + print_warning(' No NVMe or SMART data available') + + def smart_check(self, silent=False): + """Check SMART attributes for errors.""" + override_disabled = False + for k, v in self.smart_attributes.items(): + if k in ATTRIBUTES['SMART']: + if 'Error' not in ATTRIBUTES['SMART'][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES['SMART'][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES['SMART'][k].get( + 'Critical', False) + + # SMART overall assessment + ## NOTE: Only fail drives if the overall value exists and reports failed + if not self.smartctl.get('smart_status', {}).get('passed', True): + self.disk_ok = False + override_disabled = True + + # Print errors + if not silent: + if self.disk_ok: + # 199/C7 warning + if self.smart_attributes.get(199, {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the disk cable?)') + else: + # Override? + self.show_attributes() + print_warning('SMART error(s) detected.') + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') class State(): """Object to track device objects and overall state.""" @@ -288,82 +399,6 @@ class TestObj(): self.label, 'Working', self.info_label) # Functions -def attributes_ok_nvme(disk): - """Check NVMe attributes for errors, returns bool.""" - disk_ok = True - override_disabled = False - for k, v in disk.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - if 'Error' not in ATTRIBUTES['NVMe'][k]: - # Only worried about error thresholds - continue - if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: - disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['NVMe'][k].get( - 'Critical', False) - - # Print errors - if not disk_ok: - show_disk_attributes(disk) - if override_disabled: - print_error('NVMe error(s) detected.') - print_standard('Tests disabled for this device') - pause() - else: - print_warning('NVMe error(s) detected.') - disk_ok = ask('Run tests on this device anyway?') - - return disk_ok - -def attributes_ok_smart(disk): - """Check SMART attributes for errors, returns bool.""" - disk_ok = True - override_disabled = False - smart_overall_pass = True - for k, v in disk.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - if 'Error' not in ATTRIBUTES['SMART'][k]: - # Only worried about error thresholds - continue - if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: - disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['SMART'][k].get( - 'Critical', False) - - # SMART overall assessment - if not disk.smartctl.get('smart_status', {}).get('passed', False): - smart_overall_pass = False - disk_ok = False - override_disabled = True - - # Print errors - if not disk_ok: - show_disk_attributes(disk) - - # 199/C7 warning - if disk.smart_attributes.get('199', {}).get('raw', 0) > 0: - print_warning('199/C7 error detected') - print_standard(' (Have you tried swapping the drive cable?)') - - # Override? - if not smart_overall_pass: - print_error('SMART overall self-assessment: Failed') - print_standard('Tests disabled for this device') - pause() - elif override_disabled: - print_error('SMART error(s) detected.') - print_standard('Tests disabled for this device') - pause() - else: - print_warning('SMART error(s) detected.') - disk_ok = ask('Run tests on this device anyway?') - - return disk_ok - def build_outer_panes(state): """Build top and side panes.""" clear_screen() @@ -915,14 +950,15 @@ def run_nvme_smart_tests(state, test): text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **test.dev.lsblk)) if test.dev.nvme_attributes: - if attributes_ok_nvme(test.dev): + # NOTE: Pass/Fail is just the attribute check + if test.dev.disk_ok: test.passed = True test.update_status('CS') else: test.failed = True test.update_status('NS') elif test.dev.smart_attributes: - if attributes_ok_smart(test.dev): + if test.dev.disk_ok: test.passed = True test.update_status('CS') else: @@ -946,22 +982,6 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def show_disk_attributes(disk): - """Show NVMe/SMART attributes for disk.""" - print_info('Device: {}'.format(disk.path)) - print_standard(' {size:6} ({tran}) {model} {serial}'.format(**disk.lsblk)) - print_info('Attributes') - if disk.nvme_attributes: - for k, v in disk.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - print('TODO: {} {}'.format(k, v)) - elif disk.smart_attributes: - for k, v in disk.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - print('TODO: {} {}'.format(k, v)) - else: - print_warning(' No NVMe or SMART data available') - def show_results(state): """Show results for all tests.""" clear_screen() diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 75fef0fd..7b915bdb 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -4,6 +4,8 @@ ENABLED_OPEN_LOGS = False ENABLED_TICKET_NUMBERS = False ENABLED_UPLOAD_DATA = False +HW_OVERRIDES_FORCED = False +HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED # STATIC VARIABLES (also used by BASH and BATCH files) ## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH From 47084efe1725aae5cdd302735d3f84097097d28d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 23:18:51 -0700 Subject: [PATCH 055/186] Combined nvme_check() and smart_check() --- .bin/Scripts/functions/hw_diags.py | 129 ++++++++++++----------------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 63003e8e..dd2bfa4f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -117,6 +117,54 @@ class DiskObj(): self.get_details() self.get_smart_details() + def check_attributes(self, silent=False): + """Check NVMe / SMART attributes for errors.""" + override_disabled = False + if self.nvme_attributes: + attr_type = 'NVMe' + items = self.nvme_attributes.items() + elif self.smart_attributes: + attr_type = 'SMART' + items = self.smar_attributes.items() + for k, v in items: + if k in ATTRIBUTES[attr_type]: + if 'Error' not in ATTRIBUTES[attr_type][k]: + # Only worried about error thresholds + continue + if ATTRIBUTES[attr_type][k].get('Ignore', False): + # Attribute is non-failing, skip + continue + if v['raw'] >= ATTRIBUTES[attr_type][k]['Error']: + self.disk_ok = False + + # Disable override if necessary + override_disabled |= ATTRIBUTES[attr_type][k].get( + 'Critical', False) + + # SMART overall assessment + ## NOTE: Only fail drives if the overall value exists and reports failed + if not self.smartctl.get('smart_status', {}).get('passed', True): + self.disk_ok = False + override_disabled = True + + # Print errors + if not silent: + if self.disk_ok: + # 199/C7 warning + if self.smart_attributes.get(199, {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the disk cable?)') + else: + # Override? + self.show_attributes() + print_warning('{} error(s) detected.'.format(attr_type)) + if override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): + self.disk_ok = HW_OVERRIDES_FORCED or ask( + 'Run tests on this device anyway?') + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -181,43 +229,10 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} - def nvme_check(self, silent=False): - """Check NVMe attributes for errors.""" - override_disabled = False - for k, v in self.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - if 'Error' not in ATTRIBUTES['NVMe'][k]: - # Only worried about error thresholds - continue - if ATTRIBUTES['NVMe'][k].get('Ignore', False): - # Attribute is non-failing, skip - continue - if v['raw'] >= ATTRIBUTES['NVMe'][k]['Error']: - self.disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['NVMe'][k].get( - 'Critical', False) - - # Print errors - if not self.disk_ok and not silent: - self.show_attributes() - print_warning('NVMe error(s) detected.') - - # Override? - if override_disabled: - print_standard('Tests disabled for this device') - pause() - elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') - def safety_check(self, silent=False): - """Check attributes and disable tests if necessary.""" - if self.nvme_attributes: - self.nvme_check(silent) - elif self.smart_attributes: - self.smart_check(silent) + """Run safety checks and disable tests if necessary.""" + if self.nvme_attributes or self.smart_attributes: + self.check_attributes(silent) else: # No NVMe/SMART details if silent: @@ -255,48 +270,6 @@ class DiskObj(): else: print_warning(' No NVMe or SMART data available') - def smart_check(self, silent=False): - """Check SMART attributes for errors.""" - override_disabled = False - for k, v in self.smart_attributes.items(): - if k in ATTRIBUTES['SMART']: - if 'Error' not in ATTRIBUTES['SMART'][k]: - # Only worried about error thresholds - continue - if ATTRIBUTES['SMART'][k].get('Ignore', False): - # Attribute is non-failing, skip - continue - if v['raw'] >= ATTRIBUTES['SMART'][k]['Error']: - self.disk_ok = False - - # Disable override if necessary - override_disabled |= ATTRIBUTES['SMART'][k].get( - 'Critical', False) - - # SMART overall assessment - ## NOTE: Only fail drives if the overall value exists and reports failed - if not self.smartctl.get('smart_status', {}).get('passed', True): - self.disk_ok = False - override_disabled = True - - # Print errors - if not silent: - if self.disk_ok: - # 199/C7 warning - if self.smart_attributes.get(199, {}).get('raw', 0) > 0: - print_warning('199/C7 error detected') - print_standard(' (Have you tried swapping the disk cable?)') - else: - # Override? - self.show_attributes() - print_warning('SMART error(s) detected.') - if override_disabled: - print_standard('Tests disabled for this device') - pause() - elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') - class State(): """Object to track device objects and overall state.""" def __init__(self): From b5c93317dc33716c813927097da0838a2a8ecc1b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 11 Dec 2018 23:54:02 -0700 Subject: [PATCH 056/186] Override sections working --- .bin/Scripts/functions/hw_diags.py | 66 +++++++++++++++++++----------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index dd2bfa4f..345cb279 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -125,7 +125,7 @@ class DiskObj(): items = self.nvme_attributes.items() elif self.smart_attributes: attr_type = 'SMART' - items = self.smar_attributes.items() + items = self.smart_attributes.items() for k, v in items: if k in ATTRIBUTES[attr_type]: if 'Error' not in ATTRIBUTES[attr_type][k]: @@ -359,8 +359,6 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled: - return if new_status: self.status = build_status_string( self.label, new_status, self.info_label) @@ -650,12 +648,12 @@ def run_badblocks_test(state, test): TOP_PANE_TEXT, 'badblocks')) print_standard('TODO: run_badblocks_test({})'.format( test.dev.path)) - for disk in state.disks: - disk.tests['badblocks']['Started'] = True - update_progress_pane(state) - sleep(3) - disk.tests['badblocks']['Result'] = 'OVERRIDE' - update_progress_pane(state) + test.started = True + test.update_status() + update_progress_pane(state) + sleep(3) + test.update_status('Unknown') + update_progress_pane(state) def run_hw_tests(state): """Run enabled hardware tests.""" @@ -691,7 +689,7 @@ def run_hw_tests(state): # Run safety checks for disk in state.disks: - disk.safety_check() + disk.safety_check(silent=state.quick_mode) # Run tests ## Because state.tests is an OrderedDict and the disks were added @@ -704,7 +702,7 @@ def run_hw_tests(state): # Done show_results(state) - if '--quick' in sys.argv: + if state.quick_mode: pause('Press Enter to exit...') else: pause('Press Enter to return to main menu... ') @@ -719,12 +717,12 @@ def run_io_benchmark(state, test): TOP_PANE_TEXT, 'I/O Benchmark')) print_standard('TODO: run_io_benchmark({})'.format( test.dev.path)) - for disk in state.disks: - disk.tests['I/O Benchmark']['Started'] = True - update_progress_pane(state) - sleep(3) - disk.tests['I/O Benchmark']['Result'] = 'Unknown' - update_progress_pane(state) + test.started = True + test.update_status() + update_progress_pane(state) + sleep(3) + test.update_status('Unknown') + update_progress_pane(state) def run_keyboard_test(): """Run keyboard test.""" @@ -928,22 +926,42 @@ def run_nvme_smart_tests(state, test): test.passed = True test.update_status('CS') else: + # NOTE: Other test(s) should've been disabled by DiskObj.safety_check() test.failed = True test.update_status('NS') elif test.dev.smart_attributes: + # NOTE: Pass/Fail based on both attributes and SMART short self-test if test.dev.disk_ok: - test.passed = True - test.update_status('CS') + # Run short test + pause('TODO: Run SMART short self-test') + + # Check result + # TODO + short_test_passed = True + if short_test_passed: + test.passed = True + test.update_status('CS') + else: + for t in ['badblocks', 'I/O Benchmark']: + if t in test.dev.tests: + test.dev.tests[t].disabled = True + test.dev.tests[t].update_status('Denied') + # TODO + if no_logs: + test.update_status('Unknown') + else: + test.failed = True + test.update_status('NS') else: test.failed = True test.update_status('NS') else: - print_standard('Tests disabled for this device') + # NOTE: Pass/Fail not applicable without NVMe/SMART data + # Override request earlier disabled other test(s) as appropriate test.update_status('N/A') - if not ask('Run tests on this device anyway?'): - test.failed = True - update_progress_pane(state) - sleep(3) + + # Done + update_progress_pane(state) def secret_screensaver(screensaver=None): """Show screensaver.""" From 5b748798053a97c200998286d1b67fcd64e70017 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 13 Dec 2018 19:02:28 -0700 Subject: [PATCH 057/186] Fixed OVERRIDE and N/A NVMe/SMART status handling --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 345cb279..d967ae15 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -162,8 +162,11 @@ class DiskObj(): print_standard('Tests disabled for this device') pause() elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') + if HW_OVERRIDES_FORCED or ask('Run tests on this device anyway?'): + self.disk_ok = True + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].update_status('OVERRIDE') + self.tests['NVMe / SMART'].disabled = True def get_details(self): """Get data from lsblk.""" @@ -235,6 +238,9 @@ class DiskObj(): self.check_attributes(silent) else: # No NVMe/SMART details + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].update_status('N/A') + self.tests['NVMe / SMART'].disabled = True if silent: self.disk_ok = HW_OVERRIDES_FORCED else: @@ -247,8 +253,8 @@ class DiskObj(): if not self.disk_ok: for t in ['badblocks', 'I/O Benchmark']: if t in self.tests: - self.tests[t].disabled = True self.tests[t].update_status('Denied') + self.tests[t].disabled = True def show_attributes(self): """Show NVMe/SMART attributes.""" @@ -359,6 +365,8 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" + if self.disabled: + return if new_status: self.status = build_status_string( self.label, new_status, self.info_label) @@ -944,8 +952,8 @@ def run_nvme_smart_tests(state, test): else: for t in ['badblocks', 'I/O Benchmark']: if t in test.dev.tests: - test.dev.tests[t].disabled = True test.dev.tests[t].update_status('Denied') + test.dev.tests[t].disabled = True # TODO if no_logs: test.update_status('Unknown') @@ -955,10 +963,6 @@ def run_nvme_smart_tests(state, test): else: test.failed = True test.update_status('NS') - else: - # NOTE: Pass/Fail not applicable without NVMe/SMART data - # Override request earlier disabled other test(s) as appropriate - test.update_status('N/A') # Done update_progress_pane(state) From 81f05fa79f3217cc37e000f4a9998278907508d4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 16:37:14 -0700 Subject: [PATCH 058/186] Replaced show_attributes() with generate_report() * Returns list of colored strings * Optionally includes short-test results * Optionally excludes disk info --- .bin/Scripts/functions/hw_diags.py | 157 ++++++++++++++++++++++++----- 1 file changed, 133 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index d967ae15..7c40c741 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -112,6 +112,7 @@ class DiskObj(): self.nvme_attributes = {} self.path = disk_path self.smart_attributes = {} + self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() self.get_details() @@ -156,7 +157,9 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - self.show_attributes() + for line in self.generate_report(): + print(line) + print_log(strip_colors(line)) print_warning('{} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') @@ -168,6 +171,93 @@ class DiskObj(): self.tests['NVMe / SMART'].update_status('OVERRIDE') self.tests['NVMe / SMART'].disabled = True + def generate_report(self, brief=False, short_test=False): + """Generate NVMe / SMART report, returns list.""" + report = [] + if not brief: + report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( + dev_path=self.path, **COLORS)) + report.append(' {size:>6} ({tran}) {model} {serial}'.format( + **self.lsblk)) + + # Warnings + if self.nvme_attributes: + attr_type = 'NVMe' + report.append( + ' {YELLOW}NVMe disk support is still experimental{CLEAR}'.format( + **COLORS)) + elif self.smart_attributes: + attr_type = 'SMART' + else: + # No attribute data available, return short report + report.append( + ' {YELLOW}No NVMe or SMART data available{CLEAR}'.format( + **COLORS)) + return report + if not self.smartctl.get('smart_status', {}).get('passed', True): + report.append( + ' {RED}SMART overall self-assessment: Failed{CLEAR}'.format( + **COLORS)) + + # Attributes + report.append('{BLUE}{a} Attributes{YELLOW}{u:>23} {t}{CLEAR}'.format( + a=attr_type, + u='Updated:' if brief else '', + t=time.strftime('%Y-%m-%d %H:%M %Z') if brief else '', + **COLORS)) + if self.nvme_attributes: + attr_type = 'NVMe' + items = self.nvme_attributes.items() + elif self.smart_attributes: + attr_type = 'SMART' + items = self.smart_attributes.items() + for k, v in items: + if k in ATTRIBUTES[attr_type]: + _note = '' + _color = COLORS['GREEN'] + + # Attribute ID & Name + if attr_type == 'NVMe': + _line = ' {:38}'.format(k.replace('_', ' ').title()) + else: + _line = ' {i:>3} / {h}: {n:28}'.format( + i=k, + h=ATTRIBUTES[attr_type][k]['Hex'], + n=v['name'][:28]) + + # Set color + for _t, _c in [['Warning', 'YELLOW'], ['Error', 'RED']]: + if _t in ATTRIBUTES[attr_type][k]: + if v['raw'] >= ATTRIBUTES[attr_type][k][_t]: + _color = COLORS[_c] + + # 199/C7 warning + if str(k) == '199': + _note = '(bad cable?)' + + # Attribute value + _line += '{}{} {}{}'.format( + _color, + v['raw_str'], + _note, + COLORS['CLEAR']) + + # Add line to report + report.append(_line) + + # SMART short-test + if short_test: + report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) + if 'TimedOut' in self.tests['NVMe / SMART'].status: + report.append(' {YELLOW}UNKNOWN{CLEAR}: Timed out'.format(**COLORS)) + else: + report.append(' {}'.format( + self.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) + + # Done + return report + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -219,7 +309,7 @@ class DiskObj(): except ValueError: # Ignoring invalid attribute continue - _name = str(a.get('name', 'UNKNOWN')) + _name = str(a.get('name', 'UNKNOWN')).replace('_', ' ').title() _raw = int(a.get('raw', {}).get('value', -1)) _raw_str = a.get('raw', {}).get('string', 'UNKNOWN') @@ -232,6 +322,13 @@ class DiskObj(): self.smart_attributes[_id] = { 'name': _name, 'raw': _raw, 'raw_str': _raw_str} + # Self-test data + for k in ['polling_minutes', 'status']: + self.smart_self_test[k] = self.smartctl.get( + 'ata_smart_data', {}).get( + 'self_test', {}).get( + k, {}) + def safety_check(self, silent=False): """Run safety checks and disable tests if necessary.""" if self.nvme_attributes or self.smart_attributes: @@ -251,31 +348,15 @@ class DiskObj(): 'Run tests on this device anyway?') if not self.disk_ok: + if 'NVMe / SMART' in self.tests: + # NOTE: This will not overwrite the existing status if set + self.tests['NVMe / SMART'].update_status('NS') + self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: if t in self.tests: self.tests[t].update_status('Denied') self.tests[t].disabled = True - def show_attributes(self): - """Show NVMe/SMART attributes.""" - print_info('Device: {}'.format(self.path)) - print_standard( - ' {size:>6} ({tran}) {model} {serial}'.format(**self.lsblk)) - print_info('Attributes') - if self.nvme_attributes: - for k, v in self.nvme_attributes.items(): - if k in ATTRIBUTES['NVMe']: - print('TODO: {} {}'.format(k, v)) - elif self.smart_attributes: - for k, v in self.smart_attributes.items(): - # TODO: If k == 199/C7 then append ' (bad cable?)' to line - if k in ATTRIBUTES['SMART']: - print('TODO: {} {}'.format(k, v)) - if not self.smartctl.get('smart_status', {}).get('passed', True): - print_error('SMART overall self-assessment: Failed') - else: - print_warning(' No NVMe or SMART data available') - class State(): """Object to track device objects and overall state.""" def __init__(self): @@ -402,7 +483,7 @@ def build_outer_panes(state): def build_status_string(label, status, info_label=False): """Build status string with appropriate colors.""" status_color = COLORS['CLEAR'] - if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: + if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE', 'TimedOut']: status_color = COLORS['RED'] elif status in ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working']: status_color = COLORS['YELLOW'] @@ -651,6 +732,9 @@ def run_audio_test(): def run_badblocks_test(state, test): """TODO""" + # Bail early + if test.disabled: + return tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) @@ -699,6 +783,15 @@ def run_hw_tests(state): for disk in state.disks: disk.safety_check(silent=state.quick_mode) + # TODO Remove + clear_screen() + print_info('Running tests:') + for k, v in state.tests.items(): + if v['Enabled']: + print_standard(' {}'.format(k)) + update_progress_pane(state) + pause() + # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. @@ -720,6 +813,9 @@ def run_hw_tests(state): def run_io_benchmark(state, test): """TODO""" + # Bail early + if test.disabled: + return tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) @@ -739,6 +835,9 @@ def run_keyboard_test(): def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" + # Bail early + if test.disabled: + return test.started = True test.update_status() update_progress_pane(state) @@ -924,6 +1023,7 @@ def run_network_test(): def run_nvme_smart_tests(state, test): """Run NVMe or SMART test for test.dev.""" + _include_short_test = False tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( @@ -941,10 +1041,14 @@ def run_nvme_smart_tests(state, test): # NOTE: Pass/Fail based on both attributes and SMART short self-test if test.dev.disk_ok: # Run short test - pause('TODO: Run SMART short self-test') + # TODO + _include_short_test = True + _timeout = test.dev.smart_self_test['polling_minutes'].get('short', 5) + _timeout = int(_timeout) + 5 # Check result # TODO + # if 'remaining_percent' in 'status' then we've started. short_test_passed = True if short_test_passed: test.passed = True @@ -960,10 +1064,15 @@ def run_nvme_smart_tests(state, test): else: test.failed = True test.update_status('NS') + else: test.failed = True test.update_status('NS') + # Save report + test.report = test.dev.generate_report( + short_test=_include_short_test) + # Done update_progress_pane(state) From cee825245505376a757a3de5eda45cc7eb1deb95 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:03:00 -0700 Subject: [PATCH 059/186] Added CYAN to COLORS --- .bin/Scripts/functions/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 7f14bdbd..5327d895 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -27,10 +27,11 @@ global_vars = {} COLORS = { 'CLEAR': '\033[0m', 'RED': '\033[31m', + 'ORANGE': '\033[31;1m', 'GREEN': '\033[32m', 'YELLOW': '\033[33m', - 'ORANGE': '\033[31;1m', - 'BLUE': '\033[34m' + 'BLUE': '\033[34m', + 'CYAN': '\033[36m', } try: HKU = winreg.HKEY_USERS From 99984603ed601e1af78e688b2258d20ab9a7b7ef Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:32:17 -0700 Subject: [PATCH 060/186] NVMe/SMART sections working * Added timout status for clarity * Added short-test result to report --- .bin/Scripts/functions/hw_diags.py | 123 +++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 7c40c741..141d324e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -169,7 +169,9 @@ class DiskObj(): self.disk_ok = True if 'NVMe / SMART' in self.tests: self.tests['NVMe / SMART'].update_status('OVERRIDE') - self.tests['NVMe / SMART'].disabled = True + if self.nvme_attributes or not self.smart_attributes: + # i.e. only leave enabled for SMART short-tests + self.tests['NVMe / SMART'].disabled = True def generate_report(self, brief=False, short_test=False): """Generate NVMe / SMART report, returns list.""" @@ -323,6 +325,7 @@ class DiskObj(): 'name': _name, 'raw': _raw, 'raw_str': _raw_str} # Self-test data + self.smart_self_test = {} for k in ['polling_minutes', 'status']: self.smart_self_test[k] = self.smartctl.get( 'ata_smart_data', {}).get( @@ -333,6 +336,22 @@ class DiskObj(): """Run safety checks and disable tests if necessary.""" if self.nvme_attributes or self.smart_attributes: self.check_attributes(silent) + + # Check if a self-test is currently running + if 'remaining_percent' in self.smart_self_test['status']: + _msg='SMART self-test in progress, all tests disabled' + if not silent: + print_warning('WARNING: {}'.format(_msg)) + print_standard(' ') + if ask('Abort HW Diagnostics?'): + exit_script() + if 'NVMe / SMART' in self.tests: + self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report.append( + '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) + for t in self.tests.values(): + t.update_status('Denied') + t.disabled = True else: # No NVMe/SMART details if 'NVMe / SMART' in self.tests: @@ -350,6 +369,8 @@ class DiskObj(): if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set + if not self.tests['NVMe / SMART'].report: + self.tests['NVMe / SMART'].report = self.generate_report() self.tests['NVMe / SMART'].update_status('NS') self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: @@ -446,7 +467,7 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled: + if self.disabled or 'OVERRIDE' in self.status: return if new_status: self.status = build_status_string( @@ -1023,11 +1044,19 @@ def run_network_test(): def run_nvme_smart_tests(state, test): """Run NVMe or SMART test for test.dev.""" + # Bail early + if test.disabled: + return _include_short_test = False + test.started = True + test.update_status() tmux_update_pane( state.panes['Top'], text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( t=TOP_PANE_TEXT, **test.dev.lsblk)) + update_progress_pane(state) + + # NVMe if test.dev.nvme_attributes: # NOTE: Pass/Fail is just the attribute check if test.dev.disk_ok: @@ -1037,37 +1066,80 @@ def run_nvme_smart_tests(state, test): # NOTE: Other test(s) should've been disabled by DiskObj.safety_check() test.failed = True test.update_status('NS') + + # SMART elif test.dev.smart_attributes: # NOTE: Pass/Fail based on both attributes and SMART short self-test - if test.dev.disk_ok: - # Run short test - # TODO + if not (test.dev.disk_ok or 'OVERRIDE' in test.status): + test.failed = True + test.update_status('NS') + else: + # Prep + test.timeout = test.dev.smart_self_test['polling_minutes'].get( + 'short', 5) + # TODO: fix timeout, set to polling + 5 + test.timeout = int(test.timeout) + 1 _include_short_test = True - _timeout = test.dev.smart_self_test['polling_minutes'].get('short', 5) - _timeout = int(_timeout) + 5 + _self_test_started = False - # Check result - # TODO - # if 'remaining_percent' in 'status' then we've started. - short_test_passed = True - if short_test_passed: - test.passed = True - test.update_status('CS') + # Create monitor pane + test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n Pending') + state.panes['smart'] = tmux_split_window( + lines=3, vertical=True, watch=test.smart_out) + + # Show attributes + clear_screen() + for line in test.dev.generate_report(): + # Not saving to log; that will happen after all tests have been run + print(line) + print(' ') + + # Start short test + print_standard('Running self-test...') + cmd = ['sudo', 'smartctl', '--test=short', test.dev.path] + run_program(cmd, check=False) + + # Monitor progress (in 5 second increments) + for iteration in range(int(test.timeout*60/5)): + sleep(5) + + # Update SMART data + test.dev.get_smart_details() + + if _self_test_started: + # Update progress file + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n {}'.format( + test.dev.smart_self_test['status'].get('string', 'UNKNOWN'))) + + # Check if test has finished + if 'remaining_percent' not in test.dev.smart_self_test['status']: + break + + else: + # Check if test has started + if 'remaining_percent' in test.dev.smart_self_test['status']: + _self_test_started = True + + # Check if timed out + if test.dev.smart_self_test['status'].get('passed', False): + if 'OVERRIDE' not in test.status: + test.passed = True + test.update_status('CS') else: + test.failed = True + test.update_status('NS') + if not (test.failed or test.passed): + test.update_status('TimedOut') + + # Disable other drive tests if necessary + if not test.passed: for t in ['badblocks', 'I/O Benchmark']: if t in test.dev.tests: test.dev.tests[t].update_status('Denied') test.dev.tests[t].disabled = True - # TODO - if no_logs: - test.update_status('Unknown') - else: - test.failed = True - test.update_status('NS') - - else: - test.failed = True - test.update_status('NS') # Save report test.report = test.dev.generate_report( @@ -1076,6 +1148,9 @@ def run_nvme_smart_tests(state, test): # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['smart']) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': From 37b8676b9c436968ab7d68032b8deb3bc164f445 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:57:30 -0700 Subject: [PATCH 061/186] Fixed quick check --- .bin/Scripts/functions/hw_diags.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 141d324e..a4bc4f5f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1073,6 +1073,13 @@ def run_nvme_smart_tests(state, test): if not (test.dev.disk_ok or 'OVERRIDE' in test.status): test.failed = True test.update_status('NS') + elif state.quick_mode: + if test.dev.disk_ok: + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') else: # Prep test.timeout = test.dev.smart_self_test['polling_minutes'].get( @@ -1141,6 +1148,9 @@ def run_nvme_smart_tests(state, test): test.dev.tests[t].update_status('Denied') test.dev.tests[t].disabled = True + # Cleanup + tmux_kill_pane(state.panes['smart']) + # Save report test.report = test.dev.generate_report( short_test=_include_short_test) @@ -1148,9 +1158,6 @@ def run_nvme_smart_tests(state, test): # Done update_progress_pane(state) - # Cleanup - tmux_kill_pane(state.panes['smart']) - def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': From f2a519b7ec7ed3738ad3370c350e51ade8fd1d54 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Dec 2018 18:58:32 -0700 Subject: [PATCH 062/186] Adjusted log and results screen --- .bin/Scripts/functions/hw_diags.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a4bc4f5f..8fa6d743 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -157,9 +157,7 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - for line in self.generate_report(): - print(line) - print_log(strip_colors(line)) + show_report(self.generate_report()) print_warning('{} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') @@ -756,6 +754,7 @@ def run_badblocks_test(state, test): # Bail early if test.disabled: return + print_log('Starting badblocks test for {}'.format(test.dev.path)) tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'badblocks')) @@ -837,6 +836,7 @@ def run_io_benchmark(state, test): # Bail early if test.disabled: return + print_log('Starting I/O benchmark test for {}'.format(test.dev.path)) tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'I/O Benchmark')) @@ -859,6 +859,7 @@ def run_mprime_test(state, test): # Bail early if test.disabled: return + print_log('Starting Prime95 test') test.started = True test.update_status() update_progress_pane(state) @@ -1047,6 +1048,7 @@ def run_nvme_smart_tests(state, test): # Bail early if test.disabled: return + print_log('Starting NVMe/SMART test for {}'.format(test.dev.path)) _include_short_test = False test.started = True test.update_status() @@ -1168,6 +1170,12 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) +def show_report(report): + """Show report on screen and save to log w/out color.""" + for line in report: + print(line) + print_log(strip_colors(line)) + def show_results(state): """Show results for all tests.""" clear_screen() @@ -1180,11 +1188,7 @@ def show_results(state): continue print_success('{}:'.format(k)) for obj in v['Objects']: - for line in obj.report: - print(line) - print_log(strip_colors(line)) - print_standard(' ') - if 'Prime95' not in k: + show_report(obj.report) print_standard(' ') def update_main_options(state, selection, main_options): From a5d92537f54c8af6d96db97036b721289c317290 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 16:54:48 -0700 Subject: [PATCH 063/186] Removed unused function --- .bin/Scripts/functions/hw_diags.py | 32 ------------------------------ 1 file changed, 32 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8fa6d743..8409b4c8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -517,38 +517,6 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) -def check_disk_attributes(disk): - """Check if disk should be tested and allow overrides.""" - needs_override = False - print_standard(' {size:>6} ({tran}) {model} {serial}'.format( - **disk.lsblk)) - - # General checks - if not disk.nvme_attributes and not disk.smart_attributes: - needs_override = True - print_warning( - ' WARNING: No NVMe or SMART attributes available for: {}'.format( - disk.path)) - - # NVMe checks - # TODO check all tracked attributes and set disk.failing if needed - - # SMART checks - # TODO check all tracked attributes and set disk.failing if needed - - # Ask for override if necessary - if needs_override: - if ask(' Run tests on this device anyway?'): - # TODO Set override for this disk - pass - else: - for v in disk.tests.values(): - # Started is set to True to fix the status string - v['Result'] = 'Skipped' - v['Started'] = True - v['Status'] = 'Skipped' - print_standard('') - def generate_horizontal_graph(rates, oneline=False): """Generate two-line horizontal graph from rates, returns str.""" line_1 = '' From dc8416b5f71fb708a971fe6cde9a9bec9f2dff2a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 16:55:32 -0700 Subject: [PATCH 064/186] Adjusted formatting --- .bin/Scripts/functions/hw_diags.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8409b4c8..0b88001b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -117,6 +117,8 @@ class DiskObj(): self.tests = OrderedDict() self.get_details() self.get_smart_details() + self.description = '{size:>6} ({tran}) {model} {serial}'.format( + **self.lsblk) def check_attributes(self, silent=False): """Check NVMe / SMART attributes for errors.""" @@ -177,8 +179,7 @@ class DiskObj(): if not brief: report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( dev_path=self.path, **COLORS)) - report.append(' {size:>6} ({tran}) {model} {serial}'.format( - **self.lsblk)) + report.append(' {}'.format(self.description)) # Warnings if self.nvme_attributes: @@ -718,14 +719,14 @@ def run_audio_test(): pause('Press Enter to return to main menu... ') def run_badblocks_test(state, test): - """TODO""" + """Run a read-only surface scan with badblocks.""" # Bail early if test.disabled: return print_log('Starting badblocks test for {}'.format(test.dev.path)) tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'badblocks')) + state.panes['Top'], + text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) print_standard('TODO: run_badblocks_test({})'.format( test.dev.path)) test.started = True @@ -1022,8 +1023,7 @@ def run_nvme_smart_tests(state, test): test.update_status() tmux_update_pane( state.panes['Top'], - text='{t}\nDisk Health: {size:>6} ({tran}) {model} {serial}'.format( - t=TOP_PANE_TEXT, **test.dev.lsblk)) + text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) update_progress_pane(state) # NVMe @@ -1062,7 +1062,7 @@ def run_nvme_smart_tests(state, test): # Create monitor pane test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n Pending') + f.write('SMART self-test status:\n Starting...') state.panes['smart'] = tmux_split_window( lines=3, vertical=True, watch=test.smart_out) @@ -1089,7 +1089,8 @@ def run_nvme_smart_tests(state, test): # Update progress file with open(test.smart_out, 'w') as f: f.write('SMART self-test status:\n {}'.format( - test.dev.smart_self_test['status'].get('string', 'UNKNOWN'))) + test.dev.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) # Check if test has finished if 'remaining_percent' not in test.dev.smart_self_test['status']: From e96ac5c156c1eca05e14dda41a95cff0997ae492 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:09:54 -0700 Subject: [PATCH 065/186] Added watch option to use tail instead of cat * tail -f acurately prints backspace (^H) characters * badblocks output uses them and wouldn't work with watch/cat --- .bin/Scripts/functions/tmux.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 69b906d1..5fbca65d 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -46,7 +46,7 @@ def tmux_split_window( behind=False, vertical=False, follow=False, target_pane=None, working_dir=None, command=None, - text=None, watch=None): + text=None, watch=None, watch_cmd='cat'): """Run tmux split-window command and return pane_id as str.""" # Bail early if not lines and not percent: @@ -79,19 +79,21 @@ def tmux_split_window( cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: create_file(watch) - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) + if watch_cmd == 'cat': + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + elif watch_cmd == 'tail': + cmd.extend(['tail', '-f', watch]) # Run and return pane_id result = run_program(cmd) return result.stdout.decode().strip() def tmux_update_pane( - pane_id, command=None, - text=None, watch=None, - working_dir=None): + pane_id, command=None, working_dir=None, + text=None, watch=None, watch_cmd='cat'): """Respawn with either a new command or new text.""" # Bail early if not command and not text and not watch: @@ -106,10 +108,13 @@ def tmux_update_pane( cmd.extend(['echo-and-hold "{}"'.format(text)]) elif watch: create_file(watch) - cmd.extend([ - 'watch', '--color', '--no-title', - '--interval', '1', - 'cat', watch]) + if watch_cmd == 'cat': + cmd.extend([ + 'watch', '--color', '--no-title', + '--interval', '1', + 'cat', watch]) + elif watch_cmd == 'tail': + cmd.extend(['tail', '-f', watch]) run_program(cmd) From 8b936f54137d0275fbdf398fa5722749aad8903f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:45:43 -0700 Subject: [PATCH 066/186] badblocks section working --- .bin/Scripts/functions/hw_diags.py | 73 +++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 0b88001b..54ce8ba5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -383,7 +383,6 @@ class State(): self.cpu = None self.disks = [] self.panes = {} - self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.quick_mode = False self.tests = OrderedDict({ 'Prime95': { @@ -423,6 +422,7 @@ class State(): os.makedirs(global_vars['LogDir'], exist_ok=True) global_vars['LogFile'] = '{}/Hardware Diagnostics.log'.format( global_vars['LogDir']) + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) # Add CPU self.cpu = CpuObj() @@ -723,19 +723,68 @@ def run_badblocks_test(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting badblocks test for {}'.format(test.dev.path)) - tmux_update_pane( - state.panes['Top'], - text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) - print_standard('TODO: run_badblocks_test({})'.format( - test.dev.path)) test.started = True test.update_status() update_progress_pane(state) - sleep(3) - test.update_status('Unknown') + + # Update top pane + tmux_update_pane( + state.panes['Top'], + text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + + # Create monitor pane + test.badblocks_out = '{}/badblocks.out'.format(global_vars['LogDir']) + state.panes['badblocks'] = tmux_split_window( + lines=5, vertical=True, watch=test.badblocks_out, watch_cmd='tail') + + # Show disk details + clear_screen() + show_report(test.dev.generate_report()) + print_standard(' ') + + # Start badblocks + print_standard('Running badblocks test...') + test.badblocks_proc = popen_program( + ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], + pipe=True) + test.badblocks_proc.wait() + + # Check result and create report + try: + test.badblocks_out = test.badblocks_proc.stdout.read().decode() + except Exception as err: + test.badblocks_out = 'Error: {}'.format(err) + for line in test.badblocks_out.splitlines(): + line = line.strip() + if not line or re.search(r'^Checking', line, re.IGNORECASE): + # Skip empty and progress lines + continue + if re.search(r'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): + test.report.append(' {}'.format(line)) + test.passed = True + else: + test.report.append(' {YELLOW}{line}{CLEAR}'.format( + line=line, **COLORS)) + test.failed = True + + # Update status + if test.failed: + test.update_status('NS') + elif test.passed: + test.update_status('CS') + else: + test.update_status('Unknown') + + # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['badblocks']) + pause() + def run_hw_tests(state): """Run enabled hardware tests.""" print_standard('Scanning devices...') @@ -828,6 +877,8 @@ def run_mprime_test(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting Prime95 test') test.started = True test.update_status() @@ -835,9 +886,9 @@ def run_mprime_test(state, test): test.sensor_data = get_sensor_data() # Update top pane - test.title = '{}\nPrime95: {}'.format( - TOP_PANE_TEXT, test.dev.name) - tmux_update_pane(state.panes['Top'], text=test.title) + tmux_update_pane( + state.panes['Top'], + text='{}\nPrime95: {}'.format(TOP_PANE_TEXT, test.dev.name)) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) From ef42b596d95199503380828c101eeb2d01554439 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Dec 2018 18:56:41 -0700 Subject: [PATCH 067/186] Catch CTRL+c aborts and show results --- .bin/Scripts/functions/hw_diags.py | 78 ++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 54ce8ba5..2ed8e686 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -747,10 +747,13 @@ def run_badblocks_test(state, test): # Start badblocks print_standard('Running badblocks test...') - test.badblocks_proc = popen_program( - ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], - pipe=True) - test.badblocks_proc.wait() + try: + test.badblocks_proc = popen_program( + ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], + pipe=True) + test.badblocks_proc.wait() + except KeyboardInterrupt: + raise GenericAbort('Aborted') # Check result and create report try: @@ -833,11 +836,31 @@ def run_hw_tests(state): # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. - for k, v in state.tests.items(): - if v['Enabled']: - f = v['Function'] - for test_obj in v['Objects']: - f(state, test_obj) + try: + for k, v in state.tests.items(): + if v['Enabled']: + f = v['Function'] + for test_obj in v['Objects']: + f(state, test_obj) + except GenericAbort: + # Cleanup + tmux_kill_pane(*state.panes.values()) + + # Rebuild panes + update_progress_pane(state) + build_outer_panes(state) + + # Mark unfinished tests as aborted + for k, v in state.tests.items(): + if v['Enabled']: + for test_obj in v['Objects']: + if re.search(r'(Pending|Working)', test_obj.status): + test_obj.update_status('Aborted') + test_obj.report.append(' {YELLOW}Aborted{CLEAR}'.format( + **COLORS)) + + # Update side pane + update_progress_pane(state) # Done show_results(state) @@ -1130,27 +1153,30 @@ def run_nvme_smart_tests(state, test): run_program(cmd, check=False) # Monitor progress (in 5 second increments) - for iteration in range(int(test.timeout*60/5)): - sleep(5) + try: + for iteration in range(int(test.timeout*60/5)): + sleep(5) - # Update SMART data - test.dev.get_smart_details() + # Update SMART data + test.dev.get_smart_details() - if _self_test_started: - # Update progress file - with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n {}'.format( - test.dev.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) + if _self_test_started: + # Update progress file + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n {}'.format( + test.dev.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) - # Check if test has finished - if 'remaining_percent' not in test.dev.smart_self_test['status']: - break + # Check if test has finished + if 'remaining_percent' not in test.dev.smart_self_test['status']: + break - else: - # Check if test has started - if 'remaining_percent' in test.dev.smart_self_test['status']: - _self_test_started = True + else: + # Check if test has started + if 'remaining_percent' in test.dev.smart_self_test['status']: + _self_test_started = True + except KeyboardInterrupt: + raise GenericAbort('Aborted') # Check if timed out if test.dev.smart_self_test['status'].get('passed', False): From 8993b483a633884bc6634e38c619701e4518dd3f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:30:46 -0700 Subject: [PATCH 068/186] Fix bad cable note --- .bin/Scripts/functions/hw_diags.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 2ed8e686..f0fd09b4 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -233,15 +233,15 @@ class DiskObj(): _color = COLORS[_c] # 199/C7 warning - if str(k) == '199': + if str(k) == '199' and v['raw'] > 0: _note = '(bad cable?)' # Attribute value - _line += '{}{} {}{}'.format( - _color, - v['raw_str'], - _note, - COLORS['CLEAR']) + _line += '{c}{v} {YELLOW}{n}{CLEAR}'.format( + c=_color, + v=v['raw_str'], + n=_note, + **COLORS) # Add line to report report.append(_line) From a4896a55f619ed7709a7bfa9a490bcac0445fb04 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:31:34 -0700 Subject: [PATCH 069/186] Adjust log names --- .bin/Scripts/functions/hw_diags.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index f0fd09b4..f2fa7469 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -736,7 +736,8 @@ def run_badblocks_test(state, test): text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) # Create monitor pane - test.badblocks_out = '{}/badblocks.out'.format(global_vars['LogDir']) + test.badblocks_out = '{}/badblocks_{}.out'.format( + global_vars['LogDir'], test.dev.name) state.panes['badblocks'] = tmux_split_window( lines=5, vertical=True, watch=test.badblocks_out, watch_cmd='tail') @@ -1134,7 +1135,8 @@ def run_nvme_smart_tests(state, test): _self_test_started = False # Create monitor pane - test.smart_out = '{}/smart.out'.format(global_vars['TmpDir']) + test.smart_out = '{}/smart_{}.out'.format( + global_vars['LogDir'], test.dev.name) with open(test.smart_out, 'w') as f: f.write('SMART self-test status:\n Starting...') state.panes['smart'] = tmux_split_window( From 503e6f2b4251fbf7355d413fb3df680441a45abb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 19:45:25 -0700 Subject: [PATCH 070/186] Fix SMART short-test timeout detection --- .bin/Scripts/functions/hw_diags.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index f2fa7469..361aad2d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1181,13 +1181,14 @@ def run_nvme_smart_tests(state, test): raise GenericAbort('Aborted') # Check if timed out - if test.dev.smart_self_test['status'].get('passed', False): - if 'OVERRIDE' not in test.status: - test.passed = True - test.update_status('CS') - else: - test.failed = True - test.update_status('NS') + if 'passed' in test.dev.smart_self_test['status']: + if test.dev.smart_self_test['status']['passed']: + if 'OVERRIDE' not in test.status: + test.passed = True + test.update_status('CS') + else: + test.failed = True + test.update_status('NS') if not (test.failed or test.passed): test.update_status('TimedOut') From 4c0bb1c9b7d954319f249422c1b41f2191fa0658 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:06:03 -0700 Subject: [PATCH 071/186] Group results by device instead of test --- .bin/Scripts/functions/hw_diags.py | 139 ++++++++++++++++++----------- 1 file changed, 87 insertions(+), 52 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 361aad2d..4b544303 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -79,7 +79,7 @@ class CpuObj(): """Object for tracking CPU specific data.""" def __init__(self): self.lscpu = {} - self.tests = {} + self.tests = OrderedDict() self.get_details() self.name = self.lscpu.get('Model name', 'Unknown CPU') @@ -102,6 +102,18 @@ class CpuObj(): continue self.lscpu[_field] = _data + def generate_cpu_report(self): + """Generate CPU report with data from all tests.""" + report = [] + report.append('{BLUE}Device{CLEAR}'.format(**COLORS)) + report.append(' {}'.format(self.name)) + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): @@ -115,9 +127,10 @@ class DiskObj(): self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() + self.warnings = [] self.get_details() self.get_smart_details() - self.description = '{size:>6} ({tran}) {model} {serial}'.format( + self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) def check_attributes(self, silent=False): @@ -159,8 +172,8 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - show_report(self.generate_report()) - print_warning('{} error(s) detected.'.format(attr_type)) + show_report(self.generate_attribute_report(description=True)) + print_warning(' {} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') pause() @@ -172,13 +185,15 @@ class DiskObj(): if self.nvme_attributes or not self.smart_attributes: # i.e. only leave enabled for SMART short-tests self.tests['NVMe / SMART'].disabled = True + print_standard(' ') - def generate_report(self, brief=False, short_test=False): + def generate_attribute_report( + self, description=False, short_test=False, timestamp=False): """Generate NVMe / SMART report, returns list.""" report = [] - if not brief: - report.append('{BLUE}Device: {dev_path}{CLEAR}'.format( - dev_path=self.path, **COLORS)) + if description: + report.append('{BLUE}Device ({name}){CLEAR}'.format( + name=self.name, **COLORS)) report.append(' {}'.format(self.description)) # Warnings @@ -203,8 +218,8 @@ class DiskObj(): # Attributes report.append('{BLUE}{a} Attributes{YELLOW}{u:>23} {t}{CLEAR}'.format( a=attr_type, - u='Updated:' if brief else '', - t=time.strftime('%Y-%m-%d %H:%M %Z') if brief else '', + u='Updated:' if timestamp else '', + t=time.strftime('%Y-%m-%d %H:%M %Z') if timestamp else '', **COLORS)) if self.nvme_attributes: attr_type = 'NVMe' @@ -259,6 +274,21 @@ class DiskObj(): # Done return report + def generate_disk_report(self): + """Generate disk report with data from all tests.""" + report = [] + report.append('{BLUE}Device ({name}){CLEAR}'.format( + name=self.name, **COLORS)) + report.append(' {}'.format(self.description)) + for w in self.warnings: + report.append(' {YELLOW}{w}{CLEAR}'.format(w=w, **COLORS)) + + # Tests + for test in self.tests.values(): + report.extend(test.report) + + return report + def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] @@ -338,14 +368,14 @@ class DiskObj(): # Check if a self-test is currently running if 'remaining_percent' in self.smart_self_test['status']: - _msg='SMART self-test in progress, all tests disabled' + _msg = 'SMART self-test in progress, all tests disabled' if not silent: print_warning('WARNING: {}'.format(_msg)) print_standard(' ') if ask('Abort HW Diagnostics?'): exit_script() if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].report.append( '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) for t in self.tests.values(): @@ -359,17 +389,20 @@ class DiskObj(): if silent: self.disk_ok = HW_OVERRIDES_FORCED else: - print_warning( - ' WARNING: No NVMe or SMART attributes available for: {}'.format( - self.path)) + _msg = 'No NVMe or SMART data available' + self.warnings.append(_msg) + print_info('Device ({})'.format(self.name)) + print_standard(' {}'.format(self.description)) + print_warning(' {}'.format(_msg)) self.disk_ok = HW_OVERRIDES_FORCED or ask( 'Run tests on this device anyway?') + print_standard(' ') if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set if not self.tests['NVMe / SMART'].report: - self.tests['NVMe / SMART'].report = self.generate_report() + self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].update_status('NS') self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: @@ -743,7 +776,7 @@ def run_badblocks_test(state, test): # Show disk details clear_screen() - show_report(test.dev.generate_report()) + show_report(test.dev.generate_attribute_report()) print_standard(' ') # Start badblocks @@ -757,6 +790,7 @@ def run_badblocks_test(state, test): raise GenericAbort('Aborted') # Check result and create report + test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) try: test.badblocks_out = test.badblocks_proc.stdout.read().decode() except Exception as err: @@ -787,7 +821,6 @@ def run_badblocks_test(state, test): # Cleanup tmux_kill_pane(state.panes['badblocks']) - pause() def run_hw_tests(state): """Run enabled hardware tests.""" @@ -825,15 +858,6 @@ def run_hw_tests(state): for disk in state.disks: disk.safety_check(silent=state.quick_mode) - # TODO Remove - clear_screen() - print_info('Running tests:') - for k, v in state.tests.items(): - if v['Enabled']: - print_standard(' {}'.format(k)) - update_progress_pane(state) - pause() - # Run tests ## Because state.tests is an OrderedDict and the disks were added ## in order, the tests will be run in order. @@ -1010,6 +1034,7 @@ def run_mprime_test(state, test): global_vars['LogDir'])) # Check results and build report + test.report.append('{BLUE}Prime95{CLEAR}'.format(**COLORS)) test.logs = {} for log in ['results.txt', 'prime.log']: lines = [] @@ -1026,20 +1051,18 @@ def run_mprime_test(state, test): # results.txt (NS check) if log == 'results.txt': - _tmp = [] for line in lines: + line = line.strip() if re.search(r'(error|fail)', line, re.IGNORECASE): test.failed = True test.update_status('NS') - _tmp.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) - if _tmp: - test.report.append('{BLUE}Log: results.txt{CLEAR}'.format(**COLORS)) - test.report.extend(_tmp) + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # prime.log (CS check) if log == 'prime.log': _tmp = {'Pass': {}, 'Warn': {}} for line in lines: + line = line.strip() _r = re.search( r'(completed.*(\d+) errors, (\d+) warnings)', line, @@ -1059,18 +1082,19 @@ def run_mprime_test(state, test): elif len(_tmp['Pass']) > 0: test.passed = True test.update_status('CS') - if len(_tmp['Pass']) + len(_tmp['Warn']) > 0: - test.report.append('{BLUE}Log: prime.log{CLEAR}'.format(**COLORS)) - for line in sorted(_tmp['Pass'].keys()): - test.report.append(' {}'.format(line)) - for line in sorted(_tmp['Warn'].keys()): - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) + for line in sorted(_tmp['Pass'].keys()): + test.report.append(' {}'.format(line)) + for line in sorted(_tmp['Warn'].keys()): + test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) - # Finalize report + # Unknown result if not (test.aborted or test.failed or test.passed): + test.report.append(' {YELLOW}Unknown result{CLEAR}'.format(**COLORS)) test.update_status('Unknown') + + # Add temps to report test.report.append('{BLUE}Temps{CLEAR}'.format(**COLORS)) - for line in generate_report( + for line in generate_sensor_report( test.sensor_data, 'Idle', 'Max', 'Cooldown', core_only=True): test.report.append(' {}'.format(line)) @@ -1144,10 +1168,10 @@ def run_nvme_smart_tests(state, test): # Show attributes clear_screen() - for line in test.dev.generate_report(): - # Not saving to log; that will happen after all tests have been run - print(line) - print(' ') + print_info('Device ({})'.format(test.dev.name)) + print_standard(' {}'.format(test.dev.description)) + show_report(test.dev.generate_attribute_report()) + print_standard(' ') # Start short test print_standard('Running self-test...') @@ -1203,7 +1227,7 @@ def run_nvme_smart_tests(state, test): tmux_kill_pane(state.panes['smart']) # Save report - test.report = test.dev.generate_report( + test.report = test.dev.generate_attribute_report( short_test=_include_short_test) # Done @@ -1231,13 +1255,24 @@ def show_results(state): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, 'Results')) - for k, v in state.tests.items(): - # Skip disabled tests - if not v['Enabled']: - continue - print_success('{}:'.format(k)) - for obj in v['Objects']: - show_report(obj.report) + + # CPU tests + _enabled = False + for k in TESTS_CPU: + _enabled |= state.tests[k]['Enabled'] + if _enabled: + print_success('CPU:'.format(k)) + show_report(state.cpu.generate_cpu_report()) + print_standard(' ') + + # Disk tests + for k in TESTS_DISK: + _enabled |= state.tests[k]['Enabled'] + if _enabled: + print_success('Disk{}:'.format( + '' if len(state.disks) == 1 else 's')) + for disk in state.disks: + show_report(disk.generate_disk_report()) print_standard(' ') def update_main_options(state, selection, main_options): From d8123a71ec5ca887115b76d07d197e54282bcd07 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:07:34 -0700 Subject: [PATCH 072/186] Renamed generate_report to generate_sensor_report --- .bin/Scripts/functions/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index b6319744..99e7999a 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -36,7 +36,7 @@ def fix_sensor_str(s): s = s.replace(' ', ' ') return s -def generate_report( +def generate_sensor_report( sensor_data, *temp_labels, colors=True, core_only=False): """Generate report based on temp_labels, returns list if str.""" @@ -153,7 +153,7 @@ def monitor_sensors(monitor_pane, monitor_file): while True: update_sensor_data(sensor_data) with open(monitor_file, 'w') as f: - report = generate_report(sensor_data, 'Current', 'Max') + report = generate_sensor_report(sensor_data, 'Current', 'Max') f.write('\n'.join(report)) sleep(1) if monitor_pane and not tmux_poll_pane(monitor_pane): From e0a2993c362469acaa08674bc85d28f3ef2920a2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:18:34 -0700 Subject: [PATCH 073/186] Skip disk safety checks if only testing the CPU --- .bin/Scripts/functions/hw_diags.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4b544303..6fd384a6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -854,9 +854,13 @@ def run_hw_tests(state): v['Objects'].append(test_obj) print_standard('') - # Run safety checks - for disk in state.disks: - disk.safety_check(silent=state.quick_mode) + # Run disk safety checks (if necessary) + _disk_tests_enabled = False + for k in TESTS_DISK: + _disk_tests_enabled |= state.tests[k]['Enabled'] + if _disk_tests_enabled: + for disk in state.disks: + disk.safety_check(silent=state.quick_mode) # Run tests ## Because state.tests is an OrderedDict and the disks were added @@ -1266,6 +1270,7 @@ def show_results(state): print_standard(' ') # Disk tests + _enabled = False for k in TESTS_DISK: _enabled |= state.tests[k]['Enabled'] if _enabled: From baaf1994e3312df5f39ee4bdd8c3009dacd50b9f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Dec 2018 22:44:46 -0700 Subject: [PATCH 074/186] Catch keyboard interrupt and gracefully abort --- .bin/Scripts/hw-diags-menu | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index 6f0247cd..5b6f4f76 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -13,20 +13,25 @@ from functions.tmux import * init_global_vars() if __name__ == '__main__': + # Show menu try: - # Show menu state = State() menu_diags(state, sys.argv) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() + except KeyboardInterrupt: + print_standard(' ') + print_warning('Aborted') + print_standard(' ') + sleep(1) + pause('Press Enter to exit...') except SystemExit: - tmux_kill_all_panes() + # Normal exit pass except: tmux_kill_all_panes() major_exception() + # Done + tmux_kill_all_panes() + exit_script() + # vim: sts=2 sw=2 ts=2 From c820d2ac6de623a4d6f1ef3400bca890b8f2965a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 13:20:39 -0700 Subject: [PATCH 075/186] Fixed Prime95 abort handling --- .bin/Scripts/functions/hw_diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 6fd384a6..ef411220 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1083,7 +1083,7 @@ def run_mprime_test(state, test): test.failed = True test.passed = False test.update_status('NS') - elif len(_tmp['Pass']) > 0: + elif len(_tmp['Pass']) > 0 and not test.aborted: test.passed = True test.update_status('CS') for line in sorted(_tmp['Pass'].keys()): From a25a10e616cbeaaec796bc0c0c3cf22f6e9c35f7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 14:07:19 -0700 Subject: [PATCH 076/186] More abort logic updates --- .bin/Scripts/functions/hw_diags.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index ef411220..c9a84940 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -127,7 +127,6 @@ class DiskObj(): self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() - self.warnings = [] self.get_details() self.get_smart_details() self.description = '{size} ({tran}) {model} {serial}'.format( @@ -280,8 +279,10 @@ class DiskObj(): report.append('{BLUE}Device ({name}){CLEAR}'.format( name=self.name, **COLORS)) report.append(' {}'.format(self.description)) - for w in self.warnings: - report.append(' {YELLOW}{w}{CLEAR}'.format(w=w, **COLORS)) + + # Attributes + if 'NVMe / SMART' not in self.tests: + report.extend(self.generate_attribute_report()) # Tests for test in self.tests.values(): @@ -389,11 +390,9 @@ class DiskObj(): if silent: self.disk_ok = HW_OVERRIDES_FORCED else: - _msg = 'No NVMe or SMART data available' - self.warnings.append(_msg) print_info('Device ({})'.format(self.name)) print_standard(' {}'.format(self.description)) - print_warning(' {}'.format(_msg)) + print_warning(' No NVMe or SMART data available') self.disk_ok = HW_OVERRIDES_FORCED or ask( 'Run tests on this device anyway?') print_standard(' ') @@ -787,9 +786,9 @@ def run_badblocks_test(state, test): pipe=True) test.badblocks_proc.wait() except KeyboardInterrupt: - raise GenericAbort('Aborted') + test.aborted = True - # Check result and create report + # Check result and build report test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) try: test.badblocks_out = test.badblocks_proc.stdout.read().decode() @@ -802,11 +801,16 @@ def run_badblocks_test(state, test): continue if re.search(r'^Pass completed.*0.*0/0/0', line, re.IGNORECASE): test.report.append(' {}'.format(line)) - test.passed = True + if not test.aborted: + test.passed = True else: test.report.append(' {YELLOW}{line}{CLEAR}'.format( line=line, **COLORS)) test.failed = True + if test.aborted: + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + test.update_status('Aborted') + raise GenericAbort('Aborted') # Update status if test.failed: @@ -885,8 +889,6 @@ def run_hw_tests(state): for test_obj in v['Objects']: if re.search(r'(Pending|Working)', test_obj.status): test_obj.update_status('Aborted') - test_obj.report.append(' {YELLOW}Aborted{CLEAR}'.format( - **COLORS)) # Update side pane update_progress_pane(state) @@ -1206,6 +1208,12 @@ def run_nvme_smart_tests(state, test): if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True except KeyboardInterrupt: + test.aborted = True + test.report = test.dev.generate_attribute_report() + test.report.append('{BLUE}SMART Short self-test{CLEAR}'.format( + **COLORS)) + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + test.update_status('Aborted') raise GenericAbort('Aborted') # Check if timed out From 385bdd7dbf11f3e5f6b99029263ea9f4fc2f7879 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:10:58 -0700 Subject: [PATCH 077/186] Allow resizing current pane --- .bin/Scripts/functions/tmux.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 5fbca65d..20f35921 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -28,12 +28,15 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes -def tmux_resize_pane(pane_id, x=None, y=None): +def tmux_resize_pane(pane_id=None, x=None, y=None): """Resize pane to specific hieght or width.""" if not x and not y: raise Exception('Neither height nor width specified.') - cmd = ['tmux', 'resize-pane', '-t', pane_id] + cmd = ['tmux', 'resize-pane'] + if pane_id: + # NOTE: If pane_id not specified then the current pane will be resized + cmd.extend(['-t', pane_id]) if x: cmd.extend(['-x', str(x)]) elif y: From ec8c78197b45b3a6d812474dc0f3d0d30af55ec6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:15:40 -0700 Subject: [PATCH 078/186] I/O Benchmark test is working --- .bin/Scripts/functions/hw_diags.py | 223 +++++++++++++++++++++++++++-- 1 file changed, 212 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index c9a84940..50eca7b5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -34,7 +34,6 @@ HW_OVERRIDES_FORCED = HW_OVERRIDES_FORCED and not HW_OVERRIDES_LIMITED IO_VARS = { 'Block Size': 512*1024, 'Chunk Size': 32*1024**2, - 'Minimum Dev Size': 8*1024**3, 'Minimum Test Size': 10*1024**3, 'Alt Test Size Factor': 0.01, 'Progress Refresh Rate': 5, @@ -74,6 +73,10 @@ TESTS_DISK = [ ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) +# Error Classe +class DeviceTooSmallError(Exception): + pass + # Classes class CpuObj(): """Object for tracking CPU specific data.""" @@ -132,6 +135,60 @@ class DiskObj(): self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) + def calc_io_dd_values(self): + """Calcualte I/O benchmark dd values.""" + # Get real disk size + cmd = ['lsblk', + '--bytes', '--nodeps', '--noheadings', + '--output', 'size', self.path] + result = run_program(cmd) + self.size_bytes = int(result.stdout.decode().strip()) + + # dd calculations + ## The minimum dev size is 'Graph Horizontal Width' * 'Chunk Size' + ## (e.g. 1.25 GB for a width of 40 and a chunk size of 32MB) + ## If the device is smaller than the minimum dd_chunks would be set + ## to zero which would cause a divide by zero error. + ## If the device is below the minimum size an Exception will be raised + ## + ## dd_size is the area to be read in bytes + ## If the dev is < 10Gb then it's the whole dev + ## Otherwise it's the larger of 10Gb or 1% of the dev + ## + ## dd_chunks is the number of groups of "Chunk Size" in self.dd_size + ## This number is reduced to a multiple of the graph width in + ## order to allow for the data to be condensed cleanly + ## + ## dd_chunk_blocks is the chunk size in number of blocks + ## (e.g. 64 if block size is 512KB and chunk size is 32MB + ## + ## dd_skip_blocks is the number of "Block Size" groups not tested + ## dd_skip_count is the number of blocks to skip per self.dd_chunk + ## dd_skip_extra is how often to add an additional skip block + ## This is needed to ensure an even testing across the dev + ## This is calculated by using the fractional amount left off + ## of the dd_skip_count variable + self.dd_size = min(IO_VARS['Minimum Test Size'], self.size_bytes) + self.dd_size = max( + self.dd_size, + self.size_bytes * IO_VARS['Alt Test Size Factor']) + self.dd_chunks = int(self.dd_size // IO_VARS['Chunk Size']) + self.dd_chunks -= self.dd_chunks % IO_VARS['Graph Horizontal Width'] + if self.dd_chunks < IO_VARS['Graph Horizontal Width']: + raise DeviceTooSmallError + self.dd_chunk_blocks = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) + self.dd_size = self.dd_chunks * IO_VARS['Chunk Size'] + self.dd_skip_blocks = int( + (self.size_bytes - self.dd_size) // IO_VARS['Block Size']) + self.dd_skip_count = int((self.dd_skip_blocks / self.dd_chunks) // 1) + self.dd_skip_extra = 0 + try: + self.dd_skip_extra = 1 + int( + 1 / ((self.dd_skip_blocks / self.dd_chunks) % 1)) + except ZeroDivisionError: + # self.dd_skip_extra == 0 is fine + pass + def check_attributes(self, silent=False): """Check NVMe / SMART attributes for errors.""" override_disabled = False @@ -498,7 +555,7 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled or 'OVERRIDE' in self.status: + if self.disabled or re.search(r'ERROR|OVERRIDE', self.status): return if new_status: self.status = build_status_string( @@ -904,23 +961,163 @@ def run_hw_tests(state): tmux_kill_pane(*state.panes.values()) def run_io_benchmark(state, test): - """TODO""" + """Run a read-only I/O benchmark using dd.""" # Bail early if test.disabled: return + + # Prep print_log('Starting I/O benchmark test for {}'.format(test.dev.path)) - tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'I/O Benchmark')) - print_standard('TODO: run_io_benchmark({})'.format( - test.dev.path)) test.started = True test.update_status() update_progress_pane(state) - sleep(3) - test.update_status('Unknown') + + # Update top pane + tmux_update_pane( + state.panes['Top'], + text='{}\nI/O Benchmark: {}'.format( + TOP_PANE_TEXT, test.dev.description)) + + # Create monitor pane + test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( + global_vars['LogDir'], test.dev.name) + state.panes['io_benchmark'] = tmux_split_window( + percent=75, vertical=True, + watch=test.io_benchmark_out, watch_cmd='tail') + tmux_resize_pane(y=15) + + # Show disk details + clear_screen() + show_report(test.dev.generate_attribute_report()) + print_standard(' ') + + # Start I/O Benchmark + print_standard('Running I/O benchmark test...') + try: + test.merged_rates = [] + test.read_rates = [] + test.vertical_graph = [] + test.dev.calc_io_dd_values() + + # Run dd read tests + offset = 0 + for i in range(test.dev.dd_chunks): + # Build cmd + i += 1 + skip = test.dev.dd_skip_count + if test.dev.dd_skip_extra and i % test.dev.dd_skip_extra == 0: + skip += 1 + cmd = [ + 'sudo', 'dd', + 'bs={}'.format(IO_VARS['Block Size']), + 'skip={}'.format(offset+skip), + 'count={}'.format(test.dev.dd_chunk_blocks), + 'iflag=direct', + 'if={}'.format(test.dev.path), + 'of=/dev/null'] + + # Run cmd and get read rate + result = run_program(cmd) + result_str = result.stderr.decode().replace('\n', '') + cur_rate = get_read_rate(result_str) + + # Add rate to lists + test.read_rates.append(cur_rate) + test.vertical_graph.append( + '{percent:0.1f} {rate}'.format( + percent=(i/test.dev.dd_chunks)*100, + rate=int(cur_rate/(1024**2)))) + + # Show progress + if i % IO_VARS['Progress Refresh Rate'] == 0: + update_io_progress( + percent=(i/test.dev.dd_chunks)*100, + rate=cur_rate, + progress_file=test.io_benchmark_out) + + # Update offset + offset += test.dev.dd_chunk_blocks + skip + + except DeviceTooSmallError: + # Device too small, skipping test + test.update_status('N/A') + except KeyboardInterrupt: + test.aborted = True + except (subprocess.CalledProcessError, TypeError, ValueError): + # Something went wrong, results unknown + test.update_status('ERROR') + + # Check result and build report + test.report.append('{BLUE}I/O Benchmark{CLEAR}'.format(**COLORS)) + if test.aborted: + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + raise GenericAbort('Aborted') + elif not test.read_rates: + if 'ERROR' in test.status: + test.report.append(' {RED}Unknown error{CLEAR}'.format(**COLORS)) + elif 'N/A' in test.status: + # Device too small + test.report.append(' {YELLOW}Disk too small to test{CLEAR}'.format( + **COLORS)) + else: + # Merge rates for horizontal graph + offset = 0 + width = int(test.dev.dd_chunks / IO_VARS['Graph Horizontal Width']) + for i in range(IO_VARS['Graph Horizontal Width']): + test.merged_rates.append( + sum(test.read_rates[offset:offset+width])/width) + offset += width + + # Add horizontal graph to report + for line in generate_horizontal_graph(test.merged_rates): + if not re.match(r'^\s+$', line): + test.report.append(line) + + # Add read speeds to report + avg_read = sum(test.read_rates) / len(test.read_rates) + min_read = min(test.read_rates) + max_read = max(test.read_rates) + avg_min_max = 'Read speeds avg: {:3.1f}'.format(avg_read/(1024**2)) + avg_min_max += ' min: {:3.1f}'.format(min_read/(1024**2)) + avg_min_max += ' max: {:3.1f}'.format(max_read/(1024**2)) + test.report.append(avg_min_max) + + # Compare read speeds to thresholds + if test.dev.lsblk['rota']: + # Use HDD scale + thresh_min = IO_VARS['Threshold HDD Min'] + thresh_high_avg = IO_VARS['Threshold HDD High Avg'] + thresh_low_avg = IO_VARS['Threshold HDD Low Avg'] + else: + # Use SSD scale + thresh_min = IO_VARS['Threshold SSD Min'] + thresh_high_avg = IO_VARS['Threshold SSD High Avg'] + thresh_low_avg = IO_VARS['Threshold SSD Low Avg'] + if min_read <= thresh_min and avg_read <= thresh_high_avg: + test.failed = True + elif avg_read <= thresh_low_avg: + test.failed = True + else: + test.passed = True + + # Update status + if test.failed: + test.update_status('NS') + elif test.passed: + test.update_status('CS') + elif not 'N/A' in test.status: + test.update_status('Unknown') + + # Save log + with open(test.io_benchmark_out.replace('.', '-raw.'), 'a') as f: + f.write('\n'.join(test.vertical_graph)) + + # Done update_progress_pane(state) + # Cleanup + tmux_kill_pane(state.panes['io_benchmark']) + def run_keyboard_test(): """Run keyboard test.""" clear_screen() @@ -1122,14 +1319,18 @@ def run_nvme_smart_tests(state, test): # Bail early if test.disabled: return + + # Prep print_log('Starting NVMe/SMART test for {}'.format(test.dev.path)) _include_short_test = False test.started = True test.update_status() + update_progress_pane(state) + + # Update top pane tmux_update_pane( state.panes['Top'], text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) - update_progress_pane(state) # NVMe if test.dev.nvme_attributes: From 8c5820d5aa73cbface183311655f97842854eb99 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:16:35 -0700 Subject: [PATCH 079/186] Fix horizontal graph * generate_horizontal_graph() now returns a list instead of a str --- .bin/Scripts/functions/hw_diags.py | 48 +++++++++++++----------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 50eca7b5..85c4d86e 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -608,11 +608,8 @@ def build_status_string(label, status, info_label=False): **COLORS) def generate_horizontal_graph(rates, oneline=False): - """Generate two-line horizontal graph from rates, returns str.""" - line_1 = '' - line_2 = '' - line_3 = '' - line_4 = '' + """Generate horizontal graph from rates, returns list.""" + graph = ['', '', '', ''] for r in rates: step = get_graph_step(r, scale=32) if oneline: @@ -630,33 +627,30 @@ def generate_horizontal_graph(rates, oneline=False): # Build graph full_block = '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) if step >= 24: - line_1 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) - line_2 += full_block - line_3 += full_block - line_4 += full_block + graph[0] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-24]) + graph[1] += full_block + graph[2] += full_block + graph[3] += full_block elif step >= 16: - line_1 += ' ' - line_2 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) - line_3 += full_block - line_4 += full_block + graph[0] += ' ' + graph[1] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-16]) + graph[2] += full_block + graph[3] += full_block elif step >= 8: - line_1 += ' ' - line_2 += ' ' - line_3 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) - line_4 += full_block + graph[0] += ' ' + graph[1] += ' ' + graph[2] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + graph[3] += full_block else: - line_1 += ' ' - line_2 += ' ' - line_3 += ' ' - line_4 += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) - line_1 += COLORS['CLEAR'] - line_2 += COLORS['CLEAR'] - line_3 += COLORS['CLEAR'] - line_4 += COLORS['CLEAR'] + graph[0] += ' ' + graph[1] += ' ' + graph[2] += ' ' + graph[3] += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + graph = [line+COLORS['CLEAR'] for line in graph] if oneline: - return line_4 + return graph[:-1] else: - return '\n'.join([line_1, line_2, line_3, line_4]) + return graph def get_graph_step(rate, scale=16): """Get graph step based on rate and scale, returns int.""" From 41c9a4d23fa5c9df58ceef85d6a2e35934827a59 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:29:09 -0700 Subject: [PATCH 080/186] Fixed only showing non-empty graph lines --- .bin/Scripts/functions/hw_diags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 85c4d86e..30220583 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1064,7 +1064,7 @@ def run_io_benchmark(state, test): # Add horizontal graph to report for line in generate_horizontal_graph(test.merged_rates): - if not re.match(r'^\s+$', line): + if not re.match(r'^\s+$', strip_colors(line)): test.report.append(line) # Add read speeds to report From 0c0f8e895021c7bfae0ae1a1d258bf86046f54ae Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Dec 2018 20:51:02 -0700 Subject: [PATCH 081/186] Added disable_test() to Disk class --- .bin/Scripts/functions/hw_diags.py | 44 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 30220583..4a855dc6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -237,12 +237,18 @@ class DiskObj(): if HW_OVERRIDES_FORCED or ask('Run tests on this device anyway?'): self.disk_ok = True if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].update_status('OVERRIDE') - if self.nvme_attributes or not self.smart_attributes: - # i.e. only leave enabled for SMART short-tests - self.tests['NVMe / SMART'].disabled = True + self.disable_test('NVMe / SMART', 'OVERRIDE') + if not self.nvme_attributes and self.smart_attributes: + # Re-enable for SMART short-tests + self.tests['NVMe / SMART'].disabled = False print_standard(' ') + def disable_test(self, name, status): + """Disable test by name and update status.""" + if name in self.tests: + self.tests[name].update_status(status) + self.tests[name].disabled = True + def generate_attribute_report( self, description=False, short_test=False, timestamp=False): """Generate NVMe / SMART report, returns list.""" @@ -427,23 +433,26 @@ class DiskObj(): # Check if a self-test is currently running if 'remaining_percent' in self.smart_self_test['status']: _msg = 'SMART self-test in progress, all tests disabled' + + # Ask to abort if not silent: print_warning('WARNING: {}'.format(_msg)) print_standard(' ') if ask('Abort HW Diagnostics?'): exit_script() + + # Add warning to report if 'NVMe / SMART' in self.tests: self.tests['NVMe / SMART'].report = self.generate_attribute_report() self.tests['NVMe / SMART'].report.append( '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) - for t in self.tests.values(): - t.update_status('Denied') - t.disabled = True + + # Disable all tests for this disk + for t in self.tests.keys(): + self.disable_test(k, 'Denied') else: # No NVMe/SMART details - if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].update_status('N/A') - self.tests['NVMe / SMART'].disabled = True + self.disable_test('NVMe / SMART', 'N/A') if silent: self.disk_ok = HW_OVERRIDES_FORCED else: @@ -457,14 +466,11 @@ class DiskObj(): if not self.disk_ok: if 'NVMe / SMART' in self.tests: # NOTE: This will not overwrite the existing status if set + self.disable_test('NVMe / SMART', 'NS') if not self.tests['NVMe / SMART'].report: self.tests['NVMe / SMART'].report = self.generate_attribute_report() - self.tests['NVMe / SMART'].update_status('NS') - self.tests['NVMe / SMART'].disabled = True for t in ['badblocks', 'I/O Benchmark']: - if t in self.tests: - self.tests[t].update_status('Denied') - self.tests[t].disabled = True + self.disable_test(t, 'Denied') class State(): """Object to track device objects and overall state.""" @@ -863,6 +869,10 @@ def run_badblocks_test(state, test): test.update_status('Aborted') raise GenericAbort('Aborted') + # Disable other drive tests if necessary + if not test.passed: + test.dev.disable_test('I/O Benchmark', 'Denied') + # Update status if test.failed: test.update_status('NS') @@ -1426,9 +1436,7 @@ def run_nvme_smart_tests(state, test): # Disable other drive tests if necessary if not test.passed: for t in ['badblocks', 'I/O Benchmark']: - if t in test.dev.tests: - test.dev.tests[t].update_status('Denied') - test.dev.tests[t].disabled = True + test.dev.disable_test(t, 'Denied') # Cleanup tmux_kill_pane(state.panes['smart']) From 10ae59be197c431f74e64e03758cdd15b8599432 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 00:55:57 -0700 Subject: [PATCH 082/186] Update tmux layout periodically --- .bin/Scripts/functions/hw_diags.py | 106 ++++++++++++++++++++++++++--- .bin/Scripts/functions/tmux.py | 20 +++++- 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 4a855dc6..56b3c501 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -72,6 +72,11 @@ TESTS_DISK = [ 'badblocks', ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) +TMUX_LAYOUT = OrderedDict({ + 'Top': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, +}) # Error Classe class DeviceTooSmallError(Exception): @@ -449,7 +454,7 @@ class DiskObj(): # Disable all tests for this disk for t in self.tests.keys(): - self.disable_test(k, 'Denied') + self.disable_test(t, 'Denied') else: # No NVMe/SMART details self.disable_test('NVMe / SMART', 'N/A') @@ -613,6 +618,49 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) +def fix_tmux_panes(state, tmux_layout): + """Fix pane sizes in case the window has been resized.""" + needs_fixed = False + + # Check layout + for k, v in tmux_layout.items(): + if not v.get('Check'): + # Not concerned with the size of this pane + continue + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Get pane size + x, y = tmux_get_pane_size(pane_id=target) + if v.get('x', False) and v.get['x'] != x: + needs_fixed = True + if v.get('y', False) and v.get['y'] != y: + needs_fixed = True + + # Bail? + if not needs_fixed: + return + + # Update layout + for k, v in tmux_layout.items(): + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Resize pane + tmux_resize_pane(pane_id=target, **v) + def generate_horizontal_graph(rates, oneline=False): """Generate horizontal graph from rates, returns list.""" graph = ['', '', '', ''] @@ -819,10 +867,14 @@ def run_badblocks_test(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'badblocks': {'y': 5, 'Check': True}, + }) # Create monitor pane test.badblocks_out = '{}/badblocks_{}.out'.format( @@ -841,7 +893,15 @@ def run_badblocks_test(state, test): test.badblocks_proc = popen_program( ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], pipe=True) - test.badblocks_proc.wait() + while True: + try: + test.badblocks_proc.wait(timeout=10) + except subprocess.TimeoutExpired: + fix_tmux_panes(state, test.tmux_layout) + else: + # badblocks finished, exit loop + break + except KeyboardInterrupt: test.aborted = True @@ -976,11 +1036,16 @@ def run_io_benchmark(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nI/O Benchmark: {}'.format( TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'io_benchmark': {'y': 1000, 'Check': False}, + 'Current': {'y': 15, 'Check': True}, + }) # Create monitor pane test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( @@ -1042,6 +1107,10 @@ def run_io_benchmark(state, test): # Update offset offset += test.dev.dd_chunk_blocks + skip + # Fix panes + if i % 5 == 0: + fix_tmux_panes(state, test.tmux_layout) + except DeviceTooSmallError: # Device too small, skipping test test.update_status('N/A') @@ -1140,10 +1209,16 @@ def run_mprime_test(state, test): update_progress_pane(state) test.sensor_data = get_sensor_data() - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nPrime95: {}'.format(TOP_PANE_TEXT, test.dev.name)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'Temps': {'y': 1000, 'Check': False}, + 'mprime': {'y': 11, 'Check': False}, + 'Current': {'y': 3, 'Check': True}, + }) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) @@ -1181,7 +1256,7 @@ def run_mprime_test(state, test): working_dir=global_vars['TmpDir']) #time_limit = int(MPRIME_LIMIT) * 60 # TODO: restore above line - time_limit = 10 + time_limit = 30 try: for i in range(time_limit): clear_screen() @@ -1199,6 +1274,12 @@ def run_mprime_test(state, test): print(_status_str) print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) update_sensor_data(test.sensor_data) + + # Fix panes + if i % 10 == 0: + fix_tmux_panes(state, test.tmux_layout) + + # Wait sleep(1) except KeyboardInterrupt: # Catch CTRL+C @@ -1331,10 +1412,14 @@ def run_nvme_smart_tests(state, test): test.update_status() update_progress_pane(state) - # Update top pane + # Update tmux layout tmux_update_pane( state.panes['Top'], text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) + test.tmux_layout = TMUX_LAYOUT.copy() + test.tmux_layout.update({ + 'smart': {'y': 3, 'Check': True}, + }) # NVMe if test.dev.nvme_attributes: @@ -1391,7 +1476,7 @@ def run_nvme_smart_tests(state, test): # Monitor progress (in 5 second increments) try: - for iteration in range(int(test.timeout*60/5)): + for i in range(int(test.timeout*60/5)): sleep(5) # Update SMART data @@ -1412,6 +1497,11 @@ def run_nvme_smart_tests(state, test): # Check if test has started if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True + + # Fix panes + if i % 2 == 0: + fix_tmux_panes(state, test.tmux_layout) + except KeyboardInterrupt: test.aborted = True test.report = test.dev.generate_attribute_report() diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 20f35921..0e585df6 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -8,6 +8,24 @@ def create_file(filepath): with open(filepath, 'w') as f: f.write('') +def tmux_get_pane_size(pane_id=None): + """Get target, or current, pane size, returns tuple.""" + x = -1 + y = -1 + cmd = ['tmux', 'display', '-p', '#{pane_width}x#{pane_height}'] + if pane_id: + cmd.extend(['-t', pane_id]) + + # Run cmd and set x & y + result = run_program(cmd, check=False) + try: + x, y = result.stdout.decode().strip().split() + except Exception: + # Ignore and return unrealistic values + pass + + return (x, y) + def tmux_kill_all_panes(pane_id=None): """Kill all tmux panes except the active pane or pane_id if specified.""" cmd = ['tmux', 'kill-pane', '-a'] @@ -28,7 +46,7 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes -def tmux_resize_pane(pane_id=None, x=None, y=None): +def tmux_resize_pane(pane_id=None, x=None, y=None, **kwargs): """Resize pane to specific hieght or width.""" if not x and not y: raise Exception('Neither height nor width specified.') From 932669844baee7a485ece26f83f7885875245641 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:13:33 -0700 Subject: [PATCH 083/186] Fixed tmux pane size handling --- .bin/Scripts/functions/hw_diags.py | 29 +++++++++++++++-------------- .bin/Scripts/functions/tmux.py | 5 ++++- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 56b3c501..dc71c54f 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -638,9 +638,9 @@ def fix_tmux_panes(state, tmux_layout): # Get pane size x, y = tmux_get_pane_size(pane_id=target) - if v.get('x', False) and v.get['x'] != x: + if v.get('x', False) and v['x'] != x: needs_fixed = True - if v.get('y', False) and v.get['y'] != y: + if v.get('y', False) and v['y'] != y: needs_fixed = True # Bail? @@ -895,7 +895,7 @@ def run_badblocks_test(state, test): pipe=True) while True: try: - test.badblocks_proc.wait(timeout=10) + test.badblocks_proc.wait(timeout=1) except subprocess.TimeoutExpired: fix_tmux_panes(state, test.tmux_layout) else: @@ -1108,8 +1108,7 @@ def run_io_benchmark(state, test): offset += test.dev.dd_chunk_blocks + skip # Fix panes - if i % 5 == 0: - fix_tmux_panes(state, test.tmux_layout) + fix_tmux_panes(state, test.tmux_layout) except DeviceTooSmallError: # Device too small, skipping test @@ -1276,8 +1275,7 @@ def run_mprime_test(state, test): update_sensor_data(test.sensor_data) # Fix panes - if i % 10 == 0: - fix_tmux_panes(state, test.tmux_layout) + fix_tmux_panes(state, test.tmux_layout) # Wait sleep(1) @@ -1474,10 +1472,17 @@ def run_nvme_smart_tests(state, test): cmd = ['sudo', 'smartctl', '--test=short', test.dev.path] run_program(cmd, check=False) - # Monitor progress (in 5 second increments) + # Monitor progress try: - for i in range(int(test.timeout*60/5)): - sleep(5) + for i in range(int(test.timeout*60)): + sleep(1) + + # Fix panes + fix_tmux_panes(state, test.tmux_layout) + + # Only update SMART progress every 5 seconds + if i % 5 != 0: + continue # Update SMART data test.dev.get_smart_details() @@ -1498,10 +1503,6 @@ def run_nvme_smart_tests(state, test): if 'remaining_percent' in test.dev.smart_self_test['status']: _self_test_started = True - # Fix panes - if i % 2 == 0: - fix_tmux_panes(state, test.tmux_layout) - except KeyboardInterrupt: test.aborted = True test.report = test.dev.generate_attribute_report() diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 0e585df6..ce35f5b0 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -12,14 +12,17 @@ def tmux_get_pane_size(pane_id=None): """Get target, or current, pane size, returns tuple.""" x = -1 y = -1 - cmd = ['tmux', 'display', '-p', '#{pane_width}x#{pane_height}'] + cmd = ['tmux', 'display', '-p'] if pane_id: cmd.extend(['-t', pane_id]) + cmd.append('#{pane_width} #{pane_height}') # Run cmd and set x & y result = run_program(cmd, check=False) try: x, y = result.stdout.decode().strip().split() + x = int(x) + y = int(y) except Exception: # Ignore and return unrealistic values pass From 7ac035c578b2fe62b308a14d783b96d6f3fbc98d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:21:05 -0700 Subject: [PATCH 084/186] Safety wheels off --- .bin/Scripts/functions/hw_diags.py | 22 +++++++--------------- .bin/Scripts/settings/main.py | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index dc71c54f..cfe68c69 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -837,13 +837,11 @@ def menu_diags(state, args): elif selection == 'R': print('(FAKE) reboot...') sleep(1) - # TODO uncomment below - #run_program(['systemctl', 'reboot']) + run_program(['systemctl', 'reboot']) elif selection == 'P': print('(FAKE) poweroff...') sleep(1) - # TODO uncomment below - #run_program(['systemctl', 'poweroff']) + run_program(['systemctl', 'poweroff']) elif selection == 'Q': break elif selection == 'S': @@ -1242,8 +1240,7 @@ def run_mprime_test(state, test): message='Getting idle temps...', indent=0, function=save_average_temp, cs='Done', sensor_data=test.sensor_data, temp_label='Idle', - seconds=3) - # TODO: Remove seconds kwarg above + seconds=5) # Stress CPU print_log('Starting Prime95') @@ -1253,9 +1250,7 @@ def run_mprime_test(state, test): state.panes['mprime'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) - #time_limit = int(MPRIME_LIMIT) * 60 - # TODO: restore above line - time_limit = 30 + time_limit = int(MPRIME_LIMIT) * 60 try: for i in range(time_limit): clear_screen() @@ -1301,14 +1296,12 @@ def run_mprime_test(state, test): clear_screen() try_and_print( message='Letting CPU cooldown for bit...', indent=0, - function=sleep, cs='Done', seconds=3) - # TODO: Above seconds should be 10 + function=sleep, cs='Done', seconds=10) try_and_print( message='Getting cooldown temps...', indent=0, function=save_average_temp, cs='Done', sensor_data=test.sensor_data, temp_label='Cooldown', - seconds=3) - # TODO: Remove seconds kwarg above + seconds=5) # Move logs to Ticket folder for item in os.scandir(global_vars['TmpDir']): @@ -1447,8 +1440,7 @@ def run_nvme_smart_tests(state, test): # Prep test.timeout = test.dev.smart_self_test['polling_minutes'].get( 'short', 5) - # TODO: fix timeout, set to polling + 5 - test.timeout = int(test.timeout) + 1 + test.timeout = int(test.timeout) + 5 _include_short_test = True _self_test_started = False diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 7b915bdb..9d32b3ef 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -15,7 +15,7 @@ KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux -MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags +MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags ROOT_PASSWORD='Abracadabra' TECH_PASSWORD='Abracadabra' # Server IP addresses From 91a77bb14e9bfb1f9525aaffc6390e667e32c2d9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:47:03 -0700 Subject: [PATCH 085/186] Ensure SMART timeout message is in the report --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cfe68c69..64d417da 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -132,6 +132,7 @@ class DiskObj(): self.nvme_attributes = {} self.path = disk_path self.smart_attributes = {} + self.smart_timeout = False self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() @@ -331,12 +332,11 @@ class DiskObj(): # SMART short-test if short_test: report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) - if 'TimedOut' in self.tests['NVMe / SMART'].status: - report.append(' {YELLOW}UNKNOWN{CLEAR}: Timed out'.format(**COLORS)) - else: - report.append(' {}'.format( - self.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) + report.append(' {}'.format( + self.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) + if self.smart_timeout: + report.append(' {YELLOW}Timed out{CLEAR}'.format(**COLORS)) # Done return report @@ -1443,6 +1443,7 @@ def run_nvme_smart_tests(state, test): test.timeout = int(test.timeout) + 5 _include_short_test = True _self_test_started = False + _self_test_finished = False # Create monitor pane test.smart_out = '{}/smart_{}.out'.format( @@ -1488,6 +1489,7 @@ def run_nvme_smart_tests(state, test): # Check if test has finished if 'remaining_percent' not in test.dev.smart_self_test['status']: + _self_test_finished = True break else: @@ -1505,15 +1507,16 @@ def run_nvme_smart_tests(state, test): raise GenericAbort('Aborted') # Check if timed out - if 'passed' in test.dev.smart_self_test['status']: - if test.dev.smart_self_test['status']['passed']: + if _self_test_finished: + if test.dev.smart_self_test['status'].get('passed', False): if 'OVERRIDE' not in test.status: test.passed = True test.update_status('CS') else: test.failed = True test.update_status('NS') - if not (test.failed or test.passed): + else: + test.dev.smart_timeout = True test.update_status('TimedOut') # Disable other drive tests if necessary From e5f0ccb5d51bf8d89460c3a35eff26d10e8dc333 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 15:57:48 -0700 Subject: [PATCH 086/186] Formatting cleanup --- .bin/Scripts/functions/hw_diags.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 64d417da..b09dbf05 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -591,8 +591,8 @@ def build_outer_panes(state): # Started state.panes['Started'] = tmux_split_window( lines=SIDE_PANE_WIDTH, target_pane=state.panes['Top'], - text='{BLUE}Started{CLEAR}\n{text}'.format( - text=time.strftime("%Y-%m-%d %H:%M %Z"), + text='{BLUE}Started{CLEAR}\n{s}'.format( + s=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) # Progress @@ -868,7 +868,8 @@ def run_badblocks_test(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nbadblocks: {}'.format(TOP_PANE_TEXT, test.dev.description)) + text='{}\nbadblocks: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'badblocks': {'y': 5, 'Check': True}, @@ -1037,8 +1038,8 @@ def run_io_benchmark(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nI/O Benchmark: {}'.format( - TOP_PANE_TEXT, test.dev.description)) + text='{}\nI/O Benchmark: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'io_benchmark': {'y': 1000, 'Check': False}, @@ -1406,7 +1407,8 @@ def run_nvme_smart_tests(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nDisk Health: {}'.format(TOP_PANE_TEXT, test.dev.description)) + text='{}\nDisk Health: {}'.format( + TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'smart': {'y': 3, 'Check': True}, @@ -1554,8 +1556,8 @@ def show_results(state): """Show results for all tests.""" clear_screen() tmux_update_pane( - state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, 'Results')) + state.panes['Top'], + text='{}\nResults'.format(TOP_PANE_TEXT)) # CPU tests _enabled = False From ad9662c1208f00071d6ce4e721fd996e3e320035 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 16:38:40 -0700 Subject: [PATCH 087/186] Updated to use new hw_diags.py --- .bin/Scripts/ddrescue-tui-smart-display | 52 +++++++++++++------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display index 285229d6..65b76890 100755 --- a/.bin/Scripts/ddrescue-tui-smart-display +++ b/.bin/Scripts/ddrescue-tui-smart-display @@ -10,30 +10,34 @@ import time os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.hw_diags import * -#init_global_vars() +init_global_vars() if __name__ == '__main__': - try: - # Prep - clear_screen() - dev_path = sys.argv[1] - devs = scan_disks(True, dev_path) - - # Warn if SMART unavailable - if dev_path not in devs: - print_error('SMART data not available') - exit_script() - - # Initial screen - dev = devs[dev_path] - clear_screen() - show_disk_details(dev, only_attributes=True) - - # Done - exit_script() - except SystemExit: - pass - except: - major_exception() + try: + # Prep + clear_screen() + state = State() + state.init() -# vim: sts=4 sw=4 ts=4 + # Find disk + disk = None + for d in state.disks: + if d.path == sys.argv[1]: + disk = d + + # Show details + clear_screen() + if disk: + for line in disk.generate_attribute_report(timestamp=True): + print(line) + else: + print_error('Disk "{}" not found'.format(sys.argv[1])) + + # Done + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 From e1834d517953fd6a183be960770a4c52a91a31a5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 17:26:20 -0700 Subject: [PATCH 088/186] Added silent mode to init_global_vars() --- .bin/Scripts/functions/common.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 5327d895..82c65bb1 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -652,9 +652,10 @@ def wait_for_process(name, poll_rate=3): sleep(1) # global_vars functions -def init_global_vars(): +def init_global_vars(silent=False): """Sets global variables based on system info.""" - print_info('Initializing') + if not silent: + print_info('Initializing') if psutil.WINDOWS: os.system('title Wizard Kit') if psutil.LINUX: @@ -672,10 +673,14 @@ def init_global_vars(): ['Clearing collisions...', clean_env_vars], ] try: - for f in init_functions: - try_and_print( - message=f[0], function=f[1], - cs='Done', ns='Error', catch_all=False) + if silent: + for f in init_functions: + f[1]() + else: + for f in init_functions: + try_and_print( + message=f[0], function=f[1], + cs='Done', ns='Error', catch_all=False) except: major_exception() From 04cfdff2bf7a5f2f085caf165235a5a5219f261e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 17:27:15 -0700 Subject: [PATCH 089/186] Don't show init, just disk details --- .bin/Scripts/ddrescue-tui-smart-display | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display index 65b76890..4b8afcde 100755 --- a/.bin/Scripts/ddrescue-tui-smart-display +++ b/.bin/Scripts/ddrescue-tui-smart-display @@ -10,12 +10,11 @@ import time os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.hw_diags import * -init_global_vars() +init_global_vars(silent=True) if __name__ == '__main__': try: # Prep - clear_screen() state = State() state.init() @@ -26,7 +25,6 @@ if __name__ == '__main__': disk = d # Show details - clear_screen() if disk: for line in disk.generate_attribute_report(timestamp=True): print(line) From ad15cdad5612940940ba243b4b58911924fc3018 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 17:28:06 -0700 Subject: [PATCH 090/186] Added warning if not saving map to a preferred FS * Fixes #76 --- .bin/Scripts/functions/ddrescue.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 7f1fc7aa..6d722165 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -351,7 +351,34 @@ class RecoveryState(): self.set_pass_num() def self_checks(self): - """Run self-checks for each BlockPair and update state values.""" + """Run self-checks and update state values.""" + cmd = ['findmnt', '--json', '--target', os.getcwd()] + map_allowed_fstypes = RECOMMENDED_FSTYPES.copy() + map_allowed_fstypes.extend(['cifs', 'ext2', 'vfat']) + map_allowed_fstypes.sort() + json_data = {} + + # Avoid saving map to non-persistent filesystem + try: + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + except Exception: + print_error('ERROR: Failed to verify map path') + raise GenericAbort() + fstype = json_data.get( + 'filesystems', [{}])[0].get( + 'fstype', 'unknown') + if fstype not in map_allowed_fstypes: + print_error( + "Map isn't being saved to a recommended filesystem ({})".format( + fstype.upper())) + print_info('Recommended types are: {}'.format( + ' / '.join(map_allowed_fstypes).upper())) + print_standard(' ') + if not ask('Proceed anyways? (Strongly discouraged)'): + raise GenericAbort() + + # Run BlockPair self checks and get total size self.total_size = 0 for bp in self.block_pairs: bp.self_check() From 62b8e51705367034f03635aaab1d0e1e016e4a39 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 19:45:02 -0700 Subject: [PATCH 091/186] Updated ddrescue-tui tmux pane size handling --- .bin/Scripts/functions/ddrescue.py | 190 +++++++++++++++++++---------- .bin/Scripts/functions/hw_diags.py | 4 +- .bin/Scripts/functions/tmux.py | 2 +- 3 files changed, 130 insertions(+), 66 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 6d722165..6cbc9430 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -8,8 +8,10 @@ import signal import stat import time +from collections import OrderedDict from functions.common import * from functions.data import * +from functions.tmux import * from operator import itemgetter # STATIC VARIABLES @@ -30,6 +32,11 @@ DDRESCUE_SETTINGS = { } RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] SIDE_PANE_WIDTH = 21 +TMUX_LAYOUT = OrderedDict({ + 'Source': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, +}) USAGE = """ {script_name} clone [source [destination]] {script_name} image [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) @@ -276,6 +283,7 @@ class RecoveryState(): self.current_pass_str = '0: Initializing' self.settings = DDRESCUE_SETTINGS.copy() self.finished = False + self.panes = {} self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.rescued = 0 self.resumed = False @@ -425,46 +433,22 @@ class RecoveryState(): # Functions def build_outer_panes(state): """Build top and side panes.""" - clear_screen() - result = run_program(['tput', 'cols']) - width = int( - (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 - - # Top panes - source_str = state.source.name - if len(source_str) > width: - source_str = '{}...'.format(source_str[:width-3]) - dest_str = state.dest.name - if len(dest_str) > width: - if state.mode == 'clone': - dest_str = '{}...'.format(dest_str[:width-3]) - else: - dest_str = '...{}'.format(dest_str[-width+3:]) - source_pane = tmux_splitw( - '-bdvl', '2', - '-PF', '#D', - 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( - text=source_str, - **COLORS)) - tmux_splitw( - '-t', source_pane, - '-dhl', '{}'.format(SIDE_PANE_WIDTH), - 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( - text=time.strftime("%Y-%m-%d %H:%M %Z"), - **COLORS)) - tmux_splitw( - '-t', source_pane, - '-dhp', '50', - 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( - text=dest_str, + state.panes['Source'] = tmux_split_window( + behind=True, vertical=True, lines=2, + text='{BLUE}Source{CLEAR}'.format(**COLORS)) + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Source'], + text='{BLUE}Started{CLEAR}\n{s}'.format( + s=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) + state.panes['Destination'] = tmux_split_window( + percent=50, target_pane=state.panes['Source'], + text='{BLUE}Destination{CLEAR}'.format(**COLORS)) # Side pane update_sidepane(state) - tmux_splitw( - '-dhl', str(SIDE_PANE_WIDTH), - 'watch', '--color', '--no-title', '--interval', '1', - 'cat', state.progress_out) + state.panes['Progress'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, watch=state.progress_out) def create_path_obj(path): @@ -491,6 +475,94 @@ def double_confirm_clone(): return ask('Asking again to confirm, is this correct?') +def fix_tmux_panes(state, forced=False): + """Fix pane sizes if the winodw has been resized.""" + needs_fixed = False + + # Check layout + for k, v in TMUX_LAYOUT.items(): + if not v.get('Check'): + # Not concerned with the size of this pane + continue + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Check pane size + x, y = tmux_get_pane_size(pane_id=target) + if v.get('x', False) and v['x'] != x: + needs_fixed = True + if v.get('y', False) and v['y'] != y: + needs_fixed = True + + # Bail? + if not needs_fixed and not forced: + return + + # Remove Destination pane (temporarily) + tmux_kill_pane(state.panes['Destination']) + + # Update layout + for k, v in TMUX_LAYOUT.items(): + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] + + # Resize pane + tmux_resize_pane(pane_id=target, **v) + + # Calc Source/Destination pane sizes + width, height = tmux_get_pane_size() + width = int(width / 2) - 1 + + # Update Source string + source_str = state.source.name + if len(source_str) > width: + source_str = '{}...'.format(source_str[:width-3]) + + # Update Destination string + dest_str = state.dest.name + if len(dest_str) > width: + if state.mode == 'clone': + dest_str = '{}...'.format(dest_str[:width-3]) + else: + dest_str = '...{}'.format(dest_str[-width+3:]) + + # Rebuild Source/Destination panes + tmux_update_pane( + pane_id=state.panes['Source'], + text='{BLUE}Source{CLEAR}\n{s}'.format( + s=source_str, **COLORS)) + state.panes['Destination'] = tmux_split_window( + percent=50, target_pane=state.panes['Source'], + text='{BLUE}Destination{CLEAR}\n{s}'.format( + s=dest_str, **COLORS)) + + if 'SMART' in state.panes: + # Calc SMART/ddrescue/Journal panes sizes + ratio = [12, 22, 4] + width, height = tmux_get_pane_size(pane_id=state.panes['Progress']) + height -= 2 + total = sum(ratio) + p_ratio = [int((x/total) * height) for x in ratio] + p_ratio[1] = height - p_ratio[0] - p_ratio[2] + + # Resize SMART/Journal panes + tmux_resize_pane(state.panes['SMART'], y=ratio[0]) + tmux_resize_pane(y=ratio[1]) + tmux_resize_pane(state.panes['Journal'], y=ratio[2]) + + def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -687,7 +759,9 @@ def menu_ddrescue(source_path, dest_path, run_mode): raise GenericAbort() # Main menu + clear_screen() build_outer_panes(state) + fix_tmux_panes(state, forced=True) menu_main(state) # Done @@ -877,29 +951,24 @@ def run_ddrescue(state, pass_settings): pause('Press Enter to return to main menu...') return - # Set heights - # NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) - result = run_program(['tput', 'lines']) - height = int(result.stdout.decode().strip()) - height_smart = int(height * (8 / 33)) - height_journal = int(height * (4 / 33)) - height_ddrescue = height - height_smart - height_journal - # Show SMART status smart_dev = state.source_path if state.source.parent: smart_dev = state.source.parent - smart_pane = tmux_splitw( - '-bdvl', str(height_smart), - '-PF', '#D', - 'watch', '--color', '--no-title', '--interval', '300', - 'ddrescue-tui-smart-display', smart_dev) + smart_cmd = [ + 'watch', '--color', '--no-title', '--interval', '5', + 'ddrescue-tui-smart-display', smart_dev, + ] + state.panes['SMART'] = tmux_split_window( + behind=True, lines=12, vertical=True, command=smart_cmd) # Show systemd journal output - journal_pane = tmux_splitw( - '-dvl', str(height_journal), - '-PF', '#D', - 'journalctl', '-f') + state.panes['Journal'] = tmux_split_window( + lines=4, vertical=True, + command=['sudo', 'journalctl', '-f']) + + # Fix layout + fix_tmux_panes(state, forced=True) # Run pass for each block-pair for bp in state.block_pairs: @@ -931,8 +1000,9 @@ def run_ddrescue(state, pass_settings): while True: bp.update_progress(state.current_pass) update_sidepane(state) + fix_tmux_panes(state) try: - ddrescue_proc.wait(timeout=10) + ddrescue_proc.wait(timeout=1) sleep(2) bp.update_progress(state.current_pass) update_sidepane(state) @@ -967,8 +1037,9 @@ def run_ddrescue(state, pass_settings): if str(return_code) != '0': # Pause on errors pause('Press Enter to return to main menu... ') - run_program(['tmux', 'kill-pane', '-t', smart_pane]) - run_program(['tmux', 'kill-pane', '-t', journal_pane]) + + # Cleanup + tmux_kill_pane(state.panes['SMART'], state.panes['Journal']) def select_parts(source_device): @@ -1219,13 +1290,6 @@ def show_usage(script_name): pause() -def tmux_splitw(*args): - """Run tmux split-window command and return output as str.""" - cmd = ['tmux', 'split-window', *args] - result = run_program(cmd) - return result.stdout.decode().strip() - - def update_sidepane(state): """Update progress file for side pane.""" output = [] diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index b09dbf05..9cef1d33 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -619,7 +619,7 @@ def build_status_string(label, status, info_label=False): **COLORS) def fix_tmux_panes(state, tmux_layout): - """Fix pane sizes in case the window has been resized.""" + """Fix pane sizes if the window has been resized.""" needs_fixed = False # Check layout @@ -636,7 +636,7 @@ def fix_tmux_panes(state, tmux_layout): else: target = state.panes[k] - # Get pane size + # Check pane size x, y = tmux_get_pane_size(pane_id=target) if v.get('x', False) and v['x'] != x: needs_fixed = True diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index ce35f5b0..84046d0b 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -69,7 +69,7 @@ def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, follow=False, target_pane=None, - working_dir=None, command=None, + command=None, working_dir=None, text=None, watch=None, watch_cmd='cat'): """Run tmux split-window command and return pane_id as str.""" # Bail early From 44fe8882301095ae789de3b393a19a0ee92daca0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 20:15:35 -0700 Subject: [PATCH 092/186] Replaced ddrescue-tui-smart-display * Output data to file and have tmux pane watching said file * This method handles resizing much better --- .bin/Scripts/ddrescue-tui-smart-display | 41 ----------------------- .bin/Scripts/functions/ddrescue.py | 44 +++++++++++++++++++------ 2 files changed, 34 insertions(+), 51 deletions(-) delete mode 100755 .bin/Scripts/ddrescue-tui-smart-display diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display deleted file mode 100755 index 4b8afcde..00000000 --- a/.bin/Scripts/ddrescue-tui-smart-display +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/python3 -# -## Wizard Kit: SMART attributes display for ddrescue TUI - -import os -import sys -import time - -# Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) -from functions.hw_diags import * -init_global_vars(silent=True) - -if __name__ == '__main__': - try: - # Prep - state = State() - state.init() - - # Find disk - disk = None - for d in state.disks: - if d.path == sys.argv[1]: - disk = d - - # Show details - if disk: - for line in disk.generate_attribute_report(timestamp=True): - print(line) - else: - print_error('Disk "{}" not found'.format(sys.argv[1])) - - # Done - exit_script() - except SystemExit: - pass - except: - major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 6cbc9430..00fb35f7 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -11,6 +11,7 @@ import time from collections import OrderedDict from functions.common import * from functions.data import * +from functions.hw_diags import * from functions.tmux import * from operator import itemgetter @@ -291,6 +292,7 @@ class RecoveryState(): self.total_size = 0 if mode not in ('clone', 'image'): raise GenericError('Unsupported mode') + self.get_smart_source() def add_block_pair(self, source, dest): """Run safety checks and append new BlockPair to internal list.""" @@ -349,6 +351,14 @@ class RecoveryState(): min_percent = min(min_percent, bp.rescued_percent) return min_percent + def get_smart_source(self): + """Get source for SMART dispay.""" + disk_path = self.source.path + if self.source.parent: + disk_path = self.source.parent + + self.smart_source = DiskObj(disk_path) + def retry_all_passes(self): """Mark all passes as pending for all block-pairs.""" self.finished = False @@ -951,16 +961,13 @@ def run_ddrescue(state, pass_settings): pause('Press Enter to return to main menu...') return - # Show SMART status - smart_dev = state.source_path - if state.source.parent: - smart_dev = state.source.parent - smart_cmd = [ - 'watch', '--color', '--no-title', '--interval', '5', - 'ddrescue-tui-smart-display', smart_dev, - ] + # Create SMART monitor pane + state.smart_out = '{}/smart_{}.out'.format( + global_vars['TmpDir'], state.smart_source.name) + with open(state.smart_out, 'w') as f: + f.write('Initializing...') state.panes['SMART'] = tmux_split_window( - behind=True, lines=12, vertical=True, command=smart_cmd) + behind=True, lines=12, vertical=True, watch=state.smart_out) # Show systemd journal output state.panes['Journal'] = tmux_split_window( @@ -997,10 +1004,26 @@ def run_ddrescue(state, pass_settings): clear_screen() print_info('Current dev: {}'.format(bp.source_path)) ddrescue_proc = popen_program(cmd) + i = 0 while True: + # Update SMART display (every 30 seconds) + i += 1 + if i % 30 == 0: + state.smart_source.get_smart_details() + with open(state.smart_out, 'w') as f: + report = state.smart_source.generate_attribute_report( + timestamp=True) + for line in report: + f.write('{}\n'.format(line)) + + # Update progress bp.update_progress(state.current_pass) update_sidepane(state) + + # Fix panes fix_tmux_panes(state) + + # Check if ddrescue has finished try: ddrescue_proc.wait(timeout=1) sleep(2) @@ -1008,8 +1031,9 @@ def run_ddrescue(state, pass_settings): update_sidepane(state) break except subprocess.TimeoutExpired: - # Catch to update bp/sidepane + # Catch to update smart/bp/sidepane pass + except KeyboardInterrupt: # Catch user abort pass From 42407f0eca0f75be518ecc34b99d8f09baad40c7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 18 Dec 2018 20:35:21 -0700 Subject: [PATCH 093/186] Adjusted ddrescue exit handling * Wait for ddrescue_proc after KeyboardInterrupt * ddrescue prints extra info to the screen after a CTRL+c * Explicitly mark KeyboardInterrupt events as an abort * Add 'DDRESCUE PROCESS HALTED' message in red if exiting non-zero * More clearly indicates that user interaction is required * Fixes issue #72 --- .bin/Scripts/functions/ddrescue.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 00fb35f7..6e515655 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -953,7 +953,8 @@ def read_map_file(map_path): def run_ddrescue(state, pass_settings): """Run ddrescue pass.""" - return_code = None + return_code = -1 + aborted = False if state.finished: clear_screen() @@ -1007,7 +1008,6 @@ def run_ddrescue(state, pass_settings): i = 0 while True: # Update SMART display (every 30 seconds) - i += 1 if i % 30 == 0: state.smart_source.get_smart_details() with open(state.smart_out, 'w') as f: @@ -1015,6 +1015,7 @@ def run_ddrescue(state, pass_settings): timestamp=True) for line in report: f.write('{}\n'.format(line)) + i += 1 # Update progress bp.update_progress(state.current_pass) @@ -1036,7 +1037,8 @@ def run_ddrescue(state, pass_settings): except KeyboardInterrupt: # Catch user abort - pass + aborted = True + ddrescue_proc.wait(timeout=10) # Update progress/sidepane again bp.update_progress(state.current_pass) @@ -1044,12 +1046,19 @@ def run_ddrescue(state, pass_settings): # Was ddrescue aborted? return_code = ddrescue_proc.poll() - if return_code is None or return_code is 130: - clear_screen() + if aborted: + print_standard(' ') + print_standard(' ') + print_error('DDRESCUE PROCESS HALTED') + print_standard(' ') print_warning('Aborted') break elif return_code: - # i.e. not None and not 0 + # i.e. True when non-zero + print_standard(' ') + print_standard(' ') + print_error('DDRESCUE PROCESS HALTED') + print_standard(' ') print_error('Error(s) encountered, see message above.') break else: From f022d0ca76e369aa2a7f9792c58c2e98a5eae05a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 19 Dec 2018 18:45:58 -0700 Subject: [PATCH 094/186] Fallback to HW-Diags CLI if X fails to start * Fixes issue #74 --- .linux_items/include/airootfs/etc/skel/.update_x | 3 +++ .linux_items/include/airootfs/etc/skel/.zlogin | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/.linux_items/include/airootfs/etc/skel/.update_x b/.linux_items/include/airootfs/etc/skel/.update_x index 4763a175..650d3162 100755 --- a/.linux_items/include/airootfs/etc/skel/.update_x +++ b/.linux_items/include/airootfs/etc/skel/.update_x @@ -92,3 +92,6 @@ else cbatticon --hide-notification & fi +# Prevent Xorg from being killed by .zlogin +touch "/tmp/x_ok" + diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/.linux_items/include/airootfs/etc/skel/.zlogin index bd736280..56b47242 100644 --- a/.linux_items/include/airootfs/etc/skel/.zlogin +++ b/.linux_items/include/airootfs/etc/skel/.zlogin @@ -14,7 +14,18 @@ if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then # Start X or HW-diags if ! fgrep -q "nox" /proc/cmdline; then + # Kill Xorg after 30 seconds if it doesn't fully initialize + (sleep 30s; if ! [[ -f "/tmp/x_ok" ]]; then pkill '(Xorg|startx)'; fi) & + + # Try starting X startx >/dev/null + + # Run Hw-Diags CLI if necessary + if ! [[ -f "/tmp/x_ok" ]]; then + echo "There was an issue starting Xorg, starting CLI interface..." + sleep 2s + hw-diags cli + fi else hw-diags cli fi From c6eb7cdfd6e52fd1bcd533c49c74aeb8baecc3a0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 19 Dec 2018 18:53:13 -0700 Subject: [PATCH 095/186] Use new arguments when calling hw-diags --- .linux_items/include/airootfs/etc/skel/.config/i3/config | 2 +- .linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml | 2 +- .linux_items/include/airootfs/etc/skel/.zlogin | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.linux_items/include/airootfs/etc/skel/.config/i3/config b/.linux_items/include/airootfs/etc/skel/.config/i3/config index ca1a5439..102b50f5 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/i3/config +++ b/.linux_items/include/airootfs/etc/skel/.config/i3/config @@ -72,7 +72,7 @@ bindsym $mod+d exec "urxvt -title 'Hardware Diagnostics' -e hw-diags" bindsym $mod+f exec "thunar ~" bindsym $mod+i exec "hardinfo" bindsym $mod+m exec "urxvt -title 'Mount All Volumes' -e mount-all-volumes gui" -bindsym $mod+s exec "urxvt -title 'Hardware Diagnostics' -e hw-diags quick" +bindsym $mod+s exec "urxvt -title 'Hardware Diagnostics' -e hw-diags --quick" bindsym $mod+t exec "urxvt -e zsh -c 'tmux new-session -A -t general; zsh'" bindsym $mod+v exec "urxvt -title 'Hardware Sensors' -e watch -c -n1 -t hw-sensors" bindsym $mod+w exec "firefox" diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml index 90c7b0e0..e563ec04 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml +++ b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml @@ -324,7 +324,7 @@ - urxvt -title "Hardware Diagnostics" -e hw-diags quick + urxvt -title "Hardware Diagnostics" -e hw-diags --quick diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/.linux_items/include/airootfs/etc/skel/.zlogin index 56b47242..04b1316e 100644 --- a/.linux_items/include/airootfs/etc/skel/.zlogin +++ b/.linux_items/include/airootfs/etc/skel/.zlogin @@ -24,9 +24,9 @@ if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then if ! [[ -f "/tmp/x_ok" ]]; then echo "There was an issue starting Xorg, starting CLI interface..." sleep 2s - hw-diags cli + hw-diags --cli fi else - hw-diags cli + hw-diags --cli fi fi From eed8a1e40c48b205e40c422a32cc608a1934b080 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 20 Dec 2018 15:25:39 -0700 Subject: [PATCH 096/186] Fix poweroff/reboot calls --- .bin/Scripts/functions/hw_diags.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9cef1d33..9f10f997 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -835,13 +835,9 @@ def menu_diags(state, args): # Tubes is close to pipes right? secret_screensaver('pipes') elif selection == 'R': - print('(FAKE) reboot...') - sleep(1) - run_program(['systemctl', 'reboot']) + run_program(['/usr/local/bin/wk-power-command', 'reboot']) elif selection == 'P': - print('(FAKE) poweroff...') - sleep(1) - run_program(['systemctl', 'poweroff']) + run_program(['/usr/local/bin/wk-power-command', 'poweroff']) elif selection == 'Q': break elif selection == 'S': From d60aab958408d9eba1ddd9c38dde8bdcd7c17a77 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 20 Dec 2018 16:03:54 -0700 Subject: [PATCH 097/186] Updated MemTest86 to 8.0 * Passmark is no longer providing ISOs so the UFD image is used instead * This is an alternative solution to issue #71 --- Build Linux | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Build Linux b/Build Linux index 56233d51..41e5dea5 100755 --- a/Build Linux +++ b/Build Linux @@ -170,13 +170,15 @@ function update_live_env() { # Memtest86 mkdir -p "$LIVE_DIR/EFI/memtest86/Benchmark" mkdir -p "$TEMP_DIR/memtest86" - curl -Lo "$TEMP_DIR/memtest86/memtest86.iso.tar.gz" "https://www.memtest86.com/downloads/memtest86-iso.tar.gz" - tar xvf "$TEMP_DIR/memtest86/memtest86.iso.tar.gz" -C "$TEMP_DIR/memtest86" - 7z x "$TEMP_DIR/memtest86"/*.iso -o"$TEMP_DIR/memtest86" - mv "$TEMP_DIR/memtest86/EFI/BOOT/BLACKLIS.CFG" "$LIVE_DIR/EFI/memtest86/blacklist.cfg" + curl -Lo "$TEMP_DIR/memtest86/memtest86-usb.zip" "https://www.memtest86.com/downloads/memtest86-usb.zip" + 7z e "$TEMP_DIR/memtest86/memtest86-usb.zip" -o"$TEMP_DIR/memtest86" "memtest86-usb.img" + 7z e "$TEMP_DIR/memtest86/memtest86-usb.img" -o"$TEMP_DIR/memtest86" "MemTest86.img" + 7z x "$TEMP_DIR/memtest86/MemTest86.img" -o"$TEMP_DIR/memtest86" + rm "$TEMP_DIR/memtest86/EFI/BOOT/BOOTIA32.EFI" mv "$TEMP_DIR/memtest86/EFI/BOOT/BOOTX64.EFI" "$LIVE_DIR/EFI/memtest86/memtestx64.efi" - mv "$TEMP_DIR/memtest86/EFI/BOOT/MT86.PNG" "$LIVE_DIR/EFI/memtest86/mt86.png" - mv "$TEMP_DIR/memtest86/EFI/BOOT/UNIFONT.BIN" "$LIVE_DIR/EFI/memtest86/unifont.bin" + for f in "$TEMP_DIR/memtest86/EFI/BOOT"/* "help"/* license.rtf; do + mv "$f" "$LIVE_DIR/EFI/memtest86"/ + done # build.sh if ! grep -iq 'wizardkit additions' "$LIVE_DIR/build.sh"; then From d930bdddbda5a72eccd2bbff88a8b5442b5f6c7e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 20 Dec 2018 16:10:08 -0700 Subject: [PATCH 098/186] Zero beginning of UFD before formatting * Fixes issue #68 --- .bin/Scripts/build-ufd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index c606892b..1253472c 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -533,6 +533,9 @@ echo -e "${GREEN}Building Kit${CLEAR}" touch "${LOG_FILE}" tmux split-window -dl 10 tail -f "${LOG_FILE}" +# Zero beginning of device +dd bs=4M count=16 if=/dev/zero of="${DEST_DEV}" >> "${LOG_FILE}" 2>&1 + # Format echo "Formatting drive..." if [[ "${USE_MBR}" == "True" ]]; then From d37923a31cfc7471ac37e2e98efb7b900db7383c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 20 Dec 2018 17:26:27 -0700 Subject: [PATCH 099/186] Bugfix: typo --- .bin/Scripts/settings/launchers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 43e8e158..5135ec08 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -212,6 +212,7 @@ LAUNCHERS = { r')', ], }, + }, r'Diagnostics\Extras': { 'AIDA64': { 'L_TYPE': 'Executable', @@ -556,8 +557,7 @@ LAUNCHERS = { 'L_ITEM': 'IObitUninstallerPortable.exe', }, }, - }, -} + } if __name__ == '__main__': print("This file is not meant to be called directly.") From 6d9f50629cff3e4ad60ce59abbc4edfa424229e6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 22 Dec 2018 17:55:49 -0700 Subject: [PATCH 100/186] Reworked status/color sections --- .bin/Scripts/functions/hw_diags.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 9f10f997..3642f49c 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -65,6 +65,11 @@ KEY_NVME = 'nvme_smart_health_information_log' KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 +STATUSES = { + 'RED': ['Denied', 'ERROR', 'NS', 'OVERRIDE', 'TimedOut'], + 'YELLOW': ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working'], + 'GREEN': ['CS'], +} TESTS_CPU = ['Prime95'] TESTS_DISK = [ 'I/O Benchmark', @@ -78,7 +83,10 @@ TMUX_LAYOUT = OrderedDict({ 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, }) -# Error Classe +# Regex +REGEX_ERROR_STATUS = re.compile('|'.join(STATUSES['RED'])) + +# Error Classes class DeviceTooSmallError(Exception): pass @@ -566,7 +574,8 @@ class TestObj(): def update_status(self, new_status=None): """Update status strings.""" - if self.disabled or re.search(r'ERROR|OVERRIDE', self.status): + if self.disabled or REGEX_ERROR_STATUS.search(self.status): + # Don't update error statuses if test is enabled return if new_status: self.status = build_status_string( @@ -603,12 +612,9 @@ def build_outer_panes(state): def build_status_string(label, status, info_label=False): """Build status string with appropriate colors.""" status_color = COLORS['CLEAR'] - if status in ['Denied', 'ERROR', 'NS', 'OVERRIDE', 'TimedOut']: - status_color = COLORS['RED'] - elif status in ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working']: - status_color = COLORS['YELLOW'] - elif status in ['CS']: - status_color = COLORS['GREEN'] + for k, v in STATUSES.items(): + if status in v: + status_color = COLORS[k] return '{l_c}{l}{CLEAR}{s_c}{s:>{s_w}}{CLEAR}'.format( l_c=COLORS['BLUE'] if info_label else '', From c15eb85a5e46c881c4d4ca9b0e821cf1cb3aafc2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 22 Dec 2018 18:07:06 -0700 Subject: [PATCH 101/186] Removed unused get_status_color and Skipped status --- .bin/Scripts/functions/hw_diags.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 3642f49c..c5002882 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -67,7 +67,7 @@ QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 STATUSES = { 'RED': ['Denied', 'ERROR', 'NS', 'OVERRIDE', 'TimedOut'], - 'YELLOW': ['Aborted', 'N/A', 'Skipped', 'Unknown', 'Working'], + 'YELLOW': ['Aborted', 'N/A', 'Unknown', 'Working'], 'GREEN': ['CS'], } TESTS_CPU = ['Prime95'] @@ -732,17 +732,6 @@ def get_read_rate(s): real_rate = convert_to_bytes(human_rate) return real_rate -def get_status_color(s): - """Get color based on status, returns str.""" - color = COLORS['CLEAR'] - if s in ['Denied', 'ERROR', 'NS', 'OVERRIDE']: - color = COLORS['RED'] - elif s in ['Aborted', 'N/A', 'Unknown', 'Working', 'Skipped']: - color = COLORS['YELLOW'] - elif s in ['CS']: - color = COLORS['GREEN'] - return color - def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] From 6e37736146e6c97008e324f2b4d4df647984242e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 23 Dec 2018 17:15:50 -0700 Subject: [PATCH 102/186] Only save attributes to log during show_results() --- .bin/Scripts/functions/hw_diags.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index c5002882..5e9e712b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -242,7 +242,9 @@ class DiskObj(): print_standard(' (Have you tried swapping the disk cable?)') else: # Override? - show_report(self.generate_attribute_report(description=True)) + show_report( + self.generate_attribute_report(description=True), + log_report=True) print_warning(' {} error(s) detected.'.format(attr_type)) if override_disabled: print_standard('Tests disabled for this device') @@ -1448,8 +1450,6 @@ def run_nvme_smart_tests(state, test): # Show attributes clear_screen() - print_info('Device ({})'.format(test.dev.name)) - print_standard(' {}'.format(test.dev.description)) show_report(test.dev.generate_attribute_report()) print_standard(' ') @@ -1537,11 +1537,12 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) -def show_report(report): - """Show report on screen and save to log w/out color.""" +def show_report(report, log_report=False): + """Show report on screen and optionally save to log w/out color.""" for line in report: print(line) - print_log(strip_colors(line)) + if log_report: + print_log(strip_colors(line)) def show_results(state): """Show results for all tests.""" @@ -1556,7 +1557,7 @@ def show_results(state): _enabled |= state.tests[k]['Enabled'] if _enabled: print_success('CPU:'.format(k)) - show_report(state.cpu.generate_cpu_report()) + show_report(state.cpu.generate_cpu_report(), log_report=True) print_standard(' ') # Disk tests @@ -1567,7 +1568,7 @@ def show_results(state): print_success('Disk{}:'.format( '' if len(state.disks) == 1 else 's')) for disk in state.disks: - show_report(disk.generate_disk_report()) + show_report(disk.generate_disk_report(), log_report=True) print_standard(' ') def update_main_options(state, selection, main_options): From 96d34ceb5038549ffa09451f46a95b3a4598c98e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 23 Dec 2018 17:33:16 -0700 Subject: [PATCH 103/186] Fix SMART short-test results section * OVERRIDE status reduced to yellow/warning * Allows it to be elevated to NS or TimedOut * Only disable other disk tests on test.failed * OVERRIDE doesn't work if based on test.passed for this test --- .bin/Scripts/functions/hw_diags.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 5e9e712b..cbfd4377 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -66,8 +66,8 @@ KEY_SMART = 'ata_smart_attributes' QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) SIDE_PANE_WIDTH = 20 STATUSES = { - 'RED': ['Denied', 'ERROR', 'NS', 'OVERRIDE', 'TimedOut'], - 'YELLOW': ['Aborted', 'N/A', 'Unknown', 'Working'], + 'RED': ['Denied', 'ERROR', 'NS', 'TimedOut'], + 'YELLOW': ['Aborted', 'N/A', 'OVERRIDE', 'Unknown', 'Working'], 'GREEN': ['CS'], } TESTS_CPU = ['Prime95'] @@ -1513,7 +1513,7 @@ def run_nvme_smart_tests(state, test): test.update_status('TimedOut') # Disable other drive tests if necessary - if not test.passed: + if test.failed: for t in ['badblocks', 'I/O Benchmark']: test.dev.disable_test(t, 'Denied') From 98c0c34bf8edf8d47381350621fe379a442abbff Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 26 Dec 2018 17:11:37 -0700 Subject: [PATCH 104/186] Removed unused vertical_graph data --- .bin/Scripts/functions/hw_diags.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index cbfd4377..c183a69d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1057,7 +1057,6 @@ def run_io_benchmark(state, test): try: test.merged_rates = [] test.read_rates = [] - test.vertical_graph = [] test.dev.calc_io_dd_values() # Run dd read tests @@ -1084,10 +1083,6 @@ def run_io_benchmark(state, test): # Add rate to lists test.read_rates.append(cur_rate) - test.vertical_graph.append( - '{percent:0.1f} {rate}'.format( - percent=(i/test.dev.dd_chunks)*100, - rate=int(cur_rate/(1024**2)))) # Show progress if i % IO_VARS['Progress Refresh Rate'] == 0: @@ -1172,10 +1167,6 @@ def run_io_benchmark(state, test): elif not 'N/A' in test.status: test.update_status('Unknown') - # Save log - with open(test.io_benchmark_out.replace('.', '-raw.'), 'a') as f: - f.write('\n'.join(test.vertical_graph)) - # Done update_progress_pane(state) From dcc2e5cd6a03784332a29bff51b731e64917f2ee Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 26 Dec 2018 21:00:29 -0700 Subject: [PATCH 105/186] Adjusted top pane text --- .bin/Scripts/functions/hw_diags.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index c183a69d..a4184689 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -861,7 +861,7 @@ def run_badblocks_test(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nbadblocks: {}'.format( + text='{}\n{}'.format( TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ @@ -1031,7 +1031,7 @@ def run_io_benchmark(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nI/O Benchmark: {}'.format( + text='{}\n{}'.format( TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ @@ -1194,7 +1194,7 @@ def run_mprime_test(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nPrime95: {}'.format(TOP_PANE_TEXT, test.dev.name)) + text='{}\n{}'.format(TOP_PANE_TEXT, test.dev.name)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ 'Temps': {'y': 1000, 'Check': False}, @@ -1391,7 +1391,7 @@ def run_nvme_smart_tests(state, test): # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\nDisk Health: {}'.format( + text='{}\n{}'.format( TOP_PANE_TEXT, test.dev.description)) test.tmux_layout = TMUX_LAYOUT.copy() test.tmux_layout.update({ From a39c62eabcea3707b34c22c988d021a739ef3b1f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 17:47:01 -0700 Subject: [PATCH 106/186] Fix crash if no sensors available --- .bin/Scripts/functions/sensors.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 99e7999a..eeb98922 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -99,9 +99,16 @@ def get_colored_temp_str(temp): def get_raw_sensor_data(): """Read sensor data and return dict.""" + data = {} cmd = ['sensors', '-j'] - result = run_program(cmd) - return json.loads(result.stdout.decode()) + try: + result = run_program(cmd) + data = json.loads(result.stdout.decode()) + except subprocess.CalledProcessError: + # Assuming no sensors available, return empty dict below + pass + + return data def get_sensor_data(): """Parse raw sensor data and return new dict.""" From 4b956d5eea0f3cc1641a3b5b449c00a05ad13fab Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 17:51:51 -0700 Subject: [PATCH 107/186] Fix MemTest86 extraction --- Build Linux | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Build Linux b/Build Linux index 41e5dea5..87e93e2d 100755 --- a/Build Linux +++ b/Build Linux @@ -174,11 +174,11 @@ function update_live_env() { 7z e "$TEMP_DIR/memtest86/memtest86-usb.zip" -o"$TEMP_DIR/memtest86" "memtest86-usb.img" 7z e "$TEMP_DIR/memtest86/memtest86-usb.img" -o"$TEMP_DIR/memtest86" "MemTest86.img" 7z x "$TEMP_DIR/memtest86/MemTest86.img" -o"$TEMP_DIR/memtest86" - rm "$TEMP_DIR/memtest86/EFI/BOOT/BOOTIA32.EFI" - mv "$TEMP_DIR/memtest86/EFI/BOOT/BOOTX64.EFI" "$LIVE_DIR/EFI/memtest86/memtestx64.efi" - for f in "$TEMP_DIR/memtest86/EFI/BOOT"/* "help"/* license.rtf; do - mv "$f" "$LIVE_DIR/EFI/memtest86"/ - done + rm "$TEMP_DIR/memtest86/EFI/BOOT/BOOTIA32.efi" + mv "$TEMP_DIR/memtest86/EFI/BOOT/BOOTX64.efi" "$LIVE_DIR/EFI/memtest86/memtestx64.efi" + mv "$TEMP_DIR/memtest86/EFI/BOOT"/* "$LIVE_DIR/EFI/memtest86"/ + mv "$TEMP_DIR/memtest86/help"/* "$LIVE_DIR/EFI/memtest86"/ + mv "$TEMP_DIR/memtest86/license.rtf" "$LIVE_DIR/EFI/memtest86"/ # build.sh if ! grep -iq 'wizardkit additions' "$LIVE_DIR/build.sh"; then From a124236defc24901b93f98be1ace8a8395aba952 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 18:01:42 -0700 Subject: [PATCH 108/186] Removed unused pydf config --- .linux_items/include/airootfs/etc/pydfrc | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 .linux_items/include/airootfs/etc/pydfrc diff --git a/.linux_items/include/airootfs/etc/pydfrc b/.linux_items/include/airootfs/etc/pydfrc deleted file mode 100644 index 100b1f05..00000000 --- a/.linux_items/include/airootfs/etc/pydfrc +++ /dev/null @@ -1,24 +0,0 @@ -normal_colour = 'default' -header_colour = 'blue' -local_fs_colour = 'default' -remote_fs_colour = 'green' -special_fs_colour = 'yellow' -readonly_fs_colour = 'cyan' -filled_fs_colour = 'red' -full_fs_colour = 'on_red', 'green', 'blink' -sizeformat = "-h" -column_separator = ' ' -column_separator_colour = 'none' -stretch_screen = 0.3 -FILL_THRESH = 75.0 -FULL_THRESH = 85.0 -format = [ - ('fs', 10, "l"), ('size', 5, "r"), - ('used', 5, "r"), ('avail', 5, "r"), ('perc', 5, "r"), - ('bar', 0.1, "l"), ('on', 11, "l") - ] -barchar = '#' -bar_fillchar = '.' -hidebinds = True -mountfile = ['/etc/mtab', '/etc/mnttab', '/proc/mounts'] - From 28e2ce90df96120e39a1733369d6f10e0f7279a2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 18:03:16 -0700 Subject: [PATCH 109/186] Removed nvme-cli in favor of smartctl --- .linux_items/packages/aur | 1 - .linux_items/packages/live_add | 1 - 2 files changed, 2 deletions(-) diff --git a/.linux_items/packages/aur b/.linux_items/packages/aur index 326ca732..8d752673 100644 --- a/.linux_items/packages/aur +++ b/.linux_items/packages/aur @@ -3,7 +3,6 @@ bash-pipes hfsprogs i3lock-fancy-git mprime -nvme-cli openbox-patched smartmontools-svn testdisk-wip diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index 4a7245b4..a6da5f71 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -53,7 +53,6 @@ network-manager-applet networkmanager noto-fonts noto-fonts-cjk -nvme-cli oblogout openbox-patched otf-font-awesome-4 From 765673db6605edb92c4f6d29d6c5a5bb65a1fd26 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 18:06:59 -0700 Subject: [PATCH 110/186] Updated windows_builds.py --- .bin/Scripts/settings/windows_builds.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/settings/windows_builds.py b/.bin/Scripts/settings/windows_builds.py index 2db18b6f..2817979a 100644 --- a/.bin/Scripts/settings/windows_builds.py +++ b/.bin/Scripts/settings/windows_builds.py @@ -170,7 +170,7 @@ WINDOWS_BUILDS = { '17755': ( '10', None, 'Redstone 5', None, 'preview build'), '17758': ( '10', None, 'Redstone 5', None, 'preview build'), '17760': ( '10', None, 'Redstone 5', None, 'preview build'), - '17763': ( '10', 'v1809', 'Redstone 5', 'October 2018 Update', 'preview build'), + '17763': ( '10', 'v1809', 'Redstone 5', 'October 2018 Update', None), '18204': ( '10', None, '19H1', None, 'preview build'), '18214': ( '10', None, '19H1', None, 'preview build'), '18219': ( '10', None, '19H1', None, 'preview build'), @@ -179,6 +179,14 @@ WINDOWS_BUILDS = { '18242': ( '10', None, '19H1', None, 'preview build'), '18247': ( '10', None, '19H1', None, 'preview build'), '18252': ( '10', None, '19H1', None, 'preview build'), + '18262': ( '10', None, '19H1', None, 'preview build'), + '18267': ( '10', None, '19H1', None, 'preview build'), + '18272': ( '10', None, '19H1', None, 'preview build'), + '18277': ( '10', None, '19H1', None, 'preview build'), + '18282': ( '10', None, '19H1', None, 'preview build'), + '18290': ( '10', None, '19H1', None, 'preview build'), + '18298': ( '10', None, '19H1', None, 'preview build'), + '18305': ( '10', None, '19H1', None, 'preview build'), } if __name__ == '__main__': From 36e419bca0bea606d46a870747e5fa118d49e962 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:42:28 -0700 Subject: [PATCH 111/186] Updated activation.py --- .bin/Scripts/functions/activation.py | 90 ++++++++++++++-------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/.bin/Scripts/functions/activation.py b/.bin/Scripts/functions/activation.py index 24436418..e10d088c 100644 --- a/.bin/Scripts/functions/activation.py +++ b/.bin/Scripts/functions/activation.py @@ -10,57 +10,59 @@ from os import environ SLMGR = r'{}\System32\slmgr.vbs'.format(environ.get('SYSTEMROOT')) def activate_with_bios(): - """Attempt to activate Windows with a key stored in the BIOS.""" - # Code borrowed from https://github.com/aeruder/get_win8key - ##################################################### - #script to query windows 8.x OEM key from PC firmware - #ACPI -> table MSDM -> raw content -> byte offset 56 to end - #ck, 03-Jan-2014 (christian@korneck.de) - ##################################################### - bios_key = None - table = b"MSDM" - if acpi.FindAcpiTable(table) is True: - rawtable = acpi.GetAcpiTable(table) - #http://msdn.microsoft.com/library/windows/hardware/hh673514 - #byte offset 36 from beginning \ - # = Microsoft 'software licensing data structure' \ - # / 36 + 20 bytes offset from beginning = Win Key - bios_key = rawtable[56:len(rawtable)].decode("utf-8") - if bios_key is None: - raise BIOSKeyNotFoundError + """Attempt to activate Windows with a key stored in the BIOS.""" + # Code borrowed from https://github.com/aeruder/get_win8key + ##################################################### + #script to query windows 8.x OEM key from PC firmware + #ACPI -> table MSDM -> raw content -> byte offset 56 to end + #ck, 03-Jan-2014 (christian@korneck.de) + ##################################################### + bios_key = None + table = b"MSDM" + if acpi.FindAcpiTable(table) is True: + rawtable = acpi.GetAcpiTable(table) + #http://msdn.microsoft.com/library/windows/hardware/hh673514 + #byte offset 36 from beginning \ + # = Microsoft 'software licensing data structure' \ + # / 36 + 20 bytes offset from beginning = Win Key + bios_key = rawtable[56:len(rawtable)].decode("utf-8") + if bios_key is None: + raise BIOSKeyNotFoundError - # Install Key - cmd = ['cscript', '//nologo', SLMGR, '/ipk', bios_key] - subprocess.run(cmd, check=False) - sleep(5) + # Install Key + cmd = ['cscript', '//nologo', SLMGR, '/ipk', bios_key] + subprocess.run(cmd, check=False) + sleep(5) - # Attempt activation - cmd = ['cscript', '//nologo', SLMGR, '/ato'] - subprocess.run(cmd, check=False) - sleep(5) + # Attempt activation + cmd = ['cscript', '//nologo', SLMGR, '/ato'] + subprocess.run(cmd, check=False) + sleep(5) - # Check status - if not windows_is_activated(): - raise Exception('Activation Failed') + # Check status + if not windows_is_activated(): + raise Exception('Activation Failed') def get_activation_string(): - """Get activation status, returns str.""" - act_str = subprocess.run( - ['cscript', '//nologo', SLMGR, '/xpr'], check=False, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - act_str = act_str.stdout.decode() - act_str = act_str.splitlines() - act_str = act_str[1].strip() - return act_str + """Get activation status, returns str.""" + act_str = subprocess.run( + ['cscript', '//nologo', SLMGR, '/xpr'], check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + act_str = act_str.stdout.decode() + act_str = act_str.splitlines() + act_str = act_str[1].strip() + return act_str def windows_is_activated(): - """Check if Windows is activated via slmgr.vbs and return bool.""" - activation_string = subprocess.run( - ['cscript', '//nologo', SLMGR, '/xpr'], check=False, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - activation_string = activation_string.stdout.decode() + """Check if Windows is activated via slmgr.vbs and return bool.""" + activation_string = subprocess.run( + ['cscript', '//nologo', SLMGR, '/xpr'], check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + activation_string = activation_string.stdout.decode() - return bool(activation_string and 'permanent' in activation_string) + return bool(activation_string and 'permanent' in activation_string) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From a269859b1792ee210524c40c66a15fc9f0cf4e8a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:42:58 -0700 Subject: [PATCH 112/186] Updated backup.py --- .bin/Scripts/functions/backup.py | 344 ++++++++++++++++--------------- 1 file changed, 175 insertions(+), 169 deletions(-) diff --git a/.bin/Scripts/functions/backup.py b/.bin/Scripts/functions/backup.py index d872c4d0..f7679db2 100644 --- a/.bin/Scripts/functions/backup.py +++ b/.bin/Scripts/functions/backup.py @@ -6,200 +6,206 @@ from functions.disk import * # Regex REGEX_BAD_PATH_NAMES = re.compile( - r'([<>:"/\|\?\*]' - r'|^(CON|PRN|AUX|NUL|COM\d*|LPT\d*)$)' - r'|^\s+' - r'|[\s\.]+$', - re.IGNORECASE) + r'([<>:"/\|\?\*]' + r'|^(CON|PRN|AUX|NUL|COM\d*|LPT\d*)$)' + r'|^\s+' + r'|[\s\.]+$', + re.IGNORECASE) def backup_partition(disk, par): - """Create a backup image of a partition.""" - if par.get('Image Exists', False) or par['Number'] in disk['Bad Partitions']: - raise GenericAbort + """Create a backup image of a partition.""" + if (par.get('Image Exists', False) + or par['Number'] in disk['Bad Partitions']): + raise GenericAbort - cmd = [ - global_vars['Tools']['wimlib-imagex'], - 'capture', - '{}:\\'.format(par['Letter']), - par['Image Path'], - par['Image Name'], # Image name - par['Image Name'], # Image description - '--compress=none', - ] - dest_dir = re.sub(r'(.*)\\.*$', r'\1', par['Image Path'], re.IGNORECASE) - os.makedirs(dest_dir, exist_ok=True) - run_program(cmd) + cmd = [ + global_vars['Tools']['wimlib-imagex'], + 'capture', + '{}:\\'.format(par['Letter']), + par['Image Path'], + par['Image Name'], # Image name + par['Image Name'], # Image description + '--compress=none', + ] + dest_dir = re.sub(r'(.*)\\.*$', r'\1', par['Image Path'], re.IGNORECASE) + os.makedirs(dest_dir, exist_ok=True) + run_program(cmd) def fix_path(path): - """Replace invalid filename characters with underscores.""" - local_drive = path[1:2] == ':' - new_path = REGEX_BAD_PATH_NAMES.sub('_', path) - if local_drive: - new_path = '{}:{}'.format(new_path[0:1], new_path[2:]) - return new_path + """Replace invalid filename characters with underscores.""" + local_drive = path[1:2] == ':' + new_path = REGEX_BAD_PATH_NAMES.sub('_', path) + if local_drive: + new_path = '{}:{}'.format(new_path[0:1], new_path[2:]) + return new_path def get_volume_display_name(mountpoint): - """Get display name from volume mountpoint and label, returns str.""" - name = mountpoint - try: - kernel32 = ctypes.windll.kernel32 - vol_name_buffer = ctypes.create_unicode_buffer(1024) - fs_name_buffer = ctypes.create_unicode_buffer(1024) - serial_number = None - max_component_length = None - file_system_flags = None + """Get display name from volume mountpoint and label, returns str.""" + name = mountpoint + try: + kernel32 = ctypes.windll.kernel32 + vol_name_buffer = ctypes.create_unicode_buffer(1024) + fs_name_buffer = ctypes.create_unicode_buffer(1024) + serial_number = None + max_component_length = None + file_system_flags = None - vol_info = kernel32.GetVolumeInformationW( - ctypes.c_wchar_p(mountpoint), - vol_name_buffer, - ctypes.sizeof(vol_name_buffer), - serial_number, - max_component_length, - file_system_flags, - fs_name_buffer, - ctypes.sizeof(fs_name_buffer) - ) + vol_info = kernel32.GetVolumeInformationW( + ctypes.c_wchar_p(mountpoint), + vol_name_buffer, + ctypes.sizeof(vol_name_buffer), + serial_number, + max_component_length, + file_system_flags, + fs_name_buffer, + ctypes.sizeof(fs_name_buffer) + ) - name = '{} "{}"'.format(name, vol_name_buffer.value) - except: - pass + name = '{} "{}"'.format(name, vol_name_buffer.value) + except: + pass - return name + return name def prep_disk_for_backup(destination, disk, backup_prefix): - """Gather details about the disk and its partitions. + """Gather details about the disk and its partitions. - This includes partitions that can't be backed up, - whether backups already exist on the BACKUP_SERVER, - partition names/sizes/used space, etc.""" - disk['Clobber Risk'] = [] - width = len(str(len(disk['Partitions']))) + This includes partitions that can't be backed up, + whether backups already exist on the BACKUP_SERVER, + partition names/sizes/used space, etc.""" + disk['Clobber Risk'] = [] + width = len(str(len(disk['Partitions']))) - # Get partition totals - disk['Bad Partitions'] = [par['Number'] for par in disk['Partitions'] - if is_bad_partition(par)] - num_valid_partitions = len(disk['Partitions']) - len(disk['Bad Partitions']) - disk['Valid Partitions'] = num_valid_partitions - if disk['Valid Partitions'] <= 0: - print_error('ERROR: No partitions can be backed up for this disk') - raise GenericAbort + # Get partition totals + disk['Bad Partitions'] = [par['Number'] for par in disk['Partitions'] + if is_bad_partition(par)] + num_valid_partitions = len(disk['Partitions']) - len(disk['Bad Partitions']) + disk['Valid Partitions'] = num_valid_partitions + if disk['Valid Partitions'] <= 0: + print_error('ERROR: No partitions can be backed up for this disk') + raise GenericAbort - # Prep partitions - for par in disk['Partitions']: - display = '{size} {fs}'.format( - num = par['Number'], - width = width, - size = par['Size'], - fs = par['FileSystem']) + # Prep partitions + for par in disk['Partitions']: + display = '{size} {fs}'.format( + num = par['Number'], + width = width, + size = par['Size'], + fs = par['FileSystem']) - if par['Number'] in disk['Bad Partitions']: - # Set display string using partition description & OS type - display = '* {display}\t\t{q}{name}{q}\t{desc} ({os})'.format( - display = display, - q = '"' if par['Name'] != '' else '', - name = par['Name'], - desc = par['Description'], - os = par['OS']) - else: - # Update info for WIM capturing - par['Image Name'] = par['Name'] if par['Name'] else 'Unknown' - if 'IP' in destination: - par['Image Path'] = r'\\{}\{}\{}'.format( - destination['IP'], destination['Share'], backup_prefix) - else: - par['Image Path'] = r'{}:\{}'.format( - destination['Letter'], backup_prefix) - par['Image Path'] += r'\{}_{}.wim'.format( - par['Number'], par['Image Name']) - par['Image Path'] = fix_path(par['Image Path']) + if par['Number'] in disk['Bad Partitions']: + # Set display string using partition description & OS type + display = '* {display}\t\t{q}{name}{q}\t{desc} ({os})'.format( + display = display, + q = '"' if par['Name'] != '' else '', + name = par['Name'], + desc = par['Description'], + os = par['OS']) + else: + # Update info for WIM capturing + par['Image Name'] = par['Name'] if par['Name'] else 'Unknown' + if 'IP' in destination: + par['Image Path'] = r'\\{}\{}\{}'.format( + destination['IP'], destination['Share'], backup_prefix) + else: + par['Image Path'] = r'{}:\{}'.format( + destination['Letter'], backup_prefix) + par['Image Path'] += r'\{}_{}.wim'.format( + par['Number'], par['Image Name']) + par['Image Path'] = fix_path(par['Image Path']) - # Check for existing backups - par['Image Exists'] = os.path.exists(par['Image Path']) - if par['Image Exists']: - disk['Clobber Risk'].append(par['Number']) - display = '+ {}'.format(display) - else: - display = ' {}'.format(display) + # Check for existing backups + par['Image Exists'] = os.path.exists(par['Image Path']) + if par['Image Exists']: + disk['Clobber Risk'].append(par['Number']) + display = '+ {}'.format(display) + else: + display = ' {}'.format(display) - # Append rest of Display String for valid/clobber partitions - display += ' (Used: {used})\t{q}{name}{q}'.format( - used = par['Used Space'], - q = '"' if par['Name'] != '' else '', - name = par['Name']) - # For all partitions - par['Display String'] = display + # Append rest of Display String for valid/clobber partitions + display += ' (Used: {used})\t{q}{name}{q}'.format( + used = par['Used Space'], + q = '"' if par['Name'] != '' else '', + name = par['Name']) + # For all partitions + par['Display String'] = display - # Set description for bad partitions - warnings = '\n' - if disk['Bad Partitions']: - warnings += '{} * Unsupported filesystem{}\n'.format( - COLORS['YELLOW'], COLORS['CLEAR']) - if disk['Clobber Risk']: - warnings += '{} + Backup exists on {}{}\n'.format( - COLORS['BLUE'], destination['Name'], COLORS['CLEAR']) - if disk['Bad Partitions'] or disk['Clobber Risk']: - warnings += '\n{}Marked partition(s) will NOT be backed up.{}\n'.format( - COLORS['YELLOW'], COLORS['CLEAR']) - disk['Backup Warnings'] = warnings + # Set description for bad partitions + warnings = '\n' + if disk['Bad Partitions']: + warnings += '{} * Unsupported filesystem{}\n'.format( + COLORS['YELLOW'], COLORS['CLEAR']) + if disk['Clobber Risk']: + warnings += '{} + Backup exists on {}{}\n'.format( + COLORS['BLUE'], destination['Name'], COLORS['CLEAR']) + if disk['Bad Partitions'] or disk['Clobber Risk']: + warnings += '\n{}Marked partition(s) will NOT be backed up.{}\n'.format( + COLORS['YELLOW'], COLORS['CLEAR']) + disk['Backup Warnings'] = warnings def select_backup_destination(auto_select=True): - """Select a backup destination from a menu, returns server dict.""" - destinations = [s for s in BACKUP_SERVERS if s['Mounted']] - actions = [ - {'Name': 'Main Menu', 'Letter': 'M'}, - ] + """Select a backup destination from a menu, returns server dict.""" + destinations = [s for s in BACKUP_SERVERS if s['Mounted']] + actions = [ + {'Name': 'Main Menu', 'Letter': 'M'}, + ] - # Add local disks - for d in psutil.disk_partitions(): - if re.search(r'^{}'.format(global_vars['Env']['SYSTEMDRIVE']), d.mountpoint, re.IGNORECASE): - # Skip current OS drive - pass - elif 'fixed' in d.opts: - # Skip DVD, etc - destinations.append({ - 'Name': 'Local Disk - {}'.format( - get_volume_display_name(d.mountpoint)), - 'Letter': re.sub(r'^(\w):\\.*$', r'\1', d.mountpoint), - }) + # Add local disks + for d in psutil.disk_partitions(): + if re.search( + r'^{}'.format(global_vars['Env']['SYSTEMDRIVE']), + d.mountpoint, + re.IGNORECASE): + # Skip current OS drive + pass + elif 'fixed' in d.opts: + # Skip DVD, etc + destinations.append({ + 'Name': 'Local Disk - {}'.format( + get_volume_display_name(d.mountpoint)), + 'Letter': re.sub(r'^(\w):\\.*$', r'\1', d.mountpoint), + }) - # Size check - for dest in destinations: - if 'IP' in dest: - dest['Usage'] = shutil.disk_usage(r'\\{IP}\{Share}'.format(**dest)) - else: - dest['Usage'] = shutil.disk_usage('{}:\\'.format(dest['Letter'])) - dest['Free Space'] = human_readable_size(dest['Usage'].free) - dest['Display Name'] = '{Name} ({Free Space} available)'.format(**dest) - - # Bail - if not destinations: - print_warning('No backup destinations found.') - raise GenericAbort - - # Skip menu? - if len(destinations) == 1 and auto_select: - return destinations[0] - - selection = menu_select( - title = 'Where are we backing up to?', - main_entries = destinations, - action_entries = actions) - if selection == 'M': - raise GenericAbort + # Size check + for dest in destinations: + if 'IP' in dest: + dest['Usage'] = shutil.disk_usage(r'\\{IP}\{Share}'.format(**dest)) else: - return destinations[int(selection)-1] + dest['Usage'] = shutil.disk_usage('{}:\\'.format(dest['Letter'])) + dest['Free Space'] = human_readable_size(dest['Usage'].free) + dest['Display Name'] = '{Name} ({Free Space} available)'.format(**dest) + + # Bail + if not destinations: + print_warning('No backup destinations found.') + raise GenericAbort + + # Skip menu? + if len(destinations) == 1 and auto_select: + return destinations[0] + + selection = menu_select( + title = 'Where are we backing up to?', + main_entries = destinations, + action_entries = actions) + if selection == 'M': + raise GenericAbort + else: + return destinations[int(selection)-1] def verify_wim_backup(partition): - """Verify WIM integrity.""" - if not os.path.exists(partition['Image Path']): - raise PathNotFoundError - cmd = [ - global_vars['Tools']['wimlib-imagex'], - 'verify', - partition['Image Path'], - '--nocheck', - ] - run_program(cmd) + """Verify WIM integrity.""" + if not os.path.exists(partition['Image Path']): + raise PathNotFoundError + cmd = [ + global_vars['Tools']['wimlib-imagex'], + 'verify', + partition['Image Path'], + '--nocheck', + ] + run_program(cmd) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 6a3e2251922e7da89b6265774b6b66253cb2ca4c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:43:12 -0700 Subject: [PATCH 113/186] Updated browsers.py --- .bin/Scripts/functions/browsers.py | 847 +++++++++++++++-------------- 1 file changed, 425 insertions(+), 422 deletions(-) diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index 143d018d..750f8af7 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -7,42 +7,42 @@ from operator import itemgetter # Define other_results for later try_and_print browser_data = {} other_results = { - 'Error': { - 'MultipleInstallationsError': 'Multiple installations detected', - }, - 'Warning': { - 'NotInstalledError': 'Not installed', - 'NoProfilesError': 'No profiles found', - } + 'Error': { + 'MultipleInstallationsError': 'Multiple installations detected', + }, + 'Warning': { + 'NotInstalledError': 'Not installed', + 'NoProfilesError': 'No profiles found', } + } # Regex REGEX_BACKUP = re.compile( - r'\.\w*bak.*', - re.IGNORECASE) + r'\.\w*bak.*', + re.IGNORECASE) REGEX_CHROMIUM_PROFILE = re.compile( - r'^(Default|Profile)', - re.IGNORECASE) + r'^(Default|Profile)', + re.IGNORECASE) REGEX_CHROMIUM_ITEMS = re.compile( - r'^(Bookmarks|Cookies|Favicons|Google Profile' - r'|History|Login Data|Top Sites|TransportSecurity' - r'|Visited Links|Web Data)', - re.IGNORECASE) + r'^(Bookmarks|Cookies|Favicons|Google Profile' + r'|History|Login Data|Top Sites|TransportSecurity' + r'|Visited Links|Web Data)', + re.IGNORECASE) REGEX_MOZILLA = re.compile( - r'^(bookmarkbackups|(cookies|formhistory|places).sqlite' - r'|key3.db|logins.json|persdict.dat)$', - re.IGNORECASE) + r'^(bookmarkbackups|(cookies|formhistory|places).sqlite' + r'|key3.db|logins.json|persdict.dat)$', + re.IGNORECASE) # STATIC VARIABLES DEFAULT_HOMEPAGE = 'https://www.google.com/' IE_GALLERY = 'https://www.microsoft.com/en-us/iegallery' MOZILLA_PREFS = { - 'browser.search.defaultenginename': '"Google"', - 'browser.search.defaultenginename.US': '"Google"', - 'browser.search.geoSpecificDefaults': 'false', - 'browser.startup.homepage': '"{}"'.format(DEFAULT_HOMEPAGE), - 'extensions.ui.lastCategory': '"addons://list/extension"', - } + 'browser.search.defaultenginename': '"Google"', + 'browser.search.defaultenginename.US': '"Google"', + 'browser.search.geoSpecificDefaults': 'false', + 'browser.startup.homepage': '"{}"'.format(DEFAULT_HOMEPAGE), + 'extensions.ui.lastCategory': '"addons://list/extension"', + } UBO_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin/cjpalhdlnbpafiamejdnhcphjbkeiagm?hl=en' UBO_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm' UBO_EXTRA_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin-extra/pgdnlhfefecpicbbihgmbmffkjpaplco?hl=en' @@ -53,456 +53,459 @@ UBO_MOZILLA_REG = r'Software\Mozilla\Firefox\Extensions' UBO_MOZILLA_REG_NAME = 'uBlock0@raymondhill.net' UBO_OPERA = 'https://addons.opera.com/en/extensions/details/ublock/?display=en' SUPPORTED_BROWSERS = { - 'Internet Explorer': { - 'base': 'ie', - 'exe_name': 'iexplore.exe', - 'rel_install_path': 'Internet Explorer', - 'user_data_path': r'{USERPROFILE}\Favorites', - }, - 'Google Chrome': { - 'base': 'chromium', - 'exe_name': 'chrome.exe', - 'rel_install_path': r'Google\Chrome\Application', - 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome\User Data', - }, - 'Google Chrome Canary': { - 'base': 'chromium', - 'exe_name': 'chrome.exe', - 'rel_install_path': r'Google\Chrome SxS\Application', - 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome SxS\User Data', - }, - 'Mozilla Firefox': { - 'base': 'mozilla', - 'exe_name': 'firefox.exe', - 'rel_install_path': 'Mozilla Firefox', - 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', - }, - 'Mozilla Firefox Dev': { - 'base': 'mozilla', - 'exe_name': 'firefox.exe', - 'rel_install_path': 'Firefox Developer Edition', - 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', - }, - 'Opera': { - 'base': 'chromium', - 'exe_name': 'launcher.exe', - 'rel_install_path': 'Opera', - 'user_data_path': r'{APPDATA}\Opera Software\Opera Stable', - }, - 'Opera Beta': { - 'base': 'chromium', - 'exe_name': 'launcher.exe', - 'rel_install_path': 'Opera beta', - 'user_data_path': r'{APPDATA}\Opera Software\Opera Next', - }, - 'Opera Dev': { - 'base': 'chromium', - 'exe_name': 'launcher.exe', - 'rel_install_path': 'Opera developer', - 'user_data_path': r'{APPDATA}\Opera Software\Opera Developer', - }, - } + 'Internet Explorer': { + 'base': 'ie', + 'exe_name': 'iexplore.exe', + 'rel_install_path': 'Internet Explorer', + 'user_data_path': r'{USERPROFILE}\Favorites', + }, + 'Google Chrome': { + 'base': 'chromium', + 'exe_name': 'chrome.exe', + 'rel_install_path': r'Google\Chrome\Application', + 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome\User Data', + }, + 'Google Chrome Canary': { + 'base': 'chromium', + 'exe_name': 'chrome.exe', + 'rel_install_path': r'Google\Chrome SxS\Application', + 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome SxS\User Data', + }, + 'Mozilla Firefox': { + 'base': 'mozilla', + 'exe_name': 'firefox.exe', + 'rel_install_path': 'Mozilla Firefox', + 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', + }, + 'Mozilla Firefox Dev': { + 'base': 'mozilla', + 'exe_name': 'firefox.exe', + 'rel_install_path': 'Firefox Developer Edition', + 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', + }, + 'Opera': { + 'base': 'chromium', + 'exe_name': 'launcher.exe', + 'rel_install_path': 'Opera', + 'user_data_path': r'{APPDATA}\Opera Software\Opera Stable', + }, + 'Opera Beta': { + 'base': 'chromium', + 'exe_name': 'launcher.exe', + 'rel_install_path': 'Opera beta', + 'user_data_path': r'{APPDATA}\Opera Software\Opera Next', + }, + 'Opera Dev': { + 'base': 'chromium', + 'exe_name': 'launcher.exe', + 'rel_install_path': 'Opera developer', + 'user_data_path': r'{APPDATA}\Opera Software\Opera Developer', + }, + } def archive_all_users(): - """Create backups for all browsers for all users.""" - users_root = r'{}\Users'.format(global_vars['Env']['SYSTEMDRIVE']) - user_envs = [] + """Create backups for all browsers for all users.""" + users_root = r'{}\Users'.format(global_vars['Env']['SYSTEMDRIVE']) + user_envs = [] - # Build list of valid users - for user_name in os.listdir(users_root): - valid_user = True - if user_name in ('Default', 'Default User'): - # Skip default users - continue - user_path = os.path.join(users_root, user_name) - appdata_local = os.path.join(user_path, r'AppData\Local') - appdata_roaming = os.path.join(user_path, r'AppData\Roaming') - valid_user &= os.path.exists(appdata_local) - valid_user &= os.path.exists(appdata_roaming) - if valid_user: - user_envs.append({ - 'USERNAME': user_name, - 'USERPROFILE': user_path, - 'APPDATA': appdata_roaming, - 'LOCALAPPDATA': appdata_local}) + # Build list of valid users + for user_name in os.listdir(users_root): + valid_user = True + if user_name in ('Default', 'Default User'): + # Skip default users + continue + user_path = os.path.join(users_root, user_name) + appdata_local = os.path.join(user_path, r'AppData\Local') + appdata_roaming = os.path.join(user_path, r'AppData\Roaming') + valid_user &= os.path.exists(appdata_local) + valid_user &= os.path.exists(appdata_roaming) + if valid_user: + user_envs.append({ + 'USERNAME': user_name, + 'USERPROFILE': user_path, + 'APPDATA': appdata_roaming, + 'LOCALAPPDATA': appdata_local}) - # Backup browsers for all valid users - print_info('Backing up browsers') - for fake_env in sorted(user_envs, key=itemgetter('USERPROFILE')): - print_standard(' {}'.format(fake_env['USERNAME'])) - for b_k, b_v in sorted(SUPPORTED_BROWSERS.items()): - if b_k == 'Mozilla Firefox Dev': - continue - source_path = b_v['user_data_path'].format(**fake_env) - if not os.path.exists(source_path): - continue - source_items = source_path + '*' - archive_path = r'{BackupDir}\Browsers ({USERNAME})\{Date}'.format( - **global_vars, **fake_env) - os.makedirs(archive_path, exist_ok=True) - archive_path += r'\{}.7z'.format(b_k) - cmd = [ - global_vars['Tools']['SevenZip'], - 'a', '-aoa', '-bso0', '-bse0', '-mx=1', - archive_path, source_items] - try_and_print(message='{}...'.format(b_k), - function=run_program, cmd=cmd) - print_standard(' ') - -def archive_browser(name): - """Create backup of Browser saved in the BackupDir.""" - source = '{}*'.format(browser_data[name]['user_data_path']) - dest = r'{BackupDir}\Browsers ({USERNAME})\{Date}'.format( - **global_vars, **global_vars['Env']) - archive = r'{}\{}.7z'.format(dest, name) - os.makedirs(dest, exist_ok=True) - cmd = [ + # Backup browsers for all valid users + print_info('Backing up browsers') + for fake_env in sorted(user_envs, key=itemgetter('USERPROFILE')): + print_standard(' {}'.format(fake_env['USERNAME'])) + for b_k, b_v in sorted(SUPPORTED_BROWSERS.items()): + if b_k == 'Mozilla Firefox Dev': + continue + source_path = b_v['user_data_path'].format(**fake_env) + if not os.path.exists(source_path): + continue + source_items = source_path + '*' + archive_path = r'{BackupDir}\Browsers ({USERNAME})\{Date}'.format( + **global_vars, **fake_env) + os.makedirs(archive_path, exist_ok=True) + archive_path += r'\{}.7z'.format(b_k) + cmd = [ global_vars['Tools']['SevenZip'], 'a', '-aoa', '-bso0', '-bse0', '-mx=1', - '-mhe=on', '-p{}'.format(ARCHIVE_PASSWORD), - archive, source] - run_program(cmd) + archive_path, source_items] + try_and_print(message='{}...'.format(b_k), + function=run_program, cmd=cmd) + print_standard(' ') + +def archive_browser(name): + """Create backup of Browser saved in the BackupDir.""" + source = '{}*'.format(browser_data[name]['user_data_path']) + dest = r'{BackupDir}\Browsers ({USERNAME})\{Date}'.format( + **global_vars, **global_vars['Env']) + archive = r'{}\{}.7z'.format(dest, name) + os.makedirs(dest, exist_ok=True) + cmd = [ + global_vars['Tools']['SevenZip'], + 'a', '-aoa', '-bso0', '-bse0', '-mx=1', + '-mhe=on', '-p{}'.format(ARCHIVE_PASSWORD), + archive, source] + run_program(cmd) def backup_browsers(): - """Create backup of all detected browser profiles.""" - for name in [k for k, v in sorted(browser_data.items()) if v['profiles']]: - try_and_print(message='{}...'.format(name), - function=archive_browser, name=name) + """Create backup of all detected browser profiles.""" + for name in [k for k, v in sorted(browser_data.items()) if v['profiles']]: + try_and_print(message='{}...'.format(name), + function=archive_browser, name=name) def clean_chromium_profile(profile): - """Renames profile, creates a new folder, and copies the user data to it.""" - if profile is None: - raise Exception - backup_path = '{path}_{Date}.bak'.format( - path=profile['path'], **global_vars) - backup_path = non_clobber_rename(backup_path) - shutil.move(profile['path'], backup_path) - os.makedirs(profile['path'], exist_ok=True) + """Renames profile, creates a new folder, and copies the user data to it.""" + if profile is None: + raise Exception + backup_path = '{path}_{Date}.bak'.format( + path=profile['path'], **global_vars) + backup_path = non_clobber_rename(backup_path) + shutil.move(profile['path'], backup_path) + os.makedirs(profile['path'], exist_ok=True) - # Restore essential files from backup_path - for entry in os.scandir(backup_path): - if REGEX_CHROMIUM_ITEMS.search(entry.name): - shutil.copy(entry.path, r'{}\{}'.format( - profile['path'], entry.name)) + # Restore essential files from backup_path + for entry in os.scandir(backup_path): + if REGEX_CHROMIUM_ITEMS.search(entry.name): + shutil.copy(entry.path, r'{}\{}'.format( + profile['path'], entry.name)) def clean_internet_explorer(**kwargs): - """Uses the built-in function to reset IE and sets the homepage. + """Uses the built-in function to reset IE and sets the homepage. - NOTE: kwargs set but unused as a workaround.""" - kill_process('iexplore.exe') - run_program(['rundll32.exe', 'inetcpl.cpl,ResetIEtoDefaults'], check=False) - key = r'Software\Microsoft\Internet Explorer\Main' + NOTE: kwargs set but unused as a workaround.""" + kill_process('iexplore.exe') + run_program(['rundll32.exe', 'inetcpl.cpl,ResetIEtoDefaults'], check=False) + key = r'Software\Microsoft\Internet Explorer\Main' - # Set homepage - with winreg.OpenKey(HKCU, key, access=winreg.KEY_WRITE) as _key: - winreg.SetValueEx(_key, 'Start Page', 0, - winreg.REG_SZ, DEFAULT_HOMEPAGE) - try: - winreg.DeleteValue(_key, 'Secondary Start Pages') - except FileNotFoundError: - pass + # Set homepage + with winreg.OpenKey(HKCU, key, access=winreg.KEY_WRITE) as _key: + winreg.SetValueEx(_key, 'Start Page', 0, + winreg.REG_SZ, DEFAULT_HOMEPAGE) + try: + winreg.DeleteValue(_key, 'Secondary Start Pages') + except FileNotFoundError: + pass def clean_mozilla_profile(profile): - """Renames profile, creates a new folder, and copies the user data to it.""" - if profile is None: - raise Exception - backup_path = '{path}_{Date}.bak'.format( - path=profile['path'], **global_vars) - backup_path = non_clobber_rename(backup_path) - shutil.move(profile['path'], backup_path) - homepages = [] - os.makedirs(profile['path'], exist_ok=True) + """Renames profile, creates a new folder, and copies the user data to it.""" + if profile is None: + raise Exception + backup_path = '{path}_{Date}.bak'.format( + path=profile['path'], **global_vars) + backup_path = non_clobber_rename(backup_path) + shutil.move(profile['path'], backup_path) + homepages = [] + os.makedirs(profile['path'], exist_ok=True) - # Restore essential files from backup_path - for entry in os.scandir(backup_path): - if REGEX_MOZILLA.search(entry.name): - if entry.is_dir(): - shutil.copytree(entry.path, r'{}\{}'.format( - profile['path'], entry.name)) - else: - shutil.copy(entry.path, r'{}\{}'.format( - profile['path'], entry.name)) + # Restore essential files from backup_path + for entry in os.scandir(backup_path): + if REGEX_MOZILLA.search(entry.name): + if entry.is_dir(): + shutil.copytree(entry.path, r'{}\{}'.format( + profile['path'], entry.name)) + else: + shutil.copy(entry.path, r'{}\{}'.format( + profile['path'], entry.name)) - # Set profile defaults - with open(r'{path}\prefs.js'.format(**profile), 'a', encoding='ascii') as f: - for k, v in MOZILLA_PREFS.items(): - f.write('user_pref("{}", {});\n'.format(k, v)) + # Set profile defaults + with open(r'{path}\prefs.js'.format(**profile), 'a', encoding='ascii') as f: + for k, v in MOZILLA_PREFS.items(): + f.write('user_pref("{}", {});\n'.format(k, v)) def get_browser_details(name): - """Get installation status and profile details for all supported browsers.""" - browser = SUPPORTED_BROWSERS[name].copy() + """Get install status and profile details for all supported browsers.""" + browser = SUPPORTED_BROWSERS[name].copy() - # Update user_data_path - browser['user_data_path'] = browser['user_data_path'].format( - **global_vars['Env']) + # Update user_data_path + browser['user_data_path'] = browser['user_data_path'].format( + **global_vars['Env']) - # Find executable (if multiple files are found, the last one is used) - exe_path = None - num_installs = 0 - for install_path in ['LOCALAPPDATA', 'PROGRAMFILES(X86)', 'PROGRAMFILES']: - test_path = r'{install_path}\{rel_install_path}\{exe_name}'.format( - install_path = global_vars['Env'].get(install_path, ''), - **browser) - if os.path.exists(test_path): - num_installs += 1 - exe_path = test_path + # Find executable (if multiple files are found, the last one is used) + exe_path = None + num_installs = 0 + for install_path in ['LOCALAPPDATA', 'PROGRAMFILES(X86)', 'PROGRAMFILES']: + test_path = r'{install_path}\{rel_install_path}\{exe_name}'.format( + install_path = global_vars['Env'].get(install_path, ''), + **browser) + if os.path.exists(test_path): + num_installs += 1 + exe_path = test_path - # Find profile(s) - profiles = [] - if browser['base'] == 'ie': - profiles.append({'name': 'Default', 'path': None}) - elif 'Google Chrome' in name: - profiles.extend( - get_chromium_profiles( - search_path=browser['user_data_path'])) - elif browser['base'] == 'mozilla': - dev = 'Dev' in name - profiles.extend( - get_mozilla_profiles( - search_path=browser['user_data_path'], dev=dev)) - if exe_path and not dev and len(profiles) == 0: - # e.g. If Firefox is installed but no profiles were found. - ## Rename profiles.ini and create a new default profile - profiles_ini_path = browser['user_data_path'].replace( - 'Profiles', 'profiles.ini') - if os.path.exists(profiles_ini_path): - backup_path = '{path}_{Date}.bak'.format( - path=profiles_ini_path, **global_vars) - backup_path = non_clobber_rename(backup_path) - shutil.move(profiles_ini_path, backup_path) - run_program([exe_path, '-createprofile', 'default'], check=False) - profiles.extend( - get_mozilla_profiles( - search_path=browser['user_data_path'], dev=dev)) + # Find profile(s) + profiles = [] + if browser['base'] == 'ie': + profiles.append({'name': 'Default', 'path': None}) + elif 'Google Chrome' in name: + profiles.extend( + get_chromium_profiles( + search_path=browser['user_data_path'])) + elif browser['base'] == 'mozilla': + dev = 'Dev' in name + profiles.extend( + get_mozilla_profiles( + search_path=browser['user_data_path'], dev=dev)) + if exe_path and not dev and len(profiles) == 0: + # e.g. If Firefox is installed but no profiles were found. + ## Rename profiles.ini and create a new default profile + profiles_ini_path = browser['user_data_path'].replace( + 'Profiles', 'profiles.ini') + if os.path.exists(profiles_ini_path): + backup_path = '{path}_{Date}.bak'.format( + path=profiles_ini_path, **global_vars) + backup_path = non_clobber_rename(backup_path) + shutil.move(profiles_ini_path, backup_path) + run_program([exe_path, '-createprofile', 'default'], check=False) + profiles.extend( + get_mozilla_profiles( + search_path=browser['user_data_path'], dev=dev)) - elif 'Opera' in name: - if os.path.exists(browser['user_data_path']): - profiles.append( - {'name': 'Default', 'path': browser['user_data_path']}) + elif 'Opera' in name: + if os.path.exists(browser['user_data_path']): + profiles.append( + {'name': 'Default', 'path': browser['user_data_path']}) - # Get homepages - if browser['base'] == 'ie': - # IE is set to only have one profile above - profiles[0]['homepages'] = get_ie_homepages() - elif browser['base'] == 'mozilla': - for profile in profiles: - prefs_path = r'{path}\prefs.js'.format(**profile) - profile['homepages'] = get_mozilla_homepages(prefs_path=prefs_path) + # Get homepages + if browser['base'] == 'ie': + # IE is set to only have one profile above + profiles[0]['homepages'] = get_ie_homepages() + elif browser['base'] == 'mozilla': + for profile in profiles: + prefs_path = r'{path}\prefs.js'.format(**profile) + profile['homepages'] = get_mozilla_homepages(prefs_path=prefs_path) - # Add to browser_data - browser_data[name] = browser - browser_data[name].update({ - 'exe_path': exe_path, - 'profiles': profiles, - }) + # Add to browser_data + browser_data[name] = browser + browser_data[name].update({ + 'exe_path': exe_path, + 'profiles': profiles, + }) - # Raise installation warnings (if any) - if num_installs == 0: - raise NotInstalledError - elif num_installs > 1 and browser['base'] != 'ie': - raise MultipleInstallationsError + # Raise installation warnings (if any) + if num_installs == 0: + raise NotInstalledError + elif num_installs > 1 and browser['base'] != 'ie': + raise MultipleInstallationsError def get_chromium_profiles(search_path): - """Find any chromium-style profiles and return as a list of dicts.""" - profiles = [] - try: - for entry in os.scandir(search_path): - if entry.is_dir() and REGEX_CHROMIUM_PROFILE.search(entry.name): - profiles.append(entry) - REGEX_PROFILE_BACKUP = r'\.\w+bak.*' - profiles = [p for p in profiles if not REGEX_BACKUP.search(p.name)] - # Convert os.DirEntries to dicts - profiles = [{'name': p.name, 'path': p.path} for p in profiles] - except Exception: - pass + """Find any chromium-style profiles and return as a list of dicts.""" + profiles = [] + try: + for entry in os.scandir(search_path): + if entry.is_dir() and REGEX_CHROMIUM_PROFILE.search(entry.name): + profiles.append(entry) + REGEX_PROFILE_BACKUP = r'\.\w+bak.*' + profiles = [p for p in profiles if not REGEX_BACKUP.search(p.name)] + # Convert os.DirEntries to dicts + profiles = [{'name': p.name, 'path': p.path} for p in profiles] + except Exception: + pass - return profiles + return profiles def get_ie_homepages(): - """Read homepages from the registry and return as a list.""" - homepages = [] - main_page = '' - extra_pages = [] - key = r'Software\Microsoft\Internet Explorer\Main' - with winreg.OpenKey(HKCU, key) as _key: - try: - main_page = winreg.QueryValueEx(_key, 'Start Page')[0] - except FileNotFoundError: - pass - try: - extra_pages = winreg.QueryValueEx(_key, 'Secondary Start Pages')[0] - except FileNotFoundError: - pass - if main_page != '': - homepages.append(main_page) - if len(extra_pages) > 0: - homepages.extend(extra_pages) + """Read homepages from the registry and return as a list.""" + homepages = [] + main_page = '' + extra_pages = [] + key = r'Software\Microsoft\Internet Explorer\Main' + with winreg.OpenKey(HKCU, key) as _key: + try: + main_page = winreg.QueryValueEx(_key, 'Start Page')[0] + except FileNotFoundError: + pass + try: + extra_pages = winreg.QueryValueEx(_key, 'Secondary Start Pages')[0] + except FileNotFoundError: + pass + if main_page != '': + homepages.append(main_page) + if len(extra_pages) > 0: + homepages.extend(extra_pages) - # Remove all curly braces - homepages = [h.replace('{', '').replace('}', '') for h in homepages] - return homepages + # Remove all curly braces + homepages = [h.replace('{', '').replace('}', '') for h in homepages] + return homepages def get_mozilla_homepages(prefs_path): - """Read homepages from prefs.js and return as a list.""" - homepages = [] - try: - with open(prefs_path, 'r') as f: - search = re.search( - r'browser\.startup\.homepage", "([^"]*)"', - f.read(), re.IGNORECASE) - if search: - homepages = search.group(1).split('|') - except Exception: - pass + """Read homepages from prefs.js and return as a list.""" + homepages = [] + try: + with open(prefs_path, 'r') as f: + search = re.search( + r'browser\.startup\.homepage", "([^"]*)"', + f.read(), re.IGNORECASE) + if search: + homepages = search.group(1).split('|') + except Exception: + pass - return homepages + return homepages def get_mozilla_profiles(search_path, dev=False): - """Find any mozilla-style profiles and return as a list of dicts.""" - profiles = [] - try: - for entry in os.scandir(search_path): - if entry.is_dir(): - if 'dev-edition' in entry.name: - # NOTE: Not always present which can lead - # to Dev profiles being marked as non-Dev - ## NOTE 2: It is possible that a non-Dev profile - ## to be created with 'dev-edition' in the name. - ## (It wouldn't make sense, but possible) - if dev: - profiles.append(entry) - elif not dev: - profiles.append(entry) - profiles = [p for p in profiles if not REGEX_BACKUP.search(p.name)] - # Convert os.DirEntries to dicts - profiles = [{'name': p.name, 'path': p.path} for p in profiles] - except Exception: - pass + """Find any mozilla-style profiles and return as a list of dicts.""" + profiles = [] + try: + for entry in os.scandir(search_path): + if entry.is_dir(): + if 'dev-edition' in entry.name: + # NOTE: Not always present which can lead + # to Dev profiles being marked as non-Dev + ## NOTE 2: It is possible that a non-Dev profile + ## to be created with 'dev-edition' in the name. + ## (It wouldn't make sense, but possible) + if dev: + profiles.append(entry) + elif not dev: + profiles.append(entry) + profiles = [p for p in profiles if not REGEX_BACKUP.search(p.name)] + # Convert os.DirEntries to dicts + profiles = [{'name': p.name, 'path': p.path} for p in profiles] + except Exception: + pass - return profiles + return profiles def install_adblock(indent=8, width=32, just_firefox=False): - """Install adblock for all supported browsers.""" - for browser in sorted(browser_data): - if just_firefox and browser_data[browser]['base'] != 'mozilla': - continue - exe_path = browser_data[browser].get('exe_path', None) - function=run_program - if not exe_path: - if browser_data[browser]['profiles']: - print_standard( - '{indent}{browser:<{width}}'.format( - indent=' '*indent, width=width, browser=browser+'...'), - end='', flush=True) - print_warning('Profile(s) detected but browser not installed', - timestamp=False) - else: - # Only warn if profile(s) are detected. - pass + """Install adblock for all supported browsers.""" + for browser in sorted(browser_data): + if just_firefox and browser_data[browser]['base'] != 'mozilla': + continue + exe_path = browser_data[browser].get('exe_path', None) + function=run_program + if not exe_path: + if browser_data[browser]['profiles']: + print_standard( + '{indent}{browser:<{width}}'.format( + indent=' '*indent, width=width, browser=browser+'...'), + end='', flush=True) + print_warning('Profile(s) detected but browser not installed', + timestamp=False) + else: + # Only warn if profile(s) are detected. + pass + else: + # Set urls to open + urls = [] + if browser_data[browser]['base'] == 'chromium': + if browser == 'Google Chrome': + # Check for system exensions + try: + winreg.QueryValue(HKLM, UBO_CHROME_REG) + except FileNotFoundError: + urls.append(UBO_CHROME) + try: + winreg.QueryValue(HKLM, UBO_EXTRA_CHROME_REG) + except FileNotFoundError: + urls.append(UBO_EXTRA_CHROME) + + if len(urls) == 0: + urls = ['chrome://extensions'] + elif 'Opera' in browser: + urls.append(UBO_OPERA) else: - # Set urls to open - urls = [] - if browser_data[browser]['base'] == 'chromium': - if browser == 'Google Chrome': - # Check for system exensions - try: - winreg.QueryValue(HKLM, UBO_CHROME_REG) - except FileNotFoundError: - urls.append(UBO_CHROME) - try: - winreg.QueryValue(HKLM, UBO_EXTRA_CHROME_REG) - except FileNotFoundError: - urls.append(UBO_EXTRA_CHROME) + urls.append(UBO_CHROME) + urls.append(UBO_EXTRA_CHROME) - if len(urls) == 0: - urls = ['chrome://extensions'] - elif 'Opera' in browser: - urls.append(UBO_OPERA) - else: - urls.append(UBO_CHROME) - urls.append(UBO_EXTRA_CHROME) + elif browser_data[browser]['base'] == 'mozilla': + # Check for system extensions + try: + with winreg.OpenKey(HKLM, UBO_MOZILLA_REG) as key: + winreg.QueryValueEx(key, UBO_MOZILLA_REG_NAME) + except FileNotFoundError: + urls = [UBO_MOZILLA] + else: + if os.path.exists(UBO_MOZZILA_PATH): + urls = ['about:addons'] + else: + urls = [UBO_MOZILLA] - elif browser_data[browser]['base'] == 'mozilla': - # Check for system extensions - try: - with winreg.OpenKey(HKLM, UBO_MOZILLA_REG) as key: - winreg.QueryValueEx(key, UBO_MOZILLA_REG_NAME) - except FileNotFoundError: - urls = [UBO_MOZILLA] - else: - if os.path.exists(UBO_MOZZILA_PATH): - urls = ['about:addons'] - else: - urls = [UBO_MOZILLA] + elif browser_data[browser]['base'] == 'ie': + urls.append(IE_GALLERY) + function=popen_program - elif browser_data[browser]['base'] == 'ie': - urls.append(IE_GALLERY) - function=popen_program - - # By using check=False we're skipping any return codes so - # it should only fail if the program can't be run - # (or can't be found). - # In other words, this isn't tracking the addon/extension's - # installation status. - try_and_print(message='{}...'.format(browser), - indent=indent, width=width, - cs='Done', function=function, - cmd=[exe_path, *urls], check=False) + # By using check=False we're skipping any return codes so + # it should only fail if the program can't be run + # (or can't be found). + # In other words, this isn't tracking the addon/extension's + # installation status. + try_and_print(message='{}...'.format(browser), + indent=indent, width=width, + cs='Done', function=function, + cmd=[exe_path, *urls], check=False) def list_homepages(indent=8, width=32): - """List current homepages for reference.""" + """List current homepages for reference.""" + browser_list = [k for k, v in sorted(browser_data.items()) if v['exe_path']] + for browser in browser_list: + # Skip Chromium-based browsers + if browser_data[browser]['base'] == 'chromium': + print_info( + '{indent}{browser:<{width}}'.format( + indent=' '*indent, width=width, browser=browser+'...'), + end='', flush=True) + print_warning('Not implemented', timestamp=False) + continue - for browser in [k for k, v in sorted(browser_data.items()) if v['exe_path']]: - # Skip Chromium-based browsers - if browser_data[browser]['base'] == 'chromium': - print_info( - '{indent}{browser:<{width}}'.format( - indent=' '*indent, width=width, browser=browser+'...'), - end='', flush=True) - print_warning('Not implemented', timestamp=False) - continue - - # All other browsers - print_info('{indent}{browser:<{width}}'.format( - indent=' '*indent, width=width, browser=browser+'...')) - for profile in browser_data[browser].get('profiles', []): - name = profile.get('name', '?') - homepages = profile.get('homepages', []) - if len(homepages) == 0: - print_standard( - '{indent}{name:<{width}}'.format( - indent=' '*indent, width=width, name=name), - end='', flush=True) - print_warning('None found', timestamp=False) - else: - for page in homepages: - print_standard('{indent}{name:<{width}}{page}'.format( - indent=' '*indent, width=width, name=name, page=page)) + # All other browsers + print_info('{indent}{browser:<{width}}'.format( + indent=' '*indent, width=width, browser=browser+'...')) + for profile in browser_data[browser].get('profiles', []): + name = profile.get('name', '?') + homepages = profile.get('homepages', []) + if len(homepages) == 0: + print_standard( + '{indent}{name:<{width}}'.format( + indent=' '*indent, width=width, name=name), + end='', flush=True) + print_warning('None found', timestamp=False) + else: + for page in homepages: + print_standard('{indent}{name:<{width}}{page}'.format( + indent=' '*indent, width=width, name=name, page=page)) def reset_browsers(indent=8, width=32): - """Reset all detected browsers to safe defaults.""" - for browser in [k for k, v in sorted(browser_data.items()) if v['profiles']]: - print_info('{indent}{name}'.format(indent=' '*indent, name=browser)) - for profile in browser_data[browser]['profiles']: - if browser_data[browser]['base'] == 'chromium': - function = clean_chromium_profile - elif browser_data[browser]['base'] == 'ie': - function = clean_internet_explorer - elif browser_data[browser]['base'] == 'mozilla': - function = clean_mozilla_profile - try_and_print( - message='{}...'.format(profile['name']), - indent=indent, width=width, function=function, - other_results=other_results, profile=profile) + """Reset all detected browsers to safe defaults.""" + browser_list = [k for k, v in sorted(browser_data.items()) if v['profiles']] + for browser in browser_list: + print_info('{indent}{name}'.format(indent=' '*indent, name=browser)) + for profile in browser_data[browser]['profiles']: + if browser_data[browser]['base'] == 'chromium': + function = clean_chromium_profile + elif browser_data[browser]['base'] == 'ie': + function = clean_internet_explorer + elif browser_data[browser]['base'] == 'mozilla': + function = clean_mozilla_profile + try_and_print( + message='{}...'.format(profile['name']), + indent=indent, width=width, function=function, + other_results=other_results, profile=profile) def scan_for_browsers(just_firefox=False): - """Scan system for any supported browsers.""" - for name, details in sorted(SUPPORTED_BROWSERS.items()): - if just_firefox and details['base'] != 'mozilla': - continue - try_and_print(message='{}...'.format(name), - function=get_browser_details, cs='Detected', - other_results=other_results, name=name) + """Scan system for any supported browsers.""" + for name, details in sorted(SUPPORTED_BROWSERS.items()): + if just_firefox and details['base'] != 'mozilla': + continue + try_and_print(message='{}...'.format(name), + function=get_browser_details, cs='Detected', + other_results=other_results, name=name) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 327c5b8a3393e7d1960c3f33b943e5a4d2114be9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:43:25 -0700 Subject: [PATCH 114/186] Updated cleanup.py --- .bin/Scripts/functions/cleanup.py | 196 +++++++++++++++--------------- 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/.bin/Scripts/functions/cleanup.py b/.bin/Scripts/functions/cleanup.py index d401b555..80ffd879 100644 --- a/.bin/Scripts/functions/cleanup.py +++ b/.bin/Scripts/functions/cleanup.py @@ -3,127 +3,127 @@ from functions.common import * def cleanup_adwcleaner(): - """Move AdwCleaner folders into the ClientDir.""" - source_path = r'{SYSTEMDRIVE}\AdwCleaner'.format(**global_vars['Env']) - source_quarantine = r'{}\Quarantine'.format(source_path) + """Move AdwCleaner folders into the ClientDir.""" + source_path = r'{SYSTEMDRIVE}\AdwCleaner'.format(**global_vars['Env']) + source_quarantine = r'{}\Quarantine'.format(source_path) - # Quarantine - if os.path.exists(source_quarantine): - os.makedirs(global_vars['QuarantineDir'], exist_ok=True) - dest_name = r'{QuarantineDir}\AdwCleaner_{Date-Time}'.format( - **global_vars) - dest_name = non_clobber_rename(dest_name) - shutil.move(source_quarantine, dest_name) + # Quarantine + if os.path.exists(source_quarantine): + os.makedirs(global_vars['QuarantineDir'], exist_ok=True) + dest_name = r'{QuarantineDir}\AdwCleaner_{Date-Time}'.format( + **global_vars) + dest_name = non_clobber_rename(dest_name) + shutil.move(source_quarantine, dest_name) - # Delete source folder if empty - delete_empty_folders(source_path) + # Delete source folder if empty + delete_empty_folders(source_path) - # Main folder - if os.path.exists(source_path): - os.makedirs(global_vars['LogDir'], exist_ok=True) - dest_name = r'{LogDir}\Tools\AdwCleaner'.format( - **global_vars) - dest_name = non_clobber_rename(dest_name) - shutil.move(source_path, dest_name) + # Main folder + if os.path.exists(source_path): + os.makedirs(global_vars['LogDir'], exist_ok=True) + dest_name = r'{LogDir}\Tools\AdwCleaner'.format( + **global_vars) + dest_name = non_clobber_rename(dest_name) + shutil.move(source_path, dest_name) def cleanup_cbs(dest_folder): - """Safely cleanup a known CBS archive bug under Windows 7. + """Safely cleanup a known CBS archive bug under Windows 7. - If a CbsPersist file is larger than 2 Gb then the auto archive feature - continually fails and will fill up the system drive with temp files. + If a CbsPersist file is larger than 2 Gb then the auto archive feature + continually fails and will fill up the system drive with temp files. - This function moves the temp files and CbsPersist file to a temp folder, - compresses the CbsPersist files with 7-Zip, and then opens the temp folder - for the user to manually save the backup files and delete the temp files. - """ - backup_folder = r'{dest_folder}\CbsFix'.format(dest_folder=dest_folder) - temp_folder = r'{backup_folder}\Temp'.format(backup_folder=backup_folder) - os.makedirs(backup_folder, exist_ok=True) - os.makedirs(temp_folder, exist_ok=True) + This function moves the temp files and CbsPersist file to a temp folder, + compresses the CbsPersist files with 7-Zip, and then opens the temp folder + for the user to manually save the backup files and delete the temp files. + """ + backup_folder = r'{dest_folder}\CbsFix'.format(dest_folder=dest_folder) + temp_folder = r'{backup_folder}\Temp'.format(backup_folder=backup_folder) + os.makedirs(backup_folder, exist_ok=True) + os.makedirs(temp_folder, exist_ok=True) - # Move files into temp folder - cbs_path = r'{SYSTEMROOT}\Logs\CBS'.format(**global_vars['Env']) - for entry in os.scandir(cbs_path): - # CbsPersist files - if entry.name.lower().startswith('cbspersist'): - dest_name = r'{}\{}'.format(temp_folder, entry.name) - dest_name = non_clobber_rename(dest_name) - shutil.move(entry.path, dest_name) - temp_path = r'{SYSTEMROOT}\Temp'.format(**global_vars['Env']) - for entry in os.scandir(temp_path): - # cab_ files - if entry.name.lower().startswith('cab_'): - dest_name = r'{}\{}'.format(temp_folder, entry.name) - dest_name = non_clobber_rename(dest_name) - shutil.move(entry.path, dest_name) + # Move files into temp folder + cbs_path = r'{SYSTEMROOT}\Logs\CBS'.format(**global_vars['Env']) + for entry in os.scandir(cbs_path): + # CbsPersist files + if entry.name.lower().startswith('cbspersist'): + dest_name = r'{}\{}'.format(temp_folder, entry.name) + dest_name = non_clobber_rename(dest_name) + shutil.move(entry.path, dest_name) + temp_path = r'{SYSTEMROOT}\Temp'.format(**global_vars['Env']) + for entry in os.scandir(temp_path): + # cab_ files + if entry.name.lower().startswith('cab_'): + dest_name = r'{}\{}'.format(temp_folder, entry.name) + dest_name = non_clobber_rename(dest_name) + shutil.move(entry.path, dest_name) - # Compress CbsPersist files with 7-Zip - cmd = [ - global_vars['Tools']['SevenZip'], - 'a', '-t7z', '-mx=3', '-bso0', '-bse0', - r'{}\CbsPersists.7z'.format(backup_folder), - r'{}\CbsPersist*'.format(temp_folder)] - run_program(cmd) + # Compress CbsPersist files with 7-Zip + cmd = [ + global_vars['Tools']['SevenZip'], + 'a', '-t7z', '-mx=3', '-bso0', '-bse0', + r'{}\CbsPersists.7z'.format(backup_folder), + r'{}\CbsPersist*'.format(temp_folder)] + run_program(cmd) def cleanup_desktop(): - """Move known backup files and reports into the ClientDir.""" - dest_folder = r'{LogDir}\Tools'.format(**global_vars) - os.makedirs(dest_folder, exist_ok=True) + """Move known backup files and reports into the ClientDir.""" + dest_folder = r'{LogDir}\Tools'.format(**global_vars) + os.makedirs(dest_folder, exist_ok=True) - desktop_path = r'{USERPROFILE}\Desktop'.format(**global_vars['Env']) - for entry in os.scandir(desktop_path): - # JRT, RKill, Shortcut cleaner - if re.search(r'^(JRT|RKill|sc-cleaner)', entry.name, re.IGNORECASE): - dest_name = r'{}\{}'.format(dest_folder, entry.name) - dest_name = non_clobber_rename(dest_name) - shutil.move(entry.path, dest_name) + desktop_path = r'{USERPROFILE}\Desktop'.format(**global_vars['Env']) + for entry in os.scandir(desktop_path): + # JRT, RKill, Shortcut cleaner + if re.search(r'^(JRT|RKill|sc-cleaner)', entry.name, re.IGNORECASE): + dest_name = r'{}\{}'.format(dest_folder, entry.name) + dest_name = non_clobber_rename(dest_name) + shutil.move(entry.path, dest_name) - # Remove dir if empty - delete_empty_folders(dest_folder) + # Remove dir if empty + delete_empty_folders(dest_folder) def delete_empty_folders(folder_path): - """Delete all empty folders in path (depth first).""" - if not os.path.exists(folder_path) or not os.path.isdir(folder_path): - # Bail early (silently) - return + """Delete all empty folders in path (depth first).""" + if not os.path.exists(folder_path) or not os.path.isdir(folder_path): + # Bail early (silently) + return - # Delete empty subfolders first - for item in os.scandir(folder_path): - if item.is_dir(): - delete_empty_folders(item.path) + # Delete empty subfolders first + for item in os.scandir(folder_path): + if item.is_dir(): + delete_empty_folders(item.path) - # Remove top folder - try: - os.rmdir(folder_path) - except OSError: - pass + # Remove top folder + try: + os.rmdir(folder_path) + except OSError: + pass def delete_registry_key(hive, key, recurse=False): - """Delete a registry key and all it's subkeys.""" - access = winreg.KEY_ALL_ACCESS + """Delete a registry key and all it's subkeys.""" + access = winreg.KEY_ALL_ACCESS - try: - if recurse: - # Delete all subkeys first - with winreg.OpenKeyEx(hive, key, 0, access) as k: - key_info = winreg.QueryInfoKey(k) - for x in range(key_info[0]): - subkey = r'{}\{}'.format(key, winreg.EnumKey(k, 0)) - delete_registry_key(hive, subkey) + try: + if recurse: + # Delete all subkeys first + with winreg.OpenKeyEx(hive, key, 0, access) as k: + key_info = winreg.QueryInfoKey(k) + for x in range(key_info[0]): + subkey = r'{}\{}'.format(key, winreg.EnumKey(k, 0)) + delete_registry_key(hive, subkey) - # Delete key - winreg.DeleteKey(hive, key) - except FileNotFoundError: - # Ignore - pass + # Delete key + winreg.DeleteKey(hive, key) + except FileNotFoundError: + # Ignore + pass def delete_registry_value(hive, key, value): - """Delete a registry value.""" - access = winreg.KEY_ALL_ACCESS - with winreg.OpenKeyEx(hive, key, 0, access) as k: - winreg.DeleteValue(k, value) + """Delete a registry value.""" + access = winreg.KEY_ALL_ACCESS + with winreg.OpenKeyEx(hive, key, 0, access) as k: + winreg.DeleteValue(k, value) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 From c74e2c766717e02468bc590ae932c7f2c24ef67c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:43:37 -0700 Subject: [PATCH 115/186] Updated common.py --- .bin/Scripts/functions/common.py | 1272 +++++++++++++++--------------- 1 file changed, 642 insertions(+), 630 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 82c65bb1..24f8c1d5 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -9,10 +9,10 @@ import sys import time import traceback try: - import winreg + import winreg except ModuleNotFoundError: - if psutil.WINDOWS: - raise + if psutil.WINDOWS: + raise from subprocess import CalledProcessError @@ -25,809 +25,821 @@ global_vars = {} # STATIC VARIABLES COLORS = { - 'CLEAR': '\033[0m', - 'RED': '\033[31m', - 'ORANGE': '\033[31;1m', - 'GREEN': '\033[32m', - 'YELLOW': '\033[33m', - 'BLUE': '\033[34m', - 'CYAN': '\033[36m', + 'CLEAR': '\033[0m', + 'RED': '\033[31m', + 'ORANGE': '\033[31;1m', + 'GREEN': '\033[32m', + 'YELLOW': '\033[33m', + 'BLUE': '\033[34m', + 'CYAN': '\033[36m', } try: - HKU = winreg.HKEY_USERS - HKCR = winreg.HKEY_CLASSES_ROOT - HKCU = winreg.HKEY_CURRENT_USER - HKLM = winreg.HKEY_LOCAL_MACHINE + HKU = winreg.HKEY_USERS + HKCR = winreg.HKEY_CLASSES_ROOT + HKCU = winreg.HKEY_CURRENT_USER + HKLM = winreg.HKEY_LOCAL_MACHINE except NameError: - if psutil.WINDOWS: - raise + if psutil.WINDOWS: + raise # Error Classes class BIOSKeyNotFoundError(Exception): - pass + pass class BinNotFoundError(Exception): - pass + pass class GenericAbort(Exception): - pass + pass class GenericError(Exception): - pass + pass class GenericRepair(Exception): - pass + pass class MultipleInstallationsError(Exception): - pass + pass class NotInstalledError(Exception): - pass + pass class NoProfilesError(Exception): - pass + pass class OSInstalledLegacyError(Exception): - pass + pass class PathNotFoundError(Exception): - pass + pass class UnsupportedOSError(Exception): - pass + pass class SecureBootDisabledError(Exception): - pass + pass class SecureBootNotAvailError(Exception): - pass + pass class SecureBootUnknownError(Exception): - pass + pass # General functions def abort(): - """Abort script.""" - print_warning('Aborted.') - sleep(1) - pause(prompt='Press Enter to exit... ') - exit_script() + """Abort script.""" + print_warning('Aborted.') + sleep(1) + pause(prompt='Press Enter to exit... ') + exit_script() def ask(prompt='Kotaero!'): - """Prompt the user with a Y/N question, log answer, and return a bool.""" - answer = None - prompt = '{} [Y/N]: '.format(prompt) - while answer is None: - tmp = input(prompt) - if re.search(r'^y(es|)$', tmp, re.IGNORECASE): - answer = True - elif re.search(r'^n(o|ope|)$', tmp, re.IGNORECASE): - answer = False - message = '{prompt}{answer_text}'.format( - prompt = prompt, - answer_text = 'Yes' if answer else 'No') - print_log(message=message) - return answer + """Prompt the user with a Y/N question, log answer, and return a bool.""" + answer = None + prompt = '{} [Y/N]: '.format(prompt) + while answer is None: + tmp = input(prompt) + if re.search(r'^y(es|)$', tmp, re.IGNORECASE): + answer = True + elif re.search(r'^n(o|ope|)$', tmp, re.IGNORECASE): + answer = False + message = '{prompt}{answer_text}'.format( + prompt = prompt, + answer_text = 'Yes' if answer else 'No') + print_log(message=message) + return answer def choice(choices, prompt='Kotaero!'): - """Prompt the user with a choice question, log answer, and returns str.""" - answer = None - choices = [str(c) for c in choices] - choices_short = {c[:1].upper(): c for c in choices} - prompt = '{} [{}]: '.format(prompt, '/'.join(choices)) - regex = '^({}|{})$'.format( - '|'.join([c[:1] for c in choices]), - '|'.join(choices)) + """Prompt the user with a choice question, log answer, and returns str.""" + answer = None + choices = [str(c) for c in choices] + choices_short = {c[:1].upper(): c for c in choices} + prompt = '{} [{}]: '.format(prompt, '/'.join(choices)) + regex = '^({}|{})$'.format( + '|'.join([c[:1] for c in choices]), + '|'.join(choices)) - # Get user's choice - while answer is None: - tmp = input(prompt) - if re.search(regex, tmp, re.IGNORECASE): - answer = tmp + # Get user's choice + while answer is None: + tmp = input(prompt) + if re.search(regex, tmp, re.IGNORECASE): + answer = tmp - # Log result - message = '{prompt}{answer_text}'.format( - prompt = prompt, - answer_text = 'Yes' if answer else 'No') - print_log(message=message) + # Log result + message = '{prompt}{answer_text}'.format( + prompt = prompt, + answer_text = 'Yes' if answer else 'No') + print_log(message=message) - # Fix answer formatting to match provided values - answer = choices_short[answer[:1].upper()] + # Fix answer formatting to match provided values + answer = choices_short[answer[:1].upper()] - # Done - return answer + # Done + return answer def clear_screen(): - """Simple wrapper for cls/clear.""" - if psutil.WINDOWS: - os.system('cls') - else: - os.system('clear') + """Simple wrapper for cls/clear.""" + if psutil.WINDOWS: + os.system('cls') + else: + os.system('clear') def convert_to_bytes(size): - """Convert human-readable size str to bytes and return an int.""" - size = str(size) - tmp = re.search(r'(\d+\.?\d*)\s+([KMGT]B)', size.upper()) - if tmp: - size = float(tmp.group(1)) - units = tmp.group(2) - if units == 'TB': - size *= 1099511627776 - elif units == 'GB': - size *= 1073741824 - elif units == 'MB': - size *= 1048576 - elif units == 'KB': - size *= 1024 - size = int(size) - else: - return -1 + """Convert human-readable size str to bytes and return an int.""" + size = str(size) + tmp = re.search(r'(\d+\.?\d*)\s+([KMGT]B)', size.upper()) + if tmp: + size = float(tmp.group(1)) + units = tmp.group(2) + if units == 'TB': + size *= 1099511627776 + elif units == 'GB': + size *= 1073741824 + elif units == 'MB': + size *= 1048576 + elif units == 'KB': + size *= 1024 + size = int(size) + else: + return -1 - return size + return size def exit_script(return_value=0): - """Exits the script after some cleanup and opens the log (if set).""" - # Remove dirs (if empty) - for dir in ['BackupDir', 'LogDir', 'TmpDir']: - try: - os.rmdir(global_vars[dir]) - except Exception: - pass + """Exits the script after some cleanup and opens the log (if set).""" + # Remove dirs (if empty) + for dir in ['BackupDir', 'LogDir', 'TmpDir']: + try: + os.rmdir(global_vars[dir]) + except Exception: + pass - # Open Log (if it exists) - log = global_vars.get('LogFile', '') - if log and os.path.exists(log) and psutil.WINDOWS and ENABLED_OPEN_LOGS: - try: - extract_item('NotepadPlusPlus', silent=True) - popen_program( - [global_vars['Tools']['NotepadPlusPlus'], - global_vars['LogFile']]) - except Exception: - print_error('ERROR: Failed to extract Notepad++ and open log.') - pause('Press Enter to exit...') + # Open Log (if it exists) + log = global_vars.get('LogFile', '') + if log and os.path.exists(log) and psutil.WINDOWS and ENABLED_OPEN_LOGS: + try: + extract_item('NotepadPlusPlus', silent=True) + popen_program( + [global_vars['Tools']['NotepadPlusPlus'], + global_vars['LogFile']]) + except Exception: + print_error('ERROR: Failed to extract Notepad++ and open log.') + pause('Press Enter to exit...') - # Kill Caffeine if still running - kill_process('caffeine.exe') + # Kill Caffeine if still running + kill_process('caffeine.exe') - # Exit - sys.exit(return_value) + # Exit + sys.exit(return_value) def extract_item(item, filter='', silent=False): - """Extract item from .cbin into .bin.""" - cmd = [ - global_vars['Tools']['SevenZip'], 'x', '-aos', '-bso0', '-bse0', - '-p{ArchivePassword}'.format(**global_vars), - r'-o{BinDir}\{item}'.format(item=item, **global_vars), - r'{CBinDir}\{item}.7z'.format(item=item, **global_vars), - filter] + """Extract item from .cbin into .bin.""" + cmd = [ + global_vars['Tools']['SevenZip'], 'x', '-aos', '-bso0', '-bse0', + '-p{ArchivePassword}'.format(**global_vars), + r'-o{BinDir}\{item}'.format(item=item, **global_vars), + r'{CBinDir}\{item}.7z'.format(item=item, **global_vars), + filter] + if not silent: + print_standard('Extracting "{item}"...'.format(item=item)) + try: + run_program(cmd) + except FileNotFoundError: if not silent: - print_standard('Extracting "{item}"...'.format(item=item)) - try: - run_program(cmd) - except FileNotFoundError: - if not silent: - print_warning('WARNING: Archive not found') - except subprocess.CalledProcessError: - if not silent: - print_warning('WARNING: Errors encountered while exctracting data') + print_warning('WARNING: Archive not found') + except subprocess.CalledProcessError: + if not silent: + print_warning('WARNING: Errors encountered while exctracting data') def get_process(name=None): - """Get process by name, returns psutil.Process obj.""" - proc = None - if not name: - raise GenericError + """Get process by name, returns psutil.Process obj.""" + proc = None + if not name: + raise GenericError - for p in psutil.process_iter(): - try: - if p.name() == name: - proc = p - except psutil._exceptions.NoSuchProcess: - # Process finished during iteration? Going to ignore - pass - return proc + for p in psutil.process_iter(): + try: + if p.name() == name: + proc = p + except psutil._exceptions.NoSuchProcess: + # Process finished during iteration? Going to ignore + pass + return proc def get_simple_string(prompt='Enter string'): - """Get string from user (minimal allowed character set) and return as str.""" - simple_string = None - while simple_string is None: - _input = input('{}: '.format(prompt)) - if re.match(r"^(\w|-| |\.|')+$", _input, re.ASCII): - simple_string = _input.strip() - return simple_string + """Get string from user (restricted character set), returns str.""" + simple_string = None + while simple_string is None: + _input = input('{}: '.format(prompt)) + if re.match(r"^(\w|-| |\.|')+$", _input, re.ASCII): + simple_string = _input.strip() + return simple_string def get_ticket_number(): - """Get TicketNumber from user, save in LogDir, and return as str.""" - if not ENABLED_TICKET_NUMBERS: - return None - ticket_number = None - while ticket_number is None: - _input = input('Enter ticket number: ') - if re.match(r'^([0-9]+([-_]?\w+|))$', _input): - ticket_number = _input - out_file = r'{}\TicketNumber'.format(global_vars['LogDir']) - if not psutil.WINDOWS: - out_file = out_file.replace('\\', '/') - with open(out_file, 'w', encoding='utf-8') as f: - f.write(ticket_number) - return ticket_number + """Get TicketNumber from user, save in LogDir, and return as str.""" + if not ENABLED_TICKET_NUMBERS: + return None + ticket_number = None + while ticket_number is None: + _input = input('Enter ticket number: ') + if re.match(r'^([0-9]+([-_]?\w+|))$', _input): + ticket_number = _input + out_file = r'{}\TicketNumber'.format(global_vars['LogDir']) + if not psutil.WINDOWS: + out_file = out_file.replace('\\', '/') + with open(out_file, 'w', encoding='utf-8') as f: + f.write(ticket_number) + return ticket_number def human_readable_size(size, decimals=0): - """Convert size in bytes to a human-readable format and return a str.""" - # Prep string formatting - width = 3+decimals - if decimals > 0: - width += 1 + """Convert size in bytes to a human-readable format and return a str.""" + # Prep string formatting + width = 3+decimals + if decimals > 0: + width += 1 - # Convert size to int - try: - size = int(size) - except ValueError: - size = convert_to_bytes(size) - except TypeError: - size = -1 + # Convert size to int + try: + size = int(size) + except ValueError: + size = convert_to_bytes(size) + except TypeError: + size = -1 - # Verify we have a valid size - if size < 0: - return '{size:>{width}} b'.format(size='???', width=width) + # Verify we have a valid size + if size < 0: + return '{size:>{width}} b'.format(size='???', width=width) - # Convert to sensible units - if size >= 1099511627776: - size /= 1099511627776 - units = 'Tb' - elif size >= 1073741824: - size /= 1073741824 - units = 'Gb' - elif size >= 1048576: - size /= 1048576 - units = 'Mb' - elif size >= 1024: - size /= 1024 - units = 'Kb' - else: - units = ' b' + # Convert to sensible units + if size >= 1099511627776: + size /= 1099511627776 + units = 'Tb' + elif size >= 1073741824: + size /= 1073741824 + units = 'Gb' + elif size >= 1048576: + size /= 1048576 + units = 'Mb' + elif size >= 1024: + size /= 1024 + units = 'Kb' + else: + units = ' b' - # Return - return '{size:>{width}.{decimals}f} {units}'.format( - size=size, width=width, decimals=decimals, units=units) + # Return + return '{size:>{width}.{decimals}f} {units}'.format( + size=size, width=width, decimals=decimals, units=units) def kill_process(name): - """Kill any running caffeine.exe processes.""" - for proc in psutil.process_iter(): - if proc.name() == name: - proc.kill() + """Kill any running caffeine.exe processes.""" + for proc in psutil.process_iter(): + if proc.name() == name: + proc.kill() def major_exception(): - """Display traceback and exit""" - print_error('Major exception') - print_warning(SUPPORT_MESSAGE) - print(traceback.format_exc()) - print_log(traceback.format_exc()) - try: - upload_crash_details() - except GenericAbort: - # User declined upload - print_warning('Upload: Aborted') - sleep(10) - except GenericError: - # No log file or uploading disabled - sleep(10) - except: - print_error('Upload: NS') - sleep(10) - else: - print_success('Upload: CS') - pause('Press Enter to exit...') - exit_script(1) + """Display traceback and exit""" + print_error('Major exception') + print_warning(SUPPORT_MESSAGE) + print(traceback.format_exc()) + print_log(traceback.format_exc()) + try: + upload_crash_details() + except GenericAbort: + # User declined upload + print_warning('Upload: Aborted') + sleep(10) + except GenericError: + # No log file or uploading disabled + sleep(10) + except: + print_error('Upload: NS') + sleep(10) + else: + print_success('Upload: CS') + pause('Press Enter to exit...') + exit_script(1) -def menu_select(title='~ Untitled Menu ~', +def menu_select( + title='[Untitled Menu]', prompt='Please make a selection', secret_actions=[], secret_exit=False, main_entries=[], action_entries=[], disabled_label='DISABLED', spacer=''): - """Display options in a menu and return selected option as a str.""" - # Bail early - if not main_entries and not action_entries: - raise Exception("MenuError: No items given") + """Display options in a menu and return selected option as a str.""" + # Bail early + if not main_entries and not action_entries: + raise Exception("MenuError: No items given") - # Set title - if 'Title' in global_vars: - title = '{}\n\n{}'.format(global_vars['Title'], title) + # Set title + if 'Title' in global_vars: + title = '{}\n\n{}'.format(global_vars['Title'], title) - # Build menu - menu_splash = '{}\n{}\n'.format(title, spacer) - width = len(str(len(main_entries))) - valid_answers = [] - if secret_exit: - valid_answers.append('Q') - if secret_actions: - valid_answers.extend(secret_actions) + # Build menu + menu_splash = '{}\n{}\n'.format(title, spacer) + width = len(str(len(main_entries))) + valid_answers = [] + if secret_exit: + valid_answers.append('Q') + if secret_actions: + valid_answers.extend(secret_actions) - # Add main entries - for i in range(len(main_entries)): - entry = main_entries[i] - # Add Spacer - if ('CRLF' in entry): - menu_splash += '{}\n'.format(spacer) - entry_str = '{number:>{width}}: {name}'.format( - number = i+1, - width = width, - name = entry.get('Display Name', entry['Name'])) - if entry.get('Disabled', False): - entry_str = '{YELLOW}{entry_str} ({disabled}){CLEAR}'.format( - entry_str = entry_str, - disabled = disabled_label, - **COLORS) - else: - valid_answers.append(str(i+1)) - menu_splash += '{}\n'.format(entry_str) - menu_splash += '{}\n'.format(spacer) + # Add main entries + for i in range(len(main_entries)): + entry = main_entries[i] + # Add Spacer + if ('CRLF' in entry): + menu_splash += '{}\n'.format(spacer) + entry_str = '{number:>{width}}: {name}'.format( + number = i+1, + width = width, + name = entry.get('Display Name', entry['Name'])) + if entry.get('Disabled', False): + entry_str = '{YELLOW}{entry_str} ({disabled}){CLEAR}'.format( + entry_str = entry_str, + disabled = disabled_label, + **COLORS) + else: + valid_answers.append(str(i+1)) + menu_splash += '{}\n'.format(entry_str) + menu_splash += '{}\n'.format(spacer) - # Add action entries - for entry in action_entries: - # Add Spacer - if ('CRLF' in entry): - menu_splash += '{}\n'.format(spacer) - valid_answers.append(entry['Letter']) - menu_splash += '{letter:>{width}}: {name}\n'.format( - letter = entry['Letter'].upper(), - width = len(str(len(action_entries))), - name = entry['Name']) + # Add action entries + for entry in action_entries: + # Add Spacer + if ('CRLF' in entry): + menu_splash += '{}\n'.format(spacer) + valid_answers.append(entry['Letter']) + menu_splash += '{letter:>{width}}: {name}\n'.format( + letter = entry['Letter'].upper(), + width = len(str(len(action_entries))), + name = entry['Name']) - answer = '' + answer = '' - while (answer.upper() not in valid_answers): - clear_screen() - print(menu_splash) - answer = input('{}: '.format(prompt)) + while (answer.upper() not in valid_answers): + clear_screen() + print(menu_splash) + answer = input('{}: '.format(prompt)) - return answer.upper() + return answer.upper() def non_clobber_rename(full_path): - """Append suffix to path, if necessary, to avoid clobbering path""" - new_path = full_path - _i = 1; - while os.path.exists(new_path): - new_path = '{path}_{i}'.format(i=_i, path=full_path) - _i += 1 + """Append suffix to path, if necessary, to avoid clobbering path""" + new_path = full_path + _i = 1; + while os.path.exists(new_path): + new_path = '{path}_{i}'.format(i=_i, path=full_path) + _i += 1 - return new_path + return new_path def pause(prompt='Press Enter to continue... '): - """Simple pause implementation.""" - input(prompt) + """Simple pause implementation.""" + input(prompt) def ping(addr='google.com'): - """Attempt to ping addr.""" - cmd = [ - 'ping', - '-n' if psutil.WINDOWS else '-c', - '2', - addr] - run_program(cmd) + """Attempt to ping addr.""" + cmd = [ + 'ping', + '-n' if psutil.WINDOWS else '-c', + '2', + addr] + run_program(cmd) def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): - """Run program and return a subprocess.Popen object.""" - cmd_kwargs = {'args': cmd, 'shell': shell} + """Run program and return a subprocess.Popen object.""" + cmd_kwargs = {'args': cmd, 'shell': shell} - if minimized: - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = 6 - cmd_kwargs['startupinfo'] = startupinfo + if minimized: + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = 6 + cmd_kwargs['startupinfo'] = startupinfo - if pipe: - cmd_kwargs.update({ - 'stdout': subprocess.PIPE, - 'stderr': subprocess.PIPE, - }) + if pipe: + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) - if 'cwd' in kwargs: - cmd_kwargs['cwd'] = kwargs['cwd'] + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] - return subprocess.Popen(**cmd_kwargs) + return subprocess.Popen(**cmd_kwargs) def print_error(*args, **kwargs): - """Prints message to screen in RED.""" - print_standard(*args, color=COLORS['RED'], **kwargs) + """Prints message to screen in RED.""" + print_standard(*args, color=COLORS['RED'], **kwargs) def print_info(*args, **kwargs): - """Prints message to screen in BLUE.""" - print_standard(*args, color=COLORS['BLUE'], **kwargs) + """Prints message to screen in BLUE.""" + print_standard(*args, color=COLORS['BLUE'], **kwargs) def print_standard(message='Generic info', - color=None, end='\n', timestamp=True, **kwargs): - """Prints message to screen and log (if set).""" - display_message = message - if color: - display_message = color + message + COLORS['CLEAR'] - # **COLORS is used below to support non-"standard" color printing - print(display_message.format(**COLORS), end=end, **kwargs) - print_log(message, end, timestamp) + color=None, end='\n', timestamp=True, **kwargs): + """Prints message to screen and log (if set).""" + display_message = message + if color: + display_message = color + message + COLORS['CLEAR'] + # **COLORS is used below to support non-"standard" color printing + print(display_message.format(**COLORS), end=end, **kwargs) + print_log(message, end, timestamp) def print_success(*args, **kwargs): - """Prints message to screen in GREEN.""" - print_standard(*args, color=COLORS['GREEN'], **kwargs) + """Prints message to screen in GREEN.""" + print_standard(*args, color=COLORS['GREEN'], **kwargs) def print_warning(*args, **kwargs): - """Prints message to screen in YELLOW.""" - print_standard(*args, color=COLORS['YELLOW'], **kwargs) + """Prints message to screen in YELLOW.""" + print_standard(*args, color=COLORS['YELLOW'], **kwargs) def print_log(message='', end='\n', timestamp=True): - """Writes message to a log if LogFile is set.""" - time_str = time.strftime("%Y-%m-%d %H%M%z: ") if timestamp else '' - if 'LogFile' in global_vars and global_vars['LogFile']: - with open(global_vars['LogFile'], 'a', encoding='utf-8') as f: - for line in message.splitlines(): - f.write('{timestamp}{line}{end}'.format( - timestamp = time_str, - line = line, - end = end)) + """Writes message to a log if LogFile is set.""" + time_str = time.strftime("%Y-%m-%d %H%M%z: ") if timestamp else '' + if 'LogFile' in global_vars and global_vars['LogFile']: + with open(global_vars['LogFile'], 'a', encoding='utf-8') as f: + for line in message.splitlines(): + f.write('{timestamp}{line}{end}'.format( + timestamp = time_str, + line = line, + end = end)) def run_program(cmd, args=[], check=True, pipe=True, shell=False, **kwargs): - """Run program and return a subprocess.CompletedProcess object.""" - if args: - # Deprecated so let's raise an exception to find & fix all occurances - print_error('ERROR: Using args is no longer supported.') - raise Exception - cmd = [c for c in cmd if c] - if shell: - cmd = ' '.join(cmd) + """Run program and return a subprocess.CompletedProcess object.""" + if args: + # Deprecated so let's raise an exception to find & fix all occurances + print_error('ERROR: Using args is no longer supported.') + raise Exception + cmd = [c for c in cmd if c] + if shell: + cmd = ' '.join(cmd) - cmd_kwargs = {'args': cmd, 'check': check, 'shell': shell} + cmd_kwargs = {'args': cmd, 'check': check, 'shell': shell} - if pipe: - cmd_kwargs.update({ - 'stdout': subprocess.PIPE, - 'stderr': subprocess.PIPE, - }) + if pipe: + cmd_kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) - if 'cwd' in kwargs: - cmd_kwargs['cwd'] = kwargs['cwd'] + if 'cwd' in kwargs: + cmd_kwargs['cwd'] = kwargs['cwd'] - return subprocess.run(**cmd_kwargs) + return subprocess.run(**cmd_kwargs) -def set_title(title='~Some Title~'): - """Set title. +def set_title(title='[Some Title]'): + """Set title. - Used for window title and menu titles.""" - global_vars['Title'] = title - os.system('title {}'.format(title)) + Used for window title and menu titles.""" + global_vars['Title'] = title + os.system('title {}'.format(title)) -def show_data(message='~Some message~', data='~Some data~', indent=8, width=32, +def show_data( + message='[Some message]', data='[Some data]', + indent=8, width=32, info=False, warning=False, error=False): - """Display info with formatting.""" - message = '{indent}{message:<{width}}{data}'.format( - indent=' '*indent, width=width, message=message, data=data) - if error: - print_error(message) - elif warning: - print_warning(message) - elif info: - print_info(message) - else: - print_standard(message) + """Display info with formatting.""" + message = '{indent}{message:<{width}}{data}'.format( + indent=' '*indent, width=width, message=message, data=data) + if error: + print_error(message) + elif warning: + print_warning(message) + elif info: + print_info(message) + else: + print_standard(message) def sleep(seconds=2): - """Wait for a while.""" - time.sleep(seconds) + """Wait for a while.""" + time.sleep(seconds) def stay_awake(): - """Prevent the system from sleeping or hibernating.""" - # DISABLED due to VCR2008 dependency - return - # Bail if caffeine is already running - for proc in psutil.process_iter(): - if proc.name() == 'caffeine.exe': - return - # Extract and run - extract_item('Caffeine', silent=True) - try: - popen_program([global_vars['Tools']['Caffeine']]) - except Exception: - print_error('ERROR: No caffeine available.') - print_warning('Please set the power setting to High Performance.') + """Prevent the system from sleeping or hibernating.""" + # DISABLED due to VCR2008 dependency + return + # Bail if caffeine is already running + for proc in psutil.process_iter(): + if proc.name() == 'caffeine.exe': + return + # Extract and run + extract_item('Caffeine', silent=True) + try: + popen_program([global_vars['Tools']['Caffeine']]) + except Exception: + print_error('ERROR: No caffeine available.') + print_warning('Please set the power setting to High Performance.') def strip_colors(s): - """Remove all ASCII color escapes from string, returns str.""" - for c in COLORS.values(): - s = s.replace(c, '') - return s + """Remove all ASCII color escapes from string, returns str.""" + for c in COLORS.values(): + s = s.replace(c, '') + return s def get_exception(s): - """Get exception by name, returns Exception object.""" - try: - obj = getattr(sys.modules[__name__], s) - except AttributeError: - # Try builtin classes - obj = getattr(sys.modules['builtins'], s) - return obj + """Get exception by name, returns Exception object.""" + try: + obj = getattr(sys.modules[__name__], s) + except AttributeError: + # Try builtin classes + obj = getattr(sys.modules['builtins'], s) + return obj def try_and_print(message='Trying...', - function=None, cs='CS', ns='NS', other_results={}, - catch_all=True, print_return=False, silent_function=True, - indent=8, width=32, *args, **kwargs): - """Run function, print if successful or not, and return dict. + function=None, cs='CS', ns='NS', other_results={}, + catch_all=True, print_return=False, silent_function=True, + indent=8, width=32, *args, **kwargs): + """Run function, print if successful or not, and return dict. - other_results is in the form of - { - 'Warning': {'ExceptionClassName': 'Result Message'}, - 'Error': {'ExceptionClassName': 'Result Message'} - } - The the ExceptionClassNames will be excepted conditions - and the result string will be printed in the correct color. - catch_all=False will result in unspecified exceptions being re-raised.""" - err = None - out = None - w_exceptions = other_results.get('Warning', {}).keys() - w_exceptions = tuple(get_exception(e) for e in w_exceptions) - e_exceptions = other_results.get('Error', {}).keys() - e_exceptions = tuple(get_exception(e) for e in e_exceptions) - w_results = other_results.get('Warning', {}) - e_results = other_results.get('Error', {}) + other_results is in the form of + { + 'Warning': {'ExceptionClassName': 'Result Message'}, + 'Error': {'ExceptionClassName': 'Result Message'} + } + The the ExceptionClassNames will be excepted conditions + and the result string will be printed in the correct color. + catch_all=False will re-raise unspecified exceptions.""" + err = None + out = None + w_exceptions = other_results.get('Warning', {}).keys() + w_exceptions = tuple(get_exception(e) for e in w_exceptions) + e_exceptions = other_results.get('Error', {}).keys() + e_exceptions = tuple(get_exception(e) for e in e_exceptions) + w_results = other_results.get('Warning', {}) + e_results = other_results.get('Error', {}) - # Run function and catch errors - print_standard('{indent}{message:<{width}}'.format( - indent=' '*indent, message=message, width=width), end='', flush=True) - try: - out = function(*args, **kwargs) - if print_return: - str_list = out - if isinstance(out, subprocess.CompletedProcess): - str_list = out.stdout.decode().strip().splitlines() - print_standard(str_list[0].strip(), timestamp=False) - for item in str_list[1:]: - print_standard('{indent}{item}'.format( - indent=' '*(indent+width), item=item.strip())) - elif silent_function: - print_success(cs, timestamp=False) - except w_exceptions as e: - _result = w_results.get(e.__class__.__name__, 'Warning') - print_warning(_result, timestamp=False) - err = e - except e_exceptions as e: - _result = e_results.get(e.__class__.__name__, 'Error') - print_error(_result, timestamp=False) - err = e - except Exception: - print_error(ns, timestamp=False) - err = traceback.format_exc() + # Run function and catch errors + print_standard('{indent}{message:<{width}}'.format( + indent=' '*indent, message=message, width=width), end='', flush=True) + try: + out = function(*args, **kwargs) + if print_return: + str_list = out + if isinstance(out, subprocess.CompletedProcess): + str_list = out.stdout.decode().strip().splitlines() + print_standard(str_list[0].strip(), timestamp=False) + for item in str_list[1:]: + print_standard('{indent}{item}'.format( + indent=' '*(indent+width), item=item.strip())) + elif silent_function: + print_success(cs, timestamp=False) + except w_exceptions as e: + _result = w_results.get(e.__class__.__name__, 'Warning') + print_warning(_result, timestamp=False) + err = e + except e_exceptions as e: + _result = e_results.get(e.__class__.__name__, 'Error') + print_error(_result, timestamp=False) + err = e + except Exception: + print_error(ns, timestamp=False) + err = traceback.format_exc() - # Return or raise? - if err and not catch_all: - raise - else: - return {'CS': not bool(err), 'Error': err, 'Out': out} + # Return or raise? + if err and not catch_all: + raise + else: + return {'CS': not bool(err), 'Error': err, 'Out': out} def upload_crash_details(): - """Upload log and runtime data to the CRASH_SERVER. + """Upload log and runtime data to the CRASH_SERVER. - Intended for uploading to a public Nextcloud share.""" - if not ENABLED_UPLOAD_DATA: - raise GenericError + Intended for uploading to a public Nextcloud share.""" + if not ENABLED_UPLOAD_DATA: + raise GenericError - import requests - if 'LogFile' in global_vars and global_vars['LogFile']: - if ask('Upload crash details to {}?'.format(CRASH_SERVER['Name'])): - with open(global_vars['LogFile']) as f: - data = '''{} + import requests + if 'LogFile' in global_vars and global_vars['LogFile']: + if ask('Upload crash details to {}?'.format(CRASH_SERVER['Name'])): + with open(global_vars['LogFile']) as f: + data = '''{} ############################# Runtime Details: sys.argv: {} global_vars: {}'''.format(f.read(), sys.argv, global_vars) - filename = global_vars.get('LogFile', 'Unknown') - filename = re.sub(r'.*(\\|/)', '', filename) - filename += '.txt' - url = '{}/Crash_{}__{}'.format( - CRASH_SERVER['Url'], - global_vars.get('Date-Time', 'Unknown Date-Time'), - filename) - r = requests.put( - url, data=data, - headers={'X-Requested-With': 'XMLHttpRequest'}, - auth=(CRASH_SERVER['User'], CRASH_SERVER['Pass'])) - # Raise exception if upload NS - if not r.ok: - raise Exception - else: - # User said no - raise GenericAbort + filename = global_vars.get('LogFile', 'Unknown') + filename = re.sub(r'.*(\\|/)', '', filename) + filename += '.txt' + url = '{}/Crash_{}__{}'.format( + CRASH_SERVER['Url'], + global_vars.get('Date-Time', 'Unknown Date-Time'), + filename) + r = requests.put( + url, data=data, + headers={'X-Requested-With': 'XMLHttpRequest'}, + auth=(CRASH_SERVER['User'], CRASH_SERVER['Pass'])) + # Raise exception if upload NS + if not r.ok: + raise Exception else: - # No LogFile defined (or invalid LogFile) - raise GenericError + # User said no + raise GenericAbort + else: + # No LogFile defined (or invalid LogFile) + raise GenericError def wait_for_process(name, poll_rate=3): - """Wait for process by name.""" - running = True - while running: - sleep(poll_rate) - running = False - for proc in psutil.process_iter(): - try: - if re.search(r'^{}'.format(name), proc.name(), re.IGNORECASE): - running = True - except psutil._exceptions.NoSuchProcess: - # Assuming process closed during iteration - pass - sleep(1) + """Wait for process by name.""" + running = True + while running: + sleep(poll_rate) + running = False + for proc in psutil.process_iter(): + try: + if re.search(r'^{}'.format(name), proc.name(), re.IGNORECASE): + running = True + except psutil._exceptions.NoSuchProcess: + # Assuming process closed during iteration + pass + sleep(1) # global_vars functions def init_global_vars(silent=False): - """Sets global variables based on system info.""" - if not silent: - print_info('Initializing') - if psutil.WINDOWS: - os.system('title Wizard Kit') - if psutil.LINUX: - init_functions = [ - ['Checking environment...', set_linux_vars], - ['Clearing collisions...', clean_env_vars], - ] + """Sets global variables based on system info.""" + if not silent: + print_info('Initializing') + if psutil.WINDOWS: + os.system('title Wizard Kit') + if psutil.LINUX: + init_functions = [ + ['Checking environment...', set_linux_vars], + ['Clearing collisions...', clean_env_vars], + ] + else: + init_functions = [ + ['Checking .bin...', find_bin], + ['Checking environment...', set_common_vars], + ['Checking OS...', check_os], + ['Checking tools...', check_tools], + ['Creating folders...', make_tmp_dirs], + ['Clearing collisions...', clean_env_vars], + ] + try: + if silent: + for f in init_functions: + f[1]() else: - init_functions = [ - ['Checking .bin...', find_bin], - ['Checking environment...', set_common_vars], - ['Checking OS...', check_os], - ['Checking tools...', check_tools], - ['Creating folders...', make_tmp_dirs], - ['Clearing collisions...', clean_env_vars], - ] - try: - if silent: - for f in init_functions: - f[1]() - else: - for f in init_functions: - try_and_print( - message=f[0], function=f[1], - cs='Done', ns='Error', catch_all=False) - except: - major_exception() + for f in init_functions: + try_and_print( + message=f[0], function=f[1], + cs='Done', ns='Error', catch_all=False) + except: + major_exception() def check_os(): - """Set OS specific variables.""" - tmp = {} + """Set OS specific variables.""" + tmp = {} - # Query registry - path = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion' - with winreg.OpenKey(HKLM, path) as key: - for name in ['CurrentBuild', 'CurrentVersion', 'ProductName']: - try: - tmp[name] = winreg.QueryValueEx(key, name)[0] - except FileNotFoundError: - tmp[name] = 'Unknown' + # Query registry + path = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion' + with winreg.OpenKey(HKLM, path) as key: + for name in ['CurrentBuild', 'CurrentVersion', 'ProductName']: + try: + tmp[name] = winreg.QueryValueEx(key, name)[0] + except FileNotFoundError: + tmp[name] = 'Unknown' - # Handle CurrentBuild collision - if tmp['CurrentBuild'] == '9200': - if tmp['CurrentVersion'] == '6.2': - # Windown 8, set to fake build number - tmp['CurrentBuild'] = '9199' - else: - # Windows 8.1, leave alone - pass + # Handle CurrentBuild collision + if tmp['CurrentBuild'] == '9200': + if tmp['CurrentVersion'] == '6.2': + # Windown 8, set to fake build number + tmp['CurrentBuild'] = '9199' + else: + # Windows 8.1, leave alone + pass - # Check bit depth - tmp['Arch'] = 32 - if 'PROGRAMFILES(X86)' in global_vars['Env']: - tmp['Arch'] = 64 + # Check bit depth + tmp['Arch'] = 32 + if 'PROGRAMFILES(X86)' in global_vars['Env']: + tmp['Arch'] = 64 - # Get Windows build info - build_info = WINDOWS_BUILDS.get( - tmp['CurrentBuild'], - ('Unknown', 'Build {}'.format(tmp['CurrentBuild']), None, None, 'unrecognized')) + # Get Windows build info + build_info = WINDOWS_BUILDS.get(tmp['CurrentBuild'], None) + if build_info is None: + # Not in windows_builds.py + build_info = [ + 'Unknown', + 'Build {}'.format(tmp['CurrentBuild']), + None, + None, + 'unrecognized'] + else: build_info = list(build_info) - tmp['Version'] = build_info.pop(0) - tmp['Release'] = build_info.pop(0) - tmp['Codename'] = build_info.pop(0) - tmp['Marketing Name'] = build_info.pop(0) - tmp['Notes'] = build_info.pop(0) + tmp['Version'] = build_info.pop(0) + tmp['Release'] = build_info.pop(0) + tmp['Codename'] = build_info.pop(0) + tmp['Marketing Name'] = build_info.pop(0) + tmp['Notes'] = build_info.pop(0) - # Set name - tmp['Name'] = tmp['ProductName'] - if tmp['Release']: - tmp['Name'] += ' {}'.format(tmp['Release']) - if tmp['Codename']: - tmp['Name'] += ' "{}"'.format(tmp['Codename']) - if tmp['Marketing Name']: - tmp['Name'] += ' / "{}"'.format(tmp['Marketing Name']) - tmp['Name'] = re.sub(r'\s+', ' ', tmp['Name']) + # Set name + tmp['Name'] = tmp['ProductName'] + if tmp['Release']: + tmp['Name'] += ' {}'.format(tmp['Release']) + if tmp['Codename']: + tmp['Name'] += ' "{}"'.format(tmp['Codename']) + if tmp['Marketing Name']: + tmp['Name'] += ' / "{}"'.format(tmp['Marketing Name']) + tmp['Name'] = re.sub(r'\s+', ' ', tmp['Name']) - # Set display name - tmp['DisplayName'] = '{} x{}'.format(tmp['Name'], tmp['Arch']) - if tmp['Notes']: - tmp['DisplayName'] += ' ({})'.format(tmp['Notes']) + # Set display name + tmp['DisplayName'] = '{} x{}'.format(tmp['Name'], tmp['Arch']) + if tmp['Notes']: + tmp['DisplayName'] += ' ({})'.format(tmp['Notes']) - global_vars['OS'] = tmp + global_vars['OS'] = tmp def check_tools(): - """Set tool variables based on OS bit-depth and tool availability.""" - if global_vars['OS'].get('Arch', 32) == 64: - global_vars['Tools'] = { - k: v.get('64', v.get('32')) for (k, v) in TOOLS.items()} - else: - global_vars['Tools'] = {k: v.get('32') for (k, v) in TOOLS.items()} + """Set tool variables based on OS bit-depth and tool availability.""" + if global_vars['OS'].get('Arch', 32) == 64: + global_vars['Tools'] = { + k: v.get('64', v.get('32')) for (k, v) in TOOLS.items()} + else: + global_vars['Tools'] = {k: v.get('32') for (k, v) in TOOLS.items()} - # Fix paths - global_vars['Tools'] = {k: os.path.join(global_vars['BinDir'], v) - for (k, v) in global_vars['Tools'].items()} + # Fix paths + global_vars['Tools'] = {k: os.path.join(global_vars['BinDir'], v) + for (k, v) in global_vars['Tools'].items()} def clean_env_vars(): - """Remove conflicting global_vars and env variables. + """Remove conflicting global_vars and env variables. - This fixes an issue where both global_vars and - global_vars['Env'] are expanded at the same time.""" - for key in global_vars.keys(): - global_vars['Env'].pop(key, None) + This fixes an issue where both global_vars and + global_vars['Env'] are expanded at the same time.""" + for key in global_vars.keys(): + global_vars['Env'].pop(key, None) def find_bin(): - """Find .bin folder in the cwd or it's parents.""" - wd = os.getcwd() - base = None - while base is None: - if os.path.exists('.bin'): - base = os.getcwd() - break - if re.fullmatch(r'\w:\\', os.getcwd()): - break - os.chdir('..') - os.chdir(wd) - if base is None: - raise BinNotFoundError - global_vars['BaseDir'] = base + """Find .bin folder in the cwd or it's parents.""" + wd = os.getcwd() + base = None + while base is None: + if os.path.exists('.bin'): + base = os.getcwd() + break + if re.fullmatch(r'\w:\\', os.getcwd()): + break + os.chdir('..') + os.chdir(wd) + if base is None: + raise BinNotFoundError + global_vars['BaseDir'] = base def make_tmp_dirs(): - """Make temp directories.""" - os.makedirs(global_vars['BackupDir'], exist_ok=True) - os.makedirs(global_vars['LogDir'], exist_ok=True) - os.makedirs(r'{}\{}'.format( - global_vars['LogDir'], KIT_NAME_FULL), exist_ok=True) - os.makedirs(r'{}\Tools'.format(global_vars['LogDir']), exist_ok=True) - os.makedirs(global_vars['TmpDir'], exist_ok=True) + """Make temp directories.""" + os.makedirs(global_vars['BackupDir'], exist_ok=True) + os.makedirs(global_vars['LogDir'], exist_ok=True) + os.makedirs(r'{}\{}'.format( + global_vars['LogDir'], KIT_NAME_FULL), exist_ok=True) + os.makedirs(r'{}\Tools'.format(global_vars['LogDir']), exist_ok=True) + os.makedirs(global_vars['TmpDir'], exist_ok=True) def set_common_vars(): - """Set common variables.""" - global_vars['Date'] = time.strftime("%Y-%m-%d") - global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") - global_vars['Env'] = os.environ.copy() + """Set common variables.""" + global_vars['Date'] = time.strftime("%Y-%m-%d") + global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") + global_vars['Env'] = os.environ.copy() - global_vars['ArchivePassword'] = ARCHIVE_PASSWORD - global_vars['BinDir'] = r'{BaseDir}\.bin'.format( - **global_vars) - global_vars['CBinDir'] = r'{BaseDir}\.cbin'.format( - **global_vars) - global_vars['ClientDir'] = r'{SYSTEMDRIVE}\{prefix}'.format( - prefix=KIT_NAME_SHORT, **global_vars['Env']) - global_vars['BackupDir'] = r'{ClientDir}\Backups'.format( - **global_vars) - global_vars['LogDir'] = r'{ClientDir}\Logs\{Date}'.format( - **global_vars) - global_vars['QuarantineDir'] = r'{ClientDir}\Quarantine'.format( - **global_vars) - global_vars['TmpDir'] = r'{BinDir}\tmp'.format( - **global_vars) + global_vars['ArchivePassword'] = ARCHIVE_PASSWORD + global_vars['BinDir'] = r'{BaseDir}\.bin'.format( + **global_vars) + global_vars['CBinDir'] = r'{BaseDir}\.cbin'.format( + **global_vars) + global_vars['ClientDir'] = r'{SYSTEMDRIVE}\{prefix}'.format( + prefix=KIT_NAME_SHORT, **global_vars['Env']) + global_vars['BackupDir'] = r'{ClientDir}\Backups'.format( + **global_vars) + global_vars['LogDir'] = r'{ClientDir}\Logs\{Date}'.format( + **global_vars) + global_vars['QuarantineDir'] = r'{ClientDir}\Quarantine'.format( + **global_vars) + global_vars['TmpDir'] = r'{BinDir}\tmp'.format( + **global_vars) def set_linux_vars(): - """Set common variables in a Linux environment. + """Set common variables in a Linux environment. - These assume we're running under a WK-Linux build.""" - result = run_program(['mktemp', '-d']) - global_vars['TmpDir'] = result.stdout.decode().strip() - global_vars['Date'] = time.strftime("%Y-%m-%d") - global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") - global_vars['Env'] = os.environ.copy() - global_vars['BinDir'] = '/usr/local/bin' - global_vars['LogDir'] = global_vars['TmpDir'] - global_vars['Tools'] = { - 'wimlib-imagex': 'wimlib-imagex', - 'SevenZip': '7z', - } + These assume we're running under a WK-Linux build.""" + result = run_program(['mktemp', '-d']) + global_vars['TmpDir'] = result.stdout.decode().strip() + global_vars['Date'] = time.strftime("%Y-%m-%d") + global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") + global_vars['Env'] = os.environ.copy() + global_vars['BinDir'] = '/usr/local/bin' + global_vars['LogDir'] = global_vars['TmpDir'] + global_vars['Tools'] = { + 'wimlib-imagex': 'wimlib-imagex', + 'SevenZip': '7z', + } def set_log_file(log_name): - """Sets global var LogFile and creates path as needed.""" - folder_path = r'{}\{}'.format(global_vars['LogDir'], KIT_NAME_FULL) - log_file = r'{}\{}'.format(folder_path, log_name) - os.makedirs(folder_path, exist_ok=True) - global_vars['LogFile'] = log_file + """Sets global var LogFile and creates path as needed.""" + folder_path = r'{}\{}'.format(global_vars['LogDir'], KIT_NAME_FULL) + log_file = r'{}\{}'.format(folder_path, log_name) + os.makedirs(folder_path, exist_ok=True) + global_vars['LogFile'] = log_file if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 4049272cbb52278fc8ee6c69b078dd79ac18d442 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:48:32 -0700 Subject: [PATCH 116/186] Updated data.py --- .bin/Scripts/functions/data.py | 1536 ++++++++++++++++---------------- 1 file changed, 769 insertions(+), 767 deletions(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 9557ebdb..ed4bfa9e 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -9,107 +9,107 @@ from functions.common import * # Classes class LocalDisk(): - def __init__(self, disk): - self.disk = disk - self.name = disk.mountpoint.upper() - self.path = self.name - def is_dir(self): - # Should always be true - return True - def is_file(self): - # Should always be false - return False + def __init__(self, disk): + self.disk = disk + self.name = disk.mountpoint.upper() + self.path = self.name + def is_dir(self): + # Should always be true + return True + def is_file(self): + # Should always be false + return False class SourceItem(): - def __init__(self, name, path): - self.name = name - self.path = path + def __init__(self, name, path): + self.name = name + self.path = path # Regex REGEX_EXCL_ITEMS = re.compile( - r'^(\.(AppleDB|AppleDesktop|AppleDouble' - r'|com\.apple\.timemachine\.supported|dbfseventsd' - r'|DocumentRevisions-V100.*|DS_Store|fseventsd|PKInstallSandboxManager' - r'|Spotlight.*|SymAV.*|symSchedScanLockxz|TemporaryItems|Trash.*' - r'|vol|VolumeIcon\.icns)|desktop\.(ini|.*DB|.*DF)' - r'|(hiberfil|pagefile)\.sys|lost\+found|Network\.*Trash\.*Folder' - r'|Recycle[dr]|System\.*Volume\.*Information|Temporary\.*Items' - r'|Thumbs\.db)$', - re.IGNORECASE) + r'^(\.(AppleDB|AppleDesktop|AppleDouble' + r'|com\.apple\.timemachine\.supported|dbfseventsd' + r'|DocumentRevisions-V100.*|DS_Store|fseventsd|PKInstallSandboxManager' + r'|Spotlight.*|SymAV.*|symSchedScanLockxz|TemporaryItems|Trash.*' + r'|vol|VolumeIcon\.icns)|desktop\.(ini|.*DB|.*DF)' + r'|(hiberfil|pagefile)\.sys|lost\+found|Network\.*Trash\.*Folder' + r'|Recycle[dr]|System\.*Volume\.*Information|Temporary\.*Items' + r'|Thumbs\.db)$', + re.IGNORECASE) REGEX_EXCL_ROOT_ITEMS = re.compile( - r'^(boot(mgr|nxt)$|Config.msi' - r'|(eula|globdata|install|vc_?red)' - r'|.*.sys$|System Volume Information|RECYCLER?|\$Recycle\.bin' - r'|\$?Win(dows(.old.*|\.~BT|)$|RE_)|\$GetCurrent|Windows10Upgrade' - r'|PerfLogs|Program Files|SYSTEM.SAV' - r'|.*\.(esd|swm|wim|dd|map|dmg|image)$)', - re.IGNORECASE) + r'^(boot(mgr|nxt)$|Config.msi' + r'|(eula|globdata|install|vc_?red)' + r'|.*.sys$|System Volume Information|RECYCLER?|\$Recycle\.bin' + r'|\$?Win(dows(.old.*|\. BT|)$|RE_)|\$GetCurrent|Windows10Upgrade' + r'|PerfLogs|Program Files|SYSTEM.SAV' + r'|.*\.(esd|swm|wim|dd|map|dmg|image)$)', + re.IGNORECASE) REGEX_INCL_ROOT_ITEMS = re.compile( - r'^(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' - r'|Media|Music|Pic(ture|)s?|Vid(eo|)s?)' - r'|{prefix}(-?Info|-?Transfer|)' - r'|(ProgramData|Recovery|Temp.*|Users)$' - r'|.*\.(log|txt|rtf|qb\w*|avi|m4a|m4v|mp4|mkv|jpg|png|tiff?)$)' - r''.format(prefix=KIT_NAME_SHORT), - re.IGNORECASE) + r'^(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' + r'|Media|Music|Pic(ture|)s?|Vid(eo|)s?)' + r'|{prefix}(-?Info|-?Transfer|)' + r'|(ProgramData|Recovery|Temp.*|Users)$' + r'|.*\.(log|txt|rtf|qb\w*|avi|m4a|m4v|mp4|mkv|jpg|png|tiff?)$)' + r''.format(prefix=KIT_NAME_SHORT), + re.IGNORECASE) REGEX_WIM_FILE = re.compile( - r'\.wim$', - re.IGNORECASE) + r'\.wim$', + re.IGNORECASE) REGEX_WINDOWS_OLD = re.compile( - r'^Win(dows|)\.old', - re.IGNORECASE) + r'^Win(dows|)\.old', + re.IGNORECASE) # STATIC VARIABLES FAST_COPY_EXCLUDES = [ - r'\*.esd', - r'\*.swm', - r'\*.wim', - r'\*.dd', - r'\*.dd.tgz', - r'\*.dd.txz', - r'\*.map', - r'\*.dmg', - r'\*.image', - r'$RECYCLE.BIN', - r'$Recycle.Bin', - r'.AppleDB', - r'.AppleDesktop', - r'.AppleDouble', - r'.com.apple.timemachine.supported', - r'.dbfseventsd', - r'.DocumentRevisions-V100*', - r'.DS_Store', - r'.fseventsd', - r'.PKInstallSandboxManager', - r'.Spotlight*', - r'.SymAV*', - r'.symSchedScanLockxz', - r'.TemporaryItems', - r'.Trash*', - r'.vol', - r'.VolumeIcon.icns', - r'desktop.ini', - r'Desktop?DB', - r'Desktop?DF', - r'hiberfil.sys', - r'lost+found', - r'Network?Trash?Folder', - r'pagefile.sys', - r'Recycled', - r'RECYCLER', - r'System?Volume?Information', - r'Temporary?Items', - r'Thumbs.db', - ] + r'\*.esd', + r'\*.swm', + r'\*.wim', + r'\*.dd', + r'\*.dd.tgz', + r'\*.dd.txz', + r'\*.map', + r'\*.dmg', + r'\*.image', + r'$RECYCLE.BIN', + r'$Recycle.Bin', + r'.AppleDB', + r'.AppleDesktop', + r'.AppleDouble', + r'.com.apple.timemachine.supported', + r'.dbfseventsd', + r'.DocumentRevisions-V100*', + r'.DS_Store', + r'.fseventsd', + r'.PKInstallSandboxManager', + r'.Spotlight*', + r'.SymAV*', + r'.symSchedScanLockxz', + r'.TemporaryItems', + r'.Trash*', + r'.vol', + r'.VolumeIcon.icns', + r'desktop.ini', + r'Desktop?DB', + r'Desktop?DF', + r'hiberfil.sys', + r'lost+found', + r'Network?Trash?Folder', + r'pagefile.sys', + r'Recycled', + r'RECYCLER', + r'System?Volume?Information', + r'Temporary?Items', + r'Thumbs.db', + ] FAST_COPY_ARGS = [ - '/cmd=noexist_only', - '/utf8', - '/skip_empty_dir', - '/linkdest', - '/no_ui', - '/auto_close', - '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), - ] + '/cmd=noexist_only', + '/utf8', + '/skip_empty_dir', + '/linkdest', + '/no_ui', + '/auto_close', + '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), + ] # Code borrowed from: https://stackoverflow.com/a/29075319 SEM_NORMAL = ctypes.c_uint() SEM_FAILCRITICALERRORS = 1 @@ -117,770 +117,772 @@ SEM_NOOPENFILEERRORBOX = 0x8000 SEM_FAIL = SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS def cleanup_transfer(dest_path): - """Fix attributes and move extraneous items outside the Transfer folder.""" - try: - # Remove dest_path if empty - os.rmdir(dest_path) - except OSError: + """Fix attributes and move extraneous items outside the Transfer folder.""" + try: + # Remove dest_path if empty + os.rmdir(dest_path) + except OSError: + pass + if not os.path.exists(dest_path): + # Bail if dest_path was empty and removed + raise Exception + + # Fix attributes + cmd = ['attrib', '-a', '-h', '-r', '-s', dest_path] + run_program(cmd, check=False) + + for root, dirs, files in os.walk(dest_path, topdown=False): + for name in dirs: + # Remove empty directories and junction points + try: + os.rmdir(os.path.join(root, name)) + except OSError: pass - if not os.path.exists(dest_path): - # Bail if dest_path was empty and removed - raise Exception - - # Fix attributes - cmd = ['attrib', '-a', '-h', '-r', '-s', dest_path] - run_program(cmd, check=False) - - for root, dirs, files in os.walk(dest_path, topdown=False): - for name in dirs: - # Remove empty directories and junction points - try: - os.rmdir(os.path.join(root, name)) - except OSError: - pass - for name in files: - # "Remove" files based on exclusion regex - if REGEX_EXCL_ITEMS.search(name): - # Make dest folder - dest_name = root.replace(dest_path, dest_path+'.Removed') - os.makedirs(dest_name, exist_ok=True) - # Set dest filename - dest_name = os.path.join(dest_name, name) - dest_name = non_clobber_rename(dest_name) - source_name = os.path.join(root, name) - try: - shutil.move(source_name, dest_name) - except Exception: - pass + for name in files: + # "Remove" files based on exclusion regex + if REGEX_EXCL_ITEMS.search(name): + # Make dest folder + dest_name = root.replace(dest_path, dest_path+'.Removed') + os.makedirs(dest_name, exist_ok=True) + # Set dest filename + dest_name = os.path.join(dest_name, name) + dest_name = non_clobber_rename(dest_name) + source_name = os.path.join(root, name) + try: + shutil.move(source_name, dest_name) + except Exception: + pass def find_core_storage_volumes(device_path=None): - """Try to create block devices for any Apple CoreStorage volumes.""" - corestorage_uuid = '53746f72-6167-11aa-aa11-00306543ecac' - dmsetup_cmd_file = '{TmpDir}/dmsetup_command'.format(**global_vars) + """Try to create block devices for any Apple CoreStorage volumes.""" + corestorage_uuid = '53746f72-6167-11aa-aa11-00306543ecac' + dmsetup_cmd_file = '{TmpDir}/dmsetup_command'.format(**global_vars) - # Get CoreStorage devices + # Get CoreStorage devices + cmd = [ + 'lsblk', '--json', '--list', '--paths', + '--output', 'NAME,PARTTYPE'] + if device_path: + cmd.append(device_path) + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + devs = json_data.get('blockdevices', []) + devs = [d for d in devs if d.get('parttype', '') == corestorage_uuid] + if devs: + print_standard(' ') + print_standard('Detected CoreStorage partition{}'.format( + '' if len(devs) == 1 else 's')) + print_standard(' Scanning for inner volume(s)....') + + # Search for inner volumes and setup dev mappers + for dev in devs: + dev_path = dev.get('name', '') + if not dev_path: + # Can't setup block device without the dev path + continue + dev_name = re.sub(r'.*/', '', dev_path) + log_path = '{LogDir}/testdisk_{dev_name}.log'.format( + dev_name=dev_name, **global_vars) + + # Run TestDisk cmd = [ - 'lsblk', '--json', '--list', '--paths', - '--output', 'NAME,PARTTYPE'] - if device_path: - cmd.append(device_path) - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - devs = json_data.get('blockdevices', []) - devs = [d for d in devs if d.get('parttype', '') == corestorage_uuid] - if devs: - print_standard(' ') - print_standard('Detected CoreStorage partition{}'.format( - '' if len(devs) == 1 else 's')) - print_standard(' Scanning for inner volume(s)....') + 'sudo', 'testdisk', + '/logname', log_path, '/debug', '/log', + '/cmd', dev_path, 'partition_none,analyze'] + result = run_program(cmd, check=False) + if result.returncode: + # i.e. return code is non-zero + continue + if not os.path.exists(log_path): + # TestDisk failed to write log + continue - # Search for inner volumes and setup dev mappers - for dev in devs: - dev_path = dev.get('name', '') - if not dev_path: - # Can't setup block device without the dev path - continue - dev_name = re.sub(r'.*/', '', dev_path) - log_path = '{LogDir}/testdisk_{dev_name}.log'.format( - dev_name=dev_name, **global_vars) + # Check log for found volumes + cs_vols = {} + with open(log_path, 'r') as f: + for line in f.readlines(): + r = re.match( + r'^.*echo "([^"]+)" . dmsetup create test(\d)$', + line.strip(), + re.IGNORECASE) + if r: + cs_name = 'CoreStorage_{}_{}'.format(dev_name, r.group(2)) + cs_vols[cs_name] = r.group(1) - # Run TestDisk - cmd = [ - 'sudo', 'testdisk', - '/logname', log_path, '/debug', '/log', - '/cmd', dev_path, 'partition_none,analyze'] - result = run_program(cmd, check=False) - if result.returncode: - # i.e. return code is non-zero - continue - if not os.path.exists(log_path): - # TestDisk failed to write log - continue - - # Check log for found volumes - cs_vols = {} - with open(log_path, 'r') as f: - for line in f.readlines(): - r = re.match( - r'^.*echo "([^"]+)" . dmsetup create test(\d)$', - line.strip(), - re.IGNORECASE) - if r: - cs_name = 'CoreStorage_{}_{}'.format(dev_name, r.group(2)) - cs_vols[cs_name] = r.group(1) - - # Create mapper device(s) - for name, dm_cmd in sorted(cs_vols.items()): - with open(dmsetup_cmd_file, 'w') as f: - f.write(dm_cmd) - cmd = ['sudo', 'dmsetup', 'create', name, dmsetup_cmd_file] - run_program(cmd, check=False) + # Create mapper device(s) + for name, dm_cmd in sorted(cs_vols.items()): + with open(dmsetup_cmd_file, 'w') as f: + f.write(dm_cmd) + cmd = ['sudo', 'dmsetup', 'create', name, dmsetup_cmd_file] + run_program(cmd, check=False) def fix_path_sep(path_str): - """Replace non-native and duplicate dir separators, returns str.""" - return re.sub(r'(\\|/)+', lambda s: os.sep, path_str) + """Replace non-native and duplicate dir separators, returns str.""" + return re.sub(r'(\\|/)+', lambda s: os.sep, path_str) def is_valid_wim_file(item): - """Checks if the provided os.DirEntry is a valid WIM file, returns bool.""" - valid = bool(item.is_file() and REGEX_WIM_FILE.search(item.name)) - if valid: - extract_item('wimlib', silent=True) - cmd = [global_vars['Tools']['wimlib-imagex'], 'info', item.path] - try: - run_program(cmd) - except subprocess.CalledProcessError: - valid = False - print_log('WARNING: Image "{}" damaged.'.format(item.name)) - return valid + """Checks if the provided os.DirEntry is a valid WIM file, returns bool.""" + valid = bool(item.is_file() and REGEX_WIM_FILE.search(item.name)) + if valid: + extract_item('wimlib', silent=True) + cmd = [global_vars['Tools']['wimlib-imagex'], 'info', item.path] + try: + run_program(cmd) + except subprocess.CalledProcessError: + valid = False + print_log('WARNING: Image "{}" damaged.'.format(item.name)) + return valid def get_mounted_volumes(): - """Get mounted volumes, returns dict.""" - cmd = [ - 'findmnt', '-J', '-b', '-i', - '-t', ( - 'autofs,binfmt_misc,bpf,cgroup,cgroup2,configfs,debugfs,devpts,devtmpfs,' - 'hugetlbfs,mqueue,proc,pstore,securityfs,sysfs,tmpfs' - ), - '-o', 'SOURCE,TARGET,FSTYPE,LABEL,SIZE,AVAIL,USED'] - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - mounted_volumes = [] - for item in json_data.get('filesystems', []): - mounted_volumes.append(item) - mounted_volumes.extend(item.get('children', [])) - return {item['source']: item for item in mounted_volumes} + """Get mounted volumes, returns dict.""" + cmd = [ + 'findmnt', '-J', '-b', '-i', + '-t', ( + 'autofs,binfmt_misc,bpf,cgroup,cgroup2,configfs,debugfs,devpts,' + 'devtmpfs,hugetlbfs,mqueue,proc,pstore,securityfs,sysfs,tmpfs' + ), + '-o', 'SOURCE,TARGET,FSTYPE,LABEL,SIZE,AVAIL,USED'] + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + mounted_volumes = [] + for item in json_data.get('filesystems', []): + mounted_volumes.append(item) + mounted_volumes.extend(item.get('children', [])) + return {item['source']: item for item in mounted_volumes} def mount_volumes( - all_devices=True, device_path=None, - read_write=False, core_storage=True): - """Mount all detected filesystems.""" - report = {} - cmd = [ - 'lsblk', '--json', '--paths', - '--output', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE'] - if not all_devices and device_path: - # Only mount volumes for specific device - cmd.append(device_path) + all_devices=True, device_path=None, + read_write=False, core_storage=True): + """Mount all detected filesystems.""" + report = {} + cmd = [ + 'lsblk', '--json', '--paths', + '--output', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE'] + if not all_devices and device_path: + # Only mount volumes for specific device + cmd.append(device_path) - # Check for Apple CoreStorage volumes first - if core_storage: - find_core_storage_volumes(device_path) + # Check for Apple CoreStorage volumes first + if core_storage: + find_core_storage_volumes(device_path) - # Get list of block devices - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - devs = json_data.get('blockdevices', []) + # Get list of block devices + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + devs = json_data.get('blockdevices', []) - # Get list of volumes - volumes = {} - for dev in devs: - if not dev.get('children', []): - volumes.update({dev['name']: dev}) - for child in dev.get('children', []): - if not child.get('children', []): - volumes.update({child['name']: child}) - for grandchild in child.get('children', []): - volumes.update({grandchild['name']: grandchild}) + # Get list of volumes + volumes = {} + for dev in devs: + if not dev.get('children', []): + volumes.update({dev['name']: dev}) + for child in dev.get('children', []): + if not child.get('children', []): + volumes.update({child['name']: child}) + for grandchild in child.get('children', []): + volumes.update({grandchild['name']: grandchild}) - # Get list of mounted volumes - mounted_volumes = get_mounted_volumes() + # Get list of mounted volumes + mounted_volumes = get_mounted_volumes() - # Loop over volumes - for vol_path, vol_data in volumes.items(): - vol_data['show_data'] = { - 'message': vol_path.replace('/dev/mapper/', ''), - 'data': None, - } - if re.search(r'^loop\d', vol_path, re.IGNORECASE): - # Skip loopback devices - vol_data['show_data']['data'] = 'Skipped' - vol_data['show_data']['warning'] = True - report[vol_path] = vol_data - elif 'children' in vol_data: - # Skip LVM/RAID partitions (the real volume is mounted separately) - vol_data['show_data']['data'] = vol_data.get('fstype', 'UNKNOWN') - if vol_data.get('label', None): - vol_data['show_data']['data'] += ' "{}"'.format(vol_data['label']) - vol_data['show_data']['info'] = True - report[vol_path] = vol_data - else: - if vol_path in mounted_volumes: - vol_data['show_data']['warning'] = True - else: - # Mount volume - cmd = ['udevil', 'mount', - '-o', 'rw' if read_write else 'ro', - vol_path] - try: - run_program(cmd) - except subprocess.CalledProcessError: - vol_data['show_data']['data'] = 'Failed to mount' - vol_data['show_data']['error'] = True - # Update mounted_volumes data - mounted_volumes = get_mounted_volumes() + # Loop over volumes + for vol_path, vol_data in volumes.items(): + vol_data['show_data'] = { + 'message': vol_path.replace('/dev/mapper/', ''), + 'data': None, + } + if re.search(r'^loop\d', vol_path, re.IGNORECASE): + # Skip loopback devices + vol_data['show_data']['data'] = 'Skipped' + vol_data['show_data']['warning'] = True + report[vol_path] = vol_data + elif 'children' in vol_data: + # Skip LVM/RAID partitions (the real volume is mounted separately) + vol_data['show_data']['data'] = vol_data.get('fstype', 'UNKNOWN') + if vol_data.get('label', None): + vol_data['show_data']['data'] += ' "{}"'.format(vol_data['label']) + vol_data['show_data']['info'] = True + report[vol_path] = vol_data + else: + if vol_path in mounted_volumes: + vol_data['show_data']['warning'] = True + else: + # Mount volume + cmd = ['udevil', 'mount', + '-o', 'rw' if read_write else 'ro', + vol_path] + try: + run_program(cmd) + except subprocess.CalledProcessError: + vol_data['show_data']['data'] = 'Failed to mount' + vol_data['show_data']['error'] = True + # Update mounted_volumes data + mounted_volumes = get_mounted_volumes() - # Format pretty result string - if vol_data['show_data']['data'] == 'Failed to mount': - vol_data['mount_point'] = None - else: - size_used = human_readable_size( - mounted_volumes[vol_path]['used']) - size_avail = human_readable_size( - mounted_volumes[vol_path]['avail']) - vol_data['size_avail'] = size_avail - vol_data['size_used'] = size_used - vol_data['mount_point'] = mounted_volumes[vol_path]['target'] - vol_data['show_data']['data'] = 'Mounted on {}'.format( - mounted_volumes[vol_path]['target']) - vol_data['show_data']['data'] = '{:40} ({} used, {} free)'.format( - vol_data['show_data']['data'], - size_used, - size_avail) + # Format pretty result string + if vol_data['show_data']['data'] == 'Failed to mount': + vol_data['mount_point'] = None + else: + size_used = human_readable_size( + mounted_volumes[vol_path]['used']) + size_avail = human_readable_size( + mounted_volumes[vol_path]['avail']) + vol_data['size_avail'] = size_avail + vol_data['size_used'] = size_used + vol_data['mount_point'] = mounted_volumes[vol_path]['target'] + vol_data['show_data']['data'] = 'Mounted on {}'.format( + mounted_volumes[vol_path]['target']) + vol_data['show_data']['data'] = '{:40} ({} used, {} free)'.format( + vol_data['show_data']['data'], + size_used, + size_avail) - # Update report - report[vol_path] = vol_data + # Update report + report[vol_path] = vol_data - return report + return report def mount_backup_shares(read_write=False): - """Mount the backup shares unless labeled as already mounted.""" + """Mount the backup shares unless labeled as already mounted.""" + if psutil.LINUX: + mounted_volumes = get_mounted_volumes() + for server in BACKUP_SERVERS: if psutil.LINUX: - mounted_volumes = get_mounted_volumes() - for server in BACKUP_SERVERS: - if psutil.LINUX: - # Update mounted status - source = '//{IP}/{Share}'.format(**server) - dest = '/Backups/{Name}'.format(**server) - mounted_str = '(Already) Mounted {}'.format(dest) - data = mounted_volumes.get(source, {}) - if dest == data.get('target', ''): - server['Mounted'] = True - elif psutil.WINDOWS: - mounted_str = '(Already) Mounted {Name}'.format(**server) - if server['Mounted']: - print_warning(mounted_str) - continue + # Update mounted status + source = '//{IP}/{Share}'.format(**server) + dest = '/Backups/{Name}'.format(**server) + mounted_str = '(Already) Mounted {}'.format(dest) + data = mounted_volumes.get(source, {}) + if dest == data.get('target', ''): + server['Mounted'] = True + elif psutil.WINDOWS: + mounted_str = '(Already) Mounted {Name}'.format(**server) + if server['Mounted']: + print_warning(mounted_str) + continue - mount_network_share(server, read_write) + mount_network_share(server, read_write) def mount_network_share(server, read_write=False): - """Mount a network share defined by server.""" - if read_write: - username = server['RW-User'] - password = server['RW-Pass'] - else: - username = server['User'] - password = server['Pass'] - if psutil.WINDOWS: - cmd = [ - 'net', 'use', r'\\{IP}\{Share}'.format(**server), - '/user:{}'.format(username), password] - warning = r'Failed to mount \\{Name}\{Share}, {IP} unreachable.'.format( - **server) - error = r'Failed to mount \\{Name}\{Share} ({IP})'.format(**server) - success = 'Mounted {Name}'.format(**server) - elif psutil.LINUX: - cmd = [ - 'sudo', 'mkdir', '-p', - '/Backups/{Name}'.format(**server)] - run_program(cmd) - cmd = [ - 'sudo', 'mount', - '//{IP}/{Share}'.format(**server), - '/Backups/{Name}'.format(**server), - '-o', '{}username={},password={}'.format( - '' if read_write else 'ro,', - username, - password)] - warning = 'Failed to mount /Backups/{Name}, {IP} unreachable.'.format( - **server) - error = 'Failed to mount /Backups/{Name}'.format(**server) - success = 'Mounted /Backups/{Name}'.format(**server) + """Mount a network share defined by server.""" + if read_write: + username = server['RW-User'] + password = server['RW-Pass'] + else: + username = server['User'] + password = server['Pass'] + if psutil.WINDOWS: + cmd = [ + 'net', 'use', r'\\{IP}\{Share}'.format(**server), + '/user:{}'.format(username), password] + warning = r'Failed to mount \\{Name}\{Share}, {IP} unreachable.'.format( + **server) + error = r'Failed to mount \\{Name}\{Share} ({IP})'.format(**server) + success = 'Mounted {Name}'.format(**server) + elif psutil.LINUX: + cmd = [ + 'sudo', 'mkdir', '-p', + '/Backups/{Name}'.format(**server)] + run_program(cmd) + cmd = [ + 'sudo', 'mount', + '//{IP}/{Share}'.format(**server), + '/Backups/{Name}'.format(**server), + '-o', '{}username={},password={}'.format( + '' if read_write else 'ro,', + username, + password)] + warning = 'Failed to mount /Backups/{Name}, {IP} unreachable.'.format( + **server) + error = 'Failed to mount /Backups/{Name}'.format(**server) + success = 'Mounted /Backups/{Name}'.format(**server) - # Test connection - try: - ping(server['IP']) - except subprocess.CalledProcessError: - print_warning(warning) - sleep(1) - return False + # Test connection + try: + ping(server['IP']) + except subprocess.CalledProcessError: + print_warning(warning) + sleep(1) + return False - # Mount - try: - run_program(cmd) - except Exception: - print_error(error) - sleep(1) - else: - print_info(success) - server['Mounted'] = True + # Mount + try: + run_program(cmd) + except Exception: + print_error(error) + sleep(1) + else: + print_info(success) + server['Mounted'] = True def run_fast_copy(items, dest): - """Copy items to dest using FastCopy.""" - if not items: - raise Exception + """Copy items to dest using FastCopy.""" + if not items: + raise Exception - cmd = [global_vars['Tools']['FastCopy'], *FAST_COPY_ARGS] - cmd.append(r'/logfile={LogDir}\Tools\FastCopy.log'.format(**global_vars)) - cmd.extend(items) - cmd.append('/to={}\\'.format(dest)) + cmd = [global_vars['Tools']['FastCopy'], *FAST_COPY_ARGS] + cmd.append(r'/logfile={LogDir}\Tools\FastCopy.log'.format(**global_vars)) + cmd.extend(items) + cmd.append('/to={}\\'.format(dest)) - run_program(cmd) + run_program(cmd) def run_wimextract(source, items, dest): - """Extract items from source WIM to dest folder.""" - if not items: - raise Exception - extract_item('wimlib', silent=True) + """Extract items from source WIM to dest folder.""" + if not items: + raise Exception + extract_item('wimlib', silent=True) - # Write files.txt - with open(r'{}\wim_files.txt'.format(global_vars['TmpDir']), 'w', - encoding='utf-8') as f: - # Defaults - for item in items: - f.write('{}\n'.format(item)) - sleep(1) # For safety? + # Write files.txt + with open(r'{}\wim_files.txt'.format(global_vars['TmpDir']), 'w', + encoding='utf-8') as f: + # Defaults + for item in items: + f.write('{}\n'.format(item)) + sleep(1) # For safety? - # Extract files - cmd = [ - global_vars['Tools']['wimlib-imagex'], - 'extract', - source, '1', - r'@{}\wim_files.txt'.format(global_vars['TmpDir']), - '--dest-dir={}\\'.format(dest), - '--no-acls', - '--nullglob'] - run_program(cmd) + # Extract files + cmd = [ + global_vars['Tools']['wimlib-imagex'], + 'extract', + source, '1', + r'@{}\wim_files.txt'.format(global_vars['TmpDir']), + '--dest-dir={}\\'.format(dest), + '--no-acls', + '--nullglob'] + run_program(cmd) def list_source_items(source_obj, rel_path=None): - """List items in a dir or WIM, returns a list of SourceItem objects.""" - items = [] - rel_path = '{}{}'.format(os.sep, rel_path) if rel_path else '' - if source_obj.is_dir(): - source_path = '{}{}'.format(source_obj.path, rel_path) - items = [SourceItem(name=item.name, path=item.path) - for item in os.scandir(source_path)] - else: - # Prep wimlib-imagex - if psutil.WINDOWS: - extract_item('wimlib', silent=True) - cmd = [ - global_vars['Tools']['wimlib-imagex'], 'dir', - source_obj.path, '1'] - if rel_path: - cmd.append('--path={}'.format(rel_path)) + """List items in a dir or WIM, returns list of SourceItem objects.""" + items = [] + rel_path = '{}{}'.format(os.sep, rel_path) if rel_path else '' + if source_obj.is_dir(): + source_path = '{}{}'.format(source_obj.path, rel_path) + items = [SourceItem(name=item.name, path=item.path) + for item in os.scandir(source_path)] + else: + # Prep wimlib-imagex + if psutil.WINDOWS: + extract_item('wimlib', silent=True) + cmd = [ + global_vars['Tools']['wimlib-imagex'], 'dir', + source_obj.path, '1'] + if rel_path: + cmd.append('--path={}'.format(rel_path)) - # Get item list - try: - items = run_program(cmd) - except subprocess.CalledProcessError: - print_error('ERROR: Failed to get file list.') - raise + # Get item list + try: + items = run_program(cmd) + except subprocess.CalledProcessError: + print_error('ERROR: Failed to get file list.') + raise - # Strip non-root items - items = [fix_path_sep(i.strip()) - for i in items.stdout.decode('utf-8', 'ignore').splitlines()] - if rel_path: - items = [i.replace(rel_path, '') for i in items] - items = [i for i in items - if i.count(os.sep) == 1 and i.strip() != os.sep] - items = [SourceItem(name=i[1:], path=rel_path+i) for i in items] + # Strip non-root items + items = [fix_path_sep(i.strip()) + for i in items.stdout.decode('utf-8', 'ignore').splitlines()] + if rel_path: + items = [i.replace(rel_path, '') for i in items] + items = [i for i in items + if i.count(os.sep) == 1 and i.strip() != os.sep] + items = [SourceItem(name=i[1:], path=rel_path+i) for i in items] - # Done - return items + # Done + return items def scan_source(source_obj, dest_path, rel_path='', interactive=True): - """Scan source for files/folders to transfer, returns list. + """Scan source for files/folders to transfer, returns list. - This will scan the root and (recursively) any Windows.old folders.""" - selected_items = [] - win_olds = [] + This will scan the root and (recursively) any Windows.old folders.""" + selected_items = [] + win_olds = [] - # Root Items - root_items = [] - item_list = list_source_items(source_obj, rel_path) - for item in item_list: - if REGEX_WINDOWS_OLD.search(item.name): - item.name = '{}{}{}'.format( - rel_path, - os.sep if rel_path else '', - item.name) - win_olds.append(item) - elif REGEX_INCL_ROOT_ITEMS.search(item.name): - print_success('Auto-Selected: {}'.format(item.path)) - root_items.append('{}'.format(item.path)) - elif not REGEX_EXCL_ROOT_ITEMS.search(item.name): - if not interactive: - print_success('Auto-Selected: {}'.format(item.path)) - root_items.append('{}'.format(item.path)) - else: - prompt = 'Transfer: "{}{}{}" ?'.format( - rel_path, - os.sep if rel_path else '', - item.name) - choices = ['Yes', 'No', 'All', 'Quit'] - answer = choice(prompt=prompt, choices=choices) - if answer == 'Quit': - abort() - elif answer == 'All': - interactive = False - if answer in ['Yes', 'All']: - root_items.append('{}'.format(item.path)) - if root_items: - selected_items.append({ - 'Message': '{}{}Root Items...'.format( - rel_path, - ' ' if rel_path else ''), - 'Items': root_items.copy(), - 'Destination': dest_path}) + # Root Items + root_items = [] + item_list = list_source_items(source_obj, rel_path) + for item in item_list: + if REGEX_WINDOWS_OLD.search(item.name): + item.name = '{}{}{}'.format( + rel_path, + os.sep if rel_path else '', + item.name) + win_olds.append(item) + elif REGEX_INCL_ROOT_ITEMS.search(item.name): + print_success('Auto-Selected: {}'.format(item.path)) + root_items.append('{}'.format(item.path)) + elif not REGEX_EXCL_ROOT_ITEMS.search(item.name): + if not interactive: + print_success('Auto-Selected: {}'.format(item.path)) + root_items.append('{}'.format(item.path)) + else: + prompt = 'Transfer: "{}{}{}" ?'.format( + rel_path, + os.sep if rel_path else '', + item.name) + choices = ['Yes', 'No', 'All', 'Quit'] + answer = choice(prompt=prompt, choices=choices) + if answer == 'Quit': + abort() + elif answer == 'All': + interactive = False + if answer in ['Yes', 'All']: + root_items.append('{}'.format(item.path)) + if root_items: + selected_items.append({ + 'Message': '{}{}Root Items...'.format( + rel_path, + ' ' if rel_path else ''), + 'Items': root_items.copy(), + 'Destination': dest_path}) - # Fonts - font_obj = get_source_item_obj(source_obj, rel_path, 'Windows/Fonts') - if font_obj: - selected_items.append({ - 'Message': '{}{}Fonts...'.format( - rel_path, - ' ' if rel_path else ''), - 'Items': [font_obj.path], - 'Destination': '{}{}Windows'.format( - dest_path, os.sep)}) + # Fonts + font_obj = get_source_item_obj(source_obj, rel_path, 'Windows/Fonts') + if font_obj: + selected_items.append({ + 'Message': '{}{}Fonts...'.format( + rel_path, + ' ' if rel_path else ''), + 'Items': [font_obj.path], + 'Destination': '{}{}Windows'.format( + dest_path, os.sep)}) - # Registry - registry_items = [] - for folder in ['config', 'OEM']: - folder_obj = get_source_item_obj( - source_obj, rel_path, 'Windows/System32/{}'.format(folder)) - if folder_obj: - registry_items.append(folder_obj.path) - if registry_items: - selected_items.append({ - 'Message': '{}{}Registry...'.format( - rel_path, - ' ' if rel_path else ''), - 'Items': registry_items.copy(), - 'Destination': '{}{}Windows{}System32'.format( - dest_path, os.sep, os.sep)}) + # Registry + registry_items = [] + for folder in ['config', 'OEM']: + folder_obj = get_source_item_obj( + source_obj, rel_path, 'Windows/System32/{}'.format(folder)) + if folder_obj: + registry_items.append(folder_obj.path) + if registry_items: + selected_items.append({ + 'Message': '{}{}Registry...'.format( + rel_path, + ' ' if rel_path else ''), + 'Items': registry_items.copy(), + 'Destination': '{}{}Windows{}System32'.format( + dest_path, os.sep, os.sep)}) - # Windows.old(s) - for old in win_olds: - selected_items.extend(scan_source( - source_obj, - '{}{}{}'.format(dest_path, os.sep, old.name), - rel_path=old.name, - interactive=False)) + # Windows.old(s) + for old in win_olds: + selected_items.extend(scan_source( + source_obj, + '{}{}{}'.format(dest_path, os.sep, old.name), + rel_path=old.name, + interactive=False)) - # Done - return selected_items + # Done + return selected_items def get_source_item_obj(source_obj, rel_path, item_path): - """Check if the item exists and return a SourceItem object if it does.""" - item_obj = None - item_path = fix_path_sep(item_path) - if source_obj.is_dir(): - item_obj = SourceItem( - name = item_path, - path = '{}{}{}{}{}'.format( - source_obj.path, - os.sep, - rel_path, - os.sep if rel_path else '', - item_path)) - if not os.path.exists(item_obj.path): - item_obj = None + """Check if the item exists, returns SourceItem object or None.""" + item_obj = None + item_path = fix_path_sep(item_path) + if source_obj.is_dir(): + item_obj = SourceItem( + name = item_path, + path = '{}{}{}{}{}'.format( + source_obj.path, + os.sep, + rel_path, + os.sep if rel_path else '', + item_path)) + if not os.path.exists(item_obj.path): + item_obj = None + else: + # Assuming WIM file + if psutil.WINDOWS: + extract_item('wimlib', silent=True) + cmd = [ + global_vars['Tools']['wimlib-imagex'], 'dir', + source_obj.path, '1', + '--path={}'.format(item_path), + '--one-file-only'] + try: + run_program(cmd) + except subprocess.CalledProcessError: + # function will return None below + pass else: - # Assuming WIM file - if psutil.WINDOWS: - extract_item('wimlib', silent=True) - cmd = [ - global_vars['Tools']['wimlib-imagex'], 'dir', - source_obj.path, '1', - '--path={}'.format(item_path), - '--one-file-only'] - try: - run_program(cmd) - except subprocess.CalledProcessError: - # function will return None below - pass - else: - item_obj = SourceItem( - name = item_path, - path = '{}{}{}{}'.format( - os.sep, - rel_path, - os.sep if rel_path else '', - item_path)) - return item_obj + item_obj = SourceItem( + name = item_path, + path = '{}{}{}{}'.format( + os.sep, + rel_path, + os.sep if rel_path else '', + item_path)) + return item_obj def select_destination(folder_path, prompt='Select destination'): - """Select destination drive, returns path as string.""" - disk = select_volume(prompt) - if 'fixed' not in disk['Disk'].opts: - folder_path = folder_path.replace('\\', '-') - path = '{disk}{folder_path}_{Date}'.format( - disk = disk['Disk'].mountpoint, - folder_path = folder_path, - **global_vars) + """Select destination drive, returns path as string.""" + disk = select_volume(prompt) + if 'fixed' not in disk['Disk'].opts: + folder_path = folder_path.replace('\\', '-') + path = '{disk}{folder_path}_{Date}'.format( + disk = disk['Disk'].mountpoint, + folder_path = folder_path, + **global_vars) - # Avoid merging with existing folder - path = non_clobber_rename(path) - os.makedirs(path, exist_ok=True) + # Avoid merging with existing folder + path = non_clobber_rename(path) + os.makedirs(path, exist_ok=True) - return path + return path def select_source(backup_prefix): - """Select backup from those found on the BACKUP_SERVERS matching the prefix.""" - selected_source = None - local_sources = [] - remote_sources = [] - sources = [] - mount_backup_shares(read_write=False) + """Select matching backup from BACKUP_SERVERS, returns obj.""" + selected_source = None + local_sources = [] + remote_sources = [] + sources = [] + mount_backup_shares(read_write=False) - # Check for prefix folders on servers - for server in BACKUP_SERVERS: - if server['Mounted']: - print_standard('Scanning {}...'.format(server['Name'])) - for d in os.scandir(r'\\{IP}\{Share}'.format(**server)): - if (d.is_dir() - and d.name.lower().startswith(backup_prefix.lower())): - # Add folder to remote_sources - remote_sources.append({ - 'Name': '{:9}| File-Based: [DIR] {}'.format( - server['Name'], d.name), - 'Server': server, - 'Sort': d.name, - 'Source': d}) + # Check for prefix folders on servers + for server in BACKUP_SERVERS: + if server['Mounted']: + print_standard('Scanning {}...'.format(server['Name'])) + for d in os.scandir(r'\\{IP}\{Share}'.format(**server)): + if (d.is_dir() + and d.name.lower().startswith(backup_prefix.lower())): + # Add folder to remote_sources + remote_sources.append({ + 'Name': '{:9}| File-Based: [DIR] {}'.format( + server['Name'], d.name), + 'Server': server, + 'Sort': d.name, + 'Source': d}) - # Check for images and subfolders - for prefix_path in remote_sources.copy(): - for item in os.scandir(prefix_path['Source'].path): - if item.is_dir(): - # Add folder to remote_sources - remote_sources.append({ - 'Name': r'{:9}| File-Based: [DIR] {}\{}'.format( - prefix_path['Server']['Name'], # Server - prefix_path['Source'].name, # Prefix folder - item.name, # Sub-folder - ), - 'Server': prefix_path['Server'], - 'Sort': r'{}\{}'.format( - prefix_path['Source'].name, # Prefix folder - item.name, # Sub-folder - ), - 'Source': item}) + # Check for images and subfolders + for prefix_path in remote_sources.copy(): + for item in os.scandir(prefix_path['Source'].path): + if item.is_dir(): + # Add folder to remote_sources + remote_sources.append({ + 'Name': r'{:9}| File-Based: [DIR] {}\{}'.format( + prefix_path['Server']['Name'], # Server + prefix_path['Source'].name, # Prefix folder + item.name, # Sub-folder + ), + 'Server': prefix_path['Server'], + 'Sort': r'{}\{}'.format( + prefix_path['Source'].name, # Prefix folder + item.name, # Sub-folder + ), + 'Source': item}) - # Check for images in folder - for subitem in os.scandir(item.path): - if REGEX_WIM_FILE.search(item.name): - # Add image to remote_sources - try: - size = human_readable_size(item.stat().st_size) - except Exception: - size = ' ? ?' # unknown - remote_sources.append({ - 'Disabled': bool(not is_valid_wim_file(subitem)), - 'Name': r'{:9}| Image-Based: {:>7} {}\{}\{}'.format( - prefix_path['Server']['Name'], # Server - size, # Size (duh) - prefix_path['Source'].name, # Prefix folder - item.name, # Sub-folder - subitem.name, # Image file - ), - 'Server': prefix_path['Server'], - 'Sort': r'{}\{}\{}'.format( - prefix_path['Source'].name, # Prefix folder - item.name, # Sub-folder - subitem.name, # Image file - ), - 'Source': subitem}) - elif REGEX_WIM_FILE.search(item.name): - # Add image to remote_sources - try: - size = human_readable_size(item.stat().st_size) - except Exception: - size = ' ? ?' # unknown - remote_sources.append({ - 'Disabled': bool(not is_valid_wim_file(item)), - 'Name': r'{:9}| Image-Based: {:>7} {}\{}'.format( - prefix_path['Server']['Name'], # Server - size, # Size (duh) - prefix_path['Source'].name, # Prefix folder - item.name, # Image file - ), - 'Server': prefix_path['Server'], - 'Sort': r'{}\{}'.format( - prefix_path['Source'].name, # Prefix folder - item.name, # Image file - ), - 'Source': item}) + # Check for images in folder + for subitem in os.scandir(item.path): + if REGEX_WIM_FILE.search(item.name): + # Add image to remote_sources + try: + size = human_readable_size(item.stat().st_size) + except Exception: + size = ' ? ?' # unknown + remote_sources.append({ + 'Disabled': bool(not is_valid_wim_file(subitem)), + 'Name': r'{:9}| Image-Based: {:>7} {}\{}\{}'.format( + prefix_path['Server']['Name'], # Server + size, # Size (duh) + prefix_path['Source'].name, # Prefix folder + item.name, # Sub-folder + subitem.name, # Image file + ), + 'Server': prefix_path['Server'], + 'Sort': r'{}\{}\{}'.format( + prefix_path['Source'].name, # Prefix folder + item.name, # Sub-folder + subitem.name, # Image file + ), + 'Source': subitem}) + elif REGEX_WIM_FILE.search(item.name): + # Add image to remote_sources + try: + size = human_readable_size(item.stat().st_size) + except Exception: + size = ' ? ?' # unknown + remote_sources.append({ + 'Disabled': bool(not is_valid_wim_file(item)), + 'Name': r'{:9}| Image-Based: {:>7} {}\{}'.format( + prefix_path['Server']['Name'], # Server + size, # Size (duh) + prefix_path['Source'].name, # Prefix folder + item.name, # Image file + ), + 'Server': prefix_path['Server'], + 'Sort': r'{}\{}'.format( + prefix_path['Source'].name, # Prefix folder + item.name, # Image file + ), + 'Source': item}) - # Check for local sources - print_standard('Scanning for local sources...') - set_thread_error_mode(silent=True) # Prevents "No disk" popups - sys_drive = global_vars['Env']['SYSTEMDRIVE'] - for d in psutil.disk_partitions(): - if re.search(r'^{}'.format(sys_drive), d.mountpoint, re.IGNORECASE): - # Skip current OS drive - continue - if 'fixed' in d.opts: - # Skip DVD, etc - local_sources.append({ - 'Name': '{:9}| File-Based: [DISK] {}'.format( - ' Local', d.mountpoint), - 'Sort': d.mountpoint, - 'Source': LocalDisk(d)}) - # Check for images and subfolders - for item in os.scandir(d.mountpoint): - if REGEX_WIM_FILE.search(item.name): - try: - size = human_readable_size(item.stat().st_size) - except Exception: - size = ' ? ?' # unknown - local_sources.append({ - 'Disabled': bool(not is_valid_wim_file(item)), - 'Name': r'{:9}| Image-Based: {:>7} {}{}'.format( - ' Local', size, d.mountpoint, item.name), - 'Sort': r'{}{}'.format(d.mountpoint, item.name), - 'Source': item}) - elif REGEX_EXCL_ROOT_ITEMS.search(item.name): - pass - elif REGEX_EXCL_ITEMS.search(item.name): - pass - elif item.is_dir(): - # Add folder to local_sources - local_sources.append({ - 'Name': r'{:9}| File-Based: [DIR] {}{}'.format( - ' Local', d.mountpoint, item.name), - 'Sort': r'{}{}'.format(d.mountpoint, item.name), - 'Source': item}) + # Check for local sources + print_standard('Scanning for local sources...') + set_thread_error_mode(silent=True) # Prevents "No disk" popups + sys_drive = global_vars['Env']['SYSTEMDRIVE'] + for d in psutil.disk_partitions(): + if re.search(r'^{}'.format(sys_drive), d.mountpoint, re.IGNORECASE): + # Skip current OS drive + continue + if 'fixed' in d.opts: + # Skip DVD, etc + local_sources.append({ + 'Name': '{:9}| File-Based: [DISK] {}'.format( + ' Local', d.mountpoint), + 'Sort': d.mountpoint, + 'Source': LocalDisk(d)}) + # Check for images and subfolders + for item in os.scandir(d.mountpoint): + if REGEX_WIM_FILE.search(item.name): + try: + size = human_readable_size(item.stat().st_size) + except Exception: + size = ' ? ?' # unknown + local_sources.append({ + 'Disabled': bool(not is_valid_wim_file(item)), + 'Name': r'{:9}| Image-Based: {:>7} {}{}'.format( + ' Local', size, d.mountpoint, item.name), + 'Sort': r'{}{}'.format(d.mountpoint, item.name), + 'Source': item}) + elif REGEX_EXCL_ROOT_ITEMS.search(item.name): + pass + elif REGEX_EXCL_ITEMS.search(item.name): + pass + elif item.is_dir(): + # Add folder to local_sources + local_sources.append({ + 'Name': r'{:9}| File-Based: [DIR] {}{}'.format( + ' Local', d.mountpoint, item.name), + 'Sort': r'{}{}'.format(d.mountpoint, item.name), + 'Source': item}) - set_thread_error_mode(silent=False) # Return to normal + set_thread_error_mode(silent=False) # Return to normal - # Build Menu - local_sources.sort(key=itemgetter('Sort')) - remote_sources.sort(key=itemgetter('Sort')) - sources.extend(local_sources) - sources.extend(remote_sources) - actions = [{'Name': 'Quit', 'Letter': 'Q'}] + # Build Menu + local_sources.sort(key=itemgetter('Sort')) + remote_sources.sort(key=itemgetter('Sort')) + sources.extend(local_sources) + sources.extend(remote_sources) + actions = [{'Name': 'Quit', 'Letter': 'Q'}] - # Select backup from sources - if len(sources) > 0: - selection = menu_select( - 'Which backup are we using?', - main_entries=sources, - action_entries=actions, - disabled_label='DAMAGED') - if selection == 'Q': - umount_backup_shares() - exit_script() - else: - selected_source = sources[int(selection)-1]['Source'] + # Select backup from sources + if len(sources) > 0: + selection = menu_select( + 'Which backup are we using?', + main_entries=sources, + action_entries=actions, + disabled_label='DAMAGED') + if selection == 'Q': + umount_backup_shares() + exit_script() else: - print_error('ERROR: No backups found using prefix: {}.'.format( - backup_prefix)) - umount_backup_shares() - pause("Press Enter to exit...") - exit_script() + selected_source = sources[int(selection)-1]['Source'] + else: + print_error('ERROR: No backups found using prefix: {}.'.format( + backup_prefix)) + umount_backup_shares() + pause("Press Enter to exit...") + exit_script() - # Sanity check - if selected_source.is_file(): - # Image-Based - if not REGEX_WIM_FILE.search(selected_source.name): - print_error('ERROR: Unsupported image: {}'.format( - selected_source.path)) - raise GenericError + # Sanity check + if selected_source.is_file(): + # Image-Based + if not REGEX_WIM_FILE.search(selected_source.name): + print_error('ERROR: Unsupported image: {}'.format( + selected_source.path)) + raise GenericError - # Done - return selected_source + # Done + return selected_source def select_volume(title='Select disk', auto_select=True): - """Select disk from attached disks. returns dict.""" - actions = [{'Name': 'Quit', 'Letter': 'Q'}] - disks = [] + """Select disk from attached disks. returns dict.""" + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + disks = [] - # Build list of disks - set_thread_error_mode(silent=True) # Prevents "No disk" popups - for d in psutil.disk_partitions(): - info = { - 'Disk': d, - 'Name': d.mountpoint} - try: - usage = psutil.disk_usage(d.device) - free = '{free} / {total} available'.format( - free = human_readable_size(usage.free, 2), - total = human_readable_size(usage.total, 2)) - except Exception: - # Meh, leaving unsupported destinations out - pass - # free = 'Unknown' - # info['Disabled'] = True - else: - info['Display Name'] = '{} ({})'.format(info['Name'], free) - disks.append(info) - set_thread_error_mode(silent=False) # Return to normal - - # Skip menu? - if len(disks) == 1 and auto_select: - return disks[0] - - # Show menu - selection = menu_select(title, main_entries=disks, action_entries=actions) - if selection == 'Q': - exit_script() + # Build list of disks + set_thread_error_mode(silent=True) # Prevents "No disk" popups + for d in psutil.disk_partitions(): + info = { + 'Disk': d, + 'Name': d.mountpoint} + try: + usage = psutil.disk_usage(d.device) + free = '{free} / {total} available'.format( + free = human_readable_size(usage.free, 2), + total = human_readable_size(usage.total, 2)) + except Exception: + # Meh, leaving unsupported destinations out + pass + # free = 'Unknown' + # info['Disabled'] = True else: - return disks[int(selection)-1] + info['Display Name'] = '{} ({})'.format(info['Name'], free) + disks.append(info) + set_thread_error_mode(silent=False) # Return to normal + + # Skip menu? + if len(disks) == 1 and auto_select: + return disks[0] + + # Show menu + selection = menu_select(title, main_entries=disks, action_entries=actions) + if selection == 'Q': + exit_script() + else: + return disks[int(selection)-1] def set_thread_error_mode(silent=True): - """Disable or Enable Windows error message dialogs. + """Disable or Enable Windows error message dialogs. - Disable when scanning for disks to avoid popups for empty cardreaders, etc - """ - # Code borrowed from: https://stackoverflow.com/a/29075319 - kernel32 = ctypes.WinDLL('kernel32') + Disable when scanning disks to avoid popups for empty cardreaders, etc + """ + # Code borrowed from: https://stackoverflow.com/a/29075319 + kernel32 = ctypes.WinDLL('kernel32') - if silent: - kernel32.SetThreadErrorMode(SEM_FAIL, ctypes.byref(SEM_NORMAL)) - else: - kernel32.SetThreadErrorMode(SEM_NORMAL, ctypes.byref(SEM_NORMAL)) + if silent: + kernel32.SetThreadErrorMode(SEM_FAIL, ctypes.byref(SEM_NORMAL)) + else: + kernel32.SetThreadErrorMode(SEM_NORMAL, ctypes.byref(SEM_NORMAL)) def transfer_source(source_obj, dest_path, selected_items): - """Transfer, or extract, files/folders from source to destination.""" - if source_obj.is_dir(): - # Run FastCopy for each selection "group" - for group in selected_items: - try_and_print(message=group['Message'], - function=run_fast_copy, cs='Done', - items=group['Items'], - dest=group['Destination']) + """Transfer, or extract, files/folders from source to destination.""" + if source_obj.is_dir(): + # Run FastCopy for each selection "group" + for group in selected_items: + try_and_print(message=group['Message'], + function=run_fast_copy, cs='Done', + items=group['Items'], + dest=group['Destination']) + else: + if REGEX_WIM_FILE.search(source_obj.name): + # Extract files from WIM + for group in selected_items: + try_and_print(message=group['Message'], + function=run_wimextract, cs='Done', + source=source_obj.path, + items=group['Items'], + dest=dest_path) else: - if REGEX_WIM_FILE.search(source_obj.name): - # Extract files from WIM - for group in selected_items: - try_and_print(message=group['Message'], - function=run_wimextract, cs='Done', - source=source_obj.path, - items=group['Items'], - dest=dest_path) - else: - print_error('ERROR: Unsupported image: {}'.format(source_obj.path)) - raise GenericError + print_error('ERROR: Unsupported image: {}'.format(source_obj.path)) + raise GenericError def umount_backup_shares(): - """Unmount the backup shares regardless of current status.""" - for server in BACKUP_SERVERS: - umount_network_share(server) + """Unmount the backup shares regardless of current status.""" + for server in BACKUP_SERVERS: + umount_network_share(server) def umount_network_share(server): - """Unmount a network share defined by server.""" - cmd = r'net use \\{IP}\{Share} /delete'.format(**server) - cmd = cmd.split(' ') - try: - run_program(cmd) - except Exception: - print_error(r'Failed to umount \\{Name}\{Share}.'.format(**server)) - sleep(1) - else: - print_info('Umounted {Name}'.format(**server)) - server['Mounted'] = False + """Unmount a network share defined by server.""" + cmd = r'net use \\{IP}\{Share} /delete'.format(**server) + cmd = cmd.split(' ') + try: + run_program(cmd) + except Exception: + print_error(r'Failed to umount \\{Name}\{Share}.'.format(**server)) + sleep(1) + else: + print_info('Umounted {Name}'.format(**server)) + server['Mounted'] = False if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 10e978d4c5657a412999a96609cd0191d3137461 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:50:51 -0700 Subject: [PATCH 117/186] Updated ddrescue.py --- .bin/Scripts/functions/ddrescue.py | 2282 ++++++++++++++-------------- 1 file changed, 1141 insertions(+), 1141 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 6e515655..7e3a06df 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -19,1345 +19,1345 @@ from operator import itemgetter AUTO_PASS_1_THRESHOLD = 95 AUTO_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { - '--binary-prefixes': {'Enabled': True, 'Hidden': True}, - '--data-preview': {'Enabled': True, 'Hidden': True, 'Value': '5'}, - '--idirect': {'Enabled': True}, - '--odirect': {'Enabled': True}, - '--max-read-rate': {'Enabled': False, 'Value': '1MiB'}, - '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, - '--reopen-on-error': {'Enabled': True}, - '--retry-passes': {'Enabled': True, 'Value': '0'}, - '--test-mode': {'Enabled': False, 'Value': 'test.map'}, - '--timeout': {'Enabled': True, 'Value': '5m'}, - '-vvvv': {'Enabled': True, 'Hidden': True}, - } + '--binary-prefixes': {'Enabled': True, 'Hidden': True}, + '--data-preview': {'Enabled': True, 'Hidden': True, 'Value': '5'}, + '--idirect': {'Enabled': True}, + '--odirect': {'Enabled': True}, + '--max-read-rate': {'Enabled': False, 'Value': '1MiB'}, + '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, + '--reopen-on-error': {'Enabled': True}, + '--retry-passes': {'Enabled': True, 'Value': '0'}, + '--test-mode': {'Enabled': False, 'Value': 'test.map'}, + '--timeout': {'Enabled': True, 'Value': '5m'}, + '-vvvv': {'Enabled': True, 'Hidden': True}, + } RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] SIDE_PANE_WIDTH = 21 TMUX_LAYOUT = OrderedDict({ - 'Source': {'y': 2, 'Check': True}, + 'Source': {'y': 2, 'Check': True}, 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, }) -USAGE = """ {script_name} clone [source [destination]] - {script_name} image [source [destination]] - (e.g. {script_name} clone /dev/sda /dev/sdb) +USAGE = """ {script_name} clone [source [destination]] + {script_name} image [source [destination]] + (e.g. {script_name} clone /dev/sda /dev/sdb) """ # Clases class BaseObj(): - """Base object used by DevObj, DirObj, and ImageObj.""" - def __init__(self, path): - self.type = 'base' - self.parent = None - self.path = os.path.realpath(path) - self.set_details() + """Base object used by DevObj, DirObj, and ImageObj.""" + def __init__(self, path): + self.type = 'base' + self.parent = None + self.path = os.path.realpath(path) + self.set_details() - def is_dev(self): - return self.type == 'dev' + def is_dev(self): + return self.type == 'dev' - def is_dir(self): - return self.type == 'dir' + def is_dir(self): + return self.type == 'dir' - def is_image(self): - return self.type == 'image' + def is_image(self): + return self.type == 'image' - def self_check(self): - pass + def self_check(self): + pass - def set_details(self): - self.details = {} + def set_details(self): + self.details = {} class BlockPair(): - """Object to track data and methods together for source and dest.""" - def __init__(self, mode, source, dest): - self.mode = mode - self.source = source - self.source_path = source.path - self.dest = dest - self.pass_done = [False, False, False] - self.resumed = False - self.rescued = 0 - self.rescued_percent = 0 - self.status = ['Pending', 'Pending', 'Pending'] - self.size = source.size - # Set dest paths - if self.mode == 'clone': - # Cloning - self.dest_path = dest.path - self.map_path = '{pwd}/Clone_{prefix}.map'.format( - pwd=os.path.realpath(global_vars['Env']['PWD']), - prefix=source.prefix) - else: - # Imaging - self.dest_path = '{path}/{prefix}.dd'.format( - path=dest.path, - prefix=source.prefix) - self.map_path = '{path}/{prefix}.map'.format( - path=dest.path, - prefix=source.prefix) - if os.path.exists(self.map_path): - self.load_map_data() - self.resumed = True - self.fix_status_strings() + """Object to track data and methods together for source and dest.""" + def __init__(self, mode, source, dest): + self.mode = mode + self.source = source + self.source_path = source.path + self.dest = dest + self.pass_done = [False, False, False] + self.resumed = False + self.rescued = 0 + self.rescued_percent = 0 + self.status = ['Pending', 'Pending', 'Pending'] + self.size = source.size + # Set dest paths + if self.mode == 'clone': + # Cloning + self.dest_path = dest.path + self.map_path = '{pwd}/Clone_{prefix}.map'.format( + pwd=os.path.realpath(global_vars['Env']['PWD']), + prefix=source.prefix) + else: + # Imaging + self.dest_path = '{path}/{prefix}.dd'.format( + path=dest.path, + prefix=source.prefix) + self.map_path = '{path}/{prefix}.map'.format( + path=dest.path, + prefix=source.prefix) + if os.path.exists(self.map_path): + self.load_map_data() + self.resumed = True + self.fix_status_strings() - def fix_status_strings(self): - """Format status strings via get_formatted_status().""" - for pass_num in [1, 2, 3]: - self.status[pass_num-1] = get_formatted_status( - label='Pass {}'.format(pass_num), - data=self.status[pass_num-1]) + def fix_status_strings(self): + """Format status strings via get_formatted_status().""" + for pass_num in [1, 2, 3]: + self.status[pass_num-1] = get_formatted_status( + label='Pass {}'.format(pass_num), + data=self.status[pass_num-1]) - def finish_pass(self, pass_num): - """Mark pass as done and check if 100% recovered.""" - map_data = read_map_file(self.map_path) - if map_data['full recovery']: - self.pass_done = [True, True, True] - self.rescued = self.size - self.status[pass_num] = get_formatted_status( - label='Pass {}'.format(pass_num+1), - data=100) - # Mark future passes as Skipped - pass_num += 1 - while pass_num <= 2: - self.status[pass_num] = get_formatted_status( - label='Pass {}'.format(pass_num+1), - data='Skipped') - pass_num += 1 - else: - self.pass_done[pass_num] = True + def finish_pass(self, pass_num): + """Mark pass as done and check if 100% recovered.""" + map_data = read_map_file(self.map_path) + if map_data['full recovery']: + self.pass_done = [True, True, True] + self.rescued = self.size + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num+1), + data=100) + # Mark future passes as Skipped + pass_num += 1 + while pass_num <= 2: + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num+1), + data='Skipped') + pass_num += 1 + else: + self.pass_done[pass_num] = True - def load_map_data(self): - """Load data from map file and set progress.""" - map_data = read_map_file(self.map_path) - self.rescued_percent = map_data['rescued'] - self.rescued = (self.rescued_percent * self.size) / 100 - if map_data['full recovery']: - self.pass_done = [True, True, True] - self.rescued = self.size - self.status = ['Skipped', 'Skipped', 'Skipped'] - elif map_data['non-tried'] > 0: - # Initial pass incomplete - pass - elif map_data['non-trimmed'] > 0: - self.pass_done = [True, False, False] - self.status = ['Skipped', 'Pending', 'Pending'] - elif map_data['non-scraped'] > 0: - self.pass_done = [True, True, False] - self.status = ['Skipped', 'Skipped', 'Pending'] - else: - self.pass_done = [True, True, True] - self.status = ['Skipped', 'Skipped', 'Skipped'] + def load_map_data(self): + """Load data from map file and set progress.""" + map_data = read_map_file(self.map_path) + self.rescued_percent = map_data['rescued'] + self.rescued = (self.rescued_percent * self.size) / 100 + if map_data['full recovery']: + self.pass_done = [True, True, True] + self.rescued = self.size + self.status = ['Skipped', 'Skipped', 'Skipped'] + elif map_data['non-tried'] > 0: + # Initial pass incomplete + pass + elif map_data['non-trimmed'] > 0: + self.pass_done = [True, False, False] + self.status = ['Skipped', 'Pending', 'Pending'] + elif map_data['non-scraped'] > 0: + self.pass_done = [True, True, False] + self.status = ['Skipped', 'Skipped', 'Pending'] + else: + self.pass_done = [True, True, True] + self.status = ['Skipped', 'Skipped', 'Skipped'] - def self_check(self): - """Self check to abort on bad dest/map combinations.""" - dest_exists = os.path.exists(self.dest_path) - map_exists = os.path.exists(self.map_path) - if self.mode == 'image': - if dest_exists and not map_exists: - raise GenericError( - 'Detected image "{}" but not the matching map'.format( - self.dest_path)) - elif not dest_exists and map_exists: - raise GenericError( - 'Detected map "{}" but not the matching image'.format( - self.map_path)) - elif not dest_exists: - raise GenericError('Destination device "{}" missing'.format( - self.dest_path)) + def self_check(self): + """Self check to abort on bad dest/map combinations.""" + dest_exists = os.path.exists(self.dest_path) + map_exists = os.path.exists(self.map_path) + if self.mode == 'image': + if dest_exists and not map_exists: + raise GenericError( + 'Detected image "{}" but not the matching map'.format( + self.dest_path)) + elif not dest_exists and map_exists: + raise GenericError( + 'Detected map "{}" but not the matching image'.format( + self.map_path)) + elif not dest_exists: + raise GenericError('Destination device "{}" missing'.format( + self.dest_path)) - def update_progress(self, pass_num): - """Update progress using map file.""" - if os.path.exists(self.map_path): - map_data = read_map_file(self.map_path) - self.rescued_percent = map_data.get('rescued', 0) - self.rescued = (self.rescued_percent * self.size) / 100 - self.status[pass_num] = get_formatted_status( - label='Pass {}'.format(pass_num+1), - data=(self.rescued/self.size)*100) + def update_progress(self, pass_num): + """Update progress using map file.""" + if os.path.exists(self.map_path): + map_data = read_map_file(self.map_path) + self.rescued_percent = map_data.get('rescued', 0) + self.rescued = (self.rescued_percent * self.size) / 100 + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num+1), + data=(self.rescued/self.size)*100) class DevObj(BaseObj): - """Block device object.""" - def self_check(self): - """Verify that self.path points to a block device.""" - if not pathlib.Path(self.path).is_block_device(): - raise GenericError('Path "{}" is not a block device.'.format( - self.path)) - if self.parent: - print_warning('"{}" is a child device.'.format(self.path)) - if ask('Use parent device "{}" instead?'.format(self.parent)): - self.path = os.path.realpath(self.parent) - self.set_details() + """Block device object.""" + def self_check(self): + """Verify that self.path points to a block device.""" + if not pathlib.Path(self.path).is_block_device(): + raise GenericError('Path "{}" is not a block device.'.format( + self.path)) + if self.parent: + print_warning('"{}" is a child device.'.format(self.path)) + if ask('Use parent device "{}" instead?'.format(self.parent)): + self.path = os.path.realpath(self.parent) + self.set_details() - def set_details(self): - """Set details via lsblk.""" - self.type = 'dev' - self.details = get_device_details(self.path) - self.name = '{name} {size} {model} {serial}'.format( - name=self.details.get('name', 'UNKNOWN'), - size=self.details.get('size', 'UNKNOWN'), - model=self.details.get('model', 'UNKNOWN'), - serial=self.details.get('serial', 'UNKNOWN')) - self.model = self.details.get('model', 'UNKNOWN') - self.model_size = self.details.get('size', 'UNKNOWN') - self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) - self.report = get_device_report(self.path) - self.parent = self.details.get('pkname', '') - self.label = self.details.get('label', '') - if not self.label: - # Force empty string in case it's set to None - self.label = '' - self.update_filename_prefix() + def set_details(self): + """Set details via lsblk.""" + self.type = 'dev' + self.details = get_device_details(self.path) + self.name = '{name} {size} {model} {serial}'.format( + name=self.details.get('name', 'UNKNOWN'), + size=self.details.get('size', 'UNKNOWN'), + model=self.details.get('model', 'UNKNOWN'), + serial=self.details.get('serial', 'UNKNOWN')) + self.model = self.details.get('model', 'UNKNOWN') + self.model_size = self.details.get('size', 'UNKNOWN') + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_device_report(self.path) + self.parent = self.details.get('pkname', '') + self.label = self.details.get('label', '') + if not self.label: + # Force empty string in case it's set to None + self.label = '' + self.update_filename_prefix() - def update_filename_prefix(self): - """Set filename prefix based on details.""" - self.prefix = '{m_size}_{model}'.format( - m_size=self.model_size, - model=self.model) - self.prefix = self.prefix.strip() - if self.parent: - # Add child device details - c_num = self.path.replace(self.parent, '') - self.prefix += '_{c_prefix}{c_num}_{c_size}{sep}{c_label}'.format( - c_prefix='p' if len(c_num) == 1 else '', - c_num=c_num, - c_size=self.details.get('size', 'UNKNOWN'), - sep='_' if self.label else '', - c_label=self.label) - self.prefix = self.prefix.strip().replace(' ', '_') - self.prefix = self.prefix.strip().replace('/', '_') + def update_filename_prefix(self): + """Set filename prefix based on details.""" + self.prefix = '{m_size}_{model}'.format( + m_size=self.model_size, + model=self.model) + self.prefix = self.prefix.strip() + if self.parent: + # Add child device details + c_num = self.path.replace(self.parent, '') + self.prefix += '_{c_prefix}{c_num}_{c_size}{sep}{c_label}'.format( + c_prefix='p' if len(c_num) == 1 else '', + c_num=c_num, + c_size=self.details.get('size', 'UNKNOWN'), + sep='_' if self.label else '', + c_label=self.label) + self.prefix = self.prefix.strip().replace(' ', '_') + self.prefix = self.prefix.strip().replace('/', '_') class DirObj(BaseObj): - def self_check(self): - """Verify that self.path points to a directory.""" - if not pathlib.Path(self.path).is_dir(): - raise GenericError('Path "{}" is not a directory.'.format( - self.path)) + def self_check(self): + """Verify that self.path points to a directory.""" + if not pathlib.Path(self.path).is_dir(): + raise GenericError('Path "{}" is not a directory.'.format( + self.path)) - def set_details(self): - """Set details via findmnt.""" - self.type = 'dir' - self.details = get_dir_details(self.path) - self.fstype = self.details.get('fstype', 'UNKNOWN') - self.name = self.path + '/' - self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) - self.report = get_dir_report(self.path) + def set_details(self): + """Set details via findmnt.""" + self.type = 'dir' + self.details = get_dir_details(self.path) + self.fstype = self.details.get('fstype', 'UNKNOWN') + self.name = self.path + '/' + self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) + self.report = get_dir_report(self.path) class ImageObj(BaseObj): - def self_check(self): - """Verify that self.path points to a file.""" - if not pathlib.Path(self.path).is_file(): - raise GenericError('Path "{}" is not an image file.'.format( - self.path)) + def self_check(self): + """Verify that self.path points to a file.""" + if not pathlib.Path(self.path).is_file(): + raise GenericError('Path "{}" is not an image file.'.format( + self.path)) - def set_details(self): - """Setup loopback device, set details via lsblk, then detach device.""" - self.type = 'image' - self.loop_dev = setup_loopback_device(self.path) - self.details = get_device_details(self.loop_dev) - self.details['model'] = 'ImageFile' - self.name = '{name} {size}'.format( - name=self.path[self.path.rfind('/')+1:], - size=self.details.get('size', 'UNKNOWN')) - self.prefix = '{}_ImageFile'.format( - self.details.get('size', 'UNKNOWN')) - self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) - self.report = get_device_report(self.loop_dev) - self.report = self.report.replace( - self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)') - run_program(['losetup', '--detach', self.loop_dev], check=False) + def set_details(self): + """Setup loopback device, set details via lsblk, then detach device.""" + self.type = 'image' + self.loop_dev = setup_loopback_device(self.path) + self.details = get_device_details(self.loop_dev) + self.details['model'] = 'ImageFile' + self.name = '{name} {size}'.format( + name=self.path[self.path.rfind('/')+1:], + size=self.details.get('size', 'UNKNOWN')) + self.prefix = '{}_ImageFile'.format( + self.details.get('size', 'UNKNOWN')) + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_device_report(self.loop_dev) + self.report = self.report.replace( + self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)') + run_program(['losetup', '--detach', self.loop_dev], check=False) class RecoveryState(): - """Object to track BlockPair objects and overall state.""" - def __init__(self, mode, source, dest): - self.mode = mode.lower() - self.source = source - self.source_path = source.path - self.dest = dest - self.block_pairs = [] - self.current_pass = 0 - self.current_pass_str = '0: Initializing' - self.settings = DDRESCUE_SETTINGS.copy() - self.finished = False - self.panes = {} - self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) - self.rescued = 0 - self.resumed = False - self.started = False - self.total_size = 0 - if mode not in ('clone', 'image'): - raise GenericError('Unsupported mode') - self.get_smart_source() + """Object to track BlockPair objects and overall state.""" + def __init__(self, mode, source, dest): + self.mode = mode.lower() + self.source = source + self.source_path = source.path + self.dest = dest + self.block_pairs = [] + self.current_pass = 0 + self.current_pass_str = '0: Initializing' + self.settings = DDRESCUE_SETTINGS.copy() + self.finished = False + self.panes = {} + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.rescued = 0 + self.resumed = False + self.started = False + self.total_size = 0 + if mode not in ('clone', 'image'): + raise GenericError('Unsupported mode') + self.get_smart_source() - def add_block_pair(self, source, dest): - """Run safety checks and append new BlockPair to internal list.""" - if self.mode == 'clone': - # Cloning safety checks - if source.is_dir(): - raise GenericError('Invalid source "{}"'.format( - source.path)) - elif not dest.is_dev(): - raise GenericError('Invalid destination "{}"'.format( - dest.path)) - elif source.size > dest.size: - raise GenericError( - 'Destination is too small, refusing to continue.') - else: - # Imaging safety checks - if not source.is_dev(): - raise GenericError('Invalid source "{}"'.format( - source.path)) - elif not dest.is_dir(): - raise GenericError('Invalid destination "{}"'.format( - dest.path)) - elif (source.size * 1.2) > dest.size: - raise GenericError( - 'Not enough free space, refusing to continue.') - elif dest.fstype.lower() not in RECOMMENDED_FSTYPES: - print_error( - 'Destination filesystem "{}" is not recommended.'.format( - dest.fstype.upper())) - print_info('Recommended types are: {}'.format( - ' / '.join(RECOMMENDED_FSTYPES).upper())) - print_standard(' ') - if not ask('Proceed anyways? (Strongly discouraged)'): - raise GenericAbort() - elif not is_writable_dir(dest): - raise GenericError( - 'Destination is not writable, refusing to continue.') - elif not is_writable_filesystem(dest): - raise GenericError( - 'Destination is mounted read-only, refusing to continue.') + def add_block_pair(self, source, dest): + """Run safety checks and append new BlockPair to internal list.""" + if self.mode == 'clone': + # Cloning safety checks + if source.is_dir(): + raise GenericError('Invalid source "{}"'.format( + source.path)) + elif not dest.is_dev(): + raise GenericError('Invalid destination "{}"'.format( + dest.path)) + elif source.size > dest.size: + raise GenericError( + 'Destination is too small, refusing to continue.') + else: + # Imaging safety checks + if not source.is_dev(): + raise GenericError('Invalid source "{}"'.format( + source.path)) + elif not dest.is_dir(): + raise GenericError('Invalid destination "{}"'.format( + dest.path)) + elif (source.size * 1.2) > dest.size: + raise GenericError( + 'Not enough free space, refusing to continue.') + elif dest.fstype.lower() not in RECOMMENDED_FSTYPES: + print_error( + 'Destination filesystem "{}" is not recommended.'.format( + dest.fstype.upper())) + print_info('Recommended types are: {}'.format( + ' / '.join(RECOMMENDED_FSTYPES).upper())) + print_standard(' ') + if not ask('Proceed anyways? (Strongly discouraged)'): + raise GenericAbort() + elif not is_writable_dir(dest): + raise GenericError( + 'Destination is not writable, refusing to continue.') + elif not is_writable_filesystem(dest): + raise GenericError( + 'Destination is mounted read-only, refusing to continue.') - # Safety checks passed - self.block_pairs.append(BlockPair(self.mode, source, dest)) + # Safety checks passed + self.block_pairs.append(BlockPair(self.mode, source, dest)) - def current_pass_done(self): - """Checks if pass is done for all block-pairs, returns bool.""" - done = True - for bp in self.block_pairs: - done &= bp.pass_done[self.current_pass] - return done + def current_pass_done(self): + """Checks if pass is done for all block-pairs, returns bool.""" + done = True + for bp in self.block_pairs: + done &= bp.pass_done[self.current_pass] + return done - def current_pass_min(self): - """Gets minimum pass rescued percentage, returns float.""" - min_percent = 100 - for bp in self.block_pairs: - min_percent = min(min_percent, bp.rescued_percent) - return min_percent + def current_pass_min(self): + """Gets minimum pass rescued percentage, returns float.""" + min_percent = 100 + for bp in self.block_pairs: + min_percent = min(min_percent, bp.rescued_percent) + return min_percent - def get_smart_source(self): - """Get source for SMART dispay.""" - disk_path = self.source.path - if self.source.parent: - disk_path = self.source.parent + def get_smart_source(self): + """Get source for SMART dispay.""" + disk_path = self.source.path + if self.source.parent: + disk_path = self.source.parent - self.smart_source = DiskObj(disk_path) + self.smart_source = DiskObj(disk_path) - def retry_all_passes(self): - """Mark all passes as pending for all block-pairs.""" - self.finished = False - for bp in self.block_pairs: - bp.pass_done = [False, False, False] - bp.status = ['Pending', 'Pending', 'Pending'] - bp.fix_status_strings() - self.set_pass_num() + def retry_all_passes(self): + """Mark all passes as pending for all block-pairs.""" + self.finished = False + for bp in self.block_pairs: + bp.pass_done = [False, False, False] + bp.status = ['Pending', 'Pending', 'Pending'] + bp.fix_status_strings() + self.set_pass_num() - def self_checks(self): - """Run self-checks and update state values.""" - cmd = ['findmnt', '--json', '--target', os.getcwd()] - map_allowed_fstypes = RECOMMENDED_FSTYPES.copy() - map_allowed_fstypes.extend(['cifs', 'ext2', 'vfat']) - map_allowed_fstypes.sort() - json_data = {} + def self_checks(self): + """Run self-checks and update state values.""" + cmd = ['findmnt', '--json', '--target', os.getcwd()] + map_allowed_fstypes = RECOMMENDED_FSTYPES.copy() + map_allowed_fstypes.extend(['cifs', 'ext2', 'vfat']) + map_allowed_fstypes.sort() + json_data = {} - # Avoid saving map to non-persistent filesystem - try: - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - except Exception: - print_error('ERROR: Failed to verify map path') - raise GenericAbort() - fstype = json_data.get( - 'filesystems', [{}])[0].get( - 'fstype', 'unknown') - if fstype not in map_allowed_fstypes: - print_error( - "Map isn't being saved to a recommended filesystem ({})".format( - fstype.upper())) - print_info('Recommended types are: {}'.format( - ' / '.join(map_allowed_fstypes).upper())) - print_standard(' ') - if not ask('Proceed anyways? (Strongly discouraged)'): - raise GenericAbort() + # Avoid saving map to non-persistent filesystem + try: + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + except Exception: + print_error('ERROR: Failed to verify map path') + raise GenericAbort() + fstype = json_data.get( + 'filesystems', [{}])[0].get( + 'fstype', 'unknown') + if fstype not in map_allowed_fstypes: + print_error( + "Map isn't being saved to a recommended filesystem ({})".format( + fstype.upper())) + print_info('Recommended types are: {}'.format( + ' / '.join(map_allowed_fstypes).upper())) + print_standard(' ') + if not ask('Proceed anyways? (Strongly discouraged)'): + raise GenericAbort() - # Run BlockPair self checks and get total size - self.total_size = 0 - for bp in self.block_pairs: - bp.self_check() - self.resumed |= bp.resumed - self.total_size += bp.size + # Run BlockPair self checks and get total size + self.total_size = 0 + for bp in self.block_pairs: + bp.self_check() + self.resumed |= bp.resumed + self.total_size += bp.size - def set_pass_num(self): - """Set current pass based on all block-pair's progress.""" - self.current_pass = 0 - for pass_num in (2, 1, 0): - # Iterate backwards through passes - pass_done = True - for bp in self.block_pairs: - pass_done &= bp.pass_done[pass_num] - if pass_done: - # All block-pairs reported being done - # Set to next pass, unless we're on the last pass (2) - self.current_pass = min(2, pass_num + 1) - if pass_num == 2: - # Also mark overall recovery as finished if on last pass - self.finished = True - break - if self.finished: - self.current_pass_str = '- "Done"' - elif self.current_pass == 0: - self.current_pass_str = '1 "Initial Read"' - elif self.current_pass == 1: - self.current_pass_str = '2 "Trimming bad areas"' - elif self.current_pass == 2: - self.current_pass_str = '3 "Scraping bad areas"' + def set_pass_num(self): + """Set current pass based on all block-pair's progress.""" + self.current_pass = 0 + for pass_num in (2, 1, 0): + # Iterate backwards through passes + pass_done = True + for bp in self.block_pairs: + pass_done &= bp.pass_done[pass_num] + if pass_done: + # All block-pairs reported being done + # Set to next pass, unless we're on the last pass (2) + self.current_pass = min(2, pass_num + 1) + if pass_num == 2: + # Also mark overall recovery as finished if on last pass + self.finished = True + break + if self.finished: + self.current_pass_str = '- "Done"' + elif self.current_pass == 0: + self.current_pass_str = '1 "Initial Read"' + elif self.current_pass == 1: + self.current_pass_str = '2 "Trimming bad areas"' + elif self.current_pass == 2: + self.current_pass_str = '3 "Scraping bad areas"' - def update_progress(self): - """Update overall progress using block_pairs.""" - self.rescued = 0 - for bp in self.block_pairs: - self.rescued += bp.rescued - self.rescued_percent = (self.rescued / self.total_size) * 100 - self.status_percent = get_formatted_status( - label='Recovered:', data=self.rescued_percent) - self.status_amount = get_formatted_status( - label='', data=human_readable_size(self.rescued, decimals=2)) + def update_progress(self): + """Update overall progress using block_pairs.""" + self.rescued = 0 + for bp in self.block_pairs: + self.rescued += bp.rescued + self.rescued_percent = (self.rescued / self.total_size) * 100 + self.status_percent = get_formatted_status( + label='Recovered:', data=self.rescued_percent) + self.status_amount = get_formatted_status( + label='', data=human_readable_size(self.rescued, decimals=2)) # Functions def build_outer_panes(state): - """Build top and side panes.""" - state.panes['Source'] = tmux_split_window( - behind=True, vertical=True, lines=2, - text='{BLUE}Source{CLEAR}'.format(**COLORS)) - state.panes['Started'] = tmux_split_window( - lines=SIDE_PANE_WIDTH, target_pane=state.panes['Source'], - text='{BLUE}Started{CLEAR}\n{s}'.format( - s=time.strftime("%Y-%m-%d %H:%M %Z"), - **COLORS)) - state.panes['Destination'] = tmux_split_window( - percent=50, target_pane=state.panes['Source'], - text='{BLUE}Destination{CLEAR}'.format(**COLORS)) + """Build top and side panes.""" + state.panes['Source'] = tmux_split_window( + behind=True, vertical=True, lines=2, + text='{BLUE}Source{CLEAR}'.format(**COLORS)) + state.panes['Started'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, target_pane=state.panes['Source'], + text='{BLUE}Started{CLEAR}\n{s}'.format( + s=time.strftime("%Y-%m-%d %H:%M %Z"), + **COLORS)) + state.panes['Destination'] = tmux_split_window( + percent=50, target_pane=state.panes['Source'], + text='{BLUE}Destination{CLEAR}'.format(**COLORS)) - # Side pane - update_sidepane(state) - state.panes['Progress'] = tmux_split_window( - lines=SIDE_PANE_WIDTH, watch=state.progress_out) + # Side pane + update_sidepane(state) + state.panes['Progress'] = tmux_split_window( + lines=SIDE_PANE_WIDTH, watch=state.progress_out) def create_path_obj(path): - """Create Dev, Dir, or Image obj based on path given.""" - obj = None - if pathlib.Path(path).is_block_device(): - obj = DevObj(path) - elif pathlib.Path(path).is_dir(): - obj = DirObj(path) - elif pathlib.Path(path).is_file(): - obj = ImageObj(path) - else: - raise GenericError('Invalid path "{}"'.format(path)) - return obj + """Create Dev, Dir, or Image obj based on path given.""" + obj = None + if pathlib.Path(path).is_block_device(): + obj = DevObj(path) + elif pathlib.Path(path).is_dir(): + obj = DirObj(path) + elif pathlib.Path(path).is_file(): + obj = ImageObj(path) + else: + raise GenericError('Invalid path "{}"'.format(path)) + return obj def double_confirm_clone(): - """Display warning and get 2nd confirmation from user, returns bool.""" - print_standard('\nSAFETY CHECK') - print_warning('All data will be DELETED from the ' - 'destination device and partition(s) listed above.') - print_warning('This is irreversible and will lead ' - 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) - return ask('Asking again to confirm, is this correct?') + """Display warning and get 2nd confirmation, returns bool.""" + print_standard('\nSAFETY CHECK') + print_warning('All data will be DELETED from the ' + 'destination device and partition(s) listed above.') + print_warning('This is irreversible and will lead ' + 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) + return ask('Asking again to confirm, is this correct?') def fix_tmux_panes(state, forced=False): - """Fix pane sizes if the winodw has been resized.""" - needs_fixed = False + """Fix pane sizes if the winodw has been resized.""" + needs_fixed = False - # Check layout - for k, v in TMUX_LAYOUT.items(): - if not v.get('Check'): - # Not concerned with the size of this pane - continue - # Get target - target = None - if k != 'Current': - if k not in state.panes: - # Skip missing panes - continue - else: - target = state.panes[k] + # Check layout + for k, v in TMUX_LAYOUT.items(): + if not v.get('Check'): + # Not concerned with the size of this pane + continue + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] - # Check pane size - x, y = tmux_get_pane_size(pane_id=target) - if v.get('x', False) and v['x'] != x: - needs_fixed = True - if v.get('y', False) and v['y'] != y: - needs_fixed = True + # Check pane size + x, y = tmux_get_pane_size(pane_id=target) + if v.get('x', False) and v['x'] != x: + needs_fixed = True + if v.get('y', False) and v['y'] != y: + needs_fixed = True - # Bail? - if not needs_fixed and not forced: - return + # Bail? + if not needs_fixed and not forced: + return - # Remove Destination pane (temporarily) - tmux_kill_pane(state.panes['Destination']) + # Remove Destination pane (temporarily) + tmux_kill_pane(state.panes['Destination']) - # Update layout - for k, v in TMUX_LAYOUT.items(): - # Get target - target = None - if k != 'Current': - if k not in state.panes: - # Skip missing panes - continue - else: - target = state.panes[k] + # Update layout + for k, v in TMUX_LAYOUT.items(): + # Get target + target = None + if k != 'Current': + if k not in state.panes: + # Skip missing panes + continue + else: + target = state.panes[k] - # Resize pane - tmux_resize_pane(pane_id=target, **v) + # Resize pane + tmux_resize_pane(pane_id=target, **v) - # Calc Source/Destination pane sizes - width, height = tmux_get_pane_size() - width = int(width / 2) - 1 + # Calc Source/Destination pane sizes + width, height = tmux_get_pane_size() + width = int(width / 2) - 1 - # Update Source string - source_str = state.source.name - if len(source_str) > width: - source_str = '{}...'.format(source_str[:width-3]) + # Update Source string + source_str = state.source.name + if len(source_str) > width: + source_str = '{}...'.format(source_str[:width-3]) - # Update Destination string - dest_str = state.dest.name - if len(dest_str) > width: - if state.mode == 'clone': - dest_str = '{}...'.format(dest_str[:width-3]) - else: - dest_str = '...{}'.format(dest_str[-width+3:]) + # Update Destination string + dest_str = state.dest.name + if len(dest_str) > width: + if state.mode == 'clone': + dest_str = '{}...'.format(dest_str[:width-3]) + else: + dest_str = '...{}'.format(dest_str[-width+3:]) - # Rebuild Source/Destination panes - tmux_update_pane( - pane_id=state.panes['Source'], - text='{BLUE}Source{CLEAR}\n{s}'.format( - s=source_str, **COLORS)) - state.panes['Destination'] = tmux_split_window( - percent=50, target_pane=state.panes['Source'], - text='{BLUE}Destination{CLEAR}\n{s}'.format( - s=dest_str, **COLORS)) + # Rebuild Source/Destination panes + tmux_update_pane( + pane_id=state.panes['Source'], + text='{BLUE}Source{CLEAR}\n{s}'.format( + s=source_str, **COLORS)) + state.panes['Destination'] = tmux_split_window( + percent=50, target_pane=state.panes['Source'], + text='{BLUE}Destination{CLEAR}\n{s}'.format( + s=dest_str, **COLORS)) - if 'SMART' in state.panes: - # Calc SMART/ddrescue/Journal panes sizes - ratio = [12, 22, 4] - width, height = tmux_get_pane_size(pane_id=state.panes['Progress']) - height -= 2 - total = sum(ratio) - p_ratio = [int((x/total) * height) for x in ratio] - p_ratio[1] = height - p_ratio[0] - p_ratio[2] + if 'SMART' in state.panes: + # Calc SMART/ddrescue/Journal panes sizes + ratio = [12, 22, 4] + width, height = tmux_get_pane_size(pane_id=state.panes['Progress']) + height -= 2 + total = sum(ratio) + p_ratio = [int((x/total) * height) for x in ratio] + p_ratio[1] = height - p_ratio[0] - p_ratio[2] - # Resize SMART/Journal panes - tmux_resize_pane(state.panes['SMART'], y=ratio[0]) - tmux_resize_pane(y=ratio[1]) - tmux_resize_pane(state.panes['Journal'], y=ratio[2]) + # Resize SMART/Journal panes + tmux_resize_pane(state.panes['SMART'], y=ratio[0]) + tmux_resize_pane(y=ratio[1]) + tmux_resize_pane(state.panes['Journal'], y=ratio[2]) def get_device_details(dev_path): - """Get device details via lsblk, returns JSON dict.""" - try: - cmd = ( - 'lsblk', - '--json', - '--output-all', - '--paths', - dev_path) - result = run_program(cmd) - except CalledProcessError: - # Return empty dict and let calling section deal with the issue - return {} + """Get device details via lsblk, returns JSON dict.""" + try: + cmd = ( + 'lsblk', + '--json', + '--output-all', + '--paths', + dev_path) + result = run_program(cmd) + except CalledProcessError: + # Return empty dict and let calling section deal with the issue + return {} - json_data = json.loads(result.stdout.decode()) - # Just return the first device (there should only be one) - return json_data['blockdevices'][0] + json_data = json.loads(result.stdout.decode()) + # Just return the first device (there should only be one) + return json_data['blockdevices'][0] def get_device_report(dev_path): - """Build colored device report using lsblk, returns str.""" - result = run_program([ - 'lsblk', '--nodeps', - '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', - dev_path]) - lines = result.stdout.decode().strip().splitlines() - lines.append('') + """Build colored device report using lsblk, returns str.""" + result = run_program([ + 'lsblk', '--nodeps', + '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', + dev_path]) + lines = result.stdout.decode().strip().splitlines() + lines.append('') - # FS details (if any) - result = run_program([ - 'lsblk', - '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', - dev_path]) - lines.extend(result.stdout.decode().strip().splitlines()) + # FS details (if any) + result = run_program([ + 'lsblk', + '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', + dev_path]) + lines.extend(result.stdout.decode().strip().splitlines()) - # Color label lines - output = [] - for line in lines: - if line[0:4] == 'NAME': - output.append('{BLUE}{line}{CLEAR}'.format(line=line, **COLORS)) - else: - output.append(line) + # Color label lines + output = [] + for line in lines: + if line[0:4] == 'NAME': + output.append('{BLUE}{line}{CLEAR}'.format(line=line, **COLORS)) + else: + output.append(line) - # Done - return '\n'.join(output) + # Done + return '\n'.join(output) def get_dir_details(dir_path): - """Get dir details via findmnt, returns JSON dict.""" - try: - result = run_program([ - 'findmnt', '-J', - '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', - '-T', dir_path]) - json_data = json.loads(result.stdout.decode()) - except Exception: - raise GenericError( - 'Failed to get directory details for "{}".'.format(self.path)) - else: - return json_data['filesystems'][0] + """Get dir details via findmnt, returns JSON dict.""" + try: + result = run_program([ + 'findmnt', '-J', + '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', + '-T', dir_path]) + json_data = json.loads(result.stdout.decode()) + except Exception: + raise GenericError( + 'Failed to get directory details for "{}".'.format(self.path)) + else: + return json_data['filesystems'][0] def get_dir_report(dir_path): - """Build colored dir report using findmnt, returns str.""" - dir_path = dir_path - output = [] - width = len(dir_path)+1 - result = run_program([ - 'findmnt', - '--output', 'SIZE,AVAIL,USED,FSTYPE,OPTIONS', - '--target', dir_path]) - for line in result.stdout.decode().splitlines(): - if 'FSTYPE' in line: - output.append('{BLUE}{label:<{width}}{line}{CLEAR}'.format( - label='PATH', - width=width, - line=line.replace('\n',''), - **COLORS)) - else: - output.append('{path:<{width}}{line}'.format( - path=dir_path, - width=width, - line=line.replace('\n',''))) + """Build colored dir report using findmnt, returns str.""" + dir_path = dir_path + output = [] + width = len(dir_path)+1 + result = run_program([ + 'findmnt', + '--output', 'SIZE,AVAIL,USED,FSTYPE,OPTIONS', + '--target', dir_path]) + for line in result.stdout.decode().splitlines(): + if 'FSTYPE' in line: + output.append('{BLUE}{label:<{width}}{line}{CLEAR}'.format( + label='PATH', + width=width, + line=line.replace('\n',''), + **COLORS)) + else: + output.append('{path:<{width}}{line}'.format( + path=dir_path, + width=width, + line=line.replace('\n',''))) - # Done - return '\n'.join(output) + # Done + return '\n'.join(output) def get_size_in_bytes(s): - """Convert size string from lsblk string to bytes, returns int.""" - s = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', s, re.IGNORECASE) - return convert_to_bytes(s) + """Convert size string from lsblk string to bytes, returns int.""" + s = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', s, re.IGNORECASE) + return convert_to_bytes(s) def get_formatted_status(label, data): - """Build status string using provided info, returns str.""" - data_width = SIDE_PANE_WIDTH - len(label) - try: - data_str = '{data:>{data_width}.2f} %'.format( - data=data, - data_width=data_width-2) - except ValueError: - # Assuming non-numeric data - data_str = '{data:>{data_width}}'.format( - data=data, - data_width=data_width) - status = '{label}{s_color}{data_str}{CLEAR}'.format( - label=label, - s_color=get_status_color(data), - data_str=data_str, - **COLORS) - return status + """Build status string using provided info, returns str.""" + data_width = SIDE_PANE_WIDTH - len(label) + try: + data_str = '{data:>{data_width}.2f} %'.format( + data=data, + data_width=data_width-2) + except ValueError: + # Assuming non-numeric data + data_str = '{data:>{data_width}}'.format( + data=data, + data_width=data_width) + status = '{label}{s_color}{data_str}{CLEAR}'.format( + label=label, + s_color=get_status_color(data), + data_str=data_str, + **COLORS) + return status def get_status_color(s, t_success=99, t_warn=90): - """Get color based on status, returns str.""" - color = COLORS['CLEAR'] - p_recovered = -1 - try: - p_recovered = float(s) - except ValueError: - # Status is either in lists below or will default to red - pass + """Get color based on status, returns str.""" + color = COLORS['CLEAR'] + p_recovered = -1 + try: + p_recovered = float(s) + except ValueError: + # Status is either in lists below or will default to red + pass - if s in ('Pending',) or str(s)[-2:] in (' b', 'Kb', 'Mb', 'Gb', 'Tb'): - color = COLORS['CLEAR'] - elif s in ('Skipped', 'Unknown'): - color = COLORS['YELLOW'] - elif p_recovered >= t_success: - color = COLORS['GREEN'] - elif p_recovered >= t_warn: - color = COLORS['YELLOW'] - else: - color = COLORS['RED'] - return color + if s in ('Pending',) or str(s)[-2:] in (' b', 'Kb', 'Mb', 'Gb', 'Tb'): + color = COLORS['CLEAR'] + elif s in ('Skipped', 'Unknown'): + color = COLORS['YELLOW'] + elif p_recovered >= t_success: + color = COLORS['GREEN'] + elif p_recovered >= t_warn: + color = COLORS['YELLOW'] + else: + color = COLORS['RED'] + return color def is_writable_dir(dir_obj): - """Check if we have read-write-execute permissions, returns bool.""" - is_ok = True - path_st_mode = os.stat(dir_obj.path).st_mode - is_ok == is_ok and path_st_mode & stat.S_IRUSR - is_ok == is_ok and path_st_mode & stat.S_IWUSR - is_ok == is_ok and path_st_mode & stat.S_IXUSR - return is_ok + """Check if we have read-write-execute permissions, returns bool.""" + is_ok = True + path_st_mode = os.stat(dir_obj.path).st_mode + is_ok == is_ok and path_st_mode & stat.S_IRUSR + is_ok == is_ok and path_st_mode & stat.S_IWUSR + is_ok == is_ok and path_st_mode & stat.S_IXUSR + return is_ok def is_writable_filesystem(dir_obj): - """Check if filesystem is mounted read-write, returns bool.""" - return 'rw' in dir_obj.details.get('options', '') + """Check if filesystem is mounted read-write, returns bool.""" + return 'rw' in dir_obj.details.get('options', '') def menu_ddrescue(source_path, dest_path, run_mode): - """ddrescue menu.""" - source = None - dest = None - if source_path: - source = create_path_obj(source_path) - else: - source = select_device('source') - source.self_check() - if dest_path: - dest = create_path_obj(dest_path) - else: - if run_mode == 'clone': - dest = select_device('destination', skip_device=source) - else: - dest = select_path(skip_device=source) - dest.self_check() - - # Build BlockPairs - state = RecoveryState(run_mode, source, dest) + """ddrescue menu.""" + source = None + dest = None + if source_path: + source = create_path_obj(source_path) + else: + source = select_device('source') + source.self_check() + if dest_path: + dest = create_path_obj(dest_path) + else: if run_mode == 'clone': - state.add_block_pair(source, dest) + dest = select_device('destination', skip_device=source) else: - for part in select_parts(source): - state.add_block_pair(part, dest) + dest = select_path(skip_device=source) + dest.self_check() - # Update state - state.self_checks() - state.set_pass_num() - state.update_progress() + # Build BlockPairs + state = RecoveryState(run_mode, source, dest) + if run_mode == 'clone': + state.add_block_pair(source, dest) + else: + for part in select_parts(source): + state.add_block_pair(part, dest) - # Confirmations - clear_screen() - show_selection_details(state) - prompt = 'Start {}?'.format(state.mode.replace('e', 'ing')) - if state.resumed: - print_info('Map data detected and loaded.') - prompt = prompt.replace('Start', 'Resume') - if not ask(prompt): - raise GenericAbort() - if state.mode == 'clone' and not double_confirm_clone(): - raise GenericAbort() + # Update state + state.self_checks() + state.set_pass_num() + state.update_progress() - # Main menu - clear_screen() - build_outer_panes(state) - fix_tmux_panes(state, forced=True) - menu_main(state) + # Confirmations + clear_screen() + show_selection_details(state) + prompt = 'Start {}?'.format(state.mode.replace('e', 'ing')) + if state.resumed: + print_info('Map data detected and loaded.') + prompt = prompt.replace('Start', 'Resume') + if not ask(prompt): + raise GenericAbort() + if state.mode == 'clone' and not double_confirm_clone(): + raise GenericAbort() - # Done - run_program(['tmux', 'kill-window']) - exit_script() + # Main menu + clear_screen() + build_outer_panes(state) + fix_tmux_panes(state, forced=True) + menu_main(state) + + # Done + run_program(['tmux', 'kill-window']) + exit_script() def menu_main(state): - """Main menu is used to set ddrescue settings.""" - title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) - title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) + """Main menu is used to set ddrescue settings.""" + title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) + title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) - # Build menu - main_options = [ - {'Base Name': 'Auto continue (if recovery % over threshold)', - 'Enabled': True}, - {'Base Name': 'Retry (mark non-rescued sectors "non-tried")', - 'Enabled': False}, - {'Base Name': 'Reverse direction', 'Enabled': False}, - ] - actions = [ - {'Name': 'Start', 'Letter': 'S'}, - {'Name': 'Change settings {YELLOW}(experts only){CLEAR}'.format( - **COLORS), - 'Letter': 'C'}, - {'Name': 'Quit', 'Letter': 'Q', 'CRLF': True}, - ] + # Build menu + main_options = [ + {'Base Name': 'Auto continue (if recovery % over threshold)', + 'Enabled': True}, + {'Base Name': 'Retry (mark non-rescued sectors "non-tried")', + 'Enabled': False}, + {'Base Name': 'Reverse direction', 'Enabled': False}, + ] + actions = [ + {'Name': 'Start', 'Letter': 'S'}, + {'Name': 'Change settings {YELLOW}(experts only){CLEAR}'.format( + **COLORS), + 'Letter': 'C'}, + {'Name': 'Quit', 'Letter': 'Q', 'CRLF': True}, + ] - # Show menu - while True: - # Update entries - for opt in main_options: - opt['Name'] = '{} {}'.format( - '[✓]' if opt['Enabled'] else '[ ]', - opt['Base Name']) + # Show menu + while True: + # Update entries + for opt in main_options: + opt['Name'] = '{} {}'.format( + '[✓]' if opt['Enabled'] else '[ ]', + opt['Base Name']) - selection = menu_select( - title=title+state.current_pass_str, - main_entries=main_options, - action_entries=actions) + selection = menu_select( + title=title+state.current_pass_str, + main_entries=main_options, + action_entries=actions) - if selection.isnumeric(): - # Toggle selection - index = int(selection) - 1 - main_options[index]['Enabled'] = not main_options[index]['Enabled'] - elif selection == 'S': - # Set settings for pass - pass_settings = [] - for k, v in state.settings.items(): - if not v['Enabled']: - continue - if 'Value' in v: - pass_settings.append('{}={}'.format(k, v['Value'])) - else: - pass_settings.append(k) - for opt in main_options: - if 'Auto' in opt['Base Name']: - auto_run = opt['Enabled'] - if 'Retry' in opt['Base Name'] and opt['Enabled']: - pass_settings.extend(['--retrim', '--try-again']) - state.retry_all_passes() - if 'Reverse' in opt['Base Name'] and opt['Enabled']: - pass_settings.append('--reverse') - # Disable for next pass - if 'Auto' not in opt['Base Name']: - opt['Enabled'] = False + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + elif selection == 'S': + # Set settings for pass + pass_settings = [] + for k, v in state.settings.items(): + if not v['Enabled']: + continue + if 'Value' in v: + pass_settings.append('{}={}'.format(k, v['Value'])) + else: + pass_settings.append(k) + for opt in main_options: + if 'Auto' in opt['Base Name']: + auto_run = opt['Enabled'] + if 'Retry' in opt['Base Name'] and opt['Enabled']: + pass_settings.extend(['--retrim', '--try-again']) + state.retry_all_passes() + if 'Reverse' in opt['Base Name'] and opt['Enabled']: + pass_settings.append('--reverse') + # Disable for next pass + if 'Auto' not in opt['Base Name']: + opt['Enabled'] = False - # Run ddrescue - state.started = False - while auto_run or not state.started: - state.started = True - run_ddrescue(state, pass_settings) - if state.current_pass_done(): - if (state.current_pass == 0 and - state.current_pass_min() < AUTO_PASS_1_THRESHOLD): - auto_run = False - elif (state.current_pass == 1 and - state.current_pass_min() < AUTO_PASS_2_THRESHOLD): - auto_run = False - else: - auto_run = False - state.set_pass_num() - if state.finished: - break + # Run ddrescue + state.started = False + while auto_run or not state.started: + state.started = True + run_ddrescue(state, pass_settings) + if state.current_pass_done(): + if (state.current_pass == 0 and + state.current_pass_min() < AUTO_PASS_1_THRESHOLD): + auto_run = False + elif (state.current_pass == 1 and + state.current_pass_min() < AUTO_PASS_2_THRESHOLD): + auto_run = False + else: + auto_run = False + state.set_pass_num() + if state.finished: + break - elif selection == 'C': - menu_settings(state) - elif selection == 'Q': - if state.rescued_percent < 100: - print_warning('Recovery is less than 100%') - if ask('Are you sure you want to quit?'): - break - else: - break + elif selection == 'C': + menu_settings(state) + elif selection == 'Q': + if state.rescued_percent < 100: + print_warning('Recovery is less than 100%') + if ask('Are you sure you want to quit?'): + break + else: + break def menu_settings(state): - """Change advanced ddrescue settings.""" - title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) - title += '{YELLOW}These settings can cause {CLEAR}'.format(**COLORS) - title += '{RED}MAJOR DAMAGE{CLEAR}{YELLOW} to drives{CLEAR}\n'.format( - **COLORS) - title += 'Please read the manual before making any changes' + """Change advanced ddrescue settings.""" + title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) + title += '{YELLOW}These settings can cause {CLEAR}'.format(**COLORS) + title += '{RED}MAJOR DAMAGE{CLEAR}{YELLOW} to drives{CLEAR}\n'.format( + **COLORS) + title += 'Please read the manual before making any changes' - # Build menu - settings = [] - for k, v in sorted(state.settings.items()): - if not v.get('Hidden', False): - settings.append({'Base Name': k, 'Flag': k}) - actions = [{'Name': 'Main Menu', 'Letter': 'M'}] + # Build menu + settings = [] + for k, v in sorted(state.settings.items()): + if not v.get('Hidden', False): + settings.append({'Base Name': k, 'Flag': k}) + actions = [{'Name': 'Main Menu', 'Letter': 'M'}] - # Show menu - while True: - for s in settings: - s['Name'] = '{}{}{}'.format( - s['Base Name'], - ' = ' if 'Value' in state.settings[s['Flag']] else '', - state.settings[s['Flag']].get('Value', '')) - if not state.settings[s['Flag']]['Enabled']: - s['Name'] = '{YELLOW}{name} (Disabled){CLEAR}'.format( - name=s['Name'], - **COLORS) - selection = menu_select( - title=title, - main_entries=settings, - action_entries=actions) - if selection.isnumeric(): - index = int(selection) - 1 - flag = settings[index]['Flag'] - enabled = state.settings[flag]['Enabled'] - if 'Value' in state.settings[flag]: - answer = choice( - choices=['T', 'C'], - prompt='Toggle or change value for "{}"'.format(flag)) - if answer == 'T': - # Toggle - state.settings[flag]['Enabled'] = not enabled - else: - # Update value - state.settings[flag]['Value'] = get_simple_string( - prompt='Enter new value') - else: - state.settings[flag]['Enabled'] = not enabled - elif selection == 'M': - break + # Show menu + while True: + for s in settings: + s['Name'] = '{}{}{}'.format( + s['Base Name'], + ' = ' if 'Value' in state.settings[s['Flag']] else '', + state.settings[s['Flag']].get('Value', '')) + if not state.settings[s['Flag']]['Enabled']: + s['Name'] = '{YELLOW}{name} (Disabled){CLEAR}'.format( + name=s['Name'], + **COLORS) + selection = menu_select( + title=title, + main_entries=settings, + action_entries=actions) + if selection.isnumeric(): + index = int(selection) - 1 + flag = settings[index]['Flag'] + enabled = state.settings[flag]['Enabled'] + if 'Value' in state.settings[flag]: + answer = choice( + choices=['T', 'C'], + prompt='Toggle or change value for "{}"'.format(flag)) + if answer == 'T': + # Toggle + state.settings[flag]['Enabled'] = not enabled + else: + # Update value + state.settings[flag]['Value'] = get_simple_string( + prompt='Enter new value') + else: + state.settings[flag]['Enabled'] = not enabled + elif selection == 'M': + break def read_map_file(map_path): - """Read map file with ddrescuelog and return data as dict.""" - map_data = {'full recovery': False} - try: - result = run_program(['ddrescuelog', '-t', map_path]) - except CalledProcessError: - # (Grossly) assuming map_data hasn't been saved yet, return empty dict - return map_data - - # Parse output - for line in result.stdout.decode().splitlines(): - m = re.match( - r'^\s*(?P\S+):.*\(\s*(?P\d+\.?\d*)%.*', line.strip()) - if m: - try: - map_data[m.group('key')] = float(m.group('value')) - except ValueError: - raise GenericError('Failed to read map data') - m = re.match(r'.*current status:\s+(?P.*)', line.strip()) - if m: - map_data['pass completed'] = bool(m.group('status') == 'finished') - - # Check if 100% done - try: - run_program(['ddrescuelog', '-D', map_path]) - except CalledProcessError: - map_data['full recovery'] = False - else: - map_data['full recovery'] = True - + """Read map file with ddrescuelog and return data as dict.""" + map_data = {'full recovery': False} + try: + result = run_program(['ddrescuelog', '-t', map_path]) + except CalledProcessError: + # (Grossly) assuming map_data hasn't been saved yet, return empty dict return map_data + # Parse output + for line in result.stdout.decode().splitlines(): + m = re.match( + r'^\s*(?P\S+):.*\(\s*(?P\d+\.?\d*)%.*', line.strip()) + if m: + try: + map_data[m.group('key')] = float(m.group('value')) + except ValueError: + raise GenericError('Failed to read map data') + m = re.match(r'.*current status:\s+(?P.*)', line.strip()) + if m: + map_data['pass completed'] = bool(m.group('status') == 'finished') + + # Check if 100% done + try: + run_program(['ddrescuelog', '-D', map_path]) + except CalledProcessError: + map_data['full recovery'] = False + else: + map_data['full recovery'] = True + + return map_data + def run_ddrescue(state, pass_settings): - """Run ddrescue pass.""" - return_code = -1 - aborted = False + """Run ddrescue pass.""" + return_code = -1 + aborted = False - if state.finished: - clear_screen() - print_warning('Recovery already completed?') - pause('Press Enter to return to main menu...') - return + if state.finished: + clear_screen() + print_warning('Recovery already completed?') + pause('Press Enter to return to main menu...') + return - # Create SMART monitor pane - state.smart_out = '{}/smart_{}.out'.format( - global_vars['TmpDir'], state.smart_source.name) - with open(state.smart_out, 'w') as f: - f.write('Initializing...') - state.panes['SMART'] = tmux_split_window( - behind=True, lines=12, vertical=True, watch=state.smart_out) + # Create SMART monitor pane + state.smart_out = '{}/smart_{}.out'.format( + global_vars['TmpDir'], state.smart_source.name) + with open(state.smart_out, 'w') as f: + f.write('Initializing...') + state.panes['SMART'] = tmux_split_window( + behind=True, lines=12, vertical=True, watch=state.smart_out) - # Show systemd journal output - state.panes['Journal'] = tmux_split_window( - lines=4, vertical=True, - command=['sudo', 'journalctl', '-f']) + # Show systemd journal output + state.panes['Journal'] = tmux_split_window( + lines=4, vertical=True, + command=['sudo', 'journalctl', '-f']) - # Fix layout - fix_tmux_panes(state, forced=True) + # Fix layout + fix_tmux_panes(state, forced=True) - # Run pass for each block-pair - for bp in state.block_pairs: - if bp.pass_done[state.current_pass]: - # Skip to next block-pair - continue - update_sidepane(state) + # Run pass for each block-pair + for bp in state.block_pairs: + if bp.pass_done[state.current_pass]: + # Skip to next block-pair + continue + update_sidepane(state) - # Set ddrescue cmd - cmd = [ - 'ddrescue', *pass_settings, - bp.source_path, bp.dest_path, bp.map_path] - if state.mode == 'clone': - cmd.append('--force') - if state.current_pass == 0: - cmd.extend(['--no-trim', '--no-scrape']) - elif state.current_pass == 1: - # Allow trimming - cmd.append('--no-scrape') - elif state.current_pass == 2: - # Allow trimming and scraping - pass + # Set ddrescue cmd + cmd = [ + 'ddrescue', *pass_settings, + bp.source_path, bp.dest_path, bp.map_path] + if state.mode == 'clone': + cmd.append('--force') + if state.current_pass == 0: + cmd.extend(['--no-trim', '--no-scrape']) + elif state.current_pass == 1: + # Allow trimming + cmd.append('--no-scrape') + elif state.current_pass == 2: + # Allow trimming and scraping + pass - # Start ddrescue - try: - clear_screen() - print_info('Current dev: {}'.format(bp.source_path)) - ddrescue_proc = popen_program(cmd) - i = 0 - while True: - # Update SMART display (every 30 seconds) - if i % 30 == 0: - state.smart_source.get_smart_details() - with open(state.smart_out, 'w') as f: - report = state.smart_source.generate_attribute_report( - timestamp=True) - for line in report: - f.write('{}\n'.format(line)) - i += 1 + # Start ddrescue + try: + clear_screen() + print_info('Current dev: {}'.format(bp.source_path)) + ddrescue_proc = popen_program(cmd) + i = 0 + while True: + # Update SMART display (every 30 seconds) + if i % 30 == 0: + state.smart_source.get_smart_details() + with open(state.smart_out, 'w') as f: + report = state.smart_source.generate_attribute_report( + timestamp=True) + for line in report: + f.write('{}\n'.format(line)) + i += 1 - # Update progress - bp.update_progress(state.current_pass) - update_sidepane(state) - - # Fix panes - fix_tmux_panes(state) - - # Check if ddrescue has finished - try: - ddrescue_proc.wait(timeout=1) - sleep(2) - bp.update_progress(state.current_pass) - update_sidepane(state) - break - except subprocess.TimeoutExpired: - # Catch to update smart/bp/sidepane - pass - - except KeyboardInterrupt: - # Catch user abort - aborted = True - ddrescue_proc.wait(timeout=10) - - # Update progress/sidepane again + # Update progress bp.update_progress(state.current_pass) update_sidepane(state) - # Was ddrescue aborted? - return_code = ddrescue_proc.poll() - if aborted: - print_standard(' ') - print_standard(' ') - print_error('DDRESCUE PROCESS HALTED') - print_standard(' ') - print_warning('Aborted') - break - elif return_code: - # i.e. True when non-zero - print_standard(' ') - print_standard(' ') - print_error('DDRESCUE PROCESS HALTED') - print_standard(' ') - print_error('Error(s) encountered, see message above.') - break - else: - # Mark pass finished - bp.finish_pass(state.current_pass) - update_sidepane(state) + # Fix panes + fix_tmux_panes(state) - # Done - if str(return_code) != '0': - # Pause on errors - pause('Press Enter to return to main menu... ') + # Check if ddrescue has finished + try: + ddrescue_proc.wait(timeout=1) + sleep(2) + bp.update_progress(state.current_pass) + update_sidepane(state) + break + except subprocess.TimeoutExpired: + # Catch to update smart/bp/sidepane + pass - # Cleanup - tmux_kill_pane(state.panes['SMART'], state.panes['Journal']) + except KeyboardInterrupt: + # Catch user abort + aborted = True + ddrescue_proc.wait(timeout=10) + + # Update progress/sidepane again + bp.update_progress(state.current_pass) + update_sidepane(state) + + # Was ddrescue aborted? + return_code = ddrescue_proc.poll() + if aborted: + print_standard(' ') + print_standard(' ') + print_error('DDRESCUE PROCESS HALTED') + print_standard(' ') + print_warning('Aborted') + break + elif return_code: + # i.e. True when non-zero + print_standard(' ') + print_standard(' ') + print_error('DDRESCUE PROCESS HALTED') + print_standard(' ') + print_error('Error(s) encountered, see message above.') + break + else: + # Mark pass finished + bp.finish_pass(state.current_pass) + update_sidepane(state) + + # Done + if str(return_code) != '0': + # Pause on errors + pause('Press Enter to return to main menu... ') + + # Cleanup + tmux_kill_pane(state.panes['SMART'], state.panes['Journal']) def select_parts(source_device): - """Select partition(s) or whole device, returns list of DevObj()s.""" - selected_parts = [] - children = source_device.details.get('children', []) + """Select partition(s) or whole device, returns list of DevObj()s.""" + selected_parts = [] + children = source_device.details.get('children', []) - if not children: - # No partitions detected, auto-select whole device. - selected_parts = [source_device] - else: - # Build menu - dev_options = [{ - 'Base Name': '{:<14}(Whole device)'.format(source_device.path), - 'Dev': source_device, - 'Selected': True}] - for c_details in children: - dev_options.append({ - 'Base Name': '{:<14}({:>6} {})'.format( - c_details['name'], - c_details['size'], - c_details['fstype'] if c_details['fstype'] else 'Unknown'), - 'Details': c_details, - 'Dev': DevObj(c_details['name']), - 'Selected': False}) - actions = [ - {'Name': 'Proceed', 'Letter': 'P'}, - {'Name': 'Quit', 'Letter': 'Q'}] + if not children: + # No partitions detected, auto-select whole device. + selected_parts = [source_device] + else: + # Build menu + dev_options = [{ + 'Base Name': '{:<14}(Whole device)'.format(source_device.path), + 'Dev': source_device, + 'Selected': True}] + for c_details in children: + dev_options.append({ + 'Base Name': '{:<14}({:>6} {})'.format( + c_details['name'], + c_details['size'], + c_details['fstype'] if c_details['fstype'] else 'Unknown'), + 'Details': c_details, + 'Dev': DevObj(c_details['name']), + 'Selected': False}) + actions = [ + {'Name': 'Proceed', 'Letter': 'P'}, + {'Name': 'Quit', 'Letter': 'Q'}] - # Show menu - while True: - one_or_more_devs_selected = False - # Update entries - for dev in dev_options: - if dev['Selected']: - one_or_more_devs_selected = True - dev['Name'] = '* {}'.format(dev['Base Name']) - else: - dev['Name'] = ' {}'.format(dev['Base Name']) + # Show menu + while True: + one_or_more_devs_selected = False + # Update entries + for dev in dev_options: + if dev['Selected']: + one_or_more_devs_selected = True + dev['Name'] = '* {}'.format(dev['Base Name']) + else: + dev['Name'] = ' {}'.format(dev['Base Name']) - selection = menu_select( - title='Please select part(s) to image', - main_entries=dev_options, - action_entries=actions) + selection = menu_select( + title='Please select part(s) to image', + main_entries=dev_options, + action_entries=actions) - if selection.isnumeric(): - # Toggle selection - index = int(selection) - 1 - dev_options[index]['Selected'] = not dev_options[index]['Selected'] + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + dev_options[index]['Selected'] = not dev_options[index]['Selected'] - # Deselect whole device if child selected (this round) - if index > 0: - dev_options[0]['Selected'] = False + # Deselect whole device if child selected (this round) + if index > 0: + dev_options[0]['Selected'] = False - # Deselect all children if whole device selected - if dev_options[0]['Selected']: - for dev in dev_options[1:]: - dev['Selected'] = False - elif selection == 'P' and one_or_more_devs_selected: - break - elif selection == 'Q': - raise GenericAbort() + # Deselect all children if whole device selected + if dev_options[0]['Selected']: + for dev in dev_options[1:]: + dev['Selected'] = False + elif selection == 'P' and one_or_more_devs_selected: + break + elif selection == 'Q': + raise GenericAbort() - # Build list of selected parts - for d in dev_options: - if d['Selected']: - d['Dev'].model = source_device.model - d['Dev'].model_size = source_device.model_size - d['Dev'].update_filename_prefix() - selected_parts.append(d['Dev']) + # Build list of selected parts + for d in dev_options: + if d['Selected']: + d['Dev'].model = source_device.model + d['Dev'].model_size = source_device.model_size + d['Dev'].update_filename_prefix() + selected_parts.append(d['Dev']) - return selected_parts + return selected_parts def select_path(skip_device=None): - """Optionally mount local dev and select path, returns DirObj.""" - wd = os.path.realpath(global_vars['Env']['PWD']) - selected_path = None + """Optionally mount local dev and select path, returns DirObj.""" + wd = os.path.realpath(global_vars['Env']['PWD']) + selected_path = None - # Build menu - path_options = [ - {'Name': 'Current directory: {}'.format(wd), 'Path': wd}, - {'Name': 'Local device', 'Path': None}, - {'Name': 'Enter manually', 'Path': None}] - actions = [{'Name': 'Quit', 'Letter': 'Q'}] + # Build menu + path_options = [ + {'Name': 'Current directory: {}'.format(wd), 'Path': wd}, + {'Name': 'Local device', 'Path': None}, + {'Name': 'Enter manually', 'Path': None}] + actions = [{'Name': 'Quit', 'Letter': 'Q'}] - # Show Menu - selection = menu_select( - title='Please make a selection', - main_entries=path_options, + # Show Menu + selection = menu_select( + title='Please make a selection', + main_entries=path_options, + action_entries=actions) + + if selection == 'Q': + raise GenericAbort() + elif selection.isnumeric(): + index = int(selection) - 1 + if path_options[index]['Path'] == wd: + # Current directory + selected_path = DirObj(wd) + + elif path_options[index]['Name'] == 'Local device': + # Local device + local_device = select_device( + skip_device=skip_device) + s_path = '' + + # Mount device volume(s) + report = mount_volumes( + all_devices=False, + device_path=local_device.path, + read_write=True) + + # Select volume + vol_options = [] + for k, v in sorted(report.items()): + disabled = v['show_data']['data'] == 'Failed to mount' + if disabled: + name = '{name} (Failed to mount)'.format(**v) + else: + name = '{name} (mounted on "{mount_point}")'.format(**v) + vol_options.append({ + 'Name': name, + 'Path': v['mount_point'], + 'Disabled': disabled}) + selection = menu_select( + title='Please select a volume', + main_entries=vol_options, action_entries=actions) - - if selection == 'Q': + if selection.isnumeric(): + s_path = vol_options[int(selection)-1]['Path'] + elif selection == 'Q': raise GenericAbort() - elif selection.isnumeric(): - index = int(selection) - 1 - if path_options[index]['Path'] == wd: - # Current directory - selected_path = DirObj(wd) - elif path_options[index]['Name'] == 'Local device': - # Local device - local_device = select_device( - skip_device=skip_device) - s_path = '' + # Create folder + if ask('Create ticket folder?'): + ticket_folder = get_simple_string('Please enter folder name') + s_path = os.path.join(s_path, ticket_folder) + try: + os.makedirs(s_path, exist_ok=True) + except OSError: + raise GenericError( + 'Failed to create folder "{}"'.format(s_path)) - # Mount device volume(s) - report = mount_volumes( - all_devices=False, - device_path=local_device.path, - read_write=True) + # Create DirObj + selected_path = DirObj(s_path) - # Select volume - vol_options = [] - for k, v in sorted(report.items()): - disabled = v['show_data']['data'] == 'Failed to mount' - if disabled: - name = '{name} (Failed to mount)'.format(**v) - else: - name = '{name} (mounted on "{mount_point}")'.format(**v) - vol_options.append({ - 'Name': name, - 'Path': v['mount_point'], - 'Disabled': disabled}) - selection = menu_select( - title='Please select a volume', - main_entries=vol_options, - action_entries=actions) - if selection.isnumeric(): - s_path = vol_options[int(selection)-1]['Path'] - elif selection == 'Q': - raise GenericAbort() - - # Create folder - if ask('Create ticket folder?'): - ticket_folder = get_simple_string('Please enter folder name') - s_path = os.path.join(s_path, ticket_folder) - try: - os.makedirs(s_path, exist_ok=True) - except OSError: - raise GenericError( - 'Failed to create folder "{}"'.format(s_path)) - - # Create DirObj - selected_path = DirObj(s_path) - - elif path_options[index]['Name'] == 'Enter manually': - # Manual entry - while not selected_path: - manual_path = input('Please enter path: ').strip() - if manual_path and pathlib.Path(manual_path).is_dir(): - selected_path = DirObj(manual_path) - elif manual_path and pathlib.Path(manual_path).is_file(): - print_error('File "{}" exists'.format(manual_path)) - else: - print_error('Invalid path "{}"'.format(manual_path)) - return selected_path + elif path_options[index]['Name'] == 'Enter manually': + # Manual entry + while not selected_path: + manual_path = input('Please enter path: ').strip() + if manual_path and pathlib.Path(manual_path).is_dir(): + selected_path = DirObj(manual_path) + elif manual_path and pathlib.Path(manual_path).is_file(): + print_error('File "{}" exists'.format(manual_path)) + else: + print_error('Invalid path "{}"'.format(manual_path)) + return selected_path def select_device(description='device', skip_device=None): - """Select device via a menu, returns DevObj.""" - cmd = ( - 'lsblk', - '--json', - '--nodeps', - '--output-all', - '--paths') - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - skip_names = [] - if skip_device: - skip_names.append(skip_device.path) - if skip_device.parent: - skip_names.append(skip_device.parent) + """Select device via a menu, returns DevObj.""" + cmd = ( + 'lsblk', + '--json', + '--nodeps', + '--output-all', + '--paths') + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + skip_names = [] + if skip_device: + skip_names.append(skip_device.path) + if skip_device.parent: + skip_names.append(skip_device.parent) - # Build menu - dev_options = [] - for dev in json_data['blockdevices']: - # Disable dev if in skip_names - disabled = dev['name'] in skip_names or dev['pkname'] in skip_names + # Build menu + dev_options = [] + for dev in json_data['blockdevices']: + # Disable dev if in skip_names + disabled = dev['name'] in skip_names or dev['pkname'] in skip_names - # Add to options - dev_options.append({ - 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( - name=dev['name'], - tran=dev['tran'] if dev['tran'] else '', - size=dev['size'] if dev['size'] else '', - model=dev['model'] if dev['model'] else '', - serial=dev['serial'] if dev['serial'] else ''), - 'Dev': DevObj(dev['name']), - 'Disabled': disabled}) - dev_options = sorted(dev_options, key=itemgetter('Name')) - if not dev_options: - raise GenericError('No devices available.') + # Add to options + dev_options.append({ + 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( + name=dev['name'], + tran=dev['tran'] if dev['tran'] else '', + size=dev['size'] if dev['size'] else '', + model=dev['model'] if dev['model'] else '', + serial=dev['serial'] if dev['serial'] else ''), + 'Dev': DevObj(dev['name']), + 'Disabled': disabled}) + dev_options = sorted(dev_options, key=itemgetter('Name')) + if not dev_options: + raise GenericError('No devices available.') - # Show Menu - actions = [{'Name': 'Quit', 'Letter': 'Q'}] - selection = menu_select( - title='Please select the {} device'.format(description), - main_entries=dev_options, - action_entries=actions, - disabled_label='ALREADY SELECTED') + # Show Menu + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + selection = menu_select( + title='Please select the {} device'.format(description), + main_entries=dev_options, + action_entries=actions, + disabled_label='ALREADY SELECTED') - if selection.isnumeric(): - return dev_options[int(selection)-1]['Dev'] - elif selection == 'Q': - raise GenericAbort() + if selection.isnumeric(): + return dev_options[int(selection)-1]['Dev'] + elif selection == 'Q': + raise GenericAbort() def setup_loopback_device(source_path): - """Setup a loopback device for source_path, returns dev_path as str.""" - cmd = ( - 'losetup', - '--find', - '--partscan', - '--show', - source_path) - try: - out = run_program(cmd, check=True) - dev_path = out.stdout.decode().strip() - sleep(1) - except CalledProcessError: - raise GenericError('Failed to setup loopback device for source.') - else: - return dev_path + """Setup loopback device for source_path, returns dev_path as str.""" + cmd = ( + 'losetup', + '--find', + '--partscan', + '--show', + source_path) + try: + out = run_program(cmd, check=True) + dev_path = out.stdout.decode().strip() + sleep(1) + except CalledProcessError: + raise GenericError('Failed to setup loopback device for source.') + else: + return dev_path def show_selection_details(state): - """Show selection details.""" - # Source - print_success('Source') - print_standard(state.source.report) - print_standard(' ') + """Show selection details.""" + # Source + print_success('Source') + print_standard(state.source.report) + print_standard(' ') - # Destination - if state.mode == 'clone': - print_success('Destination ', end='') - print_error('(ALL DATA WILL BE DELETED)', timestamp=False) - else: - print_success('Destination') - print_standard(state.dest.report) - print_standard(' ') + # Destination + if state.mode == 'clone': + print_success('Destination ', end='') + print_error('(ALL DATA WILL BE DELETED)', timestamp=False) + else: + print_success('Destination') + print_standard(state.dest.report) + print_standard(' ') def show_usage(script_name): - print_info('Usage:') - print_standard(USAGE.format(script_name=script_name)) - pause() + print_info('Usage:') + print_standard(USAGE.format(script_name=script_name)) + pause() def update_sidepane(state): - """Update progress file for side pane.""" - output = [] - state.update_progress() - if state.mode == 'clone': - output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) + """Update progress file for side pane.""" + output = [] + state.update_progress() + if state.mode == 'clone': + output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) + else: + output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) + output.append('─────────────────────') + + # Overall progress + output.append('{BLUE}Overall Progress{CLEAR}'.format(**COLORS)) + output.append(state.status_percent) + output.append(state.status_amount) + output.append('─────────────────────') + + # Source(s) progress + for bp in state.block_pairs: + if state.source.is_image(): + output.append('{BLUE}Image File{CLEAR}'.format(**COLORS)) else: - output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) - output.append('─────────────────────') + output.append('{BLUE}{source}{CLEAR}'.format( + source=bp.source_path, + **COLORS)) + output.extend(bp.status) + output.append(' ') - # Overall progress - output.append('{BLUE}Overall Progress{CLEAR}'.format(**COLORS)) - output.append(state.status_percent) - output.append(state.status_amount) - output.append('─────────────────────') + # Add line-endings + output = ['{}\n'.format(line) for line in output] - # Source(s) progress - for bp in state.block_pairs: - if state.source.is_image(): - output.append('{BLUE}Image File{CLEAR}'.format(**COLORS)) - else: - output.append('{BLUE}{source}{CLEAR}'.format( - source=bp.source_path, - **COLORS)) - output.extend(bp.status) - output.append(' ') - - # Add line-endings - output = ['{}\n'.format(line) for line in output] - - with open(state.progress_out, 'w') as f: - f.writelines(output) + with open(state.progress_out, 'w') as f: + f.writelines(output) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 From 0a899539c933beecde2db7f8225eaa8981d3fbc7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:52:18 -0700 Subject: [PATCH 118/186] Updated diags.py --- .bin/Scripts/functions/diags.py | 300 ++++++++++++++++---------------- 1 file changed, 151 insertions(+), 149 deletions(-) diff --git a/.bin/Scripts/functions/diags.py b/.bin/Scripts/functions/diags.py index e55f5b12..317f2b4f 100644 --- a/.bin/Scripts/functions/diags.py +++ b/.bin/Scripts/functions/diags.py @@ -6,184 +6,186 @@ from functions.common import * # STATIC VARIABLES AUTORUNS_SETTINGS = { - r'Software\Sysinternals\AutoRuns': { - 'checkvirustotal': 1, - 'EulaAccepted': 1, - 'shownomicrosoft': 1, - 'shownowindows': 1, - 'showonlyvirustotal': 1, - 'submitvirustotal': 0, - 'verifysignatures': 1, - }, - r'Software\Sysinternals\AutoRuns\SigCheck': { - 'EulaAccepted': 1, - }, - r'Software\Sysinternals\AutoRuns\Streams': { - 'EulaAccepted': 1, - }, - r'Software\Sysinternals\AutoRuns\VirusTotal': { - 'VirusTotalTermsAccepted': 1, - }, - } + r'Software\Sysinternals\AutoRuns': { + 'checkvirustotal': 1, + 'EulaAccepted': 1, + 'shownomicrosoft': 1, + 'shownowindows': 1, + 'showonlyvirustotal': 1, + 'submitvirustotal': 0, + 'verifysignatures': 1, + }, + r'Software\Sysinternals\AutoRuns\SigCheck': { + 'EulaAccepted': 1, + }, + r'Software\Sysinternals\AutoRuns\Streams': { + 'EulaAccepted': 1, + }, + r'Software\Sysinternals\AutoRuns\VirusTotal': { + 'VirusTotalTermsAccepted': 1, + }, + } def check_connection(): - """Check if the system is online and optionally abort the script.""" - while True: - result = try_and_print(message='Ping test...', function=ping, cs='OK') - if result['CS']: - break - if not ask('ERROR: System appears offline, try again?'): - if ask('Continue anyway?'): - break - else: - abort() + """Check if the system is online and optionally abort the script.""" + while True: + result = try_and_print(message='Ping test...', function=ping, cs='OK') + if result['CS']: + break + if not ask('ERROR: System appears offline, try again?'): + if ask('Continue anyway?'): + break + else: + abort() def check_secure_boot_status(show_alert=False): - """Checks UEFI Secure Boot status via PowerShell.""" - boot_mode = get_boot_mode() - cmd = ['PowerShell', '-Command', 'Confirm-SecureBootUEFI'] - result = run_program(cmd, check=False) + """Checks UEFI Secure Boot status via PowerShell.""" + boot_mode = get_boot_mode() + cmd = ['PowerShell', '-Command', 'Confirm-SecureBootUEFI'] + result = run_program(cmd, check=False) - # Check results - if result.returncode == 0: - out = result.stdout.decode() - if 'True' in out: - # It's on, do nothing - return - elif 'False' in out: - if show_alert: - show_alert_box('Secure Boot DISABLED') - raise SecureBootDisabledError - else: - if show_alert: - show_alert_box('Secure Boot status UNKNOWN') - raise SecureBootUnknownError + # Check results + if result.returncode == 0: + out = result.stdout.decode() + if 'True' in out: + # It's on, do nothing + return + elif 'False' in out: + if show_alert: + show_alert_box('Secure Boot DISABLED') + raise SecureBootDisabledError else: - if boot_mode != 'UEFI': - if (show_alert and - global_vars['OS']['Version'] in ('8', '8.1', '10')): - # OS supports Secure Boot - show_alert_box('Secure Boot DISABLED\n\nOS installed LEGACY') - raise OSInstalledLegacyError - else: - # Check error message - err = result.stderr.decode() - if 'Cmdlet not supported' in err: - if show_alert: - show_alert_box('Secure Boot UNAVAILABLE?') - raise SecureBootNotAvailError - else: - if show_alert: - show_alert_box('Secure Boot ERROR') - raise GenericError + if show_alert: + show_alert_box('Secure Boot status UNKNOWN') + raise SecureBootUnknownError + else: + if boot_mode != 'UEFI': + if (show_alert and + global_vars['OS']['Version'] in ('8', '8.1', '10')): + # OS supports Secure Boot + show_alert_box('Secure Boot DISABLED\n\nOS installed LEGACY') + raise OSInstalledLegacyError + else: + # Check error message + err = result.stderr.decode() + if 'Cmdlet not supported' in err: + if show_alert: + show_alert_box('Secure Boot UNAVAILABLE?') + raise SecureBootNotAvailError + else: + if show_alert: + show_alert_box('Secure Boot ERROR') + raise GenericError def get_boot_mode(): - """Check if Windows is booted in UEFI or Legacy mode, returns str.""" - kernel = ctypes.windll.kernel32 - firmware_type = ctypes.c_uint() + """Check if Windows is booted in UEFI or Legacy mode, returns str.""" + kernel = ctypes.windll.kernel32 + firmware_type = ctypes.c_uint() - # Get value from kernel32 API - try: - kernel.GetFirmwareType(ctypes.byref(firmware_type)) - except: - # Just set to zero - firmware_type = ctypes.c_uint(0) + # Get value from kernel32 API + try: + kernel.GetFirmwareType(ctypes.byref(firmware_type)) + except: + # Just set to zero + firmware_type = ctypes.c_uint(0) - # Set return value - type_str = 'Unknown' - if firmware_type.value == 1: - type_str = 'Legacy' - elif firmware_type.value == 2: - type_str = 'UEFI' + # Set return value + type_str = 'Unknown' + if firmware_type.value == 1: + type_str = 'Legacy' + elif firmware_type.value == 2: + type_str = 'UEFI' - return type_str + return type_str def run_autoruns(): - """Run AutoRuns in the background with VirusTotal checks enabled.""" - extract_item('Autoruns', filter='autoruns*', silent=True) - # Update AutoRuns settings before running - for path, settings in AUTORUNS_SETTINGS.items(): - winreg.CreateKey(HKCU, path) - with winreg.OpenKey(HKCU, path, access=winreg.KEY_WRITE) as key: - for name, value in settings.items(): - winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) - popen_program(global_vars['Tools']['AutoRuns'], minimized=True) + """Run AutoRuns in the background with VirusTotal checks enabled.""" + extract_item('Autoruns', filter='autoruns*', silent=True) + # Update AutoRuns settings before running + for path, settings in AUTORUNS_SETTINGS.items(): + winreg.CreateKey(HKCU, path) + with winreg.OpenKey(HKCU, path, access=winreg.KEY_WRITE) as key: + for name, value in settings.items(): + winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) + popen_program(global_vars['Tools']['AutoRuns'], minimized=True) def run_hwinfo_sensors(): - """Run HWiNFO sensors.""" - path = r'{BinDir}\HWiNFO'.format(**global_vars) - for bit in [32, 64]: - # Configure - source = r'{}\general.ini'.format(path) - dest = r'{}\HWiNFO{}.ini'.format(path, bit) - shutil.copy(source, dest) - with open(dest, 'a') as f: - f.write('SensorsOnly=1\n') - f.write('SummaryOnly=0\n') - popen_program(global_vars['Tools']['HWiNFO']) + """Run HWiNFO sensors.""" + path = r'{BinDir}\HWiNFO'.format(**global_vars) + for bit in [32, 64]: + # Configure + source = r'{}\general.ini'.format(path) + dest = r'{}\HWiNFO{}.ini'.format(path, bit) + shutil.copy(source, dest) + with open(dest, 'a') as f: + f.write('SensorsOnly=1\n') + f.write('SummaryOnly=0\n') + popen_program(global_vars['Tools']['HWiNFO']) def run_nircmd(*cmd): - """Run custom NirCmd.""" - extract_item('NirCmd', silent=True) - cmd = [global_vars['Tools']['NirCmd'], *cmd] - run_program(cmd, check=False) + """Run custom NirCmd.""" + extract_item('NirCmd', silent=True) + cmd = [global_vars['Tools']['NirCmd'], *cmd] + run_program(cmd, check=False) def run_xmplay(): - """Run XMPlay to test audio.""" - extract_item('XMPlay', silent=True) - cmd = [global_vars['Tools']['XMPlay'], - r'{BinDir}\XMPlay\music.7z'.format(**global_vars)] + """Run XMPlay to test audio.""" + extract_item('XMPlay', silent=True) + cmd = [global_vars['Tools']['XMPlay'], + r'{BinDir}\XMPlay\music.7z'.format(**global_vars)] - # Unmute audio first - extract_item('NirCmd', silent=True) - run_nircmd('mutesysvolume', '0') + # Unmute audio first + extract_item('NirCmd', silent=True) + run_nircmd('mutesysvolume', '0') - # Open XMPlay - popen_program(cmd) + # Open XMPlay + popen_program(cmd) def run_hitmanpro(): - """Run HitmanPro in the background.""" - extract_item('HitmanPro', silent=True) - cmd = [ - global_vars['Tools']['HitmanPro'], - '/quiet', '/noinstall', '/noupload', - r'/log={LogDir}\Tools\HitmanPro.txt'.format(**global_vars)] - popen_program(cmd) + """Run HitmanPro in the background.""" + extract_item('HitmanPro', silent=True) + cmd = [ + global_vars['Tools']['HitmanPro'], + '/quiet', '/noinstall', '/noupload', + r'/log={LogDir}\Tools\HitmanPro.txt'.format(**global_vars)] + popen_program(cmd) def run_process_killer(): - """Kill most running processes skipping those in the whitelist.txt.""" - # borrowed from TronScript (reddit.com/r/TronScript) - # credit to /u/cuddlychops06 - prev_dir = os.getcwd() - extract_item('ProcessKiller', silent=True) - os.chdir(r'{BinDir}\ProcessKiller'.format(**global_vars)) - run_program(['ProcessKiller.exe', '/silent'], check=False) - os.chdir(prev_dir) + """Kill most running processes skipping those in the whitelist.txt.""" + # borrowed from TronScript (reddit.com/r/TronScript) + # credit to /u/cuddlychops06 + prev_dir = os.getcwd() + extract_item('ProcessKiller', silent=True) + os.chdir(r'{BinDir}\ProcessKiller'.format(**global_vars)) + run_program(['ProcessKiller.exe', '/silent'], check=False) + os.chdir(prev_dir) def run_rkill(): - """Run RKill and cleanup afterwards.""" - extract_item('RKill', silent=True) - cmd = [ - global_vars['Tools']['RKill'], - '-s', '-l', r'{LogDir}\Tools\RKill.log'.format(**global_vars), - '-new_console:n', '-new_console:s33V'] - run_program(cmd, check=False) - wait_for_process('RKill') + """Run RKill and cleanup afterwards.""" + extract_item('RKill', silent=True) + cmd = [ + global_vars['Tools']['RKill'], + '-s', '-l', r'{LogDir}\Tools\RKill.log'.format(**global_vars), + '-new_console:n', '-new_console:s33V'] + run_program(cmd, check=False) + wait_for_process('RKill') - # RKill cleanup - desktop_path = r'{USERPROFILE}\Desktop'.format(**global_vars['Env']) - if os.path.exists(desktop_path): - for item in os.scandir(desktop_path): - if re.search(r'^RKill', item.name, re.IGNORECASE): - dest = r'{LogDir}\Tools\{name}'.format( - name=dest, **global_vars) - dest = non_clobber_rename(dest) - shutil.move(item.path, dest) + # RKill cleanup + desktop_path = r'{USERPROFILE}\Desktop'.format(**global_vars['Env']) + if os.path.exists(desktop_path): + for item in os.scandir(desktop_path): + if re.search(r'^RKill', item.name, re.IGNORECASE): + dest = r'{LogDir}\Tools\{name}'.format( + name=dest, **global_vars) + dest = non_clobber_rename(dest) + shutil.move(item.path, dest) def show_alert_box(message, title='Wizard Kit Warning'): - """Show Windows alert box with message.""" - message_box = ctypes.windll.user32.MessageBoxW - message_box(None, message, title, 0x00001030) + """Show Windows alert box with message.""" + message_box = ctypes.windll.user32.MessageBoxW + message_box(None, message, title, 0x00001030) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From ffd07e07fd308e17ef1c6b7f1038708d977d385f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:53:58 -0700 Subject: [PATCH 119/186] Renamed diags.py to sw_diags.py * Avoid confusion with hw_diags.py --- .bin/Scripts/functions/{diags.py => sw_diags.py} | 0 .bin/Scripts/system_checklist.py | 2 +- .bin/Scripts/system_diagnostics.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename .bin/Scripts/functions/{diags.py => sw_diags.py} (100%) diff --git a/.bin/Scripts/functions/diags.py b/.bin/Scripts/functions/sw_diags.py similarity index 100% rename from .bin/Scripts/functions/diags.py rename to .bin/Scripts/functions/sw_diags.py diff --git a/.bin/Scripts/system_checklist.py b/.bin/Scripts/system_checklist.py index a76433ee..52720438 100644 --- a/.bin/Scripts/system_checklist.py +++ b/.bin/Scripts/system_checklist.py @@ -8,10 +8,10 @@ os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.activation import * from functions.cleanup import * -from functions.diags import * from functions.info import * from functions.product_keys import * from functions.setup import * +from functions.sw_diags import * init_global_vars() os.system('title {}: System Checklist Tool'.format(KIT_NAME_FULL)) set_log_file('System Checklist.log') diff --git a/.bin/Scripts/system_diagnostics.py b/.bin/Scripts/system_diagnostics.py index bbeb2d11..ffa28079 100644 --- a/.bin/Scripts/system_diagnostics.py +++ b/.bin/Scripts/system_diagnostics.py @@ -7,10 +7,10 @@ import sys os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.browsers import * -from functions.diags import * from functions.info import * from functions.product_keys import * from functions.repairs import * +from functions.sw_diags import * init_global_vars() os.system('title {}: System Diagnostics Tool'.format(KIT_NAME_FULL)) set_log_file('System Diagnostics.log') From 166a293864a43a9fab95e14279b2a1822654d723 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:55:41 -0700 Subject: [PATCH 120/186] Updated hw_diags.py --- .bin/Scripts/functions/hw_diags.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index a4184689..210ae870 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1320,7 +1320,8 @@ def run_mprime_test(state, test): if re.search(r'(error|fail)', line, re.IGNORECASE): test.failed = True test.update_status('NS') - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) + test.report.append( + ' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # prime.log (CS check) if log == 'prime.log': @@ -1349,7 +1350,8 @@ def run_mprime_test(state, test): for line in sorted(_tmp['Pass'].keys()): test.report.append(' {}'.format(line)) for line in sorted(_tmp['Warn'].keys()): - test.report.append(' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) + test.report.append( + ' {YELLOW}{line}{CLEAR}'.format(line=line, **COLORS)) # Unknown result if not (test.aborted or test.failed or test.passed): From 097fae866a51e527d32d26754a2e44a87954c85e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:57:39 -0700 Subject: [PATCH 121/186] Updated info.py --- .bin/Scripts/functions/info.py | 868 +++++++++++++++++---------------- 1 file changed, 437 insertions(+), 431 deletions(-) diff --git a/.bin/Scripts/functions/info.py b/.bin/Scripts/functions/info.py index 7630a766..4c691e3c 100644 --- a/.bin/Scripts/functions/info.py +++ b/.bin/Scripts/functions/info.py @@ -8,513 +8,519 @@ from functions.activation import * # Regex REGEX_OFFICE = re.compile( - r'(Microsoft (Office\s+' - r'(365|Enterprise|Home|Pro(\s|fessional)' - r'|Single|Small|Standard|Starter|Ultimate|system)' - r'|Works[-\s\d]+\d)' - r'|(Libre|Open|Star)\s*Office' - r'|WordPerfect|Gnumeric|Abiword)', - re.IGNORECASE) + r'(Microsoft (Office\s+' + r'(365|Enterprise|Home|Pro(\s|fessional)' + r'|Single|Small|Standard|Starter|Ultimate|system)' + r'|Works[-\s\d]+\d)' + r'|(Libre|Open|Star)\s*Office' + r'|WordPerfect|Gnumeric|Abiword)', + re.IGNORECASE) # STATIC VARIABLES REG_PROFILE_LIST = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList' REG_SHELL_FOLDERS = r'Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders' TMP_HIVE_PATH = 'TEMP_HIVE_MOUNT' EXTRA_FOLDERS = [ - 'Dropbox', - 'Google Drive', - 'OneDrive', - 'SkyDrive', + 'Dropbox', + 'Google Drive', + 'OneDrive', + 'SkyDrive', ] SHELL_FOLDERS = { - #GUIDs from: https://msdn.microsoft.com/en-us/library/windows/desktop/dd378457(v=vs.85).aspx - 'Desktop': ( - '{B4BFCC3A-DB2C-424C-B029-7FE99A87C641}', - ), - 'Documents': ( - 'Personal', - '{FDD39AD0-238F-46AF-ADB4-6C85480369C7}', - ), - 'Downloads': ( - '{374DE290-123F-4565-9164-39C4925E467B}', - ), - 'Favorites': ( - '{1777F761-68AD-4D8A-87BD-30B759FA33DD}', - ), - 'Music': ( - 'My Music', - '{4BD8D571-6D19-48D3-BE97-422220080E43}', - ), - 'Pictures': ( - 'My Pictures', - '{33E28130-4E1E-4676-835A-98395C3BC3BB}', - ), - 'Videos': ( - 'My Video', - '{18989B1D-99B5-455B-841C-AB7C74E4DDFC}', - ), + #GUIDs from: https://msdn.microsoft.com/en-us/library/windows/desktop/dd378457(v=vs.85).aspx + 'Desktop': ( + '{B4BFCC3A-DB2C-424C-B029-7FE99A87C641}', + ), + 'Documents': ( + 'Personal', + '{FDD39AD0-238F-46AF-ADB4-6C85480369C7}', + ), + 'Downloads': ( + '{374DE290-123F-4565-9164-39C4925E467B}', + ), + 'Favorites': ( + '{1777F761-68AD-4D8A-87BD-30B759FA33DD}', + ), + 'Music': ( + 'My Music', + '{4BD8D571-6D19-48D3-BE97-422220080E43}', + ), + 'Pictures': ( + 'My Pictures', + '{33E28130-4E1E-4676-835A-98395C3BC3BB}', + ), + 'Videos': ( + 'My Video', + '{18989B1D-99B5-455B-841C-AB7C74E4DDFC}', + ), } def backup_file_list(): - """Export current file listing for the system.""" - extract_item('Everything', silent=True) - cmd = [ - global_vars['Tools']['Everything'], - '-nodb', - '-create-filelist', - r'{LogDir}\File List.txt'.format(**global_vars), - global_vars['Env']['SYSTEMDRIVE']] - run_program(cmd) + """Export current file listing for the system.""" + extract_item('Everything', silent=True) + cmd = [ + global_vars['Tools']['Everything'], + '-nodb', + '-create-filelist', + r'{LogDir}\File List.txt'.format(**global_vars), + global_vars['Env']['SYSTEMDRIVE']] + run_program(cmd) def backup_power_plans(): - """Export current power plans.""" - os.makedirs(r'{BackupDir}\Power Plans\{Date}'.format( - **global_vars), exist_ok=True) - plans = run_program(['powercfg', '/L']) - plans = plans.stdout.decode().splitlines() - plans = [p for p in plans if re.search(r'^Power Scheme', p)] - for p in plans: - guid = re.sub(r'Power Scheme GUID:\s+([0-9a-f\-]+).*', r'\1', p) - name = re.sub( - r'Power Scheme GUID:\s+[0-9a-f\-]+\s+\(([^\)]+)\).*', r'\1', p) - out = r'{BackupDir}\Power Plans\{Date}\{name}.pow'.format( - name=name, **global_vars) - if not os.path.exists(out): - cmd = ['powercfg', '-export', out, guid] - run_program(cmd, check=False) + """Export current power plans.""" + os.makedirs(r'{BackupDir}\Power Plans\{Date}'.format( + **global_vars), exist_ok=True) + plans = run_program(['powercfg', '/L']) + plans = plans.stdout.decode().splitlines() + plans = [p for p in plans if re.search(r'^Power Scheme', p)] + for p in plans: + guid = re.sub(r'Power Scheme GUID:\s+([0-9a-f\-]+).*', r'\1', p) + name = re.sub( + r'Power Scheme GUID:\s+[0-9a-f\-]+\s+\(([^\)]+)\).*', r'\1', p) + out = r'{BackupDir}\Power Plans\{Date}\{name}.pow'.format( + name=name, **global_vars) + if not os.path.exists(out): + cmd = ['powercfg', '-export', out, guid] + run_program(cmd, check=False) def backup_registry(overwrite=False): - """Backup registry including user hives.""" - extract_item('erunt', silent=True) - cmd = [ - global_vars['Tools']['ERUNT'], - r'{BackupDir}\Registry\{Date}'.format(**global_vars), - 'sysreg', - 'curuser', - 'otherusers', - '/noprogresswindow'] - if overwrite: - cmd.append('/noconfirmdelete') - run_program(cmd) + """Backup registry including user hives.""" + extract_item('erunt', silent=True) + cmd = [ + global_vars['Tools']['ERUNT'], + r'{BackupDir}\Registry\{Date}'.format(**global_vars), + 'sysreg', + 'curuser', + 'otherusers', + '/noprogresswindow'] + if overwrite: + cmd.append('/noconfirmdelete') + run_program(cmd) def get_folder_size(path): - """Get (human-readable) size of folder passed, returns str.""" - size = 'Unknown' - cmd = [global_vars['Tools']['Du'], '-c', '-nobanner', '-q', path] + """Get (human-readable) size of folder passed, returns str.""" + size = 'Unknown' + cmd = [global_vars['Tools']['Du'], '-c', '-nobanner', '-q', path] + try: + out = run_program(cmd) + except FileNotFoundError: + # Failed to find folder + pass + except subprocess.CalledProcessError: + # Failed to get folder size + pass + else: try: - out = run_program(cmd) - except FileNotFoundError: - # Failed to find folder - pass - except subprocess.CalledProcessError: - # Failed to get folder size - pass + size = out.stdout.decode().split(',')[-2] + except IndexError: + # Failed to parse csv data + pass else: - try: - size = out.stdout.decode().split(',')[-2] - except IndexError: - # Failed to parse csv data - pass - else: - size = human_readable_size(size) - return size + size = human_readable_size(size) + return size def get_installed_antivirus(): - """Get list of installed Antivirus programs.""" - programs = [] + """Get list of installed Antivirus programs.""" + programs = [] + cmd = ['WMIC', r'/namespace:\\root\SecurityCenter2', + 'path', 'AntivirusProduct', + 'get', 'displayName', '/value'] + out = run_program(cmd) + out = out.stdout.decode().strip() + products = out.splitlines() + products = [p.split('=')[1] for p in products if p] + for prod in sorted(products): + # Get product state and check if it's enabled + # credit: https://jdhitsolutions.com/blog/powershell/5187/get-antivirus-product-status-with-powershell/ cmd = ['WMIC', r'/namespace:\\root\SecurityCenter2', - 'path', 'AntivirusProduct', - 'get', 'displayName', '/value'] + 'path', 'AntivirusProduct', + 'where', 'displayName="{}"'.format(prod), + 'get', 'productState', '/value'] out = run_program(cmd) out = out.stdout.decode().strip() - products = out.splitlines() - products = [p.split('=')[1] for p in products if p] - for prod in sorted(products): - # Get product state and check if it's enabled - # credit: https://jdhitsolutions.com/blog/powershell/5187/get-antivirus-product-status-with-powershell/ - cmd = ['WMIC', r'/namespace:\\root\SecurityCenter2', - 'path', 'AntivirusProduct', - 'where', 'displayName="{}"'.format(prod), - 'get', 'productState', '/value'] - out = run_program(cmd) - out = out.stdout.decode().strip() - state = out.split('=')[1] - state = hex(int(state)) - if str(state)[3:5] != '10': - programs.append('[Disabled] {}'.format(prod)) - else: - programs.append(prod) + state = out.split('=')[1] + state = hex(int(state)) + if str(state)[3:5] != '10': + programs.append('[Disabled] {}'.format(prod)) + else: + programs.append(prod) - if len(programs) == 0: - programs = ['No programs found'] - return programs + if len(programs) == 0: + programs = ['No programs found'] + return programs def get_installed_office(): - """Get list of installed Office programs.""" - programs = [] - log_file = r'{LogDir}\Installed Program List (AIDA64).txt'.format( - **global_vars) - with open (log_file, 'r') as f: - for line in sorted(f.readlines()): - if REGEX_OFFICE.search(line): - programs.append(line[4:82].strip()) + """Get list of installed Office programs.""" + programs = [] + log_file = r'{LogDir}\Installed Program List (AIDA64).txt'.format( + **global_vars) + with open (log_file, 'r') as f: + for line in sorted(f.readlines()): + if REGEX_OFFICE.search(line): + programs.append(line[4:82].strip()) - if len(programs) == 0: - programs = ['No programs found'] - return programs + if len(programs) == 0: + programs = ['No programs found'] + return programs def get_shell_path(folder, user='current'): - """Get shell path using SHGetKnownFolderPath via knownpaths, returns str. + """Get shell path using SHGetKnownFolderPath via knownpaths, returns str. - NOTE: Only works for the current user. - Code based on https://gist.github.com/mkropat/7550097 - """ - path = None - folderid = None - if user.lower() == 'public': - user = 'common' + NOTE: Only works for the current user. + Code based on https://gist.github.com/mkropat/7550097 + """ + path = None + folderid = None + if user.lower() == 'public': + user = 'common' + try: + folderid = getattr(knownpaths.FOLDERID, folder) + except AttributeError: + # Unknown folder ID, ignore and return None + pass + + if folderid: try: - folderid = getattr(knownpaths.FOLDERID, folder) - except AttributeError: - # Unknown folder ID, ignore and return None - pass + path = knownpaths.get_path( + folderid, getattr(knownpaths.UserHandle, user)) + except PathNotFoundError: + # Folder not found, ignore and return None + pass - if folderid: - try: - path = knownpaths.get_path(folderid, getattr(knownpaths.UserHandle, user)) - except PathNotFoundError: - # Folder not found, ignore and return None - pass - - return path + return path def get_user_data_paths(user): - """Get user data paths for provided user, returns dict.""" - hive_path = user['SID'] - paths = { - 'Profile': { - 'Path': None, - }, - 'Shell Folders': {}, - 'Extra Folders': {}, - } - unload_hive = False + """Get user data paths for provided user, returns dict.""" + hive_path = user['SID'] + paths = { + 'Profile': { + 'Path': None, + }, + 'Shell Folders': {}, + 'Extra Folders': {}, + } + unload_hive = False - if user['Name'] == global_vars['Env']['USERNAME']: - # We can use SHGetKnownFolderPath for the current user - paths['Profile']['Path'] = get_shell_path('Profile') - paths['Shell Folders'] = {f: {'Path': get_shell_path(f)} - for f in SHELL_FOLDERS.keys()} - else: - # We have to use the NTUSER.dat hives which isn't recommended by MS - try: - key_path = r'{}\{}'.format(REG_PROFILE_LIST, user['SID']) - with winreg.OpenKey(HKLM, key_path) as key: - paths['Profile']['Path'] = winreg.QueryValueEx( - key, 'ProfileImagePath')[0] - except Exception: - # Profile path not found, leaving as None. - pass + if user['Name'] == global_vars['Env']['USERNAME']: + # We can use SHGetKnownFolderPath for the current user + paths['Profile']['Path'] = get_shell_path('Profile') + paths['Shell Folders'] = {f: {'Path': get_shell_path(f)} + for f in SHELL_FOLDERS.keys()} + else: + # We have to use the NTUSER.dat hives which isn't recommended by MS + try: + key_path = r'{}\{}'.format(REG_PROFILE_LIST, user['SID']) + with winreg.OpenKey(HKLM, key_path) as key: + paths['Profile']['Path'] = winreg.QueryValueEx( + key, 'ProfileImagePath')[0] + except Exception: + # Profile path not found, leaving as None. + pass - # Shell folders (Prep) - if not reg_path_exists(HKU, hive_path) and paths['Profile']['Path']: - # User not logged-in, loading hive - # Also setting unload_hive so it will be unloaded later. - hive_path = TMP_HIVE_PATH - cmd = ['reg', 'load', r'HKU\{}'.format(TMP_HIVE_PATH), - r'{}\NTUSER.DAT'.format(paths['Profile']['Path'])] - unload_hive = True + # Shell folders (Prep) + if not reg_path_exists(HKU, hive_path) and paths['Profile']['Path']: + # User not logged-in, loading hive + # Also setting unload_hive so it will be unloaded later. + hive_path = TMP_HIVE_PATH + cmd = ['reg', 'load', r'HKU\{}'.format(TMP_HIVE_PATH), + r'{}\NTUSER.DAT'.format(paths['Profile']['Path'])] + unload_hive = True + try: + run_program(cmd) + except subprocess.CalledProcessError: + # Failed to load user hive + pass + + # Shell folders + shell_folders = r'{}\{}'.format(hive_path, REG_SHELL_FOLDERS) + if (reg_path_exists(HKU, hive_path) + and reg_path_exists(HKU, shell_folders)): + with winreg.OpenKey(HKU, shell_folders) as key: + for folder, values in SHELL_FOLDERS.items(): + for value in values: try: - run_program(cmd) - except subprocess.CalledProcessError: - # Failed to load user hive - pass + path = winreg.QueryValueEx(key, value)[0] + except FileNotFoundError: + # Skip missing values + pass + else: + paths['Shell Folders'][folder] = {'Path': path} + # Stop checking values for this folder + break - # Shell folders - shell_folders = r'{}\{}'.format(hive_path, REG_SHELL_FOLDERS) - if (reg_path_exists(HKU, hive_path) - and reg_path_exists(HKU, shell_folders)): - with winreg.OpenKey(HKU, shell_folders) as key: - for folder, values in SHELL_FOLDERS.items(): - for value in values: - try: - path = winreg.QueryValueEx(key, value)[0] - except FileNotFoundError: - # Skip missing values - pass - else: - paths['Shell Folders'][folder] = {'Path': path} - # Stop checking values for this folder - break - - # Shell folder (extra check) - if paths['Profile']['Path']: - for folder in SHELL_FOLDERS.keys(): - folder_path = r'{Path}\{folder}'.format( - folder=folder, **paths['Profile']) - if (folder not in paths['Shell Folders'] - and os.path.exists(folder_path)): - paths['Shell Folders'][folder] = {'Path': folder_path} - - # Extra folders + # Shell folder (extra check) if paths['Profile']['Path']: - for folder in EXTRA_FOLDERS: - folder_path = r'{Path}\{folder}'.format( - folder=folder, **paths['Profile']) - if os.path.exists(folder_path): - paths['Extra Folders'][folder] = {'Path': folder_path} + for folder in SHELL_FOLDERS.keys(): + folder_path = r'{Path}\{folder}'.format( + folder=folder, **paths['Profile']) + if (folder not in paths['Shell Folders'] + and os.path.exists(folder_path)): + paths['Shell Folders'][folder] = {'Path': folder_path} - # Shell folders (cleanup) - if unload_hive: - cmd = ['reg', 'unload', r'HKU\{}'.format(TMP_HIVE_PATH)] - run_program(cmd, check=False) + # Extra folders + if paths['Profile']['Path']: + for folder in EXTRA_FOLDERS: + folder_path = r'{Path}\{folder}'.format( + folder=folder, **paths['Profile']) + if os.path.exists(folder_path): + paths['Extra Folders'][folder] = {'Path': folder_path} - # Done - return paths + # Shell folders (cleanup) + if unload_hive: + cmd = ['reg', 'unload', r'HKU\{}'.format(TMP_HIVE_PATH)] + run_program(cmd, check=False) + + # Done + return paths def get_user_folder_sizes(users): - """Update list(users) to include folder paths and sizes.""" - extract_item('du', filter='du*', silent=True) - # Configure Du - winreg.CreateKey(HKCU, r'Software\Sysinternals\Du') - with winreg.OpenKey(HKCU, - r'Software\Sysinternals\Du', access=winreg.KEY_WRITE) as key: - winreg.SetValueEx(key, 'EulaAccepted', 0, winreg.REG_DWORD, 1) + """Update list(users) to include folder paths and sizes.""" + extract_item('du', filter='du*', silent=True) + # Configure Du + winreg.CreateKey(HKCU, r'Software\Sysinternals\Du') + with winreg.OpenKey(HKCU, + r'Software\Sysinternals\Du', access=winreg.KEY_WRITE) as key: + winreg.SetValueEx(key, 'EulaAccepted', 0, winreg.REG_DWORD, 1) - for u in users: - u.update(get_user_data_paths(u)) - if u['Profile']['Path']: - u['Profile']['Size'] = get_folder_size(u['Profile']['Path']) - for folder in u['Shell Folders'].keys(): - u['Shell Folders'][folder]['Size'] = get_folder_size( - u['Shell Folders'][folder]['Path']) - for folder in u['Extra Folders'].keys(): - u['Extra Folders'][folder]['Size'] = get_folder_size( - u['Extra Folders'][folder]['Path']) + for u in users: + u.update(get_user_data_paths(u)) + if u['Profile']['Path']: + u['Profile']['Size'] = get_folder_size(u['Profile']['Path']) + for folder in u['Shell Folders'].keys(): + u['Shell Folders'][folder]['Size'] = get_folder_size( + u['Shell Folders'][folder]['Path']) + for folder in u['Extra Folders'].keys(): + u['Extra Folders'][folder]['Size'] = get_folder_size( + u['Extra Folders'][folder]['Path']) def get_user_list(): - """Get user list via WMIC, returns list of dicts.""" - users = [] + """Get user list via WMIC, returns list of dicts.""" + users = [] - # Get user info from WMI - cmd = ['wmic', 'useraccount', 'get', '/format:csv'] - try: - out = run_program(cmd) - except subprocess.CalledProcessError: - # Meh, return empty list to avoid a full crash - return users - - entries = out.stdout.decode().splitlines() - entries = [e.strip().split(',') for e in entries if e.strip()] - - # Add user(s) to dict - keys = entries[0] - for e in entries[1:]: - # Create dict using 1st line (keys) - e = dict(zip(keys, e)) - # Set Active status via 'Disabled' TRUE/FALSE str - e['Active'] = bool(e['Disabled'].upper() == 'FALSE') - # Assume SIDs ending with 1000+ are "Standard" and others are "System" - e['Type'] = 'Standard' if re.search(r'-1\d+$', e['SID']) else 'System' - users.append(e) - - # Sort list - users.sort(key=itemgetter('Name')) - - # Done + # Get user info from WMI + cmd = ['wmic', 'useraccount', 'get', '/format:csv'] + try: + out = run_program(cmd) + except subprocess.CalledProcessError: + # Meh, return empty list to avoid a full crash return users + entries = out.stdout.decode().splitlines() + entries = [e.strip().split(',') for e in entries if e.strip()] + + # Add user(s) to dict + keys = entries[0] + for e in entries[1:]: + # Create dict using 1st line (keys) + e = dict(zip(keys, e)) + # Set Active status via 'Disabled' TRUE/FALSE str + e['Active'] = bool(e['Disabled'].upper() == 'FALSE') + # Assume SIDs ending with 1000+ are "Standard" and others are "System" + e['Type'] = 'Standard' if re.search(r'-1\d+$', e['SID']) else 'System' + users.append(e) + + # Sort list + users.sort(key=itemgetter('Name')) + + # Done + return users + def reg_path_exists(hive, path): - """Test if specified path exists, returns bool.""" - try: - winreg.QueryValue(hive, path) - except FileNotFoundError: - return False - else: - return True + """Test if specified path exists, returns bool.""" + try: + winreg.QueryValue(hive, path) + except FileNotFoundError: + return False + else: + return True def run_aida64(): - """Run AIDA64 to save system reports.""" - extract_item('AIDA64', silent=True) - # All system info - config = r'{BinDir}\AIDA64\full.rpf'.format(**global_vars) - report_file = r'{LogDir}\System Information (AIDA64).html'.format( - **global_vars) - if not os.path.exists(report_file): - cmd = [ - global_vars['Tools']['AIDA64'], - '/R', report_file, - '/CUSTOM', config, - '/HTML', '/SILENT', '/SAFEST'] - run_program(cmd, check=False) + """Run AIDA64 to save system reports.""" + extract_item('AIDA64', silent=True) + # All system info + config = r'{BinDir}\AIDA64\full.rpf'.format(**global_vars) + report_file = r'{LogDir}\System Information (AIDA64).html'.format( + **global_vars) + if not os.path.exists(report_file): + cmd = [ + global_vars['Tools']['AIDA64'], + '/R', report_file, + '/CUSTOM', config, + '/HTML', '/SILENT', '/SAFEST'] + run_program(cmd, check=False) - # Installed Programs - config = r'{BinDir}\AIDA64\installed_programs.rpf'.format(**global_vars) - report_file = r'{LogDir}\Installed Program List (AIDA64).txt'.format( - **global_vars) - if not os.path.exists(report_file): - cmd = [ - global_vars['Tools']['AIDA64'], - '/R', report_file, - '/CUSTOM', config, - '/TEXT', '/SILENT', '/SAFEST'] - run_program(cmd, check=False) + # Installed Programs + config = r'{BinDir}\AIDA64\installed_programs.rpf'.format(**global_vars) + report_file = r'{LogDir}\Installed Program List (AIDA64).txt'.format( + **global_vars) + if not os.path.exists(report_file): + cmd = [ + global_vars['Tools']['AIDA64'], + '/R', report_file, + '/CUSTOM', config, + '/TEXT', '/SILENT', '/SAFEST'] + run_program(cmd, check=False) - # Product Keys - config = r'{BinDir}\AIDA64\licenses.rpf'.format(**global_vars) - report_file = r'{LogDir}\Product Keys (AIDA64).txt'.format(**global_vars) - if not os.path.exists(report_file): - cmd = [ - global_vars['Tools']['AIDA64'], - '/R', report_file, - '/CUSTOM', config, - '/TEXT', '/SILENT', '/SAFEST'] - run_program(cmd, check=False) + # Product Keys + config = r'{BinDir}\AIDA64\licenses.rpf'.format(**global_vars) + report_file = r'{LogDir}\Product Keys (AIDA64).txt'.format(**global_vars) + if not os.path.exists(report_file): + cmd = [ + global_vars['Tools']['AIDA64'], + '/R', report_file, + '/CUSTOM', config, + '/TEXT', '/SILENT', '/SAFEST'] + run_program(cmd, check=False) def run_bleachbit(cleaners=None, preview=True): - """Run BleachBit preview and save log. + """Run BleachBit preview and save log. - If preview is True then no files should be deleted.""" - error_path = r'{}\Tools\BleachBit.err'.format(global_vars['LogDir']) - log_path = error_path.replace('err', 'log') - extract_item('BleachBit', silent=True) + If preview is True then no files should be deleted.""" + error_path = r'{}\Tools\BleachBit.err'.format(global_vars['LogDir']) + log_path = error_path.replace('err', 'log') + extract_item('BleachBit', silent=True) - # Safety check - if not cleaners: - # Disable cleaning and use preset config - cleaners = ['--preset'] - preview = True + # Safety check + if not cleaners: + # Disable cleaning and use preset config + cleaners = ['--preset'] + preview = True - # Run - cmd = [ - global_vars['Tools']['BleachBit'], - '--preview' if preview else '--clean'] - cmd.extend(cleaners) - out = run_program(cmd, check=False) + # Run + cmd = [ + global_vars['Tools']['BleachBit'], + '--preview' if preview else '--clean'] + cmd.extend(cleaners) + out = run_program(cmd, check=False) - # Save stderr - if out.stderr.decode().splitlines(): - with open(error_path, 'a', encoding='utf-8') as f: - for line in out.stderr.decode().splitlines(): - f.write(line.strip() + '\n') + # Save stderr + if out.stderr.decode().splitlines(): + with open(error_path, 'a', encoding='utf-8') as f: + for line in out.stderr.decode().splitlines(): + f.write(line.strip() + '\n') - # Save stdout - with open(log_path, 'a', encoding='utf-8') as f: - for line in out.stdout.decode().splitlines(): - f.write(line.strip() + '\n') + # Save stdout + with open(log_path, 'a', encoding='utf-8') as f: + for line in out.stdout.decode().splitlines(): + f.write(line.strip() + '\n') def show_disk_usage(disk): - """Show free and used space for a specified disk.""" - print_standard('{:5}'.format(disk.device.replace('/', ' ')), - end='', flush=True, timestamp=False) - try: - usage = psutil.disk_usage(disk.device) - display_string = '{percent:>5.2f}% Free ({free} / {total})'.format( - percent = 100 - usage.percent, - free = human_readable_size(usage.free, 2), - total = human_readable_size(usage.total, 2)) - if usage.percent > 85: - print_error(display_string, timestamp=False) - elif usage.percent > 75: - print_warning(display_string, timestamp=False) - else: - print_standard(display_string, timestamp=False) - except Exception: - print_warning('Unknown', timestamp=False) + """Show free and used space for a specified disk.""" + print_standard('{:5}'.format(disk.device.replace('/', ' ')), + end='', flush=True, timestamp=False) + try: + usage = psutil.disk_usage(disk.device) + display_string = '{percent:>5.2f}% Free ({free} / {total})'.format( + percent = 100 - usage.percent, + free = human_readable_size(usage.free, 2), + total = human_readable_size(usage.total, 2)) + if usage.percent > 85: + print_error(display_string, timestamp=False) + elif usage.percent > 75: + print_warning(display_string, timestamp=False) + else: + print_standard(display_string, timestamp=False) + except Exception: + print_warning('Unknown', timestamp=False) def show_free_space(indent=8, width=32): - """Show free space info for all fixed disks.""" - message = 'Free Space:' - for disk in psutil.disk_partitions(): - try: - if 'fixed' in disk.opts: - try_and_print(message=message, function=show_disk_usage, - ns='Unknown', silent_function=False, - indent=indent, width=width, disk=disk) - message = '' - except Exception: - pass + """Show free space info for all fixed disks.""" + message = 'Free Space:' + for disk in psutil.disk_partitions(): + try: + if 'fixed' in disk.opts: + try_and_print(message=message, function=show_disk_usage, + ns='Unknown', silent_function=False, + indent=indent, width=width, disk=disk) + message = '' + except Exception: + pass def show_installed_ram(): - """Show installed RAM.""" - mem = psutil.virtual_memory() - if mem.total > 5905580032: - # > 5.5 Gb so 6Gb or greater - print_standard(human_readable_size(mem.total).strip(), timestamp=False) - elif mem.total > 3758096384: - # > 3.5 Gb so 4Gb or greater - print_warning(human_readable_size(mem.total).strip(), timestamp=False) - else: - print_error(human_readable_size(mem.total).strip(), timestamp=False) + """Show installed RAM.""" + mem = psutil.virtual_memory() + if mem.total > 5905580032: + # > 5.5 Gb so 6Gb or greater + print_standard(human_readable_size(mem.total).strip(), timestamp=False) + elif mem.total > 3758096384: + # > 3.5 Gb so 4Gb or greater + print_warning(human_readable_size(mem.total).strip(), timestamp=False) + else: + print_error(human_readable_size(mem.total).strip(), timestamp=False) def show_os_activation(): - """Show OS activation info.""" - act_str = get_activation_string() - if windows_is_activated(): - print_standard(act_str, timestamp=False) - elif re.search(r'unavailable', act_str, re.IGNORECASE): - print_warning(act_str, timestamp=False) - else: - print_error(act_str, timestamp=False) + """Show OS activation info.""" + act_str = get_activation_string() + if windows_is_activated(): + print_standard(act_str, timestamp=False) + elif re.search(r'unavailable', act_str, re.IGNORECASE): + print_warning(act_str, timestamp=False) + else: + print_error(act_str, timestamp=False) def show_os_name(): - """Show extended OS name (including warnings).""" - os_name = global_vars['OS']['DisplayName'] - if global_vars['OS']['Arch'] == 32: - # Show all 32-bit installs as an error message - print_error(os_name, timestamp=False) + """Show extended OS name (including warnings).""" + os_name = global_vars['OS']['DisplayName'] + if global_vars['OS']['Arch'] == 32: + # Show all 32-bit installs as an error message + print_error(os_name, timestamp=False) + else: + if re.search( + r'(preview build|unrecognized|unsupported)', + os_name, + re.IGNORECASE): + print_error(os_name, timestamp=False) + elif re.search(r'outdated', os_name, re.IGNORECASE): + print_warning(os_name, timestamp=False) else: - if re.search(r'(preview build|unrecognized|unsupported)', os_name, re.IGNORECASE): - print_error(os_name, timestamp=False) - elif re.search(r'outdated', os_name, re.IGNORECASE): - print_warning(os_name, timestamp=False) - else: - print_standard(os_name, timestamp=False) + print_standard(os_name, timestamp=False) def show_temp_files_size(): - """Show total size of temp files identified by BleachBit.""" - size = None - with open(r'{LogDir}\Tools\BleachBit.log'.format(**global_vars), 'r') as f: - for line in f.readlines(): - if re.search(r'^disk space to be recovered:', line, re.IGNORECASE): - size = re.sub(r'.*: ', '', line.strip()) - size = re.sub(r'(\w)iB$', r' \1b', size) - if size is None: - print_warning(size, timestamp=False) - else: - print_standard(size, timestamp=False) + """Show total size of temp files identified by BleachBit.""" + size = None + with open(r'{LogDir}\Tools\BleachBit.log'.format(**global_vars), 'r') as f: + for line in f.readlines(): + if re.search(r'^disk space to be recovered:', line, re.IGNORECASE): + size = re.sub(r'.*: ', '', line.strip()) + size = re.sub(r'(\w)iB$', r' \1b', size) + if size is None: + print_warning(size, timestamp=False) + else: + print_standard(size, timestamp=False) def show_user_data_summary(indent=8, width=32): - """Print user data folder sizes for all users.""" - users = get_user_list() - users = [u for u in users if u['Active']] - get_user_folder_sizes(users) - for user in users: - if ('Size' not in user['Profile'] - and not any(user['Shell Folders']) - and not any(user['Extra Folders'])): - # Skip empty users - continue - print_success('{indent}User: {user}'.format( - indent = ' '*int(indent/2), - user = user['Name'])) - for section in ['Profile', None, 'Shell Folders', 'Extra Folders']: - folders = [] - if section is None: - # Divider - print_standard('{}{}'.format(' '*indent, '-'*(width+6))) - elif section == 'Profile': - folders = {'Profile': user['Profile']} - else: - folders = user[section] - for folder in folders: - print_standard( - '{indent}{folder:<{width}}{size:>6} ({path})'.format( - indent = ' ' * indent, - width = width, - folder = folder, - size = folders[folder].get('Size', 'Unknown'), - path = folders[folder].get('Path', 'Unknown'))) + """Print user data folder sizes for all users.""" + users = get_user_list() + users = [u for u in users if u['Active']] + get_user_folder_sizes(users) + for user in users: + if ('Size' not in user['Profile'] + and not any(user['Shell Folders']) + and not any(user['Extra Folders'])): + # Skip empty users + continue + print_success('{indent}User: {user}'.format( + indent = ' '*int(indent/2), + user = user['Name'])) + for section in ['Profile', None, 'Shell Folders', 'Extra Folders']: + folders = [] + if section is None: + # Divider + print_standard('{}{}'.format(' '*indent, '-'*(width+6))) + elif section == 'Profile': + folders = {'Profile': user['Profile']} + else: + folders = user[section] + for folder in folders: + print_standard( + '{indent}{folder:<{width}}{size:>6} ({path})'.format( + indent = ' ' * indent, + width = width, + folder = folder, + size = folders[folder].get('Size', 'Unknown'), + path = folders[folder].get('Path', 'Unknown'))) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 018aba2fe6f1b4e34e007dc24beea9c3ac544c49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 19:58:06 -0700 Subject: [PATCH 122/186] Updated network.py --- .bin/Scripts/functions/network.py | 89 ++++++++++++++++--------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index 5735c486..80935506 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -13,60 +13,61 @@ from functions.common import * # REGEX REGEX_VALID_IP = re.compile( - r'(10.\d+.\d+.\d+' - r'|172.(1[6-9]|2\d|3[0-1])' - r'|192.168.\d+.\d+)', - re.IGNORECASE) + r'(10.\d+.\d+.\d+' + r'|172.(1[6-9]|2\d|3[0-1])' + r'|192.168.\d+.\d+)', + re.IGNORECASE) def connect_to_network(): - """Connect to network if not already connected.""" - net_ifs = psutil.net_if_addrs() - net_ifs = [i[:2] for i in net_ifs.keys()] + """Connect to network if not already connected.""" + net_ifs = psutil.net_if_addrs() + net_ifs = [i[:2] for i in net_ifs.keys()] - # Bail if currently connected - if is_connected(): - return + # Bail if currently connected + if is_connected(): + return - # WiFi - if 'wl' in net_ifs: - cmd = [ - 'nmcli', 'dev', 'wifi', - 'connect', WIFI_SSID, - 'password', WIFI_PASSWORD] - try_and_print( - message = 'Connecting to {}...'.format(WIFI_SSID), - function = run_program, - cmd = cmd) + # WiFi + if 'wl' in net_ifs: + cmd = [ + 'nmcli', 'dev', 'wifi', + 'connect', WIFI_SSID, + 'password', WIFI_PASSWORD] + try_and_print( + message = 'Connecting to {}...'.format(WIFI_SSID), + function = run_program, + cmd = cmd) def is_connected(): - """Check for a valid private IP.""" - devs = psutil.net_if_addrs() - for dev in devs.values(): - for family in dev: - if REGEX_VALID_IP.search(family.address): - # Valid IP found - return True - # Else - return False + """Check for a valid private IP.""" + devs = psutil.net_if_addrs() + for dev in devs.values(): + for family in dev: + if REGEX_VALID_IP.search(family.address): + # Valid IP found + return True + # Else + return False def show_valid_addresses(): - """Show all valid private IP addresses assigned to the system.""" - devs = psutil.net_if_addrs() - for dev, families in sorted(devs.items()): - for family in families: - if REGEX_VALID_IP.search(family.address): - # Valid IP found - show_data(message=dev, data=family.address) + """Show all valid private IP addresses assigned to the system.""" + devs = psutil.net_if_addrs() + for dev, families in sorted(devs.items()): + for family in families: + if REGEX_VALID_IP.search(family.address): + # Valid IP found + show_data(message=dev, data=family.address) def speedtest(): - """Run a network speedtest using speedtest-cli.""" - result = run_program(['speedtest-cli', '--simple']) - output = [line.strip() for line in result.stdout.decode().splitlines() - if line.strip()] - output = [line.split() for line in output] - output = [(a, float(b), c) for a, b, c in output] - return ['{:10}{:6.2f} {}'.format(*line) for line in output] + """Run a network speedtest using speedtest-cli.""" + result = run_program(['speedtest-cli', '--simple']) + output = [line.strip() for line in result.stdout.decode().splitlines() + if line.strip()] + output = [line.split() for line in output] + output = [(a, float(b), c) for a, b, c in output] + return ['{:10}{:6.2f} {}'.format(*line) for line in output] if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") +# vim: sts=2 sw=2 ts=2 From 07e43307c5bfc97e53692f5846762cd3bd1187df Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:04:29 -0700 Subject: [PATCH 123/186] Moved partiton_uids.py to settings --- .bin/Scripts/functions/disk.py | 4 +- .bin/Scripts/functions/partition_uids.py | 326 ----------------------- .bin/Scripts/settings/partition_uids.py | 325 ++++++++++++++++++++++ 3 files changed, 327 insertions(+), 328 deletions(-) delete mode 100644 .bin/Scripts/functions/partition_uids.py create mode 100644 .bin/Scripts/settings/partition_uids.py diff --git a/.bin/Scripts/functions/disk.py b/.bin/Scripts/functions/disk.py index 75879ff8..257f21b0 100644 --- a/.bin/Scripts/functions/disk.py +++ b/.bin/Scripts/functions/disk.py @@ -1,7 +1,7 @@ # Wizard Kit: Functions - Disk from functions.common import * -from functions import partition_uids +from settings.partition_uids import * # Regex REGEX_BAD_PARTITION = re.compile(r'(RAW|Unknown)', re.IGNORECASE) @@ -113,7 +113,7 @@ def get_partition_details(disk, partition): details.update({key.strip(): value.strip() for (key, value) in tmp}) # Get MBR type / GPT GUID for extra details on "Unknown" partitions - guid = partition_uids.lookup_guid(details.get('Type')) + guid = PARTITION_UIDS.get(details.get('Type').upper(), {}) if guid: details.update({ 'Description': guid.get('Description', '')[:29], diff --git a/.bin/Scripts/functions/partition_uids.py b/.bin/Scripts/functions/partition_uids.py deleted file mode 100644 index 38fba0db..00000000 --- a/.bin/Scripts/functions/partition_uids.py +++ /dev/null @@ -1,326 +0,0 @@ -# Wizard Kit: Functions - PARTITION UIDs -# sources: https://en.wikipedia.org/wiki/GUID_Partition_Table -# https://en.wikipedia.org/wiki/Partition_type -# NOTE: Info has been trimmed for brevity. As such, there may be some inaccuracy. - -PARTITION_UIDS = { - '00': {'OS': 'All','Description': 'Empty partition entry'}, - '01': {'OS': 'DOS','Description': 'FAT12 as primary partition'}, - '02': {'OS': 'XENIX','Description': 'XENIX root'}, - '03': {'OS': 'XENIX','Description': 'XENIX usr'}, - '04': {'OS': 'DOS','Description': 'FAT16 with less than 32 MB'}, - '05': {'OS': 'DOS / SpeedStor','Description': 'Extended partition'}, - '06': {'OS': 'DOS1+','Description': 'FAT16B [over 65K sectors]'}, - '07': {'OS': 'Windows / OS/2 / QNX 2','Description': 'NTFS/exFAT/HPFS/IFS/QNX'}, - '08': {'OS': 'CBM / DOS / OS/2 / AIX /QNX','Description': 'FAT12-16/AIX/QNY/SplitDrive'}, - '09': {'OS': 'AIX / QNX / Coherent / OS-9','Description': 'AIX/QNZ/Coherent/RBF'}, - '0A': {'OS': 'OS/2 / Coherent','Description': 'Boot Manager / Swap'}, - '0B': {'OS': 'DOS','Description': 'FAT32 with CHS addressing'}, - '0C': {'OS': 'DOS','Description': 'FAT32 with LBA'}, - '0D': {'OS': 'Silicon Safe','Description': 'Reserved'}, - '0E': {'OS': 'DOS','Description': 'FAT16B with LBA'}, - '0F': {'OS': 'DOS','Description': 'Extended partition with LBA'}, - '10': {'OS': 'OPUS','Description': 'Unknown'}, - '11': {'OS': 'Leading Edge MS-DOS / OS/2','Description': 'FAT12/FAT16'}, - '12': {'OS': 'Compaq Contura','Description': 'conf/diag/hiber/rescue/serv'}, - '14': {'OS': 'AST DOS / OS/2 / MaverickOS','Description': 'FAT12/FAT16/Omega'}, - '15': {'OS': 'OS/2 / Maverick OS','Description': 'Hidden extended / Swap'}, - '16': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT16B'}, - '17': {'OS': 'OS/2 Boot Manager','Description': 'Hidden IFS/HPFS/NTFS/exFAT'}, - '18': {'OS': 'AST Windows','Description': '0-Volt Suspend/SmartSleep'}, - '19': {'OS': 'Willowtech Photon coS','Description': 'Willowtech Photon coS'}, - '1B': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT32'}, - '1C': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT32 with LBA'}, - '1E': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT16 with LBA'}, - '1F': {'OS': 'OS/2 Boot Manager','Description': 'Hidden extended with LBA'}, - '20': {'OS': 'Windows Mobile','Description': 'update XIP/Willowsoft OFS1'}, - '21': {'OS': 'Oxygen','Description': 'SpeedStor / FSo2'}, - '22': {'OS': 'Oxygen','Description': 'Oxygen Extended Partition'}, - '23': {'OS': 'Windows Mobile','Description': 'Reserved / boot XIP'}, - '24': {'OS': 'NEC MS-DOS0','Description': 'Logical FAT12 or FAT16'}, - '25': {'OS': 'Windows Mobile','Description': 'IMGFS[citation needed]'}, - '26': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '27': {'OS': 'Win/PQserv/MirOS/RooterBOOT','Description': 'WinRE/Rescue/MirOS/RooterBOOT'}, - '2A': {'OS': 'AtheOS','Description': 'AthFS/AFS/Reserved'}, - '2B': {'OS': 'SyllableOS','Description': 'SyllableSecure (SylStor)'}, - '31': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '32': {'OS': 'NOS','Description': 'Unknown'}, - '33': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '34': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '35': {'OS': 'OS/2 Server /eComStation','Description': 'JFS'}, - '36': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '38': {'OS': 'THEOS','Description': 'THEOS version 3.2, 2 GB'}, - '39': {'OS': 'Plan 9 / THEOS','Description': 'Plan 9 edition 3 / THEOS v4'}, - '3A': {'OS': 'THEOS','Description': 'THEOS v4, 4 GB'}, - '3B': {'OS': 'THEOS','Description': 'THEOS v4 extended'}, - '3C': {'OS': 'PartitionMagic','Description': 'PqRP (image in progress)'}, - '3D': {'OS': 'PartitionMagic','Description': 'Hidden NetWare'}, - '3F': {'OS': 'OS/32','Description': 'Unknown'}, - '40': {'OS': 'PICK / Venix','Description': 'PICK R83 / Venix 80286'}, - '41': {'OS': 'RISC / Linux / PowerPC','Description': 'Boot / Old Linux/Minix'}, - '42': {'OS': 'SFS / Linux / Win2K/XP/etc','Description': 'SFS / Old Linux Swap'}, - '43': {'OS': 'Linux','Description': 'Old Linux native'}, - '44': {'OS': 'GoBack','Description': 'Norton/WildFire/Adaptec/Roxio'}, - '45': {'OS': 'Boot-US / EUMEL/ELAN','Description': 'Priam/Boot/EUMEL/ELAN (L2)'}, - '46': {'OS': 'EUMEL/ELAN','Description': 'EUMEL/ELAN (L2)'}, - '47': {'OS': 'EUMEL/ELAN','Description': 'EUMEL/ELAN (L2)'}, - '48': {'OS': 'EUMEL/ELAN','Description': 'EUMEL/ELAN (L2), ERGOS L3'}, - '4A': {'OS': 'AdaOS / ALFS/THIN','Description': 'Aquila / ALFS/THIN'}, - '4C': {'OS': 'ETH Oberon','Description': 'Aos (A2) file system (76)'}, - '4D': {'OS': 'QNX Neutrino','Description': 'Primary QNX POSIX volume'}, - '4E': {'OS': 'QNX Neutrino','Description': 'Secondary QNX POSIX volume'}, - '4F': {'OS': 'QNX Neutrino / ETH Oberon','Description': '3rd QNX POSIX/Boot/Native'}, - '50': {'OS': 'DiskMan4/ETH/LynxOS/Novell','Description': 'Alt FS/Read-only/Lynx RTOS'}, - '51': {'OS': 'Disk Manager 4-6','Description': 'R/W partition (Aux 1)'}, - '52': {'OS': 'CP/M-80/ System V/AT, V/386','Description': 'CP/M-80'}, - '53': {'OS': 'Disk Manager 6','Description': 'Auxiliary 3 (WO)'}, - '54': {'OS': 'Disk Manager 6','Description': 'Dynamic Drive Overlay (DDO)'}, - '55': {'OS': 'EZ-Drive','Description': 'Maxtor/MaxBlast/DriveGuide'}, - '56': {'OS': 'AT&T DOS/EZ-Drive/VFeature','Description': 'FAT12~16/EZ-BIOS/VFeature'}, - '57': {'OS': 'DrivePro','Description': 'VNDI partition'}, - '5C': {'OS': 'EDISK','Description': 'Priam EDisk Volume'}, - '61': {'OS': 'SpeedStor','Description': 'Unknown'}, - '63': {'OS': 'Unix','Description': 'Unix,ISC,SysV,ix,BSD,HURD'}, - '64': {'OS': 'SpeedStor / NetWare','Description': 'NetWare FS 286/2,PC-ARMOUR'}, - '65': {'OS': 'NetWare','Description': 'NetWare File System 386'}, - '66': {'OS': 'NetWare / NetWare','Description': 'NetWare FS 386 / SMS'}, - '67': {'OS': 'NetWare','Description': 'Wolf Mountain'}, - '68': {'OS': 'NetWare','Description': 'Unknown'}, - '69': {'OS': 'NetWare 5 / NetWare','Description': 'Novell Storage Services'}, - '6E': {'Description': 'Unknown'}, - '70': {'OS': 'DiskSecure','Description': 'DiskSecure multiboot'}, - '71': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '72': {'OS': 'APTI systems / Unix V7/x86','Description': 'APTI altFAT12 / V7 / x86'}, - '73': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '74': {'OS': 'Microsoft, IBM','Description': 'Reserved / Scramdisk'}, - '75': {'OS': 'PC/IX','Description': 'Unknown'}, - '76': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, - '77': {'OS': 'Novell','Description': 'VNDI, M2FS, M2CS'}, - '78': {'OS': 'Geurt Vos','Description': 'XOSL bootloader file system'}, - '79': {'OS': 'APTI conformant systems','Description': 'APTI altFAT16 (CHS, SFN)'}, - '7A': {'OS': 'APTI conformant systems','Description': 'APTI altFAT16 (LBA, SFN)'}, - '7B': {'OS': 'APTI conformant systems','Description': 'APTI altFAT16B (CHS, SFN)'}, - '7C': {'OS': 'APTI conformant systems','Description': 'APTI altFAT32 (LBA, SFN)'}, - '7D': {'OS': 'APTI conformant systems','Description': 'APTI altFAT32 (CHS, SFN)'}, - '7E': {'OS': 'F.I.X. (claim) / PrimoCache','Description': 'Level 2 cache'}, - '7F': {'OS': 'Varies','Description': 'AltOS DevPartition Standard'}, - '80': {'OS': 'Minix 1.1-1.4a','Description': 'Minix file system (old)'}, - '81': {'OS': 'Minix 1.4b+ / Linux','Description': 'MINIX FS/Mitac AdvDiskManager'}, - '82': {'OS': 'Linux / Sun Microsystems','Description': 'Swap / Solaris x86 / Prime'}, - '83': {'OS': 'GNU/Linux','Description': 'Any native Linux FS'}, - '84': {'OS': 'OS/2 / Windows 7','Description': 'Hibernat/HiddenC/RapidStart'}, - '85': {'OS': 'GNU/Linux','Description': 'Linux extended'}, - '86': {'OS': 'Windows NT 4 Server / Linux','Description': 'FAT16B mirror/LinuxRAID-old'}, - '87': {'OS': 'Windows NT 4 Server','Description': 'HPFS/NTFS mirrored volume'}, - '88': {'OS': 'GNU/Linux','Description': 'Plaintext partition table'}, - '8A': {'OS': 'AiR-BOOT','Description': 'Linux kernel image'}, - '8B': {'OS': 'Windows NT 4 Server','Description': 'FAT32 mirrored volume set'}, - '8C': {'OS': 'Windows NT 4 Server','Description': 'FAT32 mirrored volume set'}, - '8D': {'OS': 'Free FDISK','Description': 'Hidden FAT12'}, - '8E': {'OS': 'Linux','Description': 'Linux LVM'}, - '90': {'OS': 'Free FDISK','Description': 'Hidden FAT16'}, - '91': {'OS': 'Free FDISK','Description': 'Hidden extended partition'}, - '92': {'OS': 'Free FDISK','Description': 'Hidden FAT16B'}, - '93': {'OS': 'Amoeba / Linux','Description': 'Amoeba native/Hidden Linux'}, - '94': {'OS': 'Amoeba','Description': 'Amoeba bad block table'}, - '95': {'OS': 'EXOPC','Description': 'EXOPC native'}, - '96': {'OS': 'CHRP','Description': 'ISO-9660 file system'}, - '97': {'OS': 'Free FDISK','Description': 'Hidden FAT32'}, - '98': {'OS': 'Free FDISK / ROM-DOS','Description': 'Hidden FAT32 / service part'}, - '99': {'OS': 'early Unix','Description': 'Unknown'}, - '9A': {'OS': 'Free FDISK','Description': 'Hidden FAT16'}, - '9B': {'OS': 'Free FDISK','Description': 'Hidden extended partition'}, - '9E': {'OS': 'VSTA / ForthOS','Description': 'ForthOS (eForth port)'}, - '9F': {'OS': 'BSD/OS 3.0+, BSDI','Description': 'Unknown'}, - 'A0': {'OS': 'HP/Phoenix/IBM/Toshiba/Sony','Description': 'Diagnostic for HP/Hibernate'}, - 'A1': {'OS': 'HP / Phoenix, NEC','Description': 'HP Vol Expansion/Hibernate'}, - 'A2': {'OS': 'Cyclone V','Description': 'Hard Processor System (HPS)'}, - 'A3': {'OS': 'HP','Description': 'HP Vol Expansion(SpeedStor)'}, - 'A4': {'OS': 'HP','Description': 'HP Vol Expansion(SpeedStor)'}, - 'A5': {'OS': 'BSD','Description': 'BSD slice'}, - 'A6': {'OS': 'OpenBSD','Description': 'HP Vol Expansion/BSD slice'}, - 'A7': {'OS': 'NeXT','Description': 'NeXTSTEP'}, - 'A8': {'OS': 'Darwin, Mac OS X','Description': 'Apple Darwin, Mac OS X UFS'}, - 'A9': {'OS': 'NetBSD','Description': 'NetBSD slice'}, - 'AA': {'OS': 'MS-DOS','Description': 'Olivetti DOS FAT12(1.44 MB)'}, - 'AB': {'OS': 'Darwin, Mac OS X / GO! OS','Description': 'Apple Darwin/OS X boot/GO!'}, - 'AD': {'OS': 'RISC OS','Description': 'ADFS / FileCore format'}, - 'AE': {'OS': 'ShagOS','Description': 'ShagOS file system'}, - 'AF': {'OS': 'ShagOS','Description': 'OS X HFS & HFS+/ShagOS Swap'}, - 'B0': {'OS': 'Boot-Star','Description': 'Boot-Star dummy partition'}, - 'B1': {'OS': 'QNX 6.x','Description': 'HPVolExpansion/QNX Neutrino'}, - 'B2': {'OS': 'QNX 6.x','Description': 'QNX Neutrino power-safe FS'}, - 'B3': {'OS': 'QNX 6.x','Description': 'HPVolExpansion/QNX Neutrino'}, - 'B4': {'OS': 'HP','Description': 'HP Vol Expansion(SpeedStor)'}, - 'B6': {'OS': 'Windows NT 4 Server','Description': 'HPVolExpansion/FAT16Bmirror'}, - 'B7': {'OS': 'BSDI / Windows NT 4 Server','Description': 'BSDI,Swap,HPFS/NTFS mirror'}, - 'B8': {'OS': 'BSDI (before 3.0)','Description': 'BSDI Swap / native FS'}, - 'BB': {'OS': 'Acronis/BootWizard/WinNT 4','Description': 'BootWizard/OEM/FAT32 mirror'}, - 'BC': {'OS': 'Acronis/WinNT/BackupCapsule','Description': 'FAT32RAID/SecureZone/Backup'}, - 'BD': {'OS': 'BonnyDOS/286','Description': 'Unknown'}, - 'BE': {'OS': 'Solaris 8','Description': 'Solaris 8 boot'}, - 'BF': {'OS': 'Solaris','Description': 'Solaris x86'}, - 'C0': {'OS': 'DR-DOS,MultiuserDOS,REAL/32','Description': 'Secured FAT (under 32 MB)'}, - 'C1': {'OS': 'DR DOS','Description': 'Secured FAT12'}, - 'C2': {'OS': 'Power Boot','Description': 'Hidden Linux native FS'}, - 'C3': {'OS': 'Power Boot','Description': 'Hidden Linux Swap'}, - 'C4': {'OS': 'DR DOS','Description': 'Secured FAT16'}, - 'C5': {'OS': 'DR DOS','Description': 'Secured extended partition'}, - 'C6': {'OS': 'DR DOS / WinNT 4 Server','Description': 'Secured FAT16B/FAT16Bmirror'}, - 'C7': {'OS': 'Syrinx / WinNT 4 Server','Description': 'Syrinx boot/HPFS/NTFSmirror'}, - 'C8': {'Description': "DR-DOS Reserved (since '97)"}, - 'C9': {'Description': "DR-DOS Reserved (since '97)"}, - 'CA': {'Description': "DR-DOS Reserved (since '97)"}, - 'CB': {'OS': 'DR-DOSx / WinNT 4 Server','Description': 'Secured FAT32/FAT32 mirror'}, - 'CC': {'OS': 'DR-DOSx / WinNT 4 Server','Description': 'Secured FAT32/FAT32 mirror'}, - 'CD': {'OS': 'CTOS','Description': 'Memory dump'}, - 'CE': {'OS': 'DR-DOSx','Description': 'Secured FAT16B'}, - 'CF': {'OS': 'DR-DOSx','Description': 'Secured extended partition'}, - 'D0': {'OS': 'Multiuser DOS, REAL/32','Description': 'Secured FAT (over 32 MB)'}, - 'D1': {'OS': 'Multiuser DOS','Description': 'Secured FAT12'}, - 'D4': {'OS': 'Multiuser DOS','Description': 'Secured FAT16'}, - 'D5': {'OS': 'Multiuser DOS','Description': 'Secured extended partition'}, - 'D6': {'OS': 'Multiuser DOS','Description': 'Secured FAT16B'}, - 'D8': {'OS': 'Digital Research','Description': 'CP/M-86 [citation needed]'}, - 'DA': {'OS': 'Powercopy Backup','Description': 'Non-FS data / Shielded disk'}, - 'DB': {'OS': 'CP/M-86/CDOS/CTOS/D800/DRMK','Description': 'CP/M-86/ConcDOS/Boot/FAT32'}, - 'DD': {'OS': 'CTOS','Description': 'Hidden memory dump'}, - 'DE': {'OS': 'Dell','Description': 'FAT16 utility/diagnostic'}, - 'DF': {'OS': 'DG/UX / BootIt / Aviion','Description': 'DG/UX Virt DiskMan / EMBRM'}, - 'E0': {'OS': 'STMicroelectronics','Description': 'ST AVFS'}, - 'E1': {'OS': 'SpeedStor','Description': 'ExtendedFAT12 >1023cylinder'}, - 'E2': {'Description': 'DOS read-only (XFDISK)'}, - 'E3': {'OS': 'SpeedStor','Description': 'DOS read-only'}, - 'E4': {'OS': 'SpeedStor','Description': 'ExtendedFAT16 <1024cylinder'}, - 'E5': {'OS': 'Tandy MS-DOS','Description': 'Logical FAT12 or FAT16'}, - 'E6': {'OS': 'SpeedStor','Description': 'Unknown'}, - 'E8': {'OS': 'LUKS','Description': 'Linux Unified Key Setup'}, - 'EB': {'OS': 'BeOS, Haiku','Description': 'BFS'}, - 'EC': {'OS': 'SkyOS','Description': 'SkyFS'}, - 'ED': {'OS': 'Sprytix / EDD 4','Description': 'EDC loader / GPT hybrid MBR'}, - 'EE': {'OS': 'EFI','Description': 'GPT protective MBR'}, - 'EF': {'OS': 'EFI','Description': 'EFI system partition'}, - 'F0': {'OS': 'Linux / OS/32','Description': 'PA-RISC Linux boot loader.'}, - 'F1': {'OS': 'SpeedStor','Description': 'Unknown'}, - 'F2': {'OS': 'SperryIT DOS/Unisys DOS','Description': 'Logical FAT12/FAT16'}, - 'F3': {'OS': 'SpeedStor','Description': 'Unknown'}, - 'F4': {'OS': 'SpeedStor / Prologue','Description': '"large"DOS part/NGF/TwinFS'}, - 'F5': {'OS': 'Prologue','Description': 'MD0-MD9 part for NGF/TwinFS'}, - 'F6': {'OS': 'SpeedStor','Description': 'Unknown'}, - 'F7': {'OS': 'O.S.G. / X1','Description': 'EFAT / Solid State FS'}, - 'F9': {'OS': 'Linux','Description': 'pCache ext2/ext3 cache'}, - 'FA': {'OS': 'Bochs','Description': 'x86 emulator'}, - 'FB': {'OS': 'VMware','Description': 'VMware VMFS partition'}, - 'FC': {'OS': 'VMware','Description': 'Swap / VMKCORE kernel dump'}, - 'FD': {'OS': 'Linux / FreeDOS','Description': 'LinuxRAID/Reserved4FreeDOS'}, - 'FE': {'OS': 'SpeedStor/LANstep/NT/Linux','Description': 'PS/2/DiskAdmin/old LinuxLVM'}, - 'FF': {'OS': 'XENIX','Description': 'XENIX bad block table'}, - '00000000-0000-0000-0000-000000000000': {'Description': 'Unused entry'}, - '024DEE41-33E7-11D3-9D69-0008C781F39F': {'Description': 'MBR partition scheme'}, - 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B': {'Description': 'EFI System partition'}, - '21686148-6449-6E6F-744E-656564454649': {'Description': 'BIOS Boot partition'}, - 'D3BFE2DE-3DAF-11DF-BA40-E3A556D89593': {'Description': 'Intel Fast Flash (iFFS) partition (for Intel Rapid Start technology)'}, - 'F4019732-066E-4E12-8273-346C5641494F': {'Description': 'Sony boot partition'}, - 'BFBFAFE7-A34F-448A-9A5B-6213EB736C22': {'Description': 'Lenovo boot partition'}, - 'E3C9E316-0B5C-4DB8-817D-F92DF00215AE': {'OS': 'Windows', 'Description': 'Microsoft Reserved Partition (MSR)'}, - 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7': {'OS': 'Windows', 'Description': 'Basic data partition'}, - '5808C8AA-7E8F-42E0-85D2-E1E90434CFB3': {'OS': 'Windows', 'Description': 'Logical Disk Manager (LDM) metadata partition'}, - 'AF9B60A0-1431-4F62-BC68-3311714A69AD': {'OS': 'Windows', 'Description': 'Logical Disk Manager data partition'}, - 'DE94BBA4-06D1-4D40-A16A-BFD50179D6AC': {'OS': 'Windows', 'Description': 'Windows Recovery Environment'}, - '37AFFC90-EF7D-4E96-91C3-2D7AE055B174': {'OS': 'Windows', 'Description': 'IBM General Parallel File System (GPFS) partition'}, - 'E75CAF8F-F680-4CEE-AFA3-B001E56EFC2D': {'OS': 'Windows', 'Description': 'Storage Spaces partition'}, - '75894C1E-3AEB-11D3-B7C1-7B03A0000000': {'OS': 'HP-UX', 'Description': 'Data partition'}, - 'E2A1E728-32E3-11D6-A682-7B03A0000000': {'OS': 'HP-UX', 'Description': 'Service Partition'}, - '0FC63DAF-8483-4772-8E79-3D69D8477DE4': {'OS': 'Linux', 'Description': 'Linux filesystem data'}, - 'A19D880F-05FC-4D3B-A006-743F0F84911E': {'OS': 'Linux', 'Description': 'RAID partition'}, - '44479540-F297-41B2-9AF7-D131D5F0458A': {'OS': 'Linux', 'Description': 'Root partition (x86)'}, - '4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709': {'OS': 'Linux', 'Description': 'Root partition (x86-64)'}, - '69DAD710-2CE4-4E3C-B16C-21A1D49ABED3': {'OS': 'Linux', 'Description': 'Root partition (32-bit ARM)'}, - 'B921B045-1DF0-41C3-AF44-4C6F280D3FAE': {'OS': 'Linux', 'Description': 'Root partition (64-bit ARM)/AArch64)'}, - '0657FD6D-A4AB-43C4-84E5-0933C84B4F4F': {'OS': 'Linux', 'Description': 'Swap partition'}, - 'E6D6D379-F507-44C2-A23C-238F2A3DF928': {'OS': 'Linux', 'Description': 'Logical Volume Manager (LVM) partition'}, - '933AC7E1-2EB4-4F13-B844-0E14E2AEF915': {'OS': 'Linux', 'Description': '/home partition'}, - '3B8F8425-20E0-4F3B-907F-1A25A76F98E8': {'OS': 'Linux', 'Description': '/srv (server data) partition'}, - '7FFEC5C9-2D00-49B7-8941-3EA10A5586B7': {'OS': 'Linux', 'Description': 'Plain dm-crypt partition'}, - 'CA7D7CCB-63ED-4C53-861C-1742536059CC': {'OS': 'Linux', 'Description': 'LUKS partition'}, - '8DA63339-0007-60C0-C436-083AC8230908': {'OS': 'Linux', 'Description': 'Reserved'}, - '83BD6B9D-7F41-11DC-BE0B-001560B84F0F': {'OS': 'FreeBSD', 'Description': 'Boot partition'}, - '516E7CB4-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Data partition'}, - '516E7CB5-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Swap partition'}, - '516E7CB6-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Unix File System (UFS) partition'}, - '516E7CB8-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Vinum volume manager partition'}, - '516E7CBA-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'ZFS partition'}, - '48465300-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Hierarchical File System Plus (HFS+) partition'}, - '55465300-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple UFS'}, - '6A898CC3-1DD2-11B2-99A6-080020736631': {'OS': 'OS X Darwin', 'Description': 'ZFS'}, - '52414944-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple RAID partition'}, - '52414944-5F4F-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple RAID partition, offline'}, - '426F6F74-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple Boot partition (Recovery HD)'}, - '4C616265-6C00-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple Label'}, - '5265636F-7665-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple TV Recovery partition'}, - '53746F72-6167-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple Core Storage (i.e. Lion FileVault) partition'}, - '6A82CB45-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Boot partition'}, - '6A85CF4D-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Root partition'}, - '6A87C46F-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Swap partition'}, - '6A8B642B-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Backup partition'}, - '6A898CC3-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': '/usr partition'}, - '6A8EF2E9-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': '/var partition'}, - '6A90BA39-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': '/home partition'}, - '6A9283A5-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Alternate sector'}, - '6A945A3B-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Reserved partition'}, - '6A9630D1-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, - '6A980767-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, - '6A96237F-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, - '6A8D2AC7-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, - '49F48D32-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'Swap partition'}, - '49F48D5A-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'FFS partition'}, - '49F48D82-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'LFS partition'}, - '49F48DAA-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'RAID partition'}, - '2DB519C4-B10F-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'Concatenated partition'}, - '2DB519EC-B10F-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'Encrypted partition'}, - 'FE3A2A5D-4F32-41A7-B725-ACCC3285A309': {'OS': 'ChromeOS', 'Description': 'ChromeOS kernel'}, - '3CB8E202-3B7E-47DD-8A3C-7FF2A13CFCEC': {'OS': 'ChromeOS', 'Description': 'ChromeOS rootfs'}, - '2E0A753D-9E48-43B0-8337-B15192CB1B5E': {'OS': 'ChromeOS', 'Description': 'ChromeOS future use'}, - '42465331-3BA3-10F1-802A-4861696B7521': {'OS': 'Haiku', 'Description': 'Haiku BFS'}, - '85D5E45E-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Boot partition'}, - '85D5E45A-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Data partition'}, - '85D5E45B-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Swap partition'}, - '0394EF8B-237E-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Unix File System (UFS) partition'}, - '85D5E45C-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Vinum volume manager partition'}, - '85D5E45D-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'ZFS partition'}, - '45B0969E-9B03-4F30-B4C6-B4B80CEFF106': {'OS': 'Ceph', 'Description': 'Ceph Journal'}, - '45B0969E-9B03-4F30-B4C6-5EC00CEFF106': {'OS': 'Ceph', 'Description': 'Ceph dm-crypt Encrypted Journal'}, - '4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D': {'OS': 'Ceph', 'Description': 'Ceph OSD'}, - '4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D': {'OS': 'Ceph', 'Description': 'Ceph dm-crypt OSD'}, - '89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE': {'OS': 'Ceph', 'Description': 'Ceph disk in creation'}, - '89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE': {'OS': 'Ceph', 'Description': 'Ceph dm-crypt disk in creation'}, - '824CC7A0-36A8-11E3-890A-952519AD3F61': {'OS': 'OpenBSD', 'Description': 'Data partition'}, - 'CEF5A9AD-73BC-4601-89F3-CDEEEEE321A1': {'OS': 'QNX', 'Description': 'Power-safe (QNX6) file system'}, - 'C91818F9-8025-47AF-89D2-F030D7000C2C': {'OS': 'Plan 9', 'Description': 'Plan 9 partition'}, - '9D275380-40AD-11DB-BF97-000C2911D1B8': {'OS': 'VMware ESX', 'Description': 'vmkcore (coredump partition)'}, - 'AA31E02A-400F-11DB-9590-000C2911D1B8': {'OS': 'VMware ESX', 'Description': 'VMFS filesystem partition'}, - '9198EFFC-31C0-11DB-8F78-000C2911D1B8': {'OS': 'VMware ESX', 'Description': 'VMware Reserved'}, - '2568845D-2332-4675-BC39-8FA5A4748D15': {'OS': 'Android-IA', 'Description': 'Bootloader'}, - '114EAFFE-1552-4022-B26E-9B053604CF84': {'OS': 'Android-IA', 'Description': 'Bootloader2'}, - '49A4D17F-93A3-45C1-A0DE-F50B2EBE2599': {'OS': 'Android-IA', 'Description': 'Boot'}, - '4177C722-9E92-4AAB-8644-43502BFD5506': {'OS': 'Android-IA', 'Description': 'Recovery'}, - 'EF32A33B-A409-486C-9141-9FFB711F6266': {'OS': 'Android-IA', 'Description': 'Misc'}, - '20AC26BE-20B7-11E3-84C5-6CFDB94711E9': {'OS': 'Android-IA', 'Description': 'Metadata'}, - '38F428E6-D326-425D-9140-6E0EA133647C': {'OS': 'Android-IA', 'Description': 'System'}, - 'A893EF21-E428-470A-9E55-0668FD91A2D9': {'OS': 'Android-IA', 'Description': 'Cache'}, - 'DC76DDA9-5AC1-491C-AF42-A82591580C0D': {'OS': 'Android-IA', 'Description': 'Data'}, - 'EBC597D0-2053-4B15-8B64-E0AAC75F4DB1': {'OS': 'Android-IA', 'Description': 'Persistent'}, - '8F68CC74-C5E5-48DA-BE91-A0C8C15E9C80': {'OS': 'Android-IA', 'Description': 'Factory'}, - '767941D0-2085-11E3-AD3B-6CFDB94711E9': {'OS': 'Android-IA', 'Description': 'Fastboot / Tertiary'}, - 'AC6D7924-EB71-4DF8-B48D-E267B27148FF': {'OS': 'Android-IA', 'Description': 'OEM'}, - '7412F7D5-A156-4B13-81DC-867174929325': {'OS': 'ONIE', 'Description': 'Boot'}, - 'D4E6E2CD-4469-46F3-B5CB-1BFF57AFC149': {'OS': 'ONIE', 'Description': 'Config'}, - '9E1A2D38-C612-4316-AA26-8B49521E5A8B': {'OS': 'PowerPC', 'Description': 'PReP boot'}, - 'BC13C2FF-59E6-4262-A352-B275FD6F7172': {'OS': 'Freedesktop', 'Description': 'Extended Boot Partition ($BOOT)'}, -} - -def lookup_guid(guid): - return PARTITION_UIDS.get(guid.upper(), {}) - -if __name__ == '__main__': - print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/partition_uids.py b/.bin/Scripts/settings/partition_uids.py new file mode 100644 index 00000000..f4b21df0 --- /dev/null +++ b/.bin/Scripts/settings/partition_uids.py @@ -0,0 +1,325 @@ +# Wizard Kit: Settings - Partition UIDs +# sources: https://en.wikipedia.org/wiki/GUID_Partition_Table +# https://en.wikipedia.org/wiki/Partition_type +# NOTE: Info has been trimmed for brevity. As such, there may be some inaccuracy. + +PARTITION_UIDS = { + '00': {'OS': 'All','Description': 'Empty partition entry'}, + '01': {'OS': 'DOS','Description': 'FAT12 as primary partition'}, + '02': {'OS': 'XENIX','Description': 'XENIX root'}, + '03': {'OS': 'XENIX','Description': 'XENIX usr'}, + '04': {'OS': 'DOS','Description': 'FAT16 with less than 32 MB'}, + '05': {'OS': 'DOS / SpeedStor','Description': 'Extended partition'}, + '06': {'OS': 'DOS1+','Description': 'FAT16B [over 65K sectors]'}, + '07': {'OS': 'Windows / OS/2 / QNX 2','Description': 'NTFS/exFAT/HPFS/IFS/QNX'}, + '08': {'OS': 'CBM / DOS / OS/2 / AIX /QNX','Description': 'FAT12-16/AIX/QNY/SplitDrive'}, + '09': {'OS': 'AIX / QNX / Coherent / OS-9','Description': 'AIX/QNZ/Coherent/RBF'}, + '0A': {'OS': 'OS/2 / Coherent','Description': 'Boot Manager / Swap'}, + '0B': {'OS': 'DOS','Description': 'FAT32 with CHS addressing'}, + '0C': {'OS': 'DOS','Description': 'FAT32 with LBA'}, + '0D': {'OS': 'Silicon Safe','Description': 'Reserved'}, + '0E': {'OS': 'DOS','Description': 'FAT16B with LBA'}, + '0F': {'OS': 'DOS','Description': 'Extended partition with LBA'}, + '10': {'OS': 'OPUS','Description': 'Unknown'}, + '11': {'OS': 'Leading Edge MS-DOS / OS/2','Description': 'FAT12/FAT16'}, + '12': {'OS': 'Compaq Contura','Description': 'conf/diag/hiber/rescue/serv'}, + '14': {'OS': 'AST DOS / OS/2 / MaverickOS','Description': 'FAT12/FAT16/Omega'}, + '15': {'OS': 'OS/2 / Maverick OS','Description': 'Hidden extended / Swap'}, + '16': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT16B'}, + '17': {'OS': 'OS/2 Boot Manager','Description': 'Hidden IFS/HPFS/NTFS/exFAT'}, + '18': {'OS': 'AST Windows','Description': '0-Volt Suspend/SmartSleep'}, + '19': {'OS': 'Willowtech Photon coS','Description': 'Willowtech Photon coS'}, + '1B': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT32'}, + '1C': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT32 with LBA'}, + '1E': {'OS': 'OS/2 Boot Manager','Description': 'Hidden FAT16 with LBA'}, + '1F': {'OS': 'OS/2 Boot Manager','Description': 'Hidden extended with LBA'}, + '20': {'OS': 'Windows Mobile','Description': 'update XIP/Willowsoft OFS1'}, + '21': {'OS': 'Oxygen','Description': 'SpeedStor / FSo2'}, + '22': {'OS': 'Oxygen','Description': 'Oxygen Extended Partition'}, + '23': {'OS': 'Windows Mobile','Description': 'Reserved / boot XIP'}, + '24': {'OS': 'NEC MS-DOS0','Description': 'Logical FAT12 or FAT16'}, + '25': {'OS': 'Windows Mobile','Description': 'IMGFS[citation needed]'}, + '26': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '27': {'OS': 'Win/PQserv/MirOS/RooterBOOT','Description': 'WinRE/Rescue/MirOS/RooterBOOT'}, + '2A': {'OS': 'AtheOS','Description': 'AthFS/AFS/Reserved'}, + '2B': {'OS': 'SyllableOS','Description': 'SyllableSecure (SylStor)'}, + '31': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '32': {'OS': 'NOS','Description': 'Unknown'}, + '33': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '34': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '35': {'OS': 'OS/2 Server /eComStation','Description': 'JFS'}, + '36': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '38': {'OS': 'THEOS','Description': 'THEOS version 3.2, 2 GB'}, + '39': {'OS': 'Plan 9 / THEOS','Description': 'Plan 9 edition 3 / THEOS v4'}, + '3A': {'OS': 'THEOS','Description': 'THEOS v4, 4 GB'}, + '3B': {'OS': 'THEOS','Description': 'THEOS v4 extended'}, + '3C': {'OS': 'PartitionMagic','Description': 'PqRP (image in progress)'}, + '3D': {'OS': 'PartitionMagic','Description': 'Hidden NetWare'}, + '3F': {'OS': 'OS/32','Description': 'Unknown'}, + '40': {'OS': 'PICK / Venix','Description': 'PICK R83 / Venix 80286'}, + '41': {'OS': 'RISC / Linux / PowerPC','Description': 'Boot / Old Linux/Minix'}, + '42': {'OS': 'SFS / Linux / Win2K/XP/etc','Description': 'SFS / Old Linux Swap'}, + '43': {'OS': 'Linux','Description': 'Old Linux native'}, + '44': {'OS': 'GoBack','Description': 'Norton/WildFire/Adaptec/Roxio'}, + '45': {'OS': 'Boot-US / EUMEL/ELAN','Description': 'Priam/Boot/EUMEL/ELAN (L2)'}, + '46': {'OS': 'EUMEL/ELAN','Description': 'EUMEL/ELAN (L2)'}, + '47': {'OS': 'EUMEL/ELAN','Description': 'EUMEL/ELAN (L2)'}, + '48': {'OS': 'EUMEL/ELAN','Description': 'EUMEL/ELAN (L2), ERGOS L3'}, + '4A': {'OS': 'AdaOS / ALFS/THIN','Description': 'Aquila / ALFS/THIN'}, + '4C': {'OS': 'ETH Oberon','Description': 'Aos (A2) file system (76)'}, + '4D': {'OS': 'QNX Neutrino','Description': 'Primary QNX POSIX volume'}, + '4E': {'OS': 'QNX Neutrino','Description': 'Secondary QNX POSIX volume'}, + '4F': {'OS': 'QNX Neutrino / ETH Oberon','Description': '3rd QNX POSIX/Boot/Native'}, + '50': {'OS': 'DiskMan4/ETH/LynxOS/Novell','Description': 'Alt FS/Read-only/Lynx RTOS'}, + '51': {'OS': 'Disk Manager 4-6','Description': 'R/W partition (Aux 1)'}, + '52': {'OS': 'CP/M-80/ System V/AT, V/386','Description': 'CP/M-80'}, + '53': {'OS': 'Disk Manager 6','Description': 'Auxiliary 3 (WO)'}, + '54': {'OS': 'Disk Manager 6','Description': 'Dynamic Drive Overlay (DDO)'}, + '55': {'OS': 'EZ-Drive','Description': 'Maxtor/MaxBlast/DriveGuide'}, + '56': {'OS': 'AT&T DOS/EZ-Drive/VFeature','Description': 'FAT12 16/EZ-BIOS/VFeature'}, + '57': {'OS': 'DrivePro','Description': 'VNDI partition'}, + '5C': {'OS': 'EDISK','Description': 'Priam EDisk Volume'}, + '61': {'OS': 'SpeedStor','Description': 'Unknown'}, + '63': {'OS': 'Unix','Description': 'Unix,ISC,SysV,ix,BSD,HURD'}, + '64': {'OS': 'SpeedStor / NetWare','Description': 'NetWare FS 286/2,PC-ARMOUR'}, + '65': {'OS': 'NetWare','Description': 'NetWare File System 386'}, + '66': {'OS': 'NetWare / NetWare','Description': 'NetWare FS 386 / SMS'}, + '67': {'OS': 'NetWare','Description': 'Wolf Mountain'}, + '68': {'OS': 'NetWare','Description': 'Unknown'}, + '69': {'OS': 'NetWare 5 / NetWare','Description': 'Novell Storage Services'}, + '6E': {'Description': 'Unknown'}, + '70': {'OS': 'DiskSecure','Description': 'DiskSecure multiboot'}, + '71': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '72': {'OS': 'APTI systems / Unix V7/x86','Description': 'APTI altFAT12 / V7 / x86'}, + '73': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '74': {'OS': 'Microsoft, IBM','Description': 'Reserved / Scramdisk'}, + '75': {'OS': 'PC/IX','Description': 'Unknown'}, + '76': {'OS': 'Microsoft, IBM','Description': 'Reserved'}, + '77': {'OS': 'Novell','Description': 'VNDI, M2FS, M2CS'}, + '78': {'OS': 'Geurt Vos','Description': 'XOSL bootloader file system'}, + '79': {'OS': 'APTI conformant systems','Description': 'APTI altFAT16 (CHS, SFN)'}, + '7A': {'OS': 'APTI conformant systems','Description': 'APTI altFAT16 (LBA, SFN)'}, + '7B': {'OS': 'APTI conformant systems','Description': 'APTI altFAT16B (CHS, SFN)'}, + '7C': {'OS': 'APTI conformant systems','Description': 'APTI altFAT32 (LBA, SFN)'}, + '7D': {'OS': 'APTI conformant systems','Description': 'APTI altFAT32 (CHS, SFN)'}, + '7E': {'OS': 'F.I.X. (claim) / PrimoCache','Description': 'Level 2 cache'}, + '7F': {'OS': 'Varies','Description': 'AltOS DevPartition Standard'}, + '80': {'OS': 'Minix 1.1-1.4a','Description': 'Minix file system (old)'}, + '81': {'OS': 'Minix 1.4b+ / Linux','Description': 'MINIX FS/Mitac AdvDiskManager'}, + '82': {'OS': 'Linux / Sun Microsystems','Description': 'Swap / Solaris x86 / Prime'}, + '83': {'OS': 'GNU/Linux','Description': 'Any native Linux FS'}, + '84': {'OS': 'OS/2 / Windows 7','Description': 'Hibernat/HiddenC/RapidStart'}, + '85': {'OS': 'GNU/Linux','Description': 'Linux extended'}, + '86': {'OS': 'Windows NT 4 Server / Linux','Description': 'FAT16B mirror/LinuxRAID-old'}, + '87': {'OS': 'Windows NT 4 Server','Description': 'HPFS/NTFS mirrored volume'}, + '88': {'OS': 'GNU/Linux','Description': 'Plaintext partition table'}, + '8A': {'OS': 'AiR-BOOT','Description': 'Linux kernel image'}, + '8B': {'OS': 'Windows NT 4 Server','Description': 'FAT32 mirrored volume set'}, + '8C': {'OS': 'Windows NT 4 Server','Description': 'FAT32 mirrored volume set'}, + '8D': {'OS': 'Free FDISK','Description': 'Hidden FAT12'}, + '8E': {'OS': 'Linux','Description': 'Linux LVM'}, + '90': {'OS': 'Free FDISK','Description': 'Hidden FAT16'}, + '91': {'OS': 'Free FDISK','Description': 'Hidden extended partition'}, + '92': {'OS': 'Free FDISK','Description': 'Hidden FAT16B'}, + '93': {'OS': 'Amoeba / Linux','Description': 'Amoeba native/Hidden Linux'}, + '94': {'OS': 'Amoeba','Description': 'Amoeba bad block table'}, + '95': {'OS': 'EXOPC','Description': 'EXOPC native'}, + '96': {'OS': 'CHRP','Description': 'ISO-9660 file system'}, + '97': {'OS': 'Free FDISK','Description': 'Hidden FAT32'}, + '98': {'OS': 'Free FDISK / ROM-DOS','Description': 'Hidden FAT32 / service part'}, + '99': {'OS': 'early Unix','Description': 'Unknown'}, + '9A': {'OS': 'Free FDISK','Description': 'Hidden FAT16'}, + '9B': {'OS': 'Free FDISK','Description': 'Hidden extended partition'}, + '9E': {'OS': 'VSTA / ForthOS','Description': 'ForthOS (eForth port)'}, + '9F': {'OS': 'BSD/OS 3.0+, BSDI','Description': 'Unknown'}, + 'A0': {'OS': 'HP/Phoenix/IBM/Toshiba/Sony','Description': 'Diagnostic for HP/Hibernate'}, + 'A1': {'OS': 'HP / Phoenix, NEC','Description': 'HP Vol Expansion/Hibernate'}, + 'A2': {'OS': 'Cyclone V','Description': 'Hard Processor System (HPS)'}, + 'A3': {'OS': 'HP','Description': 'HP Vol Expansion(SpeedStor)'}, + 'A4': {'OS': 'HP','Description': 'HP Vol Expansion(SpeedStor)'}, + 'A5': {'OS': 'BSD','Description': 'BSD slice'}, + 'A6': {'OS': 'OpenBSD','Description': 'HP Vol Expansion/BSD slice'}, + 'A7': {'OS': 'NeXT','Description': 'NeXTSTEP'}, + 'A8': {'OS': 'Darwin, Mac OS X','Description': 'Apple Darwin, Mac OS X UFS'}, + 'A9': {'OS': 'NetBSD','Description': 'NetBSD slice'}, + 'AA': {'OS': 'MS-DOS','Description': 'Olivetti DOS FAT12(1.44 MB)'}, + 'AB': {'OS': 'Darwin, Mac OS X / GO! OS','Description': 'Apple Darwin/OS X boot/GO!'}, + 'AD': {'OS': 'RISC OS','Description': 'ADFS / FileCore format'}, + 'AE': {'OS': 'ShagOS','Description': 'ShagOS file system'}, + 'AF': {'OS': 'ShagOS','Description': 'OS X HFS & HFS+/ShagOS Swap'}, + 'B0': {'OS': 'Boot-Star','Description': 'Boot-Star dummy partition'}, + 'B1': {'OS': 'QNX 6.x','Description': 'HPVolExpansion/QNX Neutrino'}, + 'B2': {'OS': 'QNX 6.x','Description': 'QNX Neutrino power-safe FS'}, + 'B3': {'OS': 'QNX 6.x','Description': 'HPVolExpansion/QNX Neutrino'}, + 'B4': {'OS': 'HP','Description': 'HP Vol Expansion(SpeedStor)'}, + 'B6': {'OS': 'Windows NT 4 Server','Description': 'HPVolExpansion/FAT16Bmirror'}, + 'B7': {'OS': 'BSDI / Windows NT 4 Server','Description': 'BSDI,Swap,HPFS/NTFS mirror'}, + 'B8': {'OS': 'BSDI (before 3.0)','Description': 'BSDI Swap / native FS'}, + 'BB': {'OS': 'Acronis/BootWizard/WinNT 4','Description': 'BootWizard/OEM/FAT32 mirror'}, + 'BC': {'OS': 'Acronis/WinNT/BackupCapsule','Description': 'FAT32RAID/SecureZone/Backup'}, + 'BD': {'OS': 'BonnyDOS/286','Description': 'Unknown'}, + 'BE': {'OS': 'Solaris 8','Description': 'Solaris 8 boot'}, + 'BF': {'OS': 'Solaris','Description': 'Solaris x86'}, + 'C0': {'OS': 'DR-DOS,MultiuserDOS,REAL/32','Description': 'Secured FAT (under 32 MB)'}, + 'C1': {'OS': 'DR DOS','Description': 'Secured FAT12'}, + 'C2': {'OS': 'Power Boot','Description': 'Hidden Linux native FS'}, + 'C3': {'OS': 'Power Boot','Description': 'Hidden Linux Swap'}, + 'C4': {'OS': 'DR DOS','Description': 'Secured FAT16'}, + 'C5': {'OS': 'DR DOS','Description': 'Secured extended partition'}, + 'C6': {'OS': 'DR DOS / WinNT 4 Server','Description': 'Secured FAT16B/FAT16Bmirror'}, + 'C7': {'OS': 'Syrinx / WinNT 4 Server','Description': 'Syrinx boot/HPFS/NTFSmirror'}, + 'C8': {'Description': "DR-DOS Reserved (since '97)"}, + 'C9': {'Description': "DR-DOS Reserved (since '97)"}, + 'CA': {'Description': "DR-DOS Reserved (since '97)"}, + 'CB': {'OS': 'DR-DOSx / WinNT 4 Server','Description': 'Secured FAT32/FAT32 mirror'}, + 'CC': {'OS': 'DR-DOSx / WinNT 4 Server','Description': 'Secured FAT32/FAT32 mirror'}, + 'CD': {'OS': 'CTOS','Description': 'Memory dump'}, + 'CE': {'OS': 'DR-DOSx','Description': 'Secured FAT16B'}, + 'CF': {'OS': 'DR-DOSx','Description': 'Secured extended partition'}, + 'D0': {'OS': 'Multiuser DOS, REAL/32','Description': 'Secured FAT (over 32 MB)'}, + 'D1': {'OS': 'Multiuser DOS','Description': 'Secured FAT12'}, + 'D4': {'OS': 'Multiuser DOS','Description': 'Secured FAT16'}, + 'D5': {'OS': 'Multiuser DOS','Description': 'Secured extended partition'}, + 'D6': {'OS': 'Multiuser DOS','Description': 'Secured FAT16B'}, + 'D8': {'OS': 'Digital Research','Description': 'CP/M-86 [citation needed]'}, + 'DA': {'OS': 'Powercopy Backup','Description': 'Non-FS data / Shielded disk'}, + 'DB': {'OS': 'CP/M-86/CDOS/CTOS/D800/DRMK','Description': 'CP/M-86/ConcDOS/Boot/FAT32'}, + 'DD': {'OS': 'CTOS','Description': 'Hidden memory dump'}, + 'DE': {'OS': 'Dell','Description': 'FAT16 utility/diagnostic'}, + 'DF': {'OS': 'DG/UX / BootIt / Aviion','Description': 'DG/UX Virt DiskMan / EMBRM'}, + 'E0': {'OS': 'STMicroelectronics','Description': 'ST AVFS'}, + 'E1': {'OS': 'SpeedStor','Description': 'ExtendedFAT12 >1023cylinder'}, + 'E2': {'Description': 'DOS read-only (XFDISK)'}, + 'E3': {'OS': 'SpeedStor','Description': 'DOS read-only'}, + 'E4': {'OS': 'SpeedStor','Description': 'ExtendedFAT16 <1024cylinder'}, + 'E5': {'OS': 'Tandy MS-DOS','Description': 'Logical FAT12 or FAT16'}, + 'E6': {'OS': 'SpeedStor','Description': 'Unknown'}, + 'E8': {'OS': 'LUKS','Description': 'Linux Unified Key Setup'}, + 'EB': {'OS': 'BeOS, Haiku','Description': 'BFS'}, + 'EC': {'OS': 'SkyOS','Description': 'SkyFS'}, + 'ED': {'OS': 'Sprytix / EDD 4','Description': 'EDC loader / GPT hybrid MBR'}, + 'EE': {'OS': 'EFI','Description': 'GPT protective MBR'}, + 'EF': {'OS': 'EFI','Description': 'EFI system partition'}, + 'F0': {'OS': 'Linux / OS/32','Description': 'PA-RISC Linux boot loader.'}, + 'F1': {'OS': 'SpeedStor','Description': 'Unknown'}, + 'F2': {'OS': 'SperryIT DOS/Unisys DOS','Description': 'Logical FAT12/FAT16'}, + 'F3': {'OS': 'SpeedStor','Description': 'Unknown'}, + 'F4': {'OS': 'SpeedStor / Prologue','Description': '"large"DOS part/NGF/TwinFS'}, + 'F5': {'OS': 'Prologue','Description': 'MD0-MD9 part for NGF/TwinFS'}, + 'F6': {'OS': 'SpeedStor','Description': 'Unknown'}, + 'F7': {'OS': 'O.S.G. / X1','Description': 'EFAT / Solid State FS'}, + 'F9': {'OS': 'Linux','Description': 'pCache ext2/ext3 cache'}, + 'FA': {'OS': 'Bochs','Description': 'x86 emulator'}, + 'FB': {'OS': 'VMware','Description': 'VMware VMFS partition'}, + 'FC': {'OS': 'VMware','Description': 'Swap / VMKCORE kernel dump'}, + 'FD': {'OS': 'Linux / FreeDOS','Description': 'LinuxRAID/Reserved4FreeDOS'}, + 'FE': {'OS': 'SpeedStor/LANstep/NT/Linux','Description': 'PS/2/DiskAdmin/old LinuxLVM'}, + 'FF': {'OS': 'XENIX','Description': 'XENIX bad block table'}, + '00000000-0000-0000-0000-000000000000': {'Description': 'Unused entry'}, + '024DEE41-33E7-11D3-9D69-0008C781F39F': {'Description': 'MBR partition scheme'}, + 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B': {'Description': 'EFI System partition'}, + '21686148-6449-6E6F-744E-656564454649': {'Description': 'BIOS Boot partition'}, + 'D3BFE2DE-3DAF-11DF-BA40-E3A556D89593': {'Description': 'Intel Fast Flash (iFFS) partition (for Intel Rapid Start technology)'}, + 'F4019732-066E-4E12-8273-346C5641494F': {'Description': 'Sony boot partition'}, + 'BFBFAFE7-A34F-448A-9A5B-6213EB736C22': {'Description': 'Lenovo boot partition'}, + 'E3C9E316-0B5C-4DB8-817D-F92DF00215AE': {'OS': 'Windows', 'Description': 'Microsoft Reserved Partition (MSR)'}, + 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7': {'OS': 'Windows', 'Description': 'Basic data partition'}, + '5808C8AA-7E8F-42E0-85D2-E1E90434CFB3': {'OS': 'Windows', 'Description': 'Logical Disk Manager (LDM) metadata partition'}, + 'AF9B60A0-1431-4F62-BC68-3311714A69AD': {'OS': 'Windows', 'Description': 'Logical Disk Manager data partition'}, + 'DE94BBA4-06D1-4D40-A16A-BFD50179D6AC': {'OS': 'Windows', 'Description': 'Windows Recovery Environment'}, + '37AFFC90-EF7D-4E96-91C3-2D7AE055B174': {'OS': 'Windows', 'Description': 'IBM General Parallel File System (GPFS) partition'}, + 'E75CAF8F-F680-4CEE-AFA3-B001E56EFC2D': {'OS': 'Windows', 'Description': 'Storage Spaces partition'}, + '75894C1E-3AEB-11D3-B7C1-7B03A0000000': {'OS': 'HP-UX', 'Description': 'Data partition'}, + 'E2A1E728-32E3-11D6-A682-7B03A0000000': {'OS': 'HP-UX', 'Description': 'Service Partition'}, + '0FC63DAF-8483-4772-8E79-3D69D8477DE4': {'OS': 'Linux', 'Description': 'Linux filesystem data'}, + 'A19D880F-05FC-4D3B-A006-743F0F84911E': {'OS': 'Linux', 'Description': 'RAID partition'}, + '44479540-F297-41B2-9AF7-D131D5F0458A': {'OS': 'Linux', 'Description': 'Root partition (x86)'}, + '4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709': {'OS': 'Linux', 'Description': 'Root partition (x86-64)'}, + '69DAD710-2CE4-4E3C-B16C-21A1D49ABED3': {'OS': 'Linux', 'Description': 'Root partition (32-bit ARM)'}, + 'B921B045-1DF0-41C3-AF44-4C6F280D3FAE': {'OS': 'Linux', 'Description': 'Root partition (64-bit ARM)/AArch64)'}, + '0657FD6D-A4AB-43C4-84E5-0933C84B4F4F': {'OS': 'Linux', 'Description': 'Swap partition'}, + 'E6D6D379-F507-44C2-A23C-238F2A3DF928': {'OS': 'Linux', 'Description': 'Logical Volume Manager (LVM) partition'}, + '933AC7E1-2EB4-4F13-B844-0E14E2AEF915': {'OS': 'Linux', 'Description': '/home partition'}, + '3B8F8425-20E0-4F3B-907F-1A25A76F98E8': {'OS': 'Linux', 'Description': '/srv (server data) partition'}, + '7FFEC5C9-2D00-49B7-8941-3EA10A5586B7': {'OS': 'Linux', 'Description': 'Plain dm-crypt partition'}, + 'CA7D7CCB-63ED-4C53-861C-1742536059CC': {'OS': 'Linux', 'Description': 'LUKS partition'}, + '8DA63339-0007-60C0-C436-083AC8230908': {'OS': 'Linux', 'Description': 'Reserved'}, + '83BD6B9D-7F41-11DC-BE0B-001560B84F0F': {'OS': 'FreeBSD', 'Description': 'Boot partition'}, + '516E7CB4-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Data partition'}, + '516E7CB5-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Swap partition'}, + '516E7CB6-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Unix File System (UFS) partition'}, + '516E7CB8-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'Vinum volume manager partition'}, + '516E7CBA-6ECF-11D6-8FF8-00022D09712B': {'OS': 'FreeBSD', 'Description': 'ZFS partition'}, + '48465300-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Hierarchical File System Plus (HFS+) partition'}, + '55465300-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple UFS'}, + '6A898CC3-1DD2-11B2-99A6-080020736631': {'OS': 'OS X Darwin', 'Description': 'ZFS'}, + '52414944-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple RAID partition'}, + '52414944-5F4F-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple RAID partition, offline'}, + '426F6F74-0000-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple Boot partition (Recovery HD)'}, + '4C616265-6C00-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple Label'}, + '5265636F-7665-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple TV Recovery partition'}, + '53746F72-6167-11AA-AA11-00306543ECAC': {'OS': 'OS X Darwin', 'Description': 'Apple Core Storage (i.e. Lion FileVault) partition'}, + '6A82CB45-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Boot partition'}, + '6A85CF4D-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Root partition'}, + '6A87C46F-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Swap partition'}, + '6A8B642B-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Backup partition'}, + '6A898CC3-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': '/usr partition'}, + '6A8EF2E9-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': '/var partition'}, + '6A90BA39-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': '/home partition'}, + '6A9283A5-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Alternate sector'}, + '6A945A3B-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos', 'Description': 'Reserved partition'}, + '6A9630D1-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, + '6A980767-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, + '6A96237F-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, + '6A8D2AC7-1DD2-11B2-99A6-080020736631': {'OS': 'Solaris illumos'}, + '49F48D32-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'Swap partition'}, + '49F48D5A-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'FFS partition'}, + '49F48D82-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'LFS partition'}, + '49F48DAA-B10E-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'RAID partition'}, + '2DB519C4-B10F-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'Concatenated partition'}, + '2DB519EC-B10F-11DC-B99B-0019D1879648': {'OS': 'NetBSD', 'Description': 'Encrypted partition'}, + 'FE3A2A5D-4F32-41A7-B725-ACCC3285A309': {'OS': 'ChromeOS', 'Description': 'ChromeOS kernel'}, + '3CB8E202-3B7E-47DD-8A3C-7FF2A13CFCEC': {'OS': 'ChromeOS', 'Description': 'ChromeOS rootfs'}, + '2E0A753D-9E48-43B0-8337-B15192CB1B5E': {'OS': 'ChromeOS', 'Description': 'ChromeOS future use'}, + '42465331-3BA3-10F1-802A-4861696B7521': {'OS': 'Haiku', 'Description': 'Haiku BFS'}, + '85D5E45E-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Boot partition'}, + '85D5E45A-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Data partition'}, + '85D5E45B-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Swap partition'}, + '0394EF8B-237E-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Unix File System (UFS) partition'}, + '85D5E45C-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'Vinum volume manager partition'}, + '85D5E45D-237C-11E1-B4B3-E89A8F7FC3A7': {'OS': 'MidnightBSD', 'Description': 'ZFS partition'}, + '45B0969E-9B03-4F30-B4C6-B4B80CEFF106': {'OS': 'Ceph', 'Description': 'Ceph Journal'}, + '45B0969E-9B03-4F30-B4C6-5EC00CEFF106': {'OS': 'Ceph', 'Description': 'Ceph dm-crypt Encrypted Journal'}, + '4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D': {'OS': 'Ceph', 'Description': 'Ceph OSD'}, + '4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D': {'OS': 'Ceph', 'Description': 'Ceph dm-crypt OSD'}, + '89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE': {'OS': 'Ceph', 'Description': 'Ceph disk in creation'}, + '89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE': {'OS': 'Ceph', 'Description': 'Ceph dm-crypt disk in creation'}, + '824CC7A0-36A8-11E3-890A-952519AD3F61': {'OS': 'OpenBSD', 'Description': 'Data partition'}, + 'CEF5A9AD-73BC-4601-89F3-CDEEEEE321A1': {'OS': 'QNX', 'Description': 'Power-safe (QNX6) file system'}, + 'C91818F9-8025-47AF-89D2-F030D7000C2C': {'OS': 'Plan 9', 'Description': 'Plan 9 partition'}, + '9D275380-40AD-11DB-BF97-000C2911D1B8': {'OS': 'VMware ESX', 'Description': 'vmkcore (coredump partition)'}, + 'AA31E02A-400F-11DB-9590-000C2911D1B8': {'OS': 'VMware ESX', 'Description': 'VMFS filesystem partition'}, + '9198EFFC-31C0-11DB-8F78-000C2911D1B8': {'OS': 'VMware ESX', 'Description': 'VMware Reserved'}, + '2568845D-2332-4675-BC39-8FA5A4748D15': {'OS': 'Android-IA', 'Description': 'Bootloader'}, + '114EAFFE-1552-4022-B26E-9B053604CF84': {'OS': 'Android-IA', 'Description': 'Bootloader2'}, + '49A4D17F-93A3-45C1-A0DE-F50B2EBE2599': {'OS': 'Android-IA', 'Description': 'Boot'}, + '4177C722-9E92-4AAB-8644-43502BFD5506': {'OS': 'Android-IA', 'Description': 'Recovery'}, + 'EF32A33B-A409-486C-9141-9FFB711F6266': {'OS': 'Android-IA', 'Description': 'Misc'}, + '20AC26BE-20B7-11E3-84C5-6CFDB94711E9': {'OS': 'Android-IA', 'Description': 'Metadata'}, + '38F428E6-D326-425D-9140-6E0EA133647C': {'OS': 'Android-IA', 'Description': 'System'}, + 'A893EF21-E428-470A-9E55-0668FD91A2D9': {'OS': 'Android-IA', 'Description': 'Cache'}, + 'DC76DDA9-5AC1-491C-AF42-A82591580C0D': {'OS': 'Android-IA', 'Description': 'Data'}, + 'EBC597D0-2053-4B15-8B64-E0AAC75F4DB1': {'OS': 'Android-IA', 'Description': 'Persistent'}, + '8F68CC74-C5E5-48DA-BE91-A0C8C15E9C80': {'OS': 'Android-IA', 'Description': 'Factory'}, + '767941D0-2085-11E3-AD3B-6CFDB94711E9': {'OS': 'Android-IA', 'Description': 'Fastboot / Tertiary'}, + 'AC6D7924-EB71-4DF8-B48D-E267B27148FF': {'OS': 'Android-IA', 'Description': 'OEM'}, + '7412F7D5-A156-4B13-81DC-867174929325': {'OS': 'ONIE', 'Description': 'Boot'}, + 'D4E6E2CD-4469-46F3-B5CB-1BFF57AFC149': {'OS': 'ONIE', 'Description': 'Config'}, + '9E1A2D38-C612-4316-AA26-8B49521E5A8B': {'OS': 'PowerPC', 'Description': 'PReP boot'}, + 'BC13C2FF-59E6-4262-A352-B275FD6F7172': {'OS': 'Freedesktop', 'Description': 'Extended Boot Partition ($BOOT)'}, +} + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 72eac47524f33fbc3b1b6a7aff0d36fc4ce31901 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:05:53 -0700 Subject: [PATCH 124/186] Updated disk.py --- .bin/Scripts/functions/disk.py | 636 +++++++++++++++++---------------- 1 file changed, 319 insertions(+), 317 deletions(-) diff --git a/.bin/Scripts/functions/disk.py b/.bin/Scripts/functions/disk.py index 257f21b0..88b26155 100644 --- a/.bin/Scripts/functions/disk.py +++ b/.bin/Scripts/functions/disk.py @@ -6,390 +6,392 @@ from settings.partition_uids import * # Regex REGEX_BAD_PARTITION = re.compile(r'(RAW|Unknown)', re.IGNORECASE) REGEX_DISK_GPT = re.compile( - r'Disk ID: {[A-Z0-9]+-[A-Z0-9]+-[A-Z0-9]+-[A-Z0-9]+-[A-Z0-9]+}', - re.IGNORECASE) + r'Disk ID: {[A-Z0-9]+-[A-Z0-9]+-[A-Z0-9]+-[A-Z0-9]+-[A-Z0-9]+}', + re.IGNORECASE) REGEX_DISK_MBR = re.compile(r'Disk ID: [A-Z0-9]+', re.IGNORECASE) REGEX_DISK_RAW = re.compile(r'Disk ID: 00000000', re.IGNORECASE) def assign_volume_letters(): - """Assign a volume letter to all available volumes.""" - remove_volume_letters() + """Assign a volume letter to all available volumes.""" + remove_volume_letters() - # Write script - script = [] - for vol in get_volumes(): - script.append('select volume {}'.format(vol['Number'])) - script.append('assign') + # Write script + script = [] + for vol in get_volumes(): + script.append('select volume {}'.format(vol['Number'])) + script.append('assign') - # Run - run_diskpart(script) + # Run + run_diskpart(script) def get_boot_mode(): - """Check if the boot mode was UEFI or legacy.""" - boot_mode = 'Legacy' - try: - reg_key = winreg.OpenKey( - winreg.HKEY_LOCAL_MACHINE, r'System\CurrentControlSet\Control') - reg_value = winreg.QueryValueEx(reg_key, 'PEFirmwareType')[0] - if reg_value == 2: - boot_mode = 'UEFI' - except: - boot_mode = 'Unknown' + """Check if the boot mode was UEFI or legacy.""" + boot_mode = 'Legacy' + try: + reg_key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, r'System\CurrentControlSet\Control') + reg_value = winreg.QueryValueEx(reg_key, 'PEFirmwareType')[0] + if reg_value == 2: + boot_mode = 'UEFI' + except: + boot_mode = 'Unknown' - return boot_mode + return boot_mode def get_disk_details(disk): - """Get disk details using DiskPart.""" - details = {} - script = [ - 'select disk {}'.format(disk['Number']), - 'detail disk'] + """Get disk details using DiskPart.""" + details = {} + script = [ + 'select disk {}'.format(disk['Number']), + 'detail disk'] - # Run - try: - result = run_diskpart(script) - except subprocess.CalledProcessError: - pass - else: - output = result.stdout.decode().strip() - # Remove empty lines - tmp = [s.strip() for s in output.splitlines() if s.strip() != ''] - # Set disk name - details['Name'] = tmp[4] - # Split each line on ':' skipping those without ':' - tmp = [s.split(':') for s in tmp if ':' in s] - # Add key/value pairs to the details variable and return dict - details.update({key.strip(): value.strip() for (key, value) in tmp}) + # Run + try: + result = run_diskpart(script) + except subprocess.CalledProcessError: + pass + else: + output = result.stdout.decode().strip() + # Remove empty lines + tmp = [s.strip() for s in output.splitlines() if s.strip() != ''] + # Set disk name + details['Name'] = tmp[4] + # Split each line on ':' skipping those without ':' + tmp = [s.split(':') for s in tmp if ':' in s] + # Add key/value pairs to the details variable and return dict + details.update({key.strip(): value.strip() for (key, value) in tmp}) - return details + return details def get_disks(): - """Get list of attached disks using DiskPart.""" - disks = [] + """Get list of attached disks using DiskPart.""" + disks = [] - try: - # Run script - result = run_diskpart(['list disk']) - except subprocess.CalledProcessError: - pass - else: - # Append disk numbers - output = result.stdout.decode().strip() - for tmp in re.findall(r'Disk (\d+)\s+\w+\s+(\d+\s+\w+)', output): - num = tmp[0] - size = human_readable_size(tmp[1]) - disks.append({'Number': num, 'Size': size}) + try: + # Run script + result = run_diskpart(['list disk']) + except subprocess.CalledProcessError: + pass + else: + # Append disk numbers + output = result.stdout.decode().strip() + for tmp in re.findall(r'Disk (\d+)\s+\w+\s+(\d+\s+\w+)', output): + num = tmp[0] + size = human_readable_size(tmp[1]) + disks.append({'Number': num, 'Size': size}) - return disks + return disks def get_partition_details(disk, partition): - """Get partition details using DiskPart and fsutil.""" - details = {} - script = [ - 'select disk {}'.format(disk['Number']), - 'select partition {}'.format(partition['Number']), - 'detail partition'] + """Get partition details using DiskPart and fsutil.""" + details = {} + script = [ + 'select disk {}'.format(disk['Number']), + 'select partition {}'.format(partition['Number']), + 'detail partition'] - # Diskpart details + # Diskpart details + try: + # Run script + result = run_diskpart(script) + except subprocess.CalledProcessError: + pass + else: + # Get volume letter or RAW status + output = result.stdout.decode().strip() + tmp = re.search(r'Volume\s+\d+\s+(\w|RAW)\s+', output) + if tmp: + if tmp.group(1).upper() == 'RAW': + details['FileSystem'] = RAW + else: + details['Letter'] = tmp.group(1) + # Remove empty lines from output + tmp = [s.strip() for s in output.splitlines() if s.strip() != ''] + # Split each line on ':' skipping those without ':' + tmp = [s.split(':') for s in tmp if ':' in s] + # Add key/value pairs to the details variable and return dict + details.update({key.strip(): value.strip() for (key, value) in tmp}) + + # Get MBR type / GPT GUID for extra details on "Unknown" partitions + guid = PARTITION_UIDS.get(details.get('Type').upper(), {}) + if guid: + details.update({ + 'Description': guid.get('Description', '')[:29], + 'OS': guid.get('OS', 'Unknown')[:27]}) + + if 'Letter' in details: + # Disk usage try: - # Run script - result = run_diskpart(script) - except subprocess.CalledProcessError: - pass + tmp = psutil.disk_usage('{}:\\'.format(details['Letter'])) + except OSError as err: + details['FileSystem'] = 'Unknown' + details['Error'] = err.strerror else: - # Get volume letter or RAW status - output = result.stdout.decode().strip() - tmp = re.search(r'Volume\s+\d+\s+(\w|RAW)\s+', output) - if tmp: - if tmp.group(1).upper() == 'RAW': - details['FileSystem'] = RAW - else: - details['Letter'] = tmp.group(1) - # Remove empty lines from output - tmp = [s.strip() for s in output.splitlines() if s.strip() != ''] - # Split each line on ':' skipping those without ':' - tmp = [s.split(':') for s in tmp if ':' in s] - # Add key/value pairs to the details variable and return dict - details.update({key.strip(): value.strip() for (key, value) in tmp}) + details['Used Space'] = human_readable_size(tmp.used) - # Get MBR type / GPT GUID for extra details on "Unknown" partitions - guid = PARTITION_UIDS.get(details.get('Type').upper(), {}) - if guid: - details.update({ - 'Description': guid.get('Description', '')[:29], - 'OS': guid.get('OS', 'Unknown')[:27]}) + # fsutil details + cmd = [ + 'fsutil', + 'fsinfo', + 'volumeinfo', + '{}:'.format(details['Letter']) + ] + try: + result = run_program(cmd) + except subprocess.CalledProcessError: + pass + else: + output = result.stdout.decode().strip() + # Remove empty lines from output + tmp = [s.strip() for s in output.splitlines() if s.strip() != ''] + # Add "Feature" lines + details['File System Features'] = [s.strip() for s in tmp + if ':' not in s] + # Split each line on ':' skipping those without ':' + tmp = [s.split(':') for s in tmp if ':' in s] + # Add key/value pairs to the details variable and return dict + details.update({key.strip(): value.strip() for (key, value) in tmp}) - if 'Letter' in details: - # Disk usage - try: - tmp = psutil.disk_usage('{}:\\'.format(details['Letter'])) - except OSError as err: - details['FileSystem'] = 'Unknown' - details['Error'] = err.strerror - else: - details['Used Space'] = human_readable_size(tmp.used) + # Set Volume Name + details['Name'] = details.get('Volume Name', '') - # fsutil details - cmd = [ - 'fsutil', - 'fsinfo', - 'volumeinfo', - '{}:'.format(details['Letter']) - ] - try: - result = run_program(cmd) - except subprocess.CalledProcessError: - pass - else: - output = result.stdout.decode().strip() - # Remove empty lines from output - tmp = [s.strip() for s in output.splitlines() if s.strip() != ''] - # Add "Feature" lines - details['File System Features'] = [s.strip() for s in tmp - if ':' not in s] - # Split each line on ':' skipping those without ':' - tmp = [s.split(':') for s in tmp if ':' in s] - # Add key/value pairs to the details variable and return dict - details.update({key.strip(): value.strip() for (key, value) in tmp}) + # Set FileSystem Type + if details.get('FileSystem', '') not in ['RAW', 'Unknown']: + details['FileSystem'] = details.get('File System Name', 'Unknown') - # Set Volume Name - details['Name'] = details.get('Volume Name', '') - - # Set FileSystem Type - if details.get('FileSystem', '') not in ['RAW', 'Unknown']: - details['FileSystem'] = details.get('File System Name', 'Unknown') - - return details + return details def get_partitions(disk): - """Get list of partition using DiskPart.""" - partitions = [] - script = [ - 'select disk {}'.format(disk['Number']), - 'list partition'] + """Get list of partition using DiskPart.""" + partitions = [] + script = [ + 'select disk {}'.format(disk['Number']), + 'list partition'] - try: - # Run script - result = run_diskpart(script) - except subprocess.CalledProcessError: - pass - else: - # Append partition numbers - output = result.stdout.decode().strip() - regex = r'Partition\s+(\d+)\s+\w+\s+(\d+\s+\w+)\s+' - for tmp in re.findall(regex, output, re.IGNORECASE): - num = tmp[0] - size = human_readable_size(tmp[1]) - partitions.append({'Number': num, 'Size': size}) + try: + # Run script + result = run_diskpart(script) + except subprocess.CalledProcessError: + pass + else: + # Append partition numbers + output = result.stdout.decode().strip() + regex = r'Partition\s+(\d+)\s+\w+\s+(\d+\s+\w+)\s+' + for tmp in re.findall(regex, output, re.IGNORECASE): + num = tmp[0] + size = human_readable_size(tmp[1]) + partitions.append({'Number': num, 'Size': size}) - return partitions + return partitions def get_table_type(disk): - """Get disk partition table type using DiskPart.""" - part_type = 'Unknown' - script = [ - 'select disk {}'.format(disk['Number']), - 'uniqueid disk'] + """Get disk partition table type using DiskPart.""" + part_type = 'Unknown' + script = [ + 'select disk {}'.format(disk['Number']), + 'uniqueid disk'] - try: - result = run_diskpart(script) - except subprocess.CalledProcessError: - pass - else: - output = result.stdout.decode().strip() - if REGEX_DISK_GPT.search(output): - part_type = 'GPT' - elif REGEX_DISK_MBR.search(output): - part_type = 'MBR' - elif REGEX_DISK_RAW.search(output): - part_type = 'RAW' + try: + result = run_diskpart(script) + except subprocess.CalledProcessError: + pass + else: + output = result.stdout.decode().strip() + if REGEX_DISK_GPT.search(output): + part_type = 'GPT' + elif REGEX_DISK_MBR.search(output): + part_type = 'MBR' + elif REGEX_DISK_RAW.search(output): + part_type = 'RAW' - return part_type + return part_type def get_volumes(): - """Get list of volumes using DiskPart.""" - vols = [] - try: - result = run_diskpart(['list volume']) - except subprocess.CalledProcessError: - pass - else: - # Append volume numbers - output = result.stdout.decode().strip() - for tmp in re.findall(r'Volume (\d+)\s+([A-Za-z]?)\s+', output): - vols.append({'Number': tmp[0], 'Letter': tmp[1]}) + """Get list of volumes using DiskPart.""" + vols = [] + try: + result = run_diskpart(['list volume']) + except subprocess.CalledProcessError: + pass + else: + # Append volume numbers + output = result.stdout.decode().strip() + for tmp in re.findall(r'Volume (\d+)\s+([A-Za-z]?)\s+', output): + vols.append({'Number': tmp[0], 'Letter': tmp[1]}) - return vols + return vols def is_bad_partition(par): - """Check if the partition is accessible.""" - return 'Letter' not in par or REGEX_BAD_PARTITION.search(par['FileSystem']) + """Check if the partition is accessible.""" + return 'Letter' not in par or REGEX_BAD_PARTITION.search(par['FileSystem']) def prep_disk_for_formatting(disk=None): - """Gather details about the disk and its partitions.""" - disk['Format Warnings'] = '\n' - width = len(str(len(disk['Partitions']))) + """Gather details about the disk and its partitions.""" + disk['Format Warnings'] = '\n' + width = len(str(len(disk['Partitions']))) - # Bail early - if disk is None: - raise Exception('Disk not provided.') + # Bail early + if disk is None: + raise Exception('Disk not provided.') - # Set boot method and partition table type - disk['Use GPT'] = True - if (get_boot_mode() == 'UEFI'): - if (not ask("Setup Windows to use UEFI booting?")): - disk['Use GPT'] = False + # Set boot method and partition table type + disk['Use GPT'] = True + if (get_boot_mode() == 'UEFI'): + if (not ask("Setup Windows to use UEFI booting?")): + disk['Use GPT'] = False + else: + if (ask("Setup Windows to use BIOS/Legacy booting?")): + disk['Use GPT'] = False + + # Set Display and Warning Strings + if len(disk['Partitions']) == 0: + disk['Format Warnings'] += 'No partitions found\n' + for partition in disk['Partitions']: + display = '{size} {fs}'.format( + num = partition['Number'], + width = width, + size = partition['Size'], + fs = partition['FileSystem']) + + if is_bad_partition(partition): + # Set display string using partition description & OS type + display += '\t\t{q}{name}{q}\t{desc} ({os})'.format( + display = display, + q = '"' if partition['Name'] != '' else '', + name = partition['Name'], + desc = partition['Description'], + os = partition['OS']) else: - if (ask("Setup Windows to use BIOS/Legacy booting?")): - disk['Use GPT'] = False - - # Set Display and Warning Strings - if len(disk['Partitions']) == 0: - disk['Format Warnings'] += 'No partitions found\n' - for partition in disk['Partitions']: - display = '{size} {fs}'.format( - num = partition['Number'], - width = width, - size = partition['Size'], - fs = partition['FileSystem']) - - if is_bad_partition(partition): - # Set display string using partition description & OS type - display += '\t\t{q}{name}{q}\t{desc} ({os})'.format( - display = display, - q = '"' if partition['Name'] != '' else '', - name = partition['Name'], - desc = partition['Description'], - os = partition['OS']) - else: - # List space used instead of partition description & OS type - display += ' (Used: {used})\t{q}{name}{q}'.format( - used = partition['Used Space'], - q = '"' if partition['Name'] != '' else '', - name = partition['Name']) - # For all partitions - partition['Display String'] = display + # List space used instead of partition description & OS type + display += ' (Used: {used})\t{q}{name}{q}'.format( + used = partition['Used Space'], + q = '"' if partition['Name'] != '' else '', + name = partition['Name']) + # For all partitions + partition['Display String'] = display def reassign_volume_letter(letter, new_letter='I'): - """Assign a new letter to a volume using DiskPart.""" - if not letter: - # Ignore - return None - script = [ - 'select volume {}'.format(letter), - 'remove noerr', - 'assign letter={}'.format(new_letter)] - try: - run_diskpart(script) - except subprocess.CalledProcessError: - pass - else: - return new_letter + """Assign a new letter to a volume using DiskPart.""" + if not letter: + # Ignore + return None + script = [ + 'select volume {}'.format(letter), + 'remove noerr', + 'assign letter={}'.format(new_letter)] + try: + run_diskpart(script) + except subprocess.CalledProcessError: + pass + else: + return new_letter def remove_volume_letters(keep=None): - """Remove all assigned volume letters using DiskPart.""" - if not keep: - keep = '' + """Remove all assigned volume letters using DiskPart.""" + if not keep: + keep = '' - script = [] - for vol in get_volumes(): - if vol['Letter'].upper() != keep.upper(): - script.append('select volume {}'.format(vol['Number'])) - script.append('remove noerr') + script = [] + for vol in get_volumes(): + if vol['Letter'].upper() != keep.upper(): + script.append('select volume {}'.format(vol['Number'])) + script.append('remove noerr') - # Run script - try: - run_diskpart(script) - except subprocess.CalledProcessError: - pass + # Run script + try: + run_diskpart(script) + except subprocess.CalledProcessError: + pass def run_diskpart(script): - """Run DiskPart script.""" - tempfile = r'{}\diskpart.script'.format(global_vars['Env']['TMP']) + """Run DiskPart script.""" + tempfile = r'{}\diskpart.script'.format(global_vars['Env']['TMP']) - # Write script - with open(tempfile, 'w') as f: - for line in script: - f.write('{}\n'.format(line)) + # Write script + with open(tempfile, 'w') as f: + for line in script: + f.write('{}\n'.format(line)) - # Run script - cmd = [ - r'{}\Windows\System32\diskpart.exe'.format( - global_vars['Env']['SYSTEMDRIVE']), - '/s', tempfile] - result = run_program(cmd) - sleep(2) - return result + # Run script + cmd = [ + r'{}\Windows\System32\diskpart.exe'.format( + global_vars['Env']['SYSTEMDRIVE']), + '/s', tempfile] + result = run_program(cmd) + sleep(2) + return result def scan_disks(): - """Get details about the attached disks""" - disks = get_disks() + """Get details about the attached disks""" + disks = get_disks() - # Get disk details - for disk in disks: - # Get partition style - disk['Table'] = get_table_type(disk) + # Get disk details + for disk in disks: + # Get partition style + disk['Table'] = get_table_type(disk) - # Get disk name/model and physical details - disk.update(get_disk_details(disk)) + # Get disk name/model and physical details + disk.update(get_disk_details(disk)) - # Get partition info for disk - disk['Partitions'] = get_partitions(disk) + # Get partition info for disk + disk['Partitions'] = get_partitions(disk) - for partition in disk['Partitions']: - # Get partition details - partition.update(get_partition_details(disk, partition)) + for partition in disk['Partitions']: + # Get partition details + partition.update(get_partition_details(disk, partition)) - # Done - return disks + # Done + return disks def select_disk(title='Which disk?', disks=[]): - """Select a disk from the attached disks""" - # Build menu - disk_options = [] - for disk in disks: - display_name = '{}\t[{}] ({}) {}'.format( - disk.get('Size', ''), - disk.get('Table', ''), - disk.get('Type', ''), - disk.get('Name', 'Unknown'), - ) - pwidth=len(str(len(disk['Partitions']))) - for partition in disk['Partitions']: - # Main text - p_name = 'Partition {num:>{width}}: {size} ({fs})'.format( - num = partition['Number'], - width = pwidth, - size = partition['Size'], - fs = partition['FileSystem']) - if partition['Name']: - p_name += '\t"{}"'.format(partition['Name']) + """Select a disk from the attached disks""" + # Build menu + disk_options = [] + for disk in disks: + display_name = '{}\t[{}] ({}) {}'.format( + disk.get('Size', ''), + disk.get('Table', ''), + disk.get('Type', ''), + disk.get('Name', 'Unknown'), + ) + pwidth=len(str(len(disk['Partitions']))) + for partition in disk['Partitions']: + # Main text + p_name = 'Partition {num:>{width}}: {size} ({fs})'.format( + num = partition['Number'], + width = pwidth, + size = partition['Size'], + fs = partition['FileSystem']) + if partition['Name']: + p_name += '\t"{}"'.format(partition['Name']) - # Show unsupported partition(s) - if is_bad_partition(partition): - p_name = '{YELLOW}{p_name}{CLEAR}'.format( - p_name=p_name, **COLORS) + # Show unsupported partition(s) + if is_bad_partition(partition): + p_name = '{YELLOW}{p_name}{CLEAR}'.format( + p_name=p_name, **COLORS) - display_name += '\n\t\t\t{}'.format(p_name) - if not disk['Partitions']: - display_name += '\n\t\t\t{}No partitions found.{}'.format( - COLORS['YELLOW'], COLORS['CLEAR']) + display_name += '\n\t\t\t{}'.format(p_name) + if not disk['Partitions']: + display_name += '\n\t\t\t{}No partitions found.{}'.format( + COLORS['YELLOW'], COLORS['CLEAR']) - disk_options.append({'Name': display_name, 'Disk': disk}) - actions = [ - {'Name': 'Main Menu', 'Letter': 'M'}, - ] + disk_options.append({'Name': display_name, 'Disk': disk}) + actions = [ + {'Name': 'Main Menu', 'Letter': 'M'}, + ] - # Menu loop - selection = menu_select( - title = title, - main_entries = disk_options, - action_entries = actions) + # Menu loop + selection = menu_select( + title = title, + main_entries = disk_options, + action_entries = actions) - if (selection.isnumeric()): - return disk_options[int(selection)-1]['Disk'] - elif (selection == 'M'): - raise GenericAbort + if (selection.isnumeric()): + return disk_options[int(selection)-1]['Disk'] + elif (selection == 'M'): + raise GenericAbort if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 82a2d6b74d5399791aedaa058eccf71ec13d9569 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:07:18 -0700 Subject: [PATCH 125/186] Updated product_keys.py --- .bin/Scripts/functions/product_keys.py | 174 +++++++++++++------------ 1 file changed, 88 insertions(+), 86 deletions(-) diff --git a/.bin/Scripts/functions/product_keys.py b/.bin/Scripts/functions/product_keys.py index 988d36fd..1d96ec0b 100644 --- a/.bin/Scripts/functions/product_keys.py +++ b/.bin/Scripts/functions/product_keys.py @@ -4,108 +4,110 @@ from functions.common import * # Regex REGEX_REGISTRY_DIRS = re.compile( - r'^(config$|RegBack$|System32$|Transfer|Win)', - re.IGNORECASE) + r'^(config$|RegBack$|System32$|Transfer|Win)', + re.IGNORECASE) REGEX_SOFTWARE_HIVE = re.compile(r'^Software$', re.IGNORECASE) def extract_keys(): - """Extract keys from provided hives and return a dict.""" - keys = {} + """Extract keys from provided hives and return a dict.""" + keys = {} - # Extract keys - extract_item('ProduKey', silent=True) - for hive in find_software_hives(): - cmd = [ - global_vars['Tools']['ProduKey'], - '/IEKeys', '0', - '/WindowsKeys', '1', - '/OfficeKeys', '1', - '/ExtractEdition', '1', - '/nosavereg', - '/regfile', hive, - '/scomma', ''] - try: - out = run_program(cmd) - except subprocess.CalledProcessError: - # Ignore and return empty dict - pass - else: - for line in out.stdout.decode().splitlines(): - # Add key to keys under product only if unique - tmp = line.split(',') - product = tmp[0] - key = tmp[2] - if product not in keys: - keys[product] = [] - if key not in keys[product]: - keys[product].append(key) + # Extract keys + extract_item('ProduKey', silent=True) + for hive in find_software_hives(): + cmd = [ + global_vars['Tools']['ProduKey'], + '/IEKeys', '0', + '/WindowsKeys', '1', + '/OfficeKeys', '1', + '/ExtractEdition', '1', + '/nosavereg', + '/regfile', hive, + '/scomma', ''] + try: + out = run_program(cmd) + except subprocess.CalledProcessError: + # Ignore and return empty dict + pass + else: + for line in out.stdout.decode().splitlines(): + # Add key to keys under product only if unique + tmp = line.split(',') + product = tmp[0] + key = tmp[2] + if product not in keys: + keys[product] = [] + if key not in keys[product]: + keys[product].append(key) - # Done - return keys + # Done + return keys def list_clientdir_keys(): - """List product keys found in hives inside the ClientDir.""" - keys = extract_keys() - key_list = [] - if keys: - for product in sorted(keys): - key_list.append(product) - for key in sorted(keys[product]): - key_list.append(' {key}'.format(key=key)) - else: - key_list.append('No keys found.') + """List product keys found in hives inside the ClientDir.""" + keys = extract_keys() + key_list = [] + if keys: + for product in sorted(keys): + key_list.append(product) + for key in sorted(keys[product]): + key_list.append(' {key}'.format(key=key)) + else: + key_list.append('No keys found.') - return key_list + return key_list def find_software_hives(): - """Search for transferred SW hives and return a list.""" - hives = [] - search_paths = [global_vars['ClientDir']] + """Search for transferred SW hives and return a list.""" + hives = [] + search_paths = [global_vars['ClientDir']] - while len(search_paths) > 0: - for item in os.scandir(search_paths.pop(0)): - if item.is_dir() and REGEX_REGISTRY_DIRS.search(item.name): - search_paths.append(item.path) - if item.is_file() and REGEX_SOFTWARE_HIVE.search(item.name): - hives.append(item.path) + while len(search_paths) > 0: + for item in os.scandir(search_paths.pop(0)): + if item.is_dir() and REGEX_REGISTRY_DIRS.search(item.name): + search_paths.append(item.path) + if item.is_file() and REGEX_SOFTWARE_HIVE.search(item.name): + hives.append(item.path) - return hives + return hives def get_product_keys(): - """List product keys from saved report.""" - keys = [] - log_file = r'{LogDir}\Product Keys (ProduKey).txt'.format(**global_vars) - with open (log_file, 'r') as f: - for line in f.readlines(): - if re.search(r'^Product Name', line): - line = re.sub(r'^Product Name\s+:\s+(.*)', r'\1', line.strip()) - keys.append(line) + """List product keys from saved report.""" + keys = [] + log_file = r'{LogDir}\Product Keys (ProduKey).txt'.format(**global_vars) + with open (log_file, 'r') as f: + for line in f.readlines(): + if re.search(r'^Product Name', line): + line = re.sub(r'^Product Name\s+:\s+(.*)', r'\1', line.strip()) + keys.append(line) - if keys: - return keys - else: - return ['No product keys found'] + if keys: + return keys + else: + return ['No product keys found'] def run_produkey(): - """Run ProduKey and save report in the ClientDir.""" - extract_item('ProduKey', silent=True) - log_file = r'{LogDir}\Product Keys (ProduKey).txt'.format(**global_vars) - if not os.path.exists(log_file): - # Clear current configuration - for config in ['ProduKey.cfg', 'ProduKey64.cfg']: - config = r'{BinDir}\ProduKey\{config}'.format( - config=config, **global_vars) - try: - if os.path.exists(config): - os.remove(config) - except Exception: - pass - cmd = [ - global_vars['Tools']['ProduKey'], - '/nosavereg', - '/stext', - log_file] - run_program(cmd, check=False) + """Run ProduKey and save report in the ClientDir.""" + extract_item('ProduKey', silent=True) + log_file = r'{LogDir}\Product Keys (ProduKey).txt'.format(**global_vars) + if not os.path.exists(log_file): + # Clear current configuration + for config in ['ProduKey.cfg', 'ProduKey64.cfg']: + config = r'{BinDir}\ProduKey\{config}'.format( + config=config, **global_vars) + try: + if os.path.exists(config): + os.remove(config) + except Exception: + pass + cmd = [ + global_vars['Tools']['ProduKey'], + '/nosavereg', + '/stext', + log_file] + run_program(cmd, check=False) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From deb9d9add18737af36e64d18331cf1296426bf56 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:08:56 -0700 Subject: [PATCH 126/186] Updated repairs.py --- .bin/Scripts/functions/repairs.py | 212 +++++++++++++++--------------- 1 file changed, 107 insertions(+), 105 deletions(-) diff --git a/.bin/Scripts/functions/repairs.py b/.bin/Scripts/functions/repairs.py index 589dccc3..bcaa5bbf 100644 --- a/.bin/Scripts/functions/repairs.py +++ b/.bin/Scripts/functions/repairs.py @@ -3,124 +3,126 @@ from functions.common import * def run_chkdsk(repair=False): - """Run CHKDSK scan or schedule offline repairs.""" - if repair: - run_chkdsk_offline() - else: - run_chkdsk_scan() + """Run CHKDSK scan or schedule offline repairs.""" + if repair: + run_chkdsk_offline() + else: + run_chkdsk_scan() def run_chkdsk_scan(): - """Run CHKDSK in a "split window" and report errors.""" - if global_vars['OS']['Version'] in ('8', '8.1', '10'): - cmd = ['chkdsk', global_vars['Env']['SYSTEMDRIVE'], '/scan', '/perf'] - else: - cmd = ['chkdsk', global_vars['Env']['SYSTEMDRIVE']] - out = run_program(cmd, check=False) - # retcode == 0: no issues - # retcode == 1: fixed issues (also happens when chkdsk.exe is killed?) - # retcode == 2: issues - if int(out.returncode) > 0: - # print_error(' ERROR: CHKDSK encountered errors') - raise GenericError + """Run CHKDSK in a "split window" and report errors.""" + if global_vars['OS']['Version'] in ('8', '8.1', '10'): + cmd = ['chkdsk', global_vars['Env']['SYSTEMDRIVE'], '/scan', '/perf'] + else: + cmd = ['chkdsk', global_vars['Env']['SYSTEMDRIVE']] + out = run_program(cmd, check=False) + # retcode == 0: no issues + # retcode == 1: fixed issues (also happens when chkdsk.exe is killed?) + # retcode == 2: issues + if int(out.returncode) > 0: + # print_error(' ERROR: CHKDSK encountered errors') + raise GenericError - # Save stderr - with open(r'{LogDir}\Tools\CHKDSK.err'.format(**global_vars), 'a') as f: - for line in out.stderr.decode().splitlines(): - f.write(line.strip() + '\n') - # Save stdout - with open(r'{LogDir}\Tools\CHKDSK.log'.format(**global_vars), 'a') as f: - for line in out.stdout.decode().splitlines(): - f.write(line.strip() + '\n') + # Save stderr + with open(r'{LogDir}\Tools\CHKDSK.err'.format(**global_vars), 'a') as f: + for line in out.stderr.decode().splitlines(): + f.write(line.strip() + '\n') + # Save stdout + with open(r'{LogDir}\Tools\CHKDSK.log'.format(**global_vars), 'a') as f: + for line in out.stdout.decode().splitlines(): + f.write(line.strip() + '\n') def run_chkdsk_offline(): - """Set filesystem 'dirty bit' to force a chkdsk during next boot.""" - cmd = [ - 'fsutil', 'dirty', - 'set', - global_vars['Env']['SYSTEMDRIVE']] - out = run_program(cmd, check=False) - if int(out.returncode) > 0: - raise GenericError + """Set filesystem 'dirty bit' to force a chkdsk during next boot.""" + cmd = [ + 'fsutil', 'dirty', + 'set', + global_vars['Env']['SYSTEMDRIVE']] + out = run_program(cmd, check=False) + if int(out.returncode) > 0: + raise GenericError def run_dism(repair=False): - """Run DISM /RestoreHealth, then /CheckHealth, and then report errors.""" - if global_vars['OS']['Version'] in ('8', '8.1', '10'): - if repair: - # Restore Health - cmd = [ - 'DISM', '/Online', - '/Cleanup-Image', '/RestoreHealth', - r'/LogPath:"{LogDir}\Tools\DISM_RestoreHealth.log"'.format( - **global_vars), - '-new_console:n', '-new_console:s33V'] - else: - # Scan Health - cmd = [ - 'DISM', '/Online', - '/Cleanup-Image', '/ScanHealth', - r'/LogPath:"{LogDir}\Tools\DISM_ScanHealth.log"'.format( - **global_vars), - '-new_console:n', '-new_console:s33V'] - run_program(cmd, pipe=False, check=False, shell=True) - wait_for_process('dism') - # Now check health - cmd = [ - 'DISM', '/Online', - '/Cleanup-Image', '/CheckHealth', - r'/LogPath:"{LogDir}\Tools\DISM_CheckHealth.log"'.format(**global_vars)] - result = run_program(cmd, shell=True).stdout.decode() - # Check result - if 'no component store corruption detected' not in result.lower(): - raise GenericError + """Run DISM /RestoreHealth, then /CheckHealth, and then report errors.""" + if global_vars['OS']['Version'] in ('8', '8.1', '10'): + if repair: + # Restore Health + cmd = [ + 'DISM', '/Online', + '/Cleanup-Image', '/RestoreHealth', + r'/LogPath:"{LogDir}\Tools\DISM_RestoreHealth.log"'.format( + **global_vars), + '-new_console:n', '-new_console:s33V'] else: - raise UnsupportedOSError + # Scan Health + cmd = [ + 'DISM', '/Online', + '/Cleanup-Image', '/ScanHealth', + r'/LogPath:"{LogDir}\Tools\DISM_ScanHealth.log"'.format( + **global_vars), + '-new_console:n', '-new_console:s33V'] + run_program(cmd, pipe=False, check=False, shell=True) + wait_for_process('dism') + # Now check health + cmd = [ + 'DISM', '/Online', + '/Cleanup-Image', '/CheckHealth', + r'/LogPath:"{LogDir}\Tools\DISM_CheckHealth.log"'.format(**global_vars)] + result = run_program(cmd, shell=True).stdout.decode() + # Check result + if 'no component store corruption detected' not in result.lower(): + raise GenericError + else: + raise UnsupportedOSError def run_kvrt(): - """Run KVRT.""" - extract_item('KVRT', silent=True) - os.makedirs(global_vars['QuarantineDir'], exist_ok=True) - cmd = [ - global_vars['Tools']['KVRT'], - '-accepteula', '-dontcryptsupportinfo', '-fixednames', - '-d', global_vars['QuarantineDir'], - '-processlevel', '3'] - popen_program(cmd, pipe=False) + """Run KVRT.""" + extract_item('KVRT', silent=True) + os.makedirs(global_vars['QuarantineDir'], exist_ok=True) + cmd = [ + global_vars['Tools']['KVRT'], + '-accepteula', '-dontcryptsupportinfo', '-fixednames', + '-d', global_vars['QuarantineDir'], + '-processlevel', '3'] + popen_program(cmd, pipe=False) def run_sfc_scan(): - """Run SFC in a "split window" and report errors.""" - cmd = [ - r'{SYSTEMROOT}\System32\sfc.exe'.format(**global_vars['Env']), - '/scannow'] - out = run_program(cmd, check=False) - # Save stderr - with open(r'{LogDir}\Tools\SFC.err'.format(**global_vars), 'a') as f: - for line in out.stderr.decode('utf-8', 'ignore').splitlines(): - f.write(line.strip() + '\n') - # Save stdout - with open(r'{LogDir}\Tools\SFC.log'.format(**global_vars), 'a') as f: - for line in out.stdout.decode('utf-8', 'ignore').splitlines(): - f.write(line.strip() + '\n') - # Check result - log_text = out.stdout.decode('utf-8', 'ignore').replace('\0', '') - if re.findall(r'did\s+not\s+find\s+any\s+integrity\s+violations', log_text): - pass - elif re.findall(r'successfully\s+repaired\s+them', log_text): - raise GenericRepair - else: - raise GenericError + """Run SFC in a "split window" and report errors.""" + cmd = [ + r'{SYSTEMROOT}\System32\sfc.exe'.format(**global_vars['Env']), + '/scannow'] + out = run_program(cmd, check=False) + # Save stderr + with open(r'{LogDir}\Tools\SFC.err'.format(**global_vars), 'a') as f: + for line in out.stderr.decode('utf-8', 'ignore').splitlines(): + f.write(line.strip() + '\n') + # Save stdout + with open(r'{LogDir}\Tools\SFC.log'.format(**global_vars), 'a') as f: + for line in out.stdout.decode('utf-8', 'ignore').splitlines(): + f.write(line.strip() + '\n') + # Check result + log_text = out.stdout.decode('utf-8', 'ignore').replace('\0', '') + if re.findall(r'did\s+not\s+find\s+any\s+integrity\s+violations', log_text): + pass + elif re.findall(r'successfully\s+repaired\s+them', log_text): + raise GenericRepair + else: + raise GenericError def run_tdsskiller(): - """Run TDSSKiller.""" - extract_item('TDSSKiller', silent=True) - os.makedirs(r'{QuarantineDir}\TDSSKiller'.format( - **global_vars), exist_ok=True) - cmd = [ - global_vars['Tools']['TDSSKiller'], - '-l', r'{LogDir}\Tools\TDSSKiller.log'.format(**global_vars), - '-qpath', r'{QuarantineDir}\TDSSKiller'.format(**global_vars), - '-accepteula', '-accepteulaksn', - '-dcexact', '-tdlfs'] - run_program(cmd, pipe=False) + """Run TDSSKiller.""" + extract_item('TDSSKiller', silent=True) + os.makedirs(r'{QuarantineDir}\TDSSKiller'.format( + **global_vars), exist_ok=True) + cmd = [ + global_vars['Tools']['TDSSKiller'], + '-l', r'{LogDir}\Tools\TDSSKiller.log'.format(**global_vars), + '-qpath', r'{QuarantineDir}\TDSSKiller'.format(**global_vars), + '-accepteula', '-accepteulaksn', + '-dcexact', '-tdlfs'] + run_program(cmd, pipe=False) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From cfd4eebcd426b48dbcdd6474afd86cc5e262e542 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:09:26 -0700 Subject: [PATCH 127/186] Updated safemode.py --- .bin/Scripts/functions/safemode.py | 38 ++++++++++++++++-------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.bin/Scripts/functions/safemode.py b/.bin/Scripts/functions/safemode.py index 9f44aa04..837b14e8 100644 --- a/.bin/Scripts/functions/safemode.py +++ b/.bin/Scripts/functions/safemode.py @@ -6,31 +6,33 @@ from functions.common import * REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' def disable_safemode_msi(): - """Disable MSI access under safemode.""" - cmd = ['reg', 'delete', REG_MSISERVER, '/f'] - run_program(cmd) + """Disable MSI access under safemode.""" + cmd = ['reg', 'delete', REG_MSISERVER, '/f'] + run_program(cmd) def disable_safemode(): - """Edit BCD to remove safeboot value.""" - cmd = ['bcdedit', '/deletevalue', '{default}', 'safeboot'] - run_program(cmd) + """Edit BCD to remove safeboot value.""" + cmd = ['bcdedit', '/deletevalue', '{default}', 'safeboot'] + run_program(cmd) def enable_safemode_msi(): - """Enable MSI access under safemode.""" - cmd = ['reg', 'add', REG_MSISERVER, '/f'] - run_program(cmd) - cmd = ['reg', 'add', REG_MSISERVER, '/ve', - '/t', 'REG_SZ', '/d', 'Service', '/f'] - run_program(cmd) + """Enable MSI access under safemode.""" + cmd = ['reg', 'add', REG_MSISERVER, '/f'] + run_program(cmd) + cmd = ['reg', 'add', REG_MSISERVER, '/ve', + '/t', 'REG_SZ', '/d', 'Service', '/f'] + run_program(cmd) def enable_safemode(): - """Edit BCD to set safeboot as default.""" - cmd = ['bcdedit', '/set', '{default}', 'safeboot', 'network'] - run_program(cmd) + """Edit BCD to set safeboot as default.""" + cmd = ['bcdedit', '/set', '{default}', 'safeboot', 'network'] + run_program(cmd) def reboot(delay=3): - cmd = ['shutdown', '-r', '-t', '{}'.format(delay)] - run_program(cmd, check=False) + cmd = ['shutdown', '-r', '-t', '{}'.format(delay)] + run_program(cmd, check=False) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From a47707447c5bab3e071e96c678b716cc4630346c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:10:49 -0700 Subject: [PATCH 128/186] Updated setup.py --- .bin/Scripts/functions/setup.py | 522 ++++++++++++++++---------------- 1 file changed, 262 insertions(+), 260 deletions(-) diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index 4e1e2ee7..4254a16a 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -9,313 +9,315 @@ HKCR = winreg.HKEY_CLASSES_ROOT HKCU = winreg.HKEY_CURRENT_USER HKLM = winreg.HKEY_LOCAL_MACHINE MOZILLA_FIREFOX_UBO_PATH = r'{}\{}\ublock_origin.xpi'.format( - os.environ.get('PROGRAMFILES'), - r'Mozilla Firefox\distribution\extensions') + os.environ.get('PROGRAMFILES'), + r'Mozilla Firefox\distribution\extensions') OTHER_RESULTS = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - 'FileNotFoundError': 'File not found', - }, - 'Warning': {}} + 'Error': { + 'CalledProcessError': 'Unknown Error', + 'FileNotFoundError': 'File not found', + }, + 'Warning': {}} SETTINGS_CLASSIC_START = { - r'Software\IvoSoft\ClassicShell\Settings': {}, - r'Software\IvoSoft\ClassicStartMenu': { - 'DWORD Items': {'ShowedStyle2': 1}, - }, - r'Software\IvoSoft\ClassicStartMenu\MRU': {}, - r'Software\IvoSoft\ClassicStartMenu\Settings': { - 'DWORD Items': {'SkipMetro': 1}, - 'SZ Items': { - 'MenuStyle': 'Win7', - 'RecentPrograms': 'Recent', - }, - }, - } + r'Software\IvoSoft\ClassicShell\Settings': {}, + r'Software\IvoSoft\ClassicStartMenu': { + 'DWORD Items': {'ShowedStyle2': 1}, + }, + r'Software\IvoSoft\ClassicStartMenu\MRU': {}, + r'Software\IvoSoft\ClassicStartMenu\Settings': { + 'DWORD Items': {'SkipMetro': 1}, + 'SZ Items': { + 'MenuStyle': 'Win7', + 'RecentPrograms': 'Recent', + }, + }, + } SETTINGS_EXPLORER_SYSTEM = { - # Disable Location Tracking - r'Software\Microsoft\Windows NT\CurrentVersion\Sensor\Overrides\{BFA794E4-F964-4FDB-90F6-51056BFE4B44}': { - 'DWORD Items': {'SensorPermissionState': 0}, - }, - r'System\CurrentControlSet\Services\lfsvc\Service\Configuration': { - 'Status': {'Value': 0}, - }, - # Disable Telemetry - r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { - 'DWORD Items': {'AllowTelemetry': 0}, - }, - r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { - 'DWORD Items': {'AllowTelemetry': 0}, - 'WOW64_32': True, - }, - r'Software\Policies\Microsoft\Windows\DataCollection': { - 'DWORD Items': {'AllowTelemetry': 0}, - }, - # Disable Wi-Fi Sense - r'Software\Microsoft\PolicyManager\default\WiFi\AllowWiFiHotSpotReporting': { - 'DWORD Items': {'Value': 0}, - }, - r'Software\Microsoft\PolicyManager\default\WiFi\AllowAutoConnectToWiFiSenseHotspots': { - 'DWORD Items': {'Value': 0}, - }, - } + # Disable Location Tracking + r'Software\Microsoft\Windows NT\CurrentVersion\Sensor\Overrides\{BFA794E4-F964-4FDB-90F6-51056BFE4B44}': { + 'DWORD Items': {'SensorPermissionState': 0}, + }, + r'System\CurrentControlSet\Services\lfsvc\Service\Configuration': { + 'Status': {'Value': 0}, + }, + # Disable Telemetry + r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { + 'DWORD Items': {'AllowTelemetry': 0}, + }, + r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { + 'DWORD Items': {'AllowTelemetry': 0}, + 'WOW64_32': True, + }, + r'Software\Policies\Microsoft\Windows\DataCollection': { + 'DWORD Items': {'AllowTelemetry': 0}, + }, + # Disable Wi-Fi Sense + r'Software\Microsoft\PolicyManager\default\WiFi\AllowWiFiHotSpotReporting': { + 'DWORD Items': {'Value': 0}, + }, + r'Software\Microsoft\PolicyManager\default\WiFi\AllowAutoConnectToWiFiSenseHotspots': { + 'DWORD Items': {'Value': 0}, + }, + } SETTINGS_EXPLORER_USER = { - # Disable silently installed apps - r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { - 'DWORD Items': {'SilentInstalledAppsEnabled': 0}, - }, - # Disable Tips and Tricks - r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { - 'DWORD Items': {'SoftLandingEnabled ': 0}, - }, - # Hide People bar - r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced\People': { - 'DWORD Items': {'PeopleBand': 0}, - }, - # Hide Search button / box - r'Software\Microsoft\Windows\CurrentVersion\Search': { - 'DWORD Items': {'SearchboxTaskbarMode': 0}, - }, - # Change default Explorer view to "Computer" - r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced': { - 'DWORD Items': {'LaunchTo': 1}, - }, - } + # Disable silently installed apps + r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { + 'DWORD Items': {'SilentInstalledAppsEnabled': 0}, + }, + # Disable Tips and Tricks + r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { + 'DWORD Items': {'SoftLandingEnabled ': 0}, + }, + # Hide People bar + r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced\People': { + 'DWORD Items': {'PeopleBand': 0}, + }, + # Hide Search button / box + r'Software\Microsoft\Windows\CurrentVersion\Search': { + 'DWORD Items': {'SearchboxTaskbarMode': 0}, + }, + # Change default Explorer view to "Computer" + r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced': { + 'DWORD Items': {'LaunchTo': 1}, + }, + } SETTINGS_GOOGLE_CHROME = { - r'Software\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm': { - 'SZ Items': { - 'update_url': 'https://clients2.google.com/service/update2/crx'}, - 'WOW64_32': True, - }, - r'Software\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco': { - 'SZ Items': { - 'update_url': 'https://clients2.google.com/service/update2/crx'}, - 'WOW64_32': True, - }, - } + r'Software\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm': { + 'SZ Items': { + 'update_url': 'https://clients2.google.com/service/update2/crx'}, + 'WOW64_32': True, + }, + r'Software\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco': { + 'SZ Items': { + 'update_url': 'https://clients2.google.com/service/update2/crx'}, + 'WOW64_32': True, + }, + } SETTINGS_MOZILLA_FIREFOX_32 = { - r'Software\Mozilla\Firefox\Extensions': { - 'SZ Items': { - 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, - 'WOW64_32': True, - }, - } + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + 'WOW64_32': True, + }, + } SETTINGS_MOZILLA_FIREFOX_64 = { - r'Software\Mozilla\Firefox\Extensions': { - 'SZ Items': { - 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, - }, - } + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + }, + } VCR_REDISTS = [ - {'Name': 'Visual C++ 2010 x32...', - 'Cmd': [r'2010sp1\x32\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2010 x64...', - 'Cmd': [r'2010sp1\x64\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2012 Update 4 x32...', - 'Cmd': [r'2012u4\x32\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2012 Update 4 x64...', - 'Cmd': [r'2012u4\x64\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2013 x32...', - 'Cmd': [r'2013\x32\vcredist.exe', '/install', - '/passive', '/norestart']}, - {'Name': 'Visual C++ 2013 x64...', - 'Cmd': [r'2013\x64\vcredist.exe', '/install', - '/passive', '/norestart']}, - {'Name': 'Visual C++ 2017 x32...', - 'Cmd': [r'2017\x32\vcredist.exe', '/install', - '/passive', '/norestart']}, - {'Name': 'Visual C++ 2017 x64...', - 'Cmd': [r'2017\x64\vcredist.exe', '/install', - '/passive', '/norestart']}, - ] + {'Name': 'Visual C++ 2010 x32...', + 'Cmd': [r'2010sp1\x32\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2010 x64...', + 'Cmd': [r'2010sp1\x64\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2012 Update 4 x32...', + 'Cmd': [r'2012u4\x32\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2012 Update 4 x64...', + 'Cmd': [r'2012u4\x64\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2013 x32...', + 'Cmd': [r'2013\x32\vcredist.exe', '/install', + '/passive', '/norestart']}, + {'Name': 'Visual C++ 2013 x64...', + 'Cmd': [r'2013\x64\vcredist.exe', '/install', + '/passive', '/norestart']}, + {'Name': 'Visual C++ 2017 x32...', + 'Cmd': [r'2017\x32\vcredist.exe', '/install', + '/passive', '/norestart']}, + {'Name': 'Visual C++ 2017 x64...', + 'Cmd': [r'2017\x64\vcredist.exe', '/install', + '/passive', '/norestart']}, + ] def config_classicstart(): - """Configure ClassicStart.""" - # User level, not system level - cs_exe = r'{PROGRAMFILES}\Classic Shell\ClassicStartMenu.exe'.format( - **global_vars['Env']) - skin = r'{PROGRAMFILES}\Classic Shell\Skins\Metro-Win10-Black.skin7'.format( - **global_vars['Env']) - extract_item('ClassicStartSkin', silent=True) + """Configure ClassicStart.""" + # User level, not system level + cs_exe = r'{PROGRAMFILES}\Classic Shell\ClassicStartMenu.exe'.format( + **global_vars['Env']) + skin = r'{PROGRAMFILES}\Classic Shell\Skins\Metro-Win10-Black.skin7'.format( + **global_vars['Env']) + extract_item('ClassicStartSkin', silent=True) - # Stop Classic Start - run_program([cs_exe, '-exit'], check=False) - sleep(1) - kill_process('ClassicStartMenu.exe') + # Stop Classic Start + run_program([cs_exe, '-exit'], check=False) + sleep(1) + kill_process('ClassicStartMenu.exe') - # Configure - write_registry_settings(SETTINGS_CLASSIC_START, all_users=False) - if global_vars['OS']['Version'] == '10' and os.path.exists(skin): - # Enable Win10 theme if on Win10 - key_path = r'Software\IvoSoft\ClassicStartMenu\Settings' - with winreg.OpenKey(HKCU, key_path, access=winreg.KEY_WRITE) as key: - winreg.SetValueEx( - key, 'SkinW7', 0, winreg.REG_SZ, 'Metro-Win10-Black') - winreg.SetValueEx(key, 'SkinVariationW7', 0, winreg.REG_SZ, '') + # Configure + write_registry_settings(SETTINGS_CLASSIC_START, all_users=False) + if global_vars['OS']['Version'] == '10' and os.path.exists(skin): + # Enable Win10 theme if on Win10 + key_path = r'Software\IvoSoft\ClassicStartMenu\Settings' + with winreg.OpenKey(HKCU, key_path, access=winreg.KEY_WRITE) as key: + winreg.SetValueEx( + key, 'SkinW7', 0, winreg.REG_SZ, 'Metro-Win10-Black') + winreg.SetValueEx(key, 'SkinVariationW7', 0, winreg.REG_SZ, '') - # Pin Browser to Start Menu (Classic) - firefox = r'{PROGRAMDATA}\Start Menu\Programs\Mozilla Firefox.lnk'.format( - **global_vars['Env']) - chrome = r'{PROGRAMDATA}\Start Menu\Programs\Google Chrome.lnk'.format( - **global_vars['Env']) - dest_path = r'{APPDATA}\ClassicShell\Pinned'.format(**global_vars['Env']) - source = None - dest = None - if os.path.exists(firefox): - source = firefox - dest = r'{}\Mozilla Firefox.lnk'.format(dest_path) - elif os.path.exists(chrome): - source = chrome - dest = r'{}\Google Chrome.lnk'.format(dest_path) - if source: - try: - os.makedirs(dest_path, exist_ok=True) - shutil.copy(source, dest) - except Exception: - pass # Meh, it's fine without + # Pin Browser to Start Menu (Classic) + firefox = r'{PROGRAMDATA}\Start Menu\Programs\Mozilla Firefox.lnk'.format( + **global_vars['Env']) + chrome = r'{PROGRAMDATA}\Start Menu\Programs\Google Chrome.lnk'.format( + **global_vars['Env']) + dest_path = r'{APPDATA}\ClassicShell\Pinned'.format(**global_vars['Env']) + source = None + dest = None + if os.path.exists(firefox): + source = firefox + dest = r'{}\Mozilla Firefox.lnk'.format(dest_path) + elif os.path.exists(chrome): + source = chrome + dest = r'{}\Google Chrome.lnk'.format(dest_path) + if source: + try: + os.makedirs(dest_path, exist_ok=True) + shutil.copy(source, dest) + except Exception: + pass # Meh, it's fine without - # (Re)start Classic Start - run_program([cs_exe, '-exit'], check=False) - sleep(1) - kill_process('ClassicStartMenu.exe') - sleep(1) - popen_program(cs_exe) + # (Re)start Classic Start + run_program([cs_exe, '-exit'], check=False) + sleep(1) + kill_process('ClassicStartMenu.exe') + sleep(1) + popen_program(cs_exe) def config_explorer_system(): - """Configure Windows Explorer for all users via Registry settings.""" - write_registry_settings(SETTINGS_EXPLORER_SYSTEM, all_users=True) + """Configure Windows Explorer for all users via Registry settings.""" + write_registry_settings(SETTINGS_EXPLORER_SYSTEM, all_users=True) def config_explorer_user(): - """Configure Windows Explorer for current user via Registry settings.""" - write_registry_settings(SETTINGS_EXPLORER_USER, all_users=False) + """Configure Windows Explorer for current user via Registry settings.""" + write_registry_settings(SETTINGS_EXPLORER_USER, all_users=False) def disable_windows_telemetry(): - """Disable Windows 10 telemetry settings with O&O ShutUp10.""" - extract_item('ShutUp10', silent=True) - cmd = [ - r'{BinDir}\ShutUp10\OOSU10.exe'.format(**global_vars), - r'{BinDir}\ShutUp10\1201.cfg'.format(**global_vars), - '/quiet'] - run_program(cmd) + """Disable Windows 10 telemetry settings with O&O ShutUp10.""" + extract_item('ShutUp10', silent=True) + cmd = [ + r'{BinDir}\ShutUp10\OOSU10.exe'.format(**global_vars), + r'{BinDir}\ShutUp10\1201.cfg'.format(**global_vars), + '/quiet'] + run_program(cmd) def update_clock(): - """Set Timezone and sync clock.""" - run_program(['tzutil' ,'/s', WINDOWS_TIME_ZONE], check=False) - run_program(['net', 'stop', 'w32ime'], check=False) - run_program( - ['w32tm', '/config', '/syncfromflags:manual', - '/manualpeerlist:"us.pool.ntp.org time.nist.gov time.windows.com"', - ], - check=False) - run_program(['net', 'start', 'w32ime'], check=False) - run_program(['w32tm', '/resync', '/nowait'], check=False) + """Set Timezone and sync clock.""" + run_program(['tzutil' ,'/s', WINDOWS_TIME_ZONE], check=False) + run_program(['net', 'stop', 'w32ime'], check=False) + run_program( + ['w32tm', '/config', '/syncfromflags:manual', + '/manualpeerlist:"us.pool.ntp.org time.nist.gov time.windows.com"', + ], + check=False) + run_program(['net', 'start', 'w32ime'], check=False) + run_program(['w32tm', '/resync', '/nowait'], check=False) def write_registry_settings(settings, all_users=False): - """Write registry values from custom dict of dicts.""" - hive = HKCU - if all_users: - hive = HKLM - for k, v in settings.items(): - # CreateKey - access = winreg.KEY_WRITE - if 'WOW64_32' in v: - access = access | winreg.KEY_WOW64_32KEY - winreg.CreateKeyEx(hive, k, 0, access) + """Write registry values from custom dict of dicts.""" + hive = HKCU + if all_users: + hive = HKLM + for k, v in settings.items(): + # CreateKey + access = winreg.KEY_WRITE + if 'WOW64_32' in v: + access = access | winreg.KEY_WOW64_32KEY + winreg.CreateKeyEx(hive, k, 0, access) - # Create values - with winreg.OpenKeyEx(hive, k, 0, access) as key: - for name, value in v.get('DWORD Items', {}).items(): - winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) - for name, value in v.get('SZ Items', {}).items(): - winreg.SetValueEx(key, name, 0, winreg.REG_SZ, value) + # Create values + with winreg.OpenKeyEx(hive, k, 0, access) as key: + for name, value in v.get('DWORD Items', {}).items(): + winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) + for name, value in v.get('SZ Items', {}).items(): + winreg.SetValueEx(key, name, 0, winreg.REG_SZ, value) # Installations def install_adobe_reader(): - """Install Adobe Reader.""" - cmd = [ - r'{BaseDir}\Installers\Extras\Office\Adobe Reader DC.exe'.format( - **global_vars), - '/sAll', - '/msi', '/norestart', '/quiet', - 'ALLUSERS=1', - 'EULA_ACCEPT=YES'] - run_program(cmd) + """Install Adobe Reader.""" + cmd = [ + r'{BaseDir}\Installers\Extras\Office\Adobe Reader DC.exe'.format( + **global_vars), + '/sAll', + '/msi', '/norestart', '/quiet', + 'ALLUSERS=1', + 'EULA_ACCEPT=YES'] + run_program(cmd) def install_chrome_extensions(): - """Update registry to install Google Chrome extensions for all users.""" - write_registry_settings(SETTINGS_GOOGLE_CHROME, all_users=True) + """Update registry to install Google Chrome extensions for all users.""" + write_registry_settings(SETTINGS_GOOGLE_CHROME, all_users=True) def install_classicstart_skin(): - """Extract ClassicStart skin to installation folder.""" - if global_vars['OS']['Version'] not in ('8', '8.1', '10'): - raise UnsupportedOSError - extract_item('ClassicStartSkin', silent=True) - source = r'{BinDir}\ClassicStartSkin\Metro-Win10-Black.skin7'.format( - **global_vars) - dest_path = r'{PROGRAMFILES}\Classic Shell\Skins'.format( - **global_vars['Env']) - dest = r'{}\Metro-Win10-Black.skin7'.format(dest_path) - os.makedirs(dest_path, exist_ok=True) - shutil.copy(source, dest) + """Extract ClassicStart skin to installation folder.""" + if global_vars['OS']['Version'] not in ('8', '8.1', '10'): + raise UnsupportedOSError + extract_item('ClassicStartSkin', silent=True) + source = r'{BinDir}\ClassicStartSkin\Metro-Win10-Black.skin7'.format( + **global_vars) + dest_path = r'{PROGRAMFILES}\Classic Shell\Skins'.format( + **global_vars['Env']) + dest = r'{}\Metro-Win10-Black.skin7'.format(dest_path) + os.makedirs(dest_path, exist_ok=True) + shutil.copy(source, dest) def install_firefox_extensions(): - """Update registry to install Firefox extensions for all users.""" - dist_path = r'{PROGRAMFILES}\Mozilla Firefox\distribution\extensions'.format( - **global_vars['Env']) - source_path = r'{CBinDir}\FirefoxExtensions.7z'.format(**global_vars) - if not os.path.exists(source_path): - raise FileNotFoundError + """Update registry to install Firefox extensions for all users.""" + dist_path = r'{PROGRAMFILES}\Mozilla Firefox\distribution\extensions'.format( + **global_vars['Env']) + source_path = r'{CBinDir}\FirefoxExtensions.7z'.format(**global_vars) + if not os.path.exists(source_path): + raise FileNotFoundError - # Update registry - write_registry_settings(SETTINGS_MOZILLA_FIREFOX_32, all_users=True) - write_registry_settings(SETTINGS_MOZILLA_FIREFOX_64, all_users=True) + # Update registry + write_registry_settings(SETTINGS_MOZILLA_FIREFOX_32, all_users=True) + write_registry_settings(SETTINGS_MOZILLA_FIREFOX_64, all_users=True) - # Extract extension(s) to distribution folder - cmd = [ - global_vars['Tools']['SevenZip'], 'e', '-aos', '-bso0', '-bse0', - '-p{ArchivePassword}'.format(**global_vars), - '-o{dist_path}'.format(dist_path=dist_path), - source_path] - run_program(cmd) + # Extract extension(s) to distribution folder + cmd = [ + global_vars['Tools']['SevenZip'], 'e', '-aos', '-bso0', '-bse0', + '-p{ArchivePassword}'.format(**global_vars), + '-o{dist_path}'.format(dist_path=dist_path), + source_path] + run_program(cmd) def install_ninite_bundle(mse=False): - """Run Ninite file(s) based on OS version.""" - if global_vars['OS']['Version'] in ('8', '8.1', '10'): - # Modern selection - popen_program(r'{BaseDir}\Installers\Extras\Bundles\Modern.exe'.format( - **global_vars)) - else: - # Legacy selection - if mse: - cmd = r'{BaseDir}\Installers\Extras\Security'.format(**global_vars) - cmd += r'\Microsoft Security Essentials.exe' - popen_program(cmd) - popen_program(r'{BaseDir}\Installers\Extras\Bundles\Legacy.exe'.format( - **global_vars)) + """Run Ninite file(s) based on OS version.""" + if global_vars['OS']['Version'] in ('8', '8.1', '10'): + # Modern selection + popen_program(r'{BaseDir}\Installers\Extras\Bundles\Modern.exe'.format( + **global_vars)) + else: + # Legacy selection + if mse: + cmd = r'{BaseDir}\Installers\Extras\Security'.format(**global_vars) + cmd += r'\Microsoft Security Essentials.exe' + popen_program(cmd) + popen_program(r'{BaseDir}\Installers\Extras\Bundles\Legacy.exe'.format( + **global_vars)) def install_vcredists(): - """Install all supported Visual C++ runtimes.""" - extract_item('_vcredists', silent=True) - prev_dir = os.getcwd() - try: - os.chdir(r'{BinDir}\_vcredists'.format(**global_vars)) - except FileNotFoundError: - # Ignored since the loop below will report the errors - pass - for vcr in VCR_REDISTS: - try_and_print(message=vcr['Name'], function=run_program, - cmd=vcr['Cmd'], other_results=OTHER_RESULTS) + """Install all supported Visual C++ runtimes.""" + extract_item('_vcredists', silent=True) + prev_dir = os.getcwd() + try: + os.chdir(r'{BinDir}\_vcredists'.format(**global_vars)) + except FileNotFoundError: + # Ignored since the loop below will report the errors + pass + for vcr in VCR_REDISTS: + try_and_print(message=vcr['Name'], function=run_program, + cmd=vcr['Cmd'], other_results=OTHER_RESULTS) - os.chdir(prev_dir) + os.chdir(prev_dir) # Misc def open_device_manager(): - popen_program(['mmc', 'devmgmt.msc']) + popen_program(['mmc', 'devmgmt.msc']) def open_windows_activation(): - popen_program(['slui']) + popen_program(['slui']) def open_windows_updates(): - popen_program(['control', '/name', 'Microsoft.WindowsUpdate']) + popen_program(['control', '/name', 'Microsoft.WindowsUpdate']) if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From fa424a4576169998aa91a5a4eeff11db27aa451f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:13:32 -0700 Subject: [PATCH 129/186] Updated update.py --- .bin/Scripts/functions/update.py | 1427 +++++++++++++++--------------- 1 file changed, 716 insertions(+), 711 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index b4068c22..21d2acee 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -9,947 +9,952 @@ from settings.music import * from settings.sources import * def compress_and_remove_item(item): - """Compress and delete an item unless an error is encountered.""" - try: - compress_item(item) - except: - raise GenericError - else: - remove_item(item.path) + """Compress and delete an item unless an error is encountered.""" + try: + compress_item(item) + except: + raise GenericError + else: + remove_item(item.path) def compress_item(item): - """Compress an item in a 7-Zip archive using the ARCHIVE_PASSWORD.""" - # Prep - prev_dir = os.getcwd() - dest = '{}.7z'.format(item.path) - wd = item.path - include_str = '*' - if os.path.isfile(wd): - wd = os.path.abspath(r'{}\{}'.format(wd, os.path.pardir)) - include_str = item.name - os.chdir(wd) + """Compress an item in a 7-Zip archive using the ARCHIVE_PASSWORD.""" + # Prep + prev_dir = os.getcwd() + dest = '{}.7z'.format(item.path) + wd = item.path + include_str = '*' + if os.path.isfile(wd): + wd = os.path.abspath(r'{}\{}'.format(wd, os.path.pardir)) + include_str = item.name + os.chdir(wd) - # Compress - cmd = [ - global_vars['Tools']['SevenZip'], - 'a', dest, - '-t7z', '-mx=7', '-myx=7', '-ms=on', '-mhe', '-bso0', '-bse0', - '-p{}'.format(ARCHIVE_PASSWORD), - include_str, - ] - run_program(cmd) + # Compress + cmd = [ + global_vars['Tools']['SevenZip'], + 'a', dest, + '-t7z', '-mx=7', '-myx=7', '-ms=on', '-mhe', '-bso0', '-bse0', + '-p{}'.format(ARCHIVE_PASSWORD), + include_str, + ] + run_program(cmd) - # Done - os.chdir(prev_dir) + # Done + os.chdir(prev_dir) def download_generic(out_dir, out_name, source_url): - """Downloads a file using requests.""" - ## Code based on this Q&A: https://stackoverflow.com/q/16694907 - ### Asked by: https://stackoverflow.com/users/427457/roman-podlinov - ### Edited by: https://stackoverflow.com/users/657427/christophe-roussy - ### Using answer: https://stackoverflow.com/a/39217788 - ### Answer from: https://stackoverflow.com/users/4323/john-zwinck - os.makedirs(out_dir, exist_ok=True) - out_path = '{}/{}'.format(out_dir, out_name) - try: - r = requests.get(source_url, stream=True) - with open(out_path, 'wb') as f: - shutil.copyfileobj(r.raw, f) - r.close() - except: - raise GenericError('Failed to download file.') + """Downloads a file using requests.""" + ## Code based on this Q&A: https://stackoverflow.com/q/16694907 + ### Asked by: https://stackoverflow.com/users/427457/roman-podlinov + ### Edited by: https://stackoverflow.com/users/657427/christophe-roussy + ### Using answer: https://stackoverflow.com/a/39217788 + ### Answer from: https://stackoverflow.com/users/4323/john-zwinck + os.makedirs(out_dir, exist_ok=True) + out_path = '{}/{}'.format(out_dir, out_name) + try: + r = requests.get(source_url, stream=True) + with open(out_path, 'wb') as f: + shutil.copyfileobj(r.raw, f) + r.close() + except: + raise GenericError('Failed to download file.') def download_to_temp(out_name, source_url): - """Download a file to the TmpDir.""" - download_generic(global_vars['TmpDir'], out_name, source_url) + """Download a file to the TmpDir.""" + download_generic(global_vars['TmpDir'], out_name, source_url) def extract_generic(source, dest, mode='x', sz_args=[]): - """Extract a file to a destination.""" - cmd = [ - global_vars['Tools']['SevenZip'], - mode, source, r'-o{}'.format(dest), - '-aoa', '-bso0', '-bse0', - ] - cmd.extend(sz_args) - run_program(cmd) + """Extract a file to a destination.""" + cmd = [ + global_vars['Tools']['SevenZip'], + mode, source, r'-o{}'.format(dest), + '-aoa', '-bso0', '-bse0', + ] + cmd.extend(sz_args) + run_program(cmd) def extract_temp_to_bin(source, item, mode='x', sz_args=[]): - """Extract a file to the .bin folder.""" - source = r'{}\{}'.format(global_vars['TmpDir'], source) - dest = r'{}\{}'.format(global_vars['BinDir'], item) - extract_generic(source, dest, mode, sz_args) + """Extract a file to the .bin folder.""" + source = r'{}\{}'.format(global_vars['TmpDir'], source) + dest = r'{}\{}'.format(global_vars['BinDir'], item) + extract_generic(source, dest, mode, sz_args) def extract_temp_to_cbin(source, item, mode='x', sz_args=[]): - """Extract a file to the .cbin folder.""" - source = r'{}\{}'.format(global_vars['TmpDir'], source) - dest = r'{}\{}'.format(global_vars['CBinDir'], item) - include_path = r'{}\_include\{}'.format(global_vars['CBinDir'], item) - if os.path.exists(include_path): - shutil.copytree(include_path, dest) - extract_generic(source, dest, mode, sz_args) + """Extract a file to the .cbin folder.""" + source = r'{}\{}'.format(global_vars['TmpDir'], source) + dest = r'{}\{}'.format(global_vars['CBinDir'], item) + include_path = r'{}\_include\{}'.format(global_vars['CBinDir'], item) + if os.path.exists(include_path): + shutil.copytree(include_path, dest) + extract_generic(source, dest, mode, sz_args) def generate_launcher(section, name, options): - """Generate a launcher script.""" - # Prep - dest = r'{}\{}'.format(global_vars['BaseDir'], section) - if section == '(Root)': - dest = global_vars['BaseDir'] - full_path = r'{}\{}.cmd'.format(dest, name) - template = r'{}\Scripts\Launcher_Template.cmd'.format(global_vars['BinDir']) + """Generate a launcher script.""" + # Prep + dest = r'{}\{}'.format(global_vars['BaseDir'], section) + if section == '(Root)': + dest = global_vars['BaseDir'] + full_path = r'{}\{}.cmd'.format(dest, name) + template = r'{}\Scripts\Launcher_Template.cmd'.format(global_vars['BinDir']) - # Format options - f_options = {} - for opt in options.keys(): - # Values need to be a list to support the multi-line extra code sections - if opt == 'Extra Code': - f_options['rem EXTRA_CODE'] = options['Extra Code'] - elif re.search(r'^L_\w+', opt, re.IGNORECASE): - new_opt = 'set {}='.format(opt) - f_options[new_opt] = ['set {}={}'.format(opt, options[opt])] + # Format options + f_options = {} + for opt in options.keys(): + # Values need to be a list to support the multi-line extra code sections + if opt == 'Extra Code': + f_options['rem EXTRA_CODE'] = options['Extra Code'] + elif re.search(r'^L_\w+', opt, re.IGNORECASE): + new_opt = 'set {}='.format(opt) + f_options[new_opt] = ['set {}={}'.format(opt, options[opt])] - # Read template and update using f_options - out_text = [] - with open(template, 'r') as f: - for line in f.readlines(): - # Strip all lines to let Python handle/correct the CRLF endings - line = line.strip() - if line in f_options: - # Extend instead of append to support extra code sections - out_text.extend(f_options[line]) - else: - out_text.append(line) + # Read template and update using f_options + out_text = [] + with open(template, 'r') as f: + for line in f.readlines(): + # Strip all lines to let Python handle/correct the CRLF endings + line = line.strip() + if line in f_options: + # Extend instead of append to support extra code sections + out_text.extend(f_options[line]) + else: + out_text.append(line) - # Write file - os.makedirs(dest, exist_ok=True) - with open(full_path, 'w') as f: - # f.writelines(out_text) - f.write('\n'.join(out_text)) + # Write file + os.makedirs(dest, exist_ok=True) + with open(full_path, 'w') as f: + # f.writelines(out_text) + f.write('\n'.join(out_text)) def remove_item(item_path): - """Delete a file or folder.""" - if os.path.exists(item_path): - if os.path.isdir(item_path): - shutil.rmtree(item_path, ignore_errors=True) - else: - os.remove(item_path) + """Delete a file or folder.""" + if os.path.exists(item_path): + if os.path.isdir(item_path): + shutil.rmtree(item_path, ignore_errors=True) + else: + os.remove(item_path) def remove_from_kit(item): - """Delete a file or folder from the .bin/.cbin folders.""" - item_locations = [] - for p in [global_vars['BinDir'], global_vars['CBinDir']]: - item_locations.append(r'{}\{}'.format(p, item)) - item_locations.append(r'{}\{}.7z'.format(p, item)) - item_locations.append(r'{}\_Drivers\{}'.format(p, item)) - item_locations.append(r'{}\_Drivers\{}.7z'.format(p, item)) - for item_path in item_locations: - remove_item(item_path) - -def remove_from_temp(item): - """Delete a file or folder from the TmpDir folder.""" - item_path = r'{}\{}'.format(global_vars['TmpDir'], item) + """Delete a file or folder from the .bin/.cbin folders.""" + item_locations = [] + for p in [global_vars['BinDir'], global_vars['CBinDir']]: + item_locations.append(r'{}\{}'.format(p, item)) + item_locations.append(r'{}\{}.7z'.format(p, item)) + item_locations.append(r'{}\_Drivers\{}'.format(p, item)) + item_locations.append(r'{}\_Drivers\{}.7z'.format(p, item)) + for item_path in item_locations: remove_item(item_path) +def remove_from_temp(item): + """Delete a file or folder from the TmpDir folder.""" + item_path = r'{}\{}'.format(global_vars['TmpDir'], item) + remove_item(item_path) + def resolve_dynamic_url(source_url, regex): - """Scan source_url for a url using the regex provided; returns str.""" - # Load the download page - try: - download_page = requests.get(source_url) - except Exception: - # "Fail silently as the download_to_temp() function will catch it - return None + """Scan source_url for a url using the regex provided; returns str.""" + # Load the download page + try: + download_page = requests.get(source_url) + except Exception: + # "Fail silently as the download_to_temp() function will catch it + return None - # Scan for the url using the regex provided - url = None - for line in download_page.content.decode().splitlines(): - if re.search(regex, line): - url = line.strip() - url = re.sub(r'.*(a |)href="([^"]+)".*', r'\2', url) - url = re.sub(r".*(a |)href='([^']+)'.*", r'\2', url) - break + # Scan for the url using the regex provided + url = None + for line in download_page.content.decode().splitlines(): + if re.search(regex, line): + url = line.strip() + url = re.sub(r'.*(a |)href="([^"]+)".*', r'\2', url) + url = re.sub(r".*(a |)href='([^']+)'.*", r'\2', url) + break - # Return - return url + # Return + return url def scan_for_net_installers(server, family_name, min_year): - """Scan network shares for installers.""" - if not server['Mounted']: - mount_network_share(server) + """Scan network shares for installers.""" + if not server['Mounted']: + mount_network_share(server) - if server['Mounted']: - for year in os.scandir(r'\\{IP}\{Share}'.format(**server)): - try: - year_ok = int(year.name) < min_year - except ValueError: - year_ok = False # Skip non-year items - if year_ok: - # Don't support outdated installers - continue - for version in os.scandir(year.path): - section = r'Installers\Extras\{}\{}'.format( - family_name, year.name) - if section not in LAUNCHERS: - LAUNCHERS[section] = {} - name = version.name - if re.search(r'(exe|msi)$', name, re.IGNORECASE): - name = name[:-4] - if name not in LAUNCHERS[section]: - LAUNCHERS[section][name] = { - 'L_TYPE': family_name, - 'L_PATH': year.name, - 'L_ITEM': version.name, - } - umount_network_share(server) + if server['Mounted']: + for year in os.scandir(r'\\{IP}\{Share}'.format(**server)): + try: + year_ok = int(year.name) < min_year + except ValueError: + year_ok = False # Skip non-year items + if year_ok: + # Don't support outdated installers + continue + for version in os.scandir(year.path): + section = r'Installers\Extras\{}\{}'.format( + family_name, year.name) + if section not in LAUNCHERS: + LAUNCHERS[section] = {} + name = version.name + if re.search(r'(exe|msi)$', name, re.IGNORECASE): + name = name[:-4] + if name not in LAUNCHERS[section]: + LAUNCHERS[section][name] = { + 'L_TYPE': family_name, + 'L_PATH': year.name, + 'L_ITEM': version.name, + } + umount_network_share(server) ## Data Recovery ## def update_testdisk(): - # Stop running processes - for exe in ['fidentify_win.exe', 'photorec_win.exe', - 'qphotorec_win.exe', 'testdisk_win.exe']: - kill_process(exe) + # Stop running processes + for exe in ['fidentify_win.exe', 'photorec_win.exe', + 'qphotorec_win.exe', 'testdisk_win.exe']: + kill_process(exe) - # Remove existing folders - remove_from_kit('TestDisk') + # Remove existing folders + remove_from_kit('TestDisk') - # Download - download_to_temp('testdisk_wip.zip', SOURCE_URLS['TestDisk']) + # Download + download_to_temp('testdisk_wip.zip', SOURCE_URLS['TestDisk']) - # Extract files - extract_temp_to_cbin('testdisk_wip.zip', 'TestDisk') - dest = r'{}\TestDisk'.format(global_vars['CBinDir']) - for item in os.scandir(r'{}\testdisk-7.1-WIP'.format(dest)): - dest_item = '{}\{}'.format(dest, item.name) - if not os.path.exists(dest_item): - shutil.move(item.path, dest_item) - shutil.rmtree( - r'{}\TestDisk\testdisk-7.1-WIP'.format(global_vars['CBinDir'])) + # Extract files + extract_temp_to_cbin('testdisk_wip.zip', 'TestDisk') + dest = r'{}\TestDisk'.format(global_vars['CBinDir']) + for item in os.scandir(r'{}\testdisk-7.1-WIP'.format(dest)): + dest_item = '{}\{}'.format(dest, item.name) + if not os.path.exists(dest_item): + shutil.move(item.path, dest_item) + shutil.rmtree( + r'{}\TestDisk\testdisk-7.1-WIP'.format(global_vars['CBinDir'])) - # Cleanup - remove_from_temp('testdisk_wip.zip') + # Cleanup + remove_from_temp('testdisk_wip.zip') ## Data Transfers ## def update_fastcopy(): - ## NOTE: Lives in .bin uncompressed - # Stop running processes - for process in ['FastCopy.exe', 'FastCopy64.exe']: - kill_process(process) + ## NOTE: Lives in .bin uncompressed + # Stop running processes + for process in ['FastCopy.exe', 'FastCopy64.exe']: + kill_process(process) - # Remove existing folders - remove_from_kit('FastCopy') + # Remove existing folders + remove_from_kit('FastCopy') - # Download - download_to_temp('FastCopy.zip', SOURCE_URLS['FastCopy']) + # Download + download_to_temp('FastCopy.zip', SOURCE_URLS['FastCopy']) - # Extract installer - extract_temp_to_bin('FastCopy.zip', 'FastCopy') - _path = r'{}\FastCopy'.format(global_vars['BinDir']) - _installer = 'FastCopy354_installer.exe' + # Extract installer + extract_temp_to_bin('FastCopy.zip', 'FastCopy') + _path = r'{}\FastCopy'.format(global_vars['BinDir']) + _installer = 'FastCopy354_installer.exe' - # Extract 64-bit - cmd = [ - r'{}\{}'.format(_path, _installer), - '/NOSUBDIR', '/DIR={}'.format(_path), - '/EXTRACT64'] - run_program(cmd) - shutil.move( - r'{}\FastCopy\FastCopy.exe'.format(global_vars['BinDir']), - r'{}\FastCopy\FastCopy64.exe'.format(global_vars['BinDir'])) + # Extract 64-bit + cmd = [ + r'{}\{}'.format(_path, _installer), + '/NOSUBDIR', '/DIR={}'.format(_path), + '/EXTRACT64'] + run_program(cmd) + shutil.move( + r'{}\FastCopy\FastCopy.exe'.format(global_vars['BinDir']), + r'{}\FastCopy\FastCopy64.exe'.format(global_vars['BinDir'])) - # Extract 32-bit - cmd = [ - r'{}\{}'.format(_path, _installer), - '/NOSUBDIR', '/DIR={}'.format(_path), - '/EXTRACT32'] - run_program(cmd) + # Extract 32-bit + cmd = [ + r'{}\{}'.format(_path, _installer), + '/NOSUBDIR', '/DIR={}'.format(_path), + '/EXTRACT32'] + run_program(cmd) - # Cleanup - os.remove(r'{}\{}'.format(_path, _installer)) - os.remove(r'{}\setup.exe'.format(_path, _installer)) - remove_from_temp('FastCopy.zip') + # Cleanup + os.remove(r'{}\{}'.format(_path, _installer)) + os.remove(r'{}\setup.exe'.format(_path, _installer)) + remove_from_temp('FastCopy.zip') def update_wimlib(): - # Stop running processes - kill_process('wimlib-imagex.exe') + # Stop running processes + kill_process('wimlib-imagex.exe') - # Remove existing folders - remove_from_kit('wimlib') + # Remove existing folders + remove_from_kit('wimlib') - # Download - download_to_temp('wimlib32.zip', SOURCE_URLS['wimlib32']) - download_to_temp('wimlib64.zip', SOURCE_URLS['wimlib64']) + # Download + download_to_temp('wimlib32.zip', SOURCE_URLS['wimlib32']) + download_to_temp('wimlib64.zip', SOURCE_URLS['wimlib64']) - # Extract - extract_generic( - r'{}\wimlib32.zip'.format(global_vars['TmpDir']), - r'{}\wimlib\x32'.format(global_vars['CBinDir'])) - extract_generic( - r'{}\wimlib64.zip'.format(global_vars['TmpDir']), - r'{}\wimlib\x64'.format(global_vars['CBinDir'])) + # Extract + extract_generic( + r'{}\wimlib32.zip'.format(global_vars['TmpDir']), + r'{}\wimlib\x32'.format(global_vars['CBinDir'])) + extract_generic( + r'{}\wimlib64.zip'.format(global_vars['TmpDir']), + r'{}\wimlib\x64'.format(global_vars['CBinDir'])) - # Cleanup - remove_from_temp('wimlib32.zip') - remove_from_temp('wimlib64.zip') + # Cleanup + remove_from_temp('wimlib32.zip') + remove_from_temp('wimlib64.zip') def update_xyplorer(): - # Stop running processes - kill_process('XYplorerFree.exe') + # Stop running processes + kill_process('XYplorerFree.exe') - # Remove existing folders - remove_from_kit('XYplorerFree') + # Remove existing folders + remove_from_kit('XYplorerFree') - # Download - download_to_temp('xyplorer_free.zip', SOURCE_URLS['XYplorerFree']) + # Download + download_to_temp('xyplorer_free.zip', SOURCE_URLS['XYplorerFree']) - # Extract files - extract_temp_to_cbin('xyplorer_free.zip', 'XYplorerFree') + # Extract files + extract_temp_to_cbin('xyplorer_free.zip', 'XYplorerFree') - # Cleanup - remove_from_temp('xyplorer_free.zip') + # Cleanup + remove_from_temp('xyplorer_free.zip') ## Diagnostics ## def update_aida64(): - # Stop running processes - kill_process('notepadplusplus.exe') + # Stop running processes + kill_process('notepadplusplus.exe') - # Remove existing folders - remove_from_kit('AIDA64') + # Remove existing folders + remove_from_kit('AIDA64') - # Download - download_to_temp('aida64.zip', SOURCE_URLS['AIDA64']) + # Download + download_to_temp('aida64.zip', SOURCE_URLS['AIDA64']) - # Extract files - extract_temp_to_cbin('aida64.zip', 'AIDA64') + # Extract files + extract_temp_to_cbin('aida64.zip', 'AIDA64') - # Cleanup - remove_from_temp('aida64.zip') + # Cleanup + remove_from_temp('aida64.zip') def update_autoruns(): - # Stop running processes - kill_process('Autoruns.exe') - kill_process('Autoruns64.exe') + # Stop running processes + kill_process('Autoruns.exe') + kill_process('Autoruns64.exe') - # Remove existing folders - remove_from_kit('Autoruns') + # Remove existing folders + remove_from_kit('Autoruns') - # Download - download_to_temp('Autoruns.zip', SOURCE_URLS['Autoruns']) + # Download + download_to_temp('Autoruns.zip', SOURCE_URLS['Autoruns']) - # Extract files - extract_temp_to_cbin('Autoruns.zip', 'Autoruns') + # Extract files + extract_temp_to_cbin('Autoruns.zip', 'Autoruns') - # Cleanup - remove_from_temp('Autoruns.zip') + # Cleanup + remove_from_temp('Autoruns.zip') def update_bleachbit(): - # Stop running processes - kill_process('bleachbit.exe') + # Stop running processes + kill_process('bleachbit.exe') - # Remove existing folders - remove_from_kit('BleachBit') + # Remove existing folders + remove_from_kit('BleachBit') - # Download - download_to_temp('bleachbit.zip', SOURCE_URLS['BleachBit']) - download_to_temp('Winapp2.zip', SOURCE_URLS['Winapp2']) + # Download + download_to_temp('bleachbit.zip', SOURCE_URLS['BleachBit']) + download_to_temp('Winapp2.zip', SOURCE_URLS['Winapp2']) - # Extract files - extract_temp_to_cbin('bleachbit.zip', 'BleachBit') - extract_generic( - r'{}\Winapp2.zip'.format(global_vars['TmpDir']), - r'{}\BleachBit\cleaners'.format(global_vars['CBinDir']), - mode='e', sz_args=[r'Winapp2-master\Non-CCleaner\Winapp2.ini']) + # Extract files + extract_temp_to_cbin('bleachbit.zip', 'BleachBit') + extract_generic( + r'{}\Winapp2.zip'.format(global_vars['TmpDir']), + r'{}\BleachBit\cleaners'.format(global_vars['CBinDir']), + mode='e', sz_args=[r'Winapp2-master\Non-CCleaner\Winapp2.ini']) - # Move files into place - dest = r'{}\BleachBit'.format(global_vars['CBinDir']) - for item in os.scandir(r'{}\BleachBit-Portable'.format(dest)): - dest_item = '{}\{}'.format(dest, item.name) - if not os.path.exists(dest_item): - shutil.move(item.path, dest_item) - shutil.rmtree( - r'{}\BleachBit\BleachBit-Portable'.format(global_vars['CBinDir'])) + # Move files into place + dest = r'{}\BleachBit'.format(global_vars['CBinDir']) + for item in os.scandir(r'{}\BleachBit-Portable'.format(dest)): + dest_item = '{}\{}'.format(dest, item.name) + if not os.path.exists(dest_item): + shutil.move(item.path, dest_item) + shutil.rmtree( + r'{}\BleachBit\BleachBit-Portable'.format(global_vars['CBinDir'])) - # Cleanup - remove_from_temp('bleachbit.zip') - remove_from_temp('Winapp2.zip') + # Cleanup + remove_from_temp('bleachbit.zip') + remove_from_temp('Winapp2.zip') def update_bluescreenview(): - # Stop running processes - for exe in ['BlueScreenView.exe', 'BlueScreenView64.exe']: - kill_process(exe) + # Stop running processes + for exe in ['BlueScreenView.exe', 'BlueScreenView64.exe']: + kill_process(exe) - # Remove existing folders - remove_from_kit('BlueScreenView') + # Remove existing folders + remove_from_kit('BlueScreenView') - # Download - download_to_temp('bluescreenview32.zip', SOURCE_URLS['BlueScreenView32']) - download_to_temp('bluescreenview64.zip', SOURCE_URLS['BlueScreenView64']) + # Download + download_to_temp('bluescreenview32.zip', SOURCE_URLS['BlueScreenView32']) + download_to_temp('bluescreenview64.zip', SOURCE_URLS['BlueScreenView64']) - # Extract files - extract_temp_to_cbin('bluescreenview64.zip', 'BlueScreenView', sz_args=['BlueScreenView.exe']) - shutil.move( - r'{}\BlueScreenView\BlueScreenView.exe'.format(global_vars['CBinDir']), - r'{}\BlueScreenView\BlueScreenView64.exe'.format(global_vars['CBinDir'])) - extract_temp_to_cbin('bluescreenview32.zip', 'BlueScreenView') + # Extract files + extract_temp_to_cbin( + 'bluescreenview64.zip', 'BlueScreenView', sz_args=['BlueScreenView.exe']) + shutil.move( + r'{}\BlueScreenView\BlueScreenView.exe'.format(global_vars['CBinDir']), + r'{}\BlueScreenView\BlueScreenView64.exe'.format(global_vars['CBinDir'])) + extract_temp_to_cbin('bluescreenview32.zip', 'BlueScreenView') - # Cleanup - remove_from_temp('bluescreenview32.zip') - remove_from_temp('bluescreenview64.zip') + # Cleanup + remove_from_temp('bluescreenview32.zip') + remove_from_temp('bluescreenview64.zip') def update_erunt(): - # Stop running processes - kill_process('ERUNT.EXE') + # Stop running processes + kill_process('ERUNT.EXE') - # Remove existing folders - remove_from_kit('ERUNT') + # Remove existing folders + remove_from_kit('ERUNT') - # Download - download_to_temp('erunt.zip', SOURCE_URLS['ERUNT']) + # Download + download_to_temp('erunt.zip', SOURCE_URLS['ERUNT']) - # Extract files - extract_temp_to_cbin('erunt.zip', 'ERUNT') + # Extract files + extract_temp_to_cbin('erunt.zip', 'ERUNT') - # Cleanup - remove_from_temp('erunt.zip') + # Cleanup + remove_from_temp('erunt.zip') def update_hitmanpro(): - # Stop running processes - for exe in ['HitmanPro.exe', 'HitmanPro64.exe']: - kill_process(exe) + # Stop running processes + for exe in ['HitmanPro.exe', 'HitmanPro64.exe']: + kill_process(exe) - # Remove existing folders - remove_from_kit('HitmanPro') + # Remove existing folders + remove_from_kit('HitmanPro') - # Download - dest = r'{}\HitmanPro'.format(global_vars['CBinDir']) - download_generic(dest, 'HitmanPro.exe', SOURCE_URLS['HitmanPro32']) - download_generic(dest, 'HitmanPro64.exe', SOURCE_URLS['HitmanPro64']) + # Download + dest = r'{}\HitmanPro'.format(global_vars['CBinDir']) + download_generic(dest, 'HitmanPro.exe', SOURCE_URLS['HitmanPro32']) + download_generic(dest, 'HitmanPro64.exe', SOURCE_URLS['HitmanPro64']) def update_hwinfo(): - ## NOTE: Lives in .bin uncompressed - # Stop running processes - for exe in ['HWiNFO32.exe', 'HWiNFO64.exe']: - kill_process(exe) + ## NOTE: Lives in .bin uncompressed + # Stop running processes + for exe in ['HWiNFO32.exe', 'HWiNFO64.exe']: + kill_process(exe) - # Download - download_to_temp('HWiNFO.zip', SOURCE_URLS['HWiNFO']) + # Download + download_to_temp('HWiNFO.zip', SOURCE_URLS['HWiNFO']) - # Extract files - extract_temp_to_bin('HWiNFO.zip', 'HWiNFO') + # Extract files + extract_temp_to_bin('HWiNFO.zip', 'HWiNFO') - # Cleanup - remove_from_temp('HWiNFO.zip') + # Cleanup + remove_from_temp('HWiNFO.zip') def update_nircmd(): - # Stop running processes - for exe in ['nircmdc.exe', 'nircmdc64.exe']: - kill_process(exe) + # Stop running processes + for exe in ['nircmdc.exe', 'nircmdc64.exe']: + kill_process(exe) - # Remove existing folders - remove_from_kit('NirCmd') + # Remove existing folders + remove_from_kit('NirCmd') - # Download - download_to_temp('nircmd32.zip', SOURCE_URLS['NirCmd32']) - download_to_temp('nircmd64.zip', SOURCE_URLS['NirCmd64']) + # Download + download_to_temp('nircmd32.zip', SOURCE_URLS['NirCmd32']) + download_to_temp('nircmd64.zip', SOURCE_URLS['NirCmd64']) - # Extract files - extract_temp_to_cbin('nircmd64.zip', 'NirCmd', sz_args=['nircmdc.exe']) - shutil.move( - r'{}\NirCmd\nircmdc.exe'.format(global_vars['CBinDir']), - r'{}\NirCmd\nircmdc64.exe'.format(global_vars['CBinDir'])) - extract_temp_to_cbin('nircmd32.zip', 'NirCmd', sz_args=['nircmdc.exe']) + # Extract files + extract_temp_to_cbin('nircmd64.zip', 'NirCmd', sz_args=['nircmdc.exe']) + shutil.move( + r'{}\NirCmd\nircmdc.exe'.format(global_vars['CBinDir']), + r'{}\NirCmd\nircmdc64.exe'.format(global_vars['CBinDir'])) + extract_temp_to_cbin('nircmd32.zip', 'NirCmd', sz_args=['nircmdc.exe']) - # Cleanup - remove_from_temp('nircmd32.zip') - remove_from_temp('nircmd64.zip') + # Cleanup + remove_from_temp('nircmd32.zip') + remove_from_temp('nircmd64.zip') def update_produkey(): - # Stop running processes - for exe in ['ProduKey.exe', 'ProduKey64.exe']: - kill_process(exe) + # Stop running processes + for exe in ['ProduKey.exe', 'ProduKey64.exe']: + kill_process(exe) - # Remove existing folders - remove_from_kit('ProduKey') + # Remove existing folders + remove_from_kit('ProduKey') - # Download - download_to_temp('produkey32.zip', SOURCE_URLS['ProduKey32']) - download_to_temp('produkey64.zip', SOURCE_URLS['ProduKey64']) + # Download + download_to_temp('produkey32.zip', SOURCE_URLS['ProduKey32']) + download_to_temp('produkey64.zip', SOURCE_URLS['ProduKey64']) - # Extract files - extract_temp_to_cbin('produkey64.zip', 'ProduKey', sz_args=['ProduKey.exe']) - shutil.move( - r'{}\ProduKey\ProduKey.exe'.format(global_vars['CBinDir']), - r'{}\ProduKey\ProduKey64.exe'.format(global_vars['CBinDir'])) - extract_temp_to_cbin('produkey32.zip', 'ProduKey') + # Extract files + extract_temp_to_cbin('produkey64.zip', 'ProduKey', sz_args=['ProduKey.exe']) + shutil.move( + r'{}\ProduKey\ProduKey.exe'.format(global_vars['CBinDir']), + r'{}\ProduKey\ProduKey64.exe'.format(global_vars['CBinDir'])) + extract_temp_to_cbin('produkey32.zip', 'ProduKey') - # Cleanup - remove_from_temp('produkey32.zip') - remove_from_temp('produkey64.zip') + # Cleanup + remove_from_temp('produkey32.zip') + remove_from_temp('produkey64.zip') ## Drivers ## def update_intel_rst(): - # Remove existing folders - remove_from_kit('Intel RST') + # Remove existing folders + remove_from_kit('Intel RST') - # Prep - dest = r'{}\_Drivers\Intel RST'.format(global_vars['CBinDir']) - include_path = r'{}\_include\_Drivers\Intel RST'.format( - global_vars['CBinDir']) - if os.path.exists(include_path): - shutil.copytree(include_path, dest) + # Prep + dest = r'{}\_Drivers\Intel RST'.format(global_vars['CBinDir']) + include_path = r'{}\_include\_Drivers\Intel RST'.format( + global_vars['CBinDir']) + if os.path.exists(include_path): + shutil.copytree(include_path, dest) - # Download - for name, url in RST_SOURCES.items(): - download_generic(dest, name, url) + # Download + for name, url in RST_SOURCES.items(): + download_generic(dest, name, url) def update_intel_ssd_toolbox(): - # Remove existing folders - remove_from_kit('Intel SSD Toolbox.exe') + # Remove existing folders + remove_from_kit('Intel SSD Toolbox.exe') - # Download - download_generic( - r'{}\_Drivers\Intel SSD Toolbox'.format(global_vars['CBinDir']), - 'Intel SSD Toolbox.exe', - SOURCE_URLS['Intel SSD Toolbox']) + # Download + download_generic( + r'{}\_Drivers\Intel SSD Toolbox'.format(global_vars['CBinDir']), + 'Intel SSD Toolbox.exe', + SOURCE_URLS['Intel SSD Toolbox']) def update_samsung_magician(): - # Remove existing folders - remove_from_kit('Samsung Magician.exe') + # Remove existing folders + remove_from_kit('Samsung Magician.exe') - # Download - download_to_temp('Samsung Magician.zip', SOURCE_URLS['Samsung Magician']) + # Download + download_to_temp('Samsung Magician.zip', SOURCE_URLS['Samsung Magician']) - # Extract - extract_temp_to_cbin('Samsung Magician.zip', '_Drivers\Samsung Magician') - shutil.move( - r'{}\_Drivers\Samsung Magician\Samsung_Magician_Installer.exe'.format( - global_vars['CBinDir']), - r'{}\_Drivers\Samsung Magician\Samsung Magician.exe'.format( - global_vars['CBinDir'])) + # Extract + extract_temp_to_cbin('Samsung Magician.zip', '_Drivers\Samsung Magician') + shutil.move( + r'{}\_Drivers\Samsung Magician\Samsung_Magician_Installer.exe'.format( + global_vars['CBinDir']), + r'{}\_Drivers\Samsung Magician\Samsung Magician.exe'.format( + global_vars['CBinDir'])) - # Cleanup - remove_from_temp('Samsung Magician.zip') + # Cleanup + remove_from_temp('Samsung Magician.zip') def update_sdi_origin(): - # Download aria2 - download_to_temp('aria2.zip', SOURCE_URLS['aria2']) - aria_source = r'{}\aria2.zip'.format(global_vars['TmpDir']) - aria_dest = r'{}\aria2'.format(global_vars['TmpDir']) - aria = r'{}\aria2c.exe'.format(aria_dest) - extract_generic(aria_source, aria_dest, mode='e') + # Download aria2 + download_to_temp('aria2.zip', SOURCE_URLS['aria2']) + aria_source = r'{}\aria2.zip'.format(global_vars['TmpDir']) + aria_dest = r'{}\aria2'.format(global_vars['TmpDir']) + aria = r'{}\aria2c.exe'.format(aria_dest) + extract_generic(aria_source, aria_dest, mode='e') - # Prep for torrent download - download_to_temp('sdio.torrent', SOURCE_URLS['SDIO Torrent']) - sdio_torrent = r'{}\sdio.torrent'.format(global_vars['TmpDir']) - out = run_program([aria, sdio_torrent, '-S']) - indexes = [] - for line in out.stdout.decode().splitlines(): - r = re.search(r'^\s*(\d+)\|(.*)', line) - if r and not re.search(r'(\.(bat|inf)|Video|Server|Printer|XP)', line, re.IGNORECASE): - indexes.append(int(r.group(1))) - indexes = [str(i) for i in sorted(indexes)] + # Prep for torrent download + download_to_temp('sdio.torrent', SOURCE_URLS['SDIO Torrent']) + sdio_torrent = r'{}\sdio.torrent'.format(global_vars['TmpDir']) + out = run_program([aria, sdio_torrent, '-S']) + indexes = [] + for line in out.stdout.decode().splitlines(): + r = re.search(r'^\s*(\d+)\|(.*)', line) + if (r and not re.search( + r'(\.(bat|inf)|Video|Server|Printer|XP)', line, re.IGNORECASE)): + indexes.append(int(r.group(1))) + indexes = [str(i) for i in sorted(indexes)] - # Download SDI Origin - cmd = [ - aria, - '--select-file={}'.format(','.join(indexes)), - '-d', aria_dest, - '--seed-time=0', - sdio_torrent, - '-new_console:n', '-new_console:s33V', - ] - run_program(cmd, pipe=False, check=False, shell=True) - sleep(1) - wait_for_process('aria2c') + # Download SDI Origin + cmd = [ + aria, + '--select-file={}'.format(','.join(indexes)), + '-d', aria_dest, + '--seed-time=0', + sdio_torrent, + '-new_console:n', '-new_console:s33V', + ] + run_program(cmd, pipe=False, check=False, shell=True) + sleep(1) + wait_for_process('aria2c') - # Download SDI Origin extra themes - download_to_temp('sdio_themes.zip', SOURCE_URLS['SDIO Themes']) - theme_source = r'{}\sdio_themes.zip'.format(global_vars['TmpDir']) - theme_dest = r'{}\SDIO_Update\tools\SDI\themes'.format(aria_dest) - extract_generic(theme_source, theme_dest) + # Download SDI Origin extra themes + download_to_temp('sdio_themes.zip', SOURCE_URLS['SDIO Themes']) + theme_source = r'{}\sdio_themes.zip'.format(global_vars['TmpDir']) + theme_dest = r'{}\SDIO_Update\tools\SDI\themes'.format(aria_dest) + extract_generic(theme_source, theme_dest) - # Move files into place - for item in os.scandir(r'{}\SDIO_Update'.format(aria_dest)): - dest_item = '{}\_Drivers\SDIO\{}'.format( - global_vars['BinDir'], item.name) - r = re.search(r'^SDIO_x?(64|)_?R.*exe$', item.name, re.IGNORECASE) - if r: - dest_item = dest_item.replace(item.name, 'SDIO{}.exe'.format( - r.group(1))) - if (not os.path.exists(dest_item) - and not re.search(r'\.(inf|bat)$', item.name, re.IGNORECASE)): - shutil.move(item.path, dest_item) + # Move files into place + for item in os.scandir(r'{}\SDIO_Update'.format(aria_dest)): + dest_item = '{}\_Drivers\SDIO\{}'.format( + global_vars['BinDir'], item.name) + r = re.search(r'^SDIO_x?(64|)_?R.*exe$', item.name, re.IGNORECASE) + if r: + dest_item = dest_item.replace(item.name, 'SDIO{}.exe'.format( + r.group(1))) + if (not os.path.exists(dest_item) + and not re.search(r'\.(inf|bat)$', item.name, re.IGNORECASE)): + shutil.move(item.path, dest_item) - # Cleanup - remove_from_temp('aria2') - remove_from_temp('aria2.zip') - remove_from_temp('sdio.torrent') - remove_from_temp('sdio_themes.zip') + # Cleanup + remove_from_temp('aria2') + remove_from_temp('aria2.zip') + remove_from_temp('sdio.torrent') + remove_from_temp('sdio_themes.zip') ## Installers ## def update_adobe_reader_dc(): - # Prep - dest = r'{}\Installers\Extras\Office'.format( - global_vars['BaseDir']) + # Prep + dest = r'{}\Installers\Extras\Office'.format( + global_vars['BaseDir']) - # Remove existing installer - try: - os.remove(r'{}\Adobe Reader DC.exe'.format(dest)) - except FileNotFoundError: - pass + # Remove existing installer + try: + os.remove(r'{}\Adobe Reader DC.exe'.format(dest)) + except FileNotFoundError: + pass - # Download - download_generic( - dest, 'Adobe Reader DC.exe', SOURCE_URLS['Adobe Reader DC']) + # Download + download_generic( + dest, 'Adobe Reader DC.exe', SOURCE_URLS['Adobe Reader DC']) def update_macs_fan_control(): - # Prep - dest = r'{}\Installers'.format( - global_vars['BaseDir']) + # Prep + dest = r'{}\Installers'.format( + global_vars['BaseDir']) - # Remove existing installer - try: - os.remove(r'{}\Macs Fan Control.exe'.format(dest)) - except FileNotFoundError: - pass + # Remove existing installer + try: + os.remove(r'{}\Macs Fan Control.exe'.format(dest)) + except FileNotFoundError: + pass - # Download - download_generic( - dest, 'Macs Fan Control.exe', SOURCE_URLS['Macs Fan Control']) + # Download + download_generic( + dest, 'Macs Fan Control.exe', SOURCE_URLS['Macs Fan Control']) def update_office(): - # Remove existing folders - remove_from_kit('_Office') + # Remove existing folders + remove_from_kit('_Office') - # Prep - dest = r'{}\_Office'.format(global_vars['CBinDir']) - include_path = r'{}\_include\_Office'.format(global_vars['CBinDir']) - if os.path.exists(include_path): - shutil.copytree(include_path, dest) + # Prep + dest = r'{}\_Office'.format(global_vars['CBinDir']) + include_path = r'{}\_include\_Office'.format(global_vars['CBinDir']) + if os.path.exists(include_path): + shutil.copytree(include_path, dest) - for year in ['2016']: - # Download and extract - name = 'odt{}.exe'.format(year) - url = 'Office Deployment Tool {}'.format(year) - download_to_temp(name, SOURCE_URLS[url]) - cmd = [ - r'{}\odt{}.exe'.format(global_vars['TmpDir'], year), - r'/extract:{}\{}'.format(global_vars['TmpDir'], year), - '/quiet', - ] - run_program(cmd) - shutil.move( - r'{}\{}'.format(global_vars['TmpDir'], year), - r'{}\_Office\{}'.format(global_vars['CBinDir'], year)) + for year in ['2016']: + # Download and extract + name = 'odt{}.exe'.format(year) + url = 'Office Deployment Tool {}'.format(year) + download_to_temp(name, SOURCE_URLS[url]) + cmd = [ + r'{}\odt{}.exe'.format(global_vars['TmpDir'], year), + r'/extract:{}\{}'.format(global_vars['TmpDir'], year), + '/quiet', + ] + run_program(cmd) + shutil.move( + r'{}\{}'.format(global_vars['TmpDir'], year), + r'{}\_Office\{}'.format(global_vars['CBinDir'], year)) - # Cleanup - remove_from_temp('odt{}.exe'.format(year)) + # Cleanup + remove_from_temp('odt{}.exe'.format(year)) def update_classic_start_skin(): - # Remove existing folders - remove_from_kit('ClassicStartSkin') + # Remove existing folders + remove_from_kit('ClassicStartSkin') - # Download - download_generic( - r'{}\ClassicStartSkin'.format(global_vars['CBinDir']), - 'Metro-Win10-Black.skin7', - SOURCE_URLS['ClassicStartSkin']) + # Download + download_generic( + r'{}\ClassicStartSkin'.format(global_vars['CBinDir']), + 'Metro-Win10-Black.skin7', + SOURCE_URLS['ClassicStartSkin']) def update_vcredists(): - # Remove existing folders - remove_from_kit('_vcredists') + # Remove existing folders + remove_from_kit('_vcredists') - # Prep - dest = r'{}\_vcredists'.format(global_vars['CBinDir']) - include_path = r'{}\_include\_vcredists'.format(global_vars['CBinDir']) - if os.path.exists(include_path): - shutil.copytree(include_path, dest) + # Prep + dest = r'{}\_vcredists'.format(global_vars['CBinDir']) + include_path = r'{}\_include\_vcredists'.format(global_vars['CBinDir']) + if os.path.exists(include_path): + shutil.copytree(include_path, dest) - # Download - for year in VCREDIST_SOURCES.keys(): - for bit in ['32', '64']: - dest = r'{}\_vcredists\{}\x{}'.format( - global_vars['CBinDir'], year, bit) - download_generic( - dest, - 'vcredist.exe', - VCREDIST_SOURCES[year][bit]) + # Download + for year in VCREDIST_SOURCES.keys(): + for bit in ['32', '64']: + dest = r'{}\_vcredists\{}\x{}'.format( + global_vars['CBinDir'], year, bit) + download_generic( + dest, + 'vcredist.exe', + VCREDIST_SOURCES[year][bit]) def update_one_ninite(section, dest, name, url, indent=8, width=40): - # Prep - url = 'https://ninite.com/{}/ninite.exe'.format(url) + # Prep + url = 'https://ninite.com/{}/ninite.exe'.format(url) - # Download - download_generic(out_dir=dest, out_name=name, source_url=url) + # Download + download_generic(out_dir=dest, out_name=name, source_url=url) - # Copy to Installers folder - installer_parent = r'{}\Installers\Extras\{}'.format( - global_vars['BaseDir'], section) - installer_dest = r'{}\{}'.format(installer_parent, name) - os.makedirs(installer_parent, exist_ok=True) - if os.path.exists(installer_dest): - remove_item(installer_dest) - shutil.copy(r'{}\{}'.format(dest, name), installer_dest) + # Copy to Installers folder + installer_parent = r'{}\Installers\Extras\{}'.format( + global_vars['BaseDir'], section) + installer_dest = r'{}\{}'.format(installer_parent, name) + os.makedirs(installer_parent, exist_ok=True) + if os.path.exists(installer_dest): + remove_item(installer_dest) + shutil.copy(r'{}\{}'.format(dest, name), installer_dest) def update_all_ninite(indent=8, width=40, other_results={}): - print_info('{}Ninite'.format(' '*int(indent/2))) - for section in sorted(NINITE_SOURCES.keys()): - print_success('{}{}'.format(' '*int(indent/4*3), section)) - dest = r'{}\_Ninite\{}'.format(global_vars['CBinDir'], section) - for name, url in sorted(NINITE_SOURCES[section].items()): - try_and_print(message=name, function=update_one_ninite, - other_results=other_results, indent=indent, width=width, - section=section, dest=dest, name=name, url=url) + print_info('{}Ninite'.format(' '*int(indent/2))) + for section in sorted(NINITE_SOURCES.keys()): + print_success('{}{}'.format(' '*int(indent/4*3), section)) + dest = r'{}\_Ninite\{}'.format(global_vars['CBinDir'], section) + for name, url in sorted(NINITE_SOURCES[section].items()): + try_and_print(message=name, function=update_one_ninite, + other_results=other_results, indent=indent, width=width, + section=section, dest=dest, name=name, url=url) ## Misc ## def update_caffeine(): - # Stop running processes - kill_process('caffeine.exe') + # Stop running processes + kill_process('caffeine.exe') - # Remove existing folders - remove_from_kit('Caffeine') + # Remove existing folders + remove_from_kit('Caffeine') - # Download - download_to_temp('caffeine.zip', SOURCE_URLS['Caffeine']) + # Download + download_to_temp('caffeine.zip', SOURCE_URLS['Caffeine']) - # Extract files - extract_temp_to_cbin('caffeine.zip', 'Caffeine') + # Extract files + extract_temp_to_cbin('caffeine.zip', 'Caffeine') - # Cleanup - remove_from_temp('caffeine.zip') + # Cleanup + remove_from_temp('caffeine.zip') def update_du(): - # Stop running processes - kill_process('du.exe') - kill_process('du64.exe') + # Stop running processes + kill_process('du.exe') + kill_process('du64.exe') - # Remove existing folders - remove_from_kit('Du') + # Remove existing folders + remove_from_kit('Du') - # Download - download_to_temp('du.zip', SOURCE_URLS['Du']) + # Download + download_to_temp('du.zip', SOURCE_URLS['Du']) - # Extract files - extract_temp_to_cbin('du.zip', 'Du') + # Extract files + extract_temp_to_cbin('du.zip', 'Du') - # Cleanup - remove_from_temp('du.zip') + # Cleanup + remove_from_temp('du.zip') def update_everything(): - # Stop running processes - for exe in ['Everything.exe', 'Everything64.exe']: - kill_process(exe) + # Stop running processes + for exe in ['Everything.exe', 'Everything64.exe']: + kill_process(exe) - # Remove existing folders - remove_from_kit('Everything') + # Remove existing folders + remove_from_kit('Everything') - # Download - download_to_temp('everything32.zip', SOURCE_URLS['Everything32']) - download_to_temp('everything64.zip', SOURCE_URLS['Everything64']) + # Download + download_to_temp('everything32.zip', SOURCE_URLS['Everything32']) + download_to_temp('everything64.zip', SOURCE_URLS['Everything64']) - # Extract files - extract_temp_to_cbin('everything64.zip', 'Everything', sz_args=['Everything.exe']) - shutil.move( - r'{}\Everything\Everything.exe'.format(global_vars['CBinDir']), - r'{}\Everything\Everything64.exe'.format(global_vars['CBinDir'])) - extract_temp_to_cbin('everything32.zip', 'Everything') + # Extract files + extract_temp_to_cbin( + 'everything64.zip', 'Everything', sz_args=['Everything.exe']) + shutil.move( + r'{}\Everything\Everything.exe'.format(global_vars['CBinDir']), + r'{}\Everything\Everything64.exe'.format(global_vars['CBinDir'])) + extract_temp_to_cbin('everything32.zip', 'Everything') - # Cleanup - remove_from_temp('everything32.zip') - remove_from_temp('everything64.zip') + # Cleanup + remove_from_temp('everything32.zip') + remove_from_temp('everything64.zip') def update_firefox_ublock_origin(): - # Remove existing folders - remove_from_kit('FirefoxExtensions') + # Remove existing folders + remove_from_kit('FirefoxExtensions') - # Download - download_generic( - r'{}\FirefoxExtensions'.format(global_vars['CBinDir']), - 'ublock_origin.xpi', - SOURCE_URLS['Firefox uBO']) + # Download + download_generic( + r'{}\FirefoxExtensions'.format(global_vars['CBinDir']), + 'ublock_origin.xpi', + SOURCE_URLS['Firefox uBO']) def update_notepadplusplus(): - # Stop running processes - kill_process('notepadplusplus.exe') + # Stop running processes + kill_process('notepadplusplus.exe') - # Remove existing folders - remove_from_kit('NotepadPlusPlus') + # Remove existing folders + remove_from_kit('NotepadPlusPlus') - # Download - download_to_temp('npp.7z', SOURCE_URLS['NotepadPlusPlus']) + # Download + download_to_temp('npp.7z', SOURCE_URLS['NotepadPlusPlus']) - # Extract files - extract_temp_to_cbin('npp.7z', 'NotepadPlusPlus') - shutil.move( - r'{}\NotepadPlusPlus\notepad++.exe'.format(global_vars['CBinDir']), - r'{}\NotepadPlusPlus\notepadplusplus.exe'.format(global_vars['CBinDir']) - ) + # Extract files + extract_temp_to_cbin('npp.7z', 'NotepadPlusPlus') + shutil.move( + r'{}\NotepadPlusPlus\notepad++.exe'.format(global_vars['CBinDir']), + r'{}\NotepadPlusPlus\notepadplusplus.exe'.format(global_vars['CBinDir']) + ) - # Cleanup - remove_from_temp('npp.7z') + # Cleanup + remove_from_temp('npp.7z') def update_putty(): - # Stop running processes - kill_process('PUTTY.EXE') + # Stop running processes + kill_process('PUTTY.EXE') - # Remove existing folders - remove_from_kit('PuTTY') + # Remove existing folders + remove_from_kit('PuTTY') - # Download - download_to_temp('putty.zip', SOURCE_URLS['PuTTY']) + # Download + download_to_temp('putty.zip', SOURCE_URLS['PuTTY']) - # Extract files - extract_temp_to_cbin('putty.zip', 'PuTTY') + # Extract files + extract_temp_to_cbin('putty.zip', 'PuTTY') - # Cleanup - remove_from_temp('putty.zip') + # Cleanup + remove_from_temp('putty.zip') def update_wiztree(): - # Stop running processes - for process in ['WizTree.exe', 'WizTree64.exe']: - kill_process(process) + # Stop running processes + for process in ['WizTree.exe', 'WizTree64.exe']: + kill_process(process) - # Remove existing folders - remove_from_kit('WizTree') + # Remove existing folders + remove_from_kit('WizTree') - # Download - download_to_temp( - 'wiztree.zip', SOURCE_URLS['WizTree']) + # Download + download_to_temp( + 'wiztree.zip', SOURCE_URLS['WizTree']) - # Extract files - extract_temp_to_cbin('wiztree.zip', 'WizTree') + # Extract files + extract_temp_to_cbin('wiztree.zip', 'WizTree') - # Cleanup - remove_from_temp('wiztree.zip') + # Cleanup + remove_from_temp('wiztree.zip') def update_xmplay(): - # Stop running processes - kill_process('xmplay.exe') + # Stop running processes + kill_process('xmplay.exe') - # Remove existing folders - remove_from_kit('XMPlay') + # Remove existing folders + remove_from_kit('XMPlay') - # Download - download_to_temp('xmplay.zip', SOURCE_URLS['XMPlay']) - download_to_temp('xmp-7z.zip', SOURCE_URLS['XMPlay 7z']) - download_to_temp('xmp-gme.zip', SOURCE_URLS['XMPlay Game']) - download_to_temp('xmp-rar.zip', SOURCE_URLS['XMPlay RAR']) - download_to_temp('WAModern.zip', SOURCE_URLS['XMPlay WAModern']) + # Download + download_to_temp('xmplay.zip', SOURCE_URLS['XMPlay']) + download_to_temp('xmp-7z.zip', SOURCE_URLS['XMPlay 7z']) + download_to_temp('xmp-gme.zip', SOURCE_URLS['XMPlay Game']) + download_to_temp('xmp-rar.zip', SOURCE_URLS['XMPlay RAR']) + download_to_temp('WAModern.zip', SOURCE_URLS['XMPlay WAModern']) - # Extract files - extract_temp_to_cbin('xmplay.zip', 'XMPlay', - mode='e', sz_args=['xmplay.exe', 'xmplay.txt']) - for item in ['xmp-7z', 'xmp-gme', 'xmp-rar', 'WAModern']: - filter = [] - if item == 'WAModern': - filter.append('WAModern NightVision.xmpskin') - extract_generic( - r'{}\{}.zip'.format(global_vars['TmpDir'], item), - r'{}\XMPlay\plugins'.format(global_vars['CBinDir']), - mode='e', sz_args=filter) + # Extract files + extract_temp_to_cbin('xmplay.zip', 'XMPlay', + mode='e', sz_args=['xmplay.exe', 'xmplay.txt']) + for item in ['xmp-7z', 'xmp-gme', 'xmp-rar', 'WAModern']: + filter = [] + if item == 'WAModern': + filter.append('WAModern NightVision.xmpskin') + extract_generic( + r'{}\{}.zip'.format(global_vars['TmpDir'], item), + r'{}\XMPlay\plugins'.format(global_vars['CBinDir']), + mode='e', sz_args=filter) - # Download Music - dest = r'{}\XMPlay\music_tmp\MOD'.format(global_vars['CBinDir']) - for mod in MUSIC_MOD: - name = mod.split('#')[-1] - url = 'https://api.modarchive.org/downloads.php?moduleid={}'.format(mod) - download_generic(dest, name, url) - dest = r'{}\XMPlay\music_tmp\SNES'.format(global_vars['CBinDir']) - for game in MUSIC_SNES: - name = '{}.rsn'.format(game) - url = 'http://snesmusic.org/v2/download.php?spcNow={}'.format(game) - download_generic(dest, name, url) + # Download Music + dest = r'{}\XMPlay\music_tmp\MOD'.format(global_vars['CBinDir']) + for mod in MUSIC_MOD: + name = mod.split('#')[-1] + url = 'https://api.modarchive.org/downloads.php?moduleid={}'.format(mod) + download_generic(dest, name, url) + dest = r'{}\XMPlay\music_tmp\SNES'.format(global_vars['CBinDir']) + for game in MUSIC_SNES: + name = '{}.rsn'.format(game) + url = 'http://snesmusic.org/v2/download.php?spcNow={}'.format(game) + download_generic(dest, name, url) - # Compress Music - cmd = [ - global_vars['Tools']['SevenZip'], - 'a', r'{}\XMPlay\music.7z'.format(global_vars['CBinDir']), - '-t7z', '-mx=9', '-bso0', '-bse0', - r'{}\XMPlay\music_tmp\*'.format(global_vars['CBinDir']), - ] - run_program(cmd) + # Compress Music + cmd = [ + global_vars['Tools']['SevenZip'], + 'a', r'{}\XMPlay\music.7z'.format(global_vars['CBinDir']), + '-t7z', '-mx=9', '-bso0', '-bse0', + r'{}\XMPlay\music_tmp\*'.format(global_vars['CBinDir']), + ] + run_program(cmd) - # Cleanup - remove_item(r'{}\XMPlay\music_tmp'.format(global_vars['CBinDir'])) - remove_from_temp('xmplay.zip') - remove_from_temp('xmp-7z.zip') - remove_from_temp('xmp-gme.zip') - remove_from_temp('xmp-rar.zip') - remove_from_temp('WAModern.zip') + # Cleanup + remove_item(r'{}\XMPlay\music_tmp'.format(global_vars['CBinDir'])) + remove_from_temp('xmplay.zip') + remove_from_temp('xmp-7z.zip') + remove_from_temp('xmp-gme.zip') + remove_from_temp('xmp-rar.zip') + remove_from_temp('WAModern.zip') ## Repairs ## def update_adwcleaner(): - # Stop running processes - kill_process('AdwCleaner.exe') + # Stop running processes + kill_process('AdwCleaner.exe') - # Remove existing folders - remove_from_kit('AdwCleaner') + # Remove existing folders + remove_from_kit('AdwCleaner') - # Download - download_generic( - r'{}\AdwCleaner'.format(global_vars['CBinDir']), - 'AdwCleaner.exe', - SOURCE_URLS['AdwCleaner']) + # Download + download_generic( + r'{}\AdwCleaner'.format(global_vars['CBinDir']), + 'AdwCleaner.exe', + SOURCE_URLS['AdwCleaner']) def update_kvrt(): - # Stop running processes - kill_process('KVRT.exe') + # Stop running processes + kill_process('KVRT.exe') - # Remove existing folders - remove_from_kit('KVRT') + # Remove existing folders + remove_from_kit('KVRT') - # Download - download_generic( - r'{}\KVRT'.format(global_vars['CBinDir']), - 'KVRT.exe', - SOURCE_URLS['KVRT']) + # Download + download_generic( + r'{}\KVRT'.format(global_vars['CBinDir']), + 'KVRT.exe', + SOURCE_URLS['KVRT']) def update_rkill(): - # Stop running processes - kill_process('RKill.exe') + # Stop running processes + kill_process('RKill.exe') - # Remove existing folders - remove_from_kit('RKill') + # Remove existing folders + remove_from_kit('RKill') - # Download - url = resolve_dynamic_url( - SOURCE_URLS['RKill'], - 'href.*rkill\.exe') - download_generic( - r'{}\RKill'.format(global_vars['CBinDir']), 'RKill.exe', url) + # Download + url = resolve_dynamic_url( + SOURCE_URLS['RKill'], + 'href.*rkill\.exe') + download_generic( + r'{}\RKill'.format(global_vars['CBinDir']), 'RKill.exe', url) def update_tdsskiller(): - # Stop running processes - kill_process('TDSSKiller.exe') + # Stop running processes + kill_process('TDSSKiller.exe') - # Remove existing folders - remove_from_kit('TDSSKiller') + # Remove existing folders + remove_from_kit('TDSSKiller') - # Download - download_generic( - r'{}\TDSSKiller'.format(global_vars['CBinDir']), - 'TDSSKiller.exe', - SOURCE_URLS['TDSSKiller']) + # Download + download_generic( + r'{}\TDSSKiller'.format(global_vars['CBinDir']), + 'TDSSKiller.exe', + SOURCE_URLS['TDSSKiller']) ## Uninstallers ## def update_iobit_uninstaller(): - # Stop running processes - kill_process('IObitUninstallerPortable.exe') + # Stop running processes + kill_process('IObitUninstallerPortable.exe') - # Remove existing folders - remove_from_kit('IObitUninstallerPortable') + # Remove existing folders + remove_from_kit('IObitUninstallerPortable') - # Download - download_generic( - global_vars['CBinDir'], - 'IObitUninstallerPortable.exe', - SOURCE_URLS['IOBit_Uninstaller']) + # Download + download_generic( + global_vars['CBinDir'], + 'IObitUninstallerPortable.exe', + SOURCE_URLS['IOBit_Uninstaller']) - # "Install" - cmd = r'{}\IObitUninstallerPortable.exe'.format(global_vars['CBinDir']) - popen_program(cmd) - sleep(1) - wait_for_process('IObitUninstallerPortable') + # "Install" + cmd = r'{}\IObitUninstallerPortable.exe'.format(global_vars['CBinDir']) + popen_program(cmd) + sleep(1) + wait_for_process('IObitUninstallerPortable') - # Cleanup - remove_from_kit('IObitUninstallerPortable.exe') + # Cleanup + remove_from_kit('IObitUninstallerPortable.exe') if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From ec0341027e2575400fb4e3cc93e7f6b41b1d09a0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:14:29 -0700 Subject: [PATCH 130/186] Updated windows_setup.py --- .bin/Scripts/functions/windows_setup.py | 380 ++++++++++++------------ 1 file changed, 191 insertions(+), 189 deletions(-) diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index 0952e64d..228e43e2 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -5,234 +5,236 @@ from functions.disk import * # STATIC VARIABLES WINDOWS_VERSIONS = [ - {'Name': 'Windows 7 Home Basic', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 HOMEBASIC'}, - {'Name': 'Windows 7 Home Premium', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 HOMEPREMIUM'}, - {'Name': 'Windows 7 Professional', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 PROFESSIONAL'}, - {'Name': 'Windows 7 Ultimate', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 ULTIMATE'}, + {'Name': 'Windows 7 Home Basic', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 HOMEBASIC'}, + {'Name': 'Windows 7 Home Premium', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 HOMEPREMIUM'}, + {'Name': 'Windows 7 Professional', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 PROFESSIONAL'}, + {'Name': 'Windows 7 Ultimate', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 ULTIMATE'}, - {'Name': 'Windows 8.1', - 'Image File': 'Win8', - 'Image Name': 'Windows 8.1', - 'CRLF': True}, - {'Name': 'Windows 8.1 Pro', - 'Image File': 'Win8', - 'Image Name': 'Windows 8.1 Pro'}, + {'Name': 'Windows 8.1', + 'Image File': 'Win8', + 'Image Name': 'Windows 8.1', + 'CRLF': True}, + {'Name': 'Windows 8.1 Pro', + 'Image File': 'Win8', + 'Image Name': 'Windows 8.1 Pro'}, - {'Name': 'Windows 10 Home', - 'Image File': 'Win10', - 'Image Name': 'Windows 10 Home', - 'CRLF': True}, - {'Name': 'Windows 10 Pro', - 'Image File': 'Win10', - 'Image Name': 'Windows 10 Pro'}, - ] + {'Name': 'Windows 10 Home', + 'Image File': 'Win10', + 'Image Name': 'Windows 10 Home', + 'CRLF': True}, + {'Name': 'Windows 10 Pro', + 'Image File': 'Win10', + 'Image Name': 'Windows 10 Pro'}, + ] def find_windows_image(windows_version): - """Search for a Windows source image file, returns dict. + """Search for a Windows source image file, returns dict. - Searches on local disks and then the WINDOWS_SERVER share.""" - image = {} - imagefile = windows_version['Image File'] - imagename = windows_version['Image Name'] + Searches on local disks and then the WINDOWS_SERVER share.""" + image = {} + imagefile = windows_version['Image File'] + imagename = windows_version['Image Name'] - # Search local source - set_thread_error_mode(silent=True) # Prevents "No disk" popups - for d in psutil.disk_partitions(): - for ext in ['esd', 'wim', 'swm']: - path = '{}images\{}.{}'.format(d.mountpoint, imagefile, ext) - if os.path.isfile(path) and wim_contains_image(path, imagename): - image['Path'] = path - image['Letter'] = d.mountpoint[:1].upper() - image['Local'] = True - if ext == 'swm': - image['Glob'] = '--ref="{}*.swm"'.format(image['Path'][:-4]) - break - set_thread_error_mode(silent=False) # Return to normal + # Search local source + set_thread_error_mode(silent=True) # Prevents "No disk" popups + for d in psutil.disk_partitions(): + for ext in ['esd', 'wim', 'swm']: + path = '{}images\{}.{}'.format(d.mountpoint, imagefile, ext) + if os.path.isfile(path) and wim_contains_image(path, imagename): + image['Path'] = path + image['Letter'] = d.mountpoint[:1].upper() + image['Local'] = True + if ext == 'swm': + image['Glob'] = '--ref="{}*.swm"'.format(image['Path'][:-4]) + break + set_thread_error_mode(silent=False) # Return to normal - # Check for network source - if not image: - mount_windows_share() - if WINDOWS_SERVER['Mounted']: - for ext in ['esd', 'wim', 'swm']: - path = r'\\{}\{}\images\{}.{}'.format( - WINDOWS_SERVER['IP'], - WINDOWS_SERVER['Share'], - imagefile, - ext) - if os.path.isfile(path) and wim_contains_image(path, imagename): - image['Path'] = path - image['Letter'] = None - image['Local'] = False - if ext == 'swm': - image['Glob'] = '--ref="{}*.swm"'.format( - image['Path'][:-4]) - break + # Check for network source + if not image: + mount_windows_share() + if WINDOWS_SERVER['Mounted']: + for ext in ['esd', 'wim', 'swm']: + path = r'\\{}\{}\images\{}.{}'.format( + WINDOWS_SERVER['IP'], + WINDOWS_SERVER['Share'], + imagefile, + ext) + if os.path.isfile(path) and wim_contains_image(path, imagename): + image['Path'] = path + image['Letter'] = None + image['Local'] = False + if ext == 'swm': + image['Glob'] = '--ref="{}*.swm"'.format( + image['Path'][:-4]) + break - # Display image to be used (if any) and return - if image: - print_info('Using image: {}'.format(image['Path'])) - return image - else: - print_error('Failed to find Windows source image for {}'.format( - windows_version['Name'])) - raise GenericAbort + # Display image to be used (if any) and return + if image: + print_info('Using image: {}'.format(image['Path'])) + return image + else: + print_error('Failed to find Windows source image for {}'.format( + windows_version['Name'])) + raise GenericAbort def format_disk(disk, use_gpt): - """Format disk for use as a Windows OS disk.""" - if use_gpt: - format_gpt(disk) - else: - format_mbr(disk) + """Format disk for use as a Windows OS disk.""" + if use_gpt: + format_gpt(disk) + else: + format_mbr(disk) def format_gpt(disk): - """Format disk for use as a Windows OS disk using the GPT layout.""" - script = [ - # Partition table - 'select disk {}'.format(disk['Number']), - 'clean', - 'convert gpt', + """Format disk for use as a Windows OS disk using the GPT layout.""" + script = [ + # Partition table + 'select disk {}'.format(disk['Number']), + 'clean', + 'convert gpt', - # System partition - # NOTE: ESP needs to be >= 260 for Advanced Format 4K disks - 'create partition efi size=500', - 'format quick fs=fat32 label="System"', - 'assign letter="S"', + # System partition + # NOTE: ESP needs to be >= 260 for Advanced Format 4K disks + 'create partition efi size=500', + 'format quick fs=fat32 label="System"', + 'assign letter="S"', - # Microsoft Reserved (MSR) partition - 'create partition msr size=128', + # Microsoft Reserved (MSR) partition + 'create partition msr size=128', - # Windows partition - 'create partition primary', - 'format quick fs=ntfs label="Windows"', - 'assign letter="W"', + # Windows partition + 'create partition primary', + 'format quick fs=ntfs label="Windows"', + 'assign letter="W"', - # Recovery Tools partition - 'shrink minimum=500', - 'create partition primary', - 'format quick fs=ntfs label="Recovery Tools"', - 'assign letter="T"', - 'set id="de94bba4-06d1-4d40-a16a-bfd50179d6ac"', - 'gpt attributes=0x8000000000000001', - ] + # Recovery Tools partition + 'shrink minimum=500', + 'create partition primary', + 'format quick fs=ntfs label="Recovery Tools"', + 'assign letter="T"', + 'set id="de94bba4-06d1-4d40-a16a-bfd50179d6ac"', + 'gpt attributes=0x8000000000000001', + ] - # Run - run_diskpart(script) + # Run + run_diskpart(script) def format_mbr(disk): - """Format disk for use as a Windows OS disk using the MBR layout.""" - script = [ - # Partition table - 'select disk {}'.format(disk['Number']), - 'clean', + """Format disk for use as a Windows OS disk using the MBR layout.""" + script = [ + # Partition table + 'select disk {}'.format(disk['Number']), + 'clean', - # System partition - 'create partition primary size=100', - 'format fs=ntfs quick label="System Reserved"', - 'active', - 'assign letter="S"', + # System partition + 'create partition primary size=100', + 'format fs=ntfs quick label="System Reserved"', + 'active', + 'assign letter="S"', - # Windows partition - 'create partition primary', - 'format fs=ntfs quick label="Windows"', - 'assign letter="W"', + # Windows partition + 'create partition primary', + 'format fs=ntfs quick label="Windows"', + 'assign letter="W"', - # Recovery Tools partition - 'shrink minimum=500', - 'create partition primary', - 'format quick fs=ntfs label="Recovery"', - 'assign letter="T"', - 'set id=27', - ] + # Recovery Tools partition + 'shrink minimum=500', + 'create partition primary', + 'format quick fs=ntfs label="Recovery"', + 'assign letter="T"', + 'set id=27', + ] - # Run - run_diskpart(script) + # Run + run_diskpart(script) def mount_windows_share(): - """Mount the Windows images share unless labeled as already mounted.""" - if not WINDOWS_SERVER['Mounted']: - # Mounting read-write in case a backup was done in the same session - # and the server was left mounted read-write. This avoids throwing an - # error by trying to mount the same server with multiple credentials. - mount_network_share(WINDOWS_SERVER, read_write=True) + """Mount the Windows images share unless labeled as already mounted.""" + if not WINDOWS_SERVER['Mounted']: + # Mounting read-write in case a backup was done in the same session + # and the server was left mounted read-write. This avoids throwing an + # error by trying to mount the same server with multiple credentials. + mount_network_share(WINDOWS_SERVER, read_write=True) def select_windows_version(): - """Select Windows version from a menu, returns dict.""" - actions = [ - {'Name': 'Main Menu', 'Letter': 'M'}, - ] + """Select Windows version from a menu, returns dict.""" + actions = [ + {'Name': 'Main Menu', 'Letter': 'M'}, + ] - # Menu loop - selection = menu_select( - title = 'Which version of Windows are we installing?', - main_entries = WINDOWS_VERSIONS, - action_entries = actions) + # Menu loop + selection = menu_select( + title = 'Which version of Windows are we installing?', + main_entries = WINDOWS_VERSIONS, + action_entries = actions) - if selection.isnumeric(): - return WINDOWS_VERSIONS[int(selection)-1] - elif selection == 'M': - raise GenericAbort + if selection.isnumeric(): + return WINDOWS_VERSIONS[int(selection)-1] + elif selection == 'M': + raise GenericAbort def setup_windows(windows_image, windows_version): - """Apply a Windows image to W:""" - cmd = [ - global_vars['Tools']['wimlib-imagex'], - 'apply', - windows_image['Path'], - windows_version['Image Name'], - 'W:\\'] - if 'Glob' in windows_image: - cmd.extend(windows_image['Glob']) - run_program(cmd) + """Apply a Windows image to W:""" + cmd = [ + global_vars['Tools']['wimlib-imagex'], + 'apply', + windows_image['Path'], + windows_version['Image Name'], + 'W:\\'] + if 'Glob' in windows_image: + cmd.extend(windows_image['Glob']) + run_program(cmd) def setup_windows_re(windows_version, windows_letter='W', tools_letter='T'): - """Setup the WinRE partition.""" - win = r'{}:\Windows'.format(windows_letter) - winre = r'{}\System32\Recovery\WinRE.wim'.format(win) - dest = r'{}:\Recovery\WindowsRE'.format(tools_letter) + """Setup the WinRE partition.""" + win = r'{}:\Windows'.format(windows_letter) + winre = r'{}\System32\Recovery\WinRE.wim'.format(win) + dest = r'{}:\Recovery\WindowsRE'.format(tools_letter) - # Copy WinRE.wim - os.makedirs(dest, exist_ok=True) - shutil.copy(winre, r'{}\WinRE.wim'.format(dest)) + # Copy WinRE.wim + os.makedirs(dest, exist_ok=True) + shutil.copy(winre, r'{}\WinRE.wim'.format(dest)) - # Set location - cmd = [ - r'{}\System32\ReAgentc.exe'.format(win), - '/setreimage', - '/path', dest, - '/target', win] - run_program(cmd) + # Set location + cmd = [ + r'{}\System32\ReAgentc.exe'.format(win), + '/setreimage', + '/path', dest, + '/target', win] + run_program(cmd) def update_boot_partition(system_letter='S', windows_letter='W', mode='ALL'): - """Setup the Windows boot partition.""" - cmd = [ - r'{}\Windows\System32\bcdboot.exe'.format( - global_vars['Env']['SYSTEMDRIVE']), - r'{}:\Windows'.format(windows_letter), - '/s', '{}:'.format(system_letter), - '/f', mode] - run_program(cmd) + """Setup the Windows boot partition.""" + cmd = [ + r'{}\Windows\System32\bcdboot.exe'.format( + global_vars['Env']['SYSTEMDRIVE']), + r'{}:\Windows'.format(windows_letter), + '/s', '{}:'.format(system_letter), + '/f', mode] + run_program(cmd) def wim_contains_image(filename, imagename): - """Check if an ESD/WIM contains the specified image, returns bool.""" - cmd = [ - global_vars['Tools']['wimlib-imagex'], - 'info', - filename, - imagename] - try: - run_program(cmd) - except subprocess.CalledProcessError: - return False + """Check if an ESD/WIM contains the specified image, returns bool.""" + cmd = [ + global_vars['Tools']['wimlib-imagex'], + 'info', + filename, + imagename] + try: + run_program(cmd) + except subprocess.CalledProcessError: + return False - return True + return True if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From c3ca58879cb7ea4a07e9771d0d89f1bc70e9dfab Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:15:02 -0700 Subject: [PATCH 131/186] Updated winpe_menus.py --- .bin/Scripts/functions/winpe_menus.py | 830 +++++++++++++------------- 1 file changed, 416 insertions(+), 414 deletions(-) diff --git a/.bin/Scripts/functions/winpe_menus.py b/.bin/Scripts/functions/winpe_menus.py index 1c732eca..e20a3f3a 100644 --- a/.bin/Scripts/functions/winpe_menus.py +++ b/.bin/Scripts/functions/winpe_menus.py @@ -6,466 +6,468 @@ from functions.windows_setup import * # STATIC VARIABLES FAST_COPY_PE_ARGS = [ - '/cmd=noexist_only', - '/utf8', - '/skip_empty_dir', - '/linkdest', - '/no_ui', - '/auto_close', - '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), - ] + '/cmd=noexist_only', + '/utf8', + '/skip_empty_dir', + '/linkdest', + '/no_ui', + '/auto_close', + '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), + ] PE_TOOLS = { - 'BlueScreenView': { - 'Path': r'BlueScreenView\BlueScreenView.exe', - }, - 'FastCopy': { - 'Path': r'FastCopy\FastCopy.exe', - 'Args': FAST_COPY_PE_ARGS, - }, - 'HWiNFO': { - 'Path': r'HWiNFO\HWiNFO.exe', - }, - 'NT Password Editor': { - 'Path': r'NT Password Editor\ntpwedit.exe', - }, - 'Notepad++': { - 'Path': r'NotepadPlusPlus\NotepadPlusPlus.exe', - }, - 'PhotoRec': { - 'Path': r'TestDisk\photorec_win.exe', - 'Args': ['-new_console:n'], - }, - 'Prime95': { - 'Path': r'Prime95\prime95.exe', - }, - 'ProduKey': { - 'Path': r'ProduKey\ProduKey.exe', - }, - 'Q-Dir': { - 'Path': r'Q-Dir\Q-Dir.exe', - }, - 'TestDisk': { - 'Path': r'TestDisk\testdisk_win.exe', - 'Args': ['-new_console:n'], - }, - } + 'BlueScreenView': { + 'Path': r'BlueScreenView\BlueScreenView.exe', + }, + 'FastCopy': { + 'Path': r'FastCopy\FastCopy.exe', + 'Args': FAST_COPY_PE_ARGS, + }, + 'HWiNFO': { + 'Path': r'HWiNFO\HWiNFO.exe', + }, + 'NT Password Editor': { + 'Path': r'NT Password Editor\ntpwedit.exe', + }, + 'Notepad++': { + 'Path': r'NotepadPlusPlus\NotepadPlusPlus.exe', + }, + 'PhotoRec': { + 'Path': r'TestDisk\photorec_win.exe', + 'Args': ['-new_console:n'], + }, + 'Prime95': { + 'Path': r'Prime95\prime95.exe', + }, + 'ProduKey': { + 'Path': r'ProduKey\ProduKey.exe', + }, + 'Q-Dir': { + 'Path': r'Q-Dir\Q-Dir.exe', + }, + 'TestDisk': { + 'Path': r'TestDisk\testdisk_win.exe', + 'Args': ['-new_console:n'], + }, + } def check_pe_tools(): - """Fix tool paths for WinPE layout.""" - for k in PE_TOOLS.keys(): - PE_TOOLS[k]['Path'] = r'{}\{}'.format( - global_vars['BinDir'], PE_TOOLS[k]['Path']) - global_vars['Tools']['wimlib-imagex'] = re.sub( - r'\\x(32|64)', - r'', - global_vars['Tools']['wimlib-imagex'], - re.IGNORECASE) + """Fix tool paths for WinPE layout.""" + for k in PE_TOOLS.keys(): + PE_TOOLS[k]['Path'] = r'{}\{}'.format( + global_vars['BinDir'], PE_TOOLS[k]['Path']) + global_vars['Tools']['wimlib-imagex'] = re.sub( + r'\\x(32|64)', + r'', + global_vars['Tools']['wimlib-imagex'], + re.IGNORECASE) def menu_backup(): - """Take backup images of partition(s) in the WIM format.""" - errors = False - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - 'PathNotFoundError': 'Missing', - }, - 'Warning': { - 'GenericAbort': 'Skipped', - 'GenericRepair': 'Repaired', - }} - set_title('{}: Backup Menu'.format(KIT_NAME_FULL)) + """Take backup images of partition(s) in the WIM format.""" + errors = False + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + 'PathNotFoundError': 'Missing', + }, + 'Warning': { + 'GenericAbort': 'Skipped', + 'GenericRepair': 'Repaired', + }} + set_title('{}: Backup Menu'.format(KIT_NAME_FULL)) - # Set backup prefix - clear_screen() - print_standard('{}\n'.format(global_vars['Title'])) - ticket_number = get_ticket_number() - if ENABLED_TICKET_NUMBERS: - backup_prefix = ticket_number + # Set backup prefix + clear_screen() + print_standard('{}\n'.format(global_vars['Title'])) + ticket_number = get_ticket_number() + if ENABLED_TICKET_NUMBERS: + backup_prefix = ticket_number + else: + backup_prefix = get_simple_string(prompt='Enter backup name prefix') + backup_prefix = backup_prefix.replace(' ', '_') + + # Assign drive letters + try_and_print( + message = 'Assigning letters...', + function = assign_volume_letters, + other_results = other_results) + + # Mount backup shares + mount_backup_shares(read_write=True) + + # Select destination + destination = select_backup_destination(auto_select=False) + + # Scan disks + result = try_and_print( + message = 'Getting disk info...', + function = scan_disks, + other_results = other_results) + if result['CS']: + disks = result['Out'] + else: + print_error('ERROR: No disks found.') + raise GenericAbort + + # Select disk to backup + disk = select_disk('For which disk are we creating backups?', disks) + if not disk: + raise GenericAbort + + # "Prep" disk + prep_disk_for_backup(destination, disk, backup_prefix) + + # Display details for backup task + clear_screen() + print_info('Create Backup - Details:\n') + if ENABLED_TICKET_NUMBERS: + show_data(message='Ticket:', data=ticket_number) + show_data( + message = 'Source:', + data = '[{}] ({}) {} {}'.format( + disk.get('Table', ''), + disk.get('Type', ''), + disk.get('Name', 'Unknown'), + disk.get('Size', ''), + ), + ) + show_data( + message = 'Destination:', + data = destination.get('Display Name', destination['Name']), + ) + for par in disk['Partitions']: + message = 'Partition {}:'.format(par['Number']) + data = par['Display String'] + if par['Number'] in disk['Bad Partitions']: + show_data(message=message, data=data, width=30, warning=True) + if 'Error' in par: + show_data(message='', data=par['Error'], error=True) + elif par['Image Exists']: + show_data(message=message, data=data, width=30, info=True) else: - backup_prefix = get_simple_string(prompt='Enter backup name prefix') - backup_prefix = backup_prefix.replace(' ', '_') + show_data(message=message, data=data, width=30) + print_standard(disk['Backup Warnings']) - # Assign drive letters - try_and_print( - message = 'Assigning letters...', - function = assign_volume_letters, - other_results = other_results) + # Ask to proceed + if (not ask('Proceed with backup?')): + raise GenericAbort - # Mount backup shares - mount_backup_shares(read_write=True) - - # Select destination - destination = select_backup_destination(auto_select=False) - - # Scan disks + # Backup partition(s) + print_info('\n\nStarting task.\n') + for par in disk['Partitions']: result = try_and_print( - message = 'Getting disk info...', - function = scan_disks, - other_results = other_results) - if result['CS']: - disks = result['Out'] - else: - print_error('ERROR: No disks found.') - raise GenericAbort + message = 'Partition {} Backup...'.format(par['Number']), + function = backup_partition, + other_results = other_results, + disk = disk, + par = par) + if not result['CS'] and not isinstance(result['Error'], GenericAbort): + errors = True + par['Error'] = result['Error'] - # Select disk to backup - disk = select_disk('For which disk are we creating backups?', disks) - if not disk: - raise GenericAbort - - # "Prep" disk - prep_disk_for_backup(destination, disk, backup_prefix) - - # Display details for backup task - clear_screen() - print_info('Create Backup - Details:\n') - if ENABLED_TICKET_NUMBERS: - show_data(message='Ticket:', data=ticket_number) - show_data( - message = 'Source:', - data = '[{}] ({}) {} {}'.format( - disk.get('Table', ''), - disk.get('Type', ''), - disk.get('Name', 'Unknown'), - disk.get('Size', ''), - ), - ) - show_data( - message = 'Destination:', - data = destination.get('Display Name', destination['Name']), - ) + # Verify backup(s) + if disk['Valid Partitions']: + print_info('\n\nVerifying backup images(s)\n') for par in disk['Partitions']: - message = 'Partition {}:'.format(par['Number']) - data = par['Display String'] - if par['Number'] in disk['Bad Partitions']: - show_data(message=message, data=data, width=30, warning=True) - if 'Error' in par: - show_data(message='', data=par['Error'], error=True) - elif par['Image Exists']: - show_data(message=message, data=data, width=30, info=True) - else: - show_data(message=message, data=data, width=30) - print_standard(disk['Backup Warnings']) + if par['Number'] in disk['Bad Partitions']: + continue # Skip verification + result = try_and_print( + message = 'Partition {} Image...'.format(par['Number']), + function = verify_wim_backup, + other_results = other_results, + partition = par) + if not result['CS']: + errors = True + par['Error'] = result['Error'] - # Ask to proceed - if (not ask('Proceed with backup?')): - raise GenericAbort - - # Backup partition(s) - print_info('\n\nStarting task.\n') - for par in disk['Partitions']: - result = try_and_print( - message = 'Partition {} Backup...'.format(par['Number']), - function = backup_partition, - other_results = other_results, - disk = disk, - par = par) - if not result['CS'] and not isinstance(result['Error'], GenericAbort): - errors = True - par['Error'] = result['Error'] - - # Verify backup(s) - if disk['Valid Partitions']: - print_info('\n\nVerifying backup images(s)\n') - for par in disk['Partitions']: - if par['Number'] in disk['Bad Partitions']: - continue # Skip verification - result = try_and_print( - message = 'Partition {} Image...'.format(par['Number']), - function = verify_wim_backup, - other_results = other_results, - partition = par) - if not result['CS']: - errors = True - par['Error'] = result['Error'] - - # Print summary - if errors: - print_warning('\nErrors were encountered and are detailed below.') - for par in [p for p in disk['Partitions'] if 'Error' in p]: - print_standard(' Partition {} Error:'.format(par['Number'])) - if hasattr(par['Error'], 'stderr'): - try: - par['Error'] = par['Error'].stderr.decode() - except: - # Deal with badly formatted error message - pass - try: - par['Error'] = par['Error'].splitlines() - par['Error'] = [line.strip() for line in par['Error']] - par['Error'] = [line for line in par['Error'] if line] - for line in par['Error']: - print_error('\t{}'.format(line)) - except: - print_error('\t{}'.format(par['Error'])) - else: - print_success('\nNo errors were encountered during imaging.') - if 'LogFile' in global_vars and ask('\nReview log?'): - cmd = [ - global_vars['Tools']['NotepadPlusPlus'], - global_vars['LogFile']] + # Print summary + if errors: + print_warning('\nErrors were encountered and are detailed below.') + for par in [p for p in disk['Partitions'] if 'Error' in p]: + print_standard(' Partition {} Error:'.format(par['Number'])) + if hasattr(par['Error'], 'stderr'): try: - popen_program(cmd) - except Exception: - print_error('ERROR: Failed to open log.') - sleep(30) - pause('\nPress Enter to return to main menu... ') + par['Error'] = par['Error'].stderr.decode() + except: + # Deal with badly formatted error message + pass + try: + par['Error'] = par['Error'].splitlines() + par['Error'] = [line.strip() for line in par['Error']] + par['Error'] = [line for line in par['Error'] if line] + for line in par['Error']: + print_error('\t{}'.format(line)) + except: + print_error('\t{}'.format(par['Error'])) + else: + print_success('\nNo errors were encountered during imaging.') + if 'LogFile' in global_vars and ask('\nReview log?'): + cmd = [ + global_vars['Tools']['NotepadPlusPlus'], + global_vars['LogFile']] + try: + popen_program(cmd) + except Exception: + print_error('ERROR: Failed to open log.') + sleep(30) + pause('\nPress Enter to return to main menu... ') def menu_root(): - """Main WinPE menu.""" - check_pe_tools() - menus = [ - {'Name': 'Create Backups', 'Menu': menu_backup}, - {'Name': 'Setup Windows', 'Menu': menu_setup}, - {'Name': 'Misc Tools', 'Menu': menu_tools}, - ] - actions = [ - {'Name': 'Command Prompt', 'Letter': 'C'}, - {'Name': 'Reboot', 'Letter': 'R'}, - {'Name': 'Shutdown', 'Letter': 'S'}, - ] + """Main WinPE menu.""" + check_pe_tools() + menus = [ + {'Name': 'Create Backups', 'Menu': menu_backup}, + {'Name': 'Setup Windows', 'Menu': menu_setup}, + {'Name': 'Misc Tools', 'Menu': menu_tools}, + ] + actions = [ + {'Name': 'Command Prompt', 'Letter': 'C'}, + {'Name': 'Reboot', 'Letter': 'R'}, + {'Name': 'Shutdown', 'Letter': 'S'}, + ] - # Main loop - while True: - set_title(KIT_NAME_FULL) - selection = menu_select( - title = 'Main Menu', - main_entries = menus, - action_entries = actions, - secret_exit = True) + # Main loop + while True: + set_title(KIT_NAME_FULL) + selection = menu_select( + title = 'Main Menu', + main_entries = menus, + action_entries = actions, + secret_exit = True) - if (selection.isnumeric()): - try: - menus[int(selection)-1]['Menu']() - except GenericAbort: - print_warning('\nAborted\n') - pause('Press Enter to return to main menu... ') - elif (selection == 'C'): - run_program(['cmd', '-new_console:n'], check=False) - elif (selection == 'R'): - run_program(['wpeutil', 'reboot']) - elif (selection == 'S'): - run_program(['wpeutil', 'shutdown']) - else: - sys.exit() + if (selection.isnumeric()): + try: + menus[int(selection)-1]['Menu']() + except GenericAbort: + print_warning('\nAborted\n') + pause('Press Enter to return to main menu... ') + elif (selection == 'C'): + run_program(['cmd', '-new_console:n'], check=False) + elif (selection == 'R'): + run_program(['wpeutil', 'reboot']) + elif (selection == 'S'): + run_program(['wpeutil', 'shutdown']) + else: + sys.exit() def menu_setup(): - """Format a disk (MBR/GPT), apply a Windows image, and setup boot files.""" - errors = False - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - 'PathNotFoundError': 'Missing', - }, - 'Warning': { - 'GenericAbort': 'Skipped', - 'GenericRepair': 'Repaired', - }} - set_title('{}: Setup Menu'.format(KIT_NAME_FULL)) + """Format a disk (MBR/GPT), apply a Windows image, and setup boot files.""" + errors = False + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + 'PathNotFoundError': 'Missing', + }, + 'Warning': { + 'GenericAbort': 'Skipped', + 'GenericRepair': 'Repaired', + }} + set_title('{}: Setup Menu'.format(KIT_NAME_FULL)) - # Set ticket ID - clear_screen() - print_standard('{}\n'.format(global_vars['Title'])) - ticket_number = get_ticket_number() + # Set ticket ID + clear_screen() + print_standard('{}\n'.format(global_vars['Title'])) + ticket_number = get_ticket_number() - # Select the version of Windows to apply - windows_version = select_windows_version() + # Select the version of Windows to apply + windows_version = select_windows_version() - # Find Windows image - # NOTE: Reassign volume letters to ensure all devices are scanned - try_and_print( - message = 'Assigning volume letters...', - function = assign_volume_letters, - other_results = other_results) - windows_image = find_windows_image(windows_version) + # Find Windows image + # NOTE: Reassign volume letters to ensure all devices are scanned + try_and_print( + message = 'Assigning volume letters...', + function = assign_volume_letters, + other_results = other_results) + windows_image = find_windows_image(windows_version) - # Scan disks - result = try_and_print( - message = 'Getting disk info...', - function = scan_disks, - other_results = other_results) - if result['CS']: - disks = result['Out'] - else: - print_error('ERROR: No disks found.') - raise GenericAbort + # Scan disks + result = try_and_print( + message = 'Getting disk info...', + function = scan_disks, + other_results = other_results) + if result['CS']: + disks = result['Out'] + else: + print_error('ERROR: No disks found.') + raise GenericAbort - # Select disk to use as the OS disk - dest_disk = select_disk('To which disk are we installing Windows?', disks) - if not dest_disk: - raise GenericAbort + # Select disk to use as the OS disk + dest_disk = select_disk('To which disk are we installing Windows?', disks) + if not dest_disk: + raise GenericAbort - # "Prep" disk - prep_disk_for_formatting(dest_disk) + # "Prep" disk + prep_disk_for_formatting(dest_disk) - # Display details for setup task - clear_screen() - print_info('Setup Windows - Details:\n') - if ENABLED_TICKET_NUMBERS: - show_data(message='Ticket:', data=ticket_number) - show_data(message='Installing:', data=windows_version['Name']) + # Display details for setup task + clear_screen() + print_info('Setup Windows - Details:\n') + if ENABLED_TICKET_NUMBERS: + show_data(message='Ticket:', data=ticket_number) + show_data(message='Installing:', data=windows_version['Name']) + show_data( + message = 'Boot Method:', + data = 'UEFI (GPT)' if dest_disk['Use GPT'] else 'Legacy (MBR)') + show_data(message='Using Image:', data=windows_image['Path']) + show_data( + message = 'ERASING:', + data = '[{}] ({}) {} {}\n'.format( + dest_disk.get('Table', ''), + dest_disk.get('Type', ''), + dest_disk.get('Name', 'Unknown'), + dest_disk.get('Size', ''), + ), + warning = True) + for par in dest_disk['Partitions']: show_data( - message = 'Boot Method:', - data = 'UEFI (GPT)' if dest_disk['Use GPT'] else 'Legacy (MBR)') - show_data(message='Using Image:', data=windows_image['Path']) - show_data( - message = 'ERASING:', - data = '[{}] ({}) {} {}\n'.format( - dest_disk.get('Table', ''), - dest_disk.get('Type', ''), - dest_disk.get('Name', 'Unknown'), - dest_disk.get('Size', ''), - ), - warning = True) - for par in dest_disk['Partitions']: - show_data( - message = 'Partition {}:'.format(par['Number']), - data = par['Display String'], - warning = True) - print_warning(dest_disk['Format Warnings']) + message = 'Partition {}:'.format(par['Number']), + data = par['Display String'], + warning = True) + print_warning(dest_disk['Format Warnings']) - if (not ask('Is this correct?')): - raise GenericAbort + if (not ask('Is this correct?')): + raise GenericAbort - # Safety check - print_standard('\nSAFETY CHECK') - print_warning('All data will be DELETED from the ' - 'disk and partition(s) listed above.') - print_warning('This is irreversible and will lead ' - 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) - if (not ask('Asking again to confirm, is this correct?')): - raise GenericAbort + # Safety check + print_standard('\nSAFETY CHECK') + print_warning('All data will be DELETED from the ' + 'disk and partition(s) listed above.') + print_warning('This is irreversible and will lead ' + 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) + if (not ask('Asking again to confirm, is this correct?')): + raise GenericAbort - # Remove volume letters so S, T, & W can be used below - try_and_print( - message = 'Removing volume letters...', - function = remove_volume_letters, - other_results = other_results, - keep=windows_image['Letter']) + # Remove volume letters so S, T, & W can be used below + try_and_print( + message = 'Removing volume letters...', + function = remove_volume_letters, + other_results = other_results, + keep=windows_image['Letter']) - # Assign new letter for local source if necessary - if windows_image['Local'] and windows_image['Letter'] in ['S', 'T', 'W']: - new_letter = try_and_print( - message = 'Reassigning source volume letter...', - function = reassign_volume_letter, - other_results = other_results, - letter=windows_image['Letter']) - windows_image['Path'] = '{}{}'.format( - new_letter, windows_image['Path'][1:]) - windows_image['Letter'] = new_letter + # Assign new letter for local source if necessary + if windows_image['Local'] and windows_image['Letter'] in ['S', 'T', 'W']: + new_letter = try_and_print( + message = 'Reassigning source volume letter...', + function = reassign_volume_letter, + other_results = other_results, + letter=windows_image['Letter']) + windows_image['Path'] = '{}{}'.format( + new_letter, windows_image['Path'][1:]) + windows_image['Letter'] = new_letter - # Format and partition disk - result = try_and_print( - message = 'Formatting disk...', - function = format_disk, - other_results = other_results, - disk = dest_disk, - use_gpt = dest_disk['Use GPT']) - if not result['CS']: - # We need to crash as the disk is in an unknown state - print_error('ERROR: Failed to format disk.') - raise GenericAbort + # Format and partition disk + result = try_and_print( + message = 'Formatting disk...', + function = format_disk, + other_results = other_results, + disk = dest_disk, + use_gpt = dest_disk['Use GPT']) + if not result['CS']: + # We need to crash as the disk is in an unknown state + print_error('ERROR: Failed to format disk.') + raise GenericAbort - # Apply Image - result = try_and_print( - message = 'Applying image...', - function = setup_windows, - other_results = other_results, - windows_image = windows_image, - windows_version = windows_version) - if not result['CS']: - # We need to crash as the disk is in an unknown state - print_error('ERROR: Failed to apply image.') - raise GenericAbort + # Apply Image + result = try_and_print( + message = 'Applying image...', + function = setup_windows, + other_results = other_results, + windows_image = windows_image, + windows_version = windows_version) + if not result['CS']: + # We need to crash as the disk is in an unknown state + print_error('ERROR: Failed to apply image.') + raise GenericAbort - # Create Boot files - try_and_print( - message = 'Updating boot files...', - function = update_boot_partition, - other_results = other_results) + # Create Boot files + try_and_print( + message = 'Updating boot files...', + function = update_boot_partition, + other_results = other_results) - # Setup WinRE - try_and_print( - message = 'Updating recovery tools...', - function = setup_windows_re, - other_results = other_results, - windows_version = windows_version) + # Setup WinRE + try_and_print( + message = 'Updating recovery tools...', + function = setup_windows_re, + other_results = other_results, + windows_version = windows_version) - # Copy WinPE log(s) - source = r'{}\Logs'.format(global_vars['ClientDir']) - dest = r'W:\{}\Logs\WinPE'.format(KIT_NAME_SHORT) - shutil.copytree(source, dest) + # Copy WinPE log(s) + source = r'{}\Logs'.format(global_vars['ClientDir']) + dest = r'W:\{}\Logs\WinPE'.format(KIT_NAME_SHORT) + shutil.copytree(source, dest) - # Print summary - print_standard('\nDone.') - if 'LogFile' in global_vars and ask('\nReview log?'): - cmd = [ - global_vars['Tools']['NotepadPlusPlus'], - global_vars['LogFile']] - try: - popen_program(cmd) - except Exception: - print_error('ERROR: Failed to open log.') - sleep(30) - pause('\nPress Enter to return to main menu... ') + # Print summary + print_standard('\nDone.') + if 'LogFile' in global_vars and ask('\nReview log?'): + cmd = [ + global_vars['Tools']['NotepadPlusPlus'], + global_vars['LogFile']] + try: + popen_program(cmd) + except Exception: + print_error('ERROR: Failed to open log.') + sleep(30) + pause('\nPress Enter to return to main menu... ') def menu_tools(): - """Tool launcher menu.""" - tools = [{'Name': k} for k in sorted(PE_TOOLS.keys())] - actions = [{'Name': 'Main Menu', 'Letter': 'M'},] - set_title(KIT_NAME_FULL) + """Tool launcher menu.""" + tools = [{'Name': k} for k in sorted(PE_TOOLS.keys())] + actions = [{'Name': 'Main Menu', 'Letter': 'M'},] + set_title(KIT_NAME_FULL) - # Menu loop - while True: - selection = menu_select( - title = 'Tools Menu', - main_entries = tools, - action_entries = actions) - if (selection.isnumeric()): - name = tools[int(selection)-1]['Name'] - cmd = [PE_TOOLS[name]['Path']] + PE_TOOLS[name].get('Args', []) - if name == 'Blue Screen View': - # Select path to scan - minidump_path = select_minidump_path() - if minidump_path: - cmd.extend(['/MiniDumpFolder', minidump_path]) - try: - popen_program(cmd) - except Exception: - print_error('Failed to run {}'.format(name)) - sleep(2) - pause() - elif (selection == 'M'): - break + # Menu loop + while True: + selection = menu_select( + title = 'Tools Menu', + main_entries = tools, + action_entries = actions) + if (selection.isnumeric()): + name = tools[int(selection)-1]['Name'] + cmd = [PE_TOOLS[name]['Path']] + PE_TOOLS[name].get('Args', []) + if name == 'Blue Screen View': + # Select path to scan + minidump_path = select_minidump_path() + if minidump_path: + cmd.extend(['/MiniDumpFolder', minidump_path]) + try: + popen_program(cmd) + except Exception: + print_error('Failed to run {}'.format(name)) + sleep(2) + pause() + elif (selection == 'M'): + break def select_minidump_path(): - """Select BSOD minidump path from a menu.""" - dumps = [] + """Select BSOD minidump path from a menu.""" + dumps = [] - # Assign volume letters first - assign_volume_letters() + # Assign volume letters first + assign_volume_letters() - # Search for minidumps - set_thread_error_mode(silent=True) # Prevents "No disk" popups - for d in psutil.disk_partitions(): - if global_vars['Env']['SYSTEMDRIVE'].upper() in d.mountpoint: - # Skip RAMDisk - continue - if os.path.exists(r'{}Windows\MiniDump'.format(d.mountpoint)): - dumps.append({'Name': r'{}Windows\MiniDump'.format(d.mountpoint)}) - set_thread_error_mode(silent=False) # Return to normal + # Search for minidumps + set_thread_error_mode(silent=True) # Prevents "No disk" popups + for d in psutil.disk_partitions(): + if global_vars['Env']['SYSTEMDRIVE'].upper() in d.mountpoint: + # Skip RAMDisk + continue + if os.path.exists(r'{}Windows\MiniDump'.format(d.mountpoint)): + dumps.append({'Name': r'{}Windows\MiniDump'.format(d.mountpoint)}) + set_thread_error_mode(silent=False) # Return to normal - # Check results before showing menu - if len(dumps) == 0: - print_error('ERROR: No BSoD / MiniDump paths found') - sleep(2) - return None + # Check results before showing menu + if len(dumps) == 0: + print_error('ERROR: No BSoD / MiniDump paths found') + sleep(2) + return None - # Menu - selection = menu_select( - title = 'Which BSoD / MiniDump path are we scanning?', - main_entries = dumps) - return dumps[int(selection) - 1]['Name'] + # Menu + selection = menu_select( + title = 'Which BSoD / MiniDump path are we scanning?', + main_entries = dumps) + return dumps[int(selection) - 1]['Name'] if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From 5664765f6ed3febc875ffcde731aa2829ecb2ab9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:27:44 -0700 Subject: [PATCH 132/186] Updated connect-to-network --- .bin/Scripts/connect-to-network | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.bin/Scripts/connect-to-network b/.bin/Scripts/connect-to-network index 0475c15c..e5100303 100755 --- a/.bin/Scripts/connect-to-network +++ b/.bin/Scripts/connect-to-network @@ -12,19 +12,20 @@ from functions.network import * init_global_vars() if __name__ == '__main__': - try: - # Prep - clear_screen() + try: + # Prep + clear_screen() - # Connect - connect_to_network() - - # Done - print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Connect + connect_to_network() + + # Done + print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() +# vim: sts=2 sw=2 ts=2 From 25bacb9892bd66a7453e56fbd567fdfd17c8a533 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:27:55 -0700 Subject: [PATCH 133/186] Updated ddrescue-tui-menu --- .bin/Scripts/ddrescue-tui-menu | 88 +++++++++++++++++----------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index 4bec6230..48cf66a4 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -13,51 +13,51 @@ from functions.hw_diags import * init_global_vars() if __name__ == '__main__': + try: + # Prep + clear_screen() + args = list(sys.argv) + run_mode = '' + source_path = None + dest_path = None + + # Parse args try: - # Prep - clear_screen() - args = list(sys.argv) - run_mode = '' - source_path = None - dest_path = None + script_name = os.path.basename(args.pop(0)) + run_mode = str(args.pop(0)).lower() + source_path = args.pop(0) + dest_path = args.pop(0) + except IndexError: + # We'll set the missing paths later + pass - # Parse args - try: - script_name = os.path.basename(args.pop(0)) - run_mode = str(args.pop(0)).lower() - source_path = args.pop(0) - dest_path = args.pop(0) - except IndexError: - # We'll set the missing paths later - pass + # Show usage + if re.search(r'-+(h|help)', str(sys.argv), re.IGNORECASE): + show_usage(script_name) + exit_script() - # Show usage - if re.search(r'-+(h|help)', str(sys.argv), re.IGNORECASE): - show_usage(script_name) - exit_script() + # Start cloning/imaging + if run_mode in ('clone', 'image'): + menu_ddrescue(source_path, dest_path, run_mode) + else: + if not re.search(r'^-*(h|help\?)', run_mode, re.IGNORECASE): + print_error('Invalid mode.') + + # Done + print_standard('\nDone.') + pause("Press Enter to exit...") + exit_script() + except GenericAbort: + abort() + except GenericError as ge: + msg = 'Generic Error' + if str(ge): + msg = str(ge) + print_error(msg) + abort() + except SystemExit: + pass + except: + major_exception() - # Start cloning/imaging - if run_mode in ('clone', 'image'): - menu_ddrescue(source_path, dest_path, run_mode) - else: - if not re.search(r'^-*(h|help\?)', run_mode, re.IGNORECASE): - print_error('Invalid mode.') - - # Done - print_standard('\nDone.') - pause("Press Enter to exit...") - exit_script() - except GenericAbort: - abort() - except GenericError as ge: - msg = 'Generic Error' - if str(ge): - msg = str(ge) - print_error(msg) - abort() - except SystemExit: - pass - except: - major_exception() - -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 From a9ebeee7487a5d363af0376800f790209afcd18c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:28:14 -0700 Subject: [PATCH 134/186] Updated mount-all-volumes --- .bin/Scripts/mount-all-volumes | 41 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.bin/Scripts/mount-all-volumes b/.bin/Scripts/mount-all-volumes index 9e5de0ea..2c0a6621 100755 --- a/.bin/Scripts/mount-all-volumes +++ b/.bin/Scripts/mount-all-volumes @@ -12,27 +12,28 @@ from functions.data import * init_global_vars() if __name__ == '__main__': - try: - # Prep - clear_screen() - print_standard('{}: Volume mount tool'.format(KIT_NAME_FULL)) + try: + # Prep + clear_screen() + print_standard('{}: Volume mount tool'.format(KIT_NAME_FULL)) - # Mount volumes - report = mount_volumes(all_devices=True) + # Mount volumes + report = mount_volumes(all_devices=True) - # Print report - print_info('\nResults') - for vol_name, vol_data in sorted(report.items()): - show_data(indent=4, width=20, **vol_data['show_data']) + # Print report + print_info('\nResults') + for vol_name, vol_data in sorted(report.items()): + show_data(indent=4, width=20, **vol_data['show_data']) - # Done - print_standard('\nDone.') - if 'gui' in sys.argv: - pause("Press Enter to exit...") - popen_program(['nohup', 'thunar', '/media'], pipe=True) - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + if 'gui' in sys.argv: + pause("Press Enter to exit...") + popen_program(['nohup', 'thunar', '/media'], pipe=True) + exit_script() + except SystemExit: + pass + except: + major_exception() +# vim: sts=2 sw=2 ts=2 From 4ddce7cfbee2f8548ec0bed238afb927e2b76cbc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:28:26 -0700 Subject: [PATCH 135/186] Updated mount-backup-shares --- .bin/Scripts/mount-backup-shares | 41 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.bin/Scripts/mount-backup-shares b/.bin/Scripts/mount-backup-shares index 9f3612b6..6605a81c 100755 --- a/.bin/Scripts/mount-backup-shares +++ b/.bin/Scripts/mount-backup-shares @@ -13,26 +13,27 @@ from functions.network import * init_global_vars() if __name__ == '__main__': - try: - # Prep - clear_screen() + try: + # Prep + clear_screen() - # Connect - connect_to_network() + # Connect + connect_to_network() - # Mount - if is_connected(): - mount_backup_shares(read_write=True) - else: - # Couldn't connect - print_error('ERROR: No network connectivity.') - - # Done - print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Mount + if is_connected(): + mount_backup_shares(read_write=True) + else: + # Couldn't connect + print_error('ERROR: No network connectivity.') + + # Done + print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() +# vim: sts=2 sw=2 ts=2 From 387062074a536621ea59f903bcec05194b67fc29 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:30:40 -0700 Subject: [PATCH 136/186] Updated msword-search --- .bin/Scripts/msword-search | 107 +++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/.bin/Scripts/msword-search b/.bin/Scripts/msword-search index 3e2c175c..380c6ed5 100755 --- a/.bin/Scripts/msword-search +++ b/.bin/Scripts/msword-search @@ -9,10 +9,10 @@ import sys # STATIC VARIABLES SCANDIR = os.getcwd() USAGE = '''Usage: {script} ... - e.g. {script} "Book Title" "Keyword" "etc" + e.g. {script} "Book Title" "Keyword" "etc" - This script will search all doc/docx files below the current directory for - the search-terms provided (case-insensitive).'''.format(script=__file__) + This script will search all doc/docx files below the current directory for + the search-terms provided (case-insensitive).'''.format(script=__file__) # Init os.chdir(os.path.dirname(os.path.realpath(__file__))) @@ -23,59 +23,60 @@ init_global_vars() REGEX_DOC_FILES = re.compile(r'\.docx?$', re.IGNORECASE) def scan_for_docs(path): - for entry in os.scandir(path): - if entry.is_dir(follow_symlinks=False): - yield from scantree(entry.path) - elif entry.is_file and REGEX_DOC_FILES.search(entry.name): - yield entry + for entry in os.scandir(path): + if entry.is_dir(follow_symlinks=False): + yield from scan_for_docs(entry.path) + elif entry.is_file and REGEX_DOC_FILES.search(entry.name): + yield entry def scan_file(file_path, search): - match = False - try: - if entry.name.lower().endswith('.docx'): - result = run_program(['unzip', '-p', entry.path]) - else: - # Assuming .doc - result = run_program(['antiword', entry.path]) - out = result.stdout.decode() - match = re.search(search, out, re.IGNORECASE) - except Exception: - # Ignore errors since files may be corrupted - pass - - return entry.path if match else None + match = False + try: + if entry.name.lower().endswith('.docx'): + result = run_program(['unzip', '-p', entry.path]) + else: + # Assuming .doc + result = run_program(['antiword', entry.path]) + out = result.stdout.decode() + match = re.search(search, out, re.IGNORECASE) + except Exception: + # Ignore errors since files may be corrupted + pass + + return entry.path if match else None if __name__ == '__main__': - try: - # Prep - clear_screen() - terms = [re.sub(r'\s+', r'\s*', t) for t in sys.argv[1:]] - search = '({})'.format('|'.join(terms)) + try: + # Prep + clear_screen() + terms = [re.sub(r'\s+', r'\s*', t) for t in sys.argv[1:]] + search = '({})'.format('|'.join(terms)) - if len(sys.argv) == 1: - # Print usage - print_standard(USAGE) - else: - matches = [] - for entry in scan_for_docs(SCANDIR): - matches.append(scan_file(entry.path, search)) - # Strip None values (i.e. non-matching entries) - matches = [m for m in matches if m] - if matches: - print_success('Found {} {}:'.format( - len(matches), - 'Matches' if len(matches) > 1 else 'Match')) - for match in matches: - print_standard(match) - else: - print_error('No matches found.') - - # Done - print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + if len(sys.argv) == 1: + # Print usage + print_standard(USAGE) + else: + matches = [] + for entry in scan_for_docs(SCANDIR): + matches.append(scan_file(entry.path, search)) + # Strip None values (i.e. non-matching entries) + matches = [m for m in matches if m] + if matches: + print_success('Found {} {}:'.format( + len(matches), + 'Matches' if len(matches) > 1 else 'Match')) + for match in matches: + print_standard(match) + else: + print_error('No matches found.') + + # Done + print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() +# vim: sts=2 sw=2 ts=2 From c501c8b23f6cb139ec7842379332bcf1698bd4b2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:51:56 -0700 Subject: [PATCH 137/186] Updated docstrings longer than 72 characters --- .bin/Scripts/functions/browsers.py | 18 +++++++++++++++--- .bin/Scripts/functions/common.py | 6 +++--- .bin/Scripts/functions/data.py | 4 ++-- .bin/Scripts/functions/ddrescue.py | 2 +- .bin/Scripts/functions/info.py | 2 +- .bin/Scripts/functions/repairs.py | 2 +- .bin/Scripts/functions/sensors.py | 2 +- .bin/Scripts/functions/setup.py | 8 ++++---- .bin/Scripts/functions/windows_setup.py | 2 +- .bin/Scripts/functions/winpe_menus.py | 2 +- 10 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index 750f8af7..6c93b8cb 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -170,7 +170,13 @@ def backup_browsers(): function=archive_browser, name=name) def clean_chromium_profile(profile): - """Renames profile, creates a new folder, and copies the user data to it.""" + """Recreate profile with only the essential user data. + + This is done by renaming the existing profile, creating a new folder + with the original name, then copying the essential files from the + backup folder. This way the original state is preserved in case + something goes wrong. + """ if profile is None: raise Exception backup_path = '{path}_{Date}.bak'.format( @@ -203,7 +209,13 @@ def clean_internet_explorer(**kwargs): pass def clean_mozilla_profile(profile): - """Renames profile, creates a new folder, and copies the user data to it.""" + """Recreate profile with only the essential user data. + + This is done by renaming the existing profile, creating a new folder + with the original name, then copying the essential files from the + backup folder. This way the original state is preserved in case + something goes wrong. + """ if profile is None: raise Exception backup_path = '{path}_{Date}.bak'.format( @@ -229,7 +241,7 @@ def clean_mozilla_profile(profile): f.write('user_pref("{}", {});\n'.format(k, v)) def get_browser_details(name): - """Get install status and profile details for all supported browsers.""" + """Get installation and profile details for all supported browsers.""" browser = SUPPORTED_BROWSERS[name].copy() # Update user_data_path diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 24f8c1d5..b82a8918 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -94,7 +94,7 @@ def abort(): exit_script() def ask(prompt='Kotaero!'): - """Prompt the user with a Y/N question, log answer, and return a bool.""" + """Prompt the user with a Y/N question, returns bool.""" answer = None prompt = '{} [Y/N]: '.format(prompt) while answer is None: @@ -110,7 +110,7 @@ def ask(prompt='Kotaero!'): return answer def choice(choices, prompt='Kotaero!'): - """Prompt the user with a choice question, log answer, and returns str.""" + """Prompt the user with a choice question, returns str.""" answer = None choices = [str(c) for c in choices] choices_short = {c[:1].upper(): c for c in choices} @@ -252,7 +252,7 @@ def get_ticket_number(): return ticket_number def human_readable_size(size, decimals=0): - """Convert size in bytes to a human-readable format and return a str.""" + """Convert size from bytes to a human-readable format, returns str.""" # Prep string formatting width = 3+decimals if decimals > 0: diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index ed4bfa9e..d6c4ad9b 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -117,7 +117,7 @@ SEM_NOOPENFILEERRORBOX = 0x8000 SEM_FAIL = SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS def cleanup_transfer(dest_path): - """Fix attributes and move extraneous items outside the Transfer folder.""" + """Fix attributes and move excluded items to separate folder.""" try: # Remove dest_path if empty os.rmdir(dest_path) @@ -221,7 +221,7 @@ def fix_path_sep(path_str): return re.sub(r'(\\|/)+', lambda s: os.sep, path_str) def is_valid_wim_file(item): - """Checks if the provided os.DirEntry is a valid WIM file, returns bool.""" + """Checks if the item is a valid WIM file, returns bool.""" valid = bool(item.is_file() and REGEX_WIM_FILE.search(item.name)) if valid: extract_item('wimlib', silent=True) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 7e3a06df..e15c0c71 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -255,7 +255,7 @@ class ImageObj(BaseObj): self.path)) def set_details(self): - """Setup loopback device, set details via lsblk, then detach device.""" + """Set details using a temp loopback device.""" self.type = 'image' self.loop_dev = setup_loopback_device(self.path) self.details = get_device_details(self.loop_dev) diff --git a/.bin/Scripts/functions/info.py b/.bin/Scripts/functions/info.py index 4c691e3c..70d1062a 100644 --- a/.bin/Scripts/functions/info.py +++ b/.bin/Scripts/functions/info.py @@ -164,7 +164,7 @@ def get_installed_office(): return programs def get_shell_path(folder, user='current'): - """Get shell path using SHGetKnownFolderPath via knownpaths, returns str. + """Get shell path using knownpaths, returns str. NOTE: Only works for the current user. Code based on https://gist.github.com/mkropat/7550097 diff --git a/.bin/Scripts/functions/repairs.py b/.bin/Scripts/functions/repairs.py index bcaa5bbf..a964220c 100644 --- a/.bin/Scripts/functions/repairs.py +++ b/.bin/Scripts/functions/repairs.py @@ -43,7 +43,7 @@ def run_chkdsk_offline(): raise GenericError def run_dism(repair=False): - """Run DISM /RestoreHealth, then /CheckHealth, and then report errors.""" + """Run DISM to either scan or repair component store health.""" if global_vars['OS']['Version'] in ('8', '8.1', '10'): if repair: # Restore Health diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index eeb98922..2fde69c4 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -167,7 +167,7 @@ def monitor_sensors(monitor_pane, monitor_file): break def save_average_temp(sensor_data, temp_label, seconds=10): - """Calculate average temps and record under temp_label, returns dict.""" + """Save average temps under temp_label, returns dict.""" clear_temps(sensor_data) # Get temps diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index 4254a16a..17cc37fa 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -181,11 +181,11 @@ def config_classicstart(): popen_program(cs_exe) def config_explorer_system(): - """Configure Windows Explorer for all users via Registry settings.""" + """Configure Windows Explorer for all users.""" write_registry_settings(SETTINGS_EXPLORER_SYSTEM, all_users=True) def config_explorer_user(): - """Configure Windows Explorer for current user via Registry settings.""" + """Configure Windows Explorer for current user.""" write_registry_settings(SETTINGS_EXPLORER_USER, all_users=False) def disable_windows_telemetry(): @@ -241,7 +241,7 @@ def install_adobe_reader(): run_program(cmd) def install_chrome_extensions(): - """Update registry to install Google Chrome extensions for all users.""" + """Install Google Chrome extensions for all users.""" write_registry_settings(SETTINGS_GOOGLE_CHROME, all_users=True) def install_classicstart_skin(): @@ -258,7 +258,7 @@ def install_classicstart_skin(): shutil.copy(source, dest) def install_firefox_extensions(): - """Update registry to install Firefox extensions for all users.""" + """Install Firefox extensions for all users.""" dist_path = r'{PROGRAMFILES}\Mozilla Firefox\distribution\extensions'.format( **global_vars['Env']) source_path = r'{CBinDir}\FirefoxExtensions.7z'.format(**global_vars) diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index 228e43e2..d23dce4c 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -156,7 +156,7 @@ def format_mbr(disk): run_diskpart(script) def mount_windows_share(): - """Mount the Windows images share unless labeled as already mounted.""" + """Mount the Windows images share unless already mounted.""" if not WINDOWS_SERVER['Mounted']: # Mounting read-write in case a backup was done in the same session # and the server was left mounted read-write. This avoids throwing an diff --git a/.bin/Scripts/functions/winpe_menus.py b/.bin/Scripts/functions/winpe_menus.py index e20a3f3a..f42a6144 100644 --- a/.bin/Scripts/functions/winpe_menus.py +++ b/.bin/Scripts/functions/winpe_menus.py @@ -250,7 +250,7 @@ def menu_root(): sys.exit() def menu_setup(): - """Format a disk (MBR/GPT), apply a Windows image, and setup boot files.""" + """Format a disk, apply a Windows image, and create boot files.""" errors = False other_results = { 'Error': { From b1786e088caa239043839fb9dbac34f00bd8bf44 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:58:26 -0700 Subject: [PATCH 138/186] Updated hw-diags-audio --- .bin/Scripts/hw-diags-audio | 53 +++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/.bin/Scripts/hw-diags-audio b/.bin/Scripts/hw-diags-audio index 88168263..d1b09694 100755 --- a/.bin/Scripts/hw-diags-audio +++ b/.bin/Scripts/hw-diags-audio @@ -12,31 +12,32 @@ from functions.common import * init_global_vars() if __name__ == '__main__': + try: + # Prep + clear_screen() + print_standard('Hardware Diagnostics: Audio\n') + + # Set volume try: - # Prep - clear_screen() - print_standard('Hardware Diagnostics: Audio\n') - - # Set volume - try: - run_program('amixer -q set "Master" 80% unmute'.split()) - run_program('amixer -q set "PCM" 90% unmute'.split()) - except subprocess.CalledProcessError: - print_error('Failed to set volume') - - # Run tests - for mode in ['pink', 'wav']: - run_program( - cmd = 'speaker-test -c 2 -l 1 -t {}'.format(mode).split(), - check = False, - pipe = False) - - # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + run_program('amixer -q set "Master" 80% unmute'.split()) + run_program('amixer -q set "PCM" 90% unmute'.split()) + except subprocess.CalledProcessError: + print_error('Failed to set volume') + + # Run tests + for mode in ['pink', 'wav']: + run_program( + cmd = 'speaker-test -c 2 -l 1 -t {}'.format(mode).split(), + check = False, + pipe = False) + + # Done + #print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() +# vim: sts=2 sw=2 ts=2 From c7706a115f57f18ddf88811e69363df6fa7ef746 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 20:58:37 -0700 Subject: [PATCH 139/186] Updated hw-diags-network --- .bin/Scripts/hw-diags-network | 57 ++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/.bin/Scripts/hw-diags-network b/.bin/Scripts/hw-diags-network index 5fdd26a4..7e0b0bed 100755 --- a/.bin/Scripts/hw-diags-network +++ b/.bin/Scripts/hw-diags-network @@ -11,36 +11,37 @@ sys.path.append(os.getcwd()) from functions.network import * def check_connection(): - if not is_connected(): - # Raise to cause NS in try_and_print() - raise Exception + if not is_connected(): + # Raise to cause NS in try_and_print() + raise Exception if __name__ == '__main__': - try: - # Prep - clear_screen() - print_standard('Hardware Diagnostics: Network\n') + try: + # Prep + clear_screen() + print_standard('Hardware Diagnostics: Network\n') - # Connect - print_standard('Initializing...') - connect_to_network() + # Connect + print_standard('Initializing...') + connect_to_network() - # Tests - try_and_print( - message='Network connection:', function=check_connection, cs='OK') - show_valid_addresses() - try_and_print(message='Internet connection:', function=ping, - addr='8.8.8.8', cs='OK') - try_and_print(message='DNS Resolution:', function=ping, cs='OK') - try_and_print(message='Speedtest:', function=speedtest, - print_return=True) - - # Done - print_standard('\nDone.') - #pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Tests + try_and_print( + message='Network connection:', function=check_connection, cs='OK') + show_valid_addresses() + try_and_print(message='Internet connection:', function=ping, + addr='8.8.8.8', cs='OK') + try_and_print(message='DNS Resolution:', function=ping, cs='OK') + try_and_print(message='Speedtest:', function=speedtest, + print_return=True) + + # Done + print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() +# vim: sts=2 sw=2 ts=2 From 3b4668e61ba2a289d2007c722998accfabb7419c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 21:02:04 -0700 Subject: [PATCH 140/186] Adjusted init section --- .bin/Scripts/ddrescue-tui-menu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index 48cf66a4..66ed058a 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -6,8 +6,8 @@ import os import sys # Init -sys.path.append(os.path.dirname(os.path.realpath(__file__))) - +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) from functions.ddrescue import * from functions.hw_diags import * init_global_vars() From d0c49240d8d24f9f5a8922923c07225c1cd80eaa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 21:33:37 -0700 Subject: [PATCH 141/186] Added extra line break after classes/functions/etc * Also reordered some class/regex/static sections --- .bin/Scripts/functions/activation.py | 7 +- .bin/Scripts/functions/backup.py | 8 ++ .bin/Scripts/functions/browsers.py | 51 +++++++--- .bin/Scripts/functions/cleanup.py | 7 ++ .bin/Scripts/functions/common.py | 59 +++++++++-- .bin/Scripts/functions/data.py | 128 ++++++++++++++---------- .bin/Scripts/functions/ddrescue.py | 2 + .bin/Scripts/functions/disk.py | 17 ++++ .bin/Scripts/functions/hw_diags.py | 29 ++++++ .bin/Scripts/functions/info.py | 41 ++++++-- .bin/Scripts/functions/network.py | 6 ++ .bin/Scripts/functions/product_keys.py | 7 ++ .bin/Scripts/functions/repairs.py | 8 ++ .bin/Scripts/functions/safemode.py | 7 ++ .bin/Scripts/functions/sensors.py | 14 +++ .bin/Scripts/functions/setup.py | 17 ++++ .bin/Scripts/functions/sw_diags.py | 13 +++ .bin/Scripts/functions/tmux.py | 9 ++ .bin/Scripts/functions/update.py | 67 +++++++++++-- .bin/Scripts/functions/windows_setup.py | 12 +++ .bin/Scripts/functions/winpe_menus.py | 8 ++ .bin/Scripts/hw-diags-network | 2 + .bin/Scripts/msword-search | 3 + 23 files changed, 430 insertions(+), 92 deletions(-) diff --git a/.bin/Scripts/functions/activation.py b/.bin/Scripts/functions/activation.py index e10d088c..52ea8ecd 100644 --- a/.bin/Scripts/functions/activation.py +++ b/.bin/Scripts/functions/activation.py @@ -6,9 +6,11 @@ from borrowed import acpi from functions.common import * from os import environ -# Variables + +# STATIC VARIABLES SLMGR = r'{}\System32\slmgr.vbs'.format(environ.get('SYSTEMROOT')) + def activate_with_bios(): """Attempt to activate Windows with a key stored in the BIOS.""" # Code borrowed from https://github.com/aeruder/get_win8key @@ -43,6 +45,7 @@ def activate_with_bios(): if not windows_is_activated(): raise Exception('Activation Failed') + def get_activation_string(): """Get activation status, returns str.""" act_str = subprocess.run( @@ -53,6 +56,7 @@ def get_activation_string(): act_str = act_str[1].strip() return act_str + def windows_is_activated(): """Check if Windows is activated via slmgr.vbs and return bool.""" activation_string = subprocess.run( @@ -62,6 +66,7 @@ def windows_is_activated(): return bool(activation_string and 'permanent' in activation_string) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/backup.py b/.bin/Scripts/functions/backup.py index f7679db2..ad03f822 100644 --- a/.bin/Scripts/functions/backup.py +++ b/.bin/Scripts/functions/backup.py @@ -4,6 +4,7 @@ import ctypes from functions.disk import * + # Regex REGEX_BAD_PATH_NAMES = re.compile( r'([<>:"/\|\?\*]' @@ -12,6 +13,7 @@ REGEX_BAD_PATH_NAMES = re.compile( r'|[\s\.]+$', re.IGNORECASE) + def backup_partition(disk, par): """Create a backup image of a partition.""" if (par.get('Image Exists', False) @@ -31,6 +33,7 @@ def backup_partition(disk, par): os.makedirs(dest_dir, exist_ok=True) run_program(cmd) + def fix_path(path): """Replace invalid filename characters with underscores.""" local_drive = path[1:2] == ':' @@ -39,6 +42,7 @@ def fix_path(path): new_path = '{}:{}'.format(new_path[0:1], new_path[2:]) return new_path + def get_volume_display_name(mountpoint): """Get display name from volume mountpoint and label, returns str.""" name = mountpoint @@ -67,6 +71,7 @@ def get_volume_display_name(mountpoint): return name + def prep_disk_for_backup(destination, disk, backup_prefix): """Gather details about the disk and its partitions. @@ -143,6 +148,7 @@ def prep_disk_for_backup(destination, disk, backup_prefix): COLORS['YELLOW'], COLORS['CLEAR']) disk['Backup Warnings'] = warnings + def select_backup_destination(auto_select=True): """Select a backup destination from a menu, returns server dict.""" destinations = [s for s in BACKUP_SERVERS if s['Mounted']] @@ -193,6 +199,7 @@ def select_backup_destination(auto_select=True): else: return destinations[int(selection)-1] + def verify_wim_backup(partition): """Verify WIM integrity.""" if not os.path.exists(partition['Image Path']): @@ -205,6 +212,7 @@ def verify_wim_backup(partition): ] run_program(cmd) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index 6c93b8cb..e73399bc 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -4,6 +4,7 @@ from functions.common import * from operator import itemgetter + # Define other_results for later try_and_print browser_data = {} other_results = { @@ -16,22 +17,6 @@ other_results = { } } -# Regex -REGEX_BACKUP = re.compile( - r'\.\w*bak.*', - re.IGNORECASE) -REGEX_CHROMIUM_PROFILE = re.compile( - r'^(Default|Profile)', - re.IGNORECASE) -REGEX_CHROMIUM_ITEMS = re.compile( - r'^(Bookmarks|Cookies|Favicons|Google Profile' - r'|History|Login Data|Top Sites|TransportSecurity' - r'|Visited Links|Web Data)', - re.IGNORECASE) -REGEX_MOZILLA = re.compile( - r'^(bookmarkbackups|(cookies|formhistory|places).sqlite' - r'|key3.db|logins.json|persdict.dat)$', - re.IGNORECASE) # STATIC VARIABLES DEFAULT_HOMEPAGE = 'https://www.google.com/' @@ -103,6 +88,25 @@ SUPPORTED_BROWSERS = { }, } + +# Regex +REGEX_BACKUP = re.compile( + r'\.\w*bak.*', + re.IGNORECASE) +REGEX_CHROMIUM_PROFILE = re.compile( + r'^(Default|Profile)', + re.IGNORECASE) +REGEX_CHROMIUM_ITEMS = re.compile( + r'^(Bookmarks|Cookies|Favicons|Google Profile' + r'|History|Login Data|Top Sites|TransportSecurity' + r'|Visited Links|Web Data)', + re.IGNORECASE) +REGEX_MOZILLA = re.compile( + r'^(bookmarkbackups|(cookies|formhistory|places).sqlite' + r'|key3.db|logins.json|persdict.dat)$', + re.IGNORECASE) + + def archive_all_users(): """Create backups for all browsers for all users.""" users_root = r'{}\Users'.format(global_vars['Env']['SYSTEMDRIVE']) @@ -149,6 +153,7 @@ def archive_all_users(): function=run_program, cmd=cmd) print_standard(' ') + def archive_browser(name): """Create backup of Browser saved in the BackupDir.""" source = '{}*'.format(browser_data[name]['user_data_path']) @@ -163,12 +168,14 @@ def archive_browser(name): archive, source] run_program(cmd) + def backup_browsers(): """Create backup of all detected browser profiles.""" for name in [k for k, v in sorted(browser_data.items()) if v['profiles']]: try_and_print(message='{}...'.format(name), function=archive_browser, name=name) + def clean_chromium_profile(profile): """Recreate profile with only the essential user data. @@ -191,6 +198,7 @@ def clean_chromium_profile(profile): shutil.copy(entry.path, r'{}\{}'.format( profile['path'], entry.name)) + def clean_internet_explorer(**kwargs): """Uses the built-in function to reset IE and sets the homepage. @@ -208,6 +216,7 @@ def clean_internet_explorer(**kwargs): except FileNotFoundError: pass + def clean_mozilla_profile(profile): """Recreate profile with only the essential user data. @@ -240,6 +249,7 @@ def clean_mozilla_profile(profile): for k, v in MOZILLA_PREFS.items(): f.write('user_pref("{}", {});\n'.format(k, v)) + def get_browser_details(name): """Get installation and profile details for all supported browsers.""" browser = SUPPORTED_BROWSERS[name].copy() @@ -314,6 +324,7 @@ def get_browser_details(name): elif num_installs > 1 and browser['base'] != 'ie': raise MultipleInstallationsError + def get_chromium_profiles(search_path): """Find any chromium-style profiles and return as a list of dicts.""" profiles = [] @@ -330,6 +341,7 @@ def get_chromium_profiles(search_path): return profiles + def get_ie_homepages(): """Read homepages from the registry and return as a list.""" homepages = [] @@ -354,6 +366,7 @@ def get_ie_homepages(): homepages = [h.replace('{', '').replace('}', '') for h in homepages] return homepages + def get_mozilla_homepages(prefs_path): """Read homepages from prefs.js and return as a list.""" homepages = [] @@ -369,6 +382,7 @@ def get_mozilla_homepages(prefs_path): return homepages + def get_mozilla_profiles(search_path, dev=False): """Find any mozilla-style profiles and return as a list of dicts.""" profiles = [] @@ -393,6 +407,7 @@ def get_mozilla_profiles(search_path, dev=False): return profiles + def install_adblock(indent=8, width=32, just_firefox=False): """Install adblock for all supported browsers.""" for browser in sorted(browser_data): @@ -461,6 +476,7 @@ def install_adblock(indent=8, width=32, just_firefox=False): cs='Done', function=function, cmd=[exe_path, *urls], check=False) + def list_homepages(indent=8, width=32): """List current homepages for reference.""" browser_list = [k for k, v in sorted(browser_data.items()) if v['exe_path']] @@ -491,6 +507,7 @@ def list_homepages(indent=8, width=32): print_standard('{indent}{name:<{width}}{page}'.format( indent=' '*indent, width=width, name=name, page=page)) + def reset_browsers(indent=8, width=32): """Reset all detected browsers to safe defaults.""" browser_list = [k for k, v in sorted(browser_data.items()) if v['profiles']] @@ -508,6 +525,7 @@ def reset_browsers(indent=8, width=32): indent=indent, width=width, function=function, other_results=other_results, profile=profile) + def scan_for_browsers(just_firefox=False): """Scan system for any supported browsers.""" for name, details in sorted(SUPPORTED_BROWSERS.items()): @@ -517,6 +535,7 @@ def scan_for_browsers(just_firefox=False): function=get_browser_details, cs='Detected', other_results=other_results, name=name) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/cleanup.py b/.bin/Scripts/functions/cleanup.py index 80ffd879..5ee20be1 100644 --- a/.bin/Scripts/functions/cleanup.py +++ b/.bin/Scripts/functions/cleanup.py @@ -2,6 +2,7 @@ from functions.common import * + def cleanup_adwcleaner(): """Move AdwCleaner folders into the ClientDir.""" source_path = r'{SYSTEMDRIVE}\AdwCleaner'.format(**global_vars['Env']) @@ -26,6 +27,7 @@ def cleanup_adwcleaner(): dest_name = non_clobber_rename(dest_name) shutil.move(source_path, dest_name) + def cleanup_cbs(dest_folder): """Safely cleanup a known CBS archive bug under Windows 7. @@ -65,6 +67,7 @@ def cleanup_cbs(dest_folder): r'{}\CbsPersist*'.format(temp_folder)] run_program(cmd) + def cleanup_desktop(): """Move known backup files and reports into the ClientDir.""" dest_folder = r'{LogDir}\Tools'.format(**global_vars) @@ -81,6 +84,7 @@ def cleanup_desktop(): # Remove dir if empty delete_empty_folders(dest_folder) + def delete_empty_folders(folder_path): """Delete all empty folders in path (depth first).""" if not os.path.exists(folder_path) or not os.path.isdir(folder_path): @@ -98,6 +102,7 @@ def delete_empty_folders(folder_path): except OSError: pass + def delete_registry_key(hive, key, recurse=False): """Delete a registry key and all it's subkeys.""" access = winreg.KEY_ALL_ACCESS @@ -117,12 +122,14 @@ def delete_registry_key(hive, key, recurse=False): # Ignore pass + def delete_registry_value(hive, key, value): """Delete a registry value.""" access = winreg.KEY_ALL_ACCESS with winreg.OpenKeyEx(hive, key, 0, access) as k: winreg.DeleteValue(k, value) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index b82a8918..a0a87b76 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -20,9 +20,11 @@ from settings.main import * from settings.tools import * from settings.windows_builds import * + # Global variables global_vars = {} + # STATIC VARIABLES COLORS = { 'CLEAR': '\033[0m', @@ -42,6 +44,7 @@ except NameError: if psutil.WINDOWS: raise + # Error Classes class BIOSKeyNotFoundError(Exception): pass @@ -85,6 +88,7 @@ class SecureBootNotAvailError(Exception): class SecureBootUnknownError(Exception): pass + # General functions def abort(): """Abort script.""" @@ -93,6 +97,7 @@ def abort(): pause(prompt='Press Enter to exit... ') exit_script() + def ask(prompt='Kotaero!'): """Prompt the user with a Y/N question, returns bool.""" answer = None @@ -109,6 +114,7 @@ def ask(prompt='Kotaero!'): print_log(message=message) return answer + def choice(choices, prompt='Kotaero!'): """Prompt the user with a choice question, returns str.""" answer = None @@ -137,6 +143,7 @@ def choice(choices, prompt='Kotaero!'): # Done return answer + def clear_screen(): """Simple wrapper for cls/clear.""" if psutil.WINDOWS: @@ -144,6 +151,7 @@ def clear_screen(): else: os.system('clear') + def convert_to_bytes(size): """Convert human-readable size str to bytes and return an int.""" size = str(size) @@ -165,6 +173,7 @@ def convert_to_bytes(size): return size + def exit_script(return_value=0): """Exits the script after some cleanup and opens the log (if set).""" # Remove dirs (if empty) @@ -192,6 +201,7 @@ def exit_script(return_value=0): # Exit sys.exit(return_value) + def extract_item(item, filter='', silent=False): """Extract item from .cbin into .bin.""" cmd = [ @@ -211,6 +221,7 @@ def extract_item(item, filter='', silent=False): if not silent: print_warning('WARNING: Errors encountered while exctracting data') + def get_process(name=None): """Get process by name, returns psutil.Process obj.""" proc = None @@ -226,6 +237,7 @@ def get_process(name=None): pass return proc + def get_simple_string(prompt='Enter string'): """Get string from user (restricted character set), returns str.""" simple_string = None @@ -235,6 +247,7 @@ def get_simple_string(prompt='Enter string'): simple_string = _input.strip() return simple_string + def get_ticket_number(): """Get TicketNumber from user, save in LogDir, and return as str.""" if not ENABLED_TICKET_NUMBERS: @@ -251,6 +264,7 @@ def get_ticket_number(): f.write(ticket_number) return ticket_number + def human_readable_size(size, decimals=0): """Convert size from bytes to a human-readable format, returns str.""" # Prep string formatting @@ -290,12 +304,14 @@ def human_readable_size(size, decimals=0): return '{size:>{width}.{decimals}f} {units}'.format( size=size, width=width, decimals=decimals, units=units) + def kill_process(name): """Kill any running caffeine.exe processes.""" for proc in psutil.process_iter(): if proc.name() == name: proc.kill() + def major_exception(): """Display traceback and exit""" print_error('Major exception') @@ -319,6 +335,7 @@ def major_exception(): pause('Press Enter to exit...') exit_script(1) + def menu_select( title='[Untitled Menu]', prompt='Please make a selection', secret_actions=[], secret_exit=False, @@ -382,6 +399,7 @@ def menu_select( return answer.upper() + def non_clobber_rename(full_path): """Append suffix to path, if necessary, to avoid clobbering path""" new_path = full_path @@ -392,10 +410,12 @@ def non_clobber_rename(full_path): return new_path + def pause(prompt='Press Enter to continue... '): """Simple pause implementation.""" input(prompt) + def ping(addr='google.com'): """Attempt to ping addr.""" cmd = [ @@ -405,6 +425,7 @@ def ping(addr='google.com'): addr] run_program(cmd) + def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): """Run program and return a subprocess.Popen object.""" cmd_kwargs = {'args': cmd, 'shell': shell} @@ -426,14 +447,17 @@ def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): return subprocess.Popen(**cmd_kwargs) + def print_error(*args, **kwargs): """Prints message to screen in RED.""" print_standard(*args, color=COLORS['RED'], **kwargs) + def print_info(*args, **kwargs): """Prints message to screen in BLUE.""" print_standard(*args, color=COLORS['BLUE'], **kwargs) + def print_standard(message='Generic info', color=None, end='\n', timestamp=True, **kwargs): """Prints message to screen and log (if set).""" @@ -444,14 +468,17 @@ def print_standard(message='Generic info', print(display_message.format(**COLORS), end=end, **kwargs) print_log(message, end, timestamp) + def print_success(*args, **kwargs): """Prints message to screen in GREEN.""" print_standard(*args, color=COLORS['GREEN'], **kwargs) + def print_warning(*args, **kwargs): """Prints message to screen in YELLOW.""" print_standard(*args, color=COLORS['YELLOW'], **kwargs) + def print_log(message='', end='\n', timestamp=True): """Writes message to a log if LogFile is set.""" time_str = time.strftime("%Y-%m-%d %H%M%z: ") if timestamp else '' @@ -463,6 +490,7 @@ def print_log(message='', end='\n', timestamp=True): line = line, end = end)) + def run_program(cmd, args=[], check=True, pipe=True, shell=False, **kwargs): """Run program and return a subprocess.CompletedProcess object.""" if args: @@ -486,6 +514,7 @@ def run_program(cmd, args=[], check=True, pipe=True, shell=False, **kwargs): return subprocess.run(**cmd_kwargs) + def set_title(title='[Some Title]'): """Set title. @@ -493,6 +522,7 @@ def set_title(title='[Some Title]'): global_vars['Title'] = title os.system('title {}'.format(title)) + def show_data( message='[Some message]', data='[Some data]', indent=8, width=32, @@ -509,10 +539,12 @@ def show_data( else: print_standard(message) + def sleep(seconds=2): """Wait for a while.""" time.sleep(seconds) + def stay_awake(): """Prevent the system from sleeping or hibernating.""" # DISABLED due to VCR2008 dependency @@ -529,12 +561,14 @@ def stay_awake(): print_error('ERROR: No caffeine available.') print_warning('Please set the power setting to High Performance.') + def strip_colors(s): """Remove all ASCII color escapes from string, returns str.""" for c in COLORS.values(): s = s.replace(c, '') return s + def get_exception(s): """Get exception by name, returns Exception object.""" try: @@ -544,6 +578,7 @@ def get_exception(s): obj = getattr(sys.modules['builtins'], s) return obj + def try_and_print(message='Trying...', function=None, cs='CS', ns='NS', other_results={}, catch_all=True, print_return=False, silent_function=True, @@ -600,6 +635,7 @@ def try_and_print(message='Trying...', else: return {'CS': not bool(err), 'Error': err, 'Out': out} + def upload_crash_details(): """Upload log and runtime data to the CRASH_SERVER. @@ -611,13 +647,11 @@ def upload_crash_details(): if 'LogFile' in global_vars and global_vars['LogFile']: if ask('Upload crash details to {}?'.format(CRASH_SERVER['Name'])): with open(global_vars['LogFile']) as f: - data = '''{} -############################# -Runtime Details: - -sys.argv: {} - -global_vars: {}'''.format(f.read(), sys.argv, global_vars) + data = '{}\n'.format(f.read()) + data += '#############################\n' + data += 'Runtime Details:\n\n' + data += 'sys.argv: {}\n\n'.format(sys.argv) + data += 'global_vars: {}\n'.format(global_vars) filename = global_vars.get('LogFile', 'Unknown') filename = re.sub(r'.*(\\|/)', '', filename) filename += '.txt' @@ -639,6 +673,7 @@ global_vars: {}'''.format(f.read(), sys.argv, global_vars) # No LogFile defined (or invalid LogFile) raise GenericError + def wait_for_process(name, poll_rate=3): """Wait for process by name.""" running = True @@ -654,6 +689,7 @@ def wait_for_process(name, poll_rate=3): pass sleep(1) + # global_vars functions def init_global_vars(silent=False): """Sets global variables based on system info.""" @@ -687,6 +723,7 @@ def init_global_vars(silent=False): except: major_exception() + def check_os(): """Set OS specific variables.""" tmp = {} @@ -749,6 +786,7 @@ def check_os(): global_vars['OS'] = tmp + def check_tools(): """Set tool variables based on OS bit-depth and tool availability.""" if global_vars['OS'].get('Arch', 32) == 64: @@ -761,6 +799,7 @@ def check_tools(): global_vars['Tools'] = {k: os.path.join(global_vars['BinDir'], v) for (k, v) in global_vars['Tools'].items()} + def clean_env_vars(): """Remove conflicting global_vars and env variables. @@ -769,6 +808,7 @@ def clean_env_vars(): for key in global_vars.keys(): global_vars['Env'].pop(key, None) + def find_bin(): """Find .bin folder in the cwd or it's parents.""" wd = os.getcwd() @@ -785,6 +825,7 @@ def find_bin(): raise BinNotFoundError global_vars['BaseDir'] = base + def make_tmp_dirs(): """Make temp directories.""" os.makedirs(global_vars['BackupDir'], exist_ok=True) @@ -794,6 +835,7 @@ def make_tmp_dirs(): os.makedirs(r'{}\Tools'.format(global_vars['LogDir']), exist_ok=True) os.makedirs(global_vars['TmpDir'], exist_ok=True) + def set_common_vars(): """Set common variables.""" global_vars['Date'] = time.strftime("%Y-%m-%d") @@ -816,6 +858,7 @@ def set_common_vars(): global_vars['TmpDir'] = r'{BinDir}\tmp'.format( **global_vars) + def set_linux_vars(): """Set common variables in a Linux environment. @@ -832,6 +875,7 @@ def set_linux_vars(): 'SevenZip': '7z', } + def set_log_file(log_name): """Sets global var LogFile and creates path as needed.""" folder_path = r'{}\{}'.format(global_vars['LogDir'], KIT_NAME_FULL) @@ -839,6 +883,7 @@ def set_log_file(log_name): os.makedirs(folder_path, exist_ok=True) global_vars['LogFile'] = log_file + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index d6c4ad9b..a151b24c 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -7,57 +7,6 @@ from operator import itemgetter from functions.common import * -# Classes -class LocalDisk(): - def __init__(self, disk): - self.disk = disk - self.name = disk.mountpoint.upper() - self.path = self.name - def is_dir(self): - # Should always be true - return True - def is_file(self): - # Should always be false - return False - -class SourceItem(): - def __init__(self, name, path): - self.name = name - self.path = path - -# Regex -REGEX_EXCL_ITEMS = re.compile( - r'^(\.(AppleDB|AppleDesktop|AppleDouble' - r'|com\.apple\.timemachine\.supported|dbfseventsd' - r'|DocumentRevisions-V100.*|DS_Store|fseventsd|PKInstallSandboxManager' - r'|Spotlight.*|SymAV.*|symSchedScanLockxz|TemporaryItems|Trash.*' - r'|vol|VolumeIcon\.icns)|desktop\.(ini|.*DB|.*DF)' - r'|(hiberfil|pagefile)\.sys|lost\+found|Network\.*Trash\.*Folder' - r'|Recycle[dr]|System\.*Volume\.*Information|Temporary\.*Items' - r'|Thumbs\.db)$', - re.IGNORECASE) -REGEX_EXCL_ROOT_ITEMS = re.compile( - r'^(boot(mgr|nxt)$|Config.msi' - r'|(eula|globdata|install|vc_?red)' - r'|.*.sys$|System Volume Information|RECYCLER?|\$Recycle\.bin' - r'|\$?Win(dows(.old.*|\. BT|)$|RE_)|\$GetCurrent|Windows10Upgrade' - r'|PerfLogs|Program Files|SYSTEM.SAV' - r'|.*\.(esd|swm|wim|dd|map|dmg|image)$)', - re.IGNORECASE) -REGEX_INCL_ROOT_ITEMS = re.compile( - r'^(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' - r'|Media|Music|Pic(ture|)s?|Vid(eo|)s?)' - r'|{prefix}(-?Info|-?Transfer|)' - r'|(ProgramData|Recovery|Temp.*|Users)$' - r'|.*\.(log|txt|rtf|qb\w*|avi|m4a|m4v|mp4|mkv|jpg|png|tiff?)$)' - r''.format(prefix=KIT_NAME_SHORT), - re.IGNORECASE) -REGEX_WIM_FILE = re.compile( - r'\.wim$', - re.IGNORECASE) -REGEX_WINDOWS_OLD = re.compile( - r'^Win(dows|)\.old', - re.IGNORECASE) # STATIC VARIABLES FAST_COPY_EXCLUDES = [ @@ -116,6 +65,63 @@ SEM_FAILCRITICALERRORS = 1 SEM_NOOPENFILEERRORBOX = 0x8000 SEM_FAIL = SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS + +# Regex +REGEX_EXCL_ITEMS = re.compile( + r'^(\.(AppleDB|AppleDesktop|AppleDouble' + r'|com\.apple\.timemachine\.supported|dbfseventsd' + r'|DocumentRevisions-V100.*|DS_Store|fseventsd|PKInstallSandboxManager' + r'|Spotlight.*|SymAV.*|symSchedScanLockxz|TemporaryItems|Trash.*' + r'|vol|VolumeIcon\.icns)|desktop\.(ini|.*DB|.*DF)' + r'|(hiberfil|pagefile)\.sys|lost\+found|Network\.*Trash\.*Folder' + r'|Recycle[dr]|System\.*Volume\.*Information|Temporary\.*Items' + r'|Thumbs\.db)$', + re.IGNORECASE) +REGEX_EXCL_ROOT_ITEMS = re.compile( + r'^(boot(mgr|nxt)$|Config.msi' + r'|(eula|globdata|install|vc_?red)' + r'|.*.sys$|System Volume Information|RECYCLER?|\$Recycle\.bin' + r'|\$?Win(dows(.old.*|\. BT|)$|RE_)|\$GetCurrent|Windows10Upgrade' + r'|PerfLogs|Program Files|SYSTEM.SAV' + r'|.*\.(esd|swm|wim|dd|map|dmg|image)$)', + re.IGNORECASE) +REGEX_INCL_ROOT_ITEMS = re.compile( + r'^(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' + r'|Media|Music|Pic(ture|)s?|Vid(eo|)s?)' + r'|{prefix}(-?Info|-?Transfer|)' + r'|(ProgramData|Recovery|Temp.*|Users)$' + r'|.*\.(log|txt|rtf|qb\w*|avi|m4a|m4v|mp4|mkv|jpg|png|tiff?)$)' + r''.format(prefix=KIT_NAME_SHORT), + re.IGNORECASE) +REGEX_WIM_FILE = re.compile( + r'\.wim$', + re.IGNORECASE) +REGEX_WINDOWS_OLD = re.compile( + r'^Win(dows|)\.old', + re.IGNORECASE) + + +# Classes +class LocalDisk(): + def __init__(self, disk): + self.disk = disk + self.name = disk.mountpoint.upper() + self.path = self.name + def is_dir(self): + # Should always be true + return True + def is_file(self): + # Should always be false + return False + + +class SourceItem(): + def __init__(self, name, path): + self.name = name + self.path = path + + +# Functions def cleanup_transfer(dest_path): """Fix attributes and move excluded items to separate folder.""" try: @@ -153,6 +159,7 @@ def cleanup_transfer(dest_path): except Exception: pass + def find_core_storage_volumes(device_path=None): """Try to create block devices for any Apple CoreStorage volumes.""" corestorage_uuid = '53746f72-6167-11aa-aa11-00306543ecac' @@ -216,10 +223,12 @@ def find_core_storage_volumes(device_path=None): cmd = ['sudo', 'dmsetup', 'create', name, dmsetup_cmd_file] run_program(cmd, check=False) + def fix_path_sep(path_str): """Replace non-native and duplicate dir separators, returns str.""" return re.sub(r'(\\|/)+', lambda s: os.sep, path_str) + def is_valid_wim_file(item): """Checks if the item is a valid WIM file, returns bool.""" valid = bool(item.is_file() and REGEX_WIM_FILE.search(item.name)) @@ -233,6 +242,7 @@ def is_valid_wim_file(item): print_log('WARNING: Image "{}" damaged.'.format(item.name)) return valid + def get_mounted_volumes(): """Get mounted volumes, returns dict.""" cmd = [ @@ -250,6 +260,7 @@ def get_mounted_volumes(): mounted_volumes.extend(item.get('children', [])) return {item['source']: item for item in mounted_volumes} + def mount_volumes( all_devices=True, device_path=None, read_write=False, core_storage=True): @@ -342,6 +353,7 @@ def mount_volumes( return report + def mount_backup_shares(read_write=False): """Mount the backup shares unless labeled as already mounted.""" if psutil.LINUX: @@ -363,6 +375,7 @@ def mount_backup_shares(read_write=False): mount_network_share(server, read_write) + def mount_network_share(server, read_write=False): """Mount a network share defined by server.""" if read_write: @@ -415,6 +428,7 @@ def mount_network_share(server, read_write=False): print_info(success) server['Mounted'] = True + def run_fast_copy(items, dest): """Copy items to dest using FastCopy.""" if not items: @@ -427,6 +441,7 @@ def run_fast_copy(items, dest): run_program(cmd) + def run_wimextract(source, items, dest): """Extract items from source WIM to dest folder.""" if not items: @@ -452,6 +467,7 @@ def run_wimextract(source, items, dest): '--nullglob'] run_program(cmd) + def list_source_items(source_obj, rel_path=None): """List items in a dir or WIM, returns list of SourceItem objects.""" items = [] @@ -489,6 +505,7 @@ def list_source_items(source_obj, rel_path=None): # Done return items + def scan_source(source_obj, dest_path, rel_path='', interactive=True): """Scan source for files/folders to transfer, returns list. @@ -572,6 +589,7 @@ def scan_source(source_obj, dest_path, rel_path='', interactive=True): # Done return selected_items + def get_source_item_obj(source_obj, rel_path, item_path): """Check if the item exists, returns SourceItem object or None.""" item_obj = None @@ -611,6 +629,7 @@ def get_source_item_obj(source_obj, rel_path, item_path): item_path)) return item_obj + def select_destination(folder_path, prompt='Select destination'): """Select destination drive, returns path as string.""" disk = select_volume(prompt) @@ -627,6 +646,7 @@ def select_destination(folder_path, prompt='Select destination'): return path + def select_source(backup_prefix): """Select matching backup from BACKUP_SERVERS, returns obj.""" selected_source = None @@ -792,6 +812,7 @@ def select_source(backup_prefix): # Done return selected_source + def select_volume(title='Select disk', auto_select=True): """Select disk from attached disks. returns dict.""" actions = [{'Name': 'Quit', 'Letter': 'Q'}] @@ -829,6 +850,7 @@ def select_volume(title='Select disk', auto_select=True): else: return disks[int(selection)-1] + def set_thread_error_mode(silent=True): """Disable or Enable Windows error message dialogs. @@ -842,6 +864,7 @@ def set_thread_error_mode(silent=True): else: kernel32.SetThreadErrorMode(SEM_NORMAL, ctypes.byref(SEM_NORMAL)) + def transfer_source(source_obj, dest_path, selected_items): """Transfer, or extract, files/folders from source to destination.""" if source_obj.is_dir(): @@ -864,11 +887,13 @@ def transfer_source(source_obj, dest_path, selected_items): print_error('ERROR: Unsupported image: {}'.format(source_obj.path)) raise GenericError + def umount_backup_shares(): """Unmount the backup shares regardless of current status.""" for server in BACKUP_SERVERS: umount_network_share(server) + def umount_network_share(server): """Unmount a network share defined by server.""" cmd = r'net use \\{IP}\{Share} /delete'.format(**server) @@ -882,6 +907,7 @@ def umount_network_share(server): print_info('Umounted {Name}'.format(**server)) server['Mounted'] = False + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index e15c0c71..e6def7f2 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -15,6 +15,7 @@ from functions.hw_diags import * from functions.tmux import * from operator import itemgetter + # STATIC VARIABLES AUTO_PASS_1_THRESHOLD = 95 AUTO_PASS_2_THRESHOLD = 98 @@ -778,6 +779,7 @@ def menu_ddrescue(source_path, dest_path, run_mode): run_program(['tmux', 'kill-window']) exit_script() + def menu_main(state): """Main menu is used to set ddrescue settings.""" title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) diff --git a/.bin/Scripts/functions/disk.py b/.bin/Scripts/functions/disk.py index 88b26155..31fe577d 100644 --- a/.bin/Scripts/functions/disk.py +++ b/.bin/Scripts/functions/disk.py @@ -3,6 +3,7 @@ from functions.common import * from settings.partition_uids import * + # Regex REGEX_BAD_PARTITION = re.compile(r'(RAW|Unknown)', re.IGNORECASE) REGEX_DISK_GPT = re.compile( @@ -11,6 +12,7 @@ REGEX_DISK_GPT = re.compile( REGEX_DISK_MBR = re.compile(r'Disk ID: [A-Z0-9]+', re.IGNORECASE) REGEX_DISK_RAW = re.compile(r'Disk ID: 00000000', re.IGNORECASE) + def assign_volume_letters(): """Assign a volume letter to all available volumes.""" remove_volume_letters() @@ -24,6 +26,7 @@ def assign_volume_letters(): # Run run_diskpart(script) + def get_boot_mode(): """Check if the boot mode was UEFI or legacy.""" boot_mode = 'Legacy' @@ -38,6 +41,7 @@ def get_boot_mode(): return boot_mode + def get_disk_details(disk): """Get disk details using DiskPart.""" details = {} @@ -63,6 +67,7 @@ def get_disk_details(disk): return details + def get_disks(): """Get list of attached disks using DiskPart.""" disks = [] @@ -82,6 +87,7 @@ def get_disks(): return disks + def get_partition_details(disk, partition): """Get partition details using DiskPart and fsutil.""" details = {} @@ -161,6 +167,7 @@ def get_partition_details(disk, partition): return details + def get_partitions(disk): """Get list of partition using DiskPart.""" partitions = [] @@ -184,6 +191,7 @@ def get_partitions(disk): return partitions + def get_table_type(disk): """Get disk partition table type using DiskPart.""" part_type = 'Unknown' @@ -206,6 +214,7 @@ def get_table_type(disk): return part_type + def get_volumes(): """Get list of volumes using DiskPart.""" vols = [] @@ -221,10 +230,12 @@ def get_volumes(): return vols + def is_bad_partition(par): """Check if the partition is accessible.""" return 'Letter' not in par or REGEX_BAD_PARTITION.search(par['FileSystem']) + def prep_disk_for_formatting(disk=None): """Gather details about the disk and its partitions.""" disk['Format Warnings'] = '\n' @@ -270,6 +281,7 @@ def prep_disk_for_formatting(disk=None): # For all partitions partition['Display String'] = display + def reassign_volume_letter(letter, new_letter='I'): """Assign a new letter to a volume using DiskPart.""" if not letter: @@ -286,6 +298,7 @@ def reassign_volume_letter(letter, new_letter='I'): else: return new_letter + def remove_volume_letters(keep=None): """Remove all assigned volume letters using DiskPart.""" if not keep: @@ -303,6 +316,7 @@ def remove_volume_letters(keep=None): except subprocess.CalledProcessError: pass + def run_diskpart(script): """Run DiskPart script.""" tempfile = r'{}\diskpart.script'.format(global_vars['Env']['TMP']) @@ -321,6 +335,7 @@ def run_diskpart(script): sleep(2) return result + def scan_disks(): """Get details about the attached disks""" disks = get_disks() @@ -343,6 +358,7 @@ def scan_disks(): # Done return disks + def select_disk(title='Which disk?', disks=[]): """Select a disk from the attached disks""" # Build menu @@ -391,6 +407,7 @@ def select_disk(title='Which disk?', disks=[]): elif (selection == 'M'): raise GenericAbort + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 210ae870..155031ec 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -8,6 +8,7 @@ from collections import OrderedDict from functions.sensors import * from functions.tmux import * + # STATIC VARIABLES ATTRIBUTES = { 'NVMe': { @@ -83,13 +84,16 @@ TMUX_LAYOUT = OrderedDict({ 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, }) + # Regex REGEX_ERROR_STATUS = re.compile('|'.join(STATUSES['RED'])) + # Error Classes class DeviceTooSmallError(Exception): pass + # Classes class CpuObj(): """Object for tracking CPU specific data.""" @@ -130,6 +134,7 @@ class CpuObj(): return report + class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): @@ -487,6 +492,7 @@ class DiskObj(): for t in ['badblocks', 'I/O Benchmark']: self.disable_test(t, 'Denied') + class State(): """Object to track device objects and overall state.""" def __init__(self): @@ -559,6 +565,7 @@ class State(): if not skip_disk: self.disks.append(disk_obj) + class TestObj(): """Object to track test data.""" def __init__(self, dev, label=None, info_label=False): @@ -589,6 +596,7 @@ class TestObj(): self.status = build_status_string( self.label, 'Working', self.info_label) + # Functions def build_outer_panes(state): """Build top and side panes.""" @@ -611,6 +619,7 @@ def build_outer_panes(state): lines=SIDE_PANE_WIDTH, watch=state.progress_out) + def build_status_string(label, status, info_label=False): """Build status string with appropriate colors.""" status_color = COLORS['CLEAR'] @@ -626,6 +635,7 @@ def build_status_string(label, status, info_label=False): s_w=SIDE_PANE_WIDTH-len(label), **COLORS) + def fix_tmux_panes(state, tmux_layout): """Fix pane sizes if the window has been resized.""" needs_fixed = False @@ -669,6 +679,7 @@ def fix_tmux_panes(state, tmux_layout): # Resize pane tmux_resize_pane(pane_id=target, **v) + def generate_horizontal_graph(rates, oneline=False): """Generate horizontal graph from rates, returns list.""" graph = ['', '', '', ''] @@ -714,6 +725,7 @@ def generate_horizontal_graph(rates, oneline=False): else: return graph + def get_graph_step(rate, scale=16): """Get graph step based on rate and scale, returns int.""" m_rate = rate / (1024**2) @@ -726,6 +738,7 @@ def get_graph_step(rate, scale=16): break return step + def get_read_rate(s): """Get read rate in bytes/s from dd progress output.""" real_rate = None @@ -734,6 +747,7 @@ def get_read_rate(s): real_rate = convert_to_bytes(human_rate) return real_rate + def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] @@ -840,12 +854,14 @@ def menu_diags(state, args): elif selection == 'S': run_hw_tests(state) + def run_audio_test(): """Run audio test.""" clear_screen() run_program(['hw-diags-audio'], check=False, pipe=False) pause('Press Enter to return to main menu... ') + def run_badblocks_test(state, test): """Run a read-only surface scan with badblocks.""" # Bail early @@ -939,6 +955,7 @@ def run_badblocks_test(state, test): # Cleanup tmux_kill_pane(state.panes['badblocks']) + def run_hw_tests(state): """Run enabled hardware tests.""" print_standard('Scanning devices...') @@ -1016,6 +1033,7 @@ def run_hw_tests(state): # Cleanup tmux_kill_pane(*state.panes.values()) + def run_io_benchmark(state, test): """Run a read-only I/O benchmark using dd.""" # Bail early @@ -1173,11 +1191,13 @@ def run_io_benchmark(state, test): # Cleanup tmux_kill_pane(state.panes['io_benchmark']) + def run_keyboard_test(): """Run keyboard test.""" clear_screen() run_program(['xev', '-event', 'keyboard'], check=False, pipe=False) + def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" # Bail early @@ -1371,12 +1391,14 @@ def run_mprime_test(state, test): tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) test.monitor_proc.kill() + def run_network_test(): """Run network test.""" clear_screen() run_program(['hw-diags-network'], check=False, pipe=False) pause('Press Enter to return to main menu... ') + def run_nvme_smart_tests(state, test): """Run NVMe or SMART test for test.dev.""" # Bail early @@ -1520,6 +1542,7 @@ def run_nvme_smart_tests(state, test): # Done update_progress_pane(state) + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': @@ -1530,6 +1553,7 @@ def secret_screensaver(screensaver=None): raise Exception('Invalid screensaver') run_program(cmd, check=False, pipe=False) + def show_report(report, log_report=False): """Show report on screen and optionally save to log w/out color.""" for line in report: @@ -1537,6 +1561,7 @@ def show_report(report, log_report=False): if log_report: print_log(strip_colors(line)) + def show_results(state): """Show results for all tests.""" clear_screen() @@ -1564,6 +1589,7 @@ def show_results(state): show_report(disk.generate_disk_report(), log_report=True) print_standard(' ') + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 @@ -1607,6 +1633,7 @@ def update_main_options(state, selection, main_options): # Done return main_options + def update_io_progress(percent, rate, progress_file): """Update I/O progress file.""" bar_color = COLORS['CLEAR'] @@ -1631,6 +1658,7 @@ def update_io_progress(percent, rate, progress_file): with open(progress_file, 'a') as f: f.write(line) + def update_progress_pane(state): """Update progress file for side pane.""" output = [] @@ -1658,6 +1686,7 @@ def update_progress_pane(state): with open(state.progress_out, 'w') as f: f.writelines(output) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/info.py b/.bin/Scripts/functions/info.py index 70d1062a..adee5f80 100644 --- a/.bin/Scripts/functions/info.py +++ b/.bin/Scripts/functions/info.py @@ -6,15 +6,6 @@ from operator import itemgetter from functions.common import * from functions.activation import * -# Regex -REGEX_OFFICE = re.compile( - r'(Microsoft (Office\s+' - r'(365|Enterprise|Home|Pro(\s|fessional)' - r'|Single|Small|Standard|Starter|Ultimate|system)' - r'|Works[-\s\d]+\d)' - r'|(Libre|Open|Star)\s*Office' - r'|WordPerfect|Gnumeric|Abiword)', - re.IGNORECASE) # STATIC VARIABLES REG_PROFILE_LIST = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList' @@ -55,6 +46,18 @@ SHELL_FOLDERS = { ), } + +# Regex +REGEX_OFFICE = re.compile( + r'(Microsoft (Office\s+' + r'(365|Enterprise|Home|Pro(\s|fessional)' + r'|Single|Small|Standard|Starter|Ultimate|system)' + r'|Works[-\s\d]+\d)' + r'|(Libre|Open|Star)\s*Office' + r'|WordPerfect|Gnumeric|Abiword)', + re.IGNORECASE) + + def backup_file_list(): """Export current file listing for the system.""" extract_item('Everything', silent=True) @@ -66,6 +69,7 @@ def backup_file_list(): global_vars['Env']['SYSTEMDRIVE']] run_program(cmd) + def backup_power_plans(): """Export current power plans.""" os.makedirs(r'{BackupDir}\Power Plans\{Date}'.format( @@ -83,6 +87,7 @@ def backup_power_plans(): cmd = ['powercfg', '-export', out, guid] run_program(cmd, check=False) + def backup_registry(overwrite=False): """Backup registry including user hives.""" extract_item('erunt', silent=True) @@ -97,6 +102,7 @@ def backup_registry(overwrite=False): cmd.append('/noconfirmdelete') run_program(cmd) + def get_folder_size(path): """Get (human-readable) size of folder passed, returns str.""" size = 'Unknown' @@ -119,6 +125,7 @@ def get_folder_size(path): size = human_readable_size(size) return size + def get_installed_antivirus(): """Get list of installed Antivirus programs.""" programs = [] @@ -149,6 +156,7 @@ def get_installed_antivirus(): programs = ['No programs found'] return programs + def get_installed_office(): """Get list of installed Office programs.""" programs = [] @@ -163,6 +171,7 @@ def get_installed_office(): programs = ['No programs found'] return programs + def get_shell_path(folder, user='current'): """Get shell path using knownpaths, returns str. @@ -189,6 +198,7 @@ def get_shell_path(folder, user='current'): return path + def get_user_data_paths(user): """Get user data paths for provided user, returns dict.""" hive_path = user['SID'] @@ -273,6 +283,7 @@ def get_user_data_paths(user): # Done return paths + def get_user_folder_sizes(users): """Update list(users) to include folder paths and sizes.""" extract_item('du', filter='du*', silent=True) @@ -293,6 +304,7 @@ def get_user_folder_sizes(users): u['Extra Folders'][folder]['Size'] = get_folder_size( u['Extra Folders'][folder]['Path']) + def get_user_list(): """Get user list via WMIC, returns list of dicts.""" users = [] @@ -325,6 +337,7 @@ def get_user_list(): # Done return users + def reg_path_exists(hive, path): """Test if specified path exists, returns bool.""" try: @@ -334,6 +347,7 @@ def reg_path_exists(hive, path): else: return True + def run_aida64(): """Run AIDA64 to save system reports.""" extract_item('AIDA64', silent=True) @@ -372,6 +386,7 @@ def run_aida64(): '/TEXT', '/SILENT', '/SAFEST'] run_program(cmd, check=False) + def run_bleachbit(cleaners=None, preview=True): """Run BleachBit preview and save log. @@ -404,6 +419,7 @@ def run_bleachbit(cleaners=None, preview=True): for line in out.stdout.decode().splitlines(): f.write(line.strip() + '\n') + def show_disk_usage(disk): """Show free and used space for a specified disk.""" print_standard('{:5}'.format(disk.device.replace('/', ' ')), @@ -423,6 +439,7 @@ def show_disk_usage(disk): except Exception: print_warning('Unknown', timestamp=False) + def show_free_space(indent=8, width=32): """Show free space info for all fixed disks.""" message = 'Free Space:' @@ -436,6 +453,7 @@ def show_free_space(indent=8, width=32): except Exception: pass + def show_installed_ram(): """Show installed RAM.""" mem = psutil.virtual_memory() @@ -448,6 +466,7 @@ def show_installed_ram(): else: print_error(human_readable_size(mem.total).strip(), timestamp=False) + def show_os_activation(): """Show OS activation info.""" act_str = get_activation_string() @@ -458,6 +477,7 @@ def show_os_activation(): else: print_error(act_str, timestamp=False) + def show_os_name(): """Show extended OS name (including warnings).""" os_name = global_vars['OS']['DisplayName'] @@ -475,6 +495,7 @@ def show_os_name(): else: print_standard(os_name, timestamp=False) + def show_temp_files_size(): """Show total size of temp files identified by BleachBit.""" size = None @@ -488,6 +509,7 @@ def show_temp_files_size(): else: print_standard(size, timestamp=False) + def show_user_data_summary(indent=8, width=32): """Print user data folder sizes for all users.""" users = get_user_list() @@ -520,6 +542,7 @@ def show_user_data_summary(indent=8, width=32): size = folders[folder].get('Size', 'Unknown'), path = folders[folder].get('Path', 'Unknown'))) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index 80935506..9d9b7af1 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -11,6 +11,7 @@ os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.common import * + # REGEX REGEX_VALID_IP = re.compile( r'(10.\d+.\d+.\d+' @@ -18,6 +19,7 @@ REGEX_VALID_IP = re.compile( r'|192.168.\d+.\d+)', re.IGNORECASE) + def connect_to_network(): """Connect to network if not already connected.""" net_ifs = psutil.net_if_addrs() @@ -38,6 +40,7 @@ def connect_to_network(): function = run_program, cmd = cmd) + def is_connected(): """Check for a valid private IP.""" devs = psutil.net_if_addrs() @@ -49,6 +52,7 @@ def is_connected(): # Else return False + def show_valid_addresses(): """Show all valid private IP addresses assigned to the system.""" devs = psutil.net_if_addrs() @@ -58,6 +62,7 @@ def show_valid_addresses(): # Valid IP found show_data(message=dev, data=family.address) + def speedtest(): """Run a network speedtest using speedtest-cli.""" result = run_program(['speedtest-cli', '--simple']) @@ -67,6 +72,7 @@ def speedtest(): output = [(a, float(b), c) for a, b, c in output] return ['{:10}{:6.2f} {}'.format(*line) for line in output] + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/product_keys.py b/.bin/Scripts/functions/product_keys.py index 1d96ec0b..950e509b 100644 --- a/.bin/Scripts/functions/product_keys.py +++ b/.bin/Scripts/functions/product_keys.py @@ -2,12 +2,14 @@ from functions.common import * + # Regex REGEX_REGISTRY_DIRS = re.compile( r'^(config$|RegBack$|System32$|Transfer|Win)', re.IGNORECASE) REGEX_SOFTWARE_HIVE = re.compile(r'^Software$', re.IGNORECASE) + def extract_keys(): """Extract keys from provided hives and return a dict.""" keys = {} @@ -43,6 +45,7 @@ def extract_keys(): # Done return keys + def list_clientdir_keys(): """List product keys found in hives inside the ClientDir.""" keys = extract_keys() @@ -57,6 +60,7 @@ def list_clientdir_keys(): return key_list + def find_software_hives(): """Search for transferred SW hives and return a list.""" hives = [] @@ -71,6 +75,7 @@ def find_software_hives(): return hives + def get_product_keys(): """List product keys from saved report.""" keys = [] @@ -86,6 +91,7 @@ def get_product_keys(): else: return ['No product keys found'] + def run_produkey(): """Run ProduKey and save report in the ClientDir.""" extract_item('ProduKey', silent=True) @@ -107,6 +113,7 @@ def run_produkey(): log_file] run_program(cmd, check=False) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/repairs.py b/.bin/Scripts/functions/repairs.py index a964220c..f1e10a4a 100644 --- a/.bin/Scripts/functions/repairs.py +++ b/.bin/Scripts/functions/repairs.py @@ -2,6 +2,7 @@ from functions.common import * + def run_chkdsk(repair=False): """Run CHKDSK scan or schedule offline repairs.""" if repair: @@ -9,6 +10,7 @@ def run_chkdsk(repair=False): else: run_chkdsk_scan() + def run_chkdsk_scan(): """Run CHKDSK in a "split window" and report errors.""" if global_vars['OS']['Version'] in ('8', '8.1', '10'): @@ -32,6 +34,7 @@ def run_chkdsk_scan(): for line in out.stdout.decode().splitlines(): f.write(line.strip() + '\n') + def run_chkdsk_offline(): """Set filesystem 'dirty bit' to force a chkdsk during next boot.""" cmd = [ @@ -42,6 +45,7 @@ def run_chkdsk_offline(): if int(out.returncode) > 0: raise GenericError + def run_dism(repair=False): """Run DISM to either scan or repair component store health.""" if global_vars['OS']['Version'] in ('8', '8.1', '10'): @@ -75,6 +79,7 @@ def run_dism(repair=False): else: raise UnsupportedOSError + def run_kvrt(): """Run KVRT.""" extract_item('KVRT', silent=True) @@ -86,6 +91,7 @@ def run_kvrt(): '-processlevel', '3'] popen_program(cmd, pipe=False) + def run_sfc_scan(): """Run SFC in a "split window" and report errors.""" cmd = [ @@ -109,6 +115,7 @@ def run_sfc_scan(): else: raise GenericError + def run_tdsskiller(): """Run TDSSKiller.""" extract_item('TDSSKiller', silent=True) @@ -122,6 +129,7 @@ def run_tdsskiller(): '-dcexact', '-tdlfs'] run_program(cmd, pipe=False) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/safemode.py b/.bin/Scripts/functions/safemode.py index 837b14e8..041cd506 100644 --- a/.bin/Scripts/functions/safemode.py +++ b/.bin/Scripts/functions/safemode.py @@ -2,19 +2,23 @@ from functions.common import * + # STATIC VARIABLES REG_MSISERVER = r'HKLM\SYSTEM\CurrentControlSet\Control\SafeBoot\Network\MSIServer' + def disable_safemode_msi(): """Disable MSI access under safemode.""" cmd = ['reg', 'delete', REG_MSISERVER, '/f'] run_program(cmd) + def disable_safemode(): """Edit BCD to remove safeboot value.""" cmd = ['bcdedit', '/deletevalue', '{default}', 'safeboot'] run_program(cmd) + def enable_safemode_msi(): """Enable MSI access under safemode.""" cmd = ['reg', 'add', REG_MSISERVER, '/f'] @@ -23,15 +27,18 @@ def enable_safemode_msi(): '/t', 'REG_SZ', '/d', 'Service', '/f'] run_program(cmd) + def enable_safemode(): """Edit BCD to set safeboot as default.""" cmd = ['bcdedit', '/set', '{default}', 'safeboot', 'network'] run_program(cmd) + def reboot(delay=3): cmd = ['shutdown', '-r', '-t', '{}'.format(delay)] run_program(cmd, check=False) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 2fde69c4..a50f8941 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -5,6 +5,7 @@ import re from functions.tmux import * + # STATIC VARIABLES TEMP_LIMITS = { 'GREEN': 60, @@ -13,9 +14,11 @@ TEMP_LIMITS = { 'RED': 90, } + # REGEX REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') + def clear_temps(sensor_data): """Clear saved temps but keep structure, returns dict.""" for _section, _adapters in sensor_data.items(): @@ -23,6 +26,7 @@ def clear_temps(sensor_data): for _source, _data in _sources.items(): _data['Temps'] = [] + def fix_sensor_str(s): """Cleanup string and return str.""" s = re.sub(r'^(\w+)-(\w+)-(\w+)', r'\1 (\2 \3)', s, re.IGNORECASE) @@ -36,6 +40,7 @@ def fix_sensor_str(s): s = s.replace(' ', ' ') return s + def generate_sensor_report( sensor_data, *temp_labels, colors=True, core_only=False): @@ -73,6 +78,7 @@ def generate_sensor_report( # Done return report + def get_colored_temp_str(temp): """Get colored string based on temp, returns str.""" try: @@ -97,6 +103,7 @@ def get_colored_temp_str(temp): temp = temp, **COLORS) + def get_raw_sensor_data(): """Read sensor data and return dict.""" data = {} @@ -110,6 +117,7 @@ def get_raw_sensor_data(): return data + def get_sensor_data(): """Parse raw sensor data and return new dict.""" json_data = get_raw_sensor_data() @@ -141,6 +149,7 @@ def get_sensor_data(): # Done return sensor_data + def get_temp_str(temp, colors=True): """Get temp string, returns str.""" if colors: @@ -154,6 +163,7 @@ def get_temp_str(temp, colors=True): '-' if temp < 0 else '', temp) + def monitor_sensors(monitor_pane, monitor_file): """Continually update sensor data and report to screen.""" sensor_data = get_sensor_data() @@ -166,6 +176,7 @@ def monitor_sensors(monitor_pane, monitor_file): if monitor_pane and not tmux_poll_pane(monitor_pane): break + def save_average_temp(sensor_data, temp_label, seconds=10): """Save average temps under temp_label, returns dict.""" clear_temps(sensor_data) @@ -181,6 +192,7 @@ def save_average_temp(sensor_data, temp_label, seconds=10): for _source, _data in _sources.items(): _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) + def update_sensor_data(sensor_data): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() @@ -193,12 +205,14 @@ def update_sensor_data(sensor_data): _data['Max'] = max(_temp, _data['Max']) _data['Temps'].append(_temp) + def join_columns(column1, column2, width=55): return '{:<{}}{}'.format( column1, 55+len(column1)-len(REGEX_COLORS.sub('', column1)), column2) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index 17cc37fa..749058e3 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -3,6 +3,7 @@ from functions.common import * from functions.update import * + # STATIC VARIABLES HKU = winreg.HKEY_USERS HKCR = winreg.HKEY_CLASSES_ROOT @@ -128,6 +129,7 @@ VCR_REDISTS = [ '/passive', '/norestart']}, ] + def config_classicstart(): """Configure ClassicStart.""" # User level, not system level @@ -180,14 +182,17 @@ def config_classicstart(): sleep(1) popen_program(cs_exe) + def config_explorer_system(): """Configure Windows Explorer for all users.""" write_registry_settings(SETTINGS_EXPLORER_SYSTEM, all_users=True) + def config_explorer_user(): """Configure Windows Explorer for current user.""" write_registry_settings(SETTINGS_EXPLORER_USER, all_users=False) + def disable_windows_telemetry(): """Disable Windows 10 telemetry settings with O&O ShutUp10.""" extract_item('ShutUp10', silent=True) @@ -197,6 +202,7 @@ def disable_windows_telemetry(): '/quiet'] run_program(cmd) + def update_clock(): """Set Timezone and sync clock.""" run_program(['tzutil' ,'/s', WINDOWS_TIME_ZONE], check=False) @@ -209,6 +215,7 @@ def update_clock(): run_program(['net', 'start', 'w32ime'], check=False) run_program(['w32tm', '/resync', '/nowait'], check=False) + def write_registry_settings(settings, all_users=False): """Write registry values from custom dict of dicts.""" hive = HKCU @@ -228,6 +235,7 @@ def write_registry_settings(settings, all_users=False): for name, value in v.get('SZ Items', {}).items(): winreg.SetValueEx(key, name, 0, winreg.REG_SZ, value) + # Installations def install_adobe_reader(): """Install Adobe Reader.""" @@ -240,10 +248,12 @@ def install_adobe_reader(): 'EULA_ACCEPT=YES'] run_program(cmd) + def install_chrome_extensions(): """Install Google Chrome extensions for all users.""" write_registry_settings(SETTINGS_GOOGLE_CHROME, all_users=True) + def install_classicstart_skin(): """Extract ClassicStart skin to installation folder.""" if global_vars['OS']['Version'] not in ('8', '8.1', '10'): @@ -257,6 +267,7 @@ def install_classicstart_skin(): os.makedirs(dest_path, exist_ok=True) shutil.copy(source, dest) + def install_firefox_extensions(): """Install Firefox extensions for all users.""" dist_path = r'{PROGRAMFILES}\Mozilla Firefox\distribution\extensions'.format( @@ -277,6 +288,7 @@ def install_firefox_extensions(): source_path] run_program(cmd) + def install_ninite_bundle(mse=False): """Run Ninite file(s) based on OS version.""" if global_vars['OS']['Version'] in ('8', '8.1', '10'): @@ -292,6 +304,7 @@ def install_ninite_bundle(mse=False): popen_program(r'{BaseDir}\Installers\Extras\Bundles\Legacy.exe'.format( **global_vars)) + def install_vcredists(): """Install all supported Visual C++ runtimes.""" extract_item('_vcredists', silent=True) @@ -307,16 +320,20 @@ def install_vcredists(): os.chdir(prev_dir) + # Misc def open_device_manager(): popen_program(['mmc', 'devmgmt.msc']) + def open_windows_activation(): popen_program(['slui']) + def open_windows_updates(): popen_program(['control', '/name', 'Microsoft.WindowsUpdate']) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/sw_diags.py b/.bin/Scripts/functions/sw_diags.py index 317f2b4f..8faa25b7 100644 --- a/.bin/Scripts/functions/sw_diags.py +++ b/.bin/Scripts/functions/sw_diags.py @@ -4,6 +4,7 @@ import ctypes from functions.common import * + # STATIC VARIABLES AUTORUNS_SETTINGS = { r'Software\Sysinternals\AutoRuns': { @@ -26,6 +27,7 @@ AUTORUNS_SETTINGS = { }, } + def check_connection(): """Check if the system is online and optionally abort the script.""" while True: @@ -38,6 +40,7 @@ def check_connection(): else: abort() + def check_secure_boot_status(show_alert=False): """Checks UEFI Secure Boot status via PowerShell.""" boot_mode = get_boot_mode() @@ -77,6 +80,7 @@ def check_secure_boot_status(show_alert=False): show_alert_box('Secure Boot ERROR') raise GenericError + def get_boot_mode(): """Check if Windows is booted in UEFI or Legacy mode, returns str.""" kernel = ctypes.windll.kernel32 @@ -98,6 +102,7 @@ def get_boot_mode(): return type_str + def run_autoruns(): """Run AutoRuns in the background with VirusTotal checks enabled.""" extract_item('Autoruns', filter='autoruns*', silent=True) @@ -109,6 +114,7 @@ def run_autoruns(): winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) popen_program(global_vars['Tools']['AutoRuns'], minimized=True) + def run_hwinfo_sensors(): """Run HWiNFO sensors.""" path = r'{BinDir}\HWiNFO'.format(**global_vars) @@ -122,12 +128,14 @@ def run_hwinfo_sensors(): f.write('SummaryOnly=0\n') popen_program(global_vars['Tools']['HWiNFO']) + def run_nircmd(*cmd): """Run custom NirCmd.""" extract_item('NirCmd', silent=True) cmd = [global_vars['Tools']['NirCmd'], *cmd] run_program(cmd, check=False) + def run_xmplay(): """Run XMPlay to test audio.""" extract_item('XMPlay', silent=True) @@ -141,6 +149,7 @@ def run_xmplay(): # Open XMPlay popen_program(cmd) + def run_hitmanpro(): """Run HitmanPro in the background.""" extract_item('HitmanPro', silent=True) @@ -150,6 +159,7 @@ def run_hitmanpro(): r'/log={LogDir}\Tools\HitmanPro.txt'.format(**global_vars)] popen_program(cmd) + def run_process_killer(): """Kill most running processes skipping those in the whitelist.txt.""" # borrowed from TronScript (reddit.com/r/TronScript) @@ -160,6 +170,7 @@ def run_process_killer(): run_program(['ProcessKiller.exe', '/silent'], check=False) os.chdir(prev_dir) + def run_rkill(): """Run RKill and cleanup afterwards.""" extract_item('RKill', silent=True) @@ -180,11 +191,13 @@ def run_rkill(): dest = non_clobber_rename(dest) shutil.move(item.path, dest) + def show_alert_box(message, title='Wizard Kit Warning'): """Show Windows alert box with message.""" message_box = ctypes.windll.user32.MessageBoxW message_box(None, message, title, 0x00001030) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index 84046d0b..a11560bc 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -2,12 +2,14 @@ from functions.common import * + def create_file(filepath): """Create file if it doesn't exist.""" if not os.path.exists(filepath): with open(filepath, 'w') as f: f.write('') + def tmux_get_pane_size(pane_id=None): """Get target, or current, pane size, returns tuple.""" x = -1 @@ -29,6 +31,7 @@ def tmux_get_pane_size(pane_id=None): return (x, y) + def tmux_kill_all_panes(pane_id=None): """Kill all tmux panes except the active pane or pane_id if specified.""" cmd = ['tmux', 'kill-pane', '-a'] @@ -36,12 +39,14 @@ def tmux_kill_all_panes(pane_id=None): cmd.extend(['-t', pane_id]) run_program(cmd, check=False) + def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] for pane_id in panes: run_program(cmd+[pane_id], check=False) + def tmux_poll_pane(pane_id): """Check if pane exists, returns bool.""" cmd = ['tmux', 'list-panes', '-F', '#D'] @@ -49,6 +54,7 @@ def tmux_poll_pane(pane_id): panes = result.stdout.decode().splitlines() return pane_id in panes + def tmux_resize_pane(pane_id=None, x=None, y=None, **kwargs): """Resize pane to specific hieght or width.""" if not x and not y: @@ -65,6 +71,7 @@ def tmux_resize_pane(pane_id=None, x=None, y=None, **kwargs): run_program(cmd, check=False) + def tmux_split_window( lines=None, percent=None, behind=False, vertical=False, @@ -115,6 +122,7 @@ def tmux_split_window( result = run_program(cmd) return result.stdout.decode().strip() + def tmux_update_pane( pane_id, command=None, working_dir=None, text=None, watch=None, watch_cmd='cat'): @@ -142,6 +150,7 @@ def tmux_update_pane( run_program(cmd) + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 21d2acee..439bb6bf 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -8,6 +8,7 @@ from settings.launchers import * from settings.music import * from settings.sources import * + def compress_and_remove_item(item): """Compress and delete an item unless an error is encountered.""" try: @@ -17,6 +18,7 @@ def compress_and_remove_item(item): else: remove_item(item.path) + def compress_item(item): """Compress an item in a 7-Zip archive using the ARCHIVE_PASSWORD.""" # Prep @@ -42,6 +44,7 @@ def compress_item(item): # Done os.chdir(prev_dir) + def download_generic(out_dir, out_name, source_url): """Downloads a file using requests.""" ## Code based on this Q&A: https://stackoverflow.com/q/16694907 @@ -59,10 +62,12 @@ def download_generic(out_dir, out_name, source_url): except: raise GenericError('Failed to download file.') + def download_to_temp(out_name, source_url): """Download a file to the TmpDir.""" download_generic(global_vars['TmpDir'], out_name, source_url) + def extract_generic(source, dest, mode='x', sz_args=[]): """Extract a file to a destination.""" cmd = [ @@ -73,12 +78,14 @@ def extract_generic(source, dest, mode='x', sz_args=[]): cmd.extend(sz_args) run_program(cmd) + def extract_temp_to_bin(source, item, mode='x', sz_args=[]): """Extract a file to the .bin folder.""" source = r'{}\{}'.format(global_vars['TmpDir'], source) dest = r'{}\{}'.format(global_vars['BinDir'], item) extract_generic(source, dest, mode, sz_args) + def extract_temp_to_cbin(source, item, mode='x', sz_args=[]): """Extract a file to the .cbin folder.""" source = r'{}\{}'.format(global_vars['TmpDir'], source) @@ -88,6 +95,7 @@ def extract_temp_to_cbin(source, item, mode='x', sz_args=[]): shutil.copytree(include_path, dest) extract_generic(source, dest, mode, sz_args) + def generate_launcher(section, name, options): """Generate a launcher script.""" # Prep @@ -125,6 +133,7 @@ def generate_launcher(section, name, options): # f.writelines(out_text) f.write('\n'.join(out_text)) + def remove_item(item_path): """Delete a file or folder.""" if os.path.exists(item_path): @@ -133,6 +142,7 @@ def remove_item(item_path): else: os.remove(item_path) + def remove_from_kit(item): """Delete a file or folder from the .bin/.cbin folders.""" item_locations = [] @@ -144,11 +154,13 @@ def remove_from_kit(item): for item_path in item_locations: remove_item(item_path) + def remove_from_temp(item): """Delete a file or folder from the TmpDir folder.""" item_path = r'{}\{}'.format(global_vars['TmpDir'], item) remove_item(item_path) + def resolve_dynamic_url(source_url, regex): """Scan source_url for a url using the regex provided; returns str.""" # Load the download page @@ -170,6 +182,7 @@ def resolve_dynamic_url(source_url, regex): # Return return url + def scan_for_net_installers(server, family_name, min_year): """Scan network shares for installers.""" if not server['Mounted']: @@ -200,7 +213,8 @@ def scan_for_net_installers(server, family_name, min_year): } umount_network_share(server) -## Data Recovery ## + +# Data Recovery def update_testdisk(): # Stop running processes for exe in ['fidentify_win.exe', 'photorec_win.exe', @@ -226,7 +240,8 @@ def update_testdisk(): # Cleanup remove_from_temp('testdisk_wip.zip') -## Data Transfers ## + +# Data Transfers def update_fastcopy(): ## NOTE: Lives in .bin uncompressed # Stop running processes @@ -266,6 +281,7 @@ def update_fastcopy(): os.remove(r'{}\setup.exe'.format(_path, _installer)) remove_from_temp('FastCopy.zip') + def update_wimlib(): # Stop running processes kill_process('wimlib-imagex.exe') @@ -289,6 +305,7 @@ def update_wimlib(): remove_from_temp('wimlib32.zip') remove_from_temp('wimlib64.zip') + def update_xyplorer(): # Stop running processes kill_process('XYplorerFree.exe') @@ -305,7 +322,8 @@ def update_xyplorer(): # Cleanup remove_from_temp('xyplorer_free.zip') -## Diagnostics ## + +# Diagnostics def update_aida64(): # Stop running processes kill_process('notepadplusplus.exe') @@ -322,6 +340,7 @@ def update_aida64(): # Cleanup remove_from_temp('aida64.zip') + def update_autoruns(): # Stop running processes kill_process('Autoruns.exe') @@ -339,6 +358,7 @@ def update_autoruns(): # Cleanup remove_from_temp('Autoruns.zip') + def update_bleachbit(): # Stop running processes kill_process('bleachbit.exe') @@ -370,6 +390,7 @@ def update_bleachbit(): remove_from_temp('bleachbit.zip') remove_from_temp('Winapp2.zip') + def update_bluescreenview(): # Stop running processes for exe in ['BlueScreenView.exe', 'BlueScreenView64.exe']: @@ -394,6 +415,7 @@ def update_bluescreenview(): remove_from_temp('bluescreenview32.zip') remove_from_temp('bluescreenview64.zip') + def update_erunt(): # Stop running processes kill_process('ERUNT.EXE') @@ -410,6 +432,7 @@ def update_erunt(): # Cleanup remove_from_temp('erunt.zip') + def update_hitmanpro(): # Stop running processes for exe in ['HitmanPro.exe', 'HitmanPro64.exe']: @@ -423,6 +446,7 @@ def update_hitmanpro(): download_generic(dest, 'HitmanPro.exe', SOURCE_URLS['HitmanPro32']) download_generic(dest, 'HitmanPro64.exe', SOURCE_URLS['HitmanPro64']) + def update_hwinfo(): ## NOTE: Lives in .bin uncompressed # Stop running processes @@ -438,6 +462,7 @@ def update_hwinfo(): # Cleanup remove_from_temp('HWiNFO.zip') + def update_nircmd(): # Stop running processes for exe in ['nircmdc.exe', 'nircmdc64.exe']: @@ -461,6 +486,7 @@ def update_nircmd(): remove_from_temp('nircmd32.zip') remove_from_temp('nircmd64.zip') + def update_produkey(): # Stop running processes for exe in ['ProduKey.exe', 'ProduKey64.exe']: @@ -484,7 +510,8 @@ def update_produkey(): remove_from_temp('produkey32.zip') remove_from_temp('produkey64.zip') -## Drivers ## + +# Drivers def update_intel_rst(): # Remove existing folders remove_from_kit('Intel RST') @@ -500,6 +527,7 @@ def update_intel_rst(): for name, url in RST_SOURCES.items(): download_generic(dest, name, url) + def update_intel_ssd_toolbox(): # Remove existing folders remove_from_kit('Intel SSD Toolbox.exe') @@ -510,6 +538,7 @@ def update_intel_ssd_toolbox(): 'Intel SSD Toolbox.exe', SOURCE_URLS['Intel SSD Toolbox']) + def update_samsung_magician(): # Remove existing folders remove_from_kit('Samsung Magician.exe') @@ -528,6 +557,7 @@ def update_samsung_magician(): # Cleanup remove_from_temp('Samsung Magician.zip') + def update_sdi_origin(): # Download aria2 download_to_temp('aria2.zip', SOURCE_URLS['aria2']) @@ -585,7 +615,8 @@ def update_sdi_origin(): remove_from_temp('sdio.torrent') remove_from_temp('sdio_themes.zip') -## Installers ## + +# Installers def update_adobe_reader_dc(): # Prep dest = r'{}\Installers\Extras\Office'.format( @@ -601,6 +632,7 @@ def update_adobe_reader_dc(): download_generic( dest, 'Adobe Reader DC.exe', SOURCE_URLS['Adobe Reader DC']) + def update_macs_fan_control(): # Prep dest = r'{}\Installers'.format( @@ -616,6 +648,7 @@ def update_macs_fan_control(): download_generic( dest, 'Macs Fan Control.exe', SOURCE_URLS['Macs Fan Control']) + def update_office(): # Remove existing folders remove_from_kit('_Office') @@ -644,6 +677,7 @@ def update_office(): # Cleanup remove_from_temp('odt{}.exe'.format(year)) + def update_classic_start_skin(): # Remove existing folders remove_from_kit('ClassicStartSkin') @@ -654,6 +688,7 @@ def update_classic_start_skin(): 'Metro-Win10-Black.skin7', SOURCE_URLS['ClassicStartSkin']) + def update_vcredists(): # Remove existing folders remove_from_kit('_vcredists') @@ -674,6 +709,7 @@ def update_vcredists(): 'vcredist.exe', VCREDIST_SOURCES[year][bit]) + def update_one_ninite(section, dest, name, url, indent=8, width=40): # Prep url = 'https://ninite.com/{}/ninite.exe'.format(url) @@ -690,6 +726,7 @@ def update_one_ninite(section, dest, name, url, indent=8, width=40): remove_item(installer_dest) shutil.copy(r'{}\{}'.format(dest, name), installer_dest) + def update_all_ninite(indent=8, width=40, other_results={}): print_info('{}Ninite'.format(' '*int(indent/2))) for section in sorted(NINITE_SOURCES.keys()): @@ -700,7 +737,8 @@ def update_all_ninite(indent=8, width=40, other_results={}): other_results=other_results, indent=indent, width=width, section=section, dest=dest, name=name, url=url) -## Misc ## + +# Misc def update_caffeine(): # Stop running processes kill_process('caffeine.exe') @@ -717,6 +755,7 @@ def update_caffeine(): # Cleanup remove_from_temp('caffeine.zip') + def update_du(): # Stop running processes kill_process('du.exe') @@ -734,6 +773,7 @@ def update_du(): # Cleanup remove_from_temp('du.zip') + def update_everything(): # Stop running processes for exe in ['Everything.exe', 'Everything64.exe']: @@ -758,6 +798,7 @@ def update_everything(): remove_from_temp('everything32.zip') remove_from_temp('everything64.zip') + def update_firefox_ublock_origin(): # Remove existing folders remove_from_kit('FirefoxExtensions') @@ -768,6 +809,7 @@ def update_firefox_ublock_origin(): 'ublock_origin.xpi', SOURCE_URLS['Firefox uBO']) + def update_notepadplusplus(): # Stop running processes kill_process('notepadplusplus.exe') @@ -788,6 +830,7 @@ def update_notepadplusplus(): # Cleanup remove_from_temp('npp.7z') + def update_putty(): # Stop running processes kill_process('PUTTY.EXE') @@ -804,6 +847,7 @@ def update_putty(): # Cleanup remove_from_temp('putty.zip') + def update_wiztree(): # Stop running processes for process in ['WizTree.exe', 'WizTree64.exe']: @@ -822,6 +866,7 @@ def update_wiztree(): # Cleanup remove_from_temp('wiztree.zip') + def update_xmplay(): # Stop running processes kill_process('xmplay.exe') @@ -877,7 +922,8 @@ def update_xmplay(): remove_from_temp('xmp-rar.zip') remove_from_temp('WAModern.zip') -## Repairs ## + +# Repairs def update_adwcleaner(): # Stop running processes kill_process('AdwCleaner.exe') @@ -891,6 +937,7 @@ def update_adwcleaner(): 'AdwCleaner.exe', SOURCE_URLS['AdwCleaner']) + def update_kvrt(): # Stop running processes kill_process('KVRT.exe') @@ -904,6 +951,7 @@ def update_kvrt(): 'KVRT.exe', SOURCE_URLS['KVRT']) + def update_rkill(): # Stop running processes kill_process('RKill.exe') @@ -918,6 +966,7 @@ def update_rkill(): download_generic( r'{}\RKill'.format(global_vars['CBinDir']), 'RKill.exe', url) + def update_tdsskiller(): # Stop running processes kill_process('TDSSKiller.exe') @@ -931,7 +980,8 @@ def update_tdsskiller(): 'TDSSKiller.exe', SOURCE_URLS['TDSSKiller']) -## Uninstallers ## + +# Uninstallers def update_iobit_uninstaller(): # Stop running processes kill_process('IObitUninstallerPortable.exe') @@ -954,6 +1004,7 @@ def update_iobit_uninstaller(): # Cleanup remove_from_kit('IObitUninstallerPortable.exe') + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index d23dce4c..82120843 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -3,6 +3,7 @@ from functions.data import * from functions.disk import * + # STATIC VARIABLES WINDOWS_VERSIONS = [ {'Name': 'Windows 7 Home Basic', @@ -35,6 +36,7 @@ WINDOWS_VERSIONS = [ 'Image Name': 'Windows 10 Pro'}, ] + def find_windows_image(windows_version): """Search for a Windows source image file, returns dict. @@ -85,6 +87,7 @@ def find_windows_image(windows_version): windows_version['Name'])) raise GenericAbort + def format_disk(disk, use_gpt): """Format disk for use as a Windows OS disk.""" if use_gpt: @@ -92,6 +95,7 @@ def format_disk(disk, use_gpt): else: format_mbr(disk) + def format_gpt(disk): """Format disk for use as a Windows OS disk using the GPT layout.""" script = [ @@ -126,6 +130,7 @@ def format_gpt(disk): # Run run_diskpart(script) + def format_mbr(disk): """Format disk for use as a Windows OS disk using the MBR layout.""" script = [ @@ -155,6 +160,7 @@ def format_mbr(disk): # Run run_diskpart(script) + def mount_windows_share(): """Mount the Windows images share unless already mounted.""" if not WINDOWS_SERVER['Mounted']: @@ -163,6 +169,7 @@ def mount_windows_share(): # error by trying to mount the same server with multiple credentials. mount_network_share(WINDOWS_SERVER, read_write=True) + def select_windows_version(): """Select Windows version from a menu, returns dict.""" actions = [ @@ -180,6 +187,7 @@ def select_windows_version(): elif selection == 'M': raise GenericAbort + def setup_windows(windows_image, windows_version): """Apply a Windows image to W:""" cmd = [ @@ -192,6 +200,7 @@ def setup_windows(windows_image, windows_version): cmd.extend(windows_image['Glob']) run_program(cmd) + def setup_windows_re(windows_version, windows_letter='W', tools_letter='T'): """Setup the WinRE partition.""" win = r'{}:\Windows'.format(windows_letter) @@ -210,6 +219,7 @@ def setup_windows_re(windows_version, windows_letter='W', tools_letter='T'): '/target', win] run_program(cmd) + def update_boot_partition(system_letter='S', windows_letter='W', mode='ALL'): """Setup the Windows boot partition.""" cmd = [ @@ -220,6 +230,7 @@ def update_boot_partition(system_letter='S', windows_letter='W', mode='ALL'): '/f', mode] run_program(cmd) + def wim_contains_image(filename, imagename): """Check if an ESD/WIM contains the specified image, returns bool.""" cmd = [ @@ -234,6 +245,7 @@ def wim_contains_image(filename, imagename): return True + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/winpe_menus.py b/.bin/Scripts/functions/winpe_menus.py index f42a6144..22df7449 100644 --- a/.bin/Scripts/functions/winpe_menus.py +++ b/.bin/Scripts/functions/winpe_menus.py @@ -4,6 +4,7 @@ from functions.backup import * from functions.disk import * from functions.windows_setup import * + # STATIC VARIABLES FAST_COPY_PE_ARGS = [ '/cmd=noexist_only', @@ -50,6 +51,7 @@ PE_TOOLS = { }, } + def check_pe_tools(): """Fix tool paths for WinPE layout.""" for k in PE_TOOLS.keys(): @@ -61,6 +63,7 @@ def check_pe_tools(): global_vars['Tools']['wimlib-imagex'], re.IGNORECASE) + def menu_backup(): """Take backup images of partition(s) in the WIM format.""" errors = False @@ -211,6 +214,7 @@ def menu_backup(): sleep(30) pause('\nPress Enter to return to main menu... ') + def menu_root(): """Main WinPE menu.""" check_pe_tools() @@ -249,6 +253,7 @@ def menu_root(): else: sys.exit() + def menu_setup(): """Format a disk, apply a Windows image, and create boot files.""" errors = False @@ -409,6 +414,7 @@ def menu_setup(): sleep(30) pause('\nPress Enter to return to main menu... ') + def menu_tools(): """Tool launcher menu.""" tools = [{'Name': k} for k in sorted(PE_TOOLS.keys())] @@ -438,6 +444,7 @@ def menu_tools(): elif (selection == 'M'): break + def select_minidump_path(): """Select BSOD minidump path from a menu.""" dumps = [] @@ -467,6 +474,7 @@ def select_minidump_path(): main_entries = dumps) return dumps[int(selection) - 1]['Name'] + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/hw-diags-network b/.bin/Scripts/hw-diags-network index 7e0b0bed..3caa7adb 100755 --- a/.bin/Scripts/hw-diags-network +++ b/.bin/Scripts/hw-diags-network @@ -10,11 +10,13 @@ os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.network import * + def check_connection(): if not is_connected(): # Raise to cause NS in try_and_print() raise Exception + if __name__ == '__main__': try: # Prep diff --git a/.bin/Scripts/msword-search b/.bin/Scripts/msword-search index 380c6ed5..19bbe688 100755 --- a/.bin/Scripts/msword-search +++ b/.bin/Scripts/msword-search @@ -22,6 +22,7 @@ init_global_vars() REGEX_DOC_FILES = re.compile(r'\.docx?$', re.IGNORECASE) + def scan_for_docs(path): for entry in os.scandir(path): if entry.is_dir(follow_symlinks=False): @@ -29,6 +30,7 @@ def scan_for_docs(path): elif entry.is_file and REGEX_DOC_FILES.search(entry.name): yield entry + def scan_file(file_path, search): match = False try: @@ -45,6 +47,7 @@ def scan_file(file_path, search): return entry.path if match else None + if __name__ == '__main__': try: # Prep From c96e2f252c80166acc2863a8df0e3e9ef6316858 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 21:38:25 -0700 Subject: [PATCH 142/186] Cleaned up imports --- .bin/Scripts/functions/browsers.py | 1 - .bin/Scripts/functions/common.py | 3 +-- .bin/Scripts/functions/data.py | 3 +-- .bin/Scripts/functions/ddrescue.py | 1 - .bin/Scripts/functions/info.py | 4 +--- .bin/Scripts/functions/network.py | 3 --- .bin/Scripts/functions/setup.py | 1 - .bin/Scripts/functions/update.py | 1 - 8 files changed, 3 insertions(+), 14 deletions(-) diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index e73399bc..34c0252f 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -1,7 +1,6 @@ # Wizard Kit: Functions - Browsers from functions.common import * - from operator import itemgetter diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index a0a87b76..368ce687 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -14,11 +14,10 @@ except ModuleNotFoundError: if psutil.WINDOWS: raise -from subprocess import CalledProcessError - from settings.main import * from settings.tools import * from settings.windows_builds import * +from subprocess import CalledProcessError # Global variables diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index a151b24c..7dcee3cf 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -3,9 +3,8 @@ import ctypes import json -from operator import itemgetter - from functions.common import * +from operator import itemgetter # STATIC VARIABLES diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index e6def7f2..a2b27e5e 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -9,7 +9,6 @@ import stat import time from collections import OrderedDict -from functions.common import * from functions.data import * from functions.hw_diags import * from functions.tmux import * diff --git a/.bin/Scripts/functions/info.py b/.bin/Scripts/functions/info.py index adee5f80..ad71bce7 100644 --- a/.bin/Scripts/functions/info.py +++ b/.bin/Scripts/functions/info.py @@ -1,10 +1,8 @@ # Wizard Kit: Functions - Information from borrowed import knownpaths -from operator import itemgetter - -from functions.common import * from functions.activation import * +from operator import itemgetter # STATIC VARIABLES diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index 9d9b7af1..f210a2b0 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -6,9 +6,6 @@ import os import shutil import sys -# Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) from functions.common import * diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index 749058e3..85943d2e 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -1,6 +1,5 @@ # Wizard Kit: Functions - Setup -from functions.common import * from functions.update import * diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 439bb6bf..b4aa3f7e 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -2,7 +2,6 @@ import requests -from functions.common import * from functions.data import * from settings.launchers import * from settings.music import * From 5af0996259e4bd6353c78926843c83bf5f1c3359 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 21:42:00 -0700 Subject: [PATCH 143/186] Removed whitespace on empty lines --- .bin/Scripts/connect-to-network | 2 +- .bin/Scripts/ddrescue-tui-menu | 2 +- .bin/Scripts/hw-diags-audio | 4 ++-- .bin/Scripts/hw-diags-network | 2 +- .bin/Scripts/mount-backup-shares | 2 +- .bin/Scripts/msword-search | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/connect-to-network b/.bin/Scripts/connect-to-network index e5100303..5e04e903 100755 --- a/.bin/Scripts/connect-to-network +++ b/.bin/Scripts/connect-to-network @@ -18,7 +18,7 @@ if __name__ == '__main__': # Connect connect_to_network() - + # Done print_standard('\nDone.') #pause("Press Enter to exit...") diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index 66ed058a..f1f3b08c 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -42,7 +42,7 @@ if __name__ == '__main__': else: if not re.search(r'^-*(h|help\?)', run_mode, re.IGNORECASE): print_error('Invalid mode.') - + # Done print_standard('\nDone.') pause("Press Enter to exit...") diff --git a/.bin/Scripts/hw-diags-audio b/.bin/Scripts/hw-diags-audio index d1b09694..2c52c467 100755 --- a/.bin/Scripts/hw-diags-audio +++ b/.bin/Scripts/hw-diags-audio @@ -23,14 +23,14 @@ if __name__ == '__main__': run_program('amixer -q set "PCM" 90% unmute'.split()) except subprocess.CalledProcessError: print_error('Failed to set volume') - + # Run tests for mode in ['pink', 'wav']: run_program( cmd = 'speaker-test -c 2 -l 1 -t {}'.format(mode).split(), check = False, pipe = False) - + # Done #print_standard('\nDone.') #pause("Press Enter to exit...") diff --git a/.bin/Scripts/hw-diags-network b/.bin/Scripts/hw-diags-network index 3caa7adb..2ae0e263 100755 --- a/.bin/Scripts/hw-diags-network +++ b/.bin/Scripts/hw-diags-network @@ -36,7 +36,7 @@ if __name__ == '__main__': try_and_print(message='DNS Resolution:', function=ping, cs='OK') try_and_print(message='Speedtest:', function=speedtest, print_return=True) - + # Done print_standard('\nDone.') #pause("Press Enter to exit...") diff --git a/.bin/Scripts/mount-backup-shares b/.bin/Scripts/mount-backup-shares index 6605a81c..69374a9b 100755 --- a/.bin/Scripts/mount-backup-shares +++ b/.bin/Scripts/mount-backup-shares @@ -26,7 +26,7 @@ if __name__ == '__main__': else: # Couldn't connect print_error('ERROR: No network connectivity.') - + # Done print_standard('\nDone.') #pause("Press Enter to exit...") diff --git a/.bin/Scripts/msword-search b/.bin/Scripts/msword-search index 19bbe688..c0461dda 100755 --- a/.bin/Scripts/msword-search +++ b/.bin/Scripts/msword-search @@ -44,7 +44,7 @@ def scan_file(file_path, search): except Exception: # Ignore errors since files may be corrupted pass - + return entry.path if match else None @@ -72,7 +72,7 @@ if __name__ == '__main__': print_standard(match) else: print_error('No matches found.') - + # Done print_standard('\nDone.') #pause("Press Enter to exit...") From 922d632afb49a7090686e5eac92c2bb6e9e71a2b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 21:44:46 -0700 Subject: [PATCH 144/186] Removed trailing whitespace --- .bin/Scripts/functions/update.py | 2 +- .bin/Scripts/functions/windows_setup.py | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index b4aa3f7e..a9c7811f 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -883,7 +883,7 @@ def update_xmplay(): # Extract files extract_temp_to_cbin('xmplay.zip', 'XMPlay', mode='e', sz_args=['xmplay.exe', 'xmplay.txt']) - for item in ['xmp-7z', 'xmp-gme', 'xmp-rar', 'WAModern']: + for item in ['xmp-7z', 'xmp-gme', 'xmp-rar', 'WAModern']: filter = [] if item == 'WAModern': filter.append('WAModern NightVision.xmpskin') diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index 82120843..e790a4b6 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -12,27 +12,27 @@ WINDOWS_VERSIONS = [ {'Name': 'Windows 7 Home Premium', 'Image File': 'Win7', 'Image Name': 'Windows 7 HOMEPREMIUM'}, - {'Name': 'Windows 7 Professional', - 'Image File': 'Win7', + {'Name': 'Windows 7 Professional', + 'Image File': 'Win7', 'Image Name': 'Windows 7 PROFESSIONAL'}, - {'Name': 'Windows 7 Ultimate', - 'Image File': 'Win7', + {'Name': 'Windows 7 Ultimate', + 'Image File': 'Win7', 'Image Name': 'Windows 7 ULTIMATE'}, - {'Name': 'Windows 8.1', - 'Image File': 'Win8', + {'Name': 'Windows 8.1', + 'Image File': 'Win8', 'Image Name': 'Windows 8.1', 'CRLF': True}, - {'Name': 'Windows 8.1 Pro', - 'Image File': 'Win8', + {'Name': 'Windows 8.1 Pro', + 'Image File': 'Win8', 'Image Name': 'Windows 8.1 Pro'}, - {'Name': 'Windows 10 Home', - 'Image File': 'Win10', + {'Name': 'Windows 10 Home', + 'Image File': 'Win10', 'Image Name': 'Windows 10 Home', 'CRLF': True}, - {'Name': 'Windows 10 Pro', - 'Image File': 'Win10', + {'Name': 'Windows 10 Pro', + 'Image File': 'Win10', 'Image Name': 'Windows 10 Pro'}, ] From 3e733e65e94545bab96222b21fecd186906299f3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 22:17:23 -0700 Subject: [PATCH 145/186] Updated network.py --- .bin/Scripts/functions/network.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index f210a2b0..492ba16f 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -1,6 +1,4 @@ -#!/bin/python3 -# -## Wizard Kit: Functions - Network +# Wizard Kit: Functions - Network import os import shutil From f321dee54fca9b0c8a4f5d556102ad207a224682 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 22:28:17 -0700 Subject: [PATCH 146/186] Switched indents to 2 spaces --- .bin/Scripts/activate.py | 92 ++++----- .bin/Scripts/cbs_fix.py | 50 ++--- .bin/Scripts/check_disk.py | 84 ++++---- .bin/Scripts/dism.py | 86 +++++---- .bin/Scripts/install_sw_bundle.py | 100 +++++----- .bin/Scripts/install_vcredists.py | 38 ++-- .bin/Scripts/safemode_enter.py | 46 ++--- .bin/Scripts/safemode_exit.py | 46 ++--- .bin/Scripts/sfc_scan.py | 50 ++--- .bin/Scripts/system_checklist.py | 212 ++++++++++---------- .bin/Scripts/system_diagnostics.py | 298 ++++++++++++++--------------- .bin/Scripts/transferred_keys.py | 28 +-- .bin/Scripts/update_kit.py | 234 +++++++++++----------- .bin/Scripts/user_checklist.py | 124 ++++++------ .bin/Scripts/user_data_transfer.py | 90 ++++----- .bin/Scripts/winpe_root_menu.py | 14 +- 16 files changed, 808 insertions(+), 784 deletions(-) diff --git a/.bin/Scripts/activate.py b/.bin/Scripts/activate.py index 642e5edd..cf17263f 100644 --- a/.bin/Scripts/activate.py +++ b/.bin/Scripts/activate.py @@ -11,52 +11,54 @@ init_global_vars() os.system('title {}: Windows Activation Tool'.format(KIT_NAME_FULL)) if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: Windows Activation Tool\n'.format(KIT_NAME_FULL)) - # Bail early if already activated - if windows_is_activated(): - print_info('This system is already activated') - sleep(5) - exit_script() - other_results = { - 'Error': { - 'BIOSKeyNotFoundError': 'BIOS key not found.', - }} + try: + stay_awake() + clear_screen() + print_info('{}: Windows Activation Tool\n'.format(KIT_NAME_FULL)) + # Bail early if already activated + if windows_is_activated(): + print_info('This system is already activated') + sleep(5) + exit_script() + other_results = { + 'Error': { + 'BIOSKeyNotFoundError': 'BIOS key not found.', + }} - # Determine activation method - activation_methods = [ - {'Name': 'Activate with BIOS key', 'Function': activate_with_bios}, - ] - if global_vars['OS']['Version'] not in ('8', '8.1', '10'): - activation_methods[0]['Disabled'] = True - actions = [ - {'Name': 'Quit', 'Letter': 'Q'}, - ] + # Determine activation method + activation_methods = [ + {'Name': 'Activate with BIOS key', 'Function': activate_with_bios}, + ] + if global_vars['OS']['Version'] not in ('8', '8.1', '10'): + activation_methods[0]['Disabled'] = True + actions = [ + {'Name': 'Quit', 'Letter': 'Q'}, + ] - while True: - selection = menu_select( - '{}: Windows Activation Menu'.format(KIT_NAME_FULL), - main_entries=activation_methods, action_entries=actions) + while True: + selection = menu_select( + '{}: Windows Activation Menu'.format(KIT_NAME_FULL), + main_entries=activation_methods, action_entries=actions) - if (selection.isnumeric()): - result = try_and_print( - message = activation_methods[int(selection)-1]['Name'], - function = activation_methods[int(selection)-1]['Function'], - other_results=other_results) - if result['CS']: - break - else: - sleep(2) - elif selection == 'Q': - exit_script() - - # Done - print_success('\nDone.') - pause("Press Enter to exit...") + if (selection.isnumeric()): + result = try_and_print( + message = activation_methods[int(selection)-1]['Name'], + function = activation_methods[int(selection)-1]['Function'], + other_results=other_results) + if result['CS']: + break + else: + sleep(2) + elif selection == 'Q': exit_script() - except SystemExit: - pass - except: - major_exception() + + # Done + print_success('\nDone.') + pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/cbs_fix.py b/.bin/Scripts/cbs_fix.py index 9a8ff10c..460a32a5 100644 --- a/.bin/Scripts/cbs_fix.py +++ b/.bin/Scripts/cbs_fix.py @@ -13,30 +13,32 @@ os.system('title {}: CBS Cleanup'.format(KIT_NAME_FULL)) set_log_file('CBS Cleanup.log') if __name__ == '__main__': - try: - # Prep - stay_awake() - clear_screen() - folder_path = r'{}\Backups'.format(KIT_NAME_SHORT) - dest = select_destination(folder_path=folder_path, - prompt='Which disk are we using for temp data and backup?') + try: + # Prep + stay_awake() + clear_screen() + folder_path = r'{}\Backups'.format(KIT_NAME_SHORT) + dest = select_destination(folder_path=folder_path, + prompt='Which disk are we using for temp data and backup?') - # Show details - print_info('{}: CBS Cleanup Tool\n'.format(KIT_NAME_FULL)) - show_data('Backup / Temp path:', dest) - print_standard('\n') - if (not ask('Proceed with CBS cleanup?')): - abort() + # Show details + print_info('{}: CBS Cleanup Tool\n'.format(KIT_NAME_FULL)) + show_data('Backup / Temp path:', dest) + print_standard('\n') + if (not ask('Proceed with CBS cleanup?')): + abort() - # Run Cleanup - try_and_print(message='Running cleanup...', function=cleanup_cbs, - cs='Done', dest_folder=dest) + # Run Cleanup + try_and_print(message='Running cleanup...', function=cleanup_cbs, + cs='Done', dest_folder=dest) - # Done - print_standard('\nDone.') - pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/check_disk.py b/.bin/Scripts/check_disk.py index 7e59fb2b..1140f4fc 100644 --- a/.bin/Scripts/check_disk.py +++ b/.bin/Scripts/check_disk.py @@ -12,45 +12,47 @@ os.system('title {}: Check Disk Tool'.format(KIT_NAME_FULL)) set_log_file('Check Disk.log') if __name__ == '__main__': - try: - stay_awake() - clear_screen() - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - }, - 'Warning': { - 'GenericRepair': 'Repaired', - 'UnsupportedOSError': 'Unsupported OS', - }} - options = [ - {'Name': 'Run CHKDSK scan (read-only)', 'Repair': False}, - {'Name': 'Schedule CHKDSK scan (offline repair)', 'Repair': True}] - actions = [{'Name': 'Quit', 'Letter': 'Q'}] - selection = menu_select( - '{}: Check Disk Menu\n'.format(KIT_NAME_FULL), - main_entries=options, - action_entries=actions) - print_info('{}: Check Disk Menu\n'.format(KIT_NAME_FULL)) - if selection == 'Q': - abort() - elif selection.isnumeric(): - repair = options[int(selection)-1]['Repair'] - if repair: - cs = 'Scheduled' - else: - cs = 'CS' - message = 'CHKDSK ({SYSTEMDRIVE})...'.format(**global_vars['Env']) - try_and_print(message=message, function=run_chkdsk, - cs=cs, other_results=other_results, repair=repair) - else: - abort() + try: + stay_awake() + clear_screen() + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + }, + 'Warning': { + 'GenericRepair': 'Repaired', + 'UnsupportedOSError': 'Unsupported OS', + }} + options = [ + {'Name': 'Run CHKDSK scan (read-only)', 'Repair': False}, + {'Name': 'Schedule CHKDSK scan (offline repair)', 'Repair': True}] + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + selection = menu_select( + '{}: Check Disk Menu\n'.format(KIT_NAME_FULL), + main_entries=options, + action_entries=actions) + print_info('{}: Check Disk Menu\n'.format(KIT_NAME_FULL)) + if selection == 'Q': + abort() + elif selection.isnumeric(): + repair = options[int(selection)-1]['Repair'] + if repair: + cs = 'Scheduled' + else: + cs = 'CS' + message = 'CHKDSK ({SYSTEMDRIVE})...'.format(**global_vars['Env']) + try_and_print(message=message, function=run_chkdsk, + cs=cs, other_results=other_results, repair=repair) + else: + abort() - # Done - print_success('Done.') - pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_success('Done.') + pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/dism.py b/.bin/Scripts/dism.py index e49a9512..8b227da2 100644 --- a/.bin/Scripts/dism.py +++ b/.bin/Scripts/dism.py @@ -12,46 +12,48 @@ os.system('title {}: DISM helper Tool'.format(KIT_NAME_FULL)) set_log_file('DISM Helper.log') if __name__ == '__main__': - try: - stay_awake() - clear_screen() - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - }, - 'Warning': { - 'GenericRepair': 'Repaired', - 'UnsupportedOSError': 'Unsupported OS', - }} - disabled = bool(global_vars['OS']['Version'] not in ('8', '8.1', '10')) - options = [ - {'Name': 'Check Health', 'Repair': False, 'Disabled': disabled}, - {'Name': 'Restore Health', 'Repair': True, 'Disabled': disabled}] - actions = [{'Name': 'Quit', 'Letter': 'Q'}] - selection = menu_select( - '{}: DISM Menu\n'.format(KIT_NAME_FULL), - main_entries=options, - action_entries=actions) - print_info('{}: DISM Menu\n'.format(KIT_NAME_FULL)) - if selection == 'Q': - abort() - elif selection.isnumeric(): - repair = options[int(selection)-1]['Repair'] - if repair: - message='DISM RestoreHealth...' - else: - message='DISM ScanHealth...' - try_and_print(message=message, function=run_dism, - cs='No corruption', ns='Corruption detected', - other_results=other_results, repair=repair) - else: - abort() + try: + stay_awake() + clear_screen() + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + }, + 'Warning': { + 'GenericRepair': 'Repaired', + 'UnsupportedOSError': 'Unsupported OS', + }} + disabled = bool(global_vars['OS']['Version'] not in ('8', '8.1', '10')) + options = [ + {'Name': 'Check Health', 'Repair': False, 'Disabled': disabled}, + {'Name': 'Restore Health', 'Repair': True, 'Disabled': disabled}] + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + selection = menu_select( + '{}: DISM Menu\n'.format(KIT_NAME_FULL), + main_entries=options, + action_entries=actions) + print_info('{}: DISM Menu\n'.format(KIT_NAME_FULL)) + if selection == 'Q': + abort() + elif selection.isnumeric(): + repair = options[int(selection)-1]['Repair'] + if repair: + message='DISM RestoreHealth...' + else: + message='DISM ScanHealth...' + try_and_print(message=message, function=run_dism, + cs='No corruption', ns='Corruption detected', + other_results=other_results, repair=repair) + else: + abort() - # Done - print_success('Done.') - pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_success('Done.') + pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/install_sw_bundle.py b/.bin/Scripts/install_sw_bundle.py index 5ee1a8f3..4386bfb0 100644 --- a/.bin/Scripts/install_sw_bundle.py +++ b/.bin/Scripts/install_sw_bundle.py @@ -12,55 +12,55 @@ os.system('title {}: SW Bundle Tool'.format(KIT_NAME_FULL)) set_log_file('Install SW Bundle.log') if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: SW Bundle Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - 'FileNotFoundError': 'File not found', - }, - 'Warning': { - 'GenericRepair': 'Repaired', - 'UnsupportedOSError': 'Unsupported OS', - }} - answer_extensions = ask('Install Extensions?') - answer_adobe_reader = ask('Install Adobe Reader?') - answer_vcr = ask('Install Visual C++ Runtimes?') - answer_ninite = ask('Install Ninite Bundle?') - if answer_ninite and global_vars['OS']['Version'] in ['7']: - # Vista is dead, not going to check for it - answer_mse = ask('Install MSE?') - else: - answer_mse = False + try: + stay_awake() + clear_screen() + print_info('{}: SW Bundle Tool\n'.format(KIT_NAME_FULL)) + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + 'FileNotFoundError': 'File not found', + }, + 'Warning': { + 'GenericRepair': 'Repaired', + 'UnsupportedOSError': 'Unsupported OS', + }} + answer_extensions = ask('Install Extensions?') + answer_adobe_reader = ask('Install Adobe Reader?') + answer_vcr = ask('Install Visual C++ Runtimes?') + answer_ninite = ask('Install Ninite Bundle?') + if answer_ninite and global_vars['OS']['Version'] in ['7']: + # Vista is dead, not going to check for it + answer_mse = ask('Install MSE?') + else: + answer_mse = False - print_info('Installing Programs') - if answer_adobe_reader: - try_and_print(message='Adobe Reader DC...', - function=install_adobe_reader, other_results=other_results) - if answer_vcr: - install_vcredists() - if answer_ninite: - try_and_print(message='Ninite bundle...', - function=install_ninite_bundle, cs='Started', - mse=answer_mse, other_results=other_results) - if answer_extensions: - wait_for_process('ninite.exe') - print_info('Installing Extensions') - try_and_print(message='Classic Shell skin...', - function=install_classicstart_skin, - other_results=other_results) - try_and_print(message='Google Chrome extensions...', - function=install_chrome_extensions) - try_and_print(message='Mozilla Firefox extensions...', - function=install_firefox_extensions, - other_results=other_results) - print_standard('\nDone.') - exit_script() - except SystemExit: - pass - except: - major_exception() + print_info('Installing Programs') + if answer_adobe_reader: + try_and_print(message='Adobe Reader DC...', + function=install_adobe_reader, other_results=other_results) + if answer_vcr: + install_vcredists() + if answer_ninite: + try_and_print(message='Ninite bundle...', + function=install_ninite_bundle, cs='Started', + mse=answer_mse, other_results=other_results) + if answer_extensions: + wait_for_process('ninite.exe') + print_info('Installing Extensions') + try_and_print(message='Classic Shell skin...', + function=install_classicstart_skin, + other_results=other_results) + try_and_print(message='Google Chrome extensions...', + function=install_chrome_extensions) + try_and_print(message='Mozilla Firefox extensions...', + function=install_firefox_extensions, + other_results=other_results) + print_standard('\nDone.') + exit_script() + except SystemExit: + pass + except: + major_exception() -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/install_vcredists.py b/.bin/Scripts/install_vcredists.py index fd953551..1a722bf9 100644 --- a/.bin/Scripts/install_vcredists.py +++ b/.bin/Scripts/install_vcredists.py @@ -12,23 +12,25 @@ os.system('title {}: Install Visual C++ Runtimes'.format(KIT_NAME_FULL)) set_log_file('Install Visual C++ Runtimes.log') if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: Install Visual C++ Runtimes\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - }} + try: + stay_awake() + clear_screen() + print_info('{}: Install Visual C++ Runtimes\n'.format(KIT_NAME_FULL)) + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + }} - if ask('Install Visual C++ Runtimes?'): - install_vcredists() - else: - abort() + if ask('Install Visual C++ Runtimes?'): + install_vcredists() + else: + abort() - print_standard('\nDone.') - exit_script() - except SystemExit: - pass - except: - major_exception() + print_standard('\nDone.') + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/safemode_enter.py b/.bin/Scripts/safemode_enter.py index cce7e28a..027f4987 100644 --- a/.bin/Scripts/safemode_enter.py +++ b/.bin/Scripts/safemode_enter.py @@ -11,28 +11,30 @@ init_global_vars() os.system('title {}: SafeMode Tool'.format(KIT_NAME_FULL)) if __name__ == '__main__': - try: - clear_screen() - print_info('{}: SafeMode Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': {'CalledProcessError': 'Unknown Error'}, - 'Warning': {}} + try: + clear_screen() + print_info('{}: SafeMode Tool\n'.format(KIT_NAME_FULL)) + other_results = { + 'Error': {'CalledProcessError': 'Unknown Error'}, + 'Warning': {}} - if not ask('Enable booting to SafeMode (with Networking)?'): - abort() + if not ask('Enable booting to SafeMode (with Networking)?'): + abort() - # Configure SafeMode - try_and_print(message='Set BCD option...', - function=enable_safemode, other_results=other_results) - try_and_print(message='Enable MSI in SafeMode...', - function=enable_safemode_msi, other_results=other_results) + # Configure SafeMode + try_and_print(message='Set BCD option...', + function=enable_safemode, other_results=other_results) + try_and_print(message='Enable MSI in SafeMode...', + function=enable_safemode_msi, other_results=other_results) - # Done - print_standard('\nDone.') - pause('Press Enter to reboot...') - reboot() - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + pause('Press Enter to reboot...') + reboot() + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/safemode_exit.py b/.bin/Scripts/safemode_exit.py index af66222e..85376d51 100644 --- a/.bin/Scripts/safemode_exit.py +++ b/.bin/Scripts/safemode_exit.py @@ -11,28 +11,30 @@ init_global_vars() os.system('title {}: SafeMode Tool'.format(KIT_NAME_FULL)) if __name__ == '__main__': - try: - clear_screen() - print_info('{}: SafeMode Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': {'CalledProcessError': 'Unknown Error'}, - 'Warning': {}} + try: + clear_screen() + print_info('{}: SafeMode Tool\n'.format(KIT_NAME_FULL)) + other_results = { + 'Error': {'CalledProcessError': 'Unknown Error'}, + 'Warning': {}} - if not ask('Disable booting to SafeMode?'): - abort() + if not ask('Disable booting to SafeMode?'): + abort() - # Configure SafeMode - try_and_print(message='Remove BCD option...', - function=disable_safemode, other_results=other_results) - try_and_print(message='Disable MSI in SafeMode...', - function=disable_safemode_msi, other_results=other_results) + # Configure SafeMode + try_and_print(message='Remove BCD option...', + function=disable_safemode, other_results=other_results) + try_and_print(message='Disable MSI in SafeMode...', + function=disable_safemode_msi, other_results=other_results) - # Done - print_standard('\nDone.') - pause('Press Enter to reboot...') - reboot() - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + pause('Press Enter to reboot...') + reboot() + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/sfc_scan.py b/.bin/Scripts/sfc_scan.py index d7a3d3fc..9864306a 100644 --- a/.bin/Scripts/sfc_scan.py +++ b/.bin/Scripts/sfc_scan.py @@ -12,28 +12,30 @@ os.system('title {}: SFC Tool'.format(KIT_NAME_FULL)) set_log_file('SFC Tool.log') if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: SFC Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - }, - 'Warning': { - 'GenericRepair': 'Repaired', - }} - if ask('Run a SFC scan now?'): - try_and_print(message='SFC scan...', - function=run_sfc_scan, other_results=other_results) - else: - abort() + try: + stay_awake() + clear_screen() + print_info('{}: SFC Tool\n'.format(KIT_NAME_FULL)) + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + }, + 'Warning': { + 'GenericRepair': 'Repaired', + }} + if ask('Run a SFC scan now?'): + try_and_print(message='SFC scan...', + function=run_sfc_scan, other_results=other_results) + else: + abort() - # Done - print_standard('\nDone.') - pause('Press Enter to exit...') - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + pause('Press Enter to exit...') + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/system_checklist.py b/.bin/Scripts/system_checklist.py index 52720438..1aa95d3e 100644 --- a/.bin/Scripts/system_checklist.py +++ b/.bin/Scripts/system_checklist.py @@ -17,112 +17,112 @@ os.system('title {}: System Checklist Tool'.format(KIT_NAME_FULL)) set_log_file('System Checklist.log') if __name__ == '__main__': + try: + stay_awake() + clear_screen() + print_info('{}: System Checklist Tool\n'.format(KIT_NAME_FULL)) + ticket_number = get_ticket_number() + other_results = { + 'Error': { + 'BIOSKeyNotFoundError': 'BIOS key not found', + 'CalledProcessError': 'Unknown Error', + 'FileNotFoundError': 'File not found', + 'GenericError': 'Unknown Error', + 'SecureBootDisabledError': 'Disabled', + }, + 'Warning': { + 'OSInstalledLegacyError': 'OS installed Legacy', + 'SecureBootNotAvailError': 'Not available', + 'SecureBootUnknownError': 'Unknown', + }} + if ENABLED_TICKET_NUMBERS: + print_info('Starting System Checklist for Ticket #{}\n'.format( + ticket_number)) + + # Configure + print_info('Configure') + if global_vars['OS']['Version'] == '10': + try_and_print(message='Explorer...', + function=config_explorer_system, cs='Done') + try_and_print(message='Updating Clock...', + function=update_clock, cs='Done') + + # Cleanup + print_info('Cleanup') + try_and_print(message='AdwCleaner...', + function=cleanup_adwcleaner, cs='Done', other_results=other_results) + try_and_print(message='Desktop...', + function=cleanup_desktop, cs='Done') + try_and_print(message='{}...'.format(KIT_NAME_FULL), + function=delete_empty_folders, cs='Done', + folder_path=global_vars['ClientDir']) + + # Export system info + print_info('Backup System Information') + try_and_print(message='AIDA64 reports...', + function=run_aida64, cs='Done', other_results=other_results) + try_and_print(message='File listing...', + function=backup_file_list, cs='Done', other_results=other_results) + try_and_print(message='Power plans...', + function=backup_power_plans, cs='Done') + try_and_print(message='Product Keys...', other_results=other_results, + function=run_produkey, cs='Done') + try_and_print(message='Registry...', + function=backup_registry, cs='Done', other_results=other_results) + + # User data + print_info('User Data') + show_user_data_summary() + + # Summary + print_info('Summary') + try_and_print(message='Operating System:', + function=show_os_name, ns='Unknown', silent_function=False) + try_and_print(message='Activation:', + function=show_os_activation, ns='Unknown', silent_function=False) + if (not windows_is_activated() + and global_vars['OS']['Version'] in ('8', '8.1', '10')): + try_and_print(message='BIOS Activation:', + function=activate_with_bios, + other_results=other_results) + try_and_print(message='Secure Boot Status:', + function=check_secure_boot_status, other_results=other_results) + try_and_print(message='Installed RAM:', + function=show_installed_ram, ns='Unknown', silent_function=False) + show_free_space() + try_and_print(message='Installed Antivirus:', + function=get_installed_antivirus, ns='Unknown', + other_results=other_results, print_return=True) + try_and_print(message='Installed Office:', + function=get_installed_office, ns='Unknown', + other_results=other_results, print_return=True) + + # Play audio, show devices, open Windows updates, and open Activation + try_and_print(message='Opening Device Manager...', + function=open_device_manager, cs='Started') + try_and_print(message='Opening HWiNFO (Sensors)...', + function=run_hwinfo_sensors, cs='Started', other_results=other_results) + try_and_print(message='Opening Windows Updates...', + function=open_windows_updates, cs='Started') + if not windows_is_activated(): + try_and_print(message='Opening Windows Activation...', + function=open_windows_activation, cs='Started') + sleep(3) + try_and_print(message='Running XMPlay...', + function=run_xmplay, cs='Started', other_results=other_results) try: - stay_awake() - clear_screen() - print_info('{}: System Checklist Tool\n'.format(KIT_NAME_FULL)) - ticket_number = get_ticket_number() - other_results = { - 'Error': { - 'BIOSKeyNotFoundError': 'BIOS key not found', - 'CalledProcessError': 'Unknown Error', - 'FileNotFoundError': 'File not found', - 'GenericError': 'Unknown Error', - 'SecureBootDisabledError': 'Disabled', - }, - 'Warning': { - 'OSInstalledLegacyError': 'OS installed Legacy', - 'SecureBootNotAvailError': 'Not available', - 'SecureBootUnknownError': 'Unknown', - }} - if ENABLED_TICKET_NUMBERS: - print_info('Starting System Checklist for Ticket #{}\n'.format( - ticket_number)) - - # Configure - print_info('Configure') - if global_vars['OS']['Version'] == '10': - try_and_print(message='Explorer...', - function=config_explorer_system, cs='Done') - try_and_print(message='Updating Clock...', - function=update_clock, cs='Done') - - # Cleanup - print_info('Cleanup') - try_and_print(message='AdwCleaner...', - function=cleanup_adwcleaner, cs='Done', other_results=other_results) - try_and_print(message='Desktop...', - function=cleanup_desktop, cs='Done') - try_and_print(message='{}...'.format(KIT_NAME_FULL), - function=delete_empty_folders, cs='Done', - folder_path=global_vars['ClientDir']) - - # Export system info - print_info('Backup System Information') - try_and_print(message='AIDA64 reports...', - function=run_aida64, cs='Done', other_results=other_results) - try_and_print(message='File listing...', - function=backup_file_list, cs='Done', other_results=other_results) - try_and_print(message='Power plans...', - function=backup_power_plans, cs='Done') - try_and_print(message='Product Keys...', other_results=other_results, - function=run_produkey, cs='Done') - try_and_print(message='Registry...', - function=backup_registry, cs='Done', other_results=other_results) - - # User data - print_info('User Data') - show_user_data_summary() - - # Summary - print_info('Summary') - try_and_print(message='Operating System:', - function=show_os_name, ns='Unknown', silent_function=False) - try_and_print(message='Activation:', - function=show_os_activation, ns='Unknown', silent_function=False) - if (not windows_is_activated() - and global_vars['OS']['Version'] in ('8', '8.1', '10')): - try_and_print(message='BIOS Activation:', - function=activate_with_bios, - other_results=other_results) - try_and_print(message='Secure Boot Status:', - function=check_secure_boot_status, other_results=other_results) - try_and_print(message='Installed RAM:', - function=show_installed_ram, ns='Unknown', silent_function=False) - show_free_space() - try_and_print(message='Installed Antivirus:', - function=get_installed_antivirus, ns='Unknown', - other_results=other_results, print_return=True) - try_and_print(message='Installed Office:', - function=get_installed_office, ns='Unknown', - other_results=other_results, print_return=True) - - # Play audio, show devices, open Windows updates, and open Activation - try_and_print(message='Opening Device Manager...', - function=open_device_manager, cs='Started') - try_and_print(message='Opening HWiNFO (Sensors)...', - function=run_hwinfo_sensors, cs='Started', other_results=other_results) - try_and_print(message='Opening Windows Updates...', - function=open_windows_updates, cs='Started') - if not windows_is_activated(): - try_and_print(message='Opening Windows Activation...', - function=open_windows_activation, cs='Started') - sleep(3) - try_and_print(message='Running XMPlay...', - function=run_xmplay, cs='Started', other_results=other_results) - try: - check_secure_boot_status(show_alert=True) - except: - # Only trying to open alert message boxes - pass - - # Done - print_standard('\nDone.') - pause('Press Enter exit...') - exit_script() - except SystemExit: - pass + check_secure_boot_status(show_alert=True) except: - major_exception() + # Only trying to open alert message boxes + pass -# vim: sts=4 sw=4 ts=4 + # Done + print_standard('\nDone.') + pause('Press Enter exit...') + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/system_diagnostics.py b/.bin/Scripts/system_diagnostics.py index ffa28079..c7b363d8 100644 --- a/.bin/Scripts/system_diagnostics.py +++ b/.bin/Scripts/system_diagnostics.py @@ -17,158 +17,158 @@ set_log_file('System Diagnostics.log') # Static Variables BLEACH_BIT_CLEANERS = { - 'Applications': ( - 'adobe_reader.cache', - 'adobe_reader.tmp', - 'amule.tmp', - 'flash.cache', - 'gimp.tmp', - 'hippo_opensim_viewer.cache', - 'java.cache', - 'libreoffice.cache', - 'liferea.cache', - 'miro.cache', - 'openofficeorg.cache', - 'pidgin.cache', - 'secondlife_viewer.Cache', - 'thunderbird.cache', - 'vuze.backup_files', - 'vuze.cache', - 'vuze.tmp', - 'yahoo_messenger.cache', - ), - 'Browsers': ( - 'chromium.cache', - 'chromium.current_session', - 'firefox.cache', - 'firefox.session_restore', - 'google_chrome.cache', - 'google_chrome.session', - 'google_earth.temporary_files', - 'internet_explorer.temporary_files', - 'opera.cache', - 'opera.current_session', - 'safari.cache', - 'seamonkey.cache', - ), - 'System': ( - 'system.clipboard', - 'system.tmp', - 'winapp2_windows.jump_lists', - 'winapp2_windows.ms_search', - 'windows_explorer.run', - 'windows_explorer.search_history', - 'windows_explorer.thumbnails', - ), + 'Applications': ( + 'adobe_reader.cache', + 'adobe_reader.tmp', + 'amule.tmp', + 'flash.cache', + 'gimp.tmp', + 'hippo_opensim_viewer.cache', + 'java.cache', + 'libreoffice.cache', + 'liferea.cache', + 'miro.cache', + 'openofficeorg.cache', + 'pidgin.cache', + 'secondlife_viewer.Cache', + 'thunderbird.cache', + 'vuze.backup_files', + 'vuze.cache', + 'vuze.tmp', + 'yahoo_messenger.cache', + ), + 'Browsers': ( + 'chromium.cache', + 'chromium.current_session', + 'firefox.cache', + 'firefox.session_restore', + 'google_chrome.cache', + 'google_chrome.session', + 'google_earth.temporary_files', + 'internet_explorer.temporary_files', + 'opera.cache', + 'opera.current_session', + 'safari.cache', + 'seamonkey.cache', + ), + 'System': ( + 'system.clipboard', + 'system.tmp', + 'winapp2_windows.jump_lists', + 'winapp2_windows.ms_search', + 'windows_explorer.run', + 'windows_explorer.search_history', + 'windows_explorer.thumbnails', + ), } if __name__ == '__main__': + try: + stay_awake() + clear_screen() + print_info('{}: System Diagnostics Tool\n'.format(KIT_NAME_FULL)) + ticket_number = get_ticket_number() + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + 'FileNotFoundError': 'File not found', + }, + 'Warning': { + 'GenericRepair': 'Repaired', + 'UnsupportedOSError': 'Unsupported OS', + }} + if ENABLED_TICKET_NUMBERS: + print_info('Starting System Diagnostics for Ticket #{}\n'.format( + ticket_number)) + + # Sanitize Environment + print_info('Sanitizing Environment') + try_and_print(message='Running RKill...', + function=run_rkill, cs='Done', other_results=other_results) + try_and_print(message='Running TDSSKiller...', + function=run_tdsskiller, cs='Done', other_results=other_results) + + # Re-run if earlier process was stopped. + stay_awake() + + # Start diags + print_info('Starting Background Scans') + check_connection() + try_and_print(message='Running HitmanPro...', + function=run_hitmanpro, cs='Started', other_results=other_results) + try_and_print(message='Running Autoruns...', + function=run_autoruns, cs='Started', other_results=other_results) + + # OS Health Checks + print_info('OS Health Checks') + try_and_print( + message='CHKDSK ({SYSTEMDRIVE})...'.format(**global_vars['Env']), + function=run_chkdsk, other_results=other_results) + try_and_print(message='SFC scan...', + function=run_sfc_scan, other_results=other_results) + try_and_print(message='DISM CheckHealth...', + function=run_dism, other_results=other_results, repair=False) + + # Scan for supported browsers + print_info('Scanning for browsers') + scan_for_browsers() + + # Run BleachBit cleaners + print_info('BleachBit Cleanup') + for k, v in sorted(BLEACH_BIT_CLEANERS.items()): + try_and_print(message=' {}...'.format(k), + function=run_bleachbit, + cs='Done', other_results=other_results, + cleaners=v, preview=True) + + # Export system info + print_info('Backup System Information') + try_and_print(message='AIDA64 reports...', + function=run_aida64, cs='Done', other_results=other_results) + backup_browsers() + try_and_print(message='File listing...', + function=backup_file_list, cs='Done', other_results=other_results) + try_and_print(message='Power plans...', + function=backup_power_plans, cs='Done') + try_and_print(message='Product Keys...', + function=run_produkey, cs='Done', other_results=other_results) + try_and_print(message='Registry...', + function=backup_registry, cs='Done', other_results=other_results) + + # Summary + print_info('Summary') + try_and_print(message='Operating System:', + function=show_os_name, ns='Unknown', silent_function=False) + try_and_print(message='Activation:', + function=show_os_activation, ns='Unknown', silent_function=False) + try_and_print(message='Installed RAM:', + function=show_installed_ram, ns='Unknown', silent_function=False) + show_free_space() + try_and_print(message='Temp Size:', + function=show_temp_files_size, silent_function=False) + try_and_print(message='Installed Antivirus:', + function=get_installed_antivirus, ns='Unknown', + other_results=other_results, print_return=True) + try_and_print(message='Installed Office:', + function=get_installed_office, ns='Unknown', + other_results=other_results, print_return=True) + try_and_print(message='Product Keys:', + function=get_product_keys, ns='Unknown', print_return=True) + + # User data + print_info('User Data') try: - stay_awake() - clear_screen() - print_info('{}: System Diagnostics Tool\n'.format(KIT_NAME_FULL)) - ticket_number = get_ticket_number() - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - 'FileNotFoundError': 'File not found', - }, - 'Warning': { - 'GenericRepair': 'Repaired', - 'UnsupportedOSError': 'Unsupported OS', - }} - if ENABLED_TICKET_NUMBERS: - print_info('Starting System Diagnostics for Ticket #{}\n'.format( - ticket_number)) + show_user_data_summary() + except Exception: + print_error(' Unknown error.') - # Sanitize Environment - print_info('Sanitizing Environment') - try_and_print(message='Running RKill...', - function=run_rkill, cs='Done', other_results=other_results) - try_and_print(message='Running TDSSKiller...', - function=run_tdsskiller, cs='Done', other_results=other_results) + # Done + print_standard('\nDone.') + pause('Press Enter to exit...') + exit_script() + except SystemExit: + pass + except: + major_exception() - # Re-run if earlier process was stopped. - stay_awake() - - # Start diags - print_info('Starting Background Scans') - check_connection() - try_and_print(message='Running HitmanPro...', - function=run_hitmanpro, cs='Started', other_results=other_results) - try_and_print(message='Running Autoruns...', - function=run_autoruns, cs='Started', other_results=other_results) - - # OS Health Checks - print_info('OS Health Checks') - try_and_print( - message='CHKDSK ({SYSTEMDRIVE})...'.format(**global_vars['Env']), - function=run_chkdsk, other_results=other_results) - try_and_print(message='SFC scan...', - function=run_sfc_scan, other_results=other_results) - try_and_print(message='DISM CheckHealth...', - function=run_dism, other_results=other_results, repair=False) - - # Scan for supported browsers - print_info('Scanning for browsers') - scan_for_browsers() - - # Run BleachBit cleaners - print_info('BleachBit Cleanup') - for k, v in sorted(BLEACH_BIT_CLEANERS.items()): - try_and_print(message=' {}...'.format(k), - function=run_bleachbit, - cs='Done', other_results=other_results, - cleaners=v, preview=True) - - # Export system info - print_info('Backup System Information') - try_and_print(message='AIDA64 reports...', - function=run_aida64, cs='Done', other_results=other_results) - backup_browsers() - try_and_print(message='File listing...', - function=backup_file_list, cs='Done', other_results=other_results) - try_and_print(message='Power plans...', - function=backup_power_plans, cs='Done') - try_and_print(message='Product Keys...', - function=run_produkey, cs='Done', other_results=other_results) - try_and_print(message='Registry...', - function=backup_registry, cs='Done', other_results=other_results) - - # Summary - print_info('Summary') - try_and_print(message='Operating System:', - function=show_os_name, ns='Unknown', silent_function=False) - try_and_print(message='Activation:', - function=show_os_activation, ns='Unknown', silent_function=False) - try_and_print(message='Installed RAM:', - function=show_installed_ram, ns='Unknown', silent_function=False) - show_free_space() - try_and_print(message='Temp Size:', - function=show_temp_files_size, silent_function=False) - try_and_print(message='Installed Antivirus:', - function=get_installed_antivirus, ns='Unknown', - other_results=other_results, print_return=True) - try_and_print(message='Installed Office:', - function=get_installed_office, ns='Unknown', - other_results=other_results, print_return=True) - try_and_print(message='Product Keys:', - function=get_product_keys, ns='Unknown', print_return=True) - - # User data - print_info('User Data') - try: - show_user_data_summary() - except Exception: - print_error(' Unknown error.') - - # Done - print_standard('\nDone.') - pause('Press Enter to exit...') - exit_script() - except SystemExit: - pass - except: - major_exception() - -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/transferred_keys.py b/.bin/Scripts/transferred_keys.py index b95ff3c9..3308844c 100644 --- a/.bin/Scripts/transferred_keys.py +++ b/.bin/Scripts/transferred_keys.py @@ -12,17 +12,19 @@ os.system('title {}: Transferred Key Finder'.format(KIT_NAME_FULL)) set_log_file('Transferred Keys.log') if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: Transferred Key Finder\n'.format(KIT_NAME_FULL)) - try_and_print(message='Searching for keys...', - function=list_clientdir_keys, print_return=True) + try: + stay_awake() + clear_screen() + print_info('{}: Transferred Key Finder\n'.format(KIT_NAME_FULL)) + try_and_print(message='Searching for keys...', + function=list_clientdir_keys, print_return=True) - # Done - print_standard('\nDone.') - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/update_kit.py b/.bin/Scripts/update_kit.py index 67949d93..e02acc73 100644 --- a/.bin/Scripts/update_kit.py +++ b/.bin/Scripts/update_kit.py @@ -11,135 +11,137 @@ init_global_vars() os.system('title {}: Kit Update Tool'.format(KIT_NAME_FULL)) if __name__ == '__main__': - try: - clear_screen() - print_info('{}: Kit Update Tool\n'.format(KIT_NAME_FULL)) - other_results = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - }} + try: + clear_screen() + print_info('{}: Kit Update Tool\n'.format(KIT_NAME_FULL)) + other_results = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + }} - ## Prep ## - update_sdio = ask('Update SDI Origin?') + ## Prep ## + update_sdio = ask('Update SDI Origin?') - ## Download ## - print_success('Downloading tools') + ## Download ## + print_success('Downloading tools') - # Data Recovery - print_info(' Data Recovery') - try_and_print(message='TestDisk / PhotoRec...', function=update_testdisk, other_results=other_results, width=40) + # Data Recovery + print_info(' Data Recovery') + try_and_print(message='TestDisk / PhotoRec...', function=update_testdisk, other_results=other_results, width=40) - # Data Transfers - print_info(' Data Transfers') - try_and_print(message='FastCopy...', function=update_fastcopy, other_results=other_results, width=40) - try_and_print(message='wimlib...', function=update_wimlib, other_results=other_results, width=40) - try_and_print(message='XYplorer...', function=update_xyplorer, other_results=other_results, width=40) + # Data Transfers + print_info(' Data Transfers') + try_and_print(message='FastCopy...', function=update_fastcopy, other_results=other_results, width=40) + try_and_print(message='wimlib...', function=update_wimlib, other_results=other_results, width=40) + try_and_print(message='XYplorer...', function=update_xyplorer, other_results=other_results, width=40) - # Diagnostics - print_info(' Diagnostics') - try_and_print(message='AIDA64...', function=update_aida64, other_results=other_results, width=40) - try_and_print(message='Autoruns...', function=update_autoruns, other_results=other_results, width=40) - try_and_print(message='BleachBit...', function=update_bleachbit, other_results=other_results, width=40) - try_and_print(message='Blue Screen View...', function=update_bluescreenview, other_results=other_results, width=40) - try_and_print(message='ERUNT...', function=update_erunt, other_results=other_results, width=40) - try_and_print(message='Hitman Pro...', function=update_hitmanpro, other_results=other_results, width=40) - try_and_print(message='HWiNFO...', function=update_hwinfo, other_results=other_results, width=40) - try_and_print(message='NirCmd...', function=update_nircmd, other_results=other_results, width=40) - try_and_print(message='ProduKey...', function=update_produkey, other_results=other_results, width=40) + # Diagnostics + print_info(' Diagnostics') + try_and_print(message='AIDA64...', function=update_aida64, other_results=other_results, width=40) + try_and_print(message='Autoruns...', function=update_autoruns, other_results=other_results, width=40) + try_and_print(message='BleachBit...', function=update_bleachbit, other_results=other_results, width=40) + try_and_print(message='Blue Screen View...', function=update_bluescreenview, other_results=other_results, width=40) + try_and_print(message='ERUNT...', function=update_erunt, other_results=other_results, width=40) + try_and_print(message='Hitman Pro...', function=update_hitmanpro, other_results=other_results, width=40) + try_and_print(message='HWiNFO...', function=update_hwinfo, other_results=other_results, width=40) + try_and_print(message='NirCmd...', function=update_nircmd, other_results=other_results, width=40) + try_and_print(message='ProduKey...', function=update_produkey, other_results=other_results, width=40) - # Drivers - print_info(' Drivers') - try_and_print(message='Intel RST...', function=update_intel_rst, other_results=other_results, width=40) - try_and_print(message='Intel SSD Toolbox...', function=update_intel_ssd_toolbox, other_results=other_results, width=40) - try_and_print(message='Samsing Magician...', function=update_samsung_magician, other_results=other_results, width=40) - if update_sdio: - try_and_print(message='Snappy Driver Installer Origin...', function=update_sdi_origin, other_results=other_results, width=40) + # Drivers + print_info(' Drivers') + try_and_print(message='Intel RST...', function=update_intel_rst, other_results=other_results, width=40) + try_and_print(message='Intel SSD Toolbox...', function=update_intel_ssd_toolbox, other_results=other_results, width=40) + try_and_print(message='Samsing Magician...', function=update_samsung_magician, other_results=other_results, width=40) + if update_sdio: + try_and_print(message='Snappy Driver Installer Origin...', function=update_sdi_origin, other_results=other_results, width=40) - # Installers - print_info(' Installers') - try_and_print(message='Adobe Reader DC...', function=update_adobe_reader_dc, other_results=other_results, width=40) - try_and_print(message='Macs Fan Control...', function=update_macs_fan_control, other_results=other_results, width=40) - try_and_print(message='MS Office...', function=update_office, other_results=other_results, width=40) - try_and_print(message='Visual C++ Runtimes...', function=update_vcredists, other_results=other_results, width=40) - update_all_ninite(other_results=other_results, width=40) + # Installers + print_info(' Installers') + try_and_print(message='Adobe Reader DC...', function=update_adobe_reader_dc, other_results=other_results, width=40) + try_and_print(message='Macs Fan Control...', function=update_macs_fan_control, other_results=other_results, width=40) + try_and_print(message='MS Office...', function=update_office, other_results=other_results, width=40) + try_and_print(message='Visual C++ Runtimes...', function=update_vcredists, other_results=other_results, width=40) + update_all_ninite(other_results=other_results, width=40) - # Misc - print_info(' Misc') - try_and_print(message='Caffeine...', function=update_caffeine, other_results=other_results, width=40) - try_and_print(message='Classic Start Skin...', function=update_classic_start_skin, other_results=other_results, width=40) - try_and_print(message='Du...', function=update_du, other_results=other_results, width=40) - try_and_print(message='Everything...', function=update_everything, other_results=other_results, width=40) - try_and_print(message='Firefox Extensions...', function=update_firefox_ublock_origin, other_results=other_results, width=40) - try_and_print(message='PuTTY...', function=update_putty, other_results=other_results, width=40) - try_and_print(message='Notepad++...', function=update_notepadplusplus, other_results=other_results, width=40) - try_and_print(message='WizTree...', function=update_wiztree, other_results=other_results, width=40) - try_and_print(message='XMPlay...', function=update_xmplay, other_results=other_results, width=40) + # Misc + print_info(' Misc') + try_and_print(message='Caffeine...', function=update_caffeine, other_results=other_results, width=40) + try_and_print(message='Classic Start Skin...', function=update_classic_start_skin, other_results=other_results, width=40) + try_and_print(message='Du...', function=update_du, other_results=other_results, width=40) + try_and_print(message='Everything...', function=update_everything, other_results=other_results, width=40) + try_and_print(message='Firefox Extensions...', function=update_firefox_ublock_origin, other_results=other_results, width=40) + try_and_print(message='PuTTY...', function=update_putty, other_results=other_results, width=40) + try_and_print(message='Notepad++...', function=update_notepadplusplus, other_results=other_results, width=40) + try_and_print(message='WizTree...', function=update_wiztree, other_results=other_results, width=40) + try_and_print(message='XMPlay...', function=update_xmplay, other_results=other_results, width=40) - # Repairs - print_info(' Repairs') - try_and_print(message='AdwCleaner...', function=update_adwcleaner, other_results=other_results, width=40) - try_and_print(message='KVRT...', function=update_kvrt, other_results=other_results, width=40) - try_and_print(message='RKill...', function=update_rkill, other_results=other_results, width=40) - try_and_print(message='TDSS Killer...', function=update_tdsskiller, other_results=other_results, width=40) + # Repairs + print_info(' Repairs') + try_and_print(message='AdwCleaner...', function=update_adwcleaner, other_results=other_results, width=40) + try_and_print(message='KVRT...', function=update_kvrt, other_results=other_results, width=40) + try_and_print(message='RKill...', function=update_rkill, other_results=other_results, width=40) + try_and_print(message='TDSS Killer...', function=update_tdsskiller, other_results=other_results, width=40) - # Uninstallers - print_info(' Uninstallers') - try_and_print(message='IObit Uninstaller...', function=update_iobit_uninstaller, other_results=other_results, width=40) + # Uninstallers + print_info(' Uninstallers') + try_and_print(message='IObit Uninstaller...', function=update_iobit_uninstaller, other_results=other_results, width=40) - ## Review ## - print_standard('Please review the results and download/extract any missing items to .cbin') - pause('Press Enter to compress the .cbin items') + ## Review ## + print_standard('Please review the results and download/extract any missing items to .cbin') + pause('Press Enter to compress the .cbin items') - ## Compress ## - print_success('Compressing tools') - print_info(' _Drivers') - for item in os.scandir(r'{}\_Drivers'.format(global_vars['CBinDir'])): - if not re.search(r'^(_Drivers|.*7z)$', item.name, re.IGNORECASE): - try_and_print( - message='{}...'.format(item.name), - function=compress_and_remove_item, - other_results = other_results, - width=40, - item = item) - print_info(' .cbin') - for item in os.scandir(global_vars['CBinDir']): - if not re.search(r'^(_Drivers|_include|.*7z)$', item.name, re.IGNORECASE): - try_and_print( - message='{}...'.format(item.name), - function=compress_and_remove_item, - other_results = other_results, - width=40, - item = item) + ## Compress ## + print_success('Compressing tools') + print_info(' _Drivers') + for item in os.scandir(r'{}\_Drivers'.format(global_vars['CBinDir'])): + if not re.search(r'^(_Drivers|.*7z)$', item.name, re.IGNORECASE): + try_and_print( + message='{}...'.format(item.name), + function=compress_and_remove_item, + other_results = other_results, + width=40, + item = item) + print_info(' .cbin') + for item in os.scandir(global_vars['CBinDir']): + if not re.search(r'^(_Drivers|_include|.*7z)$', item.name, re.IGNORECASE): + try_and_print( + message='{}...'.format(item.name), + function=compress_and_remove_item, + other_results = other_results, + width=40, + item = item) - ## Search for network Office/QuickBooks installers & add to LAUNCHERS - print_success('Scanning for network installers') - scan_for_net_installers(OFFICE_SERVER, 'Office', min_year=2010) - scan_for_net_installers(QUICKBOOKS_SERVER, 'QuickBooks', min_year=2015) + ## Search for network Office/QuickBooks installers & add to LAUNCHERS + print_success('Scanning for network installers') + scan_for_net_installers(OFFICE_SERVER, 'Office', min_year=2010) + scan_for_net_installers(QUICKBOOKS_SERVER, 'QuickBooks', min_year=2015) - ## Generate Launchers - print_success('Generating launchers') - for section in sorted(LAUNCHERS.keys()): - print_info(' {}'.format(section)) - for name, options in sorted(LAUNCHERS[section].items()): - try_and_print(message=name, function=generate_launcher, - section=section, name=name, options=options, - other_results=other_results, width=40) + ## Generate Launchers + print_success('Generating launchers') + for section in sorted(LAUNCHERS.keys()): + print_info(' {}'.format(section)) + for name, options in sorted(LAUNCHERS[section].items()): + try_and_print(message=name, function=generate_launcher, + section=section, name=name, options=options, + other_results=other_results, width=40) - # Rename "Copy WizardKit.cmd" (if necessary) - source = r'{}\Scripts\Copy WizardKit.cmd'.format(global_vars['BinDir']) - dest = r'{}\Copy {}.cmd'.format(global_vars['BaseDir'], KIT_NAME_FULL) - if os.path.exists(source): - try: - shutil.move(source, dest) - except Exception: - print_error(' Failed to rename "{}.cmd" to "{}.cmd"'.format( - 'Copy WizardKit', KIT_NAME_FULL)) + # Rename "Copy WizardKit.cmd" (if necessary) + source = r'{}\Scripts\Copy WizardKit.cmd'.format(global_vars['BinDir']) + dest = r'{}\Copy {}.cmd'.format(global_vars['BaseDir'], KIT_NAME_FULL) + if os.path.exists(source): + try: + shutil.move(source, dest) + except Exception: + print_error(' Failed to rename "{}.cmd" to "{}.cmd"'.format( + 'Copy WizardKit', KIT_NAME_FULL)) - # Done - print_standard('\nDone.') - pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/user_checklist.py b/.bin/Scripts/user_checklist.py index fd348dec..b57e8b29 100644 --- a/.bin/Scripts/user_checklist.py +++ b/.bin/Scripts/user_checklist.py @@ -14,74 +14,74 @@ os.system('title {}: User Checklist Tool'.format(KIT_NAME_FULL)) set_log_file('User Checklist ({USERNAME}).log'.format(**global_vars['Env'])) if __name__ == '__main__': - try: - stay_awake() - clear_screen() - print_info('{}: User Checklist\n'.format(KIT_NAME_FULL)) - other_results = { - 'Warning': { - 'NotInstalledError': 'Not installed', - 'NoProfilesError': 'No profiles found', - }} - answer_config_browsers = ask('Install adblock?') - if answer_config_browsers: - answer_reset_browsers = ask( - 'Reset browsers to safe defaults first?') - if global_vars['OS']['Version'] == '10': - answer_config_classicshell = ask('Configure ClassicShell?') - answer_config_explorer_user = ask('Configure Explorer?') + try: + stay_awake() + clear_screen() + print_info('{}: User Checklist\n'.format(KIT_NAME_FULL)) + other_results = { + 'Warning': { + 'NotInstalledError': 'Not installed', + 'NoProfilesError': 'No profiles found', + }} + answer_config_browsers = ask('Install adblock?') + if answer_config_browsers: + answer_reset_browsers = ask( + 'Reset browsers to safe defaults first?') + if global_vars['OS']['Version'] == '10': + answer_config_classicshell = ask('Configure ClassicShell?') + answer_config_explorer_user = ask('Configure Explorer?') - # Cleanup - print_info('Cleanup') - try_and_print(message='Desktop...', - function=cleanup_desktop, cs='Done') + # Cleanup + print_info('Cleanup') + try_and_print(message='Desktop...', + function=cleanup_desktop, cs='Done') - # Scan for supported browsers - print_info('Scanning for browsers') - scan_for_browsers() + # Scan for supported browsers + print_info('Scanning for browsers') + scan_for_browsers() - # Homepages - print_info('Current homepages') - list_homepages() + # Homepages + print_info('Current homepages') + list_homepages() - # Backup - print_info('Backing up browsers') - backup_browsers() + # Backup + print_info('Backing up browsers') + backup_browsers() - # Reset - if answer_config_browsers and answer_reset_browsers: - print_info('Resetting browsers') - reset_browsers() + # Reset + if answer_config_browsers and answer_reset_browsers: + print_info('Resetting browsers') + reset_browsers() - # Configure - print_info('Configuring programs') - if answer_config_browsers: - install_adblock() - if global_vars['OS']['Version'] == '10': - if answer_config_classicshell: - try_and_print(message='ClassicStart...', - function=config_classicstart, cs='Done') - if answer_config_explorer_user: - try_and_print(message='Explorer...', - function=config_explorer_user, cs='Done') - if (not answer_config_browsers - and not answer_config_classicshell - and not answer_config_explorer_user): - print_warning(' Skipped') - else: - if not answer_config_browsers: - print_warning(' Skipped') + # Configure + print_info('Configuring programs') + if answer_config_browsers: + install_adblock() + if global_vars['OS']['Version'] == '10': + if answer_config_classicshell: + try_and_print(message='ClassicStart...', + function=config_classicstart, cs='Done') + if answer_config_explorer_user: + try_and_print(message='Explorer...', + function=config_explorer_user, cs='Done') + if (not answer_config_browsers + and not answer_config_classicshell + and not answer_config_explorer_user): + print_warning(' Skipped') + else: + if not answer_config_browsers: + print_warning(' Skipped') - # Run speedtest - popen_program(['start', '', 'https://fast.com'], shell=True) + # Run speedtest + popen_program(['start', '', 'https://fast.com'], shell=True) - # Done - print_standard('\nDone.') - pause('Press Enter to exit...') - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + print_standard('\nDone.') + pause('Press Enter to exit...') + exit_script() + except SystemExit: + pass + except: + major_exception() -# vim: sts=4 sw=4 ts=4 +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/user_data_transfer.py b/.bin/Scripts/user_data_transfer.py index 981e235a..1fa06256 100644 --- a/.bin/Scripts/user_data_transfer.py +++ b/.bin/Scripts/user_data_transfer.py @@ -13,54 +13,56 @@ os.system('title {}: User Data Transfer Tool'.format(KIT_NAME_FULL)) set_log_file('User Data Transfer.log') if __name__ == '__main__': - try: - # Prep - stay_awake() - clear_screen() - print_info('{}: User Data Transfer Tool\n'.format(KIT_NAME_FULL)) + try: + # Prep + stay_awake() + clear_screen() + print_info('{}: User Data Transfer Tool\n'.format(KIT_NAME_FULL)) - # Get backup name prefix - ticket_number = get_ticket_number() - if ENABLED_TICKET_NUMBERS: - backup_prefix = ticket_number - else: - backup_prefix = get_simple_string(prompt='Enter backup name prefix') - backup_prefix = backup_prefix.replace(' ', '_') + # Get backup name prefix + ticket_number = get_ticket_number() + if ENABLED_TICKET_NUMBERS: + backup_prefix = ticket_number + else: + backup_prefix = get_simple_string(prompt='Enter backup name prefix') + backup_prefix = backup_prefix.replace(' ', '_') - # Set destination - folder_path = r'{}\Transfer'.format(KIT_NAME_SHORT) - dest = select_destination(folder_path=folder_path, - prompt='Which disk are we transferring to?') + # Set destination + folder_path = r'{}\Transfer'.format(KIT_NAME_SHORT) + dest = select_destination(folder_path=folder_path, + prompt='Which disk are we transferring to?') - # Set source items - source = select_source(backup_prefix) - items = scan_source(source, dest) + # Set source items + source = select_source(backup_prefix) + items = scan_source(source, dest) - # Transfer - clear_screen() - print_info('Transfer Details:\n') - if ENABLED_TICKET_NUMBERS: - show_data('Ticket:', ticket_number) - show_data('Source:', source.path) - show_data('Destination:', dest) + # Transfer + clear_screen() + print_info('Transfer Details:\n') + if ENABLED_TICKET_NUMBERS: + show_data('Ticket:', ticket_number) + show_data('Source:', source.path) + show_data('Destination:', dest) - if (not ask('Proceed with transfer?')): - umount_backup_shares() - abort() + if (not ask('Proceed with transfer?')): + umount_backup_shares() + abort() - print_info('Transferring Data') - transfer_source(source, dest, items) - try_and_print(message='Removing extra files...', - function=cleanup_transfer, cs='Done', dest_path=dest) - umount_backup_shares() + print_info('Transferring Data') + transfer_source(source, dest, items) + try_and_print(message='Removing extra files...', + function=cleanup_transfer, cs='Done', dest_path=dest) + umount_backup_shares() - # Done - try_and_print(message='Running KVRT...', - function=run_kvrt, cs='Started') - print_standard('\nDone.') - pause("Press Enter to exit...") - exit_script() - except SystemExit: - pass - except: - major_exception() + # Done + try_and_print(message='Running KVRT...', + function=run_kvrt, cs='Started') + print_standard('\nDone.') + pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/winpe_root_menu.py b/.bin/Scripts/winpe_root_menu.py index 743d987b..5a1f0f8b 100644 --- a/.bin/Scripts/winpe_root_menu.py +++ b/.bin/Scripts/winpe_root_menu.py @@ -14,9 +14,11 @@ set_title('{}: Root Menu'.format(KIT_NAME_FULL)) set_log_file('WinPE.log') if __name__ == '__main__': - try: - menu_root() - except SystemExit: - pass - except: - major_exception() + try: + menu_root() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 From 6df88ec021227c970adeeedd60d6ff3f5b238a90 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 22:42:42 -0700 Subject: [PATCH 147/186] Updated settings files --- .bin/Scripts/settings/launchers.py | 1118 ++++++++++++----------- .bin/Scripts/settings/main.py | 106 +-- .bin/Scripts/settings/music.py | 128 +-- .bin/Scripts/settings/sources.py | 386 ++++---- .bin/Scripts/settings/tools.py | 104 +-- .bin/Scripts/settings/windows_builds.py | 368 ++++---- 6 files changed, 1110 insertions(+), 1100 deletions(-) diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 5135ec08..3b378aca 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -1,563 +1,565 @@ # Wizard Kit: Settings - Launchers LAUNCHERS = { - r'(Root)': { - 'Activate Windows': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'activate.py', - 'L_ELEV': 'True', - }, - 'System Checklist': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'system_checklist.py', - 'L_ELEV': 'True', - }, - 'System Diagnostics': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'system_diagnostics.py', - 'L_ELEV': 'True', - }, - 'User Checklist': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'user_checklist.py', - }, - }, - r'Data Recovery': { - 'PhotoRec (CLI)': { - 'L_TYPE': 'Executable', - 'L_PATH': 'TestDisk', - 'L_ITEM': 'photorec_win.exe', - 'L_ELEV': 'True', - 'L__CLI': 'True', - }, - 'PhotoRec': { - 'L_TYPE': 'Executable', - 'L_PATH': 'TestDisk', - 'L_ITEM': 'qphotorec_win.exe', - 'L_ELEV': 'True', - }, - 'TestDisk': { - 'L_TYPE': 'Executable', - 'L_PATH': 'TestDisk', - 'L_ITEM': 'testdisk_win.exe', - 'L_ELEV': 'True', - 'L__CLI': 'True', - }, - }, - r'Data Transfers': { - 'FastCopy (as ADMIN)': { - 'L_TYPE': 'Executable', - 'L_PATH': 'FastCopy', - 'L_ITEM': 'FastCopy.exe', - 'L_ARGS': ( - r' /logfile=%log_dir%\Tools\FastCopy.log' - r' /cmd=noexist_only' - r' /utf8' - r' /skip_empty_dir' - r' /linkdest' - r' /exclude=' - r'$RECYCLE.BIN;' - r'$Recycle.Bin;' - r'.AppleDB;' - r'.AppleDesktop;' - r'.AppleDouble;' - r'.com.apple.timemachine.supported;' - r'.dbfseventsd;' - r'.DocumentRevisions-V100*;' - r'.DS_Store;' - r'.fseventsd;' - r'.PKInstallSandboxManager;' - r'.Spotlight*;' - r'.SymAV*;' - r'.symSchedScanLockxz;' - r'.TemporaryItems;' - r'.Trash*;' - r'.vol;' - r'.VolumeIcon.icns;' - r'desktop.ini;' - r'Desktop?DB;' - r'Desktop?DF;' - r'hiberfil.sys;' - r'lost+found;' - r'Network?Trash?Folder;' - r'pagefile.sys;' - r'Recycled;' - r'RECYCLER;' - r'System?Volume?Information;' - r'Temporary?Items;' - r'Thumbs.db' - r' /to=%client_dir%\Transfer_%iso_date%\ ' - ), - 'L_ELEV': 'True', - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Logs /Transfer', - ], - }, - 'FastCopy': { - 'L_TYPE': 'Executable', - 'L_PATH': 'FastCopy', - 'L_ITEM': 'FastCopy.exe', - 'L_ARGS': ( - r' /logfile=%log_dir%\Tools\FastCopy.log' - r' /cmd=noexist_only' - r' /utf8' - r' /skip_empty_dir' - r' /linkdest' - r' /exclude=' - r'$RECYCLE.BIN;' - r'$Recycle.Bin;' - r'.AppleDB;' - r'.AppleDesktop;' - r'.AppleDouble;' - r'.com.apple.timemachine.supported;' - r'.dbfseventsd;' - r'.DocumentRevisions-V100*;' - r'.DS_Store;' - r'.fseventsd;' - r'.PKInstallSandboxManager;' - r'.Spotlight*;' - r'.SymAV*;' - r'.symSchedScanLockxz;' - r'.TemporaryItems;' - r'.Trash*;' - r'.vol;' - r'.VolumeIcon.icns;' - r'desktop.ini;' - r'Desktop?DB;' - r'Desktop?DF;' - r'hiberfil.sys;' - r'lost+found;' - r'Network?Trash?Folder;' - r'pagefile.sys;' - r'Recycled;' - r'RECYCLER;' - r'System?Volume?Information;' - r'Temporary?Items;' - r'Thumbs.db' - r' /to=%client_dir%\Transfer_%iso_date%\ ' - ), - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Logs /Transfer', - ], - }, - 'KVRT': { - 'L_TYPE': 'Executable', - 'L_PATH': 'KVRT', - 'L_ITEM': 'KVRT.exe', - 'L_ARGS': ( - r' -accepteula' - r' -d %q_dir%' - r' -processlevel 3' - r' -dontcryptsupportinfo' - r' -fixednames' - ), - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Quarantine', - r'set "q_dir=%client_dir%\Quarantine\KVRT"', - r'mkdir "%q_dir%">nul 2>&1', - ], - }, - 'Transferred Keys': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'transferred_keys.py', - 'L_ELEV': 'True', - }, - 'User Data Transfer': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'user_data_transfer.py', - 'L_ELEV': 'True', - }, - 'XYplorer (as ADMIN)': { - 'L_TYPE': 'Executable', - 'L_PATH': 'XYplorerFree', - 'L_ITEM': 'XYplorerFree.exe', - 'L_ARGS': r'/exp /win=max %userprofile%', - 'L_ELEV': 'True', - }, - 'XYplorer': { - 'L_TYPE': 'Executable', - 'L_PATH': 'XYplorerFree', - 'L_ITEM': 'XYplorerFree.exe', - 'L_ARGS': r'/exp /win=max %userprofile%', - }, - }, - r'Diagnostics': { - 'HWiNFO': { - 'L_TYPE': 'Executable', - 'L_PATH': 'HWiNFO', - 'L_ITEM': 'HWiNFO.exe', - 'Extra Code': [ - r'for %%a in (32 64) do (', - r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r')', - ], - }, - 'ProduKey': { - 'L_TYPE': 'Executable', - 'L_PATH': 'ProduKey', - 'L_ITEM': 'ProduKey.exe', - 'L_ELEV': 'True', - 'Extra Code': [ - r'if exist "%bin%\ProduKey" (', - r' del "%bin%\ProduKey\ProduKey.cfg" 2>nul', - r' del "%bin%\ProduKey\ProduKey64.cfg" 2>nul', - r')', - ], - }, - }, - r'Diagnostics\Extras': { - 'AIDA64': { - 'L_TYPE': 'Executable', - 'L_PATH': 'AIDA64', - 'L_ITEM': 'aida64.exe', - }, - 'Autoruns (with VirusTotal Scan)': { - 'L_TYPE': 'Executable', - 'L_PATH': 'Autoruns', - 'L_ITEM': 'Autoruns.exe', - 'L_ARGS': '-e', - 'Extra Code': [ - r'reg add HKCU\Software\Sysinternals\AutoRuns /v checkvirustotal /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v EulaAccepted /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownomicrosoft /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownowindows /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v showonlyvirustotal /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v submitvirustotal /t REG_DWORD /d 0 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v verifysignatures /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns\SigCheck /v EulaAccepted /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns\Streams /v EulaAccepted /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns\VirusTotal /v VirusTotalTermsAccepted /t REG_DWORD /d 1 /f >nul', - ], - }, - 'BleachBit': { - 'L_TYPE': 'Executable', - 'L_PATH': 'BleachBit', - 'L_ITEM': 'bleachbit.exe', - }, - 'BlueScreenView': { - 'L_TYPE': 'Executable', - 'L_PATH': 'BlueScreenView', - 'L_ITEM': 'BlueScreenView.exe', - }, - 'ERUNT': { - 'L_TYPE': 'Executable', - 'L_PATH': 'erunt', - 'L_ITEM': 'ERUNT.EXE', - 'L_ARGS': '%client_dir%\Backups\Registry\%iso_date% sysreg curuser otherusers', - 'L_ELEV': 'True', - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', - ], - }, - 'HitmanPro': { - 'L_TYPE': 'Executable', - 'L_PATH': 'HitmanPro', - 'L_ITEM': 'HitmanPro.exe', - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', - ], - }, - 'HWiNFO (Sensors)': { - 'L_TYPE': 'Executable', - 'L_PATH': 'HWiNFO', - 'L_ITEM': 'HWiNFO.exe', - 'Extra Code': [ - r'for %%a in (32 64) do (', - r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SensorsOnly=1)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r')', - ], - }, - }, - r'Drivers': { - 'Intel RST (Current Release)': { - 'L_TYPE': 'Executable', - 'L_PATH': '_Drivers\Intel RST', - 'L_ITEM': 'SetupRST_16.5.exe', - 'L_7ZIP': 'SetupRST_16.5.exe', - }, - 'Intel RST (Previous Releases)': { - 'L_TYPE': 'Folder', - 'L_PATH': '_Drivers\Intel RST', - 'L_ITEM': '.', - 'L_NCMD': 'True', - }, - 'Intel SSD Toolbox': { - 'L_TYPE': 'Executable', - 'L_PATH': r'_Drivers\Intel SSD Toolbox', - 'L_ITEM': 'Intel SSD Toolbox.exe', - }, - 'Samsing Magician': { - 'L_TYPE': 'Executable', - 'L_PATH': r'_Drivers\Samsung Magician', - 'L_ITEM': 'Samsung Magician.exe', - }, - 'Snappy Driver Installer Origin': { - 'L_TYPE': 'Executable', - 'L_PATH': '_Drivers\SDIO', - 'L_ITEM': 'SDIO.exe', - }, - }, - r'Drivers\Extras': { - 'Acer': { - 'L_TYPE': 'Executable', - 'L_PATH': 'HWiNFO', - 'L_ITEM': 'HWiNFO.exe', - 'Extra Code': [ - r'for %%a in (32 64) do (', - r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r')', - r'start "" "http://us.acer.com/ac/en/US/content/drivers"', - ], - }, - 'Lenovo': { - 'L_TYPE': 'Executable', - 'L_PATH': 'HWiNFO', - 'L_ITEM': 'HWiNFO.exe', - 'Extra Code': [ - r'for %%a in (32 64) do (', - r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r')', - r'start "" "https://pcsupport.lenovo.com/us/en/"', - ], - }, - 'Toshiba': { - 'L_TYPE': 'Executable', - 'L_PATH': 'HWiNFO', - 'L_ITEM': 'HWiNFO.exe', - 'Extra Code': [ - r'for %%a in (32 64) do (', - r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', - r')', - r'start "" "http://support.toshiba.com/drivers"', - ], - }, - }, - r'Installers': { - 'SW Bundle': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'install_sw_bundle.py', - 'L_ELEV': 'True', - }, - }, - r'Installers\Extras\Office\2016': { - 'Home and Business 2016 (x32)': { - 'L_TYPE': 'Office', - 'L_PATH': '2016', - 'L_ITEM': 'hb_32.xml', - 'L_NCMD': 'True', - }, - 'Home and Business 2016 (x64)': { - 'L_TYPE': 'Office', - 'L_PATH': '2016', - 'L_ITEM': 'hb_64.xml', - 'L_NCMD': 'True', - }, - 'Home and Student 2016 (x32)': { - 'L_TYPE': 'Office', - 'L_PATH': '2016', - 'L_ITEM': 'hs_32.xml', - 'L_NCMD': 'True', - }, - 'Home and Student 2016 (x64)': { - 'L_TYPE': 'Office', - 'L_PATH': '2016', - 'L_ITEM': 'hs_64.xml', - 'L_NCMD': 'True', - }, - 'Office 365 2016 (x32)': { - 'L_TYPE': 'Office', - 'L_PATH': '2016', - 'L_ITEM': '365_32.xml', - 'L_NCMD': 'True', - }, - 'Office 365 2016 (x64)': { - 'L_TYPE': 'Office', - 'L_PATH': '2016', - 'L_ITEM': '365_64.xml', - 'L_NCMD': 'True', - }, - }, - r'Installers\Extras\Runtimes': { - 'Visual C++ Runtimes': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'install_vcredists.py', - 'L_ELEV': 'True', - }, - }, - r'Misc': { - 'Cleanup CBS Temp Files': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'cbs_fix.py', - 'L_ELEV': 'True', - }, - 'ConEmu (as ADMIN)': { - 'L_TYPE': 'Executable', - 'L_PATH': 'ConEmu', - 'L_ITEM': 'ConEmu.exe', - 'L_ELEV': 'True', - }, - 'ConEmu': { - 'L_TYPE': 'Executable', - 'L_PATH': 'ConEmu', - 'L_ITEM': 'ConEmu.exe', - }, - 'Enter SafeMode': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'safemode_enter.py', - 'L_ELEV': 'True', - }, - 'Everything': { - 'L_TYPE': 'Executable', - 'L_PATH': 'Everything', - 'L_ITEM': 'Everything.exe', - 'L_ARGS': '-nodb', - 'L_ELEV': 'True', - }, - 'Exit SafeMode': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'safemode_exit.py', - 'L_ELEV': 'True', - }, - 'Notepad++': { - 'L_TYPE': 'Executable', - 'L_PATH': 'notepadplusplus', - 'L_ITEM': 'notepadplusplus.exe', - }, - 'PuTTY': { - 'L_TYPE': 'Executable', - 'L_PATH': 'PuTTY', - 'L_ITEM': 'PUTTY.EXE', - }, - 'WizTree': { - 'L_TYPE': 'Executable', - 'L_PATH': 'WizTree', - 'L_ITEM': 'WizTree.exe', - 'L_ELEV': 'True', - }, - 'XMPlay': { - 'L_TYPE': 'Executable', - 'L_PATH': 'XMPlay', - 'L_ITEM': 'xmplay.exe', - 'L_ARGS': '"%bin%\XMPlay\music.7z"', - }, - }, - r'Repairs': { - 'AdwCleaner': { - 'L_TYPE': 'Executable', - 'L_PATH': 'AdwCleaner', - 'L_ITEM': 'AdwCleaner.exe', - }, - 'Autoruns': { - 'L_TYPE': 'Executable', - 'L_PATH': 'Autoruns', - 'L_ITEM': 'Autoruns.exe', - 'L_ARGS': '-e', - 'Extra Code': [ - r'reg add HKCU\Software\Sysinternals\AutoRuns /v checkvirustotal /t REG_DWORD /d 0 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v EulaAccepted /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownomicrosoft /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownowindows /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v showonlyvirustotal /t REG_DWORD /d 0 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v submitvirustotal /t REG_DWORD /d 0 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns /v verifysignatures /t REG_DWORD /d 0 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns\SigCheck /v EulaAccepted /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns\Streams /v EulaAccepted /t REG_DWORD /d 1 /f >nul', - r'reg add HKCU\Software\Sysinternals\AutoRuns\VirusTotal /v VirusTotalTermsAccepted /t REG_DWORD /d 1 /f >nul', - ], - }, - 'CHKDSK': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'check_disk.py', - 'L_ELEV': 'True', - }, - 'DISM': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'dism.py', - 'L_ELEV': 'True', - }, - 'KVRT': { - 'L_TYPE': 'Executable', - 'L_PATH': 'KVRT', - 'L_ITEM': 'KVRT.exe', - 'L_ARGS': ( - r' -accepteula' - r' -d %q_dir%' - r' -processlevel 3' - r' -dontcryptsupportinfo' - r' -fixednames' - ), - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Quarantine', - r'set "q_dir=%client_dir%\Quarantine\KVRT"', - r'mkdir "%q_dir%">nul 2>&1', - ], - }, - 'RKill': { - 'L_TYPE': 'Executable', - 'L_PATH': 'RKill', - 'L_ITEM': 'RKill.exe', - 'L_ARGS': '-s -l %log_dir%\Tools\RKill.log', - 'L_ELEV': 'True', - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', - ], - }, - 'SFC Scan': { - 'L_TYPE': 'PyScript', - 'L_PATH': 'Scripts', - 'L_ITEM': 'sfc_scan.py', - 'L_ELEV': 'True', - }, - 'TDSSKiller': { - 'L_TYPE': 'Executable', - 'L_PATH': 'TDSSKiller', - 'L_ITEM': 'TDSSKiller.exe', - 'L_ARGS': ( - r' -l %log_dir%\Tools\TDSSKiller.log' - r' -qpath %q_dir%' - r' -accepteula' - r' -accepteulaksn' - r' -dcexact' - r' -tdlfs' - ), - 'Extra Code': [ - r'call "%bin%\Scripts\init_client_dir.cmd" /Quarantine', - r'set "q_dir=%client_dir%\Quarantine\TDSSKiller"', - r'mkdir "%q_dir%">nul 2>&1', - ], - }, - }, - r'Uninstallers': { - 'IObit Uninstaller': { - 'L_TYPE': 'Executable', - 'L_PATH': 'IObitUninstallerPortable', - 'L_ITEM': 'IObitUninstallerPortable.exe', - }, - }, - } + r'(Root)': { + 'Activate Windows': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'activate.py', + 'L_ELEV': 'True', + }, + 'System Checklist': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'system_checklist.py', + 'L_ELEV': 'True', + }, + 'System Diagnostics': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'system_diagnostics.py', + 'L_ELEV': 'True', + }, + 'User Checklist': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'user_checklist.py', + }, + }, + r'Data Recovery': { + 'PhotoRec (CLI)': { + 'L_TYPE': 'Executable', + 'L_PATH': 'TestDisk', + 'L_ITEM': 'photorec_win.exe', + 'L_ELEV': 'True', + 'L__CLI': 'True', + }, + 'PhotoRec': { + 'L_TYPE': 'Executable', + 'L_PATH': 'TestDisk', + 'L_ITEM': 'qphotorec_win.exe', + 'L_ELEV': 'True', + }, + 'TestDisk': { + 'L_TYPE': 'Executable', + 'L_PATH': 'TestDisk', + 'L_ITEM': 'testdisk_win.exe', + 'L_ELEV': 'True', + 'L__CLI': 'True', + }, + }, + r'Data Transfers': { + 'FastCopy (as ADMIN)': { + 'L_TYPE': 'Executable', + 'L_PATH': 'FastCopy', + 'L_ITEM': 'FastCopy.exe', + 'L_ARGS': ( + r' /logfile=%log_dir%\Tools\FastCopy.log' + r' /cmd=noexist_only' + r' /utf8' + r' /skip_empty_dir' + r' /linkdest' + r' /exclude=' + r'$RECYCLE.BIN;' + r'$Recycle.Bin;' + r'.AppleDB;' + r'.AppleDesktop;' + r'.AppleDouble;' + r'.com.apple.timemachine.supported;' + r'.dbfseventsd;' + r'.DocumentRevisions-V100*;' + r'.DS_Store;' + r'.fseventsd;' + r'.PKInstallSandboxManager;' + r'.Spotlight*;' + r'.SymAV*;' + r'.symSchedScanLockxz;' + r'.TemporaryItems;' + r'.Trash*;' + r'.vol;' + r'.VolumeIcon.icns;' + r'desktop.ini;' + r'Desktop?DB;' + r'Desktop?DF;' + r'hiberfil.sys;' + r'lost+found;' + r'Network?Trash?Folder;' + r'pagefile.sys;' + r'Recycled;' + r'RECYCLER;' + r'System?Volume?Information;' + r'Temporary?Items;' + r'Thumbs.db' + r' /to=%client_dir%\Transfer_%iso_date%\ ' + ), + 'L_ELEV': 'True', + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs /Transfer', + ], + }, + 'FastCopy': { + 'L_TYPE': 'Executable', + 'L_PATH': 'FastCopy', + 'L_ITEM': 'FastCopy.exe', + 'L_ARGS': ( + r' /logfile=%log_dir%\Tools\FastCopy.log' + r' /cmd=noexist_only' + r' /utf8' + r' /skip_empty_dir' + r' /linkdest' + r' /exclude=' + r'$RECYCLE.BIN;' + r'$Recycle.Bin;' + r'.AppleDB;' + r'.AppleDesktop;' + r'.AppleDouble;' + r'.com.apple.timemachine.supported;' + r'.dbfseventsd;' + r'.DocumentRevisions-V100*;' + r'.DS_Store;' + r'.fseventsd;' + r'.PKInstallSandboxManager;' + r'.Spotlight*;' + r'.SymAV*;' + r'.symSchedScanLockxz;' + r'.TemporaryItems;' + r'.Trash*;' + r'.vol;' + r'.VolumeIcon.icns;' + r'desktop.ini;' + r'Desktop?DB;' + r'Desktop?DF;' + r'hiberfil.sys;' + r'lost+found;' + r'Network?Trash?Folder;' + r'pagefile.sys;' + r'Recycled;' + r'RECYCLER;' + r'System?Volume?Information;' + r'Temporary?Items;' + r'Thumbs.db' + r' /to=%client_dir%\Transfer_%iso_date%\ ' + ), + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs /Transfer', + ], + }, + 'KVRT': { + 'L_TYPE': 'Executable', + 'L_PATH': 'KVRT', + 'L_ITEM': 'KVRT.exe', + 'L_ARGS': ( + r' -accepteula' + r' -d %q_dir%' + r' -processlevel 3' + r' -dontcryptsupportinfo' + r' -fixednames' + ), + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Quarantine', + r'set "q_dir=%client_dir%\Quarantine\KVRT"', + r'mkdir "%q_dir%">nul 2>&1', + ], + }, + 'Transferred Keys': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'transferred_keys.py', + 'L_ELEV': 'True', + }, + 'User Data Transfer': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'user_data_transfer.py', + 'L_ELEV': 'True', + }, + 'XYplorer (as ADMIN)': { + 'L_TYPE': 'Executable', + 'L_PATH': 'XYplorerFree', + 'L_ITEM': 'XYplorerFree.exe', + 'L_ARGS': r'/exp /win=max %userprofile%', + 'L_ELEV': 'True', + }, + 'XYplorer': { + 'L_TYPE': 'Executable', + 'L_PATH': 'XYplorerFree', + 'L_ITEM': 'XYplorerFree.exe', + 'L_ARGS': r'/exp /win=max %userprofile%', + }, + }, + r'Diagnostics': { + 'HWiNFO': { + 'L_TYPE': 'Executable', + 'L_PATH': 'HWiNFO', + 'L_ITEM': 'HWiNFO.exe', + 'Extra Code': [ + r'for %%a in (32 64) do (', + r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r')', + ], + }, + 'ProduKey': { + 'L_TYPE': 'Executable', + 'L_PATH': 'ProduKey', + 'L_ITEM': 'ProduKey.exe', + 'L_ELEV': 'True', + 'Extra Code': [ + r'if exist "%bin%\ProduKey" (', + r' del "%bin%\ProduKey\ProduKey.cfg" 2>nul', + r' del "%bin%\ProduKey\ProduKey64.cfg" 2>nul', + r')', + ], + }, + }, + r'Diagnostics\Extras': { + 'AIDA64': { + 'L_TYPE': 'Executable', + 'L_PATH': 'AIDA64', + 'L_ITEM': 'aida64.exe', + }, + 'Autoruns (with VirusTotal Scan)': { + 'L_TYPE': 'Executable', + 'L_PATH': 'Autoruns', + 'L_ITEM': 'Autoruns.exe', + 'L_ARGS': '-e', + 'Extra Code': [ + r'reg add HKCU\Software\Sysinternals\AutoRuns /v checkvirustotal /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v EulaAccepted /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownomicrosoft /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownowindows /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v showonlyvirustotal /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v submitvirustotal /t REG_DWORD /d 0 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v verifysignatures /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns\SigCheck /v EulaAccepted /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns\Streams /v EulaAccepted /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns\VirusTotal /v VirusTotalTermsAccepted /t REG_DWORD /d 1 /f >nul', + ], + }, + 'BleachBit': { + 'L_TYPE': 'Executable', + 'L_PATH': 'BleachBit', + 'L_ITEM': 'bleachbit.exe', + }, + 'BlueScreenView': { + 'L_TYPE': 'Executable', + 'L_PATH': 'BlueScreenView', + 'L_ITEM': 'BlueScreenView.exe', + }, + 'ERUNT': { + 'L_TYPE': 'Executable', + 'L_PATH': 'erunt', + 'L_ITEM': 'ERUNT.EXE', + 'L_ARGS': '%client_dir%\Backups\Registry\%iso_date% sysreg curuser otherusers', + 'L_ELEV': 'True', + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', + ], + }, + 'HitmanPro': { + 'L_TYPE': 'Executable', + 'L_PATH': 'HitmanPro', + 'L_ITEM': 'HitmanPro.exe', + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', + ], + }, + 'HWiNFO (Sensors)': { + 'L_TYPE': 'Executable', + 'L_PATH': 'HWiNFO', + 'L_ITEM': 'HWiNFO.exe', + 'Extra Code': [ + r'for %%a in (32 64) do (', + r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SensorsOnly=1)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r')', + ], + }, + }, + r'Drivers': { + 'Intel RST (Current Release)': { + 'L_TYPE': 'Executable', + 'L_PATH': '_Drivers\Intel RST', + 'L_ITEM': 'SetupRST_16.5.exe', + 'L_7ZIP': 'SetupRST_16.5.exe', + }, + 'Intel RST (Previous Releases)': { + 'L_TYPE': 'Folder', + 'L_PATH': '_Drivers\Intel RST', + 'L_ITEM': '.', + 'L_NCMD': 'True', + }, + 'Intel SSD Toolbox': { + 'L_TYPE': 'Executable', + 'L_PATH': r'_Drivers\Intel SSD Toolbox', + 'L_ITEM': 'Intel SSD Toolbox.exe', + }, + 'Samsing Magician': { + 'L_TYPE': 'Executable', + 'L_PATH': r'_Drivers\Samsung Magician', + 'L_ITEM': 'Samsung Magician.exe', + }, + 'Snappy Driver Installer Origin': { + 'L_TYPE': 'Executable', + 'L_PATH': '_Drivers\SDIO', + 'L_ITEM': 'SDIO.exe', + }, + }, + r'Drivers\Extras': { + 'Acer': { + 'L_TYPE': 'Executable', + 'L_PATH': 'HWiNFO', + 'L_ITEM': 'HWiNFO.exe', + 'Extra Code': [ + r'for %%a in (32 64) do (', + r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r')', + r'start "" "http://us.acer.com/ac/en/US/content/drivers"', + ], + }, + 'Lenovo': { + 'L_TYPE': 'Executable', + 'L_PATH': 'HWiNFO', + 'L_ITEM': 'HWiNFO.exe', + 'Extra Code': [ + r'for %%a in (32 64) do (', + r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r')', + r'start "" "https://pcsupport.lenovo.com/us/en/"', + ], + }, + 'Toshiba': { + 'L_TYPE': 'Executable', + 'L_PATH': 'HWiNFO', + 'L_ITEM': 'HWiNFO.exe', + 'Extra Code': [ + r'for %%a in (32 64) do (', + r' copy /y "%bin%\HWiNFO\general.ini" "%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SensorsOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r' (echo SummaryOnly=0)>>"%bin%\HWiNFO\HWiNFO%%a.ini"', + r')', + r'start "" "http://support.toshiba.com/drivers"', + ], + }, + }, + r'Installers': { + 'SW Bundle': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'install_sw_bundle.py', + 'L_ELEV': 'True', + }, + }, + r'Installers\Extras\Office\2016': { + 'Home and Business 2016 (x32)': { + 'L_TYPE': 'Office', + 'L_PATH': '2016', + 'L_ITEM': 'hb_32.xml', + 'L_NCMD': 'True', + }, + 'Home and Business 2016 (x64)': { + 'L_TYPE': 'Office', + 'L_PATH': '2016', + 'L_ITEM': 'hb_64.xml', + 'L_NCMD': 'True', + }, + 'Home and Student 2016 (x32)': { + 'L_TYPE': 'Office', + 'L_PATH': '2016', + 'L_ITEM': 'hs_32.xml', + 'L_NCMD': 'True', + }, + 'Home and Student 2016 (x64)': { + 'L_TYPE': 'Office', + 'L_PATH': '2016', + 'L_ITEM': 'hs_64.xml', + 'L_NCMD': 'True', + }, + 'Office 365 2016 (x32)': { + 'L_TYPE': 'Office', + 'L_PATH': '2016', + 'L_ITEM': '365_32.xml', + 'L_NCMD': 'True', + }, + 'Office 365 2016 (x64)': { + 'L_TYPE': 'Office', + 'L_PATH': '2016', + 'L_ITEM': '365_64.xml', + 'L_NCMD': 'True', + }, + }, + r'Installers\Extras\Runtimes': { + 'Visual C++ Runtimes': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'install_vcredists.py', + 'L_ELEV': 'True', + }, + }, + r'Misc': { + 'Cleanup CBS Temp Files': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'cbs_fix.py', + 'L_ELEV': 'True', + }, + 'ConEmu (as ADMIN)': { + 'L_TYPE': 'Executable', + 'L_PATH': 'ConEmu', + 'L_ITEM': 'ConEmu.exe', + 'L_ELEV': 'True', + }, + 'ConEmu': { + 'L_TYPE': 'Executable', + 'L_PATH': 'ConEmu', + 'L_ITEM': 'ConEmu.exe', + }, + 'Enter SafeMode': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'safemode_enter.py', + 'L_ELEV': 'True', + }, + 'Everything': { + 'L_TYPE': 'Executable', + 'L_PATH': 'Everything', + 'L_ITEM': 'Everything.exe', + 'L_ARGS': '-nodb', + 'L_ELEV': 'True', + }, + 'Exit SafeMode': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'safemode_exit.py', + 'L_ELEV': 'True', + }, + 'Notepad++': { + 'L_TYPE': 'Executable', + 'L_PATH': 'notepadplusplus', + 'L_ITEM': 'notepadplusplus.exe', + }, + 'PuTTY': { + 'L_TYPE': 'Executable', + 'L_PATH': 'PuTTY', + 'L_ITEM': 'PUTTY.EXE', + }, + 'WizTree': { + 'L_TYPE': 'Executable', + 'L_PATH': 'WizTree', + 'L_ITEM': 'WizTree.exe', + 'L_ELEV': 'True', + }, + 'XMPlay': { + 'L_TYPE': 'Executable', + 'L_PATH': 'XMPlay', + 'L_ITEM': 'xmplay.exe', + 'L_ARGS': '"%bin%\XMPlay\music.7z"', + }, + }, + r'Repairs': { + 'AdwCleaner': { + 'L_TYPE': 'Executable', + 'L_PATH': 'AdwCleaner', + 'L_ITEM': 'AdwCleaner.exe', + }, + 'Autoruns': { + 'L_TYPE': 'Executable', + 'L_PATH': 'Autoruns', + 'L_ITEM': 'Autoruns.exe', + 'L_ARGS': '-e', + 'Extra Code': [ + r'reg add HKCU\Software\Sysinternals\AutoRuns /v checkvirustotal /t REG_DWORD /d 0 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v EulaAccepted /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownomicrosoft /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v shownowindows /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v showonlyvirustotal /t REG_DWORD /d 0 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v submitvirustotal /t REG_DWORD /d 0 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns /v verifysignatures /t REG_DWORD /d 0 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns\SigCheck /v EulaAccepted /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns\Streams /v EulaAccepted /t REG_DWORD /d 1 /f >nul', + r'reg add HKCU\Software\Sysinternals\AutoRuns\VirusTotal /v VirusTotalTermsAccepted /t REG_DWORD /d 1 /f >nul', + ], + }, + 'CHKDSK': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'check_disk.py', + 'L_ELEV': 'True', + }, + 'DISM': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'dism.py', + 'L_ELEV': 'True', + }, + 'KVRT': { + 'L_TYPE': 'Executable', + 'L_PATH': 'KVRT', + 'L_ITEM': 'KVRT.exe', + 'L_ARGS': ( + r' -accepteula' + r' -d %q_dir%' + r' -processlevel 3' + r' -dontcryptsupportinfo' + r' -fixednames' + ), + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Quarantine', + r'set "q_dir=%client_dir%\Quarantine\KVRT"', + r'mkdir "%q_dir%">nul 2>&1', + ], + }, + 'RKill': { + 'L_TYPE': 'Executable', + 'L_PATH': 'RKill', + 'L_ITEM': 'RKill.exe', + 'L_ARGS': '-s -l %log_dir%\Tools\RKill.log', + 'L_ELEV': 'True', + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Logs', + ], + }, + 'SFC Scan': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'sfc_scan.py', + 'L_ELEV': 'True', + }, + 'TDSSKiller': { + 'L_TYPE': 'Executable', + 'L_PATH': 'TDSSKiller', + 'L_ITEM': 'TDSSKiller.exe', + 'L_ARGS': ( + r' -l %log_dir%\Tools\TDSSKiller.log' + r' -qpath %q_dir%' + r' -accepteula' + r' -accepteulaksn' + r' -dcexact' + r' -tdlfs' + ), + 'Extra Code': [ + r'call "%bin%\Scripts\init_client_dir.cmd" /Quarantine', + r'set "q_dir=%client_dir%\Quarantine\TDSSKiller"', + r'mkdir "%q_dir%">nul 2>&1', + ], + }, + }, + r'Uninstallers': { + 'IObit Uninstaller': { + 'L_TYPE': 'Executable', + 'L_PATH': 'IObitUninstallerPortable', + 'L_ITEM': 'IObitUninstallerPortable.exe', + }, + }, + } if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 9d32b3ef..98f867fe 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -5,7 +5,7 @@ ENABLED_OPEN_LOGS = False ENABLED_TICKET_NUMBERS = False ENABLED_UPLOAD_DATA = False HW_OVERRIDES_FORCED = False -HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED +HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED # STATIC VARIABLES (also used by BASH and BATCH files) ## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH @@ -15,14 +15,14 @@ KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux -MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags +MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags ROOT_PASSWORD='Abracadabra' TECH_PASSWORD='Abracadabra' # Server IP addresses OFFICE_SERVER_IP='10.0.0.10' QUICKBOOKS_SERVER_IP='10.0.0.10' # Time Zones -LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values +LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values # WiFi WIFI_SSID='SomeWiFi' @@ -30,64 +30,66 @@ WIFI_PASSWORD='Abracadabra' # SERVER VARIABLES ## NOTE: Windows can only use one user per server. This means that if -## one server serves multiple shares then you have to use the same -## user/password for all of those shares. +## one server serves multiple shares then you have to use the same +## user/password for all of those shares. BACKUP_SERVERS = [ - { 'IP': '10.0.0.10', - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'Backups', - 'User': 'restore', - 'Pass': 'Abracadabra', - 'RW-User': 'backup', - 'RW-Pass': 'Abracadabra', - }, - { 'IP': '10.0.0.11', - 'Name': 'ServerTwo', - 'Mounted': False, - 'Share': 'Backups', - 'User': 'restore', - 'Pass': 'Abracadabra', - 'RW-User': 'backup', - 'RW-Pass': 'Abracadabra', - }, + { 'IP': '10.0.0.10', + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'Backups', + 'User': 'restore', + 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', + }, + { 'IP': '10.0.0.11', + 'Name': 'ServerTwo', + 'Mounted': False, + 'Share': 'Backups', + 'User': 'restore', + 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', + }, ] CRASH_SERVER = { - 'Name': 'CrashServer', - 'Url': '', - 'User': '', - 'Pass': '', + 'Name': 'CrashServer', + 'Url': '', + 'User': '', + 'Pass': '', } OFFICE_SERVER = { - 'IP': OFFICE_SERVER_IP, - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'Office', - 'User': 'restore', - 'Pass': 'Abracadabra', - 'RW-User': 'backup', - 'RW-Pass': 'Abracadabra', + 'IP': OFFICE_SERVER_IP, + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'Office', + 'User': 'restore', + 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', } QUICKBOOKS_SERVER = { - 'IP': QUICKBOOKS_SERVER_IP, - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'QuickBooks', - 'User': 'restore', - 'Pass': 'Abracadabra', - 'RW-User': 'backup', - 'RW-Pass': 'Abracadabra', + 'IP': QUICKBOOKS_SERVER_IP, + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'QuickBooks', + 'User': 'restore', + 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', } WINDOWS_SERVER = { - 'IP': '10.0.0.10', - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'Windows', - 'User': 'restore', - 'Pass': 'Abracadabra', - 'RW-User': 'backup', - 'RW-Pass': 'Abracadabra', + 'IP': '10.0.0.10', + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'Windows', + 'User': 'restore', + 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', } if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/music.py b/.bin/Scripts/settings/music.py index a2d49f31..37f5d178 100644 --- a/.bin/Scripts/settings/music.py +++ b/.bin/Scripts/settings/music.py @@ -1,70 +1,72 @@ # Wizard Kit: Settings - Music MUSIC_MOD = [ - '104208#banana_boat.mod', - '114971#tilbury_fair.mod', - '132563#ufo_tune.mod', - '135906#magnetik_girl.xm', - '140628#autumn_in_budapest.xm', - '143198#summer_memories_3.xm', - '144405#hillbilly_billyboy.xm', - '154795#4mat_-_eternity.xm', - '155845#bookworm.mo3', - '155914#battleofsteel.xm', - '158975#1_channel_moog.it', - '165495#trans.s3m', - '168513#necros_-_introspection.s3m', - '169628#radix_-_feng_shui_schematics.xm', - '175238#unknown48_-_twilight.mod', - '33432#ambrozia.xm', - '33460#amigatre.mod', - '34594#CHARIOT.S3M', - '34596#BUTTERFL.XM', - '34654#CTGOBLIN.S3M', - '35151#bananasplit.mod', - '35280#DEADLOCK.XM', - '38591#compo_liam.xm', - '39987#crystald.s3m', - '40475#ELYSIUM.MOD', - '42146#enigma.mod', - '42519#GHOST2.MOD', - '42560#GSLINGER.MOD', - '42872#existing.xm', - '50427#nf-stven.xm', - '51549#overture.mod', - '54250#SATELL.S3M', - '54313#realmk.s3m', - '55789#scrambld.mod', - '57934#spacedeb.mod', - '59344#stardstm.mod', - '60395#2ND_PM.S3M', - '66187#external.xm', - '66343#beek-substitutionology.it', - '67561#radix-unreal_superhero.xm', - '70829#inside_out.s3m', - '83779#beyond_music.mod', - ] + '104208#banana_boat.mod', + '114971#tilbury_fair.mod', + '132563#ufo_tune.mod', + '135906#magnetik_girl.xm', + '140628#autumn_in_budapest.xm', + '143198#summer_memories_3.xm', + '144405#hillbilly_billyboy.xm', + '154795#4mat_-_eternity.xm', + '155845#bookworm.mo3', + '155914#battleofsteel.xm', + '158975#1_channel_moog.it', + '165495#trans.s3m', + '168513#necros_-_introspection.s3m', + '169628#radix_-_feng_shui_schematics.xm', + '175238#unknown48_-_twilight.mod', + '33432#ambrozia.xm', + '33460#amigatre.mod', + '34594#CHARIOT.S3M', + '34596#BUTTERFL.XM', + '34654#CTGOBLIN.S3M', + '35151#bananasplit.mod', + '35280#DEADLOCK.XM', + '38591#compo_liam.xm', + '39987#crystald.s3m', + '40475#ELYSIUM.MOD', + '42146#enigma.mod', + '42519#GHOST2.MOD', + '42560#GSLINGER.MOD', + '42872#existing.xm', + '50427#nf-stven.xm', + '51549#overture.mod', + '54250#SATELL.S3M', + '54313#realmk.s3m', + '55789#scrambld.mod', + '57934#spacedeb.mod', + '59344#stardstm.mod', + '60395#2ND_PM.S3M', + '66187#external.xm', + '66343#beek-substitutionology.it', + '67561#radix-unreal_superhero.xm', + '70829#inside_out.s3m', + '83779#beyond_music.mod', + ] MUSIC_SNES = [ - 'actr', - 'crock', - 'ct', - 'dkc', - 'dkq', - 'ff6', - 'fz', - 'loz3', - 'mmx', - 'ptws', - 'scv4', - 'sf', - 'sf2', - 'sgng', - 'smk', - 'smw', - 'yi', - 'zamn' - ] + 'actr', + 'crock', + 'ct', + 'dkc', + 'dkq', + 'ff6', + 'fz', + 'loz3', + 'mmx', + 'ptws', + 'scv4', + 'sf', + 'sf2', + 'sgng', + 'smk', + 'smw', + 'yi', + 'zamn' + ] if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 9f4350db..9a57e8f7 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -1,202 +1,202 @@ # Wizard Kit: Settings - Sources SOURCE_URLS = { - 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1801120058/AcroRdrDC1801120058_en_US.exe', - 'AdwCleaner': 'https://downloads.malwarebytes.com/file/adwcleaner', - 'AIDA64': 'http://download.aida64.com/aida64engineer597.zip', - 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0-win-32bit-build1.zip', - 'Autoruns': 'https://download.sysinternals.com/files/Autoruns.zip', - 'BleachBit': 'https://download.bleachbit.org/BleachBit-2.0-portable.zip', - 'BlueScreenView32': 'http://www.nirsoft.net/utils/bluescreenview.zip', - 'BlueScreenView64': 'http://www.nirsoft.net/utils/bluescreenview-x64.zip', - 'Caffeine': 'http://www.zhornsoftware.co.uk/caffeine/caffeine.zip', - 'ClassicStartSkin': 'http://www.classicshell.net/forum/download/file.php?id=3001&sid=9a195960d98fd754867dcb63d9315335', - 'Du': 'https://download.sysinternals.com/files/DU.zip', - 'ERUNT': 'http://www.aumha.org/downloads/erunt.zip', - 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.895.x86.zip', - 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.895.x64.zip', - 'FastCopy': 'http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip', - 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1056733/ublock_origin-1.16.20-an+fx.xpi', - 'HitmanPro32': 'https://dl.surfright.nl/HitmanPro.exe', - 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', - 'HWiNFO': 'http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip', - 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/27656/eng/Intel%20SSD%20Toolbox%20-%20v3.5.2.exe', - 'IOBit_Uninstaller': 'https://portableapps.duckduckgo.com/IObitUninstallerPortable_7.5.0.7.paf.exe', - 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', - 'Macs Fan Control': 'https://www.crystalidea.com/downloads/macsfancontrol_setup.exe', - 'NirCmd32': 'https://www.nirsoft.net/utils/nircmd.zip', - 'NirCmd64': 'https://www.nirsoft.net/utils/nircmd-x64.zip', - 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z', - 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_10810.33603.exe', - 'ProduKey32': 'http://www.nirsoft.net/utils/produkey.zip', - 'ProduKey64': 'http://www.nirsoft.net/utils/produkey-x64.zip', - 'PuTTY': 'https://the.earth.li/~sgtatham/putty/latest/w32/putty.zip', - 'RKill': 'https://www.bleepingcomputer.com/download/rkill/dl/10/', - 'Samsung Magician': 'https://s3.ap-northeast-2.amazonaws.com/global.semi.static/SAMSUNG_SSD_v5_2_1_180523/CD0CFAC4675B9E502899B41BE00525C3909ECE3AD57CC1A2FB6B74A766B2A1EA/Samsung_Magician_Installer.zip', - 'SDIO Themes': 'http://snappy-driver-installer.org/downloads/SDIO_Themes.zip', - 'SDIO Torrent': 'http://snappy-driver-installer.org/downloads/SDIO_Update.torrent', - 'TDSSKiller': 'https://media.kaspersky.com/utilities/VirusUtilities/EN/tdsskiller.exe', - 'TestDisk': 'https://www.cgsecurity.org/testdisk-7.1-WIP.win.zip', - 'wimlib32': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip', - 'wimlib64': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip', - 'Winapp2': 'https://github.com/MoscaDotTo/Winapp2/archive/master.zip', - 'WizTree': 'https://antibody-software.com/files/wiztree_3_26_portable.zip', - 'XMPlay 7z': 'https://support.xmplay.com/files/16/xmp-7z.zip?v=800962', - 'XMPlay Game': 'https://support.xmplay.com/files/12/xmp-gme.zip?v=515637', - 'XMPlay RAR': 'https://support.xmplay.com/files/16/xmp-rar.zip?v=409646', - 'XMPlay WAModern': 'https://support.xmplay.com/files/10/WAModern.zip?v=207099', - 'XMPlay': 'https://support.xmplay.com/files/20/xmplay383.zip?v=298195', - 'XYplorerFree': 'https://www.xyplorer.com/download/xyplorer_free_noinstall.zip', - } + 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1801120058/AcroRdrDC1801120058_en_US.exe', + 'AdwCleaner': 'https://downloads.malwarebytes.com/file/adwcleaner', + 'AIDA64': 'http://download.aida64.com/aida64engineer597.zip', + 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0-win-32bit-build1.zip', + 'Autoruns': 'https://download.sysinternals.com/files/Autoruns.zip', + 'BleachBit': 'https://download.bleachbit.org/BleachBit-2.0-portable.zip', + 'BlueScreenView32': 'http://www.nirsoft.net/utils/bluescreenview.zip', + 'BlueScreenView64': 'http://www.nirsoft.net/utils/bluescreenview-x64.zip', + 'Caffeine': 'http://www.zhornsoftware.co.uk/caffeine/caffeine.zip', + 'ClassicStartSkin': 'http://www.classicshell.net/forum/download/file.php?id=3001&sid=9a195960d98fd754867dcb63d9315335', + 'Du': 'https://download.sysinternals.com/files/DU.zip', + 'ERUNT': 'http://www.aumha.org/downloads/erunt.zip', + 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.895.x86.zip', + 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.895.x64.zip', + 'FastCopy': 'http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip', + 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1056733/ublock_origin-1.16.20-an+fx.xpi', + 'HitmanPro32': 'https://dl.surfright.nl/HitmanPro.exe', + 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', + 'HWiNFO': 'http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip', + 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/27656/eng/Intel%20SSD%20Toolbox%20-%20v3.5.2.exe', + 'IOBit_Uninstaller': 'https://portableapps.duckduckgo.com/IObitUninstallerPortable_7.5.0.7.paf.exe', + 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', + 'Macs Fan Control': 'https://www.crystalidea.com/downloads/macsfancontrol_setup.exe', + 'NirCmd32': 'https://www.nirsoft.net/utils/nircmd.zip', + 'NirCmd64': 'https://www.nirsoft.net/utils/nircmd-x64.zip', + 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z', + 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_10810.33603.exe', + 'ProduKey32': 'http://www.nirsoft.net/utils/produkey.zip', + 'ProduKey64': 'http://www.nirsoft.net/utils/produkey-x64.zip', + 'PuTTY': 'https://the.earth.li/ sgtatham/putty/latest/w32/putty.zip', + 'RKill': 'https://www.bleepingcomputer.com/download/rkill/dl/10/', + 'Samsung Magician': 'https://s3.ap-northeast-2.amazonaws.com/global.semi.static/SAMSUNG_SSD_v5_2_1_180523/CD0CFAC4675B9E502899B41BE00525C3909ECE3AD57CC1A2FB6B74A766B2A1EA/Samsung_Magician_Installer.zip', + 'SDIO Themes': 'http://snappy-driver-installer.org/downloads/SDIO_Themes.zip', + 'SDIO Torrent': 'http://snappy-driver-installer.org/downloads/SDIO_Update.torrent', + 'TDSSKiller': 'https://media.kaspersky.com/utilities/VirusUtilities/EN/tdsskiller.exe', + 'TestDisk': 'https://www.cgsecurity.org/testdisk-7.1-WIP.win.zip', + 'wimlib32': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip', + 'wimlib64': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip', + 'Winapp2': 'https://github.com/MoscaDotTo/Winapp2/archive/master.zip', + 'WizTree': 'https://antibody-software.com/files/wiztree_3_26_portable.zip', + 'XMPlay 7z': 'https://support.xmplay.com/files/16/xmp-7z.zip?v=800962', + 'XMPlay Game': 'https://support.xmplay.com/files/12/xmp-gme.zip?v=515637', + 'XMPlay RAR': 'https://support.xmplay.com/files/16/xmp-rar.zip?v=409646', + 'XMPlay WAModern': 'https://support.xmplay.com/files/10/WAModern.zip?v=207099', + 'XMPlay': 'https://support.xmplay.com/files/20/xmplay383.zip?v=298195', + 'XYplorerFree': 'https://www.xyplorer.com/download/xyplorer_free_noinstall.zip', + } VCREDIST_SOURCES = { - '2010sp1': { - '32': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe', - '64': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x64.exe', - }, - '2012u4': { - '32': 'https://download.microsoft.com/download/1/6/B/16B06F60-3B20-4FF2-B699-5E9B7962F9AE/VSU_4/vcredist_x86.exe', - '64': 'https://download.microsoft.com/download/1/6/B/16B06F60-3B20-4FF2-B699-5E9B7962F9AE/VSU_4/vcredist_x64.exe', - }, - '2013': { - '32': 'https://download.microsoft.com/download/0/5/6/056dcda9-d667-4e27-8001-8a0c6971d6b1/vcredist_x86.exe', - '64': 'https://download.microsoft.com/download/0/5/6/056dcda9-d667-4e27-8001-8a0c6971d6b1/vcredist_x64.exe', - }, - '2017': { - '32': 'https://aka.ms/vs/15/release/vc_redist.x86.exe', - '64': 'https://aka.ms/vs/15/release/vc_redist.x64.exe', - }, - } + '2010sp1': { + '32': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe', + '64': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x64.exe', + }, + '2012u4': { + '32': 'https://download.microsoft.com/download/1/6/B/16B06F60-3B20-4FF2-B699-5E9B7962F9AE/VSU_4/vcredist_x86.exe', + '64': 'https://download.microsoft.com/download/1/6/B/16B06F60-3B20-4FF2-B699-5E9B7962F9AE/VSU_4/vcredist_x64.exe', + }, + '2013': { + '32': 'https://download.microsoft.com/download/0/5/6/056dcda9-d667-4e27-8001-8a0c6971d6b1/vcredist_x86.exe', + '64': 'https://download.microsoft.com/download/0/5/6/056dcda9-d667-4e27-8001-8a0c6971d6b1/vcredist_x64.exe', + }, + '2017': { + '32': 'https://aka.ms/vs/15/release/vc_redist.x86.exe', + '64': 'https://aka.ms/vs/15/release/vc_redist.x64.exe', + }, + } NINITE_SOURCES = { - 'Bundles': { - 'Legacy.exe': '.net4.7.2-7zip-chrome-firefox-vlc', - 'Modern.exe': '.net4.7.2-7zip-chrome-classicstart-firefox-vlc', - }, - 'Audio-Video': { - 'AIMP.exe': 'aimp', - 'Audacity.exe': 'audacity', - 'CCCP.exe': 'cccp', - 'Foobar2000.exe': 'foobar', - 'GOM.exe': 'gom', - 'HandBrake.exe': 'handbrake', - 'iTunes.exe': 'itunes', - 'K-Lite Codecs.exe': 'klitecodecs', - 'MediaMonkey.exe': 'mediamonkey', - 'MusicBee.exe': 'musicbee', - 'Spotify.exe': 'spotify', - 'VLC.exe': 'vlc', - 'Winamp.exe': 'winamp', - }, - 'Cloud Storage': { - 'Dropbox.exe': 'dropbox', - 'Google Backup & Sync.exe': 'googlebackupandsync', - 'Mozy.exe': 'mozy', - 'OneDrive.exe': 'onedrive', - 'SugarSync.exe': 'sugarsync', - }, - 'Communication': { - 'Discord': 'discord', - 'Pidgin.exe': 'pidgin', - 'Skype.exe': 'skype', - 'Trillian.exe': 'trillian', - }, - 'Compression': { - '7-Zip.exe': '7zip', - 'PeaZip.exe': 'peazip', - 'WinRAR.exe': 'winrar', - }, - 'Developer': { - 'Eclipse.exe': 'eclipse', - 'JDK 8.exe': 'jdk8', - 'JDK 8 (x64).exe': 'jdkx8', - 'Notepad++.exe': 'notepadplusplus', - 'PuTTY.exe': 'putty', - 'Python 2.exe': 'python', - 'Visual Studio Code.exe': 'vscode', - 'WinMerge.exe': 'winmerge', - 'WinSCP.exe': 'winscp', - }, - 'File Sharing': { - 'qBittorrent.exe': 'qbittorrent', - }, - 'Image-Photo': { - 'Blender.exe': 'blender', - 'FastStone.exe': 'faststone', - 'GIMP.exe': 'gimp', - 'Greenshot.exe': 'greenshot', - 'Inkscape.exe': 'inkscape', - 'IrfanView.exe': 'irfanview', - 'Krita.exe': 'krita', - 'Paint.NET.exe': 'paint.net', - 'ShareX.exe': 'sharex', - 'XnView.exe': 'xnview', - }, - 'Misc': { - 'Evernote.exe': 'evernote', - 'Everything.exe': 'everything', - 'KeePass 2.exe': 'keepass2', - 'Google Earth.exe': 'googleearth', - 'NV Access.exe': 'nvda', - 'Steam.exe': 'steam', - }, - 'Office': { - 'CutePDF.exe': 'cutepdf', - 'Foxit Reader.exe': 'foxit', - 'LibreOffice.exe': 'libreoffice', - 'OpenOffice.exe': 'openoffice', - 'PDFCreator.exe': 'pdfcreator', - 'SumatraPDF.exe': 'sumatrapdf', - 'Thunderbird.exe': 'thunderbird', - }, - 'Runtimes': { - 'Adobe Air.exe': 'air', - 'dotNET.exe': '.net4.7.2', - 'Java 8.exe': 'java8', - 'Shockwave.exe': 'shockwave', - 'Silverlight.exe': 'silverlight', - }, - 'Security': { - 'Avast.exe': 'avast', - 'AVG.exe': 'avg', - 'Avira.exe': 'avira', - 'Microsoft Security Essentials.exe': 'essentials', - 'Malwarebytes Anti-Malware.exe': 'malwarebytes', - 'Spybot 2.exe': 'spybot2', - 'SUPERAntiSpyware.exe': 'super', - }, - 'Utilities': { - 'CDBurnerXP.exe': 'cdburnerxp', - 'Classic Start.exe': 'classicstart', - 'Glary Utilities.exe': 'glary', - 'ImgBurn.exe': 'imgburn', - 'InfraRecorder.exe': 'infrarecorder', - 'Launchy.exe': 'launchy', - 'RealVNC.exe': 'realvnc', - 'Revo Uninstaller.exe': 'revo', - 'TeamViewer 13.exe': 'teamviewer13', - 'TeraCopy.exe': 'teracopy', - 'WinDirStat.exe': 'windirstat', - }, - 'Web Browsers': { - 'Google Chrome.exe': 'chrome', - 'Mozilla Firefox.exe': 'firefox', - 'Opera Chromium.exe': 'operaChromium', - }, - } + 'Bundles': { + 'Legacy.exe': '.net4.7.2-7zip-chrome-firefox-vlc', + 'Modern.exe': '.net4.7.2-7zip-chrome-classicstart-firefox-vlc', + }, + 'Audio-Video': { + 'AIMP.exe': 'aimp', + 'Audacity.exe': 'audacity', + 'CCCP.exe': 'cccp', + 'Foobar2000.exe': 'foobar', + 'GOM.exe': 'gom', + 'HandBrake.exe': 'handbrake', + 'iTunes.exe': 'itunes', + 'K-Lite Codecs.exe': 'klitecodecs', + 'MediaMonkey.exe': 'mediamonkey', + 'MusicBee.exe': 'musicbee', + 'Spotify.exe': 'spotify', + 'VLC.exe': 'vlc', + 'Winamp.exe': 'winamp', + }, + 'Cloud Storage': { + 'Dropbox.exe': 'dropbox', + 'Google Backup & Sync.exe': 'googlebackupandsync', + 'Mozy.exe': 'mozy', + 'OneDrive.exe': 'onedrive', + 'SugarSync.exe': 'sugarsync', + }, + 'Communication': { + 'Discord': 'discord', + 'Pidgin.exe': 'pidgin', + 'Skype.exe': 'skype', + 'Trillian.exe': 'trillian', + }, + 'Compression': { + '7-Zip.exe': '7zip', + 'PeaZip.exe': 'peazip', + 'WinRAR.exe': 'winrar', + }, + 'Developer': { + 'Eclipse.exe': 'eclipse', + 'JDK 8.exe': 'jdk8', + 'JDK 8 (x64).exe': 'jdkx8', + 'Notepad++.exe': 'notepadplusplus', + 'PuTTY.exe': 'putty', + 'Python 2.exe': 'python', + 'Visual Studio Code.exe': 'vscode', + 'WinMerge.exe': 'winmerge', + 'WinSCP.exe': 'winscp', + }, + 'File Sharing': { + 'qBittorrent.exe': 'qbittorrent', + }, + 'Image-Photo': { + 'Blender.exe': 'blender', + 'FastStone.exe': 'faststone', + 'GIMP.exe': 'gimp', + 'Greenshot.exe': 'greenshot', + 'Inkscape.exe': 'inkscape', + 'IrfanView.exe': 'irfanview', + 'Krita.exe': 'krita', + 'Paint.NET.exe': 'paint.net', + 'ShareX.exe': 'sharex', + 'XnView.exe': 'xnview', + }, + 'Misc': { + 'Evernote.exe': 'evernote', + 'Everything.exe': 'everything', + 'KeePass 2.exe': 'keepass2', + 'Google Earth.exe': 'googleearth', + 'NV Access.exe': 'nvda', + 'Steam.exe': 'steam', + }, + 'Office': { + 'CutePDF.exe': 'cutepdf', + 'Foxit Reader.exe': 'foxit', + 'LibreOffice.exe': 'libreoffice', + 'OpenOffice.exe': 'openoffice', + 'PDFCreator.exe': 'pdfcreator', + 'SumatraPDF.exe': 'sumatrapdf', + 'Thunderbird.exe': 'thunderbird', + }, + 'Runtimes': { + 'Adobe Air.exe': 'air', + 'dotNET.exe': '.net4.7.2', + 'Java 8.exe': 'java8', + 'Shockwave.exe': 'shockwave', + 'Silverlight.exe': 'silverlight', + }, + 'Security': { + 'Avast.exe': 'avast', + 'AVG.exe': 'avg', + 'Avira.exe': 'avira', + 'Microsoft Security Essentials.exe': 'essentials', + 'Malwarebytes Anti-Malware.exe': 'malwarebytes', + 'Spybot 2.exe': 'spybot2', + 'SUPERAntiSpyware.exe': 'super', + }, + 'Utilities': { + 'CDBurnerXP.exe': 'cdburnerxp', + 'Classic Start.exe': 'classicstart', + 'Glary Utilities.exe': 'glary', + 'ImgBurn.exe': 'imgburn', + 'InfraRecorder.exe': 'infrarecorder', + 'Launchy.exe': 'launchy', + 'RealVNC.exe': 'realvnc', + 'Revo Uninstaller.exe': 'revo', + 'TeamViewer 13.exe': 'teamviewer13', + 'TeraCopy.exe': 'teracopy', + 'WinDirStat.exe': 'windirstat', + }, + 'Web Browsers': { + 'Google Chrome.exe': 'chrome', + 'Mozilla Firefox.exe': 'firefox', + 'Opera Chromium.exe': 'operaChromium', + }, + } RST_SOURCES = { - #SetupRST_12.0.exe : Removed from download center? - #SetupRST_12.5.exe : Removed from download center? - #SetupRST_12.8.exe : Removed from download center? - 'SetupRST_12.9.exe': 'https://downloadmirror.intel.com/23496/eng/SetupRST.exe', - #SetupRST_13.x.exe : Broken, doesn't support > .NET 4.5 - 'SetupRST_14.0.exe': 'https://downloadmirror.intel.com/25091/eng/SetupRST.exe', - 'SetupRST_14.8.exe': 'https://downloadmirror.intel.com/26759/eng/setuprst.exe', - 'SetupRST_15.8.exe': 'https://downloadmirror.intel.com/27442/eng/SetupRST.exe', - 'SetupRST_15.9.exe': 'https://downloadmirror.intel.com/27400/eng/SetupRST.exe', - 'SetupRST_16.0.exe': 'https://downloadmirror.intel.com/27681/eng/SetupRST.exe', - 'SetupRST_16.5.exe': 'https://downloadmirror.intel.com/27984/eng/SetupRST.exe', - } + #SetupRST_12.0.exe : Removed from download center? + #SetupRST_12.5.exe : Removed from download center? + #SetupRST_12.8.exe : Removed from download center? + 'SetupRST_12.9.exe': 'https://downloadmirror.intel.com/23496/eng/SetupRST.exe', + #SetupRST_13.x.exe : Broken, doesn't support > .NET 4.5 + 'SetupRST_14.0.exe': 'https://downloadmirror.intel.com/25091/eng/SetupRST.exe', + 'SetupRST_14.8.exe': 'https://downloadmirror.intel.com/26759/eng/setuprst.exe', + 'SetupRST_15.8.exe': 'https://downloadmirror.intel.com/27442/eng/SetupRST.exe', + 'SetupRST_15.9.exe': 'https://downloadmirror.intel.com/27400/eng/SetupRST.exe', + 'SetupRST_16.0.exe': 'https://downloadmirror.intel.com/27681/eng/SetupRST.exe', + 'SetupRST_16.5.exe': 'https://downloadmirror.intel.com/27984/eng/SetupRST.exe', + } if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") -# vim: sts=4 sw=4 ts=4 tw=0 nowrap +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/tools.py b/.bin/Scripts/settings/tools.py index b470603c..0ee74dec 100644 --- a/.bin/Scripts/settings/tools.py +++ b/.bin/Scripts/settings/tools.py @@ -1,56 +1,58 @@ # Wizard Kit: Settings - Tools TOOLS = { - # NOTE: BinDir will be prepended to these paths at runtime - 'AIDA64': { - '32': r'AIDA64\aida64.exe'}, - 'AutoRuns': { - '32': r'Autoruns\autoruns.exe', - '64': r'Autoruns\autoruns64.exe'}, - 'BleachBit': { - '32': r'BleachBit\bleachbit_console.exe'}, - 'Caffeine': { - '32': r'Caffeine\caffeine.exe'}, - 'Du': { - '32': r'Du\du.exe', - '64': r'Du\du64.exe'}, - 'ERUNT': { - '32': r'ERUNT\ERUNT.EXE'}, - 'Everything': { - '32': r'Everything\Everything.exe', - '64': r'Everything\Everything64.exe'}, - 'FastCopy': { - '32': r'FastCopy\FastCopy.exe', - '64': r'FastCopy\FastCopy64.exe'}, - 'HitmanPro': { - '32': r'HitmanPro\HitmanPro.exe', - '64': r'HitmanPro\HitmanPro64.exe'}, - 'HWiNFO': { - '32': r'HWiNFO\HWiNFO.exe', - '64': r'HWiNFO\HWiNFO64.exe'}, - 'KVRT': { - '32': r'KVRT\KVRT.exe'}, - 'NirCmd': { - '32': r'NirCmd\nircmdc.exe', - '64': r'NirCmd\nircmdc64.exe'}, - 'NotepadPlusPlus': { - '32': r'NotepadPlusPlus\notepadplusplus.exe'}, - 'ProduKey': { - '32': r'ProduKey\ProduKey.exe', - '64': r'ProduKey\ProduKey64.exe'}, - 'RKill': { - '32': r'RKill\RKill.exe'}, - 'SevenZip': { - '32': r'7-Zip\7za.exe', - '64': r'7-Zip\7za64.exe'}, - 'TDSSKiller': { - '32': r'TDSSKiller\TDSSKiller.exe'}, - 'wimlib-imagex': { - '32': r'wimlib\x32\wimlib-imagex.exe', - '64': r'wimlib\x64\wimlib-imagex.exe'}, - 'XMPlay': { - '32': r'XMPlay\xmplay.exe'}, - } + # NOTE: BinDir will be prepended to these paths at runtime + 'AIDA64': { + '32': r'AIDA64\aida64.exe'}, + 'AutoRuns': { + '32': r'Autoruns\autoruns.exe', + '64': r'Autoruns\autoruns64.exe'}, + 'BleachBit': { + '32': r'BleachBit\bleachbit_console.exe'}, + 'Caffeine': { + '32': r'Caffeine\caffeine.exe'}, + 'Du': { + '32': r'Du\du.exe', + '64': r'Du\du64.exe'}, + 'ERUNT': { + '32': r'ERUNT\ERUNT.EXE'}, + 'Everything': { + '32': r'Everything\Everything.exe', + '64': r'Everything\Everything64.exe'}, + 'FastCopy': { + '32': r'FastCopy\FastCopy.exe', + '64': r'FastCopy\FastCopy64.exe'}, + 'HitmanPro': { + '32': r'HitmanPro\HitmanPro.exe', + '64': r'HitmanPro\HitmanPro64.exe'}, + 'HWiNFO': { + '32': r'HWiNFO\HWiNFO.exe', + '64': r'HWiNFO\HWiNFO64.exe'}, + 'KVRT': { + '32': r'KVRT\KVRT.exe'}, + 'NirCmd': { + '32': r'NirCmd\nircmdc.exe', + '64': r'NirCmd\nircmdc64.exe'}, + 'NotepadPlusPlus': { + '32': r'NotepadPlusPlus\notepadplusplus.exe'}, + 'ProduKey': { + '32': r'ProduKey\ProduKey.exe', + '64': r'ProduKey\ProduKey64.exe'}, + 'RKill': { + '32': r'RKill\RKill.exe'}, + 'SevenZip': { + '32': r'7-Zip\7za.exe', + '64': r'7-Zip\7za64.exe'}, + 'TDSSKiller': { + '32': r'TDSSKiller\TDSSKiller.exe'}, + 'wimlib-imagex': { + '32': r'wimlib\x32\wimlib-imagex.exe', + '64': r'wimlib\x64\wimlib-imagex.exe'}, + 'XMPlay': { + '32': r'XMPlay\xmplay.exe'}, + } if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/windows_builds.py b/.bin/Scripts/settings/windows_builds.py index 2817979a..216be21f 100644 --- a/.bin/Scripts/settings/windows_builds.py +++ b/.bin/Scripts/settings/windows_builds.py @@ -1,193 +1,195 @@ # Wizard Kit: Settings - Windows Builds WINDOWS_BUILDS = { -# Build Version Release Codename Marketing Name Notes - '6000': ( 'Vista', 'RTM', 'Longhorn', None, 'unsupported'), - '6000': ( 'Vista', 'RTM', 'Longhorn', None, 'unsupported'), - '6001': ( 'Vista', 'SP1', 'Longhorn', None, 'unsupported'), - '6002': ( 'Vista', 'SP2', 'Longhorn', None, 'unsupported'), + # Build, Version, Release, Codename, Marketing Name, Notes + '6000': ('Vista', 'RTM', 'Longhorn', None, 'unsupported'), + '6000': ('Vista', 'RTM', 'Longhorn', None, 'unsupported'), + '6001': ('Vista', 'SP1', 'Longhorn', None, 'unsupported'), + '6002': ('Vista', 'SP2', 'Longhorn', None, 'unsupported'), - '7600': ( '7', 'RTM', 'Vienna', None, 'unsupported'), - '7601': ( '7', 'SP1', 'Vienna', None, 'outdated'), + '7600': ('7', 'RTM', 'Vienna', None, 'unsupported'), + '7601': ('7', 'SP1', 'Vienna', None, 'outdated'), - #9199 is a fake build since Win 8 is 6.2.9200 but that collides with Win 8.1 (6.3.9200) - '9199': ( '8', 'RTM', None, None, 'unsupported'), + #9199 is a fake build since Win 8 is 6.2.9200 but that collides with Win 8.1 (6.3.9200) + '9199': ('8', 'RTM', None, None, 'unsupported'), - '9200': ( '8.1', None, 'Blue', None, 'outdated'), - '9600': ( '8.1', None, 'Update', None, None), + '9200': ('8.1', None, 'Blue', None, 'outdated'), + '9600': ('8.1', None, 'Update', None, None), - '9841': ( '10', None, 'Threshold 1', None, 'preview build'), - '9860': ( '10', None, 'Threshold 1', None, 'preview build'), - '9879': ( '10', None, 'Threshold 1', None, 'preview build'), - '9926': ( '10', None, 'Threshold 1', None, 'preview build'), - '10041': ( '10', None, 'Threshold 1', None, 'preview build'), - '10049': ( '10', None, 'Threshold 1', None, 'preview build'), - '10061': ( '10', None, 'Threshold 1', None, 'preview build'), - '10074': ( '10', None, 'Threshold 1', None, 'preview build'), - '10122': ( '10', None, 'Threshold 1', None, 'preview build'), - '10130': ( '10', None, 'Threshold 1', None, 'preview build'), - '10158': ( '10', None, 'Threshold 1', None, 'preview build'), - '10159': ( '10', None, 'Threshold 1', None, 'preview build'), - '10162': ( '10', None, 'Threshold 1', None, 'preview build'), - '10166': ( '10', None, 'Threshold 1', None, 'preview build'), - '10240': ( '10', 'v1507', 'Threshold 1', None, 'unsupported'), - '10525': ( '10', None, 'Threshold 2', None, 'preview build'), - '10532': ( '10', None, 'Threshold 2', None, 'preview build'), - '10547': ( '10', None, 'Threshold 2', None, 'preview build'), - '10565': ( '10', None, 'Threshold 2', None, 'preview build'), - '10576': ( '10', None, 'Threshold 2', None, 'preview build'), - '10586': ( '10', 'v1511', 'Threshold 2', 'November Update', 'unsupported'), - '11082': ( '10', None, 'Redstone 1', None, 'preview build'), - '11099': ( '10', None, 'Redstone 1', None, 'preview build'), - '11102': ( '10', None, 'Redstone 1', None, 'preview build'), - '14251': ( '10', None, 'Redstone 1', None, 'preview build'), - '14257': ( '10', None, 'Redstone 1', None, 'preview build'), - '14271': ( '10', None, 'Redstone 1', None, 'preview build'), - '14279': ( '10', None, 'Redstone 1', None, 'preview build'), - '14291': ( '10', None, 'Redstone 1', None, 'preview build'), - '14295': ( '10', None, 'Redstone 1', None, 'preview build'), - '14316': ( '10', None, 'Redstone 1', None, 'preview build'), - '14328': ( '10', None, 'Redstone 1', None, 'preview build'), - '14332': ( '10', None, 'Redstone 1', None, 'preview build'), - '14342': ( '10', None, 'Redstone 1', None, 'preview build'), - '14352': ( '10', None, 'Redstone 1', None, 'preview build'), - '14361': ( '10', None, 'Redstone 1', None, 'preview build'), - '14366': ( '10', None, 'Redstone 1', None, 'preview build'), - '14367': ( '10', None, 'Redstone 1', None, 'preview build'), - '14371': ( '10', None, 'Redstone 1', None, 'preview build'), - '14372': ( '10', None, 'Redstone 1', None, 'preview build'), - '14376': ( '10', None, 'Redstone 1', None, 'preview build'), - '14379': ( '10', None, 'Redstone 1', None, 'preview build'), - '14383': ( '10', None, 'Redstone 1', None, 'preview build'), - '14385': ( '10', None, 'Redstone 1', None, 'preview build'), - '14388': ( '10', None, 'Redstone 1', None, 'preview build'), - '14390': ( '10', None, 'Redstone 1', None, 'preview build'), - '14393': ( '10', 'v1607', 'Redstone 1', 'Anniversary Update', 'unsupported'), - '14901': ( '10', None, 'Redstone 2', None, 'preview build'), - '14905': ( '10', None, 'Redstone 2', None, 'preview build'), - '14915': ( '10', None, 'Redstone 2', None, 'preview build'), - '14926': ( '10', None, 'Redstone 2', None, 'preview build'), - '14931': ( '10', None, 'Redstone 2', None, 'preview build'), - '14936': ( '10', None, 'Redstone 2', None, 'preview build'), - '14942': ( '10', None, 'Redstone 2', None, 'preview build'), - '14946': ( '10', None, 'Redstone 2', None, 'preview build'), - '14951': ( '10', None, 'Redstone 2', None, 'preview build'), - '14955': ( '10', None, 'Redstone 2', None, 'preview build'), - '14959': ( '10', None, 'Redstone 2', None, 'preview build'), - '14965': ( '10', None, 'Redstone 2', None, 'preview build'), - '14971': ( '10', None, 'Redstone 2', None, 'preview build'), - '14986': ( '10', None, 'Redstone 2', None, 'preview build'), - '15002': ( '10', None, 'Redstone 2', None, 'preview build'), - '15007': ( '10', None, 'Redstone 2', None, 'preview build'), - '15014': ( '10', None, 'Redstone 2', None, 'preview build'), - '15019': ( '10', None, 'Redstone 2', None, 'preview build'), - '15025': ( '10', None, 'Redstone 2', None, 'preview build'), - '15031': ( '10', None, 'Redstone 2', None, 'preview build'), - '15042': ( '10', None, 'Redstone 2', None, 'preview build'), - '15046': ( '10', None, 'Redstone 2', None, 'preview build'), - '15048': ( '10', None, 'Redstone 2', None, 'preview build'), - '15055': ( '10', None, 'Redstone 2', None, 'preview build'), - '15058': ( '10', None, 'Redstone 2', None, 'preview build'), - '15060': ( '10', None, 'Redstone 2', None, 'preview build'), - '15061': ( '10', None, 'Redstone 2', None, 'preview build'), - '15063': ( '10', 'v1703', 'Redstone 2', 'Creators Update', 'outdated'), - '16170': ( '10', None, 'Redstone 3', None, 'preview build'), - '16176': ( '10', None, 'Redstone 3', None, 'preview build'), - '16179': ( '10', None, 'Redstone 3', None, 'preview build'), - '16184': ( '10', None, 'Redstone 3', None, 'preview build'), - '16188': ( '10', None, 'Redstone 3', None, 'preview build'), - '16193': ( '10', None, 'Redstone 3', None, 'preview build'), - '16199': ( '10', None, 'Redstone 3', None, 'preview build'), - '16212': ( '10', None, 'Redstone 3', None, 'preview build'), - '16215': ( '10', None, 'Redstone 3', None, 'preview build'), - '16226': ( '10', None, 'Redstone 3', None, 'preview build'), - '16232': ( '10', None, 'Redstone 3', None, 'preview build'), - '16237': ( '10', None, 'Redstone 3', None, 'preview build'), - '16241': ( '10', None, 'Redstone 3', None, 'preview build'), - '16251': ( '10', None, 'Redstone 3', None, 'preview build'), - '16257': ( '10', None, 'Redstone 3', None, 'preview build'), - '16273': ( '10', None, 'Redstone 3', None, 'preview build'), - '16275': ( '10', None, 'Redstone 3', None, 'preview build'), - '16278': ( '10', None, 'Redstone 3', None, 'preview build'), - '16281': ( '10', None, 'Redstone 3', None, 'preview build'), - '16288': ( '10', None, 'Redstone 3', None, 'preview build'), - '16291': ( '10', None, 'Redstone 3', None, 'preview build'), - '16294': ( '10', None, 'Redstone 3', None, 'preview build'), - '16296': ( '10', None, 'Redstone 3', None, 'preview build'), - '16299': ( '10', 'v1709', 'Redstone 3', 'Fall Creators Update', 'outdated'), - '16353': ( '10', None, 'Redstone 4', None, 'preview build'), - '16362': ( '10', None, 'Redstone 4', None, 'preview build'), - '17004': ( '10', None, 'Redstone 4', None, 'preview build'), - '17017': ( '10', None, 'Redstone 4', None, 'preview build'), - '17025': ( '10', None, 'Redstone 4', None, 'preview build'), - '17035': ( '10', None, 'Redstone 4', None, 'preview build'), - '17040': ( '10', None, 'Redstone 4', None, 'preview build'), - '17046': ( '10', None, 'Redstone 4', None, 'preview build'), - '17063': ( '10', None, 'Redstone 4', None, 'preview build'), - '17074': ( '10', None, 'Redstone 4', None, 'preview build'), - '17083': ( '10', None, 'Redstone 4', None, 'preview build'), - '17093': ( '10', None, 'Redstone 4', None, 'preview build'), - '17101': ( '10', None, 'Redstone 4', None, 'preview build'), - '17107': ( '10', None, 'Redstone 4', None, 'preview build'), - '17110': ( '10', None, 'Redstone 4', None, 'preview build'), - '17112': ( '10', None, 'Redstone 4', None, 'preview build'), - '17115': ( '10', None, 'Redstone 4', None, 'preview build'), - '17120': ( '10', None, 'Redstone 4', None, 'preview build'), - '17123': ( '10', None, 'Redstone 4', None, 'preview build'), - '17127': ( '10', None, 'Redstone 4', None, 'preview build'), - '17128': ( '10', None, 'Redstone 4', None, 'preview build'), - '17133': ( '10', None, 'Redstone 4', None, 'preview build'), - '17134': ( '10', 'v1803', 'Redstone 4', 'April 2018 Update', None), - '17604': ( '10', None, 'Redstone 5', None, 'preview build'), - '17618': ( '10', None, 'Redstone 5', None, 'preview build'), - '17623': ( '10', None, 'Redstone 5', None, 'preview build'), - '17627': ( '10', None, 'Redstone 5', None, 'preview build'), - '17634': ( '10', None, 'Redstone 5', None, 'preview build'), - '17639': ( '10', None, 'Redstone 5', None, 'preview build'), - '17643': ( '10', None, 'Redstone 5', None, 'preview build'), - '17650': ( '10', None, 'Redstone 5', None, 'preview build'), - '17655': ( '10', None, 'Redstone 5', None, 'preview build'), - '17661': ( '10', None, 'Redstone 5', None, 'preview build'), - '17666': ( '10', None, 'Redstone 5', None, 'preview build'), - '17677': ( '10', None, 'Redstone 5', None, 'preview build'), - '17682': ( '10', None, 'Redstone 5', None, 'preview build'), - '17686': ( '10', None, 'Redstone 5', None, 'preview build'), - '17692': ( '10', None, 'Redstone 5', None, 'preview build'), - '17704': ( '10', None, 'Redstone 5', None, 'preview build'), - '17711': ( '10', None, 'Redstone 5', None, 'preview build'), - '17713': ( '10', None, 'Redstone 5', None, 'preview build'), - '17723': ( '10', None, 'Redstone 5', None, 'preview build'), - '17728': ( '10', None, 'Redstone 5', None, 'preview build'), - '17730': ( '10', None, 'Redstone 5', None, 'preview build'), - '17733': ( '10', None, 'Redstone 5', None, 'preview build'), - '17735': ( '10', None, 'Redstone 5', None, 'preview build'), - '17738': ( '10', None, 'Redstone 5', None, 'preview build'), - '17741': ( '10', None, 'Redstone 5', None, 'preview build'), - '17744': ( '10', None, 'Redstone 5', None, 'preview build'), - '17746': ( '10', None, 'Redstone 5', None, 'preview build'), - '17751': ( '10', None, 'Redstone 5', None, 'preview build'), - '17754': ( '10', None, 'Redstone 5', None, 'preview build'), - '17755': ( '10', None, 'Redstone 5', None, 'preview build'), - '17758': ( '10', None, 'Redstone 5', None, 'preview build'), - '17760': ( '10', None, 'Redstone 5', None, 'preview build'), - '17763': ( '10', 'v1809', 'Redstone 5', 'October 2018 Update', None), - '18204': ( '10', None, '19H1', None, 'preview build'), - '18214': ( '10', None, '19H1', None, 'preview build'), - '18219': ( '10', None, '19H1', None, 'preview build'), - '18234': ( '10', None, '19H1', None, 'preview build'), - '18237': ( '10', None, '19H1', None, 'preview build'), - '18242': ( '10', None, '19H1', None, 'preview build'), - '18247': ( '10', None, '19H1', None, 'preview build'), - '18252': ( '10', None, '19H1', None, 'preview build'), - '18262': ( '10', None, '19H1', None, 'preview build'), - '18267': ( '10', None, '19H1', None, 'preview build'), - '18272': ( '10', None, '19H1', None, 'preview build'), - '18277': ( '10', None, '19H1', None, 'preview build'), - '18282': ( '10', None, '19H1', None, 'preview build'), - '18290': ( '10', None, '19H1', None, 'preview build'), - '18298': ( '10', None, '19H1', None, 'preview build'), - '18305': ( '10', None, '19H1', None, 'preview build'), + '9841': ('10', None, 'Threshold 1', None, 'preview build'), + '9860': ('10', None, 'Threshold 1', None, 'preview build'), + '9879': ('10', None, 'Threshold 1', None, 'preview build'), + '9926': ('10', None, 'Threshold 1', None, 'preview build'), + '10041': ('10', None, 'Threshold 1', None, 'preview build'), + '10049': ('10', None, 'Threshold 1', None, 'preview build'), + '10061': ('10', None, 'Threshold 1', None, 'preview build'), + '10074': ('10', None, 'Threshold 1', None, 'preview build'), + '10122': ('10', None, 'Threshold 1', None, 'preview build'), + '10130': ('10', None, 'Threshold 1', None, 'preview build'), + '10158': ('10', None, 'Threshold 1', None, 'preview build'), + '10159': ('10', None, 'Threshold 1', None, 'preview build'), + '10162': ('10', None, 'Threshold 1', None, 'preview build'), + '10166': ('10', None, 'Threshold 1', None, 'preview build'), + '10240': ('10', 'v1507', 'Threshold 1', None, 'unsupported'), + '10525': ('10', None, 'Threshold 2', None, 'preview build'), + '10532': ('10', None, 'Threshold 2', None, 'preview build'), + '10547': ('10', None, 'Threshold 2', None, 'preview build'), + '10565': ('10', None, 'Threshold 2', None, 'preview build'), + '10576': ('10', None, 'Threshold 2', None, 'preview build'), + '10586': ('10', 'v1511', 'Threshold 2', 'November Update', 'unsupported'), + '11082': ('10', None, 'Redstone 1', None, 'preview build'), + '11099': ('10', None, 'Redstone 1', None, 'preview build'), + '11102': ('10', None, 'Redstone 1', None, 'preview build'), + '14251': ('10', None, 'Redstone 1', None, 'preview build'), + '14257': ('10', None, 'Redstone 1', None, 'preview build'), + '14271': ('10', None, 'Redstone 1', None, 'preview build'), + '14279': ('10', None, 'Redstone 1', None, 'preview build'), + '14291': ('10', None, 'Redstone 1', None, 'preview build'), + '14295': ('10', None, 'Redstone 1', None, 'preview build'), + '14316': ('10', None, 'Redstone 1', None, 'preview build'), + '14328': ('10', None, 'Redstone 1', None, 'preview build'), + '14332': ('10', None, 'Redstone 1', None, 'preview build'), + '14342': ('10', None, 'Redstone 1', None, 'preview build'), + '14352': ('10', None, 'Redstone 1', None, 'preview build'), + '14361': ('10', None, 'Redstone 1', None, 'preview build'), + '14366': ('10', None, 'Redstone 1', None, 'preview build'), + '14367': ('10', None, 'Redstone 1', None, 'preview build'), + '14371': ('10', None, 'Redstone 1', None, 'preview build'), + '14372': ('10', None, 'Redstone 1', None, 'preview build'), + '14376': ('10', None, 'Redstone 1', None, 'preview build'), + '14379': ('10', None, 'Redstone 1', None, 'preview build'), + '14383': ('10', None, 'Redstone 1', None, 'preview build'), + '14385': ('10', None, 'Redstone 1', None, 'preview build'), + '14388': ('10', None, 'Redstone 1', None, 'preview build'), + '14390': ('10', None, 'Redstone 1', None, 'preview build'), + '14393': ('10', 'v1607', 'Redstone 1', 'Anniversary Update', 'unsupported'), + '14901': ('10', None, 'Redstone 2', None, 'preview build'), + '14905': ('10', None, 'Redstone 2', None, 'preview build'), + '14915': ('10', None, 'Redstone 2', None, 'preview build'), + '14926': ('10', None, 'Redstone 2', None, 'preview build'), + '14931': ('10', None, 'Redstone 2', None, 'preview build'), + '14936': ('10', None, 'Redstone 2', None, 'preview build'), + '14942': ('10', None, 'Redstone 2', None, 'preview build'), + '14946': ('10', None, 'Redstone 2', None, 'preview build'), + '14951': ('10', None, 'Redstone 2', None, 'preview build'), + '14955': ('10', None, 'Redstone 2', None, 'preview build'), + '14959': ('10', None, 'Redstone 2', None, 'preview build'), + '14965': ('10', None, 'Redstone 2', None, 'preview build'), + '14971': ('10', None, 'Redstone 2', None, 'preview build'), + '14986': ('10', None, 'Redstone 2', None, 'preview build'), + '15002': ('10', None, 'Redstone 2', None, 'preview build'), + '15007': ('10', None, 'Redstone 2', None, 'preview build'), + '15014': ('10', None, 'Redstone 2', None, 'preview build'), + '15019': ('10', None, 'Redstone 2', None, 'preview build'), + '15025': ('10', None, 'Redstone 2', None, 'preview build'), + '15031': ('10', None, 'Redstone 2', None, 'preview build'), + '15042': ('10', None, 'Redstone 2', None, 'preview build'), + '15046': ('10', None, 'Redstone 2', None, 'preview build'), + '15048': ('10', None, 'Redstone 2', None, 'preview build'), + '15055': ('10', None, 'Redstone 2', None, 'preview build'), + '15058': ('10', None, 'Redstone 2', None, 'preview build'), + '15060': ('10', None, 'Redstone 2', None, 'preview build'), + '15061': ('10', None, 'Redstone 2', None, 'preview build'), + '15063': ('10', 'v1703', 'Redstone 2', 'Creators Update', 'unsupported'), + '16170': ('10', None, 'Redstone 3', None, 'preview build'), + '16176': ('10', None, 'Redstone 3', None, 'preview build'), + '16179': ('10', None, 'Redstone 3', None, 'preview build'), + '16184': ('10', None, 'Redstone 3', None, 'preview build'), + '16188': ('10', None, 'Redstone 3', None, 'preview build'), + '16193': ('10', None, 'Redstone 3', None, 'preview build'), + '16199': ('10', None, 'Redstone 3', None, 'preview build'), + '16212': ('10', None, 'Redstone 3', None, 'preview build'), + '16215': ('10', None, 'Redstone 3', None, 'preview build'), + '16226': ('10', None, 'Redstone 3', None, 'preview build'), + '16232': ('10', None, 'Redstone 3', None, 'preview build'), + '16237': ('10', None, 'Redstone 3', None, 'preview build'), + '16241': ('10', None, 'Redstone 3', None, 'preview build'), + '16251': ('10', None, 'Redstone 3', None, 'preview build'), + '16257': ('10', None, 'Redstone 3', None, 'preview build'), + '16273': ('10', None, 'Redstone 3', None, 'preview build'), + '16275': ('10', None, 'Redstone 3', None, 'preview build'), + '16278': ('10', None, 'Redstone 3', None, 'preview build'), + '16281': ('10', None, 'Redstone 3', None, 'preview build'), + '16288': ('10', None, 'Redstone 3', None, 'preview build'), + '16291': ('10', None, 'Redstone 3', None, 'preview build'), + '16294': ('10', None, 'Redstone 3', None, 'preview build'), + '16296': ('10', None, 'Redstone 3', None, 'preview build'), + '16299': ('10', 'v1709', 'Redstone 3', 'Fall Creators Update', 'outdated'), + '16353': ('10', None, 'Redstone 4', None, 'preview build'), + '16362': ('10', None, 'Redstone 4', None, 'preview build'), + '17004': ('10', None, 'Redstone 4', None, 'preview build'), + '17017': ('10', None, 'Redstone 4', None, 'preview build'), + '17025': ('10', None, 'Redstone 4', None, 'preview build'), + '17035': ('10', None, 'Redstone 4', None, 'preview build'), + '17040': ('10', None, 'Redstone 4', None, 'preview build'), + '17046': ('10', None, 'Redstone 4', None, 'preview build'), + '17063': ('10', None, 'Redstone 4', None, 'preview build'), + '17074': ('10', None, 'Redstone 4', None, 'preview build'), + '17083': ('10', None, 'Redstone 4', None, 'preview build'), + '17093': ('10', None, 'Redstone 4', None, 'preview build'), + '17101': ('10', None, 'Redstone 4', None, 'preview build'), + '17107': ('10', None, 'Redstone 4', None, 'preview build'), + '17110': ('10', None, 'Redstone 4', None, 'preview build'), + '17112': ('10', None, 'Redstone 4', None, 'preview build'), + '17115': ('10', None, 'Redstone 4', None, 'preview build'), + '17120': ('10', None, 'Redstone 4', None, 'preview build'), + '17123': ('10', None, 'Redstone 4', None, 'preview build'), + '17127': ('10', None, 'Redstone 4', None, 'preview build'), + '17128': ('10', None, 'Redstone 4', None, 'preview build'), + '17133': ('10', None, 'Redstone 4', None, 'preview build'), + '17134': ('10', 'v1803', 'Redstone 4', 'April 2018 Update', 'outdated'), + '17604': ('10', None, 'Redstone 5', None, 'preview build'), + '17618': ('10', None, 'Redstone 5', None, 'preview build'), + '17623': ('10', None, 'Redstone 5', None, 'preview build'), + '17627': ('10', None, 'Redstone 5', None, 'preview build'), + '17634': ('10', None, 'Redstone 5', None, 'preview build'), + '17639': ('10', None, 'Redstone 5', None, 'preview build'), + '17643': ('10', None, 'Redstone 5', None, 'preview build'), + '17650': ('10', None, 'Redstone 5', None, 'preview build'), + '17655': ('10', None, 'Redstone 5', None, 'preview build'), + '17661': ('10', None, 'Redstone 5', None, 'preview build'), + '17666': ('10', None, 'Redstone 5', None, 'preview build'), + '17677': ('10', None, 'Redstone 5', None, 'preview build'), + '17682': ('10', None, 'Redstone 5', None, 'preview build'), + '17686': ('10', None, 'Redstone 5', None, 'preview build'), + '17692': ('10', None, 'Redstone 5', None, 'preview build'), + '17704': ('10', None, 'Redstone 5', None, 'preview build'), + '17711': ('10', None, 'Redstone 5', None, 'preview build'), + '17713': ('10', None, 'Redstone 5', None, 'preview build'), + '17723': ('10', None, 'Redstone 5', None, 'preview build'), + '17728': ('10', None, 'Redstone 5', None, 'preview build'), + '17730': ('10', None, 'Redstone 5', None, 'preview build'), + '17733': ('10', None, 'Redstone 5', None, 'preview build'), + '17735': ('10', None, 'Redstone 5', None, 'preview build'), + '17738': ('10', None, 'Redstone 5', None, 'preview build'), + '17741': ('10', None, 'Redstone 5', None, 'preview build'), + '17744': ('10', None, 'Redstone 5', None, 'preview build'), + '17746': ('10', None, 'Redstone 5', None, 'preview build'), + '17751': ('10', None, 'Redstone 5', None, 'preview build'), + '17754': ('10', None, 'Redstone 5', None, 'preview build'), + '17755': ('10', None, 'Redstone 5', None, 'preview build'), + '17758': ('10', None, 'Redstone 5', None, 'preview build'), + '17760': ('10', None, 'Redstone 5', None, 'preview build'), + '17763': ('10', 'v1809', 'Redstone 5', 'October 2018 Update', None), + '18204': ('10', None, '19H1', None, 'preview build'), + '18214': ('10', None, '19H1', None, 'preview build'), + '18219': ('10', None, '19H1', None, 'preview build'), + '18234': ('10', None, '19H1', None, 'preview build'), + '18237': ('10', None, '19H1', None, 'preview build'), + '18242': ('10', None, '19H1', None, 'preview build'), + '18247': ('10', None, '19H1', None, 'preview build'), + '18252': ('10', None, '19H1', None, 'preview build'), + '18262': ('10', None, '19H1', None, 'preview build'), + '18267': ('10', None, '19H1', None, 'preview build'), + '18272': ('10', None, '19H1', None, 'preview build'), + '18277': ('10', None, '19H1', None, 'preview build'), + '18282': ('10', None, '19H1', None, 'preview build'), + '18290': ('10', None, '19H1', None, 'preview build'), + '18298': ('10', None, '19H1', None, 'preview build'), + '18305': ('10', None, '19H1', None, 'preview build'), } if __name__ == '__main__': - print("This file is not meant to be called directly.") + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From b6e9e447c5868b06b839cf7cd9b659844ae697f7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 22:45:49 -0700 Subject: [PATCH 148/186] Fixed main.py --- .bin/Scripts/settings/main.py | 62 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 98f867fe..3ea1fa2b 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -5,7 +5,7 @@ ENABLED_OPEN_LOGS = False ENABLED_TICKET_NUMBERS = False ENABLED_UPLOAD_DATA = False HW_OVERRIDES_FORCED = False -HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED +HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED # STATIC VARIABLES (also used by BASH and BATCH files) ## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH @@ -15,15 +15,15 @@ KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux -MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags +MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags ROOT_PASSWORD='Abracadabra' TECH_PASSWORD='Abracadabra' # Server IP addresses OFFICE_SERVER_IP='10.0.0.10' QUICKBOOKS_SERVER_IP='10.0.0.10' # Time Zones -LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values -WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values +LINUX_TIME_ZONE='America/Denver' # See 'timedatectl list-timezones' for valid values +WINDOWS_TIME_ZONE='Mountain Standard Time' # See 'tzutil /l' for valid values # WiFi WIFI_SSID='SomeWiFi' WIFI_PASSWORD='Abracadabra' @@ -34,57 +34,57 @@ WIFI_PASSWORD='Abracadabra' ## user/password for all of those shares. BACKUP_SERVERS = [ { 'IP': '10.0.0.10', - 'Name': 'ServerOne', + 'Name': 'ServerOne', 'Mounted': False, - 'Share': 'Backups', - 'User': 'restore', - 'Pass': 'Abracadabra', + 'Share': 'Backups', + 'User': 'restore', + 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', }, { 'IP': '10.0.0.11', - 'Name': 'ServerTwo', + 'Name': 'ServerTwo', 'Mounted': False, - 'Share': 'Backups', - 'User': 'restore', - 'Pass': 'Abracadabra', + 'Share': 'Backups', + 'User': 'restore', + 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', }, ] CRASH_SERVER = { - 'Name': 'CrashServer', - 'Url': '', - 'User': '', - 'Pass': '', + 'Name': 'CrashServer', + 'Url': '', + 'User': '', + 'Pass': '', } OFFICE_SERVER = { - 'IP': OFFICE_SERVER_IP, - 'Name': 'ServerOne', + 'IP': OFFICE_SERVER_IP, + 'Name': 'ServerOne', 'Mounted': False, - 'Share': 'Office', - 'User': 'restore', - 'Pass': 'Abracadabra', + 'Share': 'Office', + 'User': 'restore', + 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', } QUICKBOOKS_SERVER = { - 'IP': QUICKBOOKS_SERVER_IP, - 'Name': 'ServerOne', + 'IP': QUICKBOOKS_SERVER_IP, + 'Name': 'ServerOne', 'Mounted': False, - 'Share': 'QuickBooks', - 'User': 'restore', - 'Pass': 'Abracadabra', + 'Share': 'QuickBooks', + 'User': 'restore', + 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', } WINDOWS_SERVER = { - 'IP': '10.0.0.10', - 'Name': 'ServerOne', + 'IP': '10.0.0.10', + 'Name': 'ServerOne', 'Mounted': False, - 'Share': 'Windows', - 'User': 'restore', - 'Pass': 'Abracadabra', + 'Share': 'Windows', + 'User': 'restore', + 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', } From 453ce9cf44dcb51d63b38c202dc85e92ba26ee01 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 22:48:20 -0700 Subject: [PATCH 149/186] More fixes for main.py --- .bin/Scripts/settings/main.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 3ea1fa2b..da83e830 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -30,10 +30,10 @@ WIFI_PASSWORD='Abracadabra' # SERVER VARIABLES ## NOTE: Windows can only use one user per server. This means that if -## one server serves multiple shares then you have to use the same -## user/password for all of those shares. +## one server serves multiple shares then you have to use the same +## user/password for all of those shares. BACKUP_SERVERS = [ - { 'IP': '10.0.0.10', + { 'IP': '10.0.0.10', 'Name': 'ServerOne', 'Mounted': False, 'Share': 'Backups', @@ -41,8 +41,8 @@ BACKUP_SERVERS = [ 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', - }, - { 'IP': '10.0.0.11', + }, + { 'IP': '10.0.0.11', 'Name': 'ServerTwo', 'Mounted': False, 'Share': 'Backups', @@ -50,14 +50,14 @@ BACKUP_SERVERS = [ 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', - }, -] + }, + ] CRASH_SERVER = { 'Name': 'CrashServer', 'Url': '', 'User': '', 'Pass': '', -} + } OFFICE_SERVER = { 'IP': OFFICE_SERVER_IP, 'Name': 'ServerOne', @@ -67,7 +67,7 @@ OFFICE_SERVER = { 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', -} + } QUICKBOOKS_SERVER = { 'IP': QUICKBOOKS_SERVER_IP, 'Name': 'ServerOne', @@ -77,7 +77,7 @@ QUICKBOOKS_SERVER = { 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', -} + } WINDOWS_SERVER = { 'IP': '10.0.0.10', 'Name': 'ServerOne', @@ -87,7 +87,7 @@ WINDOWS_SERVER = { 'Pass': 'Abracadabra', 'RW-User': 'backup', 'RW-Pass': 'Abracadabra', -} + } if __name__ == '__main__': print("This file is not meant to be called directly.") From 91649f5ee7277b884d61567475de47ece4c6215a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 27 Dec 2018 23:43:35 -0700 Subject: [PATCH 150/186] Removed args from run_program * It's been deprecated for ages * Fixes issue #79 --- .bin/Scripts/functions/common.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 368ce687..f0315aff 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -490,12 +490,8 @@ def print_log(message='', end='\n', timestamp=True): end = end)) -def run_program(cmd, args=[], check=True, pipe=True, shell=False, **kwargs): +def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): """Run program and return a subprocess.CompletedProcess object.""" - if args: - # Deprecated so let's raise an exception to find & fix all occurances - print_error('ERROR: Using args is no longer supported.') - raise Exception cmd = [c for c in cmd if c] if shell: cmd = ' '.join(cmd) From 575fa174257b9d19cad8d03b1b4e62ffd674cb0c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 28 Dec 2018 15:44:14 -0700 Subject: [PATCH 151/186] Only use Unicode checkmark if in X --- .bin/Scripts/functions/ddrescue.py | 7 +++++-- .bin/Scripts/functions/hw_diags.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index a2b27e5e..f6c30e0a 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -781,6 +781,9 @@ def menu_ddrescue(source_path, dest_path, run_mode): def menu_main(state): """Main menu is used to set ddrescue settings.""" + checkmark = '*' + if 'DISPLAY' in global_vars['Env']: + checkmark = '✓' title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) @@ -804,8 +807,8 @@ def menu_main(state): while True: # Update entries for opt in main_options: - opt['Name'] = '{} {}'.format( - '[✓]' if opt['Enabled'] else '[ ]', + opt['Name'] = '[{}] {}'.format( + checkmark if opt['Enabled'] else ' ', opt['Base Name']) selection = menu_select( diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 155031ec..fae78b21 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -751,6 +751,9 @@ def get_read_rate(s): def menu_diags(state, args): """Main menu to select and run HW tests.""" args = [a.lower() for a in args] + checkmark = '*' + if 'DISPLAY' in global_vars['Env']: + checkmark = '✓' title = '{}\nMain Menu'.format(TOP_PANE_TEXT) # NOTE: Changing the order of main_options will break everything main_options = [ @@ -819,8 +822,8 @@ def menu_diags(state, args): # Update checkboxes for opt in main_options: _nvme_smart = opt['Base Name'] == 'NVMe / SMART' - opt['Name'] = '{} {} {}'.format( - '[✓]' if opt['Enabled'] else '[ ]', + opt['Name'] = '[{}] {} {}'.format( + checkmark if opt['Enabled'] else ' ', opt['Base Name'], QUICK_LABEL if state.quick_mode and _nvme_smart else '') From 8f9bae9a6f5170697300f824cc25c154e1e390a4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 28 Dec 2018 16:51:15 -0700 Subject: [PATCH 152/186] Added option to build Linux with minimal packages * All non-minimal packages/configs have been separated from the base setup * `"Build Linux" -b` will only build the full version --- .../include/airootfs/etc/skel/.zlogin | 28 +--------- .../airootfs/etc/oblogout.conf | 0 .../airootfs/etc/skel/.Xresources | 0 .../etc/skel/.config/Thunar/accels.scm | 0 .../airootfs/etc/skel/.config/Thunar/uca.xml | 0 .../airootfs/etc/skel/.config/dunst/dunstrc | 0 .../etc/skel/.config/gtk-3.0/settings.ini | 0 .../airootfs/etc/skel/.config/i3/config | 0 .../airootfs/etc/skel/.config/i3status/config | 0 .../airootfs/etc/skel/.config/mimeapps.list | 0 .../etc/skel/.config/openbox/autostart | 0 .../etc/skel/.config/openbox/environment | 0 .../etc/skel/.config/openbox/menu.xml | 0 .../airootfs/etc/skel/.config/openbox/rc.xml | 0 .../airootfs/etc/skel/.config/rofi/config | 0 .../airootfs/etc/skel/.config/tint2/tint2rc | 0 .../etc/skel/.config/volumeicon/volumeicon | 0 .../airootfs/etc/skel/.conky_start | 0 .../airootfs/etc/skel/.conkyrc | 0 .../airootfs/etc/skel/.gtkrc-2.0 | 0 .../airootfs/etc/skel/.update_conky | 0 .../airootfs/etc/skel/.update_x | 0 .../airootfs/etc/skel/.wallpaper | 0 .../airootfs/etc/skel/.xinitrc | 0 .../include_x/airootfs/etc/skel/.zlogin | 32 +++++++++++ .../applications/Hardware Diagnostics.desktop | 0 .../applications/Hardware Information.desktop | 0 .../share/applications/NetworkTest.desktop | 0 .linux_items/packages/live_add | 56 +------------------ .linux_items/packages/live_add_x | 55 ++++++++++++++++++ Build Linux | 38 +++++++++---- 31 files changed, 118 insertions(+), 91 deletions(-) rename .linux_items/{include => include_x}/airootfs/etc/oblogout.conf (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.Xresources (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/Thunar/accels.scm (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/Thunar/uca.xml (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/dunst/dunstrc (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/gtk-3.0/settings.ini (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/i3/config (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/i3status/config (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/mimeapps.list (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/openbox/autostart (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/openbox/environment (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/openbox/menu.xml (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/openbox/rc.xml (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/rofi/config (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/tint2/tint2rc (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.config/volumeicon/volumeicon (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.conky_start (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.conkyrc (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.gtkrc-2.0 (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.update_conky (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.update_x (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.wallpaper (100%) rename .linux_items/{include => include_x}/airootfs/etc/skel/.xinitrc (100%) create mode 100644 .linux_items/include_x/airootfs/etc/skel/.zlogin rename .linux_items/{include => include_x}/airootfs/usr/share/applications/Hardware Diagnostics.desktop (100%) rename .linux_items/{include => include_x}/airootfs/usr/share/applications/Hardware Information.desktop (100%) rename .linux_items/{include => include_x}/airootfs/usr/share/applications/NetworkTest.desktop (100%) create mode 100644 .linux_items/packages/live_add_x diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/.linux_items/include/airootfs/etc/skel/.zlogin index 04b1316e..68868d8e 100644 --- a/.linux_items/include/airootfs/etc/skel/.zlogin +++ b/.linux_items/include/airootfs/etc/skel/.zlogin @@ -3,30 +3,6 @@ if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then # Connect to network and update hostname $HOME/.update_network - # Update settings if using i3 - if fgrep -q "i3" /proc/cmdline; then - sed -i -r 's/#(own_window_type override)/\1/' ~/.conkyrc - sed -i -r 's/openbox-session/i3/' ~/.xinitrc - fi - - # Update Conky - $HOME/.update_conky - - # Start X or HW-diags - if ! fgrep -q "nox" /proc/cmdline; then - # Kill Xorg after 30 seconds if it doesn't fully initialize - (sleep 30s; if ! [[ -f "/tmp/x_ok" ]]; then pkill '(Xorg|startx)'; fi) & - - # Try starting X - startx >/dev/null - - # Run Hw-Diags CLI if necessary - if ! [[ -f "/tmp/x_ok" ]]; then - echo "There was an issue starting Xorg, starting CLI interface..." - sleep 2s - hw-diags --cli - fi - else - hw-diags --cli - fi + # Start HW-diags + hw-diags --cli fi diff --git a/.linux_items/include/airootfs/etc/oblogout.conf b/.linux_items/include_x/airootfs/etc/oblogout.conf similarity index 100% rename from .linux_items/include/airootfs/etc/oblogout.conf rename to .linux_items/include_x/airootfs/etc/oblogout.conf diff --git a/.linux_items/include/airootfs/etc/skel/.Xresources b/.linux_items/include_x/airootfs/etc/skel/.Xresources similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.Xresources rename to .linux_items/include_x/airootfs/etc/skel/.Xresources diff --git a/.linux_items/include/airootfs/etc/skel/.config/Thunar/accels.scm b/.linux_items/include_x/airootfs/etc/skel/.config/Thunar/accels.scm similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/Thunar/accels.scm rename to .linux_items/include_x/airootfs/etc/skel/.config/Thunar/accels.scm diff --git a/.linux_items/include/airootfs/etc/skel/.config/Thunar/uca.xml b/.linux_items/include_x/airootfs/etc/skel/.config/Thunar/uca.xml similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/Thunar/uca.xml rename to .linux_items/include_x/airootfs/etc/skel/.config/Thunar/uca.xml diff --git a/.linux_items/include/airootfs/etc/skel/.config/dunst/dunstrc b/.linux_items/include_x/airootfs/etc/skel/.config/dunst/dunstrc similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/dunst/dunstrc rename to .linux_items/include_x/airootfs/etc/skel/.config/dunst/dunstrc diff --git a/.linux_items/include/airootfs/etc/skel/.config/gtk-3.0/settings.ini b/.linux_items/include_x/airootfs/etc/skel/.config/gtk-3.0/settings.ini similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/gtk-3.0/settings.ini rename to .linux_items/include_x/airootfs/etc/skel/.config/gtk-3.0/settings.ini diff --git a/.linux_items/include/airootfs/etc/skel/.config/i3/config b/.linux_items/include_x/airootfs/etc/skel/.config/i3/config similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/i3/config rename to .linux_items/include_x/airootfs/etc/skel/.config/i3/config diff --git a/.linux_items/include/airootfs/etc/skel/.config/i3status/config b/.linux_items/include_x/airootfs/etc/skel/.config/i3status/config similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/i3status/config rename to .linux_items/include_x/airootfs/etc/skel/.config/i3status/config diff --git a/.linux_items/include/airootfs/etc/skel/.config/mimeapps.list b/.linux_items/include_x/airootfs/etc/skel/.config/mimeapps.list similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/mimeapps.list rename to .linux_items/include_x/airootfs/etc/skel/.config/mimeapps.list diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart b/.linux_items/include_x/airootfs/etc/skel/.config/openbox/autostart similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/openbox/autostart rename to .linux_items/include_x/airootfs/etc/skel/.config/openbox/autostart diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/environment b/.linux_items/include_x/airootfs/etc/skel/.config/openbox/environment similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/openbox/environment rename to .linux_items/include_x/airootfs/etc/skel/.config/openbox/environment diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/menu.xml b/.linux_items/include_x/airootfs/etc/skel/.config/openbox/menu.xml similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/openbox/menu.xml rename to .linux_items/include_x/airootfs/etc/skel/.config/openbox/menu.xml diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml b/.linux_items/include_x/airootfs/etc/skel/.config/openbox/rc.xml similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml rename to .linux_items/include_x/airootfs/etc/skel/.config/openbox/rc.xml diff --git a/.linux_items/include/airootfs/etc/skel/.config/rofi/config b/.linux_items/include_x/airootfs/etc/skel/.config/rofi/config similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/rofi/config rename to .linux_items/include_x/airootfs/etc/skel/.config/rofi/config diff --git a/.linux_items/include/airootfs/etc/skel/.config/tint2/tint2rc b/.linux_items/include_x/airootfs/etc/skel/.config/tint2/tint2rc similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/tint2/tint2rc rename to .linux_items/include_x/airootfs/etc/skel/.config/tint2/tint2rc diff --git a/.linux_items/include/airootfs/etc/skel/.config/volumeicon/volumeicon b/.linux_items/include_x/airootfs/etc/skel/.config/volumeicon/volumeicon similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.config/volumeicon/volumeicon rename to .linux_items/include_x/airootfs/etc/skel/.config/volumeicon/volumeicon diff --git a/.linux_items/include/airootfs/etc/skel/.conky_start b/.linux_items/include_x/airootfs/etc/skel/.conky_start similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.conky_start rename to .linux_items/include_x/airootfs/etc/skel/.conky_start diff --git a/.linux_items/include/airootfs/etc/skel/.conkyrc b/.linux_items/include_x/airootfs/etc/skel/.conkyrc similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.conkyrc rename to .linux_items/include_x/airootfs/etc/skel/.conkyrc diff --git a/.linux_items/include/airootfs/etc/skel/.gtkrc-2.0 b/.linux_items/include_x/airootfs/etc/skel/.gtkrc-2.0 similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.gtkrc-2.0 rename to .linux_items/include_x/airootfs/etc/skel/.gtkrc-2.0 diff --git a/.linux_items/include/airootfs/etc/skel/.update_conky b/.linux_items/include_x/airootfs/etc/skel/.update_conky similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.update_conky rename to .linux_items/include_x/airootfs/etc/skel/.update_conky diff --git a/.linux_items/include/airootfs/etc/skel/.update_x b/.linux_items/include_x/airootfs/etc/skel/.update_x similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.update_x rename to .linux_items/include_x/airootfs/etc/skel/.update_x diff --git a/.linux_items/include/airootfs/etc/skel/.wallpaper b/.linux_items/include_x/airootfs/etc/skel/.wallpaper similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.wallpaper rename to .linux_items/include_x/airootfs/etc/skel/.wallpaper diff --git a/.linux_items/include/airootfs/etc/skel/.xinitrc b/.linux_items/include_x/airootfs/etc/skel/.xinitrc similarity index 100% rename from .linux_items/include/airootfs/etc/skel/.xinitrc rename to .linux_items/include_x/airootfs/etc/skel/.xinitrc diff --git a/.linux_items/include_x/airootfs/etc/skel/.zlogin b/.linux_items/include_x/airootfs/etc/skel/.zlogin new file mode 100644 index 00000000..04b1316e --- /dev/null +++ b/.linux_items/include_x/airootfs/etc/skel/.zlogin @@ -0,0 +1,32 @@ +setterm -blank 0 -powerdown 0 2>/dev/null +if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then + # Connect to network and update hostname + $HOME/.update_network + + # Update settings if using i3 + if fgrep -q "i3" /proc/cmdline; then + sed -i -r 's/#(own_window_type override)/\1/' ~/.conkyrc + sed -i -r 's/openbox-session/i3/' ~/.xinitrc + fi + + # Update Conky + $HOME/.update_conky + + # Start X or HW-diags + if ! fgrep -q "nox" /proc/cmdline; then + # Kill Xorg after 30 seconds if it doesn't fully initialize + (sleep 30s; if ! [[ -f "/tmp/x_ok" ]]; then pkill '(Xorg|startx)'; fi) & + + # Try starting X + startx >/dev/null + + # Run Hw-Diags CLI if necessary + if ! [[ -f "/tmp/x_ok" ]]; then + echo "There was an issue starting Xorg, starting CLI interface..." + sleep 2s + hw-diags --cli + fi + else + hw-diags --cli + fi +fi diff --git a/.linux_items/include/airootfs/usr/share/applications/Hardware Diagnostics.desktop b/.linux_items/include_x/airootfs/usr/share/applications/Hardware Diagnostics.desktop similarity index 100% rename from .linux_items/include/airootfs/usr/share/applications/Hardware Diagnostics.desktop rename to .linux_items/include_x/airootfs/usr/share/applications/Hardware Diagnostics.desktop diff --git a/.linux_items/include/airootfs/usr/share/applications/Hardware Information.desktop b/.linux_items/include_x/airootfs/usr/share/applications/Hardware Information.desktop similarity index 100% rename from .linux_items/include/airootfs/usr/share/applications/Hardware Information.desktop rename to .linux_items/include_x/airootfs/usr/share/applications/Hardware Information.desktop diff --git a/.linux_items/include/airootfs/usr/share/applications/NetworkTest.desktop b/.linux_items/include_x/airootfs/usr/share/applications/NetworkTest.desktop similarity index 100% rename from .linux_items/include/airootfs/usr/share/applications/NetworkTest.desktop rename to .linux_items/include_x/airootfs/usr/share/applications/NetworkTest.desktop diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index a6da5f71..85ea5489 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -1,86 +1,47 @@ aic94xx-firmware alsa-utils antiword -arandr -arc-gtk-theme bash-pipes bc bluez bluez-utils -cbatticon chntpw cmatrix colordiff -compton -conky cpio curl dmidecode dos2unix -dunst e2fsprogs -evince -feh -ffmpeg -firefox -gnome-keyring -gparted -gpicview-gtk3 -gsmartcontrol -hardinfo hexedit hfsprogs htop -i3-gaps -i3lock-fancy-git -i3status ldns -leafpad lha libewf -libinput linux-firmware lm_sensors lzip mdadm mediainfo -mesa-demos -mkvtoolnix-cli mprime -mpv ncdu -network-manager-applet networkmanager -noto-fonts -noto-fonts-cjk -oblogout -openbox-patched -otf-font-awesome-4 p7zip -papirus-icon-theme progsreiserfs python python-psutil python-requests -qemu-guest-agent reiserfsprogs rfkill rng-tools -rofi -rxvt-unicode +rxvt-unicode-terminfo smartmontools-svn speedtest-cli -spice-vdagent terminus-font testdisk-wip -thunar -tigervnc -tint2 -tk tmux tree -ttf-font-awesome-4 -ttf-inconsolata udevil udisks2 ufw @@ -88,23 +49,8 @@ unarj unrar unzip util-linux -veracrypt vim -virtualbox-guest-modules-arch -virtualbox-guest-utils -volumeicon wd719x-firmware wimlib -xarchiver -xf86-input-libinput -xf86-video-amdgpu -xf86-video-fbdev -xf86-video-nouveau -xf86-video-vesa -xorg-server -xorg-xdpyinfo -xorg-xev -xorg-xinit -xorg-xinput zip zsh diff --git a/.linux_items/packages/live_add_x b/.linux_items/packages/live_add_x new file mode 100644 index 00000000..ab7d23b3 --- /dev/null +++ b/.linux_items/packages/live_add_x @@ -0,0 +1,55 @@ +arandr +arc-gtk-theme +cbatticon +compton +conky +dunst +evince +feh +ffmpeg +firefox +gnome-keyring +gparted +gpicview-gtk3 +gsmartcontrol +hardinfo +i3-gaps +i3lock-fancy-git +i3status +leafpad +libinput +mesa-demos +mkvtoolnix-cli +mpv +network-manager-applet +noto-fonts +noto-fonts-cjk +oblogout +openbox-patched +otf-font-awesome-4 +papirus-icon-theme +qemu-guest-agent +rofi +rxvt-unicode +spice-vdagent +thunar +tigervnc +tint2 +tk +ttf-font-awesome-4 +ttf-inconsolata +veracrypt +virtualbox-guest-modules-arch +virtualbox-guest-utils +volumeicon +xarchiver +xf86-input-libinput +xf86-video-amdgpu +xf86-video-fbdev +xf86-video-nouveau +xf86-video-vesa +xorg-server +xorg-xdpyinfo +xorg-xev +xorg-xinit +xorg-xinput diff --git a/Build Linux b/Build Linux index 87e93e2d..52e060d2 100755 --- a/Build Linux +++ b/Build Linux @@ -119,6 +119,9 @@ function copy_live_env() { # Add items rsync -aI "$ROOT_DIR/.linux_items/include/" "$LIVE_DIR/" + if [[ "${1:-}" != "--minimal" ]]; then + rsync -aI "$ROOT_DIR/.linux_items/include_x/" "$LIVE_DIR/" + fi mkdir -p "$LIVE_DIR/airootfs/usr/local/bin" rsync -aI "$ROOT_DIR/.bin/Scripts/" "$LIVE_DIR/airootfs/usr/local/bin/" cp -a "$BUILD_DIR/main.py" "$LIVE_DIR/airootfs/usr/local/bin/settings/" @@ -195,6 +198,9 @@ function update_live_env() { sed -i "/$p/d" "$LIVE_DIR/packages.x86_64" done < "$ROOT_DIR/.linux_items/packages/live_remove" cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.x86_64" + if [[ "${1:-}" != "--minimal" ]]; then + cat "$ROOT_DIR/.linux_items/packages/live_add_x" >> "$LIVE_DIR/packages.x86_64" + fi echo "[custom]" >> "$LIVE_DIR/pacman.conf" echo "SigLevel = Optional TrustAll" >> "$LIVE_DIR/pacman.conf" echo "Server = file://$REPO_DIR" >> "$LIVE_DIR/pacman.conf" @@ -213,10 +219,12 @@ function update_live_env() { rm -Rf "$SKEL_DIR/.oh-my-zsh/.git" curl -o "$SKEL_DIR/.oh-my-zsh/themes/lean.zsh-theme" https://raw.githubusercontent.com/miekg/lean/master/prompt_lean_setup - # Openbox theme - git clone --depth=1 https://github.com/addy-dclxvi/Openbox-Theme-Collections.git "$TEMP_DIR/ob-themes" - mkdir -p "$LIVE_DIR/airootfs/usr/share/themes" - cp -a "$TEMP_DIR/ob-themes/Triste-Orange" "$LIVE_DIR/airootfs/usr/share/themes/" + if [[ "${1:-}" != "--minimal" ]]; then + # Openbox theme + git clone --depth=1 https://github.com/addy-dclxvi/Openbox-Theme-Collections.git "$TEMP_DIR/ob-themes" + mkdir -p "$LIVE_DIR/airootfs/usr/share/themes" + cp -a "$TEMP_DIR/ob-themes/Triste-Orange" "$LIVE_DIR/airootfs/usr/share/themes/" + fi # Services sed -i -r 's/^(.*pacman-init.*)$/#NOPE#\1/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" @@ -259,13 +267,15 @@ function update_live_env() { # udevil fix echo "mkdir /media" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" - # VNC password - echo "mkdir '/home/$username/.vnc'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" - echo "echo '$TECH_PASSWORD' | vncpasswd -f > '/home/$username/.vnc/passwd'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + if [[ "${1:-}" != "--minimal" ]]; then + # VNC password + echo "mkdir '/home/$username/.vnc'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + echo "echo '$TECH_PASSWORD' | vncpasswd -f > '/home/$username/.vnc/passwd'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" - # Wallpaper - mkdir -p "$LIVE_DIR/airootfs/usr/share/wallpaper" - cp "$ROOT_DIR/Images/Linux.png" "$LIVE_DIR/airootfs/usr/share/wallpaper/burned.in" + # Wallpaper + mkdir -p "$LIVE_DIR/airootfs/usr/share/wallpaper" + cp "$ROOT_DIR/Images/Linux.png" "$LIVE_DIR/airootfs/usr/share/wallpaper/burned.in" + fi } function update_repo() { @@ -386,6 +396,13 @@ case ${1:-} in echo Done ;; + -m|--prep-minimal-env) + load_settings --edit + copy_live_env --minimal + update_live_env --minimal + echo Done + ;; + -o|--build-iso) load_settings build_iso @@ -414,6 +431,7 @@ case ${1:-} in echo "Advanced options:" echo " -f --fix-perms Fix folder permissions" echo " -i --install-deps Install build dependencies" + echo " -m --prep-minimal-env Prep live & airootfs folders (minimal packages)" echo " -o --build-iso Build ISO (using current setup)" echo " -p --prep-live-env Prep live & airootfs folders" echo " -u --update-repo Update custom pacman repo" From a9c5c1c27425ef88e90038bbdcd1028a20a367ab Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 28 Dec 2018 17:46:02 -0700 Subject: [PATCH 153/186] Fixed issue #80 --- .bin/Scripts/functions/hw_diags.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index fae78b21..1b1d2ae5 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -366,6 +366,8 @@ class DiskObj(): # Attributes if 'NVMe / SMART' not in self.tests: report.extend(self.generate_attribute_report()) + elif not self.tests['NVMe / SMART'].report: + report.extend(self.generate_attribute_report()) # Tests for test in self.tests.values(): @@ -1592,6 +1594,9 @@ def show_results(state): show_report(disk.generate_disk_report(), log_report=True) print_standard(' ') + # Update progress + update_progress_pane(state) + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" From 6340bceb118736e8294dabfbe67b806c9d1ec89a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Jan 2019 17:05:08 -0700 Subject: [PATCH 154/186] Added warning if no disks detected. --- .bin/Scripts/functions/hw_diags.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 1b1d2ae5..202be76d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1010,6 +1010,10 @@ def run_hw_tests(state): f = v['Function'] for test_obj in v['Objects']: f(state, test_obj) + if not v['Objects']: + # No devices available + v['Objects'].append(TestObj(dev=None, label='')) + v['Objects'][-1].update_status('N/A') except GenericAbort: # Cleanup tmux_kill_pane(*state.panes.values()) @@ -1593,6 +1597,9 @@ def show_results(state): for disk in state.disks: show_report(disk.generate_disk_report(), log_report=True) print_standard(' ') + if not state.disks: + print_warning('No devices') + print_standard(' ') # Update progress update_progress_pane(state) From 3122a75f646252f44d11b2e86ef10d1c269e175d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Jan 2019 17:31:19 -0700 Subject: [PATCH 155/186] Skip fan RPMs * Avoids reporting fan RPMs as 6000+ *C --- .bin/Scripts/functions/sensors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index a50f8941..8b2b8b03 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -134,6 +134,9 @@ def get_sensor_data(): ## current temp is labeled xxxx_input for _source, _labels in _sources.items(): for _label, _temp in _labels.items(): + if _label.startswith('fan'): + # Skip fan RPMs + continue if 'input' in _label: sensor_data[_section][_adapter][_source] = { 'Current': _temp, From 141fe422db805a13731e146a26f348ba0d125b7c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Jan 2019 17:55:56 -0700 Subject: [PATCH 156/186] Fix NVMe attribute handling * Addresses issue #78 --- .bin/Scripts/functions/hw_diags.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 202be76d..e966dc77 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -418,7 +418,9 @@ class DiskObj(): # Check for attributes if KEY_NVME in self.smartctl: - self.nvme_attributes.update(self.smartctl[KEY_NVME]) + self.nvme_attributes = { + k: {'name': k, 'raw': int(v), 'raw_str': str(v)} + for k, v in self.smartctl[KEY_NVME].items()} elif KEY_SMART in self.smartctl: for a in self.smartctl[KEY_SMART].get('table', {}): try: @@ -453,7 +455,7 @@ class DiskObj(): self.check_attributes(silent) # Check if a self-test is currently running - if 'remaining_percent' in self.smart_self_test['status']: + if 'remaining_percent' in self.smart_self_test.get('status', ''): _msg = 'SMART self-test in progress, all tests disabled' # Ask to abort From 3d69fe773db38b3f6582c0d96af158e4948db809 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Jan 2019 18:02:24 -0700 Subject: [PATCH 157/186] Removed sensors section from hw-info * Fixes issue #81 --- .bin/Scripts/hw-info | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.bin/Scripts/hw-info b/.bin/Scripts/hw-info index 98514e3e..8321e7aa 100755 --- a/.bin/Scripts/hw-info +++ b/.bin/Scripts/hw-info @@ -105,8 +105,3 @@ echo -e "${BLUE}Drives${CLEAR}" hw-drive-info | sed 's/^/ /' echo "" -# Sensors -echo -e "${BLUE}Sensors${CLEAR}" -hw-sensors | sed 's/^/ /' -echo "" - From aa4c6a1434095e4753e169cafcab24c9c52bd5c7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 2 Jan 2019 18:04:32 -0700 Subject: [PATCH 158/186] Fix issue #82 --- .bin/Scripts/hw-sensors-monitor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 42757748..731f415e 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -10,7 +10,7 @@ os.chdir(os.path.dirname(os.path.realpath(__file__))) sys.path.append(os.getcwd()) from functions.sensors import * from functions.tmux import * -init_global_vars() +init_global_vars(silent=True) if __name__ == '__main__': background = False From 4a04e92cafe30ff68d56ea2f4e8f03d524a7b49e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 5 Jan 2019 15:54:05 -0700 Subject: [PATCH 159/186] Added threading.py * Will be used by hw_diags.py and ddrescue.py for * Better control over badblocks * Background the tmux pane fixes --- .bin/Scripts/functions/threading.py | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .bin/Scripts/functions/threading.py diff --git a/.bin/Scripts/functions/threading.py b/.bin/Scripts/functions/threading.py new file mode 100644 index 00000000..dfac69c7 --- /dev/null +++ b/.bin/Scripts/functions/threading.py @@ -0,0 +1,47 @@ +# Wizard Kit: Functions - Threading + +from threading import Thread +from queue import Queue, Empty + +# Classes +class NonBlockingStreamReader(): + """Class to allow non-blocking reads from a stream.""" + # Credits: + ## https://gist.github.com/EyalAr/7915597 + ## https://stackoverflow.com/a/4896288 + + def __init__(self, stream): + self.stream = stream + self.queue = Queue() + + def populate_queue(stream, queue): + """Collect lines from stream and put them in queue.""" + while True: + line = stream.read(1) + if line: + queue.put(line) + + self.thread = start_thread( + populate_queue, + args=(self.stream, self.queue)) + + def read(self, timeout=None): + try: + return self.queue.get(block = timeout is not None, + timeout = timeout) + except Empty: + return None + + +# Functions +def start_thread(function, args=[], daemon=True): + """Run function as thread in background, returns Thread object.""" + thread = Thread(target=function, args=args, daemon=daemon) + thread.start() + return thread + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From e40b0b98e40b56692479a306e49bdfc51b65e0ce Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 6 Jan 2019 20:57:06 -0700 Subject: [PATCH 160/186] Moved fix_tmux_panes() into a background thread --- .bin/Scripts/functions/hw_diags.py | 120 +++++++++++++++-------------- .bin/Scripts/functions/tmux.py | 3 + 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index e966dc77..5dba82f9 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -6,6 +6,7 @@ import time from collections import OrderedDict from functions.sensors import * +from functions.threading import * from functions.tmux import * @@ -79,9 +80,15 @@ TESTS_DISK = [ ] TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) TMUX_LAYOUT = OrderedDict({ - 'Top': {'y': 2, 'Check': True}, - 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, - 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Top': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, + # Testing panes + 'Prime95': {'y': 11, 'Check': False}, + 'Temps': {'y': 1000, 'Check': False}, + 'SMART': {'y': 3, 'Check': True}, + 'badblocks': {'y': 5, 'Check': True}, + 'I/O Benchmark': {'y': 1000, 'Check': False}, }) @@ -640,12 +647,25 @@ def build_status_string(label, status, info_label=False): **COLORS) -def fix_tmux_panes(state, tmux_layout): +def fix_tmux_panes_loop(state): + while True: + try: + fix_tmux_panes(state) + sleep(1) + except AttributeError: + # tmux_layout attribute has been deleted, exit function + return + except RuntimeError: + # Assuming layout definitions changes mid-run, ignoring + pass + + +def fix_tmux_panes(state): """Fix pane sizes if the window has been resized.""" needs_fixed = False # Check layout - for k, v in tmux_layout.items(): + for k, v in state.tmux_layout.items(): if not v.get('Check'): # Not concerned with the size of this pane continue @@ -670,7 +690,7 @@ def fix_tmux_panes(state, tmux_layout): return # Update layout - for k, v in tmux_layout.items(): + for k, v in state.tmux_layout.items(): # Get target target = None if k != 'Current': @@ -886,10 +906,6 @@ def run_badblocks_test(state, test): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, test.dev.description)) - test.tmux_layout = TMUX_LAYOUT.copy() - test.tmux_layout.update({ - 'badblocks': {'y': 5, 'Check': True}, - }) # Create monitor pane test.badblocks_out = '{}/badblocks_{}.out'.format( @@ -908,14 +924,7 @@ def run_badblocks_test(state, test): test.badblocks_proc = popen_program( ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], pipe=True) - while True: - try: - test.badblocks_proc.wait(timeout=1) - except subprocess.TimeoutExpired: - fix_tmux_panes(state, test.tmux_layout) - else: - # badblocks finished, exit loop - break + test.badblocks_proc.wait() except KeyboardInterrupt: test.aborted = True @@ -960,7 +969,7 @@ def run_badblocks_test(state, test): update_progress_pane(state) # Cleanup - tmux_kill_pane(state.panes['badblocks']) + tmux_kill_pane(state.panes.pop('badblocks', None)) def run_hw_tests(state): @@ -971,6 +980,7 @@ def run_hw_tests(state): # Build Panes update_progress_pane(state) build_outer_panes(state) + start_tmux_repair_thread(state) # Show selected tests and create TestObj()s print_info('Selected Tests:') @@ -1018,11 +1028,13 @@ def run_hw_tests(state): v['Objects'][-1].update_status('N/A') except GenericAbort: # Cleanup + stop_tmux_repair_thread(state) tmux_kill_pane(*state.panes.values()) # Rebuild panes update_progress_pane(state) build_outer_panes(state) + start_tmux_repair_thread(state) # Mark unfinished tests as aborted for k, v in state.tests.items(): @@ -1036,12 +1048,14 @@ def run_hw_tests(state): # Done show_results(state) + sleep(1) if state.quick_mode: - pause('Press Enter to exit...') + pause('Press Enter to exit... ') else: pause('Press Enter to return to main menu... ') # Cleanup + stop_tmux_repair_thread(state) tmux_kill_pane(*state.panes.values()) @@ -1062,11 +1076,7 @@ def run_io_benchmark(state, test): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, test.dev.description)) - test.tmux_layout = TMUX_LAYOUT.copy() - test.tmux_layout.update({ - 'io_benchmark': {'y': 1000, 'Check': False}, - 'Current': {'y': 15, 'Check': True}, - }) + state.tmux_layout['Current'] = {'y': 15, 'Check': True} # Create monitor pane test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( @@ -1123,9 +1133,6 @@ def run_io_benchmark(state, test): # Update offset offset += test.dev.dd_chunk_blocks + skip - # Fix panes - fix_tmux_panes(state, test.tmux_layout) - except DeviceTooSmallError: # Device too small, skipping test test.update_status('N/A') @@ -1200,7 +1207,8 @@ def run_io_benchmark(state, test): update_progress_pane(state) # Cleanup - tmux_kill_pane(state.panes['io_benchmark']) + state.tmux_layout.pop('Current', None) + tmux_kill_pane(state.panes.pop('io_benchmark', None)) def run_keyboard_test(): @@ -1226,12 +1234,6 @@ def run_mprime_test(state, test): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format(TOP_PANE_TEXT, test.dev.name)) - test.tmux_layout = TMUX_LAYOUT.copy() - test.tmux_layout.update({ - 'Temps': {'y': 1000, 'Check': False}, - 'mprime': {'y': 11, 'Check': False}, - 'Current': {'y': 3, 'Check': True}, - }) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) @@ -1244,11 +1246,12 @@ def run_mprime_test(state, test): pipe=True) # Create monitor and worker panes - state.panes['mprime'] = tmux_split_window( + state.panes['Prime95'] = tmux_split_window( lines=10, vertical=True, text=' ') state.panes['Temps'] = tmux_split_window( behind=True, percent=80, vertical=True, watch=test.sensors_out) tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3) + state.tmux_layout['Current'] = {'y': 3, 'Check': True} # Get idle temps clear_screen() @@ -1263,7 +1266,7 @@ def run_mprime_test(state, test): test.abort_msg = 'If running too hot, press CTRL+c to abort the test' run_program(['apple-fans', 'max']) tmux_update_pane( - state.panes['mprime'], + state.panes['Prime95'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) time_limit = int(MPRIME_LIMIT) * 60 @@ -1285,9 +1288,6 @@ def run_mprime_test(state, test): print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) update_sensor_data(test.sensor_data) - # Fix panes - fix_tmux_panes(state, test.tmux_layout) - # Wait sleep(1) except KeyboardInterrupt: @@ -1305,7 +1305,7 @@ def run_mprime_test(state, test): # Stop Prime95 (twice for good measure) run_program(['killall', '-s', 'INT', 'mprime'], check=False) sleep(1) - tmux_kill_pane(state.panes['mprime']) + tmux_kill_pane(state.panes.pop('Prime95', None)) # Get cooldown temp run_program(['apple-fans', 'auto']) @@ -1399,7 +1399,11 @@ def run_mprime_test(state, test): update_progress_pane(state) # Cleanup - tmux_kill_pane(state.panes['mprime'], state.panes['Temps']) + state.tmux_layout.pop('Current', None) + tmux_kill_pane( + state.panes.pop('Prime95', None), + state.panes.pop('Temps', None), + ) test.monitor_proc.kill() @@ -1428,10 +1432,6 @@ def run_nvme_smart_tests(state, test): state.panes['Top'], text='{}\n{}'.format( TOP_PANE_TEXT, test.dev.description)) - test.tmux_layout = TMUX_LAYOUT.copy() - test.tmux_layout.update({ - 'smart': {'y': 3, 'Check': True}, - }) # NVMe if test.dev.nvme_attributes: @@ -1471,7 +1471,7 @@ def run_nvme_smart_tests(state, test): global_vars['LogDir'], test.dev.name) with open(test.smart_out, 'w') as f: f.write('SMART self-test status:\n Starting...') - state.panes['smart'] = tmux_split_window( + state.panes['SMART'] = tmux_split_window( lines=3, vertical=True, watch=test.smart_out) # Show attributes @@ -1486,15 +1486,8 @@ def run_nvme_smart_tests(state, test): # Monitor progress try: - for i in range(int(test.timeout*60)): - sleep(1) - - # Fix panes - fix_tmux_panes(state, test.tmux_layout) - - # Only update SMART progress every 5 seconds - if i % 5 != 0: - continue + for i in range(int(test.timeout*60/5)): + sleep(5) # Update SMART data test.dev.get_smart_details() @@ -1544,7 +1537,7 @@ def run_nvme_smart_tests(state, test): test.dev.disable_test(t, 'Denied') # Cleanup - tmux_kill_pane(state.panes['smart']) + tmux_kill_pane(state.panes.pop('SMART', None)) # Save report test.report = test.dev.generate_attribute_report( @@ -1607,6 +1600,19 @@ def show_results(state): update_progress_pane(state) +def start_tmux_repair_thread(state): + """Fix tmux panes as long as state.tmux_layout attribute exists.""" + state.tmux_layout = TMUX_LAYOUT.copy() + start_thread(fix_tmux_panes_loop, args=[state]) + + +def stop_tmux_repair_thread(state): + """Stop previous thread by causing an AttributeError in the thread.""" + if hasattr(state, 'tmux_layout'): + del state.tmux_layout + sleep(1) + + def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index a11560bc..e2d1b333 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -44,6 +44,9 @@ def tmux_kill_pane(*panes): """Kill tmux pane by id.""" cmd = ['tmux', 'kill-pane', '-t'] for pane_id in panes: + if not pane_id: + # Skip empty strings, None values, etc + continue run_program(cmd+[pane_id], check=False) From 68bbee66d55416877fca5731e87bba2506b88efd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 6 Jan 2019 21:45:01 -0700 Subject: [PATCH 161/186] Replaced hw-diags-badblocks with threaded section * Should fix issue #83 --- .bin/Scripts/functions/hw_diags.py | 36 +++++++++++++++++++++--------- .bin/Scripts/hw-diags-badblocks | 18 --------------- 2 files changed, 26 insertions(+), 28 deletions(-) delete mode 100755 .bin/Scripts/hw-diags-badblocks diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 5dba82f9..c9ce87f7 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -895,6 +895,18 @@ def run_badblocks_test(state, test): if test.disabled: return + def _save_badblocks_output(read_all=False, timeout=0.1): + """Get badblocks output and append to both file and var.""" + _output = '' + while _output is not None: + _output = test.badblocks_nbsr.read(0.1) + if _output is not None: + test.badblocks_stderr += _output.decode() + with open(test.badblocks_out, 'a') as f: + f.write(_output.decode()) + if not read_all: + break + # Prep print_log('Starting badblocks test for {}'.format(test.dev.path)) test.started = True @@ -920,22 +932,26 @@ def run_badblocks_test(state, test): # Start badblocks print_standard('Running badblocks test...') - try: - test.badblocks_proc = popen_program( - ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out], - pipe=True) - test.badblocks_proc.wait() + test.badblocks_proc = popen_program( + ['sudo', 'badblocks', '-sv', '-e', '1', test.dev.path], + pipe=True, bufsize=1) + test.badblocks_nbsr = NonBlockingStreamReader(test.badblocks_proc.stderr) + test.badblocks_stderr = '' + # Update progress loop + try: + while test.badblocks_proc.poll() is None: + _save_badblocks_output() except KeyboardInterrupt: + run_program(['killall', 'badblocks'], check=False) test.aborted = True + # Save remaining badblocks output + _save_badblocks_output(read_all=True) + # Check result and build report test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS)) - try: - test.badblocks_out = test.badblocks_proc.stdout.read().decode() - except Exception as err: - test.badblocks_out = 'Error: {}'.format(err) - for line in test.badblocks_out.splitlines(): + for line in test.badblocks_stderr.splitlines(): line = line.strip() if not line or re.search(r'^Checking', line, re.IGNORECASE): # Skip empty and progress lines diff --git a/.bin/Scripts/hw-diags-badblocks b/.bin/Scripts/hw-diags-badblocks deleted file mode 100755 index 2d915766..00000000 --- a/.bin/Scripts/hw-diags-badblocks +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -## Wizard Kit: HW Diagnostics - badblocks - -function usage { - echo "Usage: $0 device log-file" - echo " e.g. $0 /dev/sda /tmp/tmp.XXXXXXX/badblocks.log" -} - -# Bail early -if [ ! -b "$1" ]; then - usage - exit 1 -fi - -# Run Badblocks -sudo badblocks -sv -e 1 "$1" 2>&1 | tee -a "$2" - From 7a9474a6a86495e1d9001605a3c18749c08bfc48 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 6 Jan 2019 21:51:45 -0700 Subject: [PATCH 162/186] Try enabling SMART before checking attributes * Fixes issue #84 --- .bin/Scripts/functions/hw_diags.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index c9ce87f7..e4f193f4 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -157,6 +157,11 @@ class DiskObj(): self.smartctl = {} self.tests = OrderedDict() self.get_details() + + # Try enabling SMART + run_program(['sudo', 'smartctl', '--smart=on', self.path], check=False) + + # Get NVMe/SMART data and set description self.get_smart_details() self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) From 50da682d76a07c7877ac5d9b918ee12ee5353ab3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 6 Jan 2019 22:12:01 -0700 Subject: [PATCH 163/186] Fix issue #85 --- .bin/Scripts/functions/sensors.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 8b2b8b03..22a27ef4 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -108,14 +108,27 @@ def get_raw_sensor_data(): """Read sensor data and return dict.""" data = {} cmd = ['sensors', '-j'] + + # Get raw data try: result = run_program(cmd) - data = json.loads(result.stdout.decode()) except subprocess.CalledProcessError: # Assuming no sensors available, return empty dict below pass - return data + # Workaround for bad sensors + raw_data = [] + for line in result.stdout.decode().splitlines(): + if line.strip() == ',': + # Assuming malformatted line caused by missing data + continue + raw_data.append(line) + + # Parse JSON data + json_data = json.loads('\n'.join(raw_data)) + + # Done + return json_data def get_sensor_data(): From ae92eea76e1df16dbd19411e39f87245e2017416 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 6 Jan 2019 22:43:33 -0700 Subject: [PATCH 164/186] Added macbook12-spi-driver-dkms to Minimal build --- .linux_items/packages/aur | 1 + .linux_items/packages/live_add_min | 1 + Build Linux | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .linux_items/packages/live_add_min diff --git a/.linux_items/packages/aur b/.linux_items/packages/aur index 8d752673..9588b129 100644 --- a/.linux_items/packages/aur +++ b/.linux_items/packages/aur @@ -2,6 +2,7 @@ aic94xx-firmware bash-pipes hfsprogs i3lock-fancy-git +macbook12-spi-driver-dkms mprime openbox-patched smartmontools-svn diff --git a/.linux_items/packages/live_add_min b/.linux_items/packages/live_add_min new file mode 100644 index 00000000..85653460 --- /dev/null +++ b/.linux_items/packages/live_add_min @@ -0,0 +1 @@ +macbook12-spi-driver-dkms diff --git a/Build Linux b/Build Linux index 52e060d2..4ccd1729 100755 --- a/Build Linux +++ b/Build Linux @@ -198,8 +198,10 @@ function update_live_env() { sed -i "/$p/d" "$LIVE_DIR/packages.x86_64" done < "$ROOT_DIR/.linux_items/packages/live_remove" cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.x86_64" - if [[ "${1:-}" != "--minimal" ]]; then - cat "$ROOT_DIR/.linux_items/packages/live_add_x" >> "$LIVE_DIR/packages.x86_64" + if [[ "${1:-}" == "--minimal" ]]; then + cat "$ROOT_DIR/.linux_items/packages/live_add_min" >> "$LIVE_DIR/packages.x86_64" + else + cat "$ROOT_DIR/.linux_items/packages/live_add_x" >> "$LIVE_DIR/packages.x86_64" fi echo "[custom]" >> "$LIVE_DIR/pacman.conf" echo "SigLevel = Optional TrustAll" >> "$LIVE_DIR/pacman.conf" From ebcd38ef50b334a68bdcec5cfc4bcca2f647bed6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 7 Jan 2019 12:53:23 -0700 Subject: [PATCH 165/186] Don't change directory during initialization * Fixes issue with ddrescue-tui --- .bin/Scripts/activate.py | 3 +-- .bin/Scripts/cbs_fix.py | 3 +-- .bin/Scripts/check_disk.py | 3 +-- .bin/Scripts/connect-to-network | 3 +-- .bin/Scripts/ddrescue-tui-menu | 3 +-- .bin/Scripts/dism.py | 3 +-- .bin/Scripts/hw-diags-audio | 3 +-- .bin/Scripts/hw-diags-menu | 3 +-- .bin/Scripts/hw-diags-network | 3 +-- .bin/Scripts/hw-sensors-monitor | 3 +-- .bin/Scripts/install_sw_bundle.py | 3 +-- .bin/Scripts/install_vcredists.py | 3 +-- .bin/Scripts/mount-all-volumes | 3 +-- .bin/Scripts/mount-backup-shares | 3 +-- .bin/Scripts/msword-search | 3 +-- .bin/Scripts/safemode_enter.py | 3 +-- .bin/Scripts/safemode_exit.py | 3 +-- .bin/Scripts/sfc_scan.py | 3 +-- .bin/Scripts/system_checklist.py | 3 +-- .bin/Scripts/system_diagnostics.py | 3 +-- .bin/Scripts/transferred_keys.py | 3 +-- .bin/Scripts/update_kit.py | 3 +-- .bin/Scripts/user_checklist.py | 3 +-- .bin/Scripts/user_data_transfer.py | 3 +-- .bin/Scripts/winpe_root_menu.py | 3 +-- 25 files changed, 25 insertions(+), 50 deletions(-) diff --git a/.bin/Scripts/activate.py b/.bin/Scripts/activate.py index cf17263f..f9a9c69d 100644 --- a/.bin/Scripts/activate.py +++ b/.bin/Scripts/activate.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.activation import * init_global_vars() os.system('title {}: Windows Activation Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/cbs_fix.py b/.bin/Scripts/cbs_fix.py index 460a32a5..36a7906f 100644 --- a/.bin/Scripts/cbs_fix.py +++ b/.bin/Scripts/cbs_fix.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.cleanup import * from functions.data import * init_global_vars() diff --git a/.bin/Scripts/check_disk.py b/.bin/Scripts/check_disk.py index 1140f4fc..5b0f3c14 100644 --- a/.bin/Scripts/check_disk.py +++ b/.bin/Scripts/check_disk.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.repairs import * init_global_vars() os.system('title {}: Check Disk Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/connect-to-network b/.bin/Scripts/connect-to-network index 5e04e903..02500b37 100755 --- a/.bin/Scripts/connect-to-network +++ b/.bin/Scripts/connect-to-network @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.network import * init_global_vars() diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index f1f3b08c..a6014d76 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.ddrescue import * from functions.hw_diags import * init_global_vars() diff --git a/.bin/Scripts/dism.py b/.bin/Scripts/dism.py index 8b227da2..2ef3ff25 100644 --- a/.bin/Scripts/dism.py +++ b/.bin/Scripts/dism.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.repairs import * init_global_vars() os.system('title {}: DISM helper Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/hw-diags-audio b/.bin/Scripts/hw-diags-audio index 2c52c467..3d3a5991 100755 --- a/.bin/Scripts/hw-diags-audio +++ b/.bin/Scripts/hw-diags-audio @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.common import * init_global_vars() diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index 5b6f4f76..1c3cfc42 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.hw_diags import * from functions.tmux import * init_global_vars() diff --git a/.bin/Scripts/hw-diags-network b/.bin/Scripts/hw-diags-network index 2ae0e263..3047e131 100755 --- a/.bin/Scripts/hw-diags-network +++ b/.bin/Scripts/hw-diags-network @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.network import * diff --git a/.bin/Scripts/hw-sensors-monitor b/.bin/Scripts/hw-sensors-monitor index 731f415e..c27d786a 100755 --- a/.bin/Scripts/hw-sensors-monitor +++ b/.bin/Scripts/hw-sensors-monitor @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.sensors import * from functions.tmux import * init_global_vars(silent=True) diff --git a/.bin/Scripts/install_sw_bundle.py b/.bin/Scripts/install_sw_bundle.py index 4386bfb0..b44874ae 100644 --- a/.bin/Scripts/install_sw_bundle.py +++ b/.bin/Scripts/install_sw_bundle.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.setup import * init_global_vars() os.system('title {}: SW Bundle Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/install_vcredists.py b/.bin/Scripts/install_vcredists.py index 1a722bf9..a22cc729 100644 --- a/.bin/Scripts/install_vcredists.py +++ b/.bin/Scripts/install_vcredists.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.setup import * init_global_vars() os.system('title {}: Install Visual C++ Runtimes'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/mount-all-volumes b/.bin/Scripts/mount-all-volumes index 2c0a6621..f49439f2 100755 --- a/.bin/Scripts/mount-all-volumes +++ b/.bin/Scripts/mount-all-volumes @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.data import * init_global_vars() diff --git a/.bin/Scripts/mount-backup-shares b/.bin/Scripts/mount-backup-shares index 69374a9b..57fbe572 100755 --- a/.bin/Scripts/mount-backup-shares +++ b/.bin/Scripts/mount-backup-shares @@ -6,8 +6,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.data import * from functions.network import * init_global_vars() diff --git a/.bin/Scripts/msword-search b/.bin/Scripts/msword-search index c0461dda..b005c3c1 100755 --- a/.bin/Scripts/msword-search +++ b/.bin/Scripts/msword-search @@ -15,8 +15,7 @@ USAGE = '''Usage: {script} ... the search-terms provided (case-insensitive).'''.format(script=__file__) # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.network import * init_global_vars() diff --git a/.bin/Scripts/safemode_enter.py b/.bin/Scripts/safemode_enter.py index 027f4987..bc6d659d 100644 --- a/.bin/Scripts/safemode_enter.py +++ b/.bin/Scripts/safemode_enter.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.safemode import * init_global_vars() os.system('title {}: SafeMode Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/safemode_exit.py b/.bin/Scripts/safemode_exit.py index 85376d51..bbbdbcf8 100644 --- a/.bin/Scripts/safemode_exit.py +++ b/.bin/Scripts/safemode_exit.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.safemode import * init_global_vars() os.system('title {}: SafeMode Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/sfc_scan.py b/.bin/Scripts/sfc_scan.py index 9864306a..884694f5 100644 --- a/.bin/Scripts/sfc_scan.py +++ b/.bin/Scripts/sfc_scan.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.repairs import * init_global_vars() os.system('title {}: SFC Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/system_checklist.py b/.bin/Scripts/system_checklist.py index 1aa95d3e..5900ef00 100644 --- a/.bin/Scripts/system_checklist.py +++ b/.bin/Scripts/system_checklist.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.activation import * from functions.cleanup import * from functions.info import * diff --git a/.bin/Scripts/system_diagnostics.py b/.bin/Scripts/system_diagnostics.py index c7b363d8..47c35eab 100644 --- a/.bin/Scripts/system_diagnostics.py +++ b/.bin/Scripts/system_diagnostics.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.browsers import * from functions.info import * from functions.product_keys import * diff --git a/.bin/Scripts/transferred_keys.py b/.bin/Scripts/transferred_keys.py index 3308844c..6dab114d 100644 --- a/.bin/Scripts/transferred_keys.py +++ b/.bin/Scripts/transferred_keys.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.product_keys import * init_global_vars() os.system('title {}: Transferred Key Finder'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/update_kit.py b/.bin/Scripts/update_kit.py index e02acc73..3bc31a72 100644 --- a/.bin/Scripts/update_kit.py +++ b/.bin/Scripts/update_kit.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.update import * init_global_vars() os.system('title {}: Kit Update Tool'.format(KIT_NAME_FULL)) diff --git a/.bin/Scripts/user_checklist.py b/.bin/Scripts/user_checklist.py index b57e8b29..91e5915a 100644 --- a/.bin/Scripts/user_checklist.py +++ b/.bin/Scripts/user_checklist.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.browsers import * from functions.cleanup import * from functions.setup import * diff --git a/.bin/Scripts/user_data_transfer.py b/.bin/Scripts/user_data_transfer.py index 1fa06256..e63e0d7d 100644 --- a/.bin/Scripts/user_data_transfer.py +++ b/.bin/Scripts/user_data_transfer.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.data import * from functions.repairs import * init_global_vars() diff --git a/.bin/Scripts/winpe_root_menu.py b/.bin/Scripts/winpe_root_menu.py index 5a1f0f8b..a9555e07 100644 --- a/.bin/Scripts/winpe_root_menu.py +++ b/.bin/Scripts/winpe_root_menu.py @@ -4,8 +4,7 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from functions.winpe_menus import * # Fix 7-Zip name TOOLS['SevenZip'].pop('64') From beb36dfc97a63f217d3919ec21ebed46f41696a3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 7 Jan 2019 12:54:28 -0700 Subject: [PATCH 166/186] Adjusted window name --- .bin/Scripts/ddrescue-tui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/ddrescue-tui b/.bin/Scripts/ddrescue-tui index d83b3a7e..e41c6bb1 100755 --- a/.bin/Scripts/ddrescue-tui +++ b/.bin/Scripts/ddrescue-tui @@ -3,7 +3,7 @@ ## Wizard Kit: ddrescue TUI Launcher SESSION_NAME="ddrescue-tui" -WINDOW_NAME="GNU ddrescue TUI" +WINDOW_NAME="ddrescue TUI" MENU="ddrescue-tui-menu" function ask() { From 4bd0cd1598b6403a8502493e753dbda6171f0eae Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 7 Jan 2019 15:29:33 -0700 Subject: [PATCH 167/186] Avoid crash when no sensor data available * This was broken when fixing issue #85 --- .bin/Scripts/functions/sensors.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 22a27ef4..65a1b93e 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -112,20 +112,25 @@ def get_raw_sensor_data(): # Get raw data try: result = run_program(cmd) + result = result.stdout.decode().splitlines() except subprocess.CalledProcessError: - # Assuming no sensors available, return empty dict below - pass + # Assuming no sensors available, set to empty list + result = [] # Workaround for bad sensors raw_data = [] - for line in result.stdout.decode().splitlines(): + for line in result: if line.strip() == ',': # Assuming malformatted line caused by missing data continue raw_data.append(line) # Parse JSON data - json_data = json.loads('\n'.join(raw_data)) + try: + json_data = json.loads('\n'.join(raw_data)) + except json.JSONDecodeError: + # Still broken, just set to empty dict + json_data = {} # Done return json_data From f8adbe074d53fbdd7d4d558ea5983caacad3eb5f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 7 Jan 2019 16:17:37 -0700 Subject: [PATCH 168/186] Added Linux headers for macbook12-spi-driver-dkms * Addresses issue #67 --- .linux_items/packages/live_add_min | 1 + 1 file changed, 1 insertion(+) diff --git a/.linux_items/packages/live_add_min b/.linux_items/packages/live_add_min index 85653460..b37bdc74 100644 --- a/.linux_items/packages/live_add_min +++ b/.linux_items/packages/live_add_min @@ -1 +1,2 @@ +linux-headers macbook12-spi-driver-dkms From ed70d1ab1872c572191b2d0aa6a76fff31930a3c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 8 Jan 2019 19:18:00 -0700 Subject: [PATCH 169/186] Simplified tmux repair thread handling * Just start once and let run until script is exited * Pretty sure this fixed the 100%+ CPU usage after returning to the menu --- .bin/Scripts/functions/hw_diags.py | 31 +++++++++++------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index e4f193f4..ecefa96c 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -581,6 +581,10 @@ class State(): if not skip_disk: self.disks.append(disk_obj) + # Start tmux thread + self.tmux_layout = TMUX_LAYOUT.copy() + start_thread(fix_tmux_panes_loop, args=[self]) + class TestObj(): """Object to track test data.""" @@ -657,9 +661,6 @@ def fix_tmux_panes_loop(state): try: fix_tmux_panes(state) sleep(1) - except AttributeError: - # tmux_layout attribute has been deleted, exit function - return except RuntimeError: # Assuming layout definitions changes mid-run, ignoring pass @@ -669,6 +670,10 @@ def fix_tmux_panes(state): """Fix pane sizes if the window has been resized.""" needs_fixed = False + # Bail? + if not state.panes: + return + # Check layout for k, v in state.tmux_layout.items(): if not v.get('Check'): @@ -1001,7 +1006,6 @@ def run_hw_tests(state): # Build Panes update_progress_pane(state) build_outer_panes(state) - start_tmux_repair_thread(state) # Show selected tests and create TestObj()s print_info('Selected Tests:') @@ -1049,13 +1053,13 @@ def run_hw_tests(state): v['Objects'][-1].update_status('N/A') except GenericAbort: # Cleanup - stop_tmux_repair_thread(state) tmux_kill_pane(*state.panes.values()) + state.panes.clear() + state.tmux_layout.pop('Current', None) # Rebuild panes update_progress_pane(state) build_outer_panes(state) - start_tmux_repair_thread(state) # Mark unfinished tests as aborted for k, v in state.tests.items(): @@ -1076,8 +1080,8 @@ def run_hw_tests(state): pause('Press Enter to return to main menu... ') # Cleanup - stop_tmux_repair_thread(state) tmux_kill_pane(*state.panes.values()) + state.panes.clear() def run_io_benchmark(state, test): @@ -1621,19 +1625,6 @@ def show_results(state): update_progress_pane(state) -def start_tmux_repair_thread(state): - """Fix tmux panes as long as state.tmux_layout attribute exists.""" - state.tmux_layout = TMUX_LAYOUT.copy() - start_thread(fix_tmux_panes_loop, args=[state]) - - -def stop_tmux_repair_thread(state): - """Stop previous thread by causing an AttributeError in the thread.""" - if hasattr(state, 'tmux_layout'): - del state.tmux_layout - sleep(1) - - def update_main_options(state, selection, main_options): """Update menu and state based on selection.""" index = int(selection) - 1 From 63f9c1c193f1dd93933d61fc32d822b9541ce40d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 8 Jan 2019 20:24:14 -0700 Subject: [PATCH 170/186] Fixed set_log_file() under Linux --- .bin/Scripts/functions/common.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index f0315aff..76a273b8 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -873,8 +873,14 @@ def set_linux_vars(): def set_log_file(log_name): """Sets global var LogFile and creates path as needed.""" - folder_path = r'{}\{}'.format(global_vars['LogDir'], KIT_NAME_FULL) - log_file = r'{}\{}'.format(folder_path, log_name) + folder_path = '{}{}{}'.format( + global_vars['LogDir'], + os.sep, + KIT_NAME_FULL) + log_file = '{}{}{}'.format( + folder_path, + os.sep, + log_name) os.makedirs(folder_path, exist_ok=True) global_vars['LogFile'] = log_file From 6ea4791dc9aac77d68f4020710aad53dce57b630 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 8 Jan 2019 20:24:43 -0700 Subject: [PATCH 171/186] Added generate_global_vars_report() * Makes crash reports more readable --- .bin/Scripts/functions/common.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 76a273b8..04af1d87 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -646,7 +646,10 @@ def upload_crash_details(): data += '#############################\n' data += 'Runtime Details:\n\n' data += 'sys.argv: {}\n\n'.format(sys.argv) - data += 'global_vars: {}\n'.format(global_vars) + try: + data += generate_global_vars_report() + except Exception: + data += 'global_vars: {}\n'.format(global_vars) filename = global_vars.get('LogFile', 'Unknown') filename = re.sub(r'.*(\\|/)', '', filename) filename += '.txt' @@ -821,6 +824,32 @@ def find_bin(): global_vars['BaseDir'] = base +def generate_global_vars_report(): + """Build readable string from global_vars, returns str.""" + report = ['global_vars: {'] + for k, v in sorted(global_vars.items()): + if k == 'Env': + continue + if isinstance(v, list): + report.append(' {}:'.format(str(k))) + for item in v: + report.append(' {}'.format(str(v))) + elif isinstance(v, dict): + report.append(' {}:'.format(str(k))) + for item_k, item_v in sorted(v.items()): + report.append(' {:<15} {}'.format( + str(item_k)+':', str(item_v))) + else: + report.append(' {:<18}{}'.format(str(k)+':', str(v))) + report.append(' Env:') + for k, v in sorted(global_vars.get('Env', {}).items()): + report.append(' {:<15} {}'.format( + str(k)+':', str(v))) + report.append('}') + + return '\n'.join(report) + + def make_tmp_dirs(): """Make temp directories.""" os.makedirs(global_vars['BackupDir'], exist_ok=True) From f2bd2a6e7509dd5b43f3a7e76b8c4d004be9ddce Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 8 Jan 2019 20:33:23 -0700 Subject: [PATCH 172/186] Adjusted global_vars report --- .bin/Scripts/functions/common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 04af1d87..2bbb4291 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -831,14 +831,16 @@ def generate_global_vars_report(): if k == 'Env': continue if isinstance(v, list): - report.append(' {}:'.format(str(k))) + report.append(' {}: ['.format(str(k))) for item in v: report.append(' {}'.format(str(v))) + report.append(' ]') elif isinstance(v, dict): - report.append(' {}:'.format(str(k))) + report.append(' {}: {{'.format(str(k))) for item_k, item_v in sorted(v.items()): report.append(' {:<15} {}'.format( str(item_k)+':', str(item_v))) + report.append(' }') else: report.append(' {:<18}{}'.format(str(k)+':', str(v))) report.append(' Env:') From e088ba134ef00c5085ce9b89174822e16f4dd2c3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 8 Jan 2019 23:48:50 -0700 Subject: [PATCH 173/186] Tool version bumps --- .bin/Scripts/build_kit.ps1 | 10 ++++---- .bin/Scripts/build_pe.ps1 | 20 ++++++++-------- .bin/Scripts/settings/launchers.py | 4 ++-- .bin/Scripts/settings/sources.py | 38 ++++++++++++++++-------------- 4 files changed, 37 insertions(+), 35 deletions(-) diff --git a/.bin/Scripts/build_kit.ps1 b/.bin/Scripts/build_kit.ps1 index bd213578..08a050c9 100644 --- a/.bin/Scripts/build_kit.ps1 +++ b/.bin/Scripts/build_kit.ps1 @@ -79,21 +79,21 @@ if ($MyInvocation.InvocationName -ne ".") { $Path = $Temp # 7-Zip - DownloadFile -Path $Path -Name "7z-installer.msi" -Url "https://www.7-zip.org/a/7z1805.msi" - DownloadFile -Path $Path -Name "7z-extra.7z" -Url "https://www.7-zip.org/a/7z1805-extra.7z" + DownloadFile -Path $Path -Name "7z-installer.msi" -Url "https://www.7-zip.org/a/7z1806.msi" + DownloadFile -Path $Path -Name "7z-extra.7z" -Url "https://www.7-zip.org/a/7z1806-extra.7z" # ConEmu $Url = "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z" DownloadFile -Path $Path -Name "ConEmuPack.7z" -Url $Url # Notepad++ - $Url = "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z" + $Url = "https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z" DownloadFile -Path $Path -Name "npp.7z" -Url $Url # Python - $Url = "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip" + $Url = "https://www.python.org/ftp/python/3.7.2/python-3.7.2.post1-embed-win32.zip" DownloadFile -Path $Path -Name "python32.zip" -Url $Url - $Url = "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip" + $Url = "https://www.python.org/ftp/python/3.7.2/python-3.7.2.post1-embed-amd64.zip" DownloadFile -Path $Path -Name "python64.zip" -Url $Url # Python: psutil diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index 560d570d..a2dd0363 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -131,25 +131,25 @@ if ($MyInvocation.InvocationName -ne ".") { ## Download Tools ## $ToolSources = @( # 7-Zip - @("7z-installer.msi", "https://www.7-zip.org/a/7z1805.msi"), - @("7z-extra.7z", "https://www.7-zip.org/a/7z1805-extra.7z"), + @("7z-installer.msi", "https://www.7-zip.org/a/7z1806.msi"), + @("7z-extra.7z", "https://www.7-zip.org/a/7z1806-extra.7z"), # Blue Screen View @("bluescreenview32.zip", "http://www.nirsoft.net/utils/bluescreenview.zip"), @("bluescreenview64.zip", "http://www.nirsoft.net/utils/bluescreenview-x64.zip"), # ConEmu @("ConEmuPack.7z", "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z"), # Fast Copy - @("fastcopy.zip", "http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip"), + @("fastcopy.zip", "http://ftp.vector.co.jp/70/93/2323/FastCopy361_installer.exe"), # HWiNFO - @("hwinfo.zip", "http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip"), + @("hwinfo.zip", "https://www.fosshub.com/HWiNFO.html?dwl=hwi_600.zip"), # Killer Network Drivers @( "killerinf.zip", ("http://www.killernetworking.com"+(FindDynamicUrl "http://www.killernetworking.com/driver-downloads/item/killer-drivers-inf" "Download Killer-Ethernet").replace('&', '&')) ), # Notepad++ - @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z"), - @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.x64.7z"), + @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z"), + @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.x64.7z"), # NT Password Editor @("ntpwed.zip", "http://cdslow.org.ru/files/ntpwedit/ntpwed07.zip"), # Prime95 @@ -159,8 +159,8 @@ if ($MyInvocation.InvocationName -ne ".") { @("produkey32.zip", "http://www.nirsoft.net/utils/produkey.zip"), @("produkey64.zip", "http://www.nirsoft.net/utils/produkey-x64.zip"), # Python - @("python32.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip"), - @("python64.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip"), + @("python32.zip", "https://www.python.org/ftp/python/3.7.2/python-3.7.2.post1-embed-win32.zip"), + @("python64.zip", "https://www.python.org/ftp/python/3.7.2/python-3.7.2.post1-embed-amd64.zip"), # Python: psutil @( "psutil64.whl", @@ -182,8 +182,8 @@ if ($MyInvocation.InvocationName -ne ".") { @("vcredist_x86.exe", "https://aka.ms/vs/15/release/vc_redist.x86.exe"), @("vcredist_x64.exe", "https://aka.ms/vs/15/release/vc_redist.x64.exe"), # wimlib-imagex - @("wimlib32.zip", "https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip"), - @("wimlib64.zip", "https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip") + @("wimlib32.zip", "https://wimlib.net/downloads/wimlib-1.13.0-windows-i686-bin.zip"), + @("wimlib64.zip", "https://wimlib.net/downloads/wimlib-1.13.0-windows-x86_64-bin.zip") ) foreach ($Tool in $ToolSources) { DownloadFile -Path $Temp -Name $Tool[0] -Url $Tool[1] diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 3b378aca..0456376d 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -282,8 +282,8 @@ LAUNCHERS = { 'Intel RST (Current Release)': { 'L_TYPE': 'Executable', 'L_PATH': '_Drivers\Intel RST', - 'L_ITEM': 'SetupRST_16.5.exe', - 'L_7ZIP': 'SetupRST_16.5.exe', + 'L_ITEM': 'SetupRST_16.8.exe', + 'L_7ZIP': 'SetupRST_16.8.exe', }, 'Intel RST (Previous Releases)': { 'L_TYPE': 'Folder', diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 9a57e8f7..2644d4b6 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -1,9 +1,9 @@ # Wizard Kit: Settings - Sources SOURCE_URLS = { - 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1801120058/AcroRdrDC1801120058_en_US.exe', + 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1901020069/AcroRdrDC1901020069_en_US.exe', 'AdwCleaner': 'https://downloads.malwarebytes.com/file/adwcleaner', - 'AIDA64': 'http://download.aida64.com/aida64engineer597.zip', + 'AIDA64': 'http://download.aida64.com/aida64engineer599.zip', 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0-win-32bit-build1.zip', 'Autoruns': 'https://download.sysinternals.com/files/Autoruns.zip', 'BleachBit': 'https://download.bleachbit.org/BleachBit-2.0-portable.zip', @@ -13,32 +13,32 @@ SOURCE_URLS = { 'ClassicStartSkin': 'http://www.classicshell.net/forum/download/file.php?id=3001&sid=9a195960d98fd754867dcb63d9315335', 'Du': 'https://download.sysinternals.com/files/DU.zip', 'ERUNT': 'http://www.aumha.org/downloads/erunt.zip', - 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.895.x86.zip', - 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.895.x64.zip', - 'FastCopy': 'http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip', - 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1056733/ublock_origin-1.16.20-an+fx.xpi', + 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.924.x86.zip', + 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.924.x64.zip', + 'FastCopy': 'http://ftp.vector.co.jp/70/93/2323/FastCopy361_installer.exe', + 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1166954/ublock_origin-1.17.4-an+fx.xpi', 'HitmanPro32': 'https://dl.surfright.nl/HitmanPro.exe', 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', - 'HWiNFO': 'http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip', - 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/27656/eng/Intel%20SSD%20Toolbox%20-%20v3.5.2.exe', + 'HWiNFO': 'https://www.fosshub.com/HWiNFO.html?dwl=hwi_600.zip', + 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/28447/eng/Intel%20SSD%20Toolbox%20-%20v3.5.8.exe', 'IOBit_Uninstaller': 'https://portableapps.duckduckgo.com/IObitUninstallerPortable_7.5.0.7.paf.exe', 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', 'Macs Fan Control': 'https://www.crystalidea.com/downloads/macsfancontrol_setup.exe', 'NirCmd32': 'https://www.nirsoft.net/utils/nircmd.zip', 'NirCmd64': 'https://www.nirsoft.net/utils/nircmd-x64.zip', - 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z', + 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z', 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_10810.33603.exe', 'ProduKey32': 'http://www.nirsoft.net/utils/produkey.zip', 'ProduKey64': 'http://www.nirsoft.net/utils/produkey-x64.zip', - 'PuTTY': 'https://the.earth.li/ sgtatham/putty/latest/w32/putty.zip', + 'PuTTY': 'https://the.earth.li/~sgtatham/putty/latest/w32/putty.zip', 'RKill': 'https://www.bleepingcomputer.com/download/rkill/dl/10/', - 'Samsung Magician': 'https://s3.ap-northeast-2.amazonaws.com/global.semi.static/SAMSUNG_SSD_v5_2_1_180523/CD0CFAC4675B9E502899B41BE00525C3909ECE3AD57CC1A2FB6B74A766B2A1EA/Samsung_Magician_Installer.zip', + 'Samsung Magician': 'https://s3.ap-northeast-2.amazonaws.com/global.semi.static/SAMSUNG_SSD_v5_3_0_181121/CD0C7CC1BE00525FAC4675B9E502899B41D5C3909ECE3AA2FB6B74A766B2A1EA/Samsung_Magician_Installer.zip', 'SDIO Themes': 'http://snappy-driver-installer.org/downloads/SDIO_Themes.zip', 'SDIO Torrent': 'http://snappy-driver-installer.org/downloads/SDIO_Update.torrent', 'TDSSKiller': 'https://media.kaspersky.com/utilities/VirusUtilities/EN/tdsskiller.exe', 'TestDisk': 'https://www.cgsecurity.org/testdisk-7.1-WIP.win.zip', - 'wimlib32': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip', - 'wimlib64': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip', + 'wimlib32': 'https://wimlib.net/downloads/wimlib-1.13.0-windows-i686-bin.zip', + 'wimlib64': 'https://wimlib.net/downloads/wimlib-1.13.0-windows-x86_64-bin.zip', 'Winapp2': 'https://github.com/MoscaDotTo/Winapp2/archive/master.zip', 'WizTree': 'https://antibody-software.com/files/wiztree_3_26_portable.zip', 'XMPlay 7z': 'https://support.xmplay.com/files/16/xmp-7z.zip?v=800962', @@ -172,7 +172,7 @@ NINITE_SOURCES = { 'Launchy.exe': 'launchy', 'RealVNC.exe': 'realvnc', 'Revo Uninstaller.exe': 'revo', - 'TeamViewer 13.exe': 'teamviewer13', + 'TeamViewer 14.exe': 'teamviewer14', 'TeraCopy.exe': 'teracopy', 'WinDirStat.exe': 'windirstat', }, @@ -191,12 +191,14 @@ RST_SOURCES = { 'SetupRST_14.0.exe': 'https://downloadmirror.intel.com/25091/eng/SetupRST.exe', 'SetupRST_14.8.exe': 'https://downloadmirror.intel.com/26759/eng/setuprst.exe', 'SetupRST_15.8.exe': 'https://downloadmirror.intel.com/27442/eng/SetupRST.exe', - 'SetupRST_15.9.exe': 'https://downloadmirror.intel.com/27400/eng/SetupRST.exe', - 'SetupRST_16.0.exe': 'https://downloadmirror.intel.com/27681/eng/SetupRST.exe', - 'SetupRST_16.5.exe': 'https://downloadmirror.intel.com/27984/eng/SetupRST.exe', + #SetupRST_15.9.exe : Deprecated by Intel + #SetupRST_16.0.exe : Deprecated by Intel + #SetupRST_16.5.exe : Deprecated by Intel + #SetupRST_16.7.exe : Deprecated by Intel + 'SetupRST_16.8.exe': 'https://downloadmirror.intel.com/28400/eng/SetupRST.exe', } if __name__ == '__main__': print("This file is not meant to be called directly.") -# vim: sts=2 sw=2 ts=2 +# vim: sts=2 sw=2 ts=2 tw=0 From 6488101cdc3ac21c724ca5f723452d6dc9833268 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 9 Jan 2019 16:29:18 -0700 Subject: [PATCH 174/186] Dumb workaround for Dell sensors --- .bin/Scripts/functions/sensors.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 65a1b93e..5d2f2fae 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -220,11 +220,15 @@ def update_sensor_data(sensor_data): for _section, _adapters in sensor_data.items(): for _adapter, _sources in _adapters.items(): for _source, _data in _sources.items(): - _label = _data['Label'] - _temp = json_data[_adapter][_source][_label] - _data['Current'] = _temp - _data['Max'] = max(_temp, _data['Max']) - _data['Temps'].append(_temp) + try: + _label = _data['Label'] + _temp = json_data[_adapter][_source][_label] + _data['Current'] = _temp + _data['Max'] = max(_temp, _data['Max']) + _data['Temps'].append(_temp) + except Exception: + # Dumb workound for Dell sensors with changing source names + pass def join_columns(column1, column2, width=55): From 969011f3f5575c3f77b0f6dc3b2fc5db7b0a4979 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 9 Jan 2019 20:55:48 -0700 Subject: [PATCH 175/186] Added new_system_setup() --- .bin/Scripts/functions/setup.py | 26 +++-- .bin/Scripts/functions/sw_diags.py | 27 +++++ .bin/Scripts/new_system_setup.py | 157 +++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 .bin/Scripts/new_system_setup.py diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index 85943d2e..f51bf40b 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -288,20 +288,32 @@ def install_firefox_extensions(): run_program(cmd) -def install_ninite_bundle(mse=False): - """Run Ninite file(s) based on OS version.""" +def install_ninite_bundle(mse=False, libreoffice=False): + """Run Ninite installer(s), returns list of Popen objects.""" + popen_objects = [] if global_vars['OS']['Version'] in ('8', '8.1', '10'): # Modern selection - popen_program(r'{BaseDir}\Installers\Extras\Bundles\Modern.exe'.format( - **global_vars)) + popen_objects.append( + popen_program(r'{BaseDir}\Installers\Extras\Bundles\Modern.exe'.format( + **global_vars))) else: # Legacy selection if mse: cmd = r'{BaseDir}\Installers\Extras\Security'.format(**global_vars) cmd += r'\Microsoft Security Essentials.exe' - popen_program(cmd) - popen_program(r'{BaseDir}\Installers\Extras\Bundles\Legacy.exe'.format( - **global_vars)) + popen_objects.append(popen_program(cmd)) + popen_objects.append( + popen_program(r'{BaseDir}\Installers\Extras\Bundles\Legacy.exe'.format( + **global_vars))) + + # LibreOffice + if libreoffice: + cmd = r'{BaseDir}\Installers\Extras\Office'.format(**global_vars) + cmd += r'\LibreOffice.exe' + popen_objects.append(popen_program(cmd)) + + # Done + return popen_objects def install_vcredists(): diff --git a/.bin/Scripts/functions/sw_diags.py b/.bin/Scripts/functions/sw_diags.py index 8faa25b7..100e5ad5 100644 --- a/.bin/Scripts/functions/sw_diags.py +++ b/.bin/Scripts/functions/sw_diags.py @@ -103,6 +103,33 @@ def get_boot_mode(): return type_str +def os_is_unsupported(show_alert=False): + """Checks if the current OS is unsupported, returns bool.""" + msg = '' + unsupported = False + + # Check OS version/notes + os_info = global_vars['OS'].copy() + if os_info['Notes'] == 'unsupported': + msg = 'The installed version of Windows is no longer supported' + unsupported = True + elif os_info['Notes'] == 'preview build': + msg = 'Preview builds are not officially supported' + unsupported = True + elif os_info['Version'] == '10' and os_info['Notes'] == 'outdated': + msg = 'The installed version of Windows is outdated' + unsupported = True + if 'Preview' not in msg: + msg += '\n\nPlease consider upgrading before continuing setup.' + + # Show alert + if unsupported and show_alert: + show_alert_box(msg) + + # Done + return unsupported + + def run_autoruns(): """Run AutoRuns in the background with VirusTotal checks enabled.""" extract_item('Autoruns', filter='autoruns*', silent=True) diff --git a/.bin/Scripts/new_system_setup.py b/.bin/Scripts/new_system_setup.py new file mode 100644 index 00000000..6ab88973 --- /dev/null +++ b/.bin/Scripts/new_system_setup.py @@ -0,0 +1,157 @@ +# Wizard Kit: New system setup + +import os +import sys + +# Init +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) +from functions.activation import * +from functions.browsers import * +from functions.cleanup import * +from functions.info import * +from functions.product_keys import * +from functions.setup import * +from functions.sw_diags import * +init_global_vars() +os.system('title {}: New System Setup'.format(KIT_NAME_FULL)) +set_log_file('New System Setup.log') + +if __name__ == '__main__': + other_results = { + 'Error': { + 'BIOSKeyNotFoundError': 'BIOS key not found', + 'CalledProcessError': 'Unknown Error', + 'FileNotFoundError': 'File not found', + 'GenericError': 'Unknown Error', + 'SecureBootDisabledError': 'Disabled', + }, + 'Warning': { + 'GenericRepair': 'Repaired', + 'NoProfilesError': 'No profiles found', + 'NotInstalledError': 'Not installed', + 'OSInstalledLegacyError': 'OS installed Legacy', + 'SecureBootNotAvailError': 'Not available', + 'SecureBootUnknownError': 'Unknown', + 'UnsupportedOSError': 'Unsupported OS', + }} + try: + stay_awake() + clear_screen() + + # Check installed OS + if os_is_unsupported(show_alert=False): + print_warning('OS version not supported by this script') + if not ask('Continue anyway? (NOT RECOMMENDED)'): + abort() + + # Install Adobe Reader? + answer_adobe_reader = ask('Install Adobe Reader?') + + # Install LibreOffice? + answer_libreoffice = ask('Install LibreOffice?') + + # Install MSE? + if global_vars['OS']['Version'] == '7': + answer_mse = ask('Install MSE?') + else: + answer_mse = False + + # Install software + print_info('Installing Programs') + install_vcredists() + if answer_adobe_reader: + try_and_print(message='Adobe Reader DC...', + function=install_adobe_reader, other_results=other_results) + result = try_and_print( + message='Ninite bundle...', + function=install_ninite_bundle, cs='Started', + mse=answer_mse, libreoffice=answer_libreoffice, + other_results=other_results) + for proc in result['Out']: + # Wait for all processes to finish + proc.wait() + + # Scan for supported browsers + print_info('Scanning for browsers') + scan_for_browsers() + + # Install extensions + print_info('Installing Extensions') + try_and_print(message='Classic Shell skin...', + function=install_classicstart_skin, + other_results=other_results) + try_and_print(message='Google Chrome extensions...', + function=install_chrome_extensions) + try_and_print(message='Mozilla Firefox extensions...', + function=install_firefox_extensions, + other_results=other_results) + + # Configure software + print_info('Configuring programs') + install_adblock() + if global_vars['OS']['Version'] == '10': + try_and_print(message='ClassicStart...', + function=config_classicstart, cs='Done') + try_and_print(message='Explorer...', + function=config_explorer_user, cs='Done') + + # Configure system + print_info('Configuring system') + if global_vars['OS']['Version'] == '10': + try_and_print(message='Explorer...', + function=config_explorer_system, cs='Done') + try_and_print(message='Disabling telemetry...', + function=disable_windows_telemetry, cs='Done') + try_and_print(message='Updating Clock...', + function=update_clock, cs='Done') + + # Summary + print_info('Summary') + try_and_print(message='Operating System:', + function=show_os_name, ns='Unknown', silent_function=False) + try_and_print(message='Activation:', + function=show_os_activation, ns='Unknown', silent_function=False) + if (not windows_is_activated() + and global_vars['OS']['Version'] in ('8', '8.1', '10')): + try_and_print(message='BIOS Activation:', + function=activate_with_bios, + other_results=other_results) + try_and_print(message='Secure Boot Status:', + function=check_secure_boot_status, other_results=other_results) + try_and_print(message='Installed RAM:', + function=show_installed_ram, ns='Unknown', silent_function=False) + show_free_space() + try_and_print(message='Installed Antivirus:', + function=get_installed_antivirus, ns='Unknown', + other_results=other_results, print_return=True) + + # Play audio, show devices, open Windows updates, and open Activation + try_and_print(message='Opening Device Manager...', + function=open_device_manager, cs='Started') + try_and_print(message='Opening HWiNFO (Sensors)...', + function=run_hwinfo_sensors, cs='Started', other_results=other_results) + try_and_print(message='Opening Windows Updates...', + function=open_windows_updates, cs='Started') + if not windows_is_activated(): + try_and_print(message='Opening Windows Activation...', + function=open_windows_activation, cs='Started') + sleep(3) + try_and_print(message='Running XMPlay...', + function=run_xmplay, cs='Started', other_results=other_results) + try: + check_secure_boot_status(show_alert=True) + except: + # Only trying to open alert message boxes + pass + + # Done + print_standard('\nDone.') + pause('Press Enter to exit...') + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=2 sw=2 ts=2 From c1324548ce1c058c82419467c9449af046d26579 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 9 Jan 2019 20:59:21 -0700 Subject: [PATCH 176/186] Track ninite processes directly --- .bin/Scripts/install_sw_bundle.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/install_sw_bundle.py b/.bin/Scripts/install_sw_bundle.py index b44874ae..0faec26e 100644 --- a/.bin/Scripts/install_sw_bundle.py +++ b/.bin/Scripts/install_sw_bundle.py @@ -41,11 +41,13 @@ if __name__ == '__main__': if answer_vcr: install_vcredists() if answer_ninite: - try_and_print(message='Ninite bundle...', + result = try_and_print(message='Ninite bundle...', function=install_ninite_bundle, cs='Started', mse=answer_mse, other_results=other_results) + for proc in result['Out']: + # Wait for all processes to finish + proc.wait() if answer_extensions: - wait_for_process('ninite.exe') print_info('Installing Extensions') try_and_print(message='Classic Shell skin...', function=install_classicstart_skin, From 624639389445edcc042410274cd7a0bdec6378a1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 11 Jan 2019 12:31:08 -0700 Subject: [PATCH 177/186] Fixed update_fastcopy() --- .bin/Scripts/functions/update.py | 42 +++++++++++++++----------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index a9c7811f..2331a604 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -250,35 +250,33 @@ def update_fastcopy(): # Remove existing folders remove_from_kit('FastCopy') - # Download - download_to_temp('FastCopy.zip', SOURCE_URLS['FastCopy']) - - # Extract installer - extract_temp_to_bin('FastCopy.zip', 'FastCopy') - _path = r'{}\FastCopy'.format(global_vars['BinDir']) - _installer = 'FastCopy354_installer.exe' - - # Extract 64-bit - cmd = [ - r'{}\{}'.format(_path, _installer), - '/NOSUBDIR', '/DIR={}'.format(_path), - '/EXTRACT64'] - run_program(cmd) - shutil.move( - r'{}\FastCopy\FastCopy.exe'.format(global_vars['BinDir']), - r'{}\FastCopy\FastCopy64.exe'.format(global_vars['BinDir'])) + # Download installer + download_to_temp('FastCopy.exe', SOURCE_URLS['FastCopy']) + _installer = r'{}\FastCopy.exe'.format(global_vars['TmpDir']) # Extract 32-bit + _path32 = r'{}\FastCopy'.format(global_vars['BinDir']) cmd = [ - r'{}\{}'.format(_path, _installer), - '/NOSUBDIR', '/DIR={}'.format(_path), + _installer, + '/NOSUBDIR', '/DIR={}'.format(_path32), '/EXTRACT32'] run_program(cmd) + # Extract 64-bit + _path64 = r'{}\FastCopyTmp'.format(global_vars['TmpDir']) + cmd = [ + _installer, + '/NOSUBDIR', '/DIR={}'.format(_path64), + '/EXTRACT64'] + run_program(cmd) + shutil.move( + r'{}\FastCopy.exe'.format(_path64), + r'{}\FastCopy64.exe'.format(_path32)) + # Cleanup - os.remove(r'{}\{}'.format(_path, _installer)) - os.remove(r'{}\setup.exe'.format(_path, _installer)) - remove_from_temp('FastCopy.zip') + remove_item(r'{}\setup.exe'.format(_path32)) + remove_item(_path64) + remove_from_temp('FastCopy.exe') def update_wimlib(): From ad1d7d71f2ae2d9dec059199d6780ad71dcbd688 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 11 Jan 2019 12:59:12 -0700 Subject: [PATCH 178/186] Fixed ODT sections * Fixes issue #86 --- .bin/Scripts/Launch.cmd | 3 +- .bin/Scripts/functions/update.py | 31 +++++++------- .bin/Scripts/settings/launchers.py | 40 +++++++++++++++---- .bin/Scripts/settings/sources.py | 2 +- .../_Office/{hb_32.xml => 2016_hb_32.xml} | 0 .../_Office/{hb_64.xml => 2016_hb_64.xml} | 0 .../_Office/{hs_32.xml => 2016_hs_32.xml} | 0 .../_Office/{hs_64.xml => 2016_hs_64.xml} | 0 .cbin/_include/_Office/2019_hb_32.xml | 7 ++++ .cbin/_include/_Office/2019_hb_64.xml | 7 ++++ .cbin/_include/_Office/2019_hs_32.xml | 7 ++++ .cbin/_include/_Office/2019_hs_64.xml | 7 ++++ 12 files changed, 78 insertions(+), 26 deletions(-) rename .cbin/_include/_Office/{hb_32.xml => 2016_hb_32.xml} (100%) rename .cbin/_include/_Office/{hb_64.xml => 2016_hb_64.xml} (100%) rename .cbin/_include/_Office/{hs_32.xml => 2016_hs_32.xml} (100%) rename .cbin/_include/_Office/{hs_64.xml => 2016_hs_64.xml} (100%) create mode 100644 .cbin/_include/_Office/2019_hb_32.xml create mode 100644 .cbin/_include/_Office/2019_hb_64.xml create mode 100644 .cbin/_include/_Office/2019_hs_32.xml create mode 100644 .cbin/_include/_Office/2019_hs_64.xml diff --git a/.bin/Scripts/Launch.cmd b/.bin/Scripts/Launch.cmd index 0a0330c6..3a3ed6a5 100644 --- a/.bin/Scripts/Launch.cmd +++ b/.bin/Scripts/Launch.cmd @@ -151,6 +151,7 @@ goto Exit call "%bin%\Scripts\init_client_dir.cmd" /Office set "_odt=False" if %L_PATH% equ 2016 (set "_odt=True") +if %L_PATH% equ 2019 (set "_odt=True") if "%_odt%" == "True" ( goto LaunchOfficeODT ) else ( @@ -161,7 +162,7 @@ if "%_odt%" == "True" ( rem Prep set "args=-aoa -bso0 -bse0 -bsp0 -p%ARCHIVE_PASSWORD%" set "config=%L_ITEM%" -set "dest=%client_dir%\Office\%L_PATH%" +set "dest=%client_dir%\Office\ODT" set "odt_exe=%L_PATH%\setup.exe" set "source=%cbin%\_Office.7z" diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 2331a604..e1cd277d 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -656,23 +656,22 @@ def update_office(): if os.path.exists(include_path): shutil.copytree(include_path, dest) - for year in ['2016']: - # Download and extract - name = 'odt{}.exe'.format(year) - url = 'Office Deployment Tool {}'.format(year) - download_to_temp(name, SOURCE_URLS[url]) - cmd = [ - r'{}\odt{}.exe'.format(global_vars['TmpDir'], year), - r'/extract:{}\{}'.format(global_vars['TmpDir'], year), - '/quiet', - ] - run_program(cmd) - shutil.move( - r'{}\{}'.format(global_vars['TmpDir'], year), - r'{}\_Office\{}'.format(global_vars['CBinDir'], year)) + # Download and extract + _odt = 'odt.exe' + _out_path = r'{}\odt' + download_to_temp('odt.exe', SOURCE_URLS['Office Deployment Tool']) + cmd = [ + odt, + r'/extract:{}\odt'.format(global_vars['TmpDir']), + '/quiet', + ] + run_program(cmd) + shutil.move( + _out_path, + r'{}\_Office'.format(global_vars['CBinDir'])) - # Cleanup - remove_from_temp('odt{}.exe'.format(year)) + # Cleanup + remove_from_temp(odt.exe) def update_classic_start_skin(): diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 0456376d..73dc14d1 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -360,36 +360,60 @@ LAUNCHERS = { 'Home and Business 2016 (x32)': { 'L_TYPE': 'Office', 'L_PATH': '2016', - 'L_ITEM': 'hb_32.xml', + 'L_ITEM': '2016_hb_32.xml', 'L_NCMD': 'True', }, 'Home and Business 2016 (x64)': { 'L_TYPE': 'Office', 'L_PATH': '2016', - 'L_ITEM': 'hb_64.xml', + 'L_ITEM': '2016_hb_64.xml', 'L_NCMD': 'True', }, 'Home and Student 2016 (x32)': { 'L_TYPE': 'Office', 'L_PATH': '2016', - 'L_ITEM': 'hs_32.xml', + 'L_ITEM': '2016_hs_32.xml', 'L_NCMD': 'True', }, 'Home and Student 2016 (x64)': { 'L_TYPE': 'Office', 'L_PATH': '2016', - 'L_ITEM': 'hs_64.xml', + 'L_ITEM': '2016_hs_64.xml', 'L_NCMD': 'True', }, - 'Office 365 2016 (x32)': { + 'Home and Business 2019 (x32)': { 'L_TYPE': 'Office', - 'L_PATH': '2016', + 'L_PATH': '2019', + 'L_ITEM': '2019_hb_32.xml', + 'L_NCMD': 'True', + }, + 'Home and Business 2019 (x64)': { + 'L_TYPE': 'Office', + 'L_PATH': '2019', + 'L_ITEM': '2019_hb_64.xml', + 'L_NCMD': 'True', + }, + 'Home and Student 2019 (x32)': { + 'L_TYPE': 'Office', + 'L_PATH': '2019', + 'L_ITEM': '2019_hs_32.xml', + 'L_NCMD': 'True', + }, + 'Home and Student 2019 (x64)': { + 'L_TYPE': 'Office', + 'L_PATH': '2019', + 'L_ITEM': '2019_hs_64.xml', + 'L_NCMD': 'True', + }, + 'Office 365 2019 (x32)': { + 'L_TYPE': 'Office', + 'L_PATH': '2019', 'L_ITEM': '365_32.xml', 'L_NCMD': 'True', }, - 'Office 365 2016 (x64)': { + 'Office 365 2019 (x64)': { 'L_TYPE': 'Office', - 'L_PATH': '2016', + 'L_PATH': '2019', 'L_ITEM': '365_64.xml', 'L_NCMD': 'True', }, diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 2644d4b6..236ba1a8 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -27,7 +27,7 @@ SOURCE_URLS = { 'NirCmd32': 'https://www.nirsoft.net/utils/nircmd.zip', 'NirCmd64': 'https://www.nirsoft.net/utils/nircmd-x64.zip', 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z', - 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_10810.33603.exe', + 'Office Deployment Tool': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_11107-33602.exe', 'ProduKey32': 'http://www.nirsoft.net/utils/produkey.zip', 'ProduKey64': 'http://www.nirsoft.net/utils/produkey-x64.zip', 'PuTTY': 'https://the.earth.li/~sgtatham/putty/latest/w32/putty.zip', diff --git a/.cbin/_include/_Office/hb_32.xml b/.cbin/_include/_Office/2016_hb_32.xml similarity index 100% rename from .cbin/_include/_Office/hb_32.xml rename to .cbin/_include/_Office/2016_hb_32.xml diff --git a/.cbin/_include/_Office/hb_64.xml b/.cbin/_include/_Office/2016_hb_64.xml similarity index 100% rename from .cbin/_include/_Office/hb_64.xml rename to .cbin/_include/_Office/2016_hb_64.xml diff --git a/.cbin/_include/_Office/hs_32.xml b/.cbin/_include/_Office/2016_hs_32.xml similarity index 100% rename from .cbin/_include/_Office/hs_32.xml rename to .cbin/_include/_Office/2016_hs_32.xml diff --git a/.cbin/_include/_Office/hs_64.xml b/.cbin/_include/_Office/2016_hs_64.xml similarity index 100% rename from .cbin/_include/_Office/hs_64.xml rename to .cbin/_include/_Office/2016_hs_64.xml diff --git a/.cbin/_include/_Office/2019_hb_32.xml b/.cbin/_include/_Office/2019_hb_32.xml new file mode 100644 index 00000000..d98dd66a --- /dev/null +++ b/.cbin/_include/_Office/2019_hb_32.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/.cbin/_include/_Office/2019_hb_64.xml b/.cbin/_include/_Office/2019_hb_64.xml new file mode 100644 index 00000000..29e10c4f --- /dev/null +++ b/.cbin/_include/_Office/2019_hb_64.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/.cbin/_include/_Office/2019_hs_32.xml b/.cbin/_include/_Office/2019_hs_32.xml new file mode 100644 index 00000000..3fc1606a --- /dev/null +++ b/.cbin/_include/_Office/2019_hs_32.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/.cbin/_include/_Office/2019_hs_64.xml b/.cbin/_include/_Office/2019_hs_64.xml new file mode 100644 index 00000000..3d2956fd --- /dev/null +++ b/.cbin/_include/_Office/2019_hs_64.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file From 4a96736592fadc6e28eaf4cc9886dd495b7c0fbc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 11 Jan 2019 13:17:25 -0700 Subject: [PATCH 179/186] Removed network installers --- .bin/Scripts/functions/update.py | 31 ------------------------------- .bin/Scripts/update_kit.py | 5 ----- 2 files changed, 36 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index e1cd277d..c703bf71 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -182,37 +182,6 @@ def resolve_dynamic_url(source_url, regex): return url -def scan_for_net_installers(server, family_name, min_year): - """Scan network shares for installers.""" - if not server['Mounted']: - mount_network_share(server) - - if server['Mounted']: - for year in os.scandir(r'\\{IP}\{Share}'.format(**server)): - try: - year_ok = int(year.name) < min_year - except ValueError: - year_ok = False # Skip non-year items - if year_ok: - # Don't support outdated installers - continue - for version in os.scandir(year.path): - section = r'Installers\Extras\{}\{}'.format( - family_name, year.name) - if section not in LAUNCHERS: - LAUNCHERS[section] = {} - name = version.name - if re.search(r'(exe|msi)$', name, re.IGNORECASE): - name = name[:-4] - if name not in LAUNCHERS[section]: - LAUNCHERS[section][name] = { - 'L_TYPE': family_name, - 'L_PATH': year.name, - 'L_ITEM': version.name, - } - umount_network_share(server) - - # Data Recovery def update_testdisk(): # Stop running processes diff --git a/.bin/Scripts/update_kit.py b/.bin/Scripts/update_kit.py index 3bc31a72..18bbb69e 100644 --- a/.bin/Scripts/update_kit.py +++ b/.bin/Scripts/update_kit.py @@ -110,11 +110,6 @@ if __name__ == '__main__': width=40, item = item) - ## Search for network Office/QuickBooks installers & add to LAUNCHERS - print_success('Scanning for network installers') - scan_for_net_installers(OFFICE_SERVER, 'Office', min_year=2010) - scan_for_net_installers(QUICKBOOKS_SERVER, 'QuickBooks', min_year=2015) - ## Generate Launchers print_success('Generating launchers') for section in sorted(LAUNCHERS.keys()): From cb072366e0659ca2d7339ab1eee978654ca8852b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 11 Jan 2019 13:25:33 -0700 Subject: [PATCH 180/186] Bugfix Office sections * Addresses issue #86 --- .bin/Scripts/Launch.cmd | 2 +- .bin/Scripts/functions/update.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/Launch.cmd b/.bin/Scripts/Launch.cmd index 3a3ed6a5..18c304f2 100644 --- a/.bin/Scripts/Launch.cmd +++ b/.bin/Scripts/Launch.cmd @@ -163,7 +163,7 @@ rem Prep set "args=-aoa -bso0 -bse0 -bsp0 -p%ARCHIVE_PASSWORD%" set "config=%L_ITEM%" set "dest=%client_dir%\Office\ODT" -set "odt_exe=%L_PATH%\setup.exe" +set "odt_exe=setup.exe" set "source=%cbin%\_Office.7z" rem Extract diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index c703bf71..07fe6e8f 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -626,21 +626,21 @@ def update_office(): shutil.copytree(include_path, dest) # Download and extract - _odt = 'odt.exe' - _out_path = r'{}\odt' + _out_path = r'{}\odt'.format(global_vars['TmpDir']) download_to_temp('odt.exe', SOURCE_URLS['Office Deployment Tool']) cmd = [ - odt, - r'/extract:{}\odt'.format(global_vars['TmpDir']), + r'{}\odt.exe'.format(global_vars['TmpDir']), + r'/extract:{}\odt'.format(_out_path), '/quiet', ] run_program(cmd) shutil.move( - _out_path, + r'{}\setup.exe'.format(_out_path), r'{}\_Office'.format(global_vars['CBinDir'])) # Cleanup - remove_from_temp(odt.exe) + remove_from_temp('odt') + remove_from_temp('odt.exe') def update_classic_start_skin(): From fd8358a8995df02e449f2b702cf296fca18cb4b8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 11 Jan 2019 14:17:41 -0700 Subject: [PATCH 181/186] Fix ODT path --- .bin/Scripts/functions/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 07fe6e8f..adb26c6d 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -630,7 +630,7 @@ def update_office(): download_to_temp('odt.exe', SOURCE_URLS['Office Deployment Tool']) cmd = [ r'{}\odt.exe'.format(global_vars['TmpDir']), - r'/extract:{}\odt'.format(_out_path), + r'/extract:{}'.format(_out_path), '/quiet', ] run_program(cmd) From 6c47650c2dc1eb97c1e46d36b7295af1cd82fb9e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 11 Jan 2019 14:18:01 -0700 Subject: [PATCH 182/186] Updated IOBit Uninstaller --- .bin/Scripts/settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 236ba1a8..2dbd8a91 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -21,7 +21,7 @@ SOURCE_URLS = { 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', 'HWiNFO': 'https://www.fosshub.com/HWiNFO.html?dwl=hwi_600.zip', 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/28447/eng/Intel%20SSD%20Toolbox%20-%20v3.5.8.exe', - 'IOBit_Uninstaller': 'https://portableapps.duckduckgo.com/IObitUninstallerPortable_7.5.0.7.paf.exe', + 'IOBit_Uninstaller': r'https://portableapps.com/redirect/?a=IObitUninstallerPortable&s=s&d=pa&f=IObitUninstallerPortable_7.5.0.7.paf.exe', 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', 'Macs Fan Control': 'https://www.crystalidea.com/downloads/macsfancontrol_setup.exe', 'NirCmd32': 'https://www.nirsoft.net/utils/nircmd.zip', From bece9837b245580b8da7988737098c41be87679a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 11 Jan 2019 14:42:18 -0700 Subject: [PATCH 183/186] Moved Office 2019 installers to their own folder --- .bin/Scripts/settings/launchers.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 73dc14d1..2e88f058 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -357,61 +357,63 @@ LAUNCHERS = { }, }, r'Installers\Extras\Office\2016': { - 'Home and Business 2016 (x32)': { + 'Home and Business (x32)': { 'L_TYPE': 'Office', 'L_PATH': '2016', 'L_ITEM': '2016_hb_32.xml', 'L_NCMD': 'True', }, - 'Home and Business 2016 (x64)': { + 'Home and Business (x64)': { 'L_TYPE': 'Office', 'L_PATH': '2016', 'L_ITEM': '2016_hb_64.xml', 'L_NCMD': 'True', }, - 'Home and Student 2016 (x32)': { + 'Home and Student (x32)': { 'L_TYPE': 'Office', 'L_PATH': '2016', 'L_ITEM': '2016_hs_32.xml', 'L_NCMD': 'True', }, - 'Home and Student 2016 (x64)': { + 'Home and Student (x64)': { 'L_TYPE': 'Office', 'L_PATH': '2016', 'L_ITEM': '2016_hs_64.xml', 'L_NCMD': 'True', }, - 'Home and Business 2019 (x32)': { + }, + r'Installers\Extras\Office\2019': { + 'Home and Business (x32)': { 'L_TYPE': 'Office', 'L_PATH': '2019', 'L_ITEM': '2019_hb_32.xml', 'L_NCMD': 'True', }, - 'Home and Business 2019 (x64)': { + 'Home and Business (x64)': { 'L_TYPE': 'Office', 'L_PATH': '2019', 'L_ITEM': '2019_hb_64.xml', 'L_NCMD': 'True', }, - 'Home and Student 2019 (x32)': { + 'Home and Student (x32)': { 'L_TYPE': 'Office', 'L_PATH': '2019', 'L_ITEM': '2019_hs_32.xml', 'L_NCMD': 'True', }, - 'Home and Student 2019 (x64)': { + 'Home and Student (x64)': { 'L_TYPE': 'Office', 'L_PATH': '2019', 'L_ITEM': '2019_hs_64.xml', 'L_NCMD': 'True', }, - 'Office 365 2019 (x32)': { + 'Office 365 (x32)': { 'L_TYPE': 'Office', 'L_PATH': '2019', 'L_ITEM': '365_32.xml', 'L_NCMD': 'True', }, - 'Office 365 2019 (x64)': { + 'Office 365 (x64)': { 'L_TYPE': 'Office', 'L_PATH': '2019', 'L_ITEM': '365_64.xml', From 5b9e91f8affc99ecf749b10fa0ecd3871f256593 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 13 Jan 2019 14:51:53 -0700 Subject: [PATCH 184/186] Updated Killer Network drivers source --- .bin/Scripts/build_pe.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index a2dd0363..00405a12 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -145,7 +145,7 @@ if ($MyInvocation.InvocationName -ne ".") { # Killer Network Drivers @( "killerinf.zip", - ("http://www.killernetworking.com"+(FindDynamicUrl "http://www.killernetworking.com/driver-downloads/item/killer-drivers-inf" "Download Killer-Ethernet").replace('&', '&')) + ("https://www.killernetworking.com"+(FindDynamicUrl "https://www.killernetworking.com/killersupport/category/other-downloads" "Download Killer-Ethernet").replace('&', '&')) ), # Notepad++ @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z"), From fcd8d67f51c5e4c31eb5f2ace45037e4010b4800 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 13 Jan 2019 14:52:31 -0700 Subject: [PATCH 185/186] Bumped year in LICENSE.txt --- LICENSE.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index d02f5a6e..f29c29e6 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,7 @@ -Copyright (c) 2018 Alan Mason +Copyright (c) 2019 Alan Mason Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 814ada0ac044750dec3f414abfc2f68de22d1f31 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 13 Jan 2019 17:59:06 -0700 Subject: [PATCH 186/186] Updated HWiNFO url --- .bin/Scripts/settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 2dbd8a91..8a35895e 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -19,7 +19,7 @@ SOURCE_URLS = { 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1166954/ublock_origin-1.17.4-an+fx.xpi', 'HitmanPro32': 'https://dl.surfright.nl/HitmanPro.exe', 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', - 'HWiNFO': 'https://www.fosshub.com/HWiNFO.html?dwl=hwi_600.zip', + 'HWiNFO': 'http://files2.majorgeeks.com/caae8849cf31a8d77c51283b720e60e49ce1dc78/systeminfo/hwi_600.zip', 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/28447/eng/Intel%20SSD%20Toolbox%20-%20v3.5.8.exe', 'IOBit_Uninstaller': r'https://portableapps.com/redirect/?a=IObitUninstallerPortable&s=s&d=pa&f=IObitUninstallerPortable_7.5.0.7.paf.exe', 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe',