From bbfcc2e3fe70efcc715f6b786663af2a5d9efb6e Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Wed, 30 May 2018 17:26:49 -0600 Subject: [PATCH 001/138] Hotfix: Handle size=None in human_readable_size() --- .bin/Scripts/functions/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index f5a2ef95..f677df45 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -234,6 +234,8 @@ def human_readable_size(size, decimals=0): size = int(size) except ValueError: size = convert_to_bytes(size) + except TypeError: + size = -1 # Verify we have a valid size if size < 0: From afef5e905273308effcd242587de09200499a9b0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 14 Jul 2018 19:41:52 -0600 Subject: [PATCH 002/138] Add safe alias for ddrescue --- .linux_items/include/airootfs/etc/skel/.aliases | 1 + 1 file changed, 1 insertion(+) diff --git a/.linux_items/include/airootfs/etc/skel/.aliases b/.linux_items/include/airootfs/etc/skel/.aliases index d20b59b3..fab61bb1 100644 --- a/.linux_items/include/airootfs/etc/skel/.aliases +++ b/.linux_items/include/airootfs/etc/skel/.aliases @@ -4,6 +4,7 @@ alias 7z3='7z a -t7z -mx=3' alias 7z5='7z a -t7z -mx=5' alias 7z7='7z a -t7z -mx=7' alias 7z9='7z a -t7z -mx=9' +alias ddrescue='sudo ddrescue --ask --min-read-rate=64k -vvvv' alias diff='colordiff -ur' alias du='du -sch --apparent-size' alias fix-perms='find -type d -exec chmod 755 "{}" \; && find -type f -exec chmod 644 "{}" \;' From d57b08ec6f8f5d7b267b547e8de4457061698578 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 14 Jul 2018 21:17:32 -0600 Subject: [PATCH 003/138] Update hw_diags so it can be used by wk-ddrescue --- .bin/Scripts/functions/hw_diags.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 63942ed2..1b61d8ca 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -588,19 +588,21 @@ def scan_disks(): TESTS['NVMe/SMART']['Devices'] = devs TESTS['badblocks']['Devices'] = devs TESTS['iobenchmark']['Devices'] = devs + return devs -def show_disk_details(dev): +def show_disk_details(dev, only_attributes=False): """Display disk details.""" dev_name = dev['lsblk']['name'] - # Device description - print_info('Device: /dev/{}'.format(dev['lsblk']['name'])) - print_standard(' {:>4} ({}) {} {}'.format( - str(dev['lsblk'].get('size', '???b')).strip(), - str(dev['lsblk'].get('tran', '???')).strip().upper().replace( - 'NVME', 'NVMe'), - str(dev['lsblk'].get('model', 'Unknown Model')).strip(), - str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(), - )) + if not only_attributes: + # Device description + print_info('Device: /dev/{}'.format(dev['lsblk']['name'])) + print_standard(' {:>4} ({}) {} {}'.format( + str(dev['lsblk'].get('size', '???b')).strip(), + str(dev['lsblk'].get('tran', '???')).strip().upper().replace( + 'NVME', 'NVMe'), + str(dev['lsblk'].get('model', 'Unknown Model')).strip(), + str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(), + )) # Warnings if dev.get('NVMe Disk', False): From 8b1e19fa4b64c26080cb47e773e8d236d5fb6ea8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 14 Jul 2018 21:19:08 -0600 Subject: [PATCH 004/138] Initial wk-ddrescue menu --- .bin/Scripts/functions/ddrescue.py | 58 ++++++++++++++++++++++++++++++ .bin/Scripts/wk-ddrescue | 32 +++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 .bin/Scripts/functions/ddrescue.py create mode 100755 .bin/Scripts/wk-ddrescue diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py new file mode 100644 index 00000000..00514fe8 --- /dev/null +++ b/.bin/Scripts/functions/ddrescue.py @@ -0,0 +1,58 @@ +# Wizard Kit: Functions - ddrescue + +import json +import re + +from functions.common import * + +# STATIC VARIABLES +USAGE=""" {script_name} clone [source [destination]] + {script_name} image [source [destination]] + (e.g. {script_name} clone /dev/sda /dev/sdb) +""" + +# Functions +def menu_clone(source_path, dest_path): + print_success('GNU ddrescue: Cloning Menu') + pass + +def menu_ddrescue(*args): + """Main ddrescue loop/menu.""" + args = list(args) + script_name = os.path.basename(args.pop(0)) + run_mode = '' + source_path = None + dest_path = None + + # Parse args + try: + run_mode = args.pop(0) + source_path = args.pop(0) + dest_path = args.pop(0) + except IndexError: + # We'll set the missing paths later + pass + + # Show proper menu or exit + if run_mode == 'clone': + menu_clone(source_path, dest_path) + elif run_mode == 'image': + menu_image(source_path, dest_path) + else: + if not re.search(r'(^$|help|-h|\?)', run_mode, re.IGNORECASE): + print_error('Invalid mode.') + show_usage(script_name) + exit_script() + +def menu_image(source_path, dest_path): + print_success('GNU ddrescue: Imaging Menu') + pass + +def show_usage(script_name): + print_info('Usage:') + print_standard(USAGE.format(script_name=script_name)) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/wk-ddrescue b/.bin/Scripts/wk-ddrescue new file mode 100755 index 00000000..28ce6fb7 --- /dev/null +++ b/.bin/Scripts/wk-ddrescue @@ -0,0 +1,32 @@ +#!/bin/python3 +# +## Wizard Kit: TUI for ddrescue cloning and imaging + +import os +import sys + +# Init +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) +from functions.ddrescue import * +from functions.hw_diags import * +init_global_vars() + +if __name__ == '__main__': + try: + # Prep + clear_screen() + + # Show menu + menu_ddrescue(*sys.argv) + + # Done + #print_standard('\nDone.') + #pause("Press Enter to exit...") + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=4 sw=4 ts=4 From 667223b3c2a96db74d0868ecdc3af615a12ecb77 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 14 Jul 2018 22:52:18 -0600 Subject: [PATCH 005/138] Check passed source * If it's an image, setup loopback dev * If it's a child block dev, prompt with option to use parent block * Show selected source details --- .bin/Scripts/functions/ddrescue.py | 94 +++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 00514fe8..1acc5e96 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -1,6 +1,7 @@ # Wizard Kit: Functions - ddrescue import json +import pathlib import re from functions.common import * @@ -12,9 +13,99 @@ USAGE=""" {script_name} clone [source [destination]] """ # Functions +def show_device_details(dev_path): + """Display device details on screen.""" + cmd = ( + 'lsblk', '--nodeps', + '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', + dev_path) + result = run_program(cmd) + output = result.stdout.decode().splitlines() + print_info(output.pop(0)) + for line in output: + print_standard(line) + + # Children info + cmd = ('lsblk', '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', dev_path) + result = run_program(cmd) + output = result.stdout.decode().splitlines() + print_info(output.pop(0)) + for line in output: + print_standard(line) + +def get_device_details(dev_path): + """Get device details via lsblk, returns JSON dict.""" + try: + cmd = ( + 'lsblk', + '--json', + '--output-all', + '--paths', + dev_path) + result = run_program(cmd) + except CalledProcessError: + print_error('Failed to get device details for {}'.format(dev_path)) + abort() + + json_data = json.loads(result.stdout.decode()) + # Just return the first device (there should only be one) + return json_data['blockdevices'][0] + +def setup_loopback_device(source_path): + """Setup a loopback device for source_path, returns dev_path as str.""" + cmd = ( + 'losetup', + '--find', + '--partscan', + '--show', + source_path) + try: + out = run_program(cmd, check=True) + dev_path = out.stdout.decode().strip() + except CalledProcessError: + print_error('Failed to setup loopback device for source.') + abort() + else: + return dev_path + def menu_clone(source_path, dest_path): + """ddrescue cloning menu.""" + source_is_image = False + source_dev_path = None print_success('GNU ddrescue: Cloning Menu') - pass + + # Select source if not preselected + if not source_path: + #TODO menu_select drive + print_warning('Select drive not implemented yet.') + source_path = '' + + # Check source + source_path = os.path.realpath(source_path) + if pathlib.Path(source_path).is_block_device(): + source_dev_path = source_path + elif pathlib.Path(source_path).is_file(): + source_dev_path = setup_loopback_device(source_path) + source_is_image = True + else: + print_error('Invalid source "{}".'.format(source_path)) + abort() + source_details = get_device_details(source_dev_path) + + # Check source type + if source_details['pkname']: + print_warning('Source "{}" is a child device.'.format(source_dev_path)) + if ask('Use parent device "{}" instead?'.format( + source_details['pkname'])): + source_dev_path = source_details['pkname'] + source_details = get_device_details(source_dev_path) + + # Show selection details + if source_is_image: + print_success('Using image file: {}'.format(source_path)) + print_success(' (via loopback device: {})'.format( + source_dev_path)) + show_device_details(source_dev_path) def menu_ddrescue(*args): """Main ddrescue loop/menu.""" @@ -45,6 +136,7 @@ def menu_ddrescue(*args): exit_script() def menu_image(source_path, dest_path): + """ddrescue imaging menu.""" print_success('GNU ddrescue: Imaging Menu') pass From cfbd0ec8f2bf47d263230d0193875cfe54094deb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 14 Jul 2018 23:45:08 -0600 Subject: [PATCH 006/138] Add device selection menu --- .bin/Scripts/functions/ddrescue.py | 50 +++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 1acc5e96..b811c7ba 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -5,6 +5,7 @@ import pathlib import re from functions.common import * +from operator import itemgetter # STATIC VARIABLES USAGE=""" {script_name} clone [source [destination]] @@ -62,6 +63,7 @@ def setup_loopback_device(source_path): try: out = run_program(cmd, check=True) dev_path = out.stdout.decode().strip() + sleep(1) except CalledProcessError: print_error('Failed to setup loopback device for source.') abort() @@ -76,9 +78,7 @@ def menu_clone(source_path, dest_path): # Select source if not preselected if not source_path: - #TODO menu_select drive - print_warning('Select drive not implemented yet.') - source_path = '' + source_path = select_device('Please select a source device') # Check source source_path = os.path.realpath(source_path) @@ -101,11 +101,13 @@ def menu_clone(source_path, dest_path): source_details = get_device_details(source_dev_path) # Show selection details + print_success('Source device') if source_is_image: - print_success('Using image file: {}'.format(source_path)) - print_success(' (via loopback device: {})'.format( + print_standard('Using image file: {}'.format(source_path)) + print_standard(' (via loopback device: {})'.format( source_dev_path)) show_device_details(source_dev_path) + print_standard(' ') def menu_ddrescue(*args): """Main ddrescue loop/menu.""" @@ -140,6 +142,44 @@ def menu_image(source_path, dest_path): print_success('GNU ddrescue: Imaging Menu') pass +def select_device(title='Which device?'): + """Select block device via a menu, returns dev_path as str.""" + try: + cmd = ( + 'lsblk', + '--json', + '--nodeps', + '--output-all', + '--paths') + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + except CalledProcessError: + print_error('Failed to get device details for {}'.format(dev_path)) + abort() + + # Build menu + dev_options = [] + for dev in json_data['blockdevices']: + dev_options.append({ + 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( + name = dev['name'], + tran = dev['tran'] if dev['tran'] else '', + size = dev['size'] if dev['size'] else '', + model = dev['model'] if dev['model'] else '', + serial = dev['serial'] if dev['serial'] else ''), + 'Path': dev['name']}) + dev_options = sorted(dev_options, key=itemgetter('Name')) + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + selection = menu_select( + title = title, + main_entries = dev_options, + action_entries = actions) + + if selection.isnumeric(): + return dev_options[int(selection)-1]['Path'] + elif selection == 'Q': + abort() + def show_usage(script_name): print_info('Usage:') print_standard(USAGE.format(script_name=script_name)) From 0c3b90eb630d55c62d01ca967f2646bd87c8d8a7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 15 Jul 2018 00:04:24 -0600 Subject: [PATCH 007/138] Add clone destination sections * Hide source device in dest selection menu --- .bin/Scripts/functions/ddrescue.py | 43 +++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index b811c7ba..c16f2ebb 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -99,7 +99,32 @@ def menu_clone(source_path, dest_path): source_details['pkname'])): source_dev_path = source_details['pkname'] source_details = get_device_details(source_dev_path) + print_standard(' ') + # Select destination if not preselected + if not dest_path: + dest_path = select_device( + 'Please select a destination device', + skip_device=source_details) + + # Check destination + dest_path = os.path.realpath(dest_path) + if pathlib.Path(dest_path).is_block_device(): + dest_dev_path = dest_path + else: + print_error('Invalid destination "{}".'.format(dest_path)) + abort() + dest_details = get_device_details(dest_dev_path) + + # Check destination type + if dest_details['pkname']: + print_warning('Destination "{}" is a child device.'.format(dest_dev_path)) + if ask('Use parent device "{}" instead?'.format( + dest_details['pkname'])): + dest_dev_path = dest_details['pkname'] + dest_details = get_device_details(dest_dev_path) + print_standard(' ') + # Show selection details print_success('Source device') if source_is_image: @@ -108,6 +133,10 @@ def menu_clone(source_path, dest_path): source_dev_path)) show_device_details(source_dev_path) print_standard(' ') + + print_success('Destination device') + show_device_details(dest_dev_path) + print_standard(' ') def menu_ddrescue(*args): """Main ddrescue loop/menu.""" @@ -142,7 +171,7 @@ def menu_image(source_path, dest_path): print_success('GNU ddrescue: Imaging Menu') pass -def select_device(title='Which device?'): +def select_device(title='Which device?', skip_device={}): """Select block device via a menu, returns dev_path as str.""" try: cmd = ( @@ -160,6 +189,13 @@ def select_device(title='Which device?'): # Build menu dev_options = [] for dev in json_data['blockdevices']: + # Skip if dev in skip_device + dev_names = [dev['name'], dev['pkname']] + dev_names = [n for n in dev_names if n] + if skip_device and skip_device.get('name', None) in dev_names: + continue + + # Append non-matching devices dev_options.append({ 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( name = dev['name'], @@ -169,6 +205,11 @@ def select_device(title='Which device?'): serial = dev['serial'] if dev['serial'] else ''), 'Path': dev['name']}) dev_options = sorted(dev_options, key=itemgetter('Name')) + if not dev_options: + print_error('No devices available.') + abort() + + # Show Menu actions = [{'Name': 'Quit', 'Letter': 'Q'}] selection = menu_select( title = title, From b2fc7ea860a33859044936e39562a94d924f34d5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 17:58:29 -0600 Subject: [PATCH 008/138] Renamed script ddrescue-tui, added launcher script Launcher script runs the python script in tmux (same as hw-diags) --- .bin/Scripts/ddrescue-tui | 43 +++++++++++++++++++ .../{wk-ddrescue => ddrescue-tui-menu} | 4 +- .bin/Scripts/hw-diags | 2 +- 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100755 .bin/Scripts/ddrescue-tui rename .bin/Scripts/{wk-ddrescue => ddrescue-tui-menu} (87%) diff --git a/.bin/Scripts/ddrescue-tui b/.bin/Scripts/ddrescue-tui new file mode 100755 index 00000000..d83b3a7e --- /dev/null +++ b/.bin/Scripts/ddrescue-tui @@ -0,0 +1,43 @@ +#!/bin/bash +# +## Wizard Kit: ddrescue TUI Launcher + +SESSION_NAME="ddrescue-tui" +WINDOW_NAME="GNU ddrescue TUI" +MENU="ddrescue-tui-menu" + +function ask() { + while :; do + read -p "$1 " -r answer + if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then + return 0 + elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then + return 1 + fi + done +} + +die () { + echo "$0:" "$@" >&2 + exit 1 +} + +# Check for running session +if tmux list-session | grep -q "$SESSION_NAME"; then + echo "WARNING: tmux session $SESSION_NAME already exists." + echo "" + if ask "Kill current session?"; then + tmux kill-session -t "$SESSION_NAME" || \ + die "Failed to kill session: $SESSION_NAME" + else + echo "Aborted." + echo "" + echo -n "Press Enter to exit... " + read -r + exit 0 + fi +fi + +# Start session +tmux new-session -s "$SESSION_NAME" -n "$WINDOW_NAME" "$MENU" $* + diff --git a/.bin/Scripts/wk-ddrescue b/.bin/Scripts/ddrescue-tui-menu similarity index 87% rename from .bin/Scripts/wk-ddrescue rename to .bin/Scripts/ddrescue-tui-menu index 28ce6fb7..ea8c1a55 100755 --- a/.bin/Scripts/wk-ddrescue +++ b/.bin/Scripts/ddrescue-tui-menu @@ -21,8 +21,8 @@ if __name__ == '__main__': menu_ddrescue(*sys.argv) # Done - #print_standard('\nDone.') - #pause("Press Enter to exit...") + print_standard('\nDone.') + pause("Press Enter to exit...") exit_script() except SystemExit: pass diff --git a/.bin/Scripts/hw-diags b/.bin/Scripts/hw-diags index c1d7d3ca..e8d838df 100755 --- a/.bin/Scripts/hw-diags +++ b/.bin/Scripts/hw-diags @@ -24,7 +24,7 @@ die () { # Check for running session if tmux list-session | grep -q "$SESSION_NAME"; then - echo "WARNING: hw-diags tmux session already exists." + echo "WARNING: tmux session $SESSION_NAME already exists." echo "" if ask "Kill current session?"; then tmux kill-session -t "$SESSION_NAME" || \ From e56296d8b07129ebacc2701ec6c622378ea225a8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 18:01:06 -0600 Subject: [PATCH 009/138] Consolidated device selection code * Common code moved to select_device() * Existing select_device() renamed menu_select_device() * Fixed skip_device code * Refactored source/dest vars into dicts * Added confirmation after source/dest are selected --- .bin/Scripts/functions/ddrescue.py | 129 +++++++++++++++-------------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index c16f2ebb..ab208c87 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -75,69 +75,35 @@ def menu_clone(source_path, dest_path): source_is_image = False source_dev_path = None print_success('GNU ddrescue: Cloning Menu') - - # Select source if not preselected - if not source_path: - source_path = select_device('Please select a source device') - - # Check source - source_path = os.path.realpath(source_path) - if pathlib.Path(source_path).is_block_device(): - source_dev_path = source_path - elif pathlib.Path(source_path).is_file(): - source_dev_path = setup_loopback_device(source_path) - source_is_image = True - else: - print_error('Invalid source "{}".'.format(source_path)) - abort() - source_details = get_device_details(source_dev_path) - - # Check source type - if source_details['pkname']: - print_warning('Source "{}" is a child device.'.format(source_dev_path)) - if ask('Use parent device "{}" instead?'.format( - source_details['pkname'])): - source_dev_path = source_details['pkname'] - source_details = get_device_details(source_dev_path) - print_standard(' ') - - # Select destination if not preselected - if not dest_path: - dest_path = select_device( - 'Please select a destination device', - skip_device=source_details) - - # Check destination - dest_path = os.path.realpath(dest_path) - if pathlib.Path(dest_path).is_block_device(): - dest_dev_path = dest_path - else: - print_error('Invalid destination "{}".'.format(dest_path)) - abort() - dest_details = get_device_details(dest_dev_path) - - # Check destination type - if dest_details['pkname']: - print_warning('Destination "{}" is a child device.'.format(dest_dev_path)) - if ask('Use parent device "{}" instead?'.format( - dest_details['pkname'])): - dest_dev_path = dest_details['pkname'] - dest_details = get_device_details(dest_dev_path) - print_standard(' ') + + # Set devices + source = select_device('source', source_path) + dest = select_device('destination', dest_path, + skip_device = source['Details'], allow_image_file = False) # Show selection details + clear_screen() print_success('Source device') - if source_is_image: - print_standard('Using image file: {}'.format(source_path)) + if source['Is Image']: + print_standard('Using image file: {}'.format(source['Path'])) print_standard(' (via loopback device: {})'.format( - source_dev_path)) - show_device_details(source_dev_path) + source['Dev Path'])) + show_device_details(source['Dev Path']) print_standard(' ') - print_success('Destination device') - show_device_details(dest_dev_path) + print_success('Destination device ', end='') + print_error('(ALL DATA WILL BE DELETED)', timestamp=False) + show_device_details(dest['Dev Path']) print_standard(' ') + # Confirm + if not ask('Proceed with clone?'): + abort() + + # Build outer panes + clear_screen() + #TODO + def menu_ddrescue(*args): """Main ddrescue loop/menu.""" args = list(args) @@ -171,8 +137,53 @@ def menu_image(source_path, dest_path): print_success('GNU ddrescue: Imaging Menu') pass -def select_device(title='Which device?', skip_device={}): +def select_device(description='device', provided_path=None, + skip_device={}, allow_image_file=True): + """Select device via provided path or menu, return dev as dict.""" + dev = {'Is Image': False} + + # Set path + if provided_path: + dev['Path'] = provided_path + else: + dev['Path'] = menu_select_device( + title='Please select a {}'.format(description), + skip_device=skip_device) + dev['Path'] = os.path.realpath(dev['Path']) + + # Check path + if pathlib.Path(dev['Path']).is_block_device(): + dev['Dev Path'] = dev['Path'] + elif allow_image_file and pathlib.Path(dev['Path']).is_file(): + dev['Dev Path'] = setup_loopback_device(dev['Path']) + dev['Is Image'] = True + else: + print_error('Invalid {} "{}".'.format(description, dev['Path'])) + abort() + + # Get device details + dev['Details'] = get_device_details(dev['Dev Path']) + + # Check for parent device(s) + while dev['Details']['pkname']: + print_warning('{} "{}" is a child device.'.format( + description.title(), dev['Dev Path'])) + if ask('Use parent device "{}" instead?'.format( + dev['Details']['pkname'])): + # Update dev with parent info + dev['Dev Path'] = dev['Details']['pkname'] + dev['Details'] = get_device_details(dev['Dev Path']) + else: + # Leave alone + break + + return dev + +def menu_select_device(title='Which device?', skip_device={}): """Select block device via a menu, returns dev_path as str.""" + skip_names = [ + skip_device.get('name', None), skip_device.get('pkname', None)] + skip_names = [n for n in skip_names if n] try: cmd = ( 'lsblk', @@ -189,10 +200,8 @@ def select_device(title='Which device?', skip_device={}): # Build menu dev_options = [] for dev in json_data['blockdevices']: - # Skip if dev in skip_device - dev_names = [dev['name'], dev['pkname']] - dev_names = [n for n in dev_names if n] - if skip_device and skip_device.get('name', None) in dev_names: + # Skip if dev is in skip_names + if dev['name'] in skip_names or dev['pkname'] in skip_names: continue # Append non-matching devices From b5d8a5503192a4a51c3c24355ac2fe3fff6802a2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 18:07:12 -0600 Subject: [PATCH 010/138] Reordered functions --- .bin/Scripts/functions/ddrescue.py | 160 ++++++++++++++--------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index ab208c87..f834107e 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -14,26 +14,6 @@ USAGE=""" {script_name} clone [source [destination]] """ # Functions -def show_device_details(dev_path): - """Display device details on screen.""" - cmd = ( - 'lsblk', '--nodeps', - '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', - dev_path) - result = run_program(cmd) - output = result.stdout.decode().splitlines() - print_info(output.pop(0)) - for line in output: - print_standard(line) - - # Children info - cmd = ('lsblk', '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', dev_path) - result = run_program(cmd) - output = result.stdout.decode().splitlines() - print_info(output.pop(0)) - for line in output: - print_standard(line) - def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -52,24 +32,6 @@ def get_device_details(dev_path): # Just return the first device (there should only be one) return json_data['blockdevices'][0] -def setup_loopback_device(source_path): - """Setup a loopback device for source_path, returns dev_path as str.""" - cmd = ( - 'losetup', - '--find', - '--partscan', - '--show', - source_path) - try: - out = run_program(cmd, check=True) - dev_path = out.stdout.decode().strip() - sleep(1) - except CalledProcessError: - print_error('Failed to setup loopback device for source.') - abort() - else: - return dev_path - def menu_clone(source_path, dest_path): """ddrescue cloning menu.""" source_is_image = False @@ -137,48 +99,6 @@ def menu_image(source_path, dest_path): print_success('GNU ddrescue: Imaging Menu') pass -def select_device(description='device', provided_path=None, - skip_device={}, allow_image_file=True): - """Select device via provided path or menu, return dev as dict.""" - dev = {'Is Image': False} - - # Set path - if provided_path: - dev['Path'] = provided_path - else: - dev['Path'] = menu_select_device( - title='Please select a {}'.format(description), - skip_device=skip_device) - dev['Path'] = os.path.realpath(dev['Path']) - - # Check path - if pathlib.Path(dev['Path']).is_block_device(): - dev['Dev Path'] = dev['Path'] - elif allow_image_file and pathlib.Path(dev['Path']).is_file(): - dev['Dev Path'] = setup_loopback_device(dev['Path']) - dev['Is Image'] = True - else: - print_error('Invalid {} "{}".'.format(description, dev['Path'])) - abort() - - # Get device details - dev['Details'] = get_device_details(dev['Dev Path']) - - # Check for parent device(s) - while dev['Details']['pkname']: - print_warning('{} "{}" is a child device.'.format( - description.title(), dev['Dev Path'])) - if ask('Use parent device "{}" instead?'.format( - dev['Details']['pkname'])): - # Update dev with parent info - dev['Dev Path'] = dev['Details']['pkname'] - dev['Details'] = get_device_details(dev['Dev Path']) - else: - # Leave alone - break - - return dev - def menu_select_device(title='Which device?', skip_device={}): """Select block device via a menu, returns dev_path as str.""" skip_names = [ @@ -230,6 +150,86 @@ def menu_select_device(title='Which device?', skip_device={}): elif selection == 'Q': abort() +def select_device(description='device', provided_path=None, + skip_device={}, allow_image_file=True): + """Select device via provided path or menu, return dev as dict.""" + dev = {'Is Image': False} + + # Set path + if provided_path: + dev['Path'] = provided_path + else: + dev['Path'] = menu_select_device( + title='Please select a {}'.format(description), + skip_device=skip_device) + dev['Path'] = os.path.realpath(dev['Path']) + + # Check path + if pathlib.Path(dev['Path']).is_block_device(): + dev['Dev Path'] = dev['Path'] + elif allow_image_file and pathlib.Path(dev['Path']).is_file(): + dev['Dev Path'] = setup_loopback_device(dev['Path']) + dev['Is Image'] = True + else: + print_error('Invalid {} "{}".'.format(description, dev['Path'])) + abort() + + # Get device details + dev['Details'] = get_device_details(dev['Dev Path']) + + # Check for parent device(s) + while dev['Details']['pkname']: + print_warning('{} "{}" is a child device.'.format( + description.title(), dev['Dev Path'])) + if ask('Use parent device "{}" instead?'.format( + dev['Details']['pkname'])): + # Update dev with parent info + dev['Dev Path'] = dev['Details']['pkname'] + dev['Details'] = get_device_details(dev['Dev Path']) + else: + # Leave alone + break + + return dev + +def setup_loopback_device(source_path): + """Setup a loopback device for source_path, returns dev_path as str.""" + cmd = ( + 'losetup', + '--find', + '--partscan', + '--show', + source_path) + try: + out = run_program(cmd, check=True) + dev_path = out.stdout.decode().strip() + sleep(1) + except CalledProcessError: + print_error('Failed to setup loopback device for source.') + abort() + else: + return dev_path + +def show_device_details(dev_path): + """Display device details on screen.""" + cmd = ( + 'lsblk', '--nodeps', + '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', + dev_path) + result = run_program(cmd) + output = result.stdout.decode().splitlines() + print_info(output.pop(0)) + for line in output: + print_standard(line) + + # Children info + cmd = ('lsblk', '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', dev_path) + result = run_program(cmd) + output = result.stdout.decode().splitlines() + print_info(output.pop(0)) + for line in output: + print_standard(line) + def show_usage(script_name): print_info('Usage:') print_standard(USAGE.format(script_name=script_name)) From b1541c0faf2a2b47325f811b9f5636553bd942d3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 18:10:30 -0600 Subject: [PATCH 011/138] Added alias for hexedit --- .linux_items/include/airootfs/etc/skel/.aliases | 1 + 1 file changed, 1 insertion(+) diff --git a/.linux_items/include/airootfs/etc/skel/.aliases b/.linux_items/include/airootfs/etc/skel/.aliases index d20b59b3..5a71d2ad 100644 --- a/.linux_items/include/airootfs/etc/skel/.aliases +++ b/.linux_items/include/airootfs/etc/skel/.aliases @@ -7,6 +7,7 @@ alias 7z9='7z a -t7z -mx=9' alias diff='colordiff -ur' alias du='du -sch --apparent-size' alias fix-perms='find -type d -exec chmod 755 "{}" \; && find -type f -exec chmod 644 "{}" \;' +alias hexedit='hexedit --color' alias hw-info='sudo hw-info | less -S' alias inxi='echo -e "\e[33mWARNING: inxi is being replaced and will be removed in a future WizardKit release\e[0m"; echo -e " \e[32mReplacements include:\e[0m 'hw-drive-info', 'hw-info', & 'hw-sensors'"; echo ""; inxi' alias less='less -S' From adc12833306d344241cc50fb6f00507769097c0d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 18:17:38 -0600 Subject: [PATCH 012/138] Add pydf and alias df to pydf * Fixes issue #40 --- .linux_items/include/airootfs/etc/pydfrc | 24 +++++++++++++++++++ .../include/airootfs/etc/skel/.aliases | 1 + .linux_items/packages/live_add | 1 + 3 files changed, 26 insertions(+) create mode 100644 .linux_items/include/airootfs/etc/pydfrc diff --git a/.linux_items/include/airootfs/etc/pydfrc b/.linux_items/include/airootfs/etc/pydfrc new file mode 100644 index 00000000..100b1f05 --- /dev/null +++ b/.linux_items/include/airootfs/etc/pydfrc @@ -0,0 +1,24 @@ +normal_colour = 'default' +header_colour = 'blue' +local_fs_colour = 'default' +remote_fs_colour = 'green' +special_fs_colour = 'yellow' +readonly_fs_colour = 'cyan' +filled_fs_colour = 'red' +full_fs_colour = 'on_red', 'green', 'blink' +sizeformat = "-h" +column_separator = ' ' +column_separator_colour = 'none' +stretch_screen = 0.3 +FILL_THRESH = 75.0 +FULL_THRESH = 85.0 +format = [ + ('fs', 10, "l"), ('size', 5, "r"), + ('used', 5, "r"), ('avail', 5, "r"), ('perc', 5, "r"), + ('bar', 0.1, "l"), ('on', 11, "l") + ] +barchar = '#' +bar_fillchar = '.' +hidebinds = True +mountfile = ['/etc/mtab', '/etc/mnttab', '/proc/mounts'] + diff --git a/.linux_items/include/airootfs/etc/skel/.aliases b/.linux_items/include/airootfs/etc/skel/.aliases index 5a71d2ad..9e36c7d1 100644 --- a/.linux_items/include/airootfs/etc/skel/.aliases +++ b/.linux_items/include/airootfs/etc/skel/.aliases @@ -4,6 +4,7 @@ alias 7z3='7z a -t7z -mx=3' alias 7z5='7z a -t7z -mx=5' alias 7z7='7z a -t7z -mx=7' alias 7z9='7z a -t7z -mx=9' +alias df='pydf' alias diff='colordiff -ur' alias du='du -sch --apparent-size' alias fix-perms='find -type d -exec chmod 755 "{}" \; && find -type f -exec chmod 644 "{}" \;' diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index a297ca05..ade34eb5 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -61,6 +61,7 @@ otf-font-awesome-4 p7zip papirus-icon-theme progsreiserfs +pydf python python-psutil python-requests From ae4f2ac680229aa72566df7ad27ea3d8f6f44fcf Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 19:40:29 -0600 Subject: [PATCH 013/138] Added outer panes --- .bin/Scripts/echo-and-hold | 12 +++++++++ .bin/Scripts/functions/ddrescue.py | 40 +++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100755 .bin/Scripts/echo-and-hold diff --git a/.bin/Scripts/echo-and-hold b/.bin/Scripts/echo-and-hold new file mode 100755 index 00000000..97c69830 --- /dev/null +++ b/.bin/Scripts/echo-and-hold @@ -0,0 +1,12 @@ +#!/bin/bash +# +## Wizard Kit: "echo" text to screen and "hold" by waiting for user input + +function usage { + echo "Usage: $(basename "$0") \"text\"" + echo " e.g. $(basename "$0") \"Some text to show\"" +} + +echo -en "$@" && read -r __dont_care +exit 0 + diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index f834107e..f05a00d8 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -3,6 +3,7 @@ import json import pathlib import re +import time from functions.common import * from operator import itemgetter @@ -64,7 +65,35 @@ def menu_clone(source_path, dest_path): # Build outer panes clear_screen() - #TODO + ## Top panes + source_pane = tmux_splitw( + '-bdvl', '2', + '-PF', '#D', + 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( + text = source['Display Name'], + **COLORS)) + tmux_splitw( + '-t', source_pane, + '-dhl', '21', + 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( + text = time.strftime("%Y-%m-%d %H:%M %Z"), + **COLORS)) + tmux_splitw( + '-t', source_pane, + '-dhp', '50', + 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( + text = dest['Display Name'], + **COLORS)) + ## Side pane + tmux_splitw('-dhl', '21', 'echo-and-hold "Status #TODO"') + pause() + run_program(['tmux', 'kill-window']) + +def tmux_splitw(*args): + """Run tmux split-window command and return output as str.""" + cmd = ['tmux', 'split-window', *args] + result = run_program(cmd) + return result.stdout.decode().strip() def menu_ddrescue(*args): """Main ddrescue loop/menu.""" @@ -190,6 +219,15 @@ def select_device(description='device', provided_path=None, # Leave alone break + # Set display name + if dev['Is Image']: + dev['Display Name'] = '{name} {size} ({image_name})'.format( + image_name = dev['Path'][dev['Path'].rfind('/')+1:], + **dev['Details']) + else: + dev['Display Name'] = '{name} {size} {model}'.format( + **dev['Details']) + return dev def setup_loopback_device(source_path): From 855884ec934b1132854dd400488b45034bc8b573 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 21:01:10 -0600 Subject: [PATCH 014/138] Added initial update_progress() sections * TODO: expand to support Image mode --- .bin/Scripts/functions/ddrescue.py | 118 +++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index f05a00d8..544ce2a6 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -15,6 +15,10 @@ USAGE=""" {script_name} clone [source [destination]] """ # Functions +def abort_ddrescue_tui(): + run_program(['losetup', '-D']) + abort() + def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -27,7 +31,7 @@ def get_device_details(dev_path): result = run_program(cmd) except CalledProcessError: print_error('Failed to get device details for {}'.format(dev_path)) - abort() + abort_ddrescue_tui() json_data = json.loads(result.stdout.decode()) # Just return the first device (there should only be one) @@ -41,6 +45,10 @@ def menu_clone(source_path, dest_path): # Set devices source = select_device('source', source_path) + source['Type'] = 'Clone' + source['Pass 1'] = 'Pending' + source['Pass 2'] = 'Pending' + source['Pass 3'] = 'Pending' dest = select_device('destination', dest_path, skip_device = source['Details'], allow_image_file = False) @@ -61,7 +69,7 @@ def menu_clone(source_path, dest_path): # Confirm if not ask('Proceed with clone?'): - abort() + abort_ddrescue_tui() # Build outer panes clear_screen() @@ -85,15 +93,18 @@ def menu_clone(source_path, dest_path): text = dest['Display Name'], **COLORS)) ## Side pane - tmux_splitw('-dhl', '21', 'echo-and-hold "Status #TODO"') - pause() - run_program(['tmux', 'kill-window']) + update_progress(source) + tmux_splitw('-dhl', '21', + 'watch', '--color', '--no-title', '--interval', '1', + 'cat', source['Progress Out']) + + # Main menu + menu_main() -def tmux_splitw(*args): - """Run tmux split-window command and return output as str.""" - cmd = ['tmux', 'split-window', *args] - result = run_program(cmd) - return result.stdout.decode().strip() + # Done + run_program(['losetup', '-D']) + run_program(['tmux', 'kill-window']) + exit_script() def menu_ddrescue(*args): """Main ddrescue loop/menu.""" @@ -126,7 +137,13 @@ def menu_ddrescue(*args): def menu_image(source_path, dest_path): """ddrescue imaging menu.""" print_success('GNU ddrescue: Imaging Menu') - pass + +def menu_main(): + print_success('Main Menu') + print_standard(' ') + print_warning('#TODO') + print_standard(' ') + pause('Press Enter to exit...') def menu_select_device(title='Which device?', skip_device={}): """Select block device via a menu, returns dev_path as str.""" @@ -144,7 +161,7 @@ def menu_select_device(title='Which device?', skip_device={}): json_data = json.loads(result.stdout.decode()) except CalledProcessError: print_error('Failed to get device details for {}'.format(dev_path)) - abort() + abort_ddrescue_tui() # Build menu dev_options = [] @@ -165,7 +182,7 @@ def menu_select_device(title='Which device?', skip_device={}): dev_options = sorted(dev_options, key=itemgetter('Name')) if not dev_options: print_error('No devices available.') - abort() + abort_ddrescue_tui() # Show Menu actions = [{'Name': 'Quit', 'Letter': 'Q'}] @@ -177,7 +194,7 @@ def menu_select_device(title='Which device?', skip_device={}): if selection.isnumeric(): return dev_options[int(selection)-1]['Path'] elif selection == 'Q': - abort() + abort_ddrescue_tui() def select_device(description='device', provided_path=None, skip_device={}, allow_image_file=True): @@ -201,7 +218,7 @@ def select_device(description='device', provided_path=None, dev['Is Image'] = True else: print_error('Invalid {} "{}".'.format(description, dev['Path'])) - abort() + abort_ddrescue_tui() # Get device details dev['Details'] = get_device_details(dev['Dev Path']) @@ -244,7 +261,7 @@ def setup_loopback_device(source_path): sleep(1) except CalledProcessError: print_error('Failed to setup loopback device for source.') - abort() + abort_ddrescue_tui() else: return dev_path @@ -272,6 +289,75 @@ def show_usage(script_name): print_info('Usage:') print_standard(USAGE.format(script_name=script_name)) +def tmux_splitw(*args): + """Run tmux split-window command and return output as str.""" + cmd = ['tmux', 'split-window', *args] + result = run_program(cmd) + return result.stdout.decode().strip() + +def get_status_color(s, t_success=99, t_warn=90): + """Get color based on status, returns str.""" + color = COLORS['CLEAR'] + p_recovered = -1 + try: + p_recovered = float(s) + except ValueError: + # Status is either in lists below or will default to red + pass + + if s in ('Pending',): + color = COLORS['CLEAR'] + elif s in ('Working',): + color = COLORS['YELLOW'] + elif p_recovered >= t_success: + color = COLORS['GREEN'] + elif p_recovered >= t_warn: + color = COLORS['YELLOW'] + else: + color = COLORS['RED'] + return color + +def update_progress(source): + """Update progress file.""" + if 'Progress Out' not in source: + source['Progress Out'] = '{}/progress.out'.format(global_vars['LogDir']) + output = [] + if source['Type'] == 'Clone': + output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) + else: + output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) + output.append('─────────────────────') + + # Main device + if source['Type'] == 'Clone': + output.append('{BLUE}{dev}{CLEAR}'.format( + dev = source['Dev Path'], + **COLORS)) + for x in (1, 2, 3): + p_num = 'Pass {}'.format(x) + s_display = source[p_num] + try: + s_display = float(s_display) + except ValueError: + # Ignore and leave s_display alone + pass + else: + s_display = '{:0.2f} %'.format(s_display) + output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( + p_num = p_num, + s_color = get_status_color(source[p_num]), + s_display = s_display, + **COLORS)) + else: + #TODO + pass + + # Add line-endings + output = ['{}\n'.format(line) for line in output] + + with open(source['Progress Out'], 'w') as f: + f.writelines(output) + if __name__ == '__main__': print("This file is not meant to be called directly.") From 69909fa34cf474f6bb150d273a1f486e914165d4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 21:02:53 -0600 Subject: [PATCH 015/138] Added safety check --- .bin/Scripts/functions/ddrescue.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 544ce2a6..0805ab79 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -70,6 +70,15 @@ def menu_clone(source_path, dest_path): # Confirm if not ask('Proceed with clone?'): abort_ddrescue_tui() + + # Safety check + print_standard('\nSAFETY CHECK') + print_warning('All data will be DELETED from the ' + 'destination device and partition(s) listed above.') + print_warning('This is irreversible and will lead ' + 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) + if not ask('Asking again to confirm, is this correct?'): + abort_ddrescue_tui() # Build outer panes clear_screen() From 552868c26e0b83b90402b36732ab27e626e5f8dd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 22:04:09 -0600 Subject: [PATCH 016/138] Moved safety check into its own function * Will allow better duplication with Image mode --- .bin/Scripts/functions/ddrescue.py | 64 +++++++++++++++--------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 0805ab79..7c869115 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -37,6 +37,28 @@ def get_device_details(dev_path): # Just return the first device (there should only be one) return json_data['blockdevices'][0] +def get_status_color(s, t_success=99, t_warn=90): + """Get color based on status, returns str.""" + color = COLORS['CLEAR'] + p_recovered = -1 + try: + p_recovered = float(s) + except ValueError: + # Status is either in lists below or will default to red + pass + + if s in ('Pending',): + color = COLORS['CLEAR'] + elif s in ('Working',): + color = COLORS['YELLOW'] + elif p_recovered >= t_success: + color = COLORS['GREEN'] + elif p_recovered >= t_warn: + color = COLORS['YELLOW'] + else: + color = COLORS['RED'] + return color + def menu_clone(source_path, dest_path): """ddrescue cloning menu.""" source_is_image = False @@ -70,16 +92,8 @@ def menu_clone(source_path, dest_path): # Confirm if not ask('Proceed with clone?'): abort_ddrescue_tui() + show_safety_check() - # Safety check - print_standard('\nSAFETY CHECK') - print_warning('All data will be DELETED from the ' - 'destination device and partition(s) listed above.') - print_warning('This is irreversible and will lead ' - 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) - if not ask('Asking again to confirm, is this correct?'): - abort_ddrescue_tui() - # Build outer panes clear_screen() ## Top panes @@ -294,6 +308,16 @@ def show_device_details(dev_path): for line in output: print_standard(line) +def show_safety_check(): + """Display safety check message and get confirmation from user.""" + print_standard('\nSAFETY CHECK') + print_warning('All data will be DELETED from the ' + 'destination device and partition(s) listed above.') + print_warning('This is irreversible and will lead ' + 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) + if not ask('Asking again to confirm, is this correct?'): + abort_ddrescue_tui() + def show_usage(script_name): print_info('Usage:') print_standard(USAGE.format(script_name=script_name)) @@ -304,28 +328,6 @@ def tmux_splitw(*args): result = run_program(cmd) return result.stdout.decode().strip() -def get_status_color(s, t_success=99, t_warn=90): - """Get color based on status, returns str.""" - color = COLORS['CLEAR'] - p_recovered = -1 - try: - p_recovered = float(s) - except ValueError: - # Status is either in lists below or will default to red - pass - - if s in ('Pending',): - color = COLORS['CLEAR'] - elif s in ('Working',): - color = COLORS['YELLOW'] - elif p_recovered >= t_success: - color = COLORS['GREEN'] - elif p_recovered >= t_warn: - color = COLORS['YELLOW'] - else: - color = COLORS['RED'] - return color - def update_progress(source): """Update progress file.""" if 'Progress Out' not in source: From 6643cf5d252b0c781e90fc5aecad4fde8454387c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 22:08:00 -0600 Subject: [PATCH 017/138] Moved outer pane section to its own function * Will allow better duplication with Image mode --- .bin/Scripts/functions/ddrescue.py | 59 ++++++++++++++++-------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 7c869115..3282ff6f 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -19,6 +19,37 @@ def abort_ddrescue_tui(): run_program(['losetup', '-D']) abort() +def build_outer_panes(source, dest): + """Build top and side panes.""" + clear_screen() + + # Top panes + source_pane = tmux_splitw( + '-bdvl', '2', + '-PF', '#D', + 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( + text = source['Display Name'], + **COLORS)) + tmux_splitw( + '-t', source_pane, + '-dhl', '21', + 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( + text = time.strftime("%Y-%m-%d %H:%M %Z"), + **COLORS)) + tmux_splitw( + '-t', source_pane, + '-dhp', '50', + 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( + text = dest['Display Name'], + **COLORS)) + + # Side pane + update_progress(source) + tmux_splitw('-dhl', '21', + 'watch', '--color', '--no-title', '--interval', '1', + 'cat', source['Progress Out']) + + def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -94,34 +125,8 @@ def menu_clone(source_path, dest_path): abort_ddrescue_tui() show_safety_check() - # Build outer panes - clear_screen() - ## Top panes - source_pane = tmux_splitw( - '-bdvl', '2', - '-PF', '#D', - 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( - text = source['Display Name'], - **COLORS)) - tmux_splitw( - '-t', source_pane, - '-dhl', '21', - 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( - text = time.strftime("%Y-%m-%d %H:%M %Z"), - **COLORS)) - tmux_splitw( - '-t', source_pane, - '-dhp', '50', - 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( - text = dest['Display Name'], - **COLORS)) - ## Side pane - update_progress(source) - tmux_splitw('-dhl', '21', - 'watch', '--color', '--no-title', '--interval', '1', - 'cat', source['Progress Out']) - # Main menu + build_outer_panes(source, dest) menu_main() # Done From 9a27afebf7100c6112e4feb016ef1db220e9a088 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 22:19:39 -0600 Subject: [PATCH 018/138] Moved selection details into its own function * Will allow for better duplication in Image mode --- .bin/Scripts/functions/ddrescue.py | 37 +++++++++++++++++++----------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 3282ff6f..ae28527a 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -106,20 +106,8 @@ def menu_clone(source_path, dest_path): skip_device = source['Details'], allow_image_file = False) # Show selection details - clear_screen() - print_success('Source device') - if source['Is Image']: - print_standard('Using image file: {}'.format(source['Path'])) - print_standard(' (via loopback device: {})'.format( - source['Dev Path'])) - show_device_details(source['Dev Path']) - print_standard(' ') + show_selection_details(source, dest) - print_success('Destination device ', end='') - print_error('(ALL DATA WILL BE DELETED)', timestamp=False) - show_device_details(dest['Dev Path']) - print_standard(' ') - # Confirm if not ask('Proceed with clone?'): abort_ddrescue_tui() @@ -323,6 +311,29 @@ def show_safety_check(): if not ask('Asking again to confirm, is this correct?'): abort_ddrescue_tui() +def show_selection_details(source, dest): + clear_screen() + + # Source + print_success('Source device') + if source['Is Image']: + print_standard('Using image file: {}'.format(source['Path'])) + print_standard(' (via loopback device: {})'.format( + source['Dev Path'])) + show_device_details(source['Dev Path']) + print_standard(' ') + + # Destination + if source['Type'] == 'Clone': + print_success('Destination device ', end='') + print_error('(ALL DATA WILL BE DELETED)', timestamp=False) + show_device_details(dest['Dev Path']) + else: + dest['Dest Path'] = '/media/SHOP/Cust Name/' + print_success('Destination path') + print_standard(dest['Dest Path']) + print_standard(' ') + def show_usage(script_name): print_info('Usage:') print_standard(USAGE.format(script_name=script_name)) From ae7ba9cba40bba29c5099de9b5760d8a0fe942c9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 16 Jul 2018 22:25:13 -0600 Subject: [PATCH 019/138] Fixed typo in mount-raw-image --- .bin/Scripts/mount-raw-image | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/mount-raw-image b/.bin/Scripts/mount-raw-image index e738c445..6a183859 100755 --- a/.bin/Scripts/mount-raw-image +++ b/.bin/Scripts/mount-raw-image @@ -24,7 +24,7 @@ if [[ -f "${1:-}" ]]; then done else # losetup did not detect partitions, attempt whole image - udevil mount -o to "${LOOPDEV}" || true + udevil mount -o ro "${LOOPDEV}" || true fi else usage From 6eb486c7700ad0229c95b90a6ce2436267d8dfd4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 00:05:47 -0600 Subject: [PATCH 020/138] Extend get_simple_string() to support underscores --- .bin/Scripts/functions/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index f677df45..1aa8cfaa 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -214,11 +214,11 @@ def get_ticket_number(): return ticket_number def get_simple_string(prompt='Enter string'): - """Get string from user (only alphanumeric/space chars) and return as str.""" + """Get string from user (minimal allowed character set) and return as str.""" simple_string = None while simple_string is None: _input = input('{}: '.format(prompt)) - if re.match(r'^(\w|-| )+$', _input, re.ASCII): + if re.match(r'^(\w|-|_| )+$', _input, re.ASCII): simple_string = _input.strip() return simple_string From c37dab58af02e7575c65e6df757932338d1e3006 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 00:06:43 -0600 Subject: [PATCH 021/138] Updated mount_all_volumes(), now mount_volumes() * Now allows mounting R/W * Can restrict to a specific device's volume(s) * Added more data to the returned report --- .bin/Scripts/functions/data.py | 17 ++++++++++++++--- .bin/Scripts/mount-all-volumes | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index ad4fe4f1..340dc529 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -187,7 +187,7 @@ def get_mounted_volumes(): mounted_volumes.extend(item.get('children', [])) return {item['source']: item for item in mounted_volumes} -def mount_all_volumes(): +def mount_volumes(all_devices=True, device_path=None, read_write=False): """Mount all detected filesystems.""" report = {} @@ -195,6 +195,9 @@ def mount_all_volumes(): cmd = [ 'lsblk', '-J', '-p', '-o', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE'] + if not all_devices and device_path: + # Only mount volumes for specific device + cmd.append(device_path) result = run_program(cmd) json_data = json.loads(result.stdout.decode()) devs = json_data.get('blockdevices', []) @@ -233,8 +236,11 @@ def mount_all_volumes(): vol_data['show_data']['warning'] = True else: # Mount volume + cmd = ['udevil', 'mount', + '-o', 'rw' if read_write else 'ro', + vol_path] try: - run_program(['udevil', 'mount', '-o', 'ro', vol_path]) + run_program(cmd) except subprocess.CalledProcessError: vol_data['show_data']['data'] = 'Failed to mount' vol_data['show_data']['error'] = True @@ -242,11 +248,16 @@ def mount_all_volumes(): mounted_volumes = get_mounted_volumes() # Format pretty result string - if vol_data['show_data']['data'] != 'Failed to mount': + if vol_data['show_data']['data'] == 'Failed to mount': + vol_data['mount_point'] = None + else: size_used = human_readable_size( mounted_volumes[vol_path]['used']) size_avail = human_readable_size( mounted_volumes[vol_path]['avail']) + vol_data['size_avail'] = size_avail + vol_data['size_used'] = size_used + vol_data['mount_point'] = mounted_volumes[vol_path]['target'] vol_data['show_data']['data'] = 'Mounted on {}'.format( mounted_volumes[vol_path]['target']) vol_data['show_data']['data'] = '{:40} ({} used, {} free)'.format( diff --git a/.bin/Scripts/mount-all-volumes b/.bin/Scripts/mount-all-volumes index d743d656..9e5de0ea 100755 --- a/.bin/Scripts/mount-all-volumes +++ b/.bin/Scripts/mount-all-volumes @@ -18,7 +18,7 @@ if __name__ == '__main__': print_standard('{}: Volume mount tool'.format(KIT_NAME_FULL)) # Mount volumes - report = mount_all_volumes() + report = mount_volumes(all_devices=True) # Print report print_info('\nResults') From 007d2ef692403163538f318d1703d34474d67b6b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 00:15:28 -0600 Subject: [PATCH 022/138] Added select_path() for Image mode * Can select the current path, a local device's volume, or enter manually * Optionally add a ticket folder to path before imaging --- .bin/Scripts/functions/ddrescue.py | 110 +++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index ae28527a..3689878f 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -6,6 +6,7 @@ import re import time from functions.common import * +from functions.data import * from operator import itemgetter # STATIC VARIABLES @@ -92,9 +93,6 @@ def get_status_color(s, t_success=99, t_warn=90): def menu_clone(source_path, dest_path): """ddrescue cloning menu.""" - source_is_image = False - source_dev_path = None - print_success('GNU ddrescue: Cloning Menu') # Set devices source = select_device('source', source_path) @@ -152,7 +150,31 @@ def menu_ddrescue(*args): def menu_image(source_path, dest_path): """ddrescue imaging menu.""" - print_success('GNU ddrescue: Imaging Menu') + + # Set devices + source = select_device('source', source_path, allow_image_file = False) + source['Type'] = 'Image' + source['Pass 1'] = 'Pending' + source['Pass 2'] = 'Pending' + source['Pass 3'] = 'Pending' + dest = select_path(dest_path, skip_device=source['Details']) + + # Show selection details + show_selection_details(source, dest) + + # Confirm + if not ask('Proceed with clone?'): + abort_ddrescue_tui() + show_safety_check() + + # Main menu + build_outer_panes(source, dest) + menu_main() + + # Done + run_program(['losetup', '-D']) + run_program(['tmux', 'kill-window']) + exit_script() def menu_main(): print_success('Main Menu') @@ -212,6 +234,83 @@ def menu_select_device(title='Which device?', skip_device={}): elif selection == 'Q': abort_ddrescue_tui() +def select_path(provided_path=None, skip_device={}): + selected_path = {} + pwd = os.path.realpath(global_vars['Env']['PWD']) + path_options = [ + {'Name': 'Current directory: {}'.format(pwd), 'Path': pwd}, + {'Name': 'Local device', 'Path': None}, + {'Name': 'Enter manually', 'Path': None}] + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + selection = menu_select( + title = 'Please make a selection', + main_entries = path_options, + action_entries = actions) + + if selection == 'Q': + abort_ddrescue_tui() + elif selection.isnumeric(): + index = int(selection) - 1 + if path_options[index]['Path']: + # Current directory + selected_path['Path'] = pwd + + elif path_options[index]['Name'] == 'Local device': + # Local device + local_device = select_device( + skip_device = skip_device, + allow_image_file = False) + + # Mount device volume(s) + report = mount_volumes( + all_devices = False, + device_path = local_device['Dev Path'], + read_write = True) + + # Select volume + vol_options = [] + for k, v in sorted(report.items()): + disabled = v['show_data']['data'] == 'Failed to mount' + if disabled: + name = '{name} (Failed to mount)'.format(**v) + else: + name = '{name} (mounted on "{mount_point}")'.format(**v) + vol_options.append({ + 'Name': name, + 'Path': v['mount_point'], + 'Disabled': disabled}) + selection = menu_select( + title = 'Please select a volume', + main_entries = vol_options, + action_entries = actions) + if selection.isnumeric(): + selected_path['Path'] = vol_options[int(selection)-1]['Path'] + elif selection == 'Q': + abort_ddrescue_tui() + + elif path_options[index]['Name'] == 'Enter manually': + # Manual entry + while not selected_path: + m_path = input('Please enter path: ').strip() + if m_path and pathlib.Path(m_path).is_dir(): + selected_path['Path'] = os.path.realpath(m_path) + elif m_path and pathlib.Path(m_path).is_file(): + print_error('File "{}" exists'.format(m_path)) + else: + print_error('Invalid path "{}"'.format(m_path)) + + if ask('Create ticket folder?'): + ticket_folder = get_simple_string('Please enter folder name') + selected_path['Path'] = os.path.join( + selected_path['Path'], ticket_folder) + try: + os.makedirs(selected_path['Path'], exist_ok=True) + except OSError: + print_error('Failed to create folder "{}"'.format( + selected_path['Path'])) + abort_ddrescue_tui() + return selected_path + def select_device(description='device', provided_path=None, skip_device={}, allow_image_file=True): """Select device via provided path or menu, return dev as dict.""" @@ -329,9 +428,8 @@ def show_selection_details(source, dest): print_error('(ALL DATA WILL BE DELETED)', timestamp=False) show_device_details(dest['Dev Path']) else: - dest['Dest Path'] = '/media/SHOP/Cust Name/' print_success('Destination path') - print_standard(dest['Dest Path']) + print_standard(dest['Path']) print_standard(' ') def show_usage(script_name): From 1e4a3b6c0ec36355e019f7581022e4b0278c0364 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 00:49:41 -0600 Subject: [PATCH 023/138] Fix provided_path for Imaging and adjust top panes * Moved select_path menu sections to menu_select_path() --- .bin/Scripts/functions/ddrescue.py | 63 ++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 3689878f..0eda4a18 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -157,7 +157,7 @@ def menu_image(source_path, dest_path): source['Pass 1'] = 'Pending' source['Pass 2'] = 'Pending' source['Pass 3'] = 'Pending' - dest = select_path(dest_path, skip_device=source['Details']) + dest = select_dest_path(dest_path, skip_device=source['Details']) # Show selection details show_selection_details(source, dest) @@ -234,14 +234,19 @@ def menu_select_device(title='Which device?', skip_device={}): elif selection == 'Q': abort_ddrescue_tui() -def select_path(provided_path=None, skip_device={}): - selected_path = {} +def menu_select_path(skip_device={}): + """Select path via menu, returns path as str.""" pwd = os.path.realpath(global_vars['Env']['PWD']) + s_path = None + + # Build Menu path_options = [ {'Name': 'Current directory: {}'.format(pwd), 'Path': pwd}, {'Name': 'Local device', 'Path': None}, {'Name': 'Enter manually', 'Path': None}] actions = [{'Name': 'Quit', 'Letter': 'Q'}] + + # Show Menu selection = menu_select( title = 'Please make a selection', main_entries = path_options, @@ -253,7 +258,7 @@ def select_path(provided_path=None, skip_device={}): index = int(selection) - 1 if path_options[index]['Path']: # Current directory - selected_path['Path'] = pwd + s_path = pwd elif path_options[index]['Name'] == 'Local device': # Local device @@ -284,32 +289,58 @@ def select_path(provided_path=None, skip_device={}): main_entries = vol_options, action_entries = actions) if selection.isnumeric(): - selected_path['Path'] = vol_options[int(selection)-1]['Path'] + s_path = vol_options[int(selection)-1]['Path'] elif selection == 'Q': abort_ddrescue_tui() elif path_options[index]['Name'] == 'Enter manually': # Manual entry - while not selected_path: + while not s_path: m_path = input('Please enter path: ').strip() if m_path and pathlib.Path(m_path).is_dir(): - selected_path['Path'] = os.path.realpath(m_path) + s_path = os.path.realpath(m_path) elif m_path and pathlib.Path(m_path).is_file(): print_error('File "{}" exists'.format(m_path)) else: print_error('Invalid path "{}"'.format(m_path)) + return s_path +def select_dest_path(provided_path=None, skip_device={}): + dest = {} + + # Set path + if provided_path: + dest['Path'] = provided_path + else: + dest['Path'] = menu_select_path(skip_device=skip_device) + dest['Path'] = os.path.realpath(dest['Path']) + + # Check path + if not pathlib.Path(dest['Path']).is_dir(): + print_error('Invalid path "{}"'.format(dest['Path'])) + abort_ddrescue_tui() + + # Create ticket folder if ask('Create ticket folder?'): ticket_folder = get_simple_string('Please enter folder name') - selected_path['Path'] = os.path.join( - selected_path['Path'], ticket_folder) + dest['Path'] = os.path.join( + dest['Path'], ticket_folder) try: - os.makedirs(selected_path['Path'], exist_ok=True) + os.makedirs(dest['Path'], exist_ok=True) except OSError: print_error('Failed to create folder "{}"'.format( - selected_path['Path'])) + dest['Path'])) abort_ddrescue_tui() - return selected_path + + # Set display name + result = run_program(['tput', 'cols']) + width = int((int(result.stdout.decode().strip()) - 21) / 2) - 2 + if len(dest['Path']) > width: + dest['Display Name'] = '...{}'.format(dest['Path'][-(width-3):]) + else: + dest['Display Name'] = dest['Path'] + + return dest def select_device(description='device', provided_path=None, skip_device={}, allow_image_file=True): @@ -332,7 +363,7 @@ def select_device(description='device', provided_path=None, dev['Dev Path'] = setup_loopback_device(dev['Path']) dev['Is Image'] = True else: - print_error('Invalid {} "{}".'.format(description, dev['Path'])) + print_error('Invalid {} "{}"'.format(description, dev['Path'])) abort_ddrescue_tui() # Get device details @@ -359,6 +390,12 @@ def select_device(description='device', provided_path=None, else: dev['Display Name'] = '{name} {size} {model}'.format( **dev['Details']) + result = run_program(['tput', 'cols']) + width = int((int(result.stdout.decode().strip()) - 21) / 2) - 2 + if len(dev['Display Name']) > width: + dev['Display Name'] = '{}...'.format(dev['Display Name'][:(width-3)]) + else: + dev['Display Name'] = dev['Display Name'] return dev From 29266f161172371ece53b9c53c865a94ac3e7cda Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 01:11:43 -0600 Subject: [PATCH 024/138] Added initial Imaging source child dev support --- .bin/Scripts/functions/ddrescue.py | 55 ++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 0eda4a18..691d6169 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -165,7 +165,16 @@ def menu_image(source_path, dest_path): # Confirm if not ask('Proceed with clone?'): abort_ddrescue_tui() - show_safety_check() + + # TODO Replace with real child dev selection menu + if 'children' in source['Details']: + source['Children'] = [] + for c in source['Details']['children']: + source['Children'].append({ + 'Dev Path': c['name'], + 'Pass 1': 'Pending', + 'Pass 2': 'Pending', + 'Pass 3': 'Pending'}) # Main menu build_outer_panes(source, dest) @@ -511,8 +520,48 @@ def update_progress(source): s_display = s_display, **COLORS)) else: - #TODO - pass + # Image mode + if 'Children' in source: + for child in source['Children']: + output.append('{BLUE}{dev}{CLEAR}'.format( + dev = child['Dev Path'], + **COLORS)) + for x in (1, 2, 3): + p_num = 'Pass {}'.format(x) + s_display = child[p_num] + try: + s_display = float(s_display) + except ValueError: + # Ignore and leave s_display alone + pass + else: + s_display = '{:0.2f} %'.format(s_display) + output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( + p_num = p_num, + s_color = get_status_color(child[p_num]), + s_display = s_display, + **COLORS)) + output.append(' ') + else: + # Whole disk + output.append('{BLUE}{dev}{CLEAR} {YELLOW}(Whole){CLEAR}'.format( + dev = source['Dev Path'], + **COLORS)) + for x in (1, 2, 3): + p_num = 'Pass {}'.format(x) + s_display = source[p_num] + try: + s_display = float(s_display) + except ValueError: + # Ignore and leave s_display alone + pass + else: + s_display = '{:0.2f} %'.format(s_display) + output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( + p_num = p_num, + s_color = get_status_color(source[p_num]), + s_display = s_display, + **COLORS)) # Add line-endings output = ['{}\n'.format(line) for line in output] From de8f3bbd2b31fa1016260cbf67df8710e15596ce Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 13:21:12 -0600 Subject: [PATCH 025/138] Use image file instead of loopback device * Still setup loopback for image details but use image directly in ddrescue * Adjusted outer/side panes to use image path instead of loopback dev --- .bin/Scripts/functions/ddrescue.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 691d6169..06e7166b 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -393,16 +393,17 @@ def select_device(description='device', provided_path=None, # Set display name if dev['Is Image']: - dev['Display Name'] = '{name} {size} ({image_name})'.format( - image_name = dev['Path'][dev['Path'].rfind('/')+1:], - **dev['Details']) + dev['Display Name'] = dev['Path'] else: dev['Display Name'] = '{name} {size} {model}'.format( **dev['Details']) result = run_program(['tput', 'cols']) width = int((int(result.stdout.decode().strip()) - 21) / 2) - 2 if len(dev['Display Name']) > width: - dev['Display Name'] = '{}...'.format(dev['Display Name'][:(width-3)]) + if dev['Is Image']: + dev['Display Name'] = '...{}'.format(dev['Display Name'][-(width-3):]) + else: + dev['Display Name'] = '{}...'.format(dev['Display Name'][:(width-3)]) else: dev['Display Name'] = dev['Display Name'] @@ -502,7 +503,7 @@ def update_progress(source): # Main device if source['Type'] == 'Clone': output.append('{BLUE}{dev}{CLEAR}'.format( - dev = source['Dev Path'], + dev = 'Image File' if source['Is Image'] else source['Dev Path'], **COLORS)) for x in (1, 2, 3): p_num = 'Pass {}'.format(x) From fd8ac7cf1ad402d371b9af25028eb962a88b78a7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 14:16:38 -0600 Subject: [PATCH 026/138] Add child device selection menu for Image mode * Can select either whole device or child dev(s), not both --- .bin/Scripts/functions/ddrescue.py | 78 +++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 06e7166b..51cd45f2 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -166,15 +166,8 @@ def menu_image(source_path, dest_path): if not ask('Proceed with clone?'): abort_ddrescue_tui() - # TODO Replace with real child dev selection menu - if 'children' in source['Details']: - source['Children'] = [] - for c in source['Details']['children']: - source['Children'].append({ - 'Dev Path': c['name'], - 'Pass 1': 'Pending', - 'Pass 2': 'Pending', - 'Pass 3': 'Pending'}) + # Select child device(s) + source['Children'] = menu_select_children(source) # Main menu build_outer_panes(source, dest) @@ -192,6 +185,68 @@ def menu_main(): print_standard(' ') pause('Press Enter to exit...') +def menu_select_children(source): + """Select child device(s) or whole disk, returns list.""" + dev_options = [{ + 'Base Name': '{:<14}(Whole device)'.format(source['Dev Path']), + 'Path': source['Dev Path'], + 'Selected': False}] + for child in source['Details'].get('children', []): + dev_options.append({ + 'Base Name': '{:<14}({:>6} {})'.format( + child['name'], + child['size'], + child['fstype'] if child['fstype'] else 'Unknown'), + 'Path': child['name'], + 'Selected': False}) + actions = [ + {'Name': 'Proceed', 'Letter': 'P'}, + {'Name': 'Quit', 'Letter': 'Q'}] + + # Skip Menu if there's no children + if len(dev_options) == 1: + return [] + + # Show Menu + while True: + # Update entries + for dev in dev_options: + dev['Name'] = '{} {}'.format( + '*' if dev['Selected'] else ' ', + dev['Base Name']) + + selection = menu_select( + title = 'Please select part(s) to image', + main_entries = dev_options, + action_entries = actions) + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + dev_options[index]['Selected'] = not dev_options[index]['Selected'] + + # Deselect whole device if child selected (this round) + if index > 0: + dev_options[0]['Selected'] = False + + # Deselect all children if whole device selected + if dev_options[0]['Selected']: + for dev in dev_options[1:]: + dev['Selected'] = False + elif selection == 'P': + break + elif selection == 'Q': + abort_ddrescue_tui() + + # Check selection + selected_children = [{ + 'Dev Path': d['Path'], + 'Pass 1': 'Pending', + 'Pass 2': 'Pending', + 'Pass 3': 'Pending'} for d in dev_options + if d['Selected'] and 'Whole device' not in d['Base Name']] + return selected_children + def menu_select_device(title='Which device?', skip_device={}): """Select block device via a menu, returns dev_path as str.""" skip_names = [ @@ -522,7 +577,8 @@ def update_progress(source): **COLORS)) else: # Image mode - if 'Children' in source: + if source['Children']: + # Just parts for child in source['Children']: output.append('{BLUE}{dev}{CLEAR}'.format( dev = child['Dev Path'], @@ -544,7 +600,7 @@ def update_progress(source): **COLORS)) output.append(' ') else: - # Whole disk + # Whole device output.append('{BLUE}{dev}{CLEAR} {YELLOW}(Whole){CLEAR}'.format( dev = source['Dev Path'], **COLORS)) From 310a2eb63a25dae6695e06db6a120de07b1cac5e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 16:01:29 -0600 Subject: [PATCH 027/138] Initial Main Menu code * Required refactoring pass status code * Need to add settings menu next --- .bin/Scripts/functions/ddrescue.py | 154 ++++++++++++++++++++++++----- 1 file changed, 128 insertions(+), 26 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 51cd45f2..85149355 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -10,7 +10,19 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES -USAGE=""" {script_name} clone [source [destination]] +DDRESCUE_SETTINGS = [ + {'Flag': '--binary-prefixes', 'Enabled': True, 'Hidden': True}, + {'Flag': '--data-preview', 'Enabled': True, 'Hidden': True}, + {'Flag': '--idirect', 'Enabled': True}, + {'Flag': '--max-read-rate', 'Enabled': False, 'Value': '128MiB'}, + {'Flag': '--min-read-rate', 'Enabled': True, 'Value': '64KiB'}, + {'Flag': '--odirect', 'Enabled': True}, + {'Flag': '--reopen-on-error', 'Enabled': True}, + {'Flag': '--retry-passes=', 'Enabled': True, 'Value': '0'}, + {'Flag': '--timeout=', 'Enabled': True, 'Value': '5m'}, + {'Flag': '-vvvv', 'Enabled': True, 'Hidden': True}, + ] +USAGE = """ {script_name} clone [source [destination]] {script_name} image [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) """ @@ -97,9 +109,9 @@ def menu_clone(source_path, dest_path): # Set devices source = select_device('source', source_path) source['Type'] = 'Clone' - source['Pass 1'] = 'Pending' - source['Pass 2'] = 'Pending' - source['Pass 3'] = 'Pending' + source['Pass 1'] = {'Status': 'Pending', 'Done': False} + source['Pass 2'] = {'Status': 'Pending', 'Done': False} + source['Pass 3'] = {'Status': 'Pending', 'Done': False} dest = select_device('destination', dest_path, skip_device = source['Details'], allow_image_file = False) @@ -113,7 +125,7 @@ def menu_clone(source_path, dest_path): # Main menu build_outer_panes(source, dest) - menu_main() + menu_main(source) # Done run_program(['losetup', '-D']) @@ -154,16 +166,16 @@ def menu_image(source_path, dest_path): # Set devices source = select_device('source', source_path, allow_image_file = False) source['Type'] = 'Image' - source['Pass 1'] = 'Pending' - source['Pass 2'] = 'Pending' - source['Pass 3'] = 'Pending' + source['Pass 1'] = {'Status': 'Pending', 'Done': False} + source['Pass 2'] = {'Status': 'Pending', 'Done': False} + source['Pass 3'] = {'Status': 'Pending', 'Done': False} dest = select_dest_path(dest_path, skip_device=source['Details']) # Show selection details show_selection_details(source, dest) # Confirm - if not ask('Proceed with clone?'): + if not ask('Proceed with imaging?'): abort_ddrescue_tui() # Select child device(s) @@ -171,26 +183,116 @@ def menu_image(source_path, dest_path): # Main menu build_outer_panes(source, dest) - menu_main() + menu_main(source) # Done run_program(['losetup', '-D']) run_program(['tmux', 'kill-window']) exit_script() -def menu_main(): - print_success('Main Menu') - print_standard(' ') - print_warning('#TODO') - print_standard(' ') - pause('Press Enter to exit...') +def menu_main(source): + """Main menu is used to set ddrescue settings.""" + if 'Settings' not in source: + source['Settings'] = DDRESCUE_SETTINGS.copy() + + # Build menu + main_options = [ + {'Base Name': 'Retry', 'Enabled': False}, + {'Base Name': 'Reverse direction', 'Enabled': False}, + ] + actions =[ + {'Name': 'Start', 'Letter': 'S'}, + {'Name': 'Change settings {YELLOW}(experts only){CLEAR}'.format( + **COLORS), + 'Letter': 'C'}, + {'Name': 'Quit', 'Letter': 'Q', 'CRLF': True}, + ] + + # Show menu + while True: + # Update entries + for opt in main_options: + opt['Name'] = '{} {}'.format( + '✓' if opt['Enabled'] else 'X', + opt['Base Name']) + + selection = menu_select( + title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}'.format(**COLORS), + main_entries = main_options, + action_entries = actions) + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + elif selection == 'S': + # Set settings for pass + settings = [] + # TODO move to new function and replace with real code + for s in source['Settings']: + if not s['Enabled']: + continue + if s['Flag'][-1:] == '=': + settings.append('{Flag}{Value}'.format(**s)) + else: + settings.append(s['Flag']) + if 'Value' in s: + settings.append(s['Value']) + for opt in main_options: + if 'Retry' in opt['Base Name'] and opt['Enabled']: + settings.extend(['--retrim', '--try-again']) + if 'Reverse' in opt['Base Name'] and opt['Enabled']: + settings.append('--reverse') + # Disable for next pass + opt['Enabled'] = False + print_success('GO!') + if source['Pass 3']['Done']: + # Go to results + print_success('Done?') + elif source['Pass 2']['Done']: + # In pass 3 + print_error('Pass 3') + print_standard(str(settings)) + source['Pass 3']['Done'] = True + source['Pass 3']['Status'] = '99.99' + elif source['Pass 1']['Done']: + # In pass 2 + print_warning('Pass 2') + settings.append('--no-scrape') + print_standard(str(settings)) + source['Pass 2']['Done'] = True + source['Pass 2']['Status'] = '98.1415' + else: + # In pass 1 + print_info('Pass 1') + settings.extend(['--no-trim', '--no-scrape']) + print_standard(str(settings)) + status = source['Pass 1']['Status'] + if status == 'Pending': + source['Pass 1']['Status'] = '78.6623' + elif float(status) < 80: + source['Pass 1']['Status'] = '86.1102' + elif float(status) < 95: + source['Pass 1']['Status'] = '97.77' + source['Pass 1']['Done'] = True + update_progress(source) + pause() + elif selection == 'C': + # TODO Move to new function and replace with real code + print_warning( + 'These settings can cause {RED}SERIOUS damage{YELLOW} to drives'.format( + **COLORS)) + print_standard('Please read the manual before making any changes') + pause() + elif selection == 'Q': + break def menu_select_children(source): """Select child device(s) or whole disk, returns list.""" dev_options = [{ 'Base Name': '{:<14}(Whole device)'.format(source['Dev Path']), 'Path': source['Dev Path'], - 'Selected': False}] + 'Selected': True}] for child in source['Details'].get('children', []): dev_options.append({ 'Base Name': '{:<14}({:>6} {})'.format( @@ -241,9 +343,9 @@ def menu_select_children(source): # Check selection selected_children = [{ 'Dev Path': d['Path'], - 'Pass 1': 'Pending', - 'Pass 2': 'Pending', - 'Pass 3': 'Pending'} for d in dev_options + 'Pass 1': {'Status': 'Pending', 'Done': False}, + 'Pass 2': {'Status': 'Pending', 'Done': False}, + 'Pass 3': {'Status': 'Pending', 'Done': False}} for d in dev_options if d['Selected'] and 'Whole device' not in d['Base Name']] return selected_children @@ -562,7 +664,7 @@ def update_progress(source): **COLORS)) for x in (1, 2, 3): p_num = 'Pass {}'.format(x) - s_display = source[p_num] + s_display = source[p_num]['Status'] try: s_display = float(s_display) except ValueError: @@ -572,7 +674,7 @@ def update_progress(source): s_display = '{:0.2f} %'.format(s_display) output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( p_num = p_num, - s_color = get_status_color(source[p_num]), + s_color = get_status_color(source[p_num]['Status']), s_display = s_display, **COLORS)) else: @@ -585,7 +687,7 @@ def update_progress(source): **COLORS)) for x in (1, 2, 3): p_num = 'Pass {}'.format(x) - s_display = child[p_num] + s_display = child[p_num]['Status'] try: s_display = float(s_display) except ValueError: @@ -595,7 +697,7 @@ def update_progress(source): s_display = '{:0.2f} %'.format(s_display) output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( p_num = p_num, - s_color = get_status_color(child[p_num]), + s_color = get_status_color(child[p_num]['Status']), s_display = s_display, **COLORS)) output.append(' ') @@ -606,7 +708,7 @@ def update_progress(source): **COLORS)) for x in (1, 2, 3): p_num = 'Pass {}'.format(x) - s_display = source[p_num] + s_display = source[p_num]['Status'] try: s_display = float(s_display) except ValueError: @@ -616,7 +718,7 @@ def update_progress(source): s_display = '{:0.2f} %'.format(s_display) output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( p_num = p_num, - s_color = get_status_color(source[p_num]), + s_color = get_status_color(source[p_num]['Status']), s_display = s_display, **COLORS)) From 7d851d222228a339270f839c327637d016f9919d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 16:59:45 -0600 Subject: [PATCH 028/138] Add settings menu --- .bin/Scripts/functions/ddrescue.py | 95 ++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 85149355..cf96bfe1 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -10,18 +10,18 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES -DDRESCUE_SETTINGS = [ - {'Flag': '--binary-prefixes', 'Enabled': True, 'Hidden': True}, - {'Flag': '--data-preview', 'Enabled': True, 'Hidden': True}, - {'Flag': '--idirect', 'Enabled': True}, - {'Flag': '--max-read-rate', 'Enabled': False, 'Value': '128MiB'}, - {'Flag': '--min-read-rate', 'Enabled': True, 'Value': '64KiB'}, - {'Flag': '--odirect', 'Enabled': True}, - {'Flag': '--reopen-on-error', 'Enabled': True}, - {'Flag': '--retry-passes=', 'Enabled': True, 'Value': '0'}, - {'Flag': '--timeout=', 'Enabled': True, 'Value': '5m'}, - {'Flag': '-vvvv', 'Enabled': True, 'Hidden': True}, - ] +DDRESCUE_SETTINGS = { + '--binary-prefixes': {'Enabled': True, 'Hidden': True}, + '--data-preview': {'Enabled': True, 'Hidden': True}, + '--idirect': {'Enabled': True}, + '--odirect': {'Enabled': True}, + '--max-read-rate': {'Enabled': False, 'Value': '128MiB'}, + '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, + '--reopen-on-error': {'Enabled': True}, + '--retry-passes=': {'Enabled': True, 'Value': '0'}, + '--timeout=': {'Enabled': True, 'Value': '5m'}, + '-vvvv': {'Enabled': True, 'Hidden': True}, + } USAGE = """ {script_name} clone [source [destination]] {script_name} image [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) @@ -229,15 +229,15 @@ def menu_main(source): # Set settings for pass settings = [] # TODO move to new function and replace with real code - for s in source['Settings']: - if not s['Enabled']: + for k, v in source['Settings'].items(): + if not v['Enabled']: continue - if s['Flag'][-1:] == '=': - settings.append('{Flag}{Value}'.format(**s)) + if k[-1:] == '=': + settings.append('{}{}'.format(k, v['Value'])) else: - settings.append(s['Flag']) - if 'Value' in s: - settings.append(s['Value']) + settings.append(k) + if 'Value' in v: + settings.append(v['Value']) for opt in main_options: if 'Retry' in opt['Base Name'] and opt['Enabled']: settings.extend(['--retrim', '--try-again']) @@ -278,12 +278,7 @@ def menu_main(source): update_progress(source) pause() elif selection == 'C': - # TODO Move to new function and replace with real code - print_warning( - 'These settings can cause {RED}SERIOUS damage{YELLOW} to drives'.format( - **COLORS)) - print_standard('Please read the manual before making any changes') - pause() + menu_settings(source) elif selection == 'Q': break @@ -471,6 +466,56 @@ def menu_select_path(skip_device={}): print_error('Invalid path "{}"'.format(m_path)) return s_path +def menu_settings(source): + """Change advanced ddrescue settings.""" + title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) + title += '{YELLOW}These settings can cause {CLEAR}'.format(**COLORS) + title += '{RED}MAJOR DAMAGE{CLEAR}{YELLOW} to drives{CLEAR}\n'.format( + **COLORS) + title += 'Please read the manual before making any changes' + + # Build menu + settings = [] + for k, v in sorted(source['Settings'].items()): + if not v.get('Hidden', False): + settings.append({'Base Name': k.replace('=', ''), 'Flag': k}) + actions = [{'Name': 'Main Menu', 'Letter': 'M'}] + + # Show menu + while True: + for s in settings: + s['Name'] = '{}{}{}'.format( + s['Base Name'], + ' = ' if 'Value' in source['Settings'][s['Flag']] else '', + source['Settings'][s['Flag']].get('Value', '')) + if not source['Settings'][s['Flag']]['Enabled']: + s['Name'] = '{YELLOW}{name} (Disabled){CLEAR}'.format( + name = s['Name'], + **COLORS) + selection = menu_select( + title = title, + main_entries = settings, + action_entries = actions) + if selection.isnumeric(): + index = int(selection) - 1 + flag = settings[index]['Flag'] + enabled = source['Settings'][flag]['Enabled'] + if 'Value' in source['Settings'][flag]: + answer = choice( + choices = ['Toggle flag', 'Change value'], + prompt = 'Please make a selection for "{}"'.format(flag)) + if answer == 'Toggle flag': + # Toggle + source['Settings'][flag]['Enabled'] = not enabled + else: + # Update value + source['Settings'][flag]['Value'] = get_simple_string( + prompt = 'Enter new value') + else: + source['Settings'][flag]['Enabled'] = not enabled + elif selection == 'M': + break + def select_dest_path(provided_path=None, skip_device={}): dest = {} From d1d3e1592e84601f3243cad6f477e94b16ec3db8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 18:11:23 -0600 Subject: [PATCH 029/138] Added get_process() --- .bin/Scripts/functions/common.py | 33 +++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 1aa8cfaa..75fe99d1 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -197,6 +197,30 @@ def extract_item(item, filter='', silent=False): if not silent: print_warning('WARNING: Errors encountered while exctracting data') +def get_process(name=None): + """Get process by name, returns psutil.Process obj.""" + proc = None + if not name: + raise GenericError + + for p in psutil.process_iter(): + try: + if p.name() == name: + proc = p + except psutil._exceptions.NoSuchProcess: + # Process finished during iteration? Going to ignore + pass + return proc + +def get_simple_string(prompt='Enter string'): + """Get string from user (minimal allowed character set) and return as str.""" + simple_string = None + while simple_string is None: + _input = input('{}: '.format(prompt)) + if re.match(r'^(\w|-|_| )+$', _input, re.ASCII): + simple_string = _input.strip() + return simple_string + def get_ticket_number(): """Get TicketNumber from user, save in LogDir, and return as str.""" if not ENABLED_TICKET_NUMBERS: @@ -213,15 +237,6 @@ def get_ticket_number(): f.write(ticket_number) return ticket_number -def get_simple_string(prompt='Enter string'): - """Get string from user (minimal allowed character set) and return as str.""" - simple_string = None - while simple_string is None: - _input = input('{}: '.format(prompt)) - if re.match(r'^(\w|-|_| )+$', _input, re.ASCII): - simple_string = _input.strip() - return simple_string - def human_readable_size(size, decimals=0): """Convert size in bytes to a human-readable format and return a str.""" # Prep string formatting From 358191539cdba297b8230e1ddb9edc59677b9d49 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 19:19:38 -0600 Subject: [PATCH 030/138] Added run_ddrescue() and update_smart_report() * Working "wait" loop while ddrescue is running. --- .bin/Scripts/functions/ddrescue.py | 162 ++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 39 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index cf96bfe1..fa454ef1 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -2,7 +2,9 @@ import json import pathlib +import psutil import re +import signal import time from functions.common import * @@ -108,10 +110,11 @@ def menu_clone(source_path, dest_path): # Set devices source = select_device('source', source_path) - source['Type'] = 'Clone' + source['Current Pass'] = 'Pass 1' source['Pass 1'] = {'Status': 'Pending', 'Done': False} source['Pass 2'] = {'Status': 'Pending', 'Done': False} source['Pass 3'] = {'Status': 'Pending', 'Done': False} + source['Type'] = 'Clone' dest = select_device('destination', dest_path, skip_device = source['Details'], allow_image_file = False) @@ -165,10 +168,11 @@ def menu_image(source_path, dest_path): # Set devices source = select_device('source', source_path, allow_image_file = False) - source['Type'] = 'Image' + source['Current Pass'] = 'Pass 1' source['Pass 1'] = {'Status': 'Pending', 'Done': False} source['Pass 2'] = {'Status': 'Pending', 'Done': False} source['Pass 3'] = {'Status': 'Pending', 'Done': False} + source['Type'] = 'Image' dest = select_dest_path(dest_path, skip_device=source['Details']) # Show selection details @@ -213,7 +217,7 @@ def menu_main(source): # Update entries for opt in main_options: opt['Name'] = '{} {}'.format( - '✓' if opt['Enabled'] else 'X', + '[✓]' if opt['Enabled'] else '[ ]', opt['Base Name']) selection = menu_select( @@ -228,7 +232,6 @@ def menu_main(source): elif selection == 'S': # Set settings for pass settings = [] - # TODO move to new function and replace with real code for k, v in source['Settings'].items(): if not v['Enabled']: continue @@ -245,38 +248,9 @@ def menu_main(source): settings.append('--reverse') # Disable for next pass opt['Enabled'] = False - print_success('GO!') - if source['Pass 3']['Done']: - # Go to results - print_success('Done?') - elif source['Pass 2']['Done']: - # In pass 3 - print_error('Pass 3') - print_standard(str(settings)) - source['Pass 3']['Done'] = True - source['Pass 3']['Status'] = '99.99' - elif source['Pass 1']['Done']: - # In pass 2 - print_warning('Pass 2') - settings.append('--no-scrape') - print_standard(str(settings)) - source['Pass 2']['Done'] = True - source['Pass 2']['Status'] = '98.1415' - else: - # In pass 1 - print_info('Pass 1') - settings.extend(['--no-trim', '--no-scrape']) - print_standard(str(settings)) - status = source['Pass 1']['Status'] - if status == 'Pending': - source['Pass 1']['Status'] = '78.6623' - elif float(status) < 80: - source['Pass 1']['Status'] = '86.1102' - elif float(status) < 95: - source['Pass 1']['Status'] = '97.77' - source['Pass 1']['Done'] = True - update_progress(source) - pause() + + # Run pass + run_ddrescue(source, settings) elif selection == 'C': menu_settings(source) elif selection == 'Q': @@ -502,9 +476,9 @@ def menu_settings(source): enabled = source['Settings'][flag]['Enabled'] if 'Value' in source['Settings'][flag]: answer = choice( - choices = ['Toggle flag', 'Change value'], - prompt = 'Please make a selection for "{}"'.format(flag)) - if answer == 'Toggle flag': + choices = ['T', 'C'], + prompt = 'Toggle or change value for "{}"'.format(flag)) + if answer == 'T': # Toggle source['Settings'][flag]['Enabled'] = not enabled else: @@ -516,6 +490,99 @@ def menu_settings(source): elif selection == 'M': break +def run_ddrescue(source, settings): + """Run ddrescue pass.""" + if source['Current Pass'] == 'Pass 1': + settings.extend(['--no-trim', '--no-scrape']) + elif source['Current Pass'] == 'Pass 2': + settings.append('--no-scrape') + elif source['Current Pass'] == 'Pass 3': + pass + else: + # Assuming Done + return + + # Set heights + ## NOTE: 10/32 is based on min heights for SMART/ddrescue panes (10 + 22) + result = run_program(['tput', 'lines']) + height = int(result.stdout.decode().strip()) + height_smart = int(height * (12 / 34)) + height_ddrescue = height - height_smart + + # Show SMART status + update_smart_report(source) + smart_pane = tmux_splitw( + '-bdvl', str(height_smart), + '-PF', '#D', + 'watch', '--color', '--no-title', '--interval', '5', + 'cat', source['SMART Report']) + + # Start ddrescue + return_code = None + try: + clear_screen() + #ddrescue_proc = popen_program('ddrescue who.dd wat.dd why.map'.split()) + ddrescue_proc = popen_program(['./__choose_exit']) + while True: + sleep(3) + with open(source['SMART Report'], 'a') as f: + f.write('heh.\n') + return_code = ddrescue_proc.poll() + if return_code: + # i.e. not None and not 0 + print_error('Error(s) encountered, see message above.') + break + elif return_code is not None: + # Assuming normal exit + break + except KeyboardInterrupt: + # Catch user abort + pass + + # Was ddrescue aborted? + if return_code is None: + print_warning('Aborted') + + # TODO + update_progress(source) + print_info('Return: {}'.format(return_code)) + pause() + run_program(['tmux', 'kill-pane', '-t', smart_pane]) + return + + ##TODO + #print_success('GO!') + #if source['Pass 3']['Done']: + # # Go to results + # print_success('Done?') + #elif source['Pass 2']['Done']: + # # In pass 3 + # print_error('Pass 3') + # print_standard(str(settings)) + # source['Pass 3']['Done'] = True + # source['Pass 3']['Status'] = '99.99' + #elif source['Pass 1']['Done']: + # # In pass 2 + # print_warning('Pass 2') + # settings.append('--no-scrape') + # print_standard(str(settings)) + # source['Pass 2']['Done'] = True + # source['Pass 2']['Status'] = '98.1415' + #else: + # # In pass 1 + # print_info('Pass 1') + # settings.extend(['--no-trim', '--no-scrape']) + # print_standard(str(settings)) + # status = source['Pass 1']['Status'] + # if status == 'Pending': + # source['Pass 1']['Status'] = '78.6623' + # elif float(status) < 80: + # source['Pass 1']['Status'] = '86.1102' + # elif float(status) < 95: + # source['Pass 1']['Status'] = '97.77' + # source['Pass 1']['Done'] = True + #update_progress(source) + def select_dest_path(provided_path=None, skip_device={}): dest = {} @@ -773,6 +840,23 @@ def update_progress(source): with open(source['Progress Out'], 'w') as f: f.writelines(output) +def update_smart_report(source): + """Update smart report file.""" + if 'SMART Report' not in source: + source['SMART Report'] = '{}/smart_report.out'.format( + global_vars['LogDir']) + output = [] + + # TODO + output.append('SMART Report') + output.append('TODO') + + # Add line-endings + output = ['{}\n'.format(line) for line in output] + + with open(source['SMART Report'], 'w') as f: + f.writelines(output) + if __name__ == '__main__': print("This file is not meant to be called directly.") From a12a5912797dbf2fd0416f479828aa9f07091fac Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 21:05:37 -0600 Subject: [PATCH 031/138] Moved SMART sections to a separate script * Refresh rate is now handled by 'watch --interval' * Allows for much simpler ddrescue execution / tracking * Removed all 'SMART Report' sections from functions/ddrescue.py * functions/hw_diags.py has been further extended * Supports full device paths (only for displaying attributes ATM) * Adds a timestamp when only displaying attributes --- .bin/Scripts/ddrescue-tui-smart-display | 39 ++++++++++++++++++++++ .bin/Scripts/functions/ddrescue.py | 43 +++++-------------------- .bin/Scripts/functions/hw_diags.py | 39 ++++++++++++++++++---- 3 files changed, 79 insertions(+), 42 deletions(-) create mode 100755 .bin/Scripts/ddrescue-tui-smart-display diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display new file mode 100755 index 00000000..a53a2dca --- /dev/null +++ b/.bin/Scripts/ddrescue-tui-smart-display @@ -0,0 +1,39 @@ +#!/bin/python3 +# +## Wizard Kit: SMART attributes display for ddrescue TUI + +import os +import sys +import time + +# Init +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) +from functions.hw_diags import * +#init_global_vars() + +if __name__ == '__main__': + try: + # Prep + clear_screen() + dev_path = sys.argv[1] + devs = scan_disks(True, dev_path) + + # Warn if SMART unavailable + if dev_path not in devs: + print_error('SMART data not available') + input('') + + # Initial screen + dev = devs[dev_path] + clear_screen() + show_disk_details(dev, only_attributes=True) + + # Done + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index fa454ef1..ebc8b0f2 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -510,38 +510,28 @@ def run_ddrescue(source, settings): height_ddrescue = height - height_smart # Show SMART status - update_smart_report(source) smart_pane = tmux_splitw( '-bdvl', str(height_smart), '-PF', '#D', - 'watch', '--color', '--no-title', '--interval', '5', - 'cat', source['SMART Report']) + 'watch', '--color', '--no-title', '--interval', '300', + 'ddrescue-tui-smart-display', source['Dev Path']) # Start ddrescue - return_code = None try: clear_screen() - #ddrescue_proc = popen_program('ddrescue who.dd wat.dd why.map'.split()) - ddrescue_proc = popen_program(['./__choose_exit']) - while True: - sleep(3) - with open(source['SMART Report'], 'a') as f: - f.write('heh.\n') - return_code = ddrescue_proc.poll() - if return_code: - # i.e. not None and not 0 - print_error('Error(s) encountered, see message above.') - break - elif return_code is not None: - # Assuming normal exit - break + ddrescue_proc = popen_program(['./__choose_exit', *settings]) + ddrescue_proc.wait() except KeyboardInterrupt: # Catch user abort pass # Was ddrescue aborted? + return_code = ddrescue_proc.poll() if return_code is None: print_warning('Aborted') + elif return_code: + # i.e. not None and not 0 + print_error('Error(s) encountered, see message above.') # TODO update_progress(source) @@ -840,23 +830,6 @@ def update_progress(source): with open(source['Progress Out'], 'w') as f: f.writelines(output) -def update_smart_report(source): - """Update smart report file.""" - if 'SMART Report' not in source: - source['SMART Report'] = '{}/smart_report.out'.format( - global_vars['LogDir']) - output = [] - - # TODO - output.append('SMART Report') - output.append('TODO') - - # Add line-endings - output = ['{}\n'.format(line) for line in output] - - with open(source['SMART Report'], 'w') as f: - f.writelines(output) - if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 1b61d8ca..26547d07 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1,6 +1,7 @@ # Wizard Kit: Functions - HW Diagnostics import json +import time from functions.common import * @@ -56,7 +57,9 @@ def get_read_rate(s): def get_smart_details(dev): """Get SMART data for dev if possible, returns dict.""" - cmd = 'sudo smartctl --all --json /dev/{}'.format(dev).split() + cmd = 'sudo smartctl --all --json {}{}'.format( + '' if '/dev/' in dev else '/dev/', + dev).split() result = run_program(cmd, check=False) try: return json.loads(result.stdout.decode()) @@ -509,12 +512,17 @@ def run_tests(tests): global_vars['LogFile'])) pause('Press Enter to exit...') -def scan_disks(): +def scan_disks(full_paths=False, only_path=None): """Scan for disks eligible for hardware testing.""" clear_screen() # Get eligible disk list - result = run_program(['lsblk', '-J', '-O']) + cmd = ['lsblk', '-J', '-O'] + if full_paths: + cmd.append('-p') + if only_path: + cmd.append(only_path) + result = run_program(cmd) json_data = json.loads(result.stdout.decode()) devs = {} for d in json_data.get('blockdevices', []): @@ -536,13 +544,18 @@ def scan_disks(): for dev, data in devs.items(): # Get SMART attributes run_program( - cmd = 'sudo smartctl -s on /dev/{}'.format(dev).split(), + cmd = 'sudo smartctl -s on {}{}'.format( + '' if full_paths else '/dev/', + dev).split(), check = False) data['smartctl'] = get_smart_details(dev) # Get NVMe attributes if data['lsblk']['tran'] == 'nvme': cmd = 'sudo nvme smart-log /dev/{} -o json'.format(dev).split() + cmd = 'sudo nvme smart-log {}{} -o json'.format( + '' if full_paths else '/dev/', + dev).split() result = run_program(cmd, check=False) try: data['nvme-cli'] = json.loads(result.stdout.decode()) @@ -595,7 +608,9 @@ def show_disk_details(dev, only_attributes=False): dev_name = dev['lsblk']['name'] if not only_attributes: # Device description - print_info('Device: /dev/{}'.format(dev['lsblk']['name'])) + print_info('Device: {}{}'.format( + '' if '/dev/' in dev['lsblk']['name'] else '/dev/', + dev['lsblk']['name'])) print_standard(' {:>4} ({}) {} {}'.format( str(dev['lsblk'].get('size', '???b')).strip(), str(dev['lsblk'].get('tran', '???')).strip().upper().replace( @@ -617,7 +632,12 @@ def show_disk_details(dev, only_attributes=False): # Attributes if dev.get('NVMe Disk', False): - print_info('Attributes:') + if only_attributes: + print_info('SMART Attributes:', end='') + print_warning(' Updated: {}'.format( + time.strftime('%Y-%m-%d %H:%M %Z'))) + else: + print_info('Attributes:') for attrib, threshold in sorted(ATTRIBUTES['NVMe'].items()): if attrib in dev['nvme-cli']: print_standard( @@ -638,7 +658,12 @@ def show_disk_details(dev, only_attributes=False): print_success(raw_str, timestamp=False) elif dev['smartctl'].get('ata_smart_attributes', None): # SMART attributes - print_info('Attributes:') + if only_attributes: + print_info('SMART Attributes:', end='') + print_warning(' Updated: {}'.format( + time.strftime('%Y-%m-%d %H:%M %Z'))) + else: + print_info('Attributes:') s_table = dev['smartctl'].get('ata_smart_attributes', {}).get( 'table', {}) s_table = {a.get('id', 'Unknown'): a for a in s_table} From c705ba6afc5700300b2c46e3bb3f8e6ff60f776a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 22:10:37 -0600 Subject: [PATCH 032/138] Add pass completion detection sections * Retry option now sets recovery back to pass 1 --- .bin/Scripts/functions/ddrescue.py | 43 ++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index ebc8b0f2..4b2fd311 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -105,6 +105,20 @@ def get_status_color(s, t_success=99, t_warn=90): color = COLORS['RED'] return color +def mark_pass_complete(source): + """Mark current pass complete and set next pass as current.""" + current_pass = source['Current Pass'] + current_pass_num = int(current_pass[-1:]) + next_pass_num = current_pass_num + 1 + next_pass = 'Pass {}'.format(next_pass_num) + + # Update source vars + source['Current Pass'] = next_pass + source[current_pass]['Done'] = True + + # TODO Remove test code + source[current_pass]['Status'] = str(11.078 * current_pass_num * 3) + def menu_clone(source_path, dest_path): """ddrescue cloning menu.""" @@ -201,7 +215,7 @@ def menu_main(source): # Build menu main_options = [ - {'Base Name': 'Retry', 'Enabled': False}, + {'Base Name': 'Retry (mark non-rescued sectors "non-tried")', 'Enabled': False}, {'Base Name': 'Reverse direction', 'Enabled': False}, ] actions =[ @@ -244,6 +258,10 @@ def menu_main(source): for opt in main_options: if 'Retry' in opt['Base Name'] and opt['Enabled']: settings.extend(['--retrim', '--try-again']) + source['Current Pass'] = 'Pass 1' + source['Pass 1']['Done'] = False + source['Pass 2']['Done'] = False + source['Pass 3']['Done'] = False if 'Reverse' in opt['Base Name'] and opt['Enabled']: settings.append('--reverse') # Disable for next pass @@ -492,21 +510,25 @@ def menu_settings(source): def run_ddrescue(source, settings): """Run ddrescue pass.""" - if source['Current Pass'] == 'Pass 1': + current_pass = source['Current Pass'] + source[current_pass]['Status'] = 'Working' + update_progress(source) + if current_pass == 'Pass 1': settings.extend(['--no-trim', '--no-scrape']) - elif source['Current Pass'] == 'Pass 2': + elif current_pass == 'Pass 2': + # Allow trimming settings.append('--no-scrape') - elif source['Current Pass'] == 'Pass 3': + elif current_pass == 'Pass 3': + # Allow trimming and scraping pass else: - # Assuming Done - return + raise GenericError("This shouldn't happen?") # Set heights - ## NOTE: 10/32 is based on min heights for SMART/ddrescue panes (10 + 22) + ## NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) result = run_program(['tput', 'lines']) height = int(result.stdout.decode().strip()) - height_smart = int(height * (12 / 34)) + height_smart = int(height * (12 / 33)) height_ddrescue = height - height_smart # Show SMART status @@ -529,9 +551,14 @@ def run_ddrescue(source, settings): return_code = ddrescue_proc.poll() if return_code is None: print_warning('Aborted') + source[current_pass]['Status'] = 'Incomplete' elif return_code: # i.e. not None and not 0 print_error('Error(s) encountered, see message above.') + source[current_pass]['Status'] = 'Incomplete' + else: + # Not None and not non-zero int, assuming 0 + mark_pass_complete(source) # TODO update_progress(source) From 9d91a28d7a0756e11e64d1783982b4efdc6f8fb3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 17 Jul 2018 23:22:08 -0600 Subject: [PATCH 033/138] Add children pass, status, and update sections * Updating device / child device status/progress done in mark_*() functions * Add current pass description to main menu * Current pass (overall) only updated if all children have passed * Fix Pass 4 crash --- .bin/Scripts/functions/ddrescue.py | 131 +++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 33 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 4b2fd311..0d0a2aa5 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -106,19 +106,53 @@ def get_status_color(s, t_success=99, t_warn=90): return color def mark_pass_complete(source): - """Mark current pass complete and set next pass as current.""" + """Mark current pass complete for device, and overall if applicable.""" current_pass = source['Current Pass'] current_pass_num = int(current_pass[-1:]) next_pass_num = current_pass_num + 1 - next_pass = 'Pass {}'.format(next_pass_num) + if 1 <= next_pass_num <= 3: + next_pass = 'Pass {}'.format(next_pass_num) + else: + next_pass = 'Done' + # Check children progress + pass_complete_for_all_devs = True + for child in source['Children']: + if child['Dev Path'] == source['Current Device']: + # This function was called for this device, mark complete + child[current_pass]['Done'] = True + # TODO remove test code + child[current_pass]['Status'] = str(12.5 * current_pass_num * 2.75) + if not child[current_pass]['Done']: + pass_complete_for_all_devs = False + # Update source vars - source['Current Pass'] = next_pass - source[current_pass]['Done'] = True + if pass_complete_for_all_devs: + source['Current Pass'] = next_pass + source[current_pass]['Done'] = True # TODO Remove test code source[current_pass]['Status'] = str(11.078 * current_pass_num * 3) +def mark_pass_incomplete(source): + """Mark current pass incomplete.""" + current_pass = source['Current Pass'] + source[current_pass]['Status'] = 'Incomplete' + for child in source['Children']: + if child['Dev Path'] == source['Current Device']: + # This function was called for this device, mark incomplete + child[current_pass]['Status'] = 'Incomplete' + +def mark_all_passes_pending(source): + """Mark all devs and passes as pending in preparation for retry.""" + source['Current Pass'] = 'Pass 1' + for p_num in ['Pass 1', 'Pass 2', 'Pass 3']: + source[p_num]['Status'] = 'Pending' + source[p_num]['Done'] = False + for child in source['Children']: + child[p_num]['Status'] = 'Pending' + child[p_num]['Done'] = False + def menu_clone(source_path, dest_path): """ddrescue cloning menu.""" @@ -210,6 +244,8 @@ def menu_image(source_path, dest_path): def menu_main(source): """Main menu is used to set ddrescue settings.""" + title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) + title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) if 'Settings' not in source: source['Settings'] = DDRESCUE_SETTINGS.copy() @@ -228,6 +264,13 @@ def menu_main(source): # Show menu while True: + display_pass = '1 "Initial Read"' + if source['Current Pass'] == 'Pass 2': + display_pass = '2 "Trimming bad areas"' + elif source['Current Pass'] == 'Pass 3': + display_pass = '3 "Scraping bad areas"' + elif source['Current Pass'] == 'Done': + display_pass = 'Done' # Update entries for opt in main_options: opt['Name'] = '{} {}'.format( @@ -235,7 +278,7 @@ def menu_main(source): opt['Base Name']) selection = menu_select( - title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}'.format(**COLORS), + title = title + display_pass, main_entries = main_options, action_entries = actions) @@ -258,10 +301,7 @@ def menu_main(source): for opt in main_options: if 'Retry' in opt['Base Name'] and opt['Enabled']: settings.extend(['--retrim', '--try-again']) - source['Current Pass'] = 'Pass 1' - source['Pass 1']['Done'] = False - source['Pass 2']['Done'] = False - source['Pass 3']['Done'] = False + mark_all_passes_pending(source) if 'Reverse' in opt['Base Name'] and opt['Enabled']: settings.append('--reverse') # Disable for next pass @@ -511,8 +551,8 @@ def menu_settings(source): def run_ddrescue(source, settings): """Run ddrescue pass.""" current_pass = source['Current Pass'] - source[current_pass]['Status'] = 'Working' - update_progress(source) + + # Set pass options if current_pass == 'Pass 1': settings.extend(['--no-trim', '--no-scrape']) elif current_pass == 'Pass 2': @@ -521,9 +561,21 @@ def run_ddrescue(source, settings): elif current_pass == 'Pass 3': # Allow trimming and scraping pass + elif current_pass == 'Done': + clear_screen() + print_warning('Recovery already completed?') + pause('Press Enter to return to main menu...') + return else: raise GenericError("This shouldn't happen?") + # Set device(s) to clone/image + source[current_pass]['Status'] = 'Working' + devs = [source] + if source['Children']: + # Use only selected child devices + devs = source['Children'] + # Set heights ## NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) result = run_program(['tput', 'lines']) @@ -537,33 +589,46 @@ def run_ddrescue(source, settings): '-PF', '#D', 'watch', '--color', '--no-title', '--interval', '300', 'ddrescue-tui-smart-display', source['Dev Path']) + + # Start pass for each selected device + for dev in devs: + if dev[current_pass]['Done']: + # Move to next device + continue + source['Current Device'] = dev['Dev Path'] + dev[current_pass]['Status'] = 'Working' + update_progress(source) + + # Start ddrescue + try: + clear_screen() + print_info('Current dev: {}'.format(dev['Dev Path'])) + ddrescue_proc = popen_program(['./__choose_exit', *settings]) + ddrescue_proc.wait() + except KeyboardInterrupt: + # Catch user abort + pass - # Start ddrescue - try: - clear_screen() - ddrescue_proc = popen_program(['./__choose_exit', *settings]) - ddrescue_proc.wait() - except KeyboardInterrupt: - # Catch user abort - pass - - # Was ddrescue aborted? - return_code = ddrescue_proc.poll() - if return_code is None: - print_warning('Aborted') - source[current_pass]['Status'] = 'Incomplete' - elif return_code: - # i.e. not None and not 0 - print_error('Error(s) encountered, see message above.') - source[current_pass]['Status'] = 'Incomplete' - else: - # Not None and not non-zero int, assuming 0 - mark_pass_complete(source) + # Was ddrescue aborted? + return_code = ddrescue_proc.poll() + if return_code is None: + print_warning('Aborted') + mark_pass_incomplete(source) + break + elif return_code: + # i.e. not None and not 0 + print_error('Error(s) encountered, see message above.') + mark_pass_incomplete(source) + break + else: + # Not None and not non-zero int, assuming 0 + mark_pass_complete(source) # TODO update_progress(source) print_info('Return: {}'.format(return_code)) - pause() + if str(return_code) != '0': + pause() run_program(['tmux', 'kill-pane', '-t', smart_pane]) return From 88c28a3f25a3ce9881edb083da626c451fec901f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 18:26:03 -0600 Subject: [PATCH 034/138] Added auto-continue code * Enabled by default * Based on static thresholds per pass. * Pass 1: 85% * Pass 2: 98% * If using child devices, all must be above the threshold to continue --- .bin/Scripts/functions/ddrescue.py | 67 ++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 0d0a2aa5..b1bd0673 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -12,6 +12,8 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES +AUTO_NEXT_PASS_1_THRESHOLD = 85 +AUTO_NEXT_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { '--binary-prefixes': {'Enabled': True, 'Hidden': True}, '--data-preview': {'Enabled': True, 'Hidden': True}, @@ -122,7 +124,9 @@ def mark_pass_complete(source): # This function was called for this device, mark complete child[current_pass]['Done'] = True # TODO remove test code - child[current_pass]['Status'] = str(12.5 * current_pass_num * 2.75) + from random import randint + status = randint((current_pass_num-1)*10+85, 110) + randint(0, 99) / 100 + child[current_pass]['Status'] = status if not child[current_pass]['Done']: pass_complete_for_all_devs = False @@ -132,7 +136,18 @@ def mark_pass_complete(source): source[current_pass]['Done'] = True # TODO Remove test code - source[current_pass]['Status'] = str(11.078 * current_pass_num * 3) + if source['Children']: + status = 100 + for child in source['Children']: + try: + status = min(status, child[current_pass]['Status']) + except TypeError: + # Force 0% to ensure we won't auto-continue to next pass + status = 0 + else: + from random import randint + status = randint((current_pass_num-1)*10+75, 100) + randint(0, 99) / 100 + source[current_pass]['Status'] = status def mark_pass_incomplete(source): """Mark current pass incomplete.""" @@ -251,7 +266,10 @@ def menu_main(source): # Build menu main_options = [ - {'Base Name': 'Retry (mark non-rescued sectors "non-tried")', 'Enabled': False}, + {'Base Name': 'Auto continue (if recovery % over threshold)', + 'Enabled': True}, + {'Base Name': 'Retry (mark non-rescued sectors "non-tried")', + 'Enabled': False}, {'Base Name': 'Reverse direction', 'Enabled': False}, ] actions =[ @@ -264,12 +282,13 @@ def menu_main(source): # Show menu while True: + current_pass = source['Current Pass'] display_pass = '1 "Initial Read"' - if source['Current Pass'] == 'Pass 2': + if current_pass == 'Pass 2': display_pass = '2 "Trimming bad areas"' - elif source['Current Pass'] == 'Pass 3': + elif current_pass == 'Pass 3': display_pass = '3 "Scraping bad areas"' - elif source['Current Pass'] == 'Done': + elif current_pass == 'Done': display_pass = 'Done' # Update entries for opt in main_options: @@ -302,13 +321,43 @@ def menu_main(source): if 'Retry' in opt['Base Name'] and opt['Enabled']: settings.extend(['--retrim', '--try-again']) mark_all_passes_pending(source) + current_pass = 'Pass 1' if 'Reverse' in opt['Base Name'] and opt['Enabled']: settings.append('--reverse') # Disable for next pass - opt['Enabled'] = False + if 'Auto' not in opt['Base Name']: + opt['Enabled'] = False - # Run pass - run_ddrescue(source, settings) + # Run ddrecue + auto_run = True + while auto_run: + run_ddrescue(source, settings) + auto_run = False + if current_pass == 'Done': + # "Pass Done" i.e. all passes done + break + if not main_options[0]['Enabled']: + # Auto next pass + break + if source[current_pass]['Done']: + try: + recovered = float(source[current_pass]['Status']) + except ValueError: + # Nope + recovered = 'Nope' + pass + else: + if current_pass == 'Pass 1' and recovered > 85: + auto_run = True + elif current_pass == 'Pass 2' and recovered > 98: + auto_run = True + # Update current pass for next iteration + print_info('State:') + print_standard(' Pass #: {}\n Auto: {}\n Recovered: {}'.format( + current_pass, auto_run, recovered)) + pause() + current_pass = source['Current Pass'] + elif selection == 'C': menu_settings(source) elif selection == 'Q': From f2c557f77c2fa3202b4cd7565ff3ee25468a9daa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 20:54:51 -0600 Subject: [PATCH 035/138] Added safety checks for the destination * Dev size / avail space checks * Permission checks * No mount option checks (yet?) --- .bin/Scripts/functions/ddrescue.py | 90 +++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index b1bd0673..f3e3c13b 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -5,6 +5,7 @@ import pathlib import psutil import re import signal +import stat import time from functions.common import * @@ -12,6 +13,7 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES +AUTHORIZED_DEST_FSTYPES = ['ext3', 'ext4', 'xfs'] AUTO_NEXT_PASS_1_THRESHOLD = 85 AUTO_NEXT_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { @@ -33,7 +35,8 @@ USAGE = """ {script_name} clone [source [destination]] # Functions def abort_ddrescue_tui(): - run_program(['losetup', '-D']) + # TODO uncomment line below + # run_program(['losetup', '-D']) abort() def build_outer_panes(source, dest): @@ -65,7 +68,72 @@ def build_outer_panes(source, dest): tmux_splitw('-dhl', '21', 'watch', '--color', '--no-title', '--interval', '1', 'cat', source['Progress Out']) - + +def dest_safety_check(source, dest): + """Verify the destination is appropriate for the source.""" + source_size = source['Details']['size'] + if dest['Is Dir']: + cmd = ['findmnt', '-D', '-J', + '-T', dest['Path']] + result = run_program(cmd) + try: + json_data = json.loads(result.stdout.decode()) + except Exception: + # Welp, let's abort + print_error('Failed to verify destination usability.') + abort_ddrescue_tui() + else: + dest_size = json_data['filesystems'][0]['avail'] + dest['Free Space'] = dest_size + dest['Filesystem'] = json_data['filesystems'][0]['fstype'] + else: + dest_size = dest['Details']['size'] + + # Fix strings before converting to bytes + source_size = re.sub( + r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', source_size.upper()) + dest_size = re.sub( + r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', dest_size.upper()) + + # Convert to bytes and compare size + source_size = convert_to_bytes(source_size) + dest_size = convert_to_bytes(dest_size) + if source['Type'] == 'Image' and dest_size < (source_size * 1.2): + # Imaging: ensure 120% of source size is available + print_error( + 'Not enough free space on destination, refusing to continue.') + print_standard(' Dest {d_size} < Required {s_size}'.format( + d_size = human_readable_size(dest_size), + s_size = human_readable_size(source_size * 1.2))) + abort_ddrescue_tui() + elif source['Type'] == 'Clone' and source_size > dest_size: + # Cloning: ensure dest >= size + print_error('Destination is too small, refusing to continue.') + print_standard(' Dest {d_size} < Source {s_size}'.format( + d_size = human_readable_size(dest_size), + s_size = human_readable_size(source_size))) + abort_ddrescue_tui() + + # Filesystem checks + if source['Type'] == 'Image': + # Filesystem Type + if dest['Filesystem'] not in AUTHORIZED_DEST_FSTYPES: + print_error( + 'Destination filesystem "{}" is not a recommended type.'.format( + dest['Filesystem'])) + if not ask('Proceed anyways? (strongly discouraged by author)'): + abort_ddrescue_tui() + # Read-Write access + ## Note: only checks path permissions, not mount options + ## if the FS is RO then ddrescue will fail later + dest_ok = True + dest_st_mode = os.stat(dest['Path']).st_mode + dest_ok = dest_ok and dest_st_mode & stat.S_IRUSR + dest_ok = dest_ok and dest_st_mode & stat.S_IWUSR + dest_ok = dest_ok and dest_st_mode & stat.S_IXUSR + if not dest_ok: + print_error('Destination is not writable, refusing to continue.') + abort_ddrescue_tui() def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" @@ -180,6 +248,7 @@ def menu_clone(source_path, dest_path): source['Type'] = 'Clone' dest = select_device('destination', dest_path, skip_device = source['Details'], allow_image_file = False) + dest_safety_check(source, dest) # Show selection details show_selection_details(source, dest) @@ -237,6 +306,7 @@ def menu_image(source_path, dest_path): source['Pass 3'] = {'Status': 'Pending', 'Done': False} source['Type'] = 'Image' dest = select_dest_path(dest_path, skip_device=source['Details']) + dest_safety_check(source, dest) # Show selection details show_selection_details(source, dest) @@ -352,10 +422,6 @@ def menu_main(source): elif current_pass == 'Pass 2' and recovered > 98: auto_run = True # Update current pass for next iteration - print_info('State:') - print_standard(' Pass #: {}\n Auto: {}\n Recovered: {}'.format( - current_pass, auto_run, recovered)) - pause() current_pass = source['Current Pass'] elif selection == 'C': @@ -652,7 +718,8 @@ def run_ddrescue(source, settings): try: clear_screen() print_info('Current dev: {}'.format(dev['Dev Path'])) - ddrescue_proc = popen_program(['./__choose_exit', *settings]) + #ddrescue_proc = popen_program(['./__choose_exit', *settings]) + ddrescue_proc = popen_program(['./__exit_ok', *settings]) ddrescue_proc.wait() except KeyboardInterrupt: # Catch user abort @@ -715,7 +782,7 @@ def run_ddrescue(source, settings): #update_progress(source) def select_dest_path(provided_path=None, skip_device={}): - dest = {} + dest = {'Is Dir': True, 'Is Image': False} # Set path if provided_path: @@ -754,7 +821,7 @@ def select_dest_path(provided_path=None, skip_device={}): def select_device(description='device', provided_path=None, skip_device={}, allow_image_file=True): """Select device via provided path or menu, return dev as dict.""" - dev = {'Is Image': False} + dev = {'Is Dir': False, 'Is Image': False} # Set path if provided_path: @@ -777,6 +844,8 @@ def select_device(description='device', provided_path=None, # Get device details dev['Details'] = get_device_details(dev['Dev Path']) + if 'Children' not in dev: + dev['Children'] = {} # Check for parent device(s) while dev['Details']['pkname']: @@ -877,6 +946,9 @@ def show_selection_details(source, dest): else: print_success('Destination path') print_standard(dest['Path']) + print_info('{:<8}{}'.format('FREE', 'FSTYPE')) + print_standard('{:<8}{}'.format( + dest['Free Space'], dest['Filesystem'])) print_standard(' ') def show_usage(script_name): From e5ce254e8b3b6ef92d680a70be9d251bb636e46e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 22:02:17 -0600 Subject: [PATCH 036/138] Verify destination FS is mounted read-write --- .bin/Scripts/functions/ddrescue.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index f3e3c13b..9f39b682 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -73,7 +73,8 @@ def dest_safety_check(source, dest): """Verify the destination is appropriate for the source.""" source_size = source['Details']['size'] if dest['Is Dir']: - cmd = ['findmnt', '-D', '-J', + cmd = ['findmnt', '-J', + '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', '-T', dest['Path']] result = run_program(cmd) try: @@ -86,6 +87,7 @@ def dest_safety_check(source, dest): dest_size = json_data['filesystems'][0]['avail'] dest['Free Space'] = dest_size dest['Filesystem'] = json_data['filesystems'][0]['fstype'] + dest['Mount options'] = json_data['filesystems'][0]['options'] else: dest_size = dest['Details']['size'] @@ -114,18 +116,16 @@ def dest_safety_check(source, dest): s_size = human_readable_size(source_size))) abort_ddrescue_tui() - # Filesystem checks + # Imaging specific checks if source['Type'] == 'Image': # Filesystem Type if dest['Filesystem'] not in AUTHORIZED_DEST_FSTYPES: print_error( 'Destination filesystem "{}" is not a recommended type.'.format( dest['Filesystem'])) - if not ask('Proceed anyways? (strongly discouraged by author)'): + if not ask('Proceed anyways? (Strongly discouraged)'): abort_ddrescue_tui() # Read-Write access - ## Note: only checks path permissions, not mount options - ## if the FS is RO then ddrescue will fail later dest_ok = True dest_st_mode = os.stat(dest['Path']).st_mode dest_ok = dest_ok and dest_st_mode & stat.S_IRUSR @@ -134,6 +134,12 @@ def dest_safety_check(source, dest): if not dest_ok: print_error('Destination is not writable, refusing to continue.') abort_ddrescue_tui() + + # Mount options check + if 'rw' not in dest['Mount options'].split(','): + print_error( + 'Destination is not mounted read-write, refusing to continue.') + abort_ddrescue_tui() def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" From 19dcc87950fe05b4eee3e2e5711838a7b0e80b66 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 22:09:47 -0600 Subject: [PATCH 037/138] Pause when showing usage --- .bin/Scripts/functions/ddrescue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 9f39b682..0e24d7b8 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -960,6 +960,7 @@ def show_selection_details(source, dest): def show_usage(script_name): print_info('Usage:') print_standard(USAGE.format(script_name=script_name)) + pause() def tmux_splitw(*args): """Run tmux split-window command and return output as str.""" From 646e1a3764f57b36ca4c3c9ec0b1acf94fdb0860 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 22:17:32 -0600 Subject: [PATCH 038/138] Show list of authorized fstypes with error --- .bin/Scripts/functions/ddrescue.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 0e24d7b8..887ab0d1 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -123,6 +123,9 @@ def dest_safety_check(source, dest): print_error( 'Destination filesystem "{}" is not a recommended type.'.format( dest['Filesystem'])) + print_info('Authorized types are: {}'.format( + ' / '.join(AUTHORIZED_DEST_FSTYPES).upper())) + print_standard(' ') if not ask('Proceed anyways? (Strongly discouraged)'): abort_ddrescue_tui() # Read-Write access From e640caee746389e49643fdae72de9b3c1ba8b801 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 23:19:42 -0600 Subject: [PATCH 039/138] Add dest image/map path sections --- .bin/Scripts/functions/ddrescue.py | 46 ++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 887ab0d1..2471f61c 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -261,6 +261,7 @@ def menu_clone(source_path, dest_path): # Show selection details show_selection_details(source, dest) + set_dest_image_paths(source, dest) # Confirm if not ask('Proceed with clone?'): @@ -326,6 +327,7 @@ def menu_image(source_path, dest_path): # Select child device(s) source['Children'] = menu_select_children(source) + set_dest_image_paths(source, dest) # Main menu build_outer_panes(source, dest) @@ -444,13 +446,14 @@ def menu_select_children(source): 'Base Name': '{:<14}(Whole device)'.format(source['Dev Path']), 'Path': source['Dev Path'], 'Selected': True}] - for child in source['Details'].get('children', []): + for c_details in source['Details'].get('children', []): dev_options.append({ 'Base Name': '{:<14}({:>6} {})'.format( - child['name'], - child['size'], - child['fstype'] if child['fstype'] else 'Unknown'), - 'Path': child['name'], + c_details['name'], + c_details['size'], + c_details['fstype'] if c_details['fstype'] else 'Unknown'), + 'Details': c_details, + 'Path': c_details['name'], 'Selected': False}) actions = [ {'Name': 'Proceed', 'Letter': 'P'}, @@ -493,6 +496,7 @@ def menu_select_children(source): # Check selection selected_children = [{ + 'Details': d['Details'], 'Dev Path': d['Path'], 'Pass 1': {'Status': 'Pending', 'Done': False}, 'Pass 2': {'Status': 'Pending', 'Done': False}, @@ -887,6 +891,38 @@ def select_device(description='device', provided_path=None, return dev +def set_dest_image_paths(source, dest): + """Set destination image path for source and any child devices.""" + if source['Type'] == 'Clone': + base = '{pwd}/Clone_{Date-Time}'.format( + pwd = os.path.realpath(global_vars['Env']['PWD']), + **global_vars) + else: + base = '{Path}/{size}_{model}'.format( + size = source['Details']['size'], + model = source['Details'].get('model', 'Unknown'), + **dest) + source['Dest Paths'] = { + 'Image': '{}.dd'.format(base), + 'Map': '{}.map'.format(base)} + + # Child devices + for child in source['Children']: + p_label = '' + if child['Details']['label']: + p_label = '_{}'.format(child['Details']['label']) + base = '{Path}/{size}_{model}_{p_num}_{p_size}{p_label}'.format( + size = source['Details']['size'], + model = source['Details'].get('model', 'Unknown'), + p_num = child['Details']['name'].replace( + child['Details']['pkname'], ''), + p_size = child['Details']['size'], + p_label = p_label, + **dest) + child['Dest Paths'] = { + 'Image': '{}.dd'.format(base), + 'Map': '{}.map'.format(base)} + def setup_loopback_device(source_path): """Setup a loopback device for source_path, returns dev_path as str.""" cmd = ( From f84413f1a9aa73289f24e7bbaef0c2a9e8bc7b85 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 23:36:15 -0600 Subject: [PATCH 040/138] Fix SMART not available warning --- .bin/Scripts/ddrescue-tui-smart-display | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display index a53a2dca..285229d6 100755 --- a/.bin/Scripts/ddrescue-tui-smart-display +++ b/.bin/Scripts/ddrescue-tui-smart-display @@ -22,7 +22,7 @@ if __name__ == '__main__': # Warn if SMART unavailable if dev_path not in devs: print_error('SMART data not available') - input('') + exit_script() # Initial screen dev = devs[dev_path] From d09664bb7d6c79601bb6287514cef4066baa4e10 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 23:37:02 -0600 Subject: [PATCH 041/138] Misc cleanup --- .bin/Scripts/functions/ddrescue.py | 47 ++++-------------------------- 1 file changed, 6 insertions(+), 41 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 2471f61c..ba0bd308 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -35,8 +35,7 @@ USAGE = """ {script_name} clone [source [destination]] # Functions def abort_ddrescue_tui(): - # TODO uncomment line below - # run_program(['losetup', '-D']) + run_program(['losetup', '-D']) abort() def build_outer_panes(source, dest): @@ -731,8 +730,8 @@ def run_ddrescue(source, settings): try: clear_screen() print_info('Current dev: {}'.format(dev['Dev Path'])) - #ddrescue_proc = popen_program(['./__choose_exit', *settings]) - ddrescue_proc = popen_program(['./__exit_ok', *settings]) + ddrescue_proc = popen_program(['./__choose_exit', *settings]) + #ddrescue_proc = popen_program(['./__exit_ok', *settings]) ddrescue_proc.wait() except KeyboardInterrupt: # Catch user abort @@ -753,46 +752,12 @@ def run_ddrescue(source, settings): # Not None and not non-zero int, assuming 0 mark_pass_complete(source) - # TODO + # Cleanup update_progress(source) - print_info('Return: {}'.format(return_code)) if str(return_code) != '0': - pause() + # Pause on errors + pause('Press Enter to return to main menu... ') run_program(['tmux', 'kill-pane', '-t', smart_pane]) - return - - ##TODO - #print_success('GO!') - #if source['Pass 3']['Done']: - # # Go to results - # print_success('Done?') - #elif source['Pass 2']['Done']: - # # In pass 3 - # print_error('Pass 3') - # print_standard(str(settings)) - # source['Pass 3']['Done'] = True - # source['Pass 3']['Status'] = '99.99' - #elif source['Pass 1']['Done']: - # # In pass 2 - # print_warning('Pass 2') - # settings.append('--no-scrape') - # print_standard(str(settings)) - # source['Pass 2']['Done'] = True - # source['Pass 2']['Status'] = '98.1415' - #else: - # # In pass 1 - # print_info('Pass 1') - # settings.extend(['--no-trim', '--no-scrape']) - # print_standard(str(settings)) - # status = source['Pass 1']['Status'] - # if status == 'Pending': - # source['Pass 1']['Status'] = '78.6623' - # elif float(status) < 80: - # source['Pass 1']['Status'] = '86.1102' - # elif float(status) < 95: - # source['Pass 1']['Status'] = '97.77' - # source['Pass 1']['Done'] = True - #update_progress(source) def select_dest_path(provided_path=None, skip_device={}): dest = {'Is Dir': True, 'Is Image': False} From 7597394d61782b58f826c8cf94696cb6194194be Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 18 Jul 2018 23:53:05 -0600 Subject: [PATCH 042/138] Build real ddrescue cmd for Cloning or Imaging * --force is only used for cloning --- .bin/Scripts/functions/ddrescue.py | 37 +++++++++++++++++++----------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index ba0bd308..d3ef9e23 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -269,7 +269,7 @@ def menu_clone(source_path, dest_path): # Main menu build_outer_panes(source, dest) - menu_main(source) + menu_main(source, dest) # Done run_program(['losetup', '-D']) @@ -330,14 +330,14 @@ def menu_image(source_path, dest_path): # Main menu build_outer_panes(source, dest) - menu_main(source) + menu_main(source, dest) # Done run_program(['losetup', '-D']) run_program(['tmux', 'kill-window']) exit_script() -def menu_main(source): +def menu_main(source, dest): """Main menu is used to set ddrescue settings.""" title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) @@ -411,7 +411,7 @@ def menu_main(source): # Run ddrecue auto_run = True while auto_run: - run_ddrescue(source, settings) + run_ddrescue(source, dest, settings) auto_run = False if current_pass == 'Done': # "Pass Done" i.e. all passes done @@ -675,7 +675,7 @@ def menu_settings(source): elif selection == 'M': break -def run_ddrescue(source, settings): +def run_ddrescue(source, dest, settings): """Run ddrescue pass.""" current_pass = source['Current Pass'] @@ -698,10 +698,10 @@ def run_ddrescue(source, settings): # Set device(s) to clone/image source[current_pass]['Status'] = 'Working' - devs = [source] + source_devs = [source] if source['Children']: # Use only selected child devices - devs = source['Children'] + source_devs = source['Children'] # Set heights ## NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) @@ -718,20 +718,29 @@ def run_ddrescue(source, settings): 'ddrescue-tui-smart-display', source['Dev Path']) # Start pass for each selected device - for dev in devs: - if dev[current_pass]['Done']: + for s_dev in source_devs: + if s_dev[current_pass]['Done']: # Move to next device continue - source['Current Device'] = dev['Dev Path'] - dev[current_pass]['Status'] = 'Working' + source['Current Device'] = s_dev['Dev Path'] + s_dev[current_pass]['Status'] = 'Working' update_progress(source) + # Set ddrescue cmd + if source['Type'] == 'Clone': + cmd = ['ddrescue', *settings, '--force', + s_dev['Dev Path'], dest['Dev Path'], s_dev['Dest Paths']['Map']] + else: + cmd = ['ddrescue', *settings, + s_dev['Dev Path'], s_dev['Dest Paths']['Image'], + s_dev['Dest Paths']['Map']] + # Start ddrescue try: clear_screen() - print_info('Current dev: {}'.format(dev['Dev Path'])) - ddrescue_proc = popen_program(['./__choose_exit', *settings]) - #ddrescue_proc = popen_program(['./__exit_ok', *settings]) + print_info('Current dev: {}'.format(s_dev['Dev Path'])) + ddrescue_proc = popen_program(['./__choose_exit', *cmd]) + #ddrescue_proc = popen_program(['./__exit_ok', *cmd]) ddrescue_proc.wait() except KeyboardInterrupt: # Catch user abort From 016f87b76c3712e2cd57a49e474fd3b9c2e3ff8d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 19 Jul 2018 00:43:41 -0600 Subject: [PATCH 043/138] Don't hide source dev when selecting dest dev * Disable the entry instead * It's more clear what's being done --- .bin/Scripts/functions/ddrescue.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index d3ef9e23..f04da365 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -524,9 +524,8 @@ def menu_select_device(title='Which device?', skip_device={}): # Build menu dev_options = [] for dev in json_data['blockdevices']: - # Skip if dev is in skip_names - if dev['name'] in skip_names or dev['pkname'] in skip_names: - continue + # Disable dev if in skip_names + disable_dev = dev['name'] in skip_names or dev['pkname'] in skip_names # Append non-matching devices dev_options.append({ @@ -536,7 +535,8 @@ def menu_select_device(title='Which device?', skip_device={}): size = dev['size'] if dev['size'] else '', model = dev['model'] if dev['model'] else '', serial = dev['serial'] if dev['serial'] else ''), - 'Path': dev['name']}) + 'Path': dev['name'], + 'Disabled': disable_dev}) dev_options = sorted(dev_options, key=itemgetter('Name')) if not dev_options: print_error('No devices available.') @@ -547,7 +547,8 @@ def menu_select_device(title='Which device?', skip_device={}): selection = menu_select( title = title, main_entries = dev_options, - action_entries = actions) + action_entries = actions, + disabled_label = 'SOURCE DEVICE') if selection.isnumeric(): return dev_options[int(selection)-1]['Path'] @@ -741,6 +742,7 @@ def run_ddrescue(source, dest, settings): print_info('Current dev: {}'.format(s_dev['Dev Path'])) ddrescue_proc = popen_program(['./__choose_exit', *cmd]) #ddrescue_proc = popen_program(['./__exit_ok', *cmd]) + #ddrescue_proc = popen_program(cmd) ddrescue_proc.wait() except KeyboardInterrupt: # Catch user abort From 5f4598814abfeca13382f91f6dce8c33c3d6e258 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 19 Jul 2018 00:45:04 -0600 Subject: [PATCH 044/138] Clear screen before printing abort warning * Otherwise the "Abort" string is in the middle of the ddrescue output * Also added secondary return code to be treated as a user abort --- .bin/Scripts/functions/ddrescue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index f04da365..5ef7b4cf 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -750,7 +750,8 @@ def run_ddrescue(source, dest, settings): # Was ddrescue aborted? return_code = ddrescue_proc.poll() - if return_code is None: + if return_code is None or return_code is 130: + clear_screen() print_warning('Aborted') mark_pass_incomplete(source) break From 658229337071030c760a6f639f2e0493269311e8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 19 Jul 2018 01:34:14 -0600 Subject: [PATCH 045/138] Initial resume code -- Needs testing --- .bin/Scripts/functions/ddrescue.py | 37 ++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 5ef7b4cf..6faa7a2e 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -868,12 +868,45 @@ def select_device(description='device', provided_path=None, return dev +def check_dest_paths(source): + """Check for image and/or map file and alert user about details.""" + dd_image_exists = os.path.exists(source['Dest Paths']['Image']) + map_exists = os.path.exists(source['Dest Paths']['Map']) + if 'Clone' in source['Dest Paths']['Map']: + if map_exists: + # We're cloning and a matching map file was detected + if ask('Matching map file detected, resume recovery?'): + print_success('TODO: ...') + exit_script() + else: + abort_ddrescue_tui() + else: + # We're imaging + if dd_image_exists and not map_exists: + # Refuce to resume without map file + print_error('Destination image "{}" exists but map missing.'.format( + source['Dest Paths']['Image'])) + abort_ddrescue_tui() + elif not dd_image_exists and map_exists: + # Can't resume without dd_image + print_error('Destination image missing but map "{}" exists.'.format( + source['Dest Paths']['Map'])) + abort_ddrescue_tui() + elif dd_image_exists and map_exists: + # Matching dd_image and map file were detected + if ask('Matching image and map file detected, resume recovery?'): + print_success('TODO: ...') + exit_script() + else: + abort_ddrescue_tui() + def set_dest_image_paths(source, dest): """Set destination image path for source and any child devices.""" if source['Type'] == 'Clone': - base = '{pwd}/Clone_{Date-Time}'.format( + base = '{pwd}/Clone_{size}_{model}'.format( pwd = os.path.realpath(global_vars['Env']['PWD']), - **global_vars) + size = source['Details']['size'], + model = source['Details'].get('model', 'Unknown')) else: base = '{Path}/{size}_{model}'.format( size = source['Details']['size'], From 93c9b206d935c9d3d9ab2a3a2d9df8a1c5fb251a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 20 Jul 2018 13:56:49 -0600 Subject: [PATCH 046/138] Added ddrescue-tui aliases wkclone and wkimage --- .linux_items/include/airootfs/etc/skel/.aliases | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.linux_items/include/airootfs/etc/skel/.aliases b/.linux_items/include/airootfs/etc/skel/.aliases index fab61bb1..852f3803 100644 --- a/.linux_items/include/airootfs/etc/skel/.aliases +++ b/.linux_items/include/airootfs/etc/skel/.aliases @@ -35,3 +35,5 @@ alias srsz='sudo rsync -avhzPS --stats --exclude-from="$HOME/.rsync_exclusions"' alias testdisk='sudo testdisk' alias umount='sudo umount' alias unmount='sudo umount' +alias wkclone='sudo ddrescue-tui clone' +alias wkimage='sudo ddrescue-tui image' From 281607f3e4deb74553776201cdf2c690d0a3455d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 20 Jul 2018 13:57:55 -0600 Subject: [PATCH 047/138] Adjusted confirm/show details order --- .bin/Scripts/functions/ddrescue.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 6faa7a2e..a35f72e7 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -261,6 +261,7 @@ def menu_clone(source_path, dest_path): # Show selection details show_selection_details(source, dest) set_dest_image_paths(source, dest) + check_dest_paths(source) # Confirm if not ask('Proceed with clone?'): @@ -316,6 +317,11 @@ def menu_image(source_path, dest_path): source['Type'] = 'Image' dest = select_dest_path(dest_path, skip_device=source['Details']) dest_safety_check(source, dest) + + # Select child device(s) + source['Children'] = menu_select_children(source) + set_dest_image_paths(source, dest) + check_dest_paths(source) # Show selection details show_selection_details(source, dest) @@ -323,10 +329,6 @@ def menu_image(source_path, dest_path): # Confirm if not ask('Proceed with imaging?'): abort_ddrescue_tui() - - # Select child device(s) - source['Children'] = menu_select_children(source) - set_dest_image_paths(source, dest) # Main menu build_outer_panes(source, dest) @@ -875,10 +877,7 @@ def check_dest_paths(source): if 'Clone' in source['Dest Paths']['Map']: if map_exists: # We're cloning and a matching map file was detected - if ask('Matching map file detected, resume recovery?'): - print_success('TODO: ...') - exit_script() - else: + if not ask('Matching map file detected, resume recovery?'): abort_ddrescue_tui() else: # We're imaging From 37734e65bfae5fb771f55e62c5106ba2ed8c0e40 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 20 Jul 2018 17:53:09 -0600 Subject: [PATCH 048/138] Bugfix: Paths are now relative to the current dir * They were relative to the script's dir before --- .bin/Scripts/ddrescue-tui-menu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index ea8c1a55..a8205ded 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -6,8 +6,8 @@ import os import sys # Init -os.chdir(os.path.dirname(os.path.realpath(__file__))) -sys.path.append(os.getcwd()) +sys.path.append(os.path.dirname(os.path.realpath(__file__))) + from functions.ddrescue import * from functions.hw_diags import * init_global_vars() From d7dfb34b0206655d5413f252c8fb8e73802443f8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 20 Jul 2018 18:06:59 -0600 Subject: [PATCH 049/138] Resume function working for imaging cases --- .bin/Scripts/functions/ddrescue.py | 46 +++++++++++++++++++----------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index a35f72e7..81cc9777 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -880,24 +880,36 @@ def check_dest_paths(source): if not ask('Matching map file detected, resume recovery?'): abort_ddrescue_tui() else: - # We're imaging - if dd_image_exists and not map_exists: - # Refuce to resume without map file - print_error('Destination image "{}" exists but map missing.'.format( - source['Dest Paths']['Image'])) + abort_imaging = False + resume_files_exist = False + source_devs = [source] + if source['Children']: + source_devs = source['Children'] + for dev in source_devs: + # We're imaging + dd_image_exists = os.path.exists(dev['Dest Paths']['Image']) + map_exists = os.path.exists(dev['Dest Paths']['Map']) + if dd_image_exists and not map_exists: + # Refuce to resume without map file + i = dev['Dest Paths']['Image'] + i = i[i.rfind('/')+1:] + print_error( + 'Detected image "{}" but not the matching map'.format(i)) + abort_imaging = True + elif not dd_image_exists and map_exists: + # Can't resume without dd_image + m = dev['Dest Paths']['Map'] + m = m[m.rfind('/')+1:] + print_error( + 'Detected map "{}" but not the matching image'.format(m)) + abort_imaging = True + elif dd_image_exists and map_exists: + # Matching dd_image and map file were detected + resume_files_exist = True + p = 'Matching image and map file{} detected, resume recovery?'.format( + 's' if len(source_devs) > 1 else '') + if abort_imaging or (resume_files_exist and not ask(p)): abort_ddrescue_tui() - elif not dd_image_exists and map_exists: - # Can't resume without dd_image - print_error('Destination image missing but map "{}" exists.'.format( - source['Dest Paths']['Map'])) - abort_ddrescue_tui() - elif dd_image_exists and map_exists: - # Matching dd_image and map file were detected - if ask('Matching image and map file detected, resume recovery?'): - print_success('TODO: ...') - exit_script() - else: - abort_ddrescue_tui() def set_dest_image_paths(source, dest): """Set destination image path for source and any child devices.""" From 6b28444c36fd6a173b7287267dbd534971ff398d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 21 Jul 2018 20:31:39 -0600 Subject: [PATCH 050/138] Fix function order --- .bin/Scripts/functions/ddrescue.py | 82 +++++++++++++++--------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 81cc9777..718f9262 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -68,6 +68,47 @@ def build_outer_panes(source, dest): 'watch', '--color', '--no-title', '--interval', '1', 'cat', source['Progress Out']) +def check_dest_paths(source): + """Check for image and/or map file and alert user about details.""" + dd_image_exists = os.path.exists(source['Dest Paths']['Image']) + map_exists = os.path.exists(source['Dest Paths']['Map']) + if 'Clone' in source['Dest Paths']['Map']: + if map_exists: + # We're cloning and a matching map file was detected + if not ask('Matching map file detected, resume recovery?'): + abort_ddrescue_tui() + else: + abort_imaging = False + resume_files_exist = False + source_devs = [source] + if source['Children']: + source_devs = source['Children'] + for dev in source_devs: + # We're imaging + dd_image_exists = os.path.exists(dev['Dest Paths']['Image']) + map_exists = os.path.exists(dev['Dest Paths']['Map']) + if dd_image_exists and not map_exists: + # Refuce to resume without map file + i = dev['Dest Paths']['Image'] + i = i[i.rfind('/')+1:] + print_error( + 'Detected image "{}" but not the matching map'.format(i)) + abort_imaging = True + elif not dd_image_exists and map_exists: + # Can't resume without dd_image + m = dev['Dest Paths']['Map'] + m = m[m.rfind('/')+1:] + print_error( + 'Detected map "{}" but not the matching image'.format(m)) + abort_imaging = True + elif dd_image_exists and map_exists: + # Matching dd_image and map file were detected + resume_files_exist = True + p = 'Matching image and map file{} detected, resume recovery?'.format( + 's' if len(source_devs) > 1 else '') + if abort_imaging or (resume_files_exist and not ask(p)): + abort_ddrescue_tui() + def dest_safety_check(source, dest): """Verify the destination is appropriate for the source.""" source_size = source['Details']['size'] @@ -870,47 +911,6 @@ def select_device(description='device', provided_path=None, return dev -def check_dest_paths(source): - """Check for image and/or map file and alert user about details.""" - dd_image_exists = os.path.exists(source['Dest Paths']['Image']) - map_exists = os.path.exists(source['Dest Paths']['Map']) - if 'Clone' in source['Dest Paths']['Map']: - if map_exists: - # We're cloning and a matching map file was detected - if not ask('Matching map file detected, resume recovery?'): - abort_ddrescue_tui() - else: - abort_imaging = False - resume_files_exist = False - source_devs = [source] - if source['Children']: - source_devs = source['Children'] - for dev in source_devs: - # We're imaging - dd_image_exists = os.path.exists(dev['Dest Paths']['Image']) - map_exists = os.path.exists(dev['Dest Paths']['Map']) - if dd_image_exists and not map_exists: - # Refuce to resume without map file - i = dev['Dest Paths']['Image'] - i = i[i.rfind('/')+1:] - print_error( - 'Detected image "{}" but not the matching map'.format(i)) - abort_imaging = True - elif not dd_image_exists and map_exists: - # Can't resume without dd_image - m = dev['Dest Paths']['Map'] - m = m[m.rfind('/')+1:] - print_error( - 'Detected map "{}" but not the matching image'.format(m)) - abort_imaging = True - elif dd_image_exists and map_exists: - # Matching dd_image and map file were detected - resume_files_exist = True - p = 'Matching image and map file{} detected, resume recovery?'.format( - 's' if len(source_devs) > 1 else '') - if abort_imaging or (resume_files_exist and not ask(p)): - abort_ddrescue_tui() - def set_dest_image_paths(source, dest): """Set destination image path for source and any child devices.""" if source['Type'] == 'Clone': From 9e48c1d1a63d8e02a6dd33d0e80d57299e5291f4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Jul 2018 02:03:04 -0600 Subject: [PATCH 051/138] Read data from MAP files (Big update) * Added read_map_file() which uses ddrescuelog to create dict of current state * Added --test-mode= option to expert menu * Add size (in bytes) to all devs * Allows to calculate real total percent recovered * Detect 100% completion via ddrescuelog -D * Moved mark_complete / mark_incomplete code to update_progress() * Update progress every 30s during ddrescue passes * Fixed auto_run logic --- .bin/Scripts/functions/ddrescue.py | 252 +++++++++++++++++++---------- 1 file changed, 169 insertions(+), 83 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 718f9262..5656c212 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -25,9 +25,12 @@ DDRESCUE_SETTINGS = { '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, '--reopen-on-error': {'Enabled': True}, '--retry-passes=': {'Enabled': True, 'Value': '0'}, + '--test-mode=': {'Enabled': True, 'Value': 'some.map'}, '--timeout=': {'Enabled': True, 'Value': '5m'}, '-vvvv': {'Enabled': True, 'Hidden': True}, } +REGEX_MAP_DATA = re.compile(r'^\s*(?P\S+):.*\(\s*(?P\d+\.?\d*)%.*') +REGEX_MAP_STATUS = re.compile(r'.*current status:\s+(?P.*)') USAGE = """ {script_name} clone [source [destination]] {script_name} image [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) @@ -131,15 +134,9 @@ def dest_safety_check(source, dest): else: dest_size = dest['Details']['size'] - # Fix strings before converting to bytes - source_size = re.sub( - r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', source_size.upper()) - dest_size = re.sub( - r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', dest_size.upper()) - # Convert to bytes and compare size - source_size = convert_to_bytes(source_size) - dest_size = convert_to_bytes(dest_size) + source_size = get_device_size_in_bytes(source_size) + dest_size = get_device_size_in_bytes(dest_size) if source['Type'] == 'Image' and dest_size < (source_size * 1.2): # Imaging: ensure 120% of source size is available print_error( @@ -202,6 +199,23 @@ def get_device_details(dev_path): # Just return the first device (there should only be one) return json_data['blockdevices'][0] +def get_device_size_in_bytes(s): + """Convert size string from lsblk string to bytes, returns int.""" + s = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', s, re.IGNORECASE) + return convert_to_bytes(s) + +def get_recovery_scope_size(source): + """Calculate total size of selected dev(s).""" + source['Total Size'] = 0 + if source['Children']: + for child in source['Children']: + child['Size'] = get_device_size_in_bytes(child['Details']['size']) + source['Total Size'] += child['Size'] + else: + # Whole dev + source['Size'] = get_device_size_in_bytes(source['Details']['size']) + source['Total Size'] = source['Size'] + def get_status_color(s, t_success=99, t_warn=90): """Get color based on status, returns str.""" color = COLORS['CLEAR'] @@ -214,7 +228,7 @@ def get_status_color(s, t_success=99, t_warn=90): if s in ('Pending',): color = COLORS['CLEAR'] - elif s in ('Working',): + elif s in ('Skipped', 'Unknown', 'Working'): color = COLORS['YELLOW'] elif p_recovered >= t_success: color = COLORS['GREEN'] @@ -224,57 +238,6 @@ def get_status_color(s, t_success=99, t_warn=90): color = COLORS['RED'] return color -def mark_pass_complete(source): - """Mark current pass complete for device, and overall if applicable.""" - current_pass = source['Current Pass'] - current_pass_num = int(current_pass[-1:]) - next_pass_num = current_pass_num + 1 - if 1 <= next_pass_num <= 3: - next_pass = 'Pass {}'.format(next_pass_num) - else: - next_pass = 'Done' - - # Check children progress - pass_complete_for_all_devs = True - for child in source['Children']: - if child['Dev Path'] == source['Current Device']: - # This function was called for this device, mark complete - child[current_pass]['Done'] = True - # TODO remove test code - from random import randint - status = randint((current_pass_num-1)*10+85, 110) + randint(0, 99) / 100 - child[current_pass]['Status'] = status - if not child[current_pass]['Done']: - pass_complete_for_all_devs = False - - # Update source vars - if pass_complete_for_all_devs: - source['Current Pass'] = next_pass - source[current_pass]['Done'] = True - - # TODO Remove test code - if source['Children']: - status = 100 - for child in source['Children']: - try: - status = min(status, child[current_pass]['Status']) - except TypeError: - # Force 0% to ensure we won't auto-continue to next pass - status = 0 - else: - from random import randint - status = randint((current_pass_num-1)*10+75, 100) + randint(0, 99) / 100 - source[current_pass]['Status'] = status - -def mark_pass_incomplete(source): - """Mark current pass incomplete.""" - current_pass = source['Current Pass'] - source[current_pass]['Status'] = 'Incomplete' - for child in source['Children']: - if child['Dev Path'] == source['Current Device']: - # This function was called for this device, mark incomplete - child[current_pass]['Status'] = 'Incomplete' - def mark_all_passes_pending(source): """Mark all devs and passes as pending in preparation for retry.""" source['Current Pass'] = 'Pass 1' @@ -294,6 +257,8 @@ def menu_clone(source_path, dest_path): source['Pass 1'] = {'Status': 'Pending', 'Done': False} source['Pass 2'] = {'Status': 'Pending', 'Done': False} source['Pass 3'] = {'Status': 'Pending', 'Done': False} + source['Recovered Size'] = 0, + source['Total Size'] = 0, source['Type'] = 'Clone' dest = select_device('destination', dest_path, skip_device = source['Details'], allow_image_file = False) @@ -303,6 +268,7 @@ def menu_clone(source_path, dest_path): show_selection_details(source, dest) set_dest_image_paths(source, dest) check_dest_paths(source) + get_recovery_scope_size(source) # Confirm if not ask('Proceed with clone?'): @@ -355,6 +321,8 @@ def menu_image(source_path, dest_path): source['Pass 1'] = {'Status': 'Pending', 'Done': False} source['Pass 2'] = {'Status': 'Pending', 'Done': False} source['Pass 3'] = {'Status': 'Pending', 'Done': False} + source['Recovered Size'] = 0, + source['Total Size'] = 0, source['Type'] = 'Image' dest = select_dest_path(dest_path, skip_device=source['Details']) dest_safety_check(source, dest) @@ -363,6 +331,7 @@ def menu_image(source_path, dest_path): source['Children'] = menu_select_children(source) set_dest_image_paths(source, dest) check_dest_paths(source) + get_recovery_scope_size(source) # Show selection details show_selection_details(source, dest) @@ -441,6 +410,8 @@ def menu_main(source, dest): if 'Value' in v: settings.append(v['Value']) for opt in main_options: + if 'Auto' in opt['Base Name']: + auto_run = opt['Enabled'] if 'Retry' in opt['Base Name'] and opt['Enabled']: settings.extend(['--retrim', '--try-again']) mark_all_passes_pending(source) @@ -452,10 +423,10 @@ def menu_main(source, dest): opt['Enabled'] = False # Run ddrecue - auto_run = True - while auto_run: + first_run = True + while auto_run or first_run: + first_run = False run_ddrescue(source, dest, settings) - auto_run = False if current_pass == 'Done': # "Pass Done" i.e. all passes done break @@ -463,17 +434,13 @@ def menu_main(source, dest): # Auto next pass break if source[current_pass]['Done']: - try: - recovered = float(source[current_pass]['Status']) - except ValueError: - # Nope - recovered = 'Nope' - pass - else: - if current_pass == 'Pass 1' and recovered > 85: - auto_run = True - elif current_pass == 'Pass 2' and recovered > 98: - auto_run = True + min_status = source[current_pass]['Min Status'] + if (current_pass == 'Pass 1' + and min_status < AUTO_NEXT_PASS_1_THRESHOLD): + auto_run = False + elif (current_pass == 'Pass 2' + and min_status < AUTO_NEXT_PASS_2_THRESHOLD): + auto_run = False # Update current pass for next iteration current_pass = source['Current Pass'] @@ -719,9 +686,42 @@ def menu_settings(source): elif selection == 'M': break +def read_map_file(map_path): + """Read map file with ddrescuelog and return data as dict.""" + map_data = {} + try: + result = run_program(['ddrescuelog', '-t', map_path]) + except subprocess.CalledProcessError: + print_error('Failed to read map data') + abort_ddrescue_tui() + + # Parse output + for line in result.stdout.decode().splitlines(): + m = REGEX_MAP_DATA.match(line.strip()) + if m: + try: + map_data[m.group('key')] = float(m.group('value')) + except ValueError: + print_error('Failed to read map data') + abort_ddrescue_tui() + m = REGEX_MAP_STATUS.match(line.strip()) + if m: + map_data['pass completed'] = bool(m.group('status') == 'finished') + + # Check if 100% done + try: + run_program(['ddrescuelog', '-D', map_path]) + except subprocess.CalledProcessError: + map_data['full recovery'] = False + else: + map_data['full recovery'] = True + + return map_data + def run_ddrescue(source, dest, settings): """Run ddrescue pass.""" current_pass = source['Current Pass'] + return_code = None # Set pass options if current_pass == 'Pass 1': @@ -786,7 +786,12 @@ def run_ddrescue(source, dest, settings): ddrescue_proc = popen_program(['./__choose_exit', *cmd]) #ddrescue_proc = popen_program(['./__exit_ok', *cmd]) #ddrescue_proc = popen_program(cmd) - ddrescue_proc.wait() + while True: + try: + ddrescue_proc.wait(timeout=30) + break + except subprocess.TimeoutExpired: + update_progress(source) except KeyboardInterrupt: # Catch user abort pass @@ -796,19 +801,14 @@ def run_ddrescue(source, dest, settings): if return_code is None or return_code is 130: clear_screen() print_warning('Aborted') - mark_pass_incomplete(source) break elif return_code: # i.e. not None and not 0 print_error('Error(s) encountered, see message above.') - mark_pass_incomplete(source) break - else: - # Not None and not non-zero int, assuming 0 - mark_pass_complete(source) # Cleanup - update_progress(source) + update_progress(source, end_run=True) if str(return_code) != '0': # Pause on errors pause('Press Enter to return to main menu... ') @@ -878,7 +878,7 @@ def select_device(description='device', provided_path=None, # Get device details dev['Details'] = get_device_details(dev['Dev Path']) if 'Children' not in dev: - dev['Children'] = {} + dev['Children'] = [] # Check for parent device(s) while dev['Details']['pkname']: @@ -1028,8 +1028,26 @@ def tmux_splitw(*args): result = run_program(cmd) return result.stdout.decode().strip() -def update_progress(source): +def update_progress(source, end_run=False): """Update progress file.""" + current_pass = source['Current Pass'] + pass_complete_for_all_devs = True + total_recovery = True + source['Recovered Size'] = 0 + if current_pass != 'Done': + source[current_pass]['Min Status'] = 100 + try: + current_pass_num = int(current_pass[-1:]) + next_pass_num = current_pass_num + 1 + except ValueError: + # Either Done or undefined? + current_pass_num = -1 + next_pass_num = -1 + if 1 <= next_pass_num <= 3: + next_pass = 'Pass {}'.format(next_pass_num) + else: + next_pass = 'Done' + if 'Progress Out' not in source: source['Progress Out'] = '{}/progress.out'.format(global_vars['LogDir']) output = [] @@ -1038,6 +1056,74 @@ def update_progress(source): else: output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) output.append('─────────────────────') + + # Update children progress + for child in source['Children']: + if os.path.exists(child['Dest Paths']['Map']): + map_data = read_map_file(child['Dest Paths']['Map']) + if child['Dev Path'] == source.get('Current Device', ''): + # Current child device + r_size = map_data['rescued']/100 * child['Size'] + child[current_pass]['Done'] = map_data['pass completed'] + child[current_pass]['Status'] = map_data['rescued'] + child['Recovered Size'] = r_size + + # All child devices + pass_complete_for_all_devs &= child[current_pass]['Done'] + total_recovery &= map_data['full recovery'] + try: + source['Recovered Size'] += child.get('Recovered Size', 0) + source[current_pass]['Min Status'] = min( + source[current_pass]['Min Status'], + child[current_pass]['Status']) + except TypeError: + # Force 0% to disable auto-continue + source[current_pass]['Min Status'] = 0 + else: + # Map missing, assuming this pass hasn't run for this dev yet + pass_complete_for_all_devs = False + total_recovery = False + + # Update source progress + if len(source['Children']) > 0: + # Imaging parts, skip updating source progress + pass + elif os.path.exists(source['Dest Paths']['Map']): + # Cloning/Imaging whole device + map_data = read_map_file(source['Dest Paths']['Map']) + source[current_pass]['Done'] = map_data['pass completed'] + source[current_pass]['Status'] = map_data['rescued'] + source['Recovered Size'] = map_data['rescued']/100 * source['Size'] + try: + source[current_pass]['Min Status'] = min( + source[current_pass]['Min Status'], + source[current_pass]['Status']) + except TypeError: + # Force 0% to disable auto-continue + source[current_pass]['Min Status'] = 0 + pass_complete_for_all_devs &= source[current_pass]['Done'] + total_recovery &= map_data['full recovery'] + else: + # Cloning/Imaging whole device and map missing + pass_complete_for_all_devs = False + total_recovery = False + + # End of pass updates + if end_run: + if total_recovery: + # Sweet! + source['Current Pass'] = 'Done' + source['Recovered Size'] = source['Total Size'] + for p_num in ['Pass 1', 'Pass 2', 'Pass 3']: + if source[p_num]['Status'] == 'Pending': + source[p_num]['Status'] = 'Skipped' + for child in source['Children']: + if child[p_num]['Status'] == 'Pending': + child[p_num]['Status'] = 'Skipped' + elif pass_complete_for_all_devs: + # Ready for next pass? + source['Current Pass'] = next_pass + source[current_pass]['Done'] = True # Main device if source['Type'] == 'Clone': From 1f63f911447a5397c2d41478222fa7db31f9d703 Mon Sep 17 00:00:00 2001 From: Alan Mason <2xShirt@gmail.com> Date: Sun, 22 Jul 2018 16:27:34 -0600 Subject: [PATCH 052/138] PEP8 Cleanup --- .bin/Scripts/functions/ddrescue.py | 297 ++++++++++++++++------------- 1 file changed, 169 insertions(+), 128 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 5656c212..ca0d94a3 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -13,7 +13,7 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES -AUTHORIZED_DEST_FSTYPES = ['ext3', 'ext4', 'xfs'] +RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] AUTO_NEXT_PASS_1_THRESHOLD = 85 AUTO_NEXT_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { @@ -36,41 +36,45 @@ USAGE = """ {script_name} clone [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) """ + # Functions def abort_ddrescue_tui(): run_program(['losetup', '-D']) abort() + def build_outer_panes(source, dest): """Build top and side panes.""" clear_screen() - + # Top panes source_pane = tmux_splitw( '-bdvl', '2', '-PF', '#D', 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( - text = source['Display Name'], + text=source['Display Name'], **COLORS)) tmux_splitw( '-t', source_pane, '-dhl', '21', 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( - text = time.strftime("%Y-%m-%d %H:%M %Z"), + text=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) tmux_splitw( '-t', source_pane, '-dhp', '50', 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( - text = dest['Display Name'], + text=dest['Display Name'], **COLORS)) - + # Side pane update_progress(source) - tmux_splitw('-dhl', '21', + tmux_splitw( + '-dhl', '21', 'watch', '--color', '--no-title', '--interval', '1', 'cat', source['Progress Out']) + def check_dest_paths(source): """Check for image and/or map file and alert user about details.""" dd_image_exists = os.path.exists(source['Dest Paths']['Image']) @@ -112,11 +116,13 @@ def check_dest_paths(source): if abort_imaging or (resume_files_exist and not ask(p)): abort_ddrescue_tui() + def dest_safety_check(source, dest): """Verify the destination is appropriate for the source.""" source_size = source['Details']['size'] if dest['Is Dir']: - cmd = ['findmnt', '-J', + cmd = [ + 'findmnt', '-J', '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', '-T', dest['Path']] result = run_program(cmd) @@ -141,27 +147,29 @@ def dest_safety_check(source, dest): # Imaging: ensure 120% of source size is available print_error( 'Not enough free space on destination, refusing to continue.') - print_standard(' Dest {d_size} < Required {s_size}'.format( - d_size = human_readable_size(dest_size), - s_size = human_readable_size(source_size * 1.2))) + print_standard( + ' Dest {d_size} < Required {s_size}'.format( + d_size=human_readable_size(dest_size), + s_size=human_readable_size(source_size * 1.2))) abort_ddrescue_tui() elif source['Type'] == 'Clone' and source_size > dest_size: # Cloning: ensure dest >= size print_error('Destination is too small, refusing to continue.') - print_standard(' Dest {d_size} < Source {s_size}'.format( - d_size = human_readable_size(dest_size), - s_size = human_readable_size(source_size))) + print_standard( + ' Dest {d_size} < Source {s_size}'.format( + d_size=human_readable_size(dest_size), + s_size=human_readable_size(source_size))) abort_ddrescue_tui() # Imaging specific checks if source['Type'] == 'Image': # Filesystem Type - if dest['Filesystem'] not in AUTHORIZED_DEST_FSTYPES: + if dest['Filesystem'] not in RECOMMENDED_FSTYPES: print_error( - 'Destination filesystem "{}" is not a recommended type.'.format( - dest['Filesystem'])) - print_info('Authorized types are: {}'.format( - ' / '.join(AUTHORIZED_DEST_FSTYPES).upper())) + 'Destination filesystem "{}" is not recommended.'.format( + dest['Filesystem'])) + print_info('Recommended types are: {}'.format( + ' / '.join(RECOMMENDED_FSTYPES).upper())) print_standard(' ') if not ask('Proceed anyways? (Strongly discouraged)'): abort_ddrescue_tui() @@ -174,13 +182,14 @@ def dest_safety_check(source, dest): if not dest_ok: print_error('Destination is not writable, refusing to continue.') abort_ddrescue_tui() - + # Mount options check if 'rw' not in dest['Mount options'].split(','): print_error( 'Destination is not mounted read-write, refusing to continue.') abort_ddrescue_tui() + def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -199,11 +208,13 @@ def get_device_details(dev_path): # Just return the first device (there should only be one) return json_data['blockdevices'][0] + def get_device_size_in_bytes(s): """Convert size string from lsblk string to bytes, returns int.""" s = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', s, re.IGNORECASE) return convert_to_bytes(s) + def get_recovery_scope_size(source): """Calculate total size of selected dev(s).""" source['Total Size'] = 0 @@ -216,6 +227,7 @@ def get_recovery_scope_size(source): source['Size'] = get_device_size_in_bytes(source['Details']['size']) source['Total Size'] = source['Size'] + def get_status_color(s, t_success=99, t_warn=90): """Get color based on status, returns str.""" color = COLORS['CLEAR'] @@ -225,7 +237,7 @@ def get_status_color(s, t_success=99, t_warn=90): except ValueError: # Status is either in lists below or will default to red pass - + if s in ('Pending',): color = COLORS['CLEAR'] elif s in ('Skipped', 'Unknown', 'Working'): @@ -238,6 +250,7 @@ def get_status_color(s, t_success=99, t_warn=90): color = COLORS['RED'] return color + def mark_all_passes_pending(source): """Mark all devs and passes as pending in preparation for retry.""" source['Current Pass'] = 'Pass 1' @@ -248,9 +261,10 @@ def mark_all_passes_pending(source): child[p_num]['Status'] = 'Pending' child[p_num]['Done'] = False + def menu_clone(source_path, dest_path): """ddrescue cloning menu.""" - + # Set devices source = select_device('source', source_path) source['Current Pass'] = 'Pass 1' @@ -260,21 +274,22 @@ def menu_clone(source_path, dest_path): source['Recovered Size'] = 0, source['Total Size'] = 0, source['Type'] = 'Clone' - dest = select_device('destination', dest_path, - skip_device = source['Details'], allow_image_file = False) + dest = select_device( + 'destination', dest_path, + skip_device=source['Details'], allow_image_file=False) dest_safety_check(source, dest) - + # Show selection details show_selection_details(source, dest) set_dest_image_paths(source, dest) check_dest_paths(source) get_recovery_scope_size(source) - + # Confirm if not ask('Proceed with clone?'): abort_ddrescue_tui() show_safety_check() - + # Main menu build_outer_panes(source, dest) menu_main(source, dest) @@ -284,6 +299,7 @@ def menu_clone(source_path, dest_path): run_program(['tmux', 'kill-window']) exit_script() + def menu_ddrescue(*args): """Main ddrescue loop/menu.""" args = list(args) @@ -312,11 +328,12 @@ def menu_ddrescue(*args): show_usage(script_name) exit_script() + def menu_image(source_path, dest_path): """ddrescue imaging menu.""" - + # Set devices - source = select_device('source', source_path, allow_image_file = False) + source = select_device('source', source_path, allow_image_file=False) source['Current Pass'] = 'Pass 1' source['Pass 1'] = {'Status': 'Pending', 'Done': False} source['Pass 2'] = {'Status': 'Pending', 'Done': False} @@ -332,14 +349,14 @@ def menu_image(source_path, dest_path): set_dest_image_paths(source, dest) check_dest_paths(source) get_recovery_scope_size(source) - + # Show selection details show_selection_details(source, dest) - + # Confirm if not ask('Proceed with imaging?'): abort_ddrescue_tui() - + # Main menu build_outer_panes(source, dest) menu_main(source, dest) @@ -349,6 +366,7 @@ def menu_image(source_path, dest_path): run_program(['tmux', 'kill-window']) exit_script() + def menu_main(source, dest): """Main menu is used to set ddrescue settings.""" title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) @@ -364,7 +382,7 @@ def menu_main(source, dest): 'Enabled': False}, {'Base Name': 'Reverse direction', 'Enabled': False}, ] - actions =[ + actions = [ {'Name': 'Start', 'Letter': 'S'}, {'Name': 'Change settings {YELLOW}(experts only){CLEAR}'.format( **COLORS), @@ -387,11 +405,11 @@ def menu_main(source, dest): opt['Name'] = '{} {}'.format( '[✓]' if opt['Enabled'] else '[ ]', opt['Base Name']) - + selection = menu_select( - title = title + display_pass, - main_entries = main_options, - action_entries = actions) + title=title+display_pass, + main_entries=main_options, + action_entries=actions) if selection.isnumeric(): # Toggle selection @@ -435,20 +453,21 @@ def menu_main(source, dest): break if source[current_pass]['Done']: min_status = source[current_pass]['Min Status'] - if (current_pass == 'Pass 1' - and min_status < AUTO_NEXT_PASS_1_THRESHOLD): + if (current_pass == 'Pass 1' and + min_status < AUTO_NEXT_PASS_1_THRESHOLD): auto_run = False - elif (current_pass == 'Pass 2' - and min_status < AUTO_NEXT_PASS_2_THRESHOLD): + elif (current_pass == 'Pass 2' and + min_status < AUTO_NEXT_PASS_2_THRESHOLD): auto_run = False # Update current pass for next iteration current_pass = source['Current Pass'] - + elif selection == 'C': menu_settings(source) elif selection == 'Q': break + def menu_select_children(source): """Select child device(s) or whole disk, returns list.""" dev_options = [{ @@ -467,7 +486,7 @@ def menu_select_children(source): actions = [ {'Name': 'Proceed', 'Letter': 'P'}, {'Name': 'Quit', 'Letter': 'Q'}] - + # Skip Menu if there's no children if len(dev_options) == 1: return [] @@ -481,9 +500,9 @@ def menu_select_children(source): dev['Base Name']) selection = menu_select( - title = 'Please select part(s) to image', - main_entries = dev_options, - action_entries = actions) + title='Please select part(s) to image', + main_entries=dev_options, + action_entries=actions) if selection.isnumeric(): # Toggle selection @@ -513,6 +532,7 @@ def menu_select_children(source): if d['Selected'] and 'Whole device' not in d['Base Name']] return selected_children + def menu_select_device(title='Which device?', skip_device={}): """Select block device via a menu, returns dev_path as str.""" skip_names = [ @@ -540,11 +560,11 @@ def menu_select_device(title='Which device?', skip_device={}): # Append non-matching devices dev_options.append({ 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( - name = dev['name'], - tran = dev['tran'] if dev['tran'] else '', - size = dev['size'] if dev['size'] else '', - model = dev['model'] if dev['model'] else '', - serial = dev['serial'] if dev['serial'] else ''), + name=dev['name'], + tran=dev['tran'] if dev['tran'] else '', + size=dev['size'] if dev['size'] else '', + model=dev['model'] if dev['model'] else '', + serial=dev['serial'] if dev['serial'] else ''), 'Path': dev['name'], 'Disabled': disable_dev}) dev_options = sorted(dev_options, key=itemgetter('Name')) @@ -555,16 +575,17 @@ def menu_select_device(title='Which device?', skip_device={}): # Show Menu actions = [{'Name': 'Quit', 'Letter': 'Q'}] selection = menu_select( - title = title, - main_entries = dev_options, - action_entries = actions, - disabled_label = 'SOURCE DEVICE') + title=title, + main_entries=dev_options, + action_entries=actions, + disabled_label='SOURCE DEVICE') if selection.isnumeric(): return dev_options[int(selection)-1]['Path'] elif selection == 'Q': abort_ddrescue_tui() + def menu_select_path(skip_device={}): """Select path via menu, returns path as str.""" pwd = os.path.realpath(global_vars['Env']['PWD']) @@ -579,9 +600,9 @@ def menu_select_path(skip_device={}): # Show Menu selection = menu_select( - title = 'Please make a selection', - main_entries = path_options, - action_entries = actions) + title='Please make a selection', + main_entries=path_options, + action_entries=actions) if selection == 'Q': abort_ddrescue_tui() @@ -594,14 +615,14 @@ def menu_select_path(skip_device={}): elif path_options[index]['Name'] == 'Local device': # Local device local_device = select_device( - skip_device = skip_device, - allow_image_file = False) + skip_device=skip_device, + allow_image_file=False) # Mount device volume(s) report = mount_volumes( - all_devices = False, - device_path = local_device['Dev Path'], - read_write = True) + all_devices=False, + device_path=local_device['Dev Path'], + read_write=True) # Select volume vol_options = [] @@ -616,9 +637,9 @@ def menu_select_path(skip_device={}): 'Path': v['mount_point'], 'Disabled': disabled}) selection = menu_select( - title = 'Please select a volume', - main_entries = vol_options, - action_entries = actions) + title='Please select a volume', + main_entries=vol_options, + action_entries=actions) if selection.isnumeric(): s_path = vol_options[int(selection)-1]['Path'] elif selection == 'Q': @@ -636,6 +657,7 @@ def menu_select_path(skip_device={}): print_error('Invalid path "{}"'.format(m_path)) return s_path + def menu_settings(source): """Change advanced ddrescue settings.""" title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) @@ -660,32 +682,33 @@ def menu_settings(source): source['Settings'][s['Flag']].get('Value', '')) if not source['Settings'][s['Flag']]['Enabled']: s['Name'] = '{YELLOW}{name} (Disabled){CLEAR}'.format( - name = s['Name'], + name=s['Name'], **COLORS) selection = menu_select( - title = title, - main_entries = settings, - action_entries = actions) + title=title, + main_entries=settings, + action_entries=actions) if selection.isnumeric(): index = int(selection) - 1 flag = settings[index]['Flag'] enabled = source['Settings'][flag]['Enabled'] if 'Value' in source['Settings'][flag]: answer = choice( - choices = ['T', 'C'], - prompt = 'Toggle or change value for "{}"'.format(flag)) + choices=['T', 'C'], + prompt='Toggle or change value for "{}"'.format(flag)) if answer == 'T': # Toggle source['Settings'][flag]['Enabled'] = not enabled else: # Update value source['Settings'][flag]['Value'] = get_simple_string( - prompt = 'Enter new value') + prompt='Enter new value') else: source['Settings'][flag]['Enabled'] = not enabled elif selection == 'M': break + def read_map_file(map_path): """Read map file with ddrescuelog and return data as dict.""" map_data = {} @@ -694,7 +717,7 @@ def read_map_file(map_path): except subprocess.CalledProcessError: print_error('Failed to read map data') abort_ddrescue_tui() - + # Parse output for line in result.stdout.decode().splitlines(): m = REGEX_MAP_DATA.match(line.strip()) @@ -718,6 +741,7 @@ def read_map_file(map_path): return map_data + def run_ddrescue(source, dest, settings): """Run ddrescue pass.""" current_pass = source['Current Pass'] @@ -739,28 +763,28 @@ def run_ddrescue(source, dest, settings): return else: raise GenericError("This shouldn't happen?") - + # Set device(s) to clone/image source[current_pass]['Status'] = 'Working' source_devs = [source] if source['Children']: # Use only selected child devices source_devs = source['Children'] - + # Set heights - ## NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) + # NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) result = run_program(['tput', 'lines']) height = int(result.stdout.decode().strip()) height_smart = int(height * (12 / 33)) height_ddrescue = height - height_smart - + # Show SMART status smart_pane = tmux_splitw( '-bdvl', str(height_smart), '-PF', '#D', 'watch', '--color', '--no-title', '--interval', '300', 'ddrescue-tui-smart-display', source['Dev Path']) - + # Start pass for each selected device for s_dev in source_devs: if s_dev[current_pass]['Done']: @@ -769,23 +793,24 @@ def run_ddrescue(source, dest, settings): source['Current Device'] = s_dev['Dev Path'] s_dev[current_pass]['Status'] = 'Working' update_progress(source) - + # Set ddrescue cmd if source['Type'] == 'Clone': - cmd = ['ddrescue', *settings, '--force', - s_dev['Dev Path'], dest['Dev Path'], s_dev['Dest Paths']['Map']] + cmd = [ + 'ddrescue', *settings, '--force', s_dev['Dev Path'], + dest['Dev Path'], s_dev['Dest Paths']['Map']] else: - cmd = ['ddrescue', *settings, - s_dev['Dev Path'], s_dev['Dest Paths']['Image'], - s_dev['Dest Paths']['Map']] + cmd = [ + 'ddrescue', *settings, s_dev['Dev Path'], + s_dev['Dest Paths']['Image'], s_dev['Dest Paths']['Map']] # Start ddrescue try: clear_screen() print_info('Current dev: {}'.format(s_dev['Dev Path'])) ddrescue_proc = popen_program(['./__choose_exit', *cmd]) - #ddrescue_proc = popen_program(['./__exit_ok', *cmd]) - #ddrescue_proc = popen_program(cmd) + # ddrescue_proc = popen_program(['./__exit_ok', *cmd]) + # ddrescue_proc = popen_program(cmd) while True: try: ddrescue_proc.wait(timeout=30) @@ -814,6 +839,7 @@ def run_ddrescue(source, dest, settings): pause('Press Enter to return to main menu... ') run_program(['tmux', 'kill-pane', '-t', smart_pane]) + def select_dest_path(provided_path=None, skip_device={}): dest = {'Is Dir': True, 'Is Image': False} @@ -851,11 +877,12 @@ def select_dest_path(provided_path=None, skip_device={}): return dest + def select_device(description='device', provided_path=None, - skip_device={}, allow_image_file=True): + skip_device={}, allow_image_file=True): """Select device via provided path or menu, return dev as dict.""" dev = {'Is Dir': False, 'Is Image': False} - + # Set path if provided_path: dev['Path'] = provided_path @@ -864,7 +891,7 @@ def select_device(description='device', provided_path=None, title='Please select a {}'.format(description), skip_device=skip_device) dev['Path'] = os.path.realpath(dev['Path']) - + # Check path if pathlib.Path(dev['Path']).is_block_device(): dev['Dev Path'] = dev['Path'] @@ -879,13 +906,13 @@ def select_device(description='device', provided_path=None, dev['Details'] = get_device_details(dev['Dev Path']) if 'Children' not in dev: dev['Children'] = [] - + # Check for parent device(s) while dev['Details']['pkname']: print_warning('{} "{}" is a child device.'.format( description.title(), dev['Dev Path'])) if ask('Use parent device "{}" instead?'.format( - dev['Details']['pkname'])): + dev['Details']['pkname'])): # Update dev with parent info dev['Dev Path'] = dev['Details']['pkname'] dev['Details'] = get_device_details(dev['Dev Path']) @@ -903,25 +930,28 @@ def select_device(description='device', provided_path=None, width = int((int(result.stdout.decode().strip()) - 21) / 2) - 2 if len(dev['Display Name']) > width: if dev['Is Image']: - dev['Display Name'] = '...{}'.format(dev['Display Name'][-(width-3):]) + dev['Display Name'] = '...{}'.format( + dev['Display Name'][-(width-3):]) else: - dev['Display Name'] = '{}...'.format(dev['Display Name'][:(width-3)]) + dev['Display Name'] = '{}...'.format( + dev['Display Name'][:(width-3)]) else: dev['Display Name'] = dev['Display Name'] return dev + def set_dest_image_paths(source, dest): """Set destination image path for source and any child devices.""" if source['Type'] == 'Clone': base = '{pwd}/Clone_{size}_{model}'.format( - pwd = os.path.realpath(global_vars['Env']['PWD']), - size = source['Details']['size'], - model = source['Details'].get('model', 'Unknown')) + pwd=os.path.realpath(global_vars['Env']['PWD']), + size=source['Details']['size'], + model=source['Details'].get('model', 'Unknown')) else: base = '{Path}/{size}_{model}'.format( - size = source['Details']['size'], - model = source['Details'].get('model', 'Unknown'), + size=source['Details']['size'], + model=source['Details'].get('model', 'Unknown'), **dest) source['Dest Paths'] = { 'Image': '{}.dd'.format(base), @@ -933,17 +963,18 @@ def set_dest_image_paths(source, dest): if child['Details']['label']: p_label = '_{}'.format(child['Details']['label']) base = '{Path}/{size}_{model}_{p_num}_{p_size}{p_label}'.format( - size = source['Details']['size'], - model = source['Details'].get('model', 'Unknown'), - p_num = child['Details']['name'].replace( + size=source['Details']['size'], + model=source['Details'].get('model', 'Unknown'), + p_num=child['Details']['name'].replace( child['Details']['pkname'], ''), - p_size = child['Details']['size'], - p_label = p_label, + p_size=child['Details']['size'], + p_label=p_label, **dest) child['Dest Paths'] = { 'Image': '{}.dd'.format(base), 'Map': '{}.map'.format(base)} + def setup_loopback_device(source_path): """Setup a loopback device for source_path, returns dev_path as str.""" cmd = ( @@ -962,6 +993,7 @@ def setup_loopback_device(source_path): else: return dev_path + def show_device_details(dev_path): """Display device details on screen.""" cmd = ( @@ -982,6 +1014,7 @@ def show_device_details(dev_path): for line in output: print_standard(line) + def show_safety_check(): """Display safety check message and get confirmation from user.""" print_standard('\nSAFETY CHECK') @@ -992,9 +1025,10 @@ def show_safety_check(): if not ask('Asking again to confirm, is this correct?'): abort_ddrescue_tui() + def show_selection_details(source, dest): clear_screen() - + # Source print_success('Source device') if source['Is Image']: @@ -1003,7 +1037,7 @@ def show_selection_details(source, dest): source['Dev Path'])) show_device_details(source['Dev Path']) print_standard(' ') - + # Destination if source['Type'] == 'Clone': print_success('Destination device ', end='') @@ -1017,17 +1051,20 @@ def show_selection_details(source, dest): dest['Free Space'], dest['Filesystem'])) print_standard(' ') + def show_usage(script_name): print_info('Usage:') print_standard(USAGE.format(script_name=script_name)) pause() + def tmux_splitw(*args): """Run tmux split-window command and return output as str.""" cmd = ['tmux', 'split-window', *args] result = run_program(cmd) return result.stdout.decode().strip() + def update_progress(source, end_run=False): """Update progress file.""" current_pass = source['Current Pass'] @@ -1047,9 +1084,10 @@ def update_progress(source, end_run=False): next_pass = 'Pass {}'.format(next_pass_num) else: next_pass = 'Done' - + if 'Progress Out' not in source: - source['Progress Out'] = '{}/progress.out'.format(global_vars['LogDir']) + source['Progress Out'] = '{}/progress.out'.format( + global_vars['LogDir']) output = [] if source['Type'] == 'Clone': output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) @@ -1067,7 +1105,7 @@ def update_progress(source, end_run=False): child[current_pass]['Done'] = map_data['pass completed'] child[current_pass]['Status'] = map_data['rescued'] child['Recovered Size'] = r_size - + # All child devices pass_complete_for_all_devs &= child[current_pass]['Done'] total_recovery &= map_data['full recovery'] @@ -1083,7 +1121,7 @@ def update_progress(source, end_run=False): # Map missing, assuming this pass hasn't run for this dev yet pass_complete_for_all_devs = False total_recovery = False - + # Update source progress if len(source['Children']) > 0: # Imaging parts, skip updating source progress @@ -1107,7 +1145,7 @@ def update_progress(source, end_run=False): # Cloning/Imaging whole device and map missing pass_complete_for_all_devs = False total_recovery = False - + # End of pass updates if end_run: if total_recovery: @@ -1124,11 +1162,11 @@ def update_progress(source, end_run=False): # Ready for next pass? source['Current Pass'] = next_pass source[current_pass]['Done'] = True - + # Main device if source['Type'] == 'Clone': output.append('{BLUE}{dev}{CLEAR}'.format( - dev = 'Image File' if source['Is Image'] else source['Dev Path'], + dev='Image File' if source['Is Image'] else source['Dev Path'], **COLORS)) for x in (1, 2, 3): p_num = 'Pass {}'.format(x) @@ -1141,9 +1179,9 @@ def update_progress(source, end_run=False): else: s_display = '{:0.2f} %'.format(s_display) output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( - p_num = p_num, - s_color = get_status_color(source[p_num]['Status']), - s_display = s_display, + p_num=p_num, + s_color=get_status_color(source[p_num]['Status']), + s_display=s_display, **COLORS)) else: # Image mode @@ -1151,7 +1189,7 @@ def update_progress(source, end_run=False): # Just parts for child in source['Children']: output.append('{BLUE}{dev}{CLEAR}'.format( - dev = child['Dev Path'], + dev=child['Dev Path'], **COLORS)) for x in (1, 2, 3): p_num = 'Pass {}'.format(x) @@ -1163,16 +1201,17 @@ def update_progress(source, end_run=False): pass else: s_display = '{:0.2f} %'.format(s_display) - output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( - p_num = p_num, - s_color = get_status_color(child[p_num]['Status']), - s_display = s_display, - **COLORS)) + output.append( + '{p_num}{s_color}{s_display:>15}{CLEAR}'.format( + p_num=p_num, + s_color=get_status_color(child[p_num]['Status']), + s_display=s_display, + **COLORS)) output.append(' ') else: # Whole device output.append('{BLUE}{dev}{CLEAR} {YELLOW}(Whole){CLEAR}'.format( - dev = source['Dev Path'], + dev=source['Dev Path'], **COLORS)) for x in (1, 2, 3): p_num = 'Pass {}'.format(x) @@ -1184,11 +1223,12 @@ def update_progress(source, end_run=False): pass else: s_display = '{:0.2f} %'.format(s_display) - output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( - p_num = p_num, - s_color = get_status_color(source[p_num]['Status']), - s_display = s_display, - **COLORS)) + output.append( + '{p_num}{s_color}{s_display:>15}{CLEAR}'.format( + p_num=p_num, + s_color=get_status_color(source[p_num]['Status']), + s_display=s_display, + **COLORS)) # Add line-endings output = ['{}\n'.format(line) for line in output] @@ -1196,6 +1236,7 @@ def update_progress(source, end_run=False): with open(source['Progress Out'], 'w') as f: f.writelines(output) + if __name__ == '__main__': print("This file is not meant to be called directly.") From cd955fe1fc871b991d8016a75694ba8bdeaffd27 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Jul 2018 21:56:39 -0600 Subject: [PATCH 053/138] Add overall recovery status to side-pane * --test-mode disabled by default * Fixed bug that prevented escaping auto_run via Ctrl-c * Fixed no-trim / no-scrape flag handling * Only proceed device(s) have been selected in menu_select_children --- .bin/Scripts/functions/ddrescue.py | 115 +++++++++++++++++------------ 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index ca0d94a3..27cac0ba 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -14,23 +14,25 @@ from operator import itemgetter # STATIC VARIABLES RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] -AUTO_NEXT_PASS_1_THRESHOLD = 85 +AUTO_NEXT_PASS_1_THRESHOLD = 90 AUTO_NEXT_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { '--binary-prefixes': {'Enabled': True, 'Hidden': True}, '--data-preview': {'Enabled': True, 'Hidden': True}, '--idirect': {'Enabled': True}, '--odirect': {'Enabled': True}, - '--max-read-rate': {'Enabled': False, 'Value': '128MiB'}, + '--max-read-rate': {'Enabled': False, 'Value': '4MiB'}, '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, '--reopen-on-error': {'Enabled': True}, '--retry-passes=': {'Enabled': True, 'Value': '0'}, - '--test-mode=': {'Enabled': True, 'Value': 'some.map'}, + '--test-mode=': {'Enabled': False, 'Value': 'some.map'}, '--timeout=': {'Enabled': True, 'Value': '5m'}, '-vvvv': {'Enabled': True, 'Hidden': True}, } REGEX_MAP_DATA = re.compile(r'^\s*(?P\S+):.*\(\s*(?P\d+\.?\d*)%.*') REGEX_MAP_STATUS = re.compile(r'.*current status:\s+(?P.*)') +STATUS_COLOR_CLEAR = ('Pending',) +STATUS_COLOR_YELLOW = ('Skipped', 'Unknown', 'Working') USAGE = """ {script_name} clone [source [destination]] {script_name} image [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) @@ -238,9 +240,9 @@ def get_status_color(s, t_success=99, t_warn=90): # Status is either in lists below or will default to red pass - if s in ('Pending',): + if s in STATUS_COLOR_CLEAR: color = COLORS['CLEAR'] - elif s in ('Skipped', 'Unknown', 'Working'): + elif s in STATUS_COLOR_YELLOW: color = COLORS['YELLOW'] elif p_recovered >= t_success: color = COLORS['GREEN'] @@ -445,6 +447,7 @@ def menu_main(source, dest): while auto_run or first_run: first_run = False run_ddrescue(source, dest, settings) + update_progress(source, end_run=True) if current_pass == 'Done': # "Pass Done" i.e. all passes done break @@ -459,6 +462,8 @@ def menu_main(source, dest): elif (current_pass == 'Pass 2' and min_status < AUTO_NEXT_PASS_2_THRESHOLD): auto_run = False + else: + auto_run = False # Update current pass for next iteration current_pass = source['Current Pass'] @@ -493,11 +498,14 @@ def menu_select_children(source): # Show Menu while True: + one_or_more_devs_selected = False # Update entries for dev in dev_options: - dev['Name'] = '{} {}'.format( - '*' if dev['Selected'] else ' ', - dev['Base Name']) + if dev['Selected']: + one_or_more_devs_selected = True + dev['Name'] = '* {}'.format(dev['Base Name']) + else: + dev['Name'] = ' {}'.format(dev['Base Name']) selection = menu_select( title='Please select part(s) to image', @@ -517,7 +525,7 @@ def menu_select_children(source): if dev_options[0]['Selected']: for dev in dev_options[1:]: dev['Selected'] = False - elif selection == 'P': + elif selection == 'P' and one_or_more_devs_selected: break elif selection == 'Q': abort_ddrescue_tui() @@ -747,22 +755,11 @@ def run_ddrescue(source, dest, settings): current_pass = source['Current Pass'] return_code = None - # Set pass options - if current_pass == 'Pass 1': - settings.extend(['--no-trim', '--no-scrape']) - elif current_pass == 'Pass 2': - # Allow trimming - settings.append('--no-scrape') - elif current_pass == 'Pass 3': - # Allow trimming and scraping - pass - elif current_pass == 'Done': + if current_pass == 'Done': clear_screen() print_warning('Recovery already completed?') pause('Press Enter to return to main menu...') return - else: - raise GenericError("This shouldn't happen?") # Set device(s) to clone/image source[current_pass]['Status'] = 'Working' @@ -803,6 +800,14 @@ def run_ddrescue(source, dest, settings): cmd = [ 'ddrescue', *settings, s_dev['Dev Path'], s_dev['Dest Paths']['Image'], s_dev['Dest Paths']['Map']] + if current_pass == 'Pass 1': + cmd.extend(['--no-trim', '--no-scrape']) + elif current_pass == 'Pass 2': + # Allow trimming + cmd.append('--no-scrape') + elif current_pass == 'Pass 3': + # Allow trimming and scraping + pass # Start ddrescue try: @@ -813,7 +818,9 @@ def run_ddrescue(source, dest, settings): # ddrescue_proc = popen_program(cmd) while True: try: - ddrescue_proc.wait(timeout=30) + ddrescue_proc.wait(timeout=10) + sleep(2) + update_progress(source) break except subprocess.TimeoutExpired: update_progress(source) @@ -832,8 +839,7 @@ def run_ddrescue(source, dest, settings): print_error('Error(s) encountered, see message above.') break - # Cleanup - update_progress(source, end_run=True) + # Done if str(return_code) != '0': # Pause on errors pause('Press Enter to return to main menu... ') @@ -1066,7 +1072,7 @@ def tmux_splitw(*args): def update_progress(source, end_run=False): - """Update progress file.""" + """Update progress for source dev(s) and update status pane file.""" current_pass = source['Current Pass'] pass_complete_for_all_devs = True total_recovery = True @@ -1085,16 +1091,6 @@ def update_progress(source, end_run=False): else: next_pass = 'Done' - if 'Progress Out' not in source: - source['Progress Out'] = '{}/progress.out'.format( - global_vars['LogDir']) - output = [] - if source['Type'] == 'Clone': - output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) - else: - output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) - output.append('─────────────────────') - # Update children progress for child in source['Children']: if os.path.exists(child['Dest Paths']['Map']): @@ -1129,17 +1125,18 @@ def update_progress(source, end_run=False): elif os.path.exists(source['Dest Paths']['Map']): # Cloning/Imaging whole device map_data = read_map_file(source['Dest Paths']['Map']) - source[current_pass]['Done'] = map_data['pass completed'] - source[current_pass]['Status'] = map_data['rescued'] + if current_pass != 'Done': + source[current_pass]['Done'] = map_data['pass completed'] + source[current_pass]['Status'] = map_data['rescued'] + try: + source[current_pass]['Min Status'] = min( + source[current_pass]['Min Status'], + source[current_pass]['Status']) + except TypeError: + # Force 0% to disable auto-continue + source[current_pass]['Min Status'] = 0 + pass_complete_for_all_devs &= source[current_pass]['Done'] source['Recovered Size'] = map_data['rescued']/100 * source['Size'] - try: - source[current_pass]['Min Status'] = min( - source[current_pass]['Min Status'], - source[current_pass]['Status']) - except TypeError: - # Force 0% to disable auto-continue - source[current_pass]['Min Status'] = 0 - pass_complete_for_all_devs &= source[current_pass]['Done'] total_recovery &= map_data['full recovery'] else: # Cloning/Imaging whole device and map missing @@ -1161,7 +1158,30 @@ def update_progress(source, end_run=False): elif pass_complete_for_all_devs: # Ready for next pass? source['Current Pass'] = next_pass - source[current_pass]['Done'] = True + if current_pass != 'Done': + source[current_pass]['Done'] = True + + # Start building output lines + if 'Progress Out' not in source: + source['Progress Out'] = '{}/progress.out'.format( + global_vars['LogDir']) + output = [] + if source['Type'] == 'Clone': + output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) + else: + output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) + output.append('─────────────────────') + + # Overall progress + recovered_p = (source['Recovered Size'] / source['Total Size']) * 100 + recovered_s = human_readable_size(source['Recovered Size']) + output.append('{BLUE}Overall Progress{CLEAR}'.format(**COLORS)) + output.append('Recovered:{s_color}{recovered_p:>9.2f} %{CLEAR}'.format( + s_color=get_status_color(recovered_p), + recovered_p=recovered_p, + **COLORS)) + output.append('{:>21}'.format(recovered_s)) + output.append('─────────────────────') # Main device if source['Type'] == 'Clone': @@ -1207,6 +1227,9 @@ def update_progress(source, end_run=False): s_color=get_status_color(child[p_num]['Status']), s_display=s_display, **COLORS)) + p = (child.get('Recovered Size', 0) / child['Size']) * 100 + output.append('Recovered:{s_color}{p:>9.2f} %{CLEAR}'.format( + s_color=get_status_color(p), p=p, **COLORS)) output.append(' ') else: # Whole device From f5994d851b58fbfb2d14a9503b3f34e81092106a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 22 Jul 2018 21:56:52 -0600 Subject: [PATCH 054/138] Allow more characters in get_simple_string() --- .bin/Scripts/functions/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 75fe99d1..dc427157 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -217,7 +217,7 @@ def get_simple_string(prompt='Enter string'): simple_string = None while simple_string is None: _input = input('{}: '.format(prompt)) - if re.match(r'^(\w|-|_| )+$', _input, re.ASCII): + if re.match(r"^(\w|-| |\.|')+$", _input, re.ASCII): simple_string = _input.strip() return simple_string From 2430ba5e00900a5577a06a46266ddf55df0f9c97 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 23 Jul 2018 00:43:22 -0600 Subject: [PATCH 055/138] Resume session via map file(s) * Read map file(s) and set progress, status, and current pass --- .bin/Scripts/functions/ddrescue.py | 110 ++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 27cac0ba..aa0655d5 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -21,11 +21,11 @@ DDRESCUE_SETTINGS = { '--data-preview': {'Enabled': True, 'Hidden': True}, '--idirect': {'Enabled': True}, '--odirect': {'Enabled': True}, - '--max-read-rate': {'Enabled': False, 'Value': '4MiB'}, + '--max-read-rate': {'Enabled': False, 'Value': '32MiB'}, '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, '--reopen-on-error': {'Enabled': True}, '--retry-passes=': {'Enabled': True, 'Value': '0'}, - '--test-mode=': {'Enabled': False, 'Value': 'some.map'}, + '--test-mode=': {'Enabled': False, 'Value': 'test.map'}, '--timeout=': {'Enabled': True, 'Value': '5m'}, '-vvvv': {'Enabled': True, 'Hidden': True}, } @@ -256,6 +256,7 @@ def get_status_color(s, t_success=99, t_warn=90): def mark_all_passes_pending(source): """Mark all devs and passes as pending in preparation for retry.""" source['Current Pass'] = 'Pass 1' + source['Started Recovery'] = False for p_num in ['Pass 1', 'Pass 2', 'Pass 3']: source[p_num]['Status'] = 'Pending' source[p_num]['Done'] = False @@ -273,8 +274,9 @@ def menu_clone(source_path, dest_path): source['Pass 1'] = {'Status': 'Pending', 'Done': False} source['Pass 2'] = {'Status': 'Pending', 'Done': False} source['Pass 3'] = {'Status': 'Pending', 'Done': False} - source['Recovered Size'] = 0, - source['Total Size'] = 0, + source['Recovered Size'] = 0 + source['Started Recovery'] = False + source['Total Size'] = 0 source['Type'] = 'Clone' dest = select_device( 'destination', dest_path, @@ -283,9 +285,12 @@ def menu_clone(source_path, dest_path): # Show selection details show_selection_details(source, dest) + + # Set status details set_dest_image_paths(source, dest) - check_dest_paths(source) get_recovery_scope_size(source) + check_dest_paths(source) + resume_from_map(source) # Confirm if not ask('Proceed with clone?'): @@ -340,8 +345,9 @@ def menu_image(source_path, dest_path): source['Pass 1'] = {'Status': 'Pending', 'Done': False} source['Pass 2'] = {'Status': 'Pending', 'Done': False} source['Pass 3'] = {'Status': 'Pending', 'Done': False} - source['Recovered Size'] = 0, - source['Total Size'] = 0, + source['Recovered Size'] = 0 + source['Started Recovery'] = False + source['Total Size'] = 0 source['Type'] = 'Image' dest = select_dest_path(dest_path, skip_device=source['Details']) dest_safety_check(source, dest) @@ -349,8 +355,9 @@ def menu_image(source_path, dest_path): # Select child device(s) source['Children'] = menu_select_children(source) set_dest_image_paths(source, dest) - check_dest_paths(source) get_recovery_scope_size(source) + check_dest_paths(source) + resume_from_map(source) # Show selection details show_selection_details(source, dest) @@ -750,6 +757,84 @@ def read_map_file(map_path): return map_data +def resume_from_map(source): + """Read map file(s) and set current progress to resume previous session.""" + map_data_read = False + non_tried = 0 + non_trimmed = 0 + non_scraped = 0 + + # Read map data + if source['Type'] != 'Clone' and source['Children']: + # Imaging child device(s) + for child in source['Children']: + if os.path.exists(child['Dest Paths']['Map']): + map_data = read_map_file(child['Dest Paths']['Map']) + map_data_read = True + non_tried += map_data['non-tried'] + non_trimmed += map_data['non-trimmed'] + non_scraped += map_data['non-scraped'] + child['Recovered Size'] = map_data['rescued']/100*child['Size'] + + # Get (dev) current pass + dev_current_pass = 1 + if map_data['non-tried'] == 0: + if map_data['non-trimmed'] > 0: + dev_current_pass = 2 + elif map_data['non-scraped'] > 0: + dev_current_pass = 3 + elif map_data['rescued'] == 100: + dev_current_pass = 4 + + # Mark passes as skipped + for x in range(1, dev_current_pass): + p_num = 'Pass {}'.format(x) + child[p_num]['Done'] = True + child[p_num]['Status'] = 'Skipped' + + elif map_data_read: + # No map but we've already read at least one map, force pass 1 + non_tried = 1 + elif os.path.exists(source['Dest Paths']['Map']): + # Cloning or Imaging whole device + map_data = read_map_file(source['Dest Paths']['Map']) + map_data_read = True + non_tried += map_data['non-tried'] + non_trimmed += map_data['non-trimmed'] + non_scraped += map_data['non-scraped'] + + # Bail + if not map_data_read: + # No map data found, assuming fresh start + return + + # Set current pass + if non_tried > 0: + current_pass = 'Pass 1' + elif non_trimmed > 0: + current_pass = 'Pass 2' + source['Pass 1']['Done'] = True + source['Pass 1']['Status'] = 'Skipped' + elif non_scraped > 0: + current_pass = 'Pass 3' + source['Pass 1']['Done'] = True + source['Pass 1']['Status'] = 'Skipped' + source['Pass 2']['Done'] = True + source['Pass 2']['Status'] = 'Skipped' + else: + source['Current Pass'] = 'Done' + update_progress(source, end_run=True) + return + source['Current Pass'] = current_pass + + # Update current pass + if not source['Children']: + if os.path.exists(source['Dest Paths']['Map']): + map_data = read_map_file(source['Dest Paths']['Map']) + source[current_pass]['Done'] = map_data['pass completed'] + source['Recovered Size'] = map_data['rescued']/100*source['Size'] + + def run_ddrescue(source, dest, settings): """Run ddrescue pass.""" current_pass = source['Current Pass'] @@ -763,6 +848,7 @@ def run_ddrescue(source, dest, settings): # Set device(s) to clone/image source[current_pass]['Status'] = 'Working' + source['Started Recovery'] = True source_devs = [source] if source['Children']: # Use only selected child devices @@ -1093,13 +1179,16 @@ def update_progress(source, end_run=False): # Update children progress for child in source['Children']: + if current_pass == 'Done': + continue if os.path.exists(child['Dest Paths']['Map']): map_data = read_map_file(child['Dest Paths']['Map']) if child['Dev Path'] == source.get('Current Device', ''): # Current child device r_size = map_data['rescued']/100 * child['Size'] child[current_pass]['Done'] = map_data['pass completed'] - child[current_pass]['Status'] = map_data['rescued'] + if source['Started Recovery']: + child[current_pass]['Status'] = map_data['rescued'] child['Recovered Size'] = r_size # All child devices @@ -1127,7 +1216,8 @@ def update_progress(source, end_run=False): map_data = read_map_file(source['Dest Paths']['Map']) if current_pass != 'Done': source[current_pass]['Done'] = map_data['pass completed'] - source[current_pass]['Status'] = map_data['rescued'] + if source['Started Recovery']: + source[current_pass]['Status'] = map_data['rescued'] try: source[current_pass]['Min Status'] = min( source[current_pass]['Min Status'], From f5ff65bfe0c68a1cbbee502fc2253e984df4d2bb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 23 Jul 2018 23:25:12 -0600 Subject: [PATCH 056/138] Started rewriting ddrescue.py Added two classes: * BlockPair() * Track source/dest pair specific data * update_progress() method for its own data * RecoveryState() * Track BlockPair objects and overall state * update_progress() method for overall data Reasons: * Code readability * Better status updates, code currently split between: * get_recovery_scope_size() * resume_from_map() * update_progress() * Functions that should probably be merged into other functions: * get_recovery_scope_size() * set_dest_image_paths() * check_dest_paths() * Logic that needs to be cleaned up: * Calculating overall recovery size * Pass "Done"ness and status strings need separated * Pass "Done"ness at the device and overall levels * Updating output for side pane status display --- .bin/Scripts/functions/ddrescue.py | 126 ++++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 19 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index aa0655d5..341d1266 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -13,7 +13,6 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES -RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] AUTO_NEXT_PASS_1_THRESHOLD = 90 AUTO_NEXT_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { @@ -29,16 +28,98 @@ DDRESCUE_SETTINGS = { '--timeout=': {'Enabled': True, 'Value': '5m'}, '-vvvv': {'Enabled': True, 'Hidden': True}, } -REGEX_MAP_DATA = re.compile(r'^\s*(?P\S+):.*\(\s*(?P\d+\.?\d*)%.*') -REGEX_MAP_STATUS = re.compile(r'.*current status:\s+(?P.*)') -STATUS_COLOR_CLEAR = ('Pending',) -STATUS_COLOR_YELLOW = ('Skipped', 'Unknown', 'Working') +RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] +SIDE_PANE_WIDTH = 21 USAGE = """ {script_name} clone [source [destination]] {script_name} image [source [destination]] (e.g. {script_name} clone /dev/sda /dev/sdb) """ +# Clases +class BlockPair(): + """Object to track data and methods together for source and dest.""" + def __init__(self, source_path, dest_path, map_path, total_size): + self.source_path = source_path + self.dest_path = dest_path + self.map_path = map_path + self.pass_done = [False, False, False] + self.rescued = 0 + self.total_size = total_size + self.status = ['Pending', 'Pending', 'Pending'] + + def finish_pass(pass_num): + """Mark pass as done and check if 100% recovered.""" + if map_data['full recovery']: + self.pass_done = [True, True, True] + self.recovered = self.total_size + self.status[pass_num] = get_formatted_status(100) + else: + self.pass_done[pass_num] = True + + def get_pass_done(pass_num): + """Return pass number's done state.""" + return self.pass_done[pass_num] + + def get_rescued(): + """Return rescued size.""" + return self.rescued + + def update_progress(pass_num): + """Update progress using map file.""" + if os.path.exists(self.map_path): + map_data = read_map_file(self.map_path) + self.rescued = map_data['rescued'] * self.total_size + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num), + data=(self.rescued/self.total_size)*100) + + +class RecoveryState(): + """Object to track BlockPair objects and overall state.""" + def __init__(self, mode): + self.block_pairs = [] + self.current_pass = 0 + self.finished = False + self.mode = mode.lower() + self.rescued = 0 + self.started = False + self.total_size = 0 + if mode not in ('clone', 'image'): + raise GenericError('Unsupported mode') + + def add_block_pair(obj): + """Append BlockPair object to internal list.""" + self.block_pairs.append(obj) + + def set_pass_num(): + """Set current pass based on all block-pair's progress.""" + self.current_pass = 0 + for pass_num in (2, 1, 0): + # Iterate backwards through passes + pass_done = True + for bp in self.block_pairs: + pass_done &= bp.get_pass_done(pass_num) + if pass_done: + # All block-pairs reported being done + # Set to next pass, unless we're on the last pass (2) + self.current_pass = min(2, pass_num + 1) + if pass_num == 2: + # Also mark overall recovery as finished if on last pass + self.finished = True + break + + def update_progress(): + """Update overall progress using block_pairs.""" + self.rescued = 0 + for bp in self.block_pairs: + self.rescued += bp.get_rescued() + self.status_percent = get_formatted_status( + label='Recovered:', data=(self.rescued/self.total_size)*100) + self.status_amount = get_formatted_status( + label='', data=human_readable_size(self.rescued)) + + # Functions def abort_ddrescue_tui(): run_program(['losetup', '-D']) @@ -58,7 +139,7 @@ def build_outer_panes(source, dest): **COLORS)) tmux_splitw( '-t', source_pane, - '-dhl', '21', + '-dhl', '{}'.format(SIDE_PANE_WIDTH), 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( text=time.strftime("%Y-%m-%d %H:%M %Z"), **COLORS)) @@ -72,7 +153,7 @@ def build_outer_panes(source, dest): # Side pane update_progress(source) tmux_splitw( - '-dhl', '21', + '-dhl', '{}'.format(SIDE_PANE_WIDTH), 'watch', '--color', '--no-title', '--interval', '1', 'cat', source['Progress Out']) @@ -217,8 +298,13 @@ def get_device_size_in_bytes(s): return convert_to_bytes(s) +def get_formatted_status(label, data): + """TODO""" + # TODO + pass def get_recovery_scope_size(source): """Calculate total size of selected dev(s).""" + # TODO function deprecated source['Total Size'] = 0 if source['Children']: for child in source['Children']: @@ -240,9 +326,9 @@ def get_status_color(s, t_success=99, t_warn=90): # Status is either in lists below or will default to red pass - if s in STATUS_COLOR_CLEAR: + if s in ('Pending',): color = COLORS['CLEAR'] - elif s in STATUS_COLOR_YELLOW: + elif s in ('Skipped', 'Unknown', 'Working'): color = COLORS['YELLOW'] elif p_recovered >= t_success: color = COLORS['GREEN'] @@ -727,22 +813,19 @@ def menu_settings(source): def read_map_file(map_path): """Read map file with ddrescuelog and return data as dict.""" map_data = {} - try: - result = run_program(['ddrescuelog', '-t', map_path]) - except subprocess.CalledProcessError: - print_error('Failed to read map data') - abort_ddrescue_tui() + result = run_program(['ddrescuelog', '-t', map_path]) # Parse output for line in result.stdout.decode().splitlines(): - m = REGEX_MAP_DATA.match(line.strip()) + m = re.match( + r'^\s*(?P\S+):.*\(\s*(?P\d+\.?\d*)%.*', line.strip()) if m: try: map_data[m.group('key')] = float(m.group('value')) except ValueError: print_error('Failed to read map data') abort_ddrescue_tui() - m = REGEX_MAP_STATUS.match(line.strip()) + m = re.match(r'.*current status:\s+(?P.*)', line.strip()) if m: map_data['pass completed'] = bool(m.group('status') == 'finished') @@ -759,6 +842,7 @@ def read_map_file(map_path): def resume_from_map(source): """Read map file(s) and set current progress to resume previous session.""" + # TODO function deprecated map_data_read = False non_tried = 0 non_trimmed = 0 @@ -961,7 +1045,8 @@ def select_dest_path(provided_path=None, skip_device={}): # Set display name result = run_program(['tput', 'cols']) - width = int((int(result.stdout.decode().strip()) - 21) / 2) - 2 + width = int( + (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 if len(dest['Path']) > width: dest['Display Name'] = '...{}'.format(dest['Path'][-(width-3):]) else: @@ -1019,7 +1104,8 @@ def select_device(description='device', provided_path=None, dev['Display Name'] = '{name} {size} {model}'.format( **dev['Details']) result = run_program(['tput', 'cols']) - width = int((int(result.stdout.decode().strip()) - 21) / 2) - 2 + width = int( + (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 if len(dev['Display Name']) > width: if dev['Is Image']: dev['Display Name'] = '...{}'.format( @@ -1035,6 +1121,7 @@ def select_device(description='device', provided_path=None, def set_dest_image_paths(source, dest): """Set destination image path for source and any child devices.""" + # TODO function deprecated if source['Type'] == 'Clone': base = '{pwd}/Clone_{size}_{model}'.format( pwd=os.path.realpath(global_vars['Env']['PWD']), @@ -1270,7 +1357,8 @@ def update_progress(source, end_run=False): s_color=get_status_color(recovered_p), recovered_p=recovered_p, **COLORS)) - output.append('{:>21}'.format(recovered_s)) + output.append('{recovered_s:>{width}}'.format( + recovered_s=recovered_s, width=SIDE_PANE_WIDTH)) output.append('─────────────────────') # Main device From fa3b3b11b05ef2ade99372ba409652d382361a94 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Jul 2018 00:39:22 -0600 Subject: [PATCH 057/138] Added methods: load_map and self_check(s) * load_map() is called on BlockPair() instantiation * This partially replaces resume_from_map() * Also fixed improper method declarations lacking the self argument --- .bin/Scripts/functions/ddrescue.py | 71 ++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 341d1266..9e1b8ee7 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -39,33 +39,75 @@ USAGE = """ {script_name} clone [source [destination]] # Clases class BlockPair(): """Object to track data and methods together for source and dest.""" - def __init__(self, source_path, dest_path, map_path, total_size): + def __init__(self, source_path, dest_path, map_path, mode, total_size): self.source_path = source_path self.dest_path = dest_path self.map_path = map_path + self.mode = mode self.pass_done = [False, False, False] self.rescued = 0 self.total_size = total_size self.status = ['Pending', 'Pending', 'Pending'] + if os.path.exists(self.map_path): + self.load_map_data() - def finish_pass(pass_num): + def finish_pass(self, pass_num): """Mark pass as done and check if 100% recovered.""" if map_data['full recovery']: self.pass_done = [True, True, True] - self.recovered = self.total_size + self.rescued = self.total_size self.status[pass_num] = get_formatted_status(100) + # Mark future passes as Skipped + pass_num += 1 + while pass_num <= 2 + self.status[pass_num] = 'Skipped' + pass_num += 1 else: self.pass_done[pass_num] = True - def get_pass_done(pass_num): + def get_pass_done(self, pass_num): """Return pass number's done state.""" return self.pass_done[pass_num] - def get_rescued(): + def get_rescued(self): """Return rescued size.""" return self.rescued - def update_progress(pass_num): + def load_map_data(self): + """Load data from map file and set progress.""" + map_data = read_map_file(self.map_path) + self.rescued = map_data['rescued'] * self.total_size + if map_data['full recovery']: + self.pass_done = [True, True, True] + self.rescued = self.total_size + self.status = ['Skipped', 'Skipped', 'Skipped'] + elif map_data['non-tried'] > 0: + # Initial pass incomplete + pass + elif map_data['non-trimmed'] > 0: + self.pass_done[0] = True + self.status[0] = 'Skipped' + elif map_data['non-scraped'] > 0: + self.pass_done = [True, True, False] + self.status = ['Skipped', 'Skipped', 'Pending'] + + def self_check(self): + """Self check to abort on bad dest/map combinations.""" + dest_exists = os.path.exists(self.dest_path) + map_exists = os.path.exists(self.map_path) + if self.mode == 'image': + if dest_exists and not map_exists: + raise GenericError( + 'Detected image "{}" but not the matching map'.format( + self.dest_path)) + elif not dest_exists and map_exists: + raise GenericError( + 'Detected map "{}" but not the matching image'.format( + self.map_path)) + elif not dest_exists: + raise Genericerror('Destination device missing') + + def update_progress(self, pass_num): """Update progress using map file.""" if os.path.exists(self.map_path): map_data = read_map_file(self.map_path) @@ -83,16 +125,27 @@ class RecoveryState(): self.finished = False self.mode = mode.lower() self.rescued = 0 + self.resumed = False self.started = False self.total_size = 0 if mode not in ('clone', 'image'): raise GenericError('Unsupported mode') - def add_block_pair(obj): + def add_block_pair(self, obj): """Append BlockPair object to internal list.""" + obj.set_mode(self.mode) self.block_pairs.append(obj) - def set_pass_num(): + def self_checks(self): + """Run self-checks for each BlockPair object.""" + for bp in self.block_pairs: + try: + bp.self_check() + except GenericError as err: + print_error(err) + abort_ddrescue_tui() + + def set_pass_num(self): """Set current pass based on all block-pair's progress.""" self.current_pass = 0 for pass_num in (2, 1, 0): @@ -109,7 +162,7 @@ class RecoveryState(): self.finished = True break - def update_progress(): + def update_progress(self): """Update overall progress using block_pairs.""" self.rescued = 0 for bp in self.block_pairs: From 98b05c93bf16c90d2ceb2c5e0cc509b0c763c6e4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 24 Jul 2018 00:54:58 -0600 Subject: [PATCH 058/138] Moved menu_ddrescue() to ddrescue-tui-menu file * Let's parse the sys.argv earlier in the process --- .bin/Scripts/ddrescue-tui-menu | 24 +++++++++++++++++++-- .bin/Scripts/functions/ddrescue.py | 34 +++++------------------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index a8205ded..798cd43e 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -16,9 +16,29 @@ if __name__ == '__main__': try: # Prep clear_screen() + args = list(sys.argv) + run_mode = '' + source_path = None + dest_path = None - # Show menu - menu_ddrescue(*sys.argv) + # Parse args + try: + script_name = os.path.basename(args.pop(0)) + run_mode = str(args.pop(0)).lower() + source_path = args.pop(0) + dest_path = args.pop(0) + except IndexError: + # We'll set the missing paths later + pass + + # Show usage? + if run_mode in ('clone', 'image'): + menu_ddrescue(source_path, dest_path, run_mode) + else: + if not re.search(r'(^$|help|-h|\?)', run_mode, re.IGNORECASE): + print_error('Invalid mode.') + show_usage(script_name) + exit_script() # Done print_standard('\nDone.') diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 9e1b8ee7..d76ee2cc 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -59,7 +59,7 @@ class BlockPair(): self.status[pass_num] = get_formatted_status(100) # Mark future passes as Skipped pass_num += 1 - while pass_num <= 2 + while pass_num <= 2: self.status[pass_num] = 'Skipped' pass_num += 1 else: @@ -446,34 +446,10 @@ def menu_clone(source_path, dest_path): exit_script() -def menu_ddrescue(*args): - """Main ddrescue loop/menu.""" - args = list(args) - script_name = os.path.basename(args.pop(0)) - run_mode = '' - source_path = None - dest_path = None - - # Parse args - try: - run_mode = args.pop(0) - source_path = args.pop(0) - dest_path = args.pop(0) - except IndexError: - # We'll set the missing paths later - pass - - # Show proper menu or exit - if run_mode == 'clone': - menu_clone(source_path, dest_path) - elif run_mode == 'image': - menu_image(source_path, dest_path) - else: - if not re.search(r'(^$|help|-h|\?)', run_mode, re.IGNORECASE): - print_error('Invalid mode.') - show_usage(script_name) - exit_script() - +def menu_ddrescue(source_path, dest_path, run_mode): + """Main ddrescue menu.""" + # TODO Merge menu_clone and menu_image here + pass def menu_image(source_path, dest_path): """ddrescue imaging menu.""" From 180eb0f9ef36b6953b4c39168db8736018193402 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 25 Jul 2018 21:44:40 -0600 Subject: [PATCH 059/138] Added base, dev, dir, and image objects --- .bin/Scripts/functions/ddrescue.py | 76 ++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index d76ee2cc..5698859a 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -173,6 +173,68 @@ class RecoveryState(): label='', data=human_readable_size(self.rescued)) +class BaseObj(): + """Base object used by DevObj, DirObj, and ImageObj.""" + def __init__(self, path): + self.type = 'Base' + self.path = os.path.realpath(path) + self.set_details() + + def is_dev(): + return self.type == 'Dev' + + def is_dir(): + return self.type == 'Dir' + + def is_image(): + return self.type == 'Image' + + def self_check(): + pass + + def set_details(self): + pass + + +class DevObj(BaseObj): + """Block device object.""" + def self_check(self): + """Verify that self.path points to a block device.""" + if not pathlib.Path(self.path).is_block_device(): + raise GenericError('TODO') + + def set_details(self): + """Set details via lsblk.""" + self.type = 'Dev' + # TODO Run lsblk + + +class DirObj(BaseObj): + def self_check(self): + """Verify that self.path points to a directory.""" + if not pathlib.Path(self.path).is_dir(): + raise GenericError('TODO') + + def set_details(self): + """Set details via findmnt.""" + self.type = 'Dir' + # TODO Run findmnt + + +class ImageObj(BaseObj): + def self_check(self): + """Verify that self.path points to a file.""" + if not pathlib.Path(self.path).is_file(): + raise GenericError('TODO') + + def set_details(self): + """Setup loopback device and set details via lsblk.""" + self.type = 'Image' + # TODO Run losetup + # TODO Run lsblk + # TODO Remove loopback device + + # Functions def abort_ddrescue_tui(): run_program(['losetup', '-D']) @@ -253,6 +315,20 @@ def check_dest_paths(source): abort_ddrescue_tui() +def create_path_obj(path): + """Create Dev, Dir, or Image obj based on path given.""" + obj = None + if pathlib.Path(self.path).is_block_device(): + obj = Dev(path) + elif pathlib.Path(self.path).is_dir(): + obj = DirObj(path) + elif pathlib.Path(self.path).is_file(): + obj = ImageObj(path) + else: + raise GenericAbort('TODO') + return obj + + def dest_safety_check(source, dest): """Verify the destination is appropriate for the source.""" source_size = source['Details']['size'] From 66c756333592efc20f175af7c86145b22780f526 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 25 Jul 2018 23:31:04 -0600 Subject: [PATCH 060/138] Set details for Dev/Dir/Image objects * Colored report data is generated during obj instantiation * Code has been moved into its own function * Entire colored string is now stored for each Obj * (Should make show_selection, etc more mode/Obj agnostic) * loopback_dev vs image_path is now better separated * losetup is called in ImageObj.set_details() * loopback -D is still called during program cleanup/wrapup * get_device_size_in_bytes() has been renamed get_size_in_bytes() --- .bin/Scripts/functions/ddrescue.py | 111 +++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 28 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 5698859a..1484b00c 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -206,7 +206,10 @@ class DevObj(BaseObj): def set_details(self): """Set details via lsblk.""" self.type = 'Dev' - # TODO Run lsblk + self.details = get_device_details(self.path) + self.name = self.details.get('name', 'UNKNOWN') + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_device_report(self.path) class DirObj(BaseObj): @@ -218,7 +221,10 @@ class DirObj(BaseObj): def set_details(self): """Set details via findmnt.""" self.type = 'Dir' - # TODO Run findmnt + self.details = get_dir_details(self.path) + self.name = self.path + self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) + self.report = get_dir_report(self.path) class ImageObj(BaseObj): @@ -230,9 +236,12 @@ class ImageObj(BaseObj): def set_details(self): """Setup loopback device and set details via lsblk.""" self.type = 'Image' - # TODO Run losetup - # TODO Run lsblk - # TODO Remove loopback device + self.loop_dev = setup_loopback_device(self.path) + self.details = get_image_details(self.loopdev) + self.name = self.path[self.path.rfind('/')+1:] + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_image_report(self.loop_dev) + self.report = self.report.replace(self.loop_dev, '{Img}') # Functions @@ -333,28 +342,14 @@ def dest_safety_check(source, dest): """Verify the destination is appropriate for the source.""" source_size = source['Details']['size'] if dest['Is Dir']: - cmd = [ - 'findmnt', '-J', - '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', - '-T', dest['Path']] - result = run_program(cmd) - try: - json_data = json.loads(result.stdout.decode()) - except Exception: - # Welp, let's abort - print_error('Failed to verify destination usability.') - abort_ddrescue_tui() - else: - dest_size = json_data['filesystems'][0]['avail'] - dest['Free Space'] = dest_size - dest['Filesystem'] = json_data['filesystems'][0]['fstype'] - dest['Mount options'] = json_data['filesystems'][0]['options'] + # MOVED + pass else: dest_size = dest['Details']['size'] # Convert to bytes and compare size - source_size = get_device_size_in_bytes(source_size) - dest_size = get_device_size_in_bytes(dest_size) + source_size = get_size_in_bytes(source_size) + dest_size = get_size_in_bytes(dest_size) if source['Type'] == 'Image' and dest_size < (source_size * 1.2): # Imaging: ensure 120% of source size is available print_error( @@ -413,15 +408,75 @@ def get_device_details(dev_path): dev_path) result = run_program(cmd) except CalledProcessError: - print_error('Failed to get device details for {}'.format(dev_path)) - abort_ddrescue_tui() + # Return empty dict and let calling section deal with the issue + return {} json_data = json.loads(result.stdout.decode()) # Just return the first device (there should only be one) return json_data['blockdevices'][0] -def get_device_size_in_bytes(s): +def get_device_report(dev_path): + """Build colored device report using lsblk, returns str.""" + result = run_program([ + 'lsblk', '--nodeps', + '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', + dev_path]) + lines = result.stdout.decode().strip().splitlines() + lines.append('') + + # FS details (if any) + result = run_program([ + 'lsblk', + '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', + dev_path]) + lines.extend(result.stdout.decode().strip().splitlines()) + + # Color label lines + output = [] + for line in lines: + if line[0:4] == 'NAME': + output.append('{BLUE}{line}{CLEAR}'.format(line=line, **COLORS)) + else: + output.append(line) + + # Done + return '\n'.join(output) + + +def get_dir_details(dir_path): + """Get dir details via findmnt, returns JSON dict.""" + try: + result = run_program([ + 'findmnt', '-J', + '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', + '-T', dir_path]) + json_data = json.loads(result.stdout.decode()) + except Exception: + raise GenericError( + 'Failed to get directory details for "{}".'.format(self.path)) + else: + return json_data['filesystems'][0] + + +def get_dir_report(dir_path): + """Build colored dir report using findmnt, returns str.""" + output = [] + result = run_program([ + 'findmnt', + '--output', 'SIZE,AVAIL,USED,FSTYPE,OPTIONS', + '--target', dir_path]) + for line in result.stdout.decode().strip().splitlines(): + if 'FSTYPE' in line: + output.append('{BLUE}{line}{CLEAR}'.format(line=line, **COLORS)) + else: + output.append(line) + + # Done + return '\n'.join(output) + + +def get_size_in_bytes(s): """Convert size string from lsblk string to bytes, returns int.""" s = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', s, re.IGNORECASE) return convert_to_bytes(s) @@ -437,11 +492,11 @@ def get_recovery_scope_size(source): source['Total Size'] = 0 if source['Children']: for child in source['Children']: - child['Size'] = get_device_size_in_bytes(child['Details']['size']) + child['Size'] = get_size_in_bytes(child['Details']['size']) source['Total Size'] += child['Size'] else: # Whole dev - source['Size'] = get_device_size_in_bytes(source['Details']['size']) + source['Size'] = get_size_in_bytes(source['Details']['size']) source['Total Size'] = source['Size'] From 6aeba34bdb3fde9e3d67e77af5b234532534d202 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 25 Jul 2018 23:57:50 -0600 Subject: [PATCH 061/138] Include path in dir report --- .bin/Scripts/functions/ddrescue.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 1484b00c..d2b1f84b 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -462,15 +462,23 @@ def get_dir_details(dir_path): def get_dir_report(dir_path): """Build colored dir report using findmnt, returns str.""" output = [] + width = len(dir_path)+1 result = run_program([ 'findmnt', '--output', 'SIZE,AVAIL,USED,FSTYPE,OPTIONS', '--target', dir_path]) for line in result.stdout.decode().strip().splitlines(): if 'FSTYPE' in line: - output.append('{BLUE}{line}{CLEAR}'.format(line=line, **COLORS)) + output.append('{BLUE}{path:<{width}}{line}{CLEAR}'.format( + path=dir_path, + width=width, + line=line, + **COLORS)) else: - output.append(line) + output.append('{path:<{width}}{line}'.format( + path=dir_path, + width=width, + line=line)) # Done return '\n'.join(output) From d1eefd05ab1c9d7b38c8c10f196bc37dd1e3fb55 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 17:31:39 -0600 Subject: [PATCH 062/138] Major update to refactor for object-centricity * Dest/map paths are now set in two steps: * The filename prefix is set when creating the DevObj() * The full paths are set when creating the BlockPair() * Merged dest safety checks into RecoveryState.add_block_pair() * Mostly check_dest_paths() and dest_safety_check() * Moved dir RWX checks to is_writable_dir() * Moved mount RW check to is_writable_filesystem() * Started merging menu_clone() and menu_image() into menu_ddrescue() --- .bin/Scripts/functions/ddrescue.py | 229 ++++++++++++++++++++++------- 1 file changed, 177 insertions(+), 52 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index d2b1f84b..6a264127 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -39,15 +39,29 @@ USAGE = """ {script_name} clone [source [destination]] # Clases class BlockPair(): """Object to track data and methods together for source and dest.""" - def __init__(self, source_path, dest_path, map_path, mode, total_size): - self.source_path = source_path - self.dest_path = dest_path - self.map_path = map_path + def __init__(self, source, dest, mode): + self.source_path = source.path self.mode = mode + self.name = source.name self.pass_done = [False, False, False] self.rescued = 0 - self.total_size = total_size self.status = ['Pending', 'Pending', 'Pending'] + self.total_size = source.size + # Set dest paths + if self.mode == 'clone': + # Cloning + self.dest_path = dest.path + self.map_path = '{pwd}/Clone_{prefix}.map'.format( + pwd=os.path.realpath(global_vars['Env']['PWD']), + prefix=source.prefix) + else: + # Imaging + self.dest_path = '{path}/{prefix}.dd'.format( + path=dest.path, + prefix=source.prefix) + self.map_path = '{path}/{prefix}.map'.format( + path=dest.path, + prefix=source.prefix) if os.path.exists(self.map_path): self.load_map_data() @@ -69,10 +83,6 @@ class BlockPair(): """Return pass number's done state.""" return self.pass_done[pass_num] - def get_rescued(self): - """Return rescued size.""" - return self.rescued - def load_map_data(self): """Load data from map file and set progress.""" map_data = read_map_file(self.map_path) @@ -105,7 +115,7 @@ class BlockPair(): 'Detected map "{}" but not the matching image'.format( self.map_path)) elif not dest_exists: - raise Genericerror('Destination device missing') + raise GenericError('Destination device missing') def update_progress(self, pass_num): """Update progress using map file.""" @@ -131,10 +141,48 @@ class RecoveryState(): if mode not in ('clone', 'image'): raise GenericError('Unsupported mode') - def add_block_pair(self, obj): - """Append BlockPair object to internal list.""" - obj.set_mode(self.mode) - self.block_pairs.append(obj) + def add_block_pair(self, source, dest): + """Run safety checks and append new BlockPair to internal list.""" + if self.mode == 'clone': + # Cloning safety checks + if source.is_dir(): + raise GenericAbort('Invalid source "{}"'.format( + source.path)) + elif not dest.is_dev(): + raise GenericAbort('Invalid destination "{}"'.format( + dest.path)) + elif source.size > dest.size: + raise GenericAbort( + 'Destination is too small, refusing to continue.') + else: + # Imaging safety checks + if not source.is_dev(): + raise GenericAbort('Invalid source "{}"'.format( + source.path)) + elif not dest.is_dir(): + raise GenericAbort('Invalid destination "{}"'.format( + dest.path)) + elif (source.size * 1.2) > dest.size: + raise GenericAbort( + 'Destination is too small, refusing to continue.') + elif dest.fstype.lower() not in RECOMMENDED_FSTYPES: + print_error( + 'Destination filesystem "{}" is not recommended.'.format( + dest.fstype.upper())) + print_info('Recommended types are: {}'.format( + ' / '.join(RECOMMENDED_FSTYPES).upper())) + print_standard(' ') + if not ask('Proceed anyways? (Strongly discouraged)'): + raise GenericAbort('Aborted.') + elif not is_writable_dir(dest): + raise GenericAbort( + 'Destination is not writable, refusing to continue.') + elif not is_writable_filesystem(dest): + raise GenericAbort( + 'Destination is mounted read-only, refusing to continue.') + + # Safety checks passed + self.block_pairs.append(BlockPair(source, dest)) def self_checks(self): """Run self-checks for each BlockPair object.""" @@ -143,7 +191,7 @@ class RecoveryState(): bp.self_check() except GenericError as err: print_error(err) - abort_ddrescue_tui() + raise GenericAbort('Aborted.') def set_pass_num(self): """Set current pass based on all block-pair's progress.""" @@ -165,8 +213,10 @@ class RecoveryState(): def update_progress(self): """Update overall progress using block_pairs.""" self.rescued = 0 + self.total_size = 0 for bp in self.block_pairs: - self.rescued += bp.get_rescued() + self.rescued += bp.rescued + self.total_size += bp.size self.status_percent = get_formatted_status( label='Recovered:', data=(self.rescued/self.total_size)*100) self.status_amount = get_formatted_status( @@ -176,24 +226,24 @@ class RecoveryState(): class BaseObj(): """Base object used by DevObj, DirObj, and ImageObj.""" def __init__(self, path): - self.type = 'Base' + self.type = 'base' self.path = os.path.realpath(path) self.set_details() - def is_dev(): - return self.type == 'Dev' + def is_dev(self): + return self.type == 'dev' - def is_dir(): - return self.type == 'Dir' + def is_dir(self): + return self.type == 'dir' - def is_image(): - return self.type == 'Image' + def is_image(self): + return self.type == 'image' - def self_check(): + def self_check(self): pass def set_details(self): - pass + self.details = {} class DevObj(BaseObj): @@ -205,11 +255,32 @@ class DevObj(BaseObj): def set_details(self): """Set details via lsblk.""" - self.type = 'Dev' + self.type = 'dev' self.details = get_device_details(self.path) self.name = self.details.get('name', 'UNKNOWN') + self.model = self.details.get('model', 'UNKNOWN') + self.model_size = self.details.get('size', 'UNKNOWN') self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) self.report = get_device_report(self.path) + self.parent = self.details.get('pkname', '') + self.label = self.details.get('label', '') + if not self.label: + # Force empty string in case it's set to None + self.label = '' + self.update_filename_prefix() + + def update_filename_prefix(self): + """Set filename prefix based on details.""" + self.prefix = '{m_size}_{model}'.format( + m_size=self.model_size, + model=self.model) + if self.parent: + # Add child device details + self.prefix += '_{c_num}_{c_size}{sep}{c_label}'.format( + c_num=self.name.replace(self.parent, ''), + c_size=self.details.get('size', 'UNKNOWN'), + sep='_' if self.label else '', + c_label=self.label) class DirObj(BaseObj): @@ -220,8 +291,9 @@ class DirObj(BaseObj): def set_details(self): """Set details via findmnt.""" - self.type = 'Dir' + self.type = 'dir' self.details = get_dir_details(self.path) + self.fstype = self.details.get('fstype', 'UNKNOWN') self.name = self.path self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) self.report = get_dir_report(self.path) @@ -234,14 +306,16 @@ class ImageObj(BaseObj): raise GenericError('TODO') def set_details(self): - """Setup loopback device and set details via lsblk.""" - self.type = 'Image' + """Setup loopback device, set details via lsblk, then detach device.""" + self.type = 'image' self.loop_dev = setup_loopback_device(self.path) - self.details = get_image_details(self.loopdev) + self.details = get_device_details(self.loopdev) + self.details['model'] = 'ImageFile' self.name = self.path[self.path.rfind('/')+1:] self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) - self.report = get_image_report(self.loop_dev) + self.report = get_device_report(self.loop_dev) self.report = self.report.replace(self.loop_dev, '{Img}') + run_program(['losetup', '--detach', loop_path], check=False) # Functions @@ -284,9 +358,9 @@ def build_outer_panes(source, dest): def check_dest_paths(source): """Check for image and/or map file and alert user about details.""" - dd_image_exists = os.path.exists(source['Dest Paths']['Image']) + dd_image_exists = os.path.exists(source['Dest Paths']['image']) map_exists = os.path.exists(source['Dest Paths']['Map']) - if 'Clone' in source['Dest Paths']['Map']: + if 'clone' in source['Dest Paths']['Map']: if map_exists: # We're cloning and a matching map file was detected if not ask('Matching map file detected, resume recovery?'): @@ -299,11 +373,11 @@ def check_dest_paths(source): source_devs = source['Children'] for dev in source_devs: # We're imaging - dd_image_exists = os.path.exists(dev['Dest Paths']['Image']) + dd_image_exists = os.path.exists(dev['Dest Paths']['image']) map_exists = os.path.exists(dev['Dest Paths']['Map']) if dd_image_exists and not map_exists: # Refuce to resume without map file - i = dev['Dest Paths']['Image'] + i = dev['Dest Paths']['image'] i = i[i.rfind('/')+1:] print_error( 'Detected image "{}" but not the matching map'.format(i)) @@ -350,7 +424,7 @@ def dest_safety_check(source, dest): # Convert to bytes and compare size source_size = get_size_in_bytes(source_size) dest_size = get_size_in_bytes(dest_size) - if source['Type'] == 'Image' and dest_size < (source_size * 1.2): + if source['Type'] == 'image' and dest_size < (source_size * 1.2): # Imaging: ensure 120% of source size is available print_error( 'Not enough free space on destination, refusing to continue.') @@ -359,7 +433,7 @@ def dest_safety_check(source, dest): d_size=human_readable_size(dest_size), s_size=human_readable_size(source_size * 1.2))) abort_ddrescue_tui() - elif source['Type'] == 'Clone' and source_size > dest_size: + elif source['Type'] == 'clone' and source_size > dest_size: # Cloning: ensure dest >= size print_error('Destination is too small, refusing to continue.') print_standard( @@ -369,7 +443,7 @@ def dest_safety_check(source, dest): abort_ddrescue_tui() # Imaging specific checks - if source['Type'] == 'Image': + if source['Type'] == 'image': # Filesystem Type if dest['Filesystem'] not in RECOMMENDED_FSTYPES: print_error( @@ -531,6 +605,21 @@ def get_status_color(s, t_success=99, t_warn=90): return color +def is_writable_dir(dir_obj): + """Check if we have read-write-execute permissions, returns bool.""" + is_ok = True + path_st_mode = os.stat(dir_obj.path).st_mode + is_ok == is_ok and path_st_mode & stat.S_IRUSR + is_ok == is_ok and path_st_mode & stat.S_IWUSR + is_ok == is_ok and path_st_mode & stat.S_IXUSR + return is_ok + + +def is_writable_filesystem(dir_obj): + """Check if filesystem is mounted read-write, returns bool.""" + return 'rw' in dir_obj.details.get('options', '') + + def mark_all_passes_pending(source): """Mark all devs and passes as pending in preparation for retry.""" source['Current Pass'] = 'Pass 1' @@ -555,7 +644,7 @@ def menu_clone(source_path, dest_path): source['Recovered Size'] = 0 source['Started Recovery'] = False source['Total Size'] = 0 - source['Type'] = 'Clone' + source['Type'] = 'clone' dest = select_device( 'destination', dest_path, skip_device=source['Details'], allow_image_file=False) @@ -586,9 +675,45 @@ def menu_clone(source_path, dest_path): def menu_ddrescue(source_path, dest_path, run_mode): - """Main ddrescue menu.""" - # TODO Merge menu_clone and menu_image here - pass + """ddrescue menu.""" + source = None + dest = None + if source_path: + source = create_path_obj(source_path) + if dest_path: + dest = create_path_obj(dest_path) + + # Show selection menus (if necessary) + if not source: + source = select_device('source') + if not dest: + if run_mode == 'clone': + dest = select_device('destination', skip_device=source) + else: + dest = select_directory() + + # Build BlockPairs + state = RecoveryState(run_mode) + if run_mode == 'clone': + state.add_block_pair(source, dest) + else: + # TODO select dev or child dev(s) + + # Confirmations + # TODO Show selection details + # TODO resume? + # TODO Proceed? (maybe merge with resume? prompt?) + # TODO double-confirm for clones for safety + + # Main menu + build_outer_panes(source, dest) + # TODO Fix + #menu_main(source, dest) + pause('Fake Main Menu... ') + + # Done + run_program(['tmux', 'kill-window']) + exit_script() def menu_image(source_path, dest_path): """ddrescue imaging menu.""" @@ -602,7 +727,7 @@ def menu_image(source_path, dest_path): source['Recovered Size'] = 0 source['Started Recovery'] = False source['Total Size'] = 0 - source['Type'] = 'Image' + source['Type'] = 'image' dest = select_dest_path(dest_path, skip_device=source['Details']) dest_safety_check(source, dest) @@ -1017,7 +1142,7 @@ def resume_from_map(source): non_scraped = 0 # Read map data - if source['Type'] != 'Clone' and source['Children']: + if source['Type'] != 'clone' and source['Children']: # Imaging child device(s) for child in source['Children']: if os.path.exists(child['Dest Paths']['Map']): @@ -1130,14 +1255,14 @@ def run_ddrescue(source, dest, settings): update_progress(source) # Set ddrescue cmd - if source['Type'] == 'Clone': + if source['Type'] == 'clone': cmd = [ 'ddrescue', *settings, '--force', s_dev['Dev Path'], dest['Dev Path'], s_dev['Dest Paths']['Map']] else: cmd = [ 'ddrescue', *settings, s_dev['Dev Path'], - s_dev['Dest Paths']['Image'], s_dev['Dest Paths']['Map']] + s_dev['Dest Paths']['image'], s_dev['Dest Paths']['Map']] if current_pass == 'Pass 1': cmd.extend(['--no-trim', '--no-scrape']) elif current_pass == 'Pass 2': @@ -1290,7 +1415,7 @@ def select_device(description='device', provided_path=None, def set_dest_image_paths(source, dest): """Set destination image path for source and any child devices.""" # TODO function deprecated - if source['Type'] == 'Clone': + if source['Type'] == 'clone': base = '{pwd}/Clone_{size}_{model}'.format( pwd=os.path.realpath(global_vars['Env']['PWD']), size=source['Details']['size'], @@ -1301,7 +1426,7 @@ def set_dest_image_paths(source, dest): model=source['Details'].get('model', 'Unknown'), **dest) source['Dest Paths'] = { - 'Image': '{}.dd'.format(base), + 'image': '{}.dd'.format(base), 'Map': '{}.map'.format(base)} # Child devices @@ -1318,7 +1443,7 @@ def set_dest_image_paths(source, dest): p_label=p_label, **dest) child['Dest Paths'] = { - 'Image': '{}.dd'.format(base), + 'image': '{}.dd'.format(base), 'Map': '{}.map'.format(base)} @@ -1386,7 +1511,7 @@ def show_selection_details(source, dest): print_standard(' ') # Destination - if source['Type'] == 'Clone': + if source['Type'] == 'clone': print_success('Destination device ', end='') print_error('(ALL DATA WILL BE DELETED)', timestamp=False) show_device_details(dest['Dev Path']) @@ -1511,7 +1636,7 @@ def update_progress(source, end_run=False): source['Progress Out'] = '{}/progress.out'.format( global_vars['LogDir']) output = [] - if source['Type'] == 'Clone': + if source['Type'] == 'clone': output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) else: output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) @@ -1530,7 +1655,7 @@ def update_progress(source, end_run=False): output.append('─────────────────────') # Main device - if source['Type'] == 'Clone': + if source['Type'] == 'clone': output.append('{BLUE}{dev}{CLEAR}'.format( dev='Image File' if source['Is Image'] else source['Dev Path'], **COLORS)) From 127c3b810d56c939782a66e40b8a15daa6faad6d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 18:07:12 -0600 Subject: [PATCH 063/138] Fixed image prefixes and removed unsused functions --- .bin/Scripts/functions/ddrescue.py | 233 +---------------------------- 1 file changed, 5 insertions(+), 228 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 6a264127..ca4226c5 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -115,7 +115,8 @@ class BlockPair(): 'Detected map "{}" but not the matching image'.format( self.map_path)) elif not dest_exists: - raise GenericError('Destination device missing') + raise GenericError('Destination device "{}" missing'.format( + self.dest_path)) def update_progress(self, pass_num): """Update progress using map file.""" @@ -312,6 +313,8 @@ class ImageObj(BaseObj): self.details = get_device_details(self.loopdev) self.details['model'] = 'ImageFile' self.name = self.path[self.path.rfind('/')+1:] + self.prefix = '{}_ImageFile'.format( + self.details.get('size', 'UNKNOWN')) self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) self.report = get_device_report(self.loop_dev) self.report = self.report.replace(self.loop_dev, '{Img}') @@ -356,48 +359,6 @@ def build_outer_panes(source, dest): 'cat', source['Progress Out']) -def check_dest_paths(source): - """Check for image and/or map file and alert user about details.""" - dd_image_exists = os.path.exists(source['Dest Paths']['image']) - map_exists = os.path.exists(source['Dest Paths']['Map']) - if 'clone' in source['Dest Paths']['Map']: - if map_exists: - # We're cloning and a matching map file was detected - if not ask('Matching map file detected, resume recovery?'): - abort_ddrescue_tui() - else: - abort_imaging = False - resume_files_exist = False - source_devs = [source] - if source['Children']: - source_devs = source['Children'] - for dev in source_devs: - # We're imaging - dd_image_exists = os.path.exists(dev['Dest Paths']['image']) - map_exists = os.path.exists(dev['Dest Paths']['Map']) - if dd_image_exists and not map_exists: - # Refuce to resume without map file - i = dev['Dest Paths']['image'] - i = i[i.rfind('/')+1:] - print_error( - 'Detected image "{}" but not the matching map'.format(i)) - abort_imaging = True - elif not dd_image_exists and map_exists: - # Can't resume without dd_image - m = dev['Dest Paths']['Map'] - m = m[m.rfind('/')+1:] - print_error( - 'Detected map "{}" but not the matching image'.format(m)) - abort_imaging = True - elif dd_image_exists and map_exists: - # Matching dd_image and map file were detected - resume_files_exist = True - p = 'Matching image and map file{} detected, resume recovery?'.format( - 's' if len(source_devs) > 1 else '') - if abort_imaging or (resume_files_exist and not ask(p)): - abort_ddrescue_tui() - - def create_path_obj(path): """Create Dev, Dir, or Image obj based on path given.""" obj = None @@ -412,65 +373,6 @@ def create_path_obj(path): return obj -def dest_safety_check(source, dest): - """Verify the destination is appropriate for the source.""" - source_size = source['Details']['size'] - if dest['Is Dir']: - # MOVED - pass - else: - dest_size = dest['Details']['size'] - - # Convert to bytes and compare size - source_size = get_size_in_bytes(source_size) - dest_size = get_size_in_bytes(dest_size) - if source['Type'] == 'image' and dest_size < (source_size * 1.2): - # Imaging: ensure 120% of source size is available - print_error( - 'Not enough free space on destination, refusing to continue.') - print_standard( - ' Dest {d_size} < Required {s_size}'.format( - d_size=human_readable_size(dest_size), - s_size=human_readable_size(source_size * 1.2))) - abort_ddrescue_tui() - elif source['Type'] == 'clone' and source_size > dest_size: - # Cloning: ensure dest >= size - print_error('Destination is too small, refusing to continue.') - print_standard( - ' Dest {d_size} < Source {s_size}'.format( - d_size=human_readable_size(dest_size), - s_size=human_readable_size(source_size))) - abort_ddrescue_tui() - - # Imaging specific checks - if source['Type'] == 'image': - # Filesystem Type - if dest['Filesystem'] not in RECOMMENDED_FSTYPES: - print_error( - 'Destination filesystem "{}" is not recommended.'.format( - dest['Filesystem'])) - print_info('Recommended types are: {}'.format( - ' / '.join(RECOMMENDED_FSTYPES).upper())) - print_standard(' ') - if not ask('Proceed anyways? (Strongly discouraged)'): - abort_ddrescue_tui() - # Read-Write access - dest_ok = True - dest_st_mode = os.stat(dest['Path']).st_mode - dest_ok = dest_ok and dest_st_mode & stat.S_IRUSR - dest_ok = dest_ok and dest_st_mode & stat.S_IWUSR - dest_ok = dest_ok and dest_st_mode & stat.S_IXUSR - if not dest_ok: - print_error('Destination is not writable, refusing to continue.') - abort_ddrescue_tui() - - # Mount options check - if 'rw' not in dest['Mount options'].split(','): - print_error( - 'Destination is not mounted read-write, refusing to continue.') - abort_ddrescue_tui() - - def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -568,18 +470,6 @@ def get_formatted_status(label, data): """TODO""" # TODO pass -def get_recovery_scope_size(source): - """Calculate total size of selected dev(s).""" - # TODO function deprecated - source['Total Size'] = 0 - if source['Children']: - for child in source['Children']: - child['Size'] = get_size_in_bytes(child['Details']['size']) - source['Total Size'] += child['Size'] - else: - # Whole dev - source['Size'] = get_size_in_bytes(source['Details']['size']) - source['Total Size'] = source['Size'] def get_status_color(s, t_success=99, t_warn=90): @@ -698,6 +588,7 @@ def menu_ddrescue(source_path, dest_path, run_mode): state.add_block_pair(source, dest) else: # TODO select dev or child dev(s) + pass # Confirmations # TODO Show selection details @@ -1133,85 +1024,6 @@ def read_map_file(map_path): return map_data -def resume_from_map(source): - """Read map file(s) and set current progress to resume previous session.""" - # TODO function deprecated - map_data_read = False - non_tried = 0 - non_trimmed = 0 - non_scraped = 0 - - # Read map data - if source['Type'] != 'clone' and source['Children']: - # Imaging child device(s) - for child in source['Children']: - if os.path.exists(child['Dest Paths']['Map']): - map_data = read_map_file(child['Dest Paths']['Map']) - map_data_read = True - non_tried += map_data['non-tried'] - non_trimmed += map_data['non-trimmed'] - non_scraped += map_data['non-scraped'] - child['Recovered Size'] = map_data['rescued']/100*child['Size'] - - # Get (dev) current pass - dev_current_pass = 1 - if map_data['non-tried'] == 0: - if map_data['non-trimmed'] > 0: - dev_current_pass = 2 - elif map_data['non-scraped'] > 0: - dev_current_pass = 3 - elif map_data['rescued'] == 100: - dev_current_pass = 4 - - # Mark passes as skipped - for x in range(1, dev_current_pass): - p_num = 'Pass {}'.format(x) - child[p_num]['Done'] = True - child[p_num]['Status'] = 'Skipped' - - elif map_data_read: - # No map but we've already read at least one map, force pass 1 - non_tried = 1 - elif os.path.exists(source['Dest Paths']['Map']): - # Cloning or Imaging whole device - map_data = read_map_file(source['Dest Paths']['Map']) - map_data_read = True - non_tried += map_data['non-tried'] - non_trimmed += map_data['non-trimmed'] - non_scraped += map_data['non-scraped'] - - # Bail - if not map_data_read: - # No map data found, assuming fresh start - return - - # Set current pass - if non_tried > 0: - current_pass = 'Pass 1' - elif non_trimmed > 0: - current_pass = 'Pass 2' - source['Pass 1']['Done'] = True - source['Pass 1']['Status'] = 'Skipped' - elif non_scraped > 0: - current_pass = 'Pass 3' - source['Pass 1']['Done'] = True - source['Pass 1']['Status'] = 'Skipped' - source['Pass 2']['Done'] = True - source['Pass 2']['Status'] = 'Skipped' - else: - source['Current Pass'] = 'Done' - update_progress(source, end_run=True) - return - source['Current Pass'] = current_pass - - # Update current pass - if not source['Children']: - if os.path.exists(source['Dest Paths']['Map']): - map_data = read_map_file(source['Dest Paths']['Map']) - source[current_pass]['Done'] = map_data['pass completed'] - source['Recovered Size'] = map_data['rescued']/100*source['Size'] - - def run_ddrescue(source, dest, settings): """Run ddrescue pass.""" current_pass = source['Current Pass'] @@ -1412,41 +1224,6 @@ def select_device(description='device', provided_path=None, return dev -def set_dest_image_paths(source, dest): - """Set destination image path for source and any child devices.""" - # TODO function deprecated - if source['Type'] == 'clone': - base = '{pwd}/Clone_{size}_{model}'.format( - pwd=os.path.realpath(global_vars['Env']['PWD']), - size=source['Details']['size'], - model=source['Details'].get('model', 'Unknown')) - else: - base = '{Path}/{size}_{model}'.format( - size=source['Details']['size'], - model=source['Details'].get('model', 'Unknown'), - **dest) - source['Dest Paths'] = { - 'image': '{}.dd'.format(base), - 'Map': '{}.map'.format(base)} - - # Child devices - for child in source['Children']: - p_label = '' - if child['Details']['label']: - p_label = '_{}'.format(child['Details']['label']) - base = '{Path}/{size}_{model}_{p_num}_{p_size}{p_label}'.format( - size=source['Details']['size'], - model=source['Details'].get('model', 'Unknown'), - p_num=child['Details']['name'].replace( - child['Details']['pkname'], ''), - p_size=child['Details']['size'], - p_label=p_label, - **dest) - child['Dest Paths'] = { - 'image': '{}.dd'.format(base), - 'Map': '{}.map'.format(base)} - - def setup_loopback_device(source_path): """Setup a loopback device for source_path, returns dev_path as str.""" cmd = ( From a19ac4772b52d78734570d797120a91d610156cf Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 18:29:14 -0600 Subject: [PATCH 064/138] Better exception handling --- .bin/Scripts/ddrescue-tui-menu | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index 798cd43e..10cde744 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -38,12 +38,21 @@ if __name__ == '__main__': if not re.search(r'(^$|help|-h|\?)', run_mode, re.IGNORECASE): print_error('Invalid mode.') show_usage(script_name) - exit_script() # Done print_standard('\nDone.') pause("Press Enter to exit...") exit_script() + except GenericAbort as ga: + if str(ga): + print_warning(str(ga)) + abort() + except GenericError as ge: + if str(ge): + print_error(str(ge)) + else: + print_error('Generic Error?') + abort() except SystemExit: pass except: From 4047b956f57f11f3ef6529eac397a04103f5b1fe Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 18:54:31 -0600 Subject: [PATCH 065/138] Even better exception handling --- .bin/Scripts/ddrescue-tui-menu | 10 ++-- .bin/Scripts/functions/ddrescue.py | 78 +++++++++++++----------------- 2 files changed, 37 insertions(+), 51 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index 10cde744..f96dec36 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -43,15 +43,13 @@ if __name__ == '__main__': print_standard('\nDone.') pause("Press Enter to exit...") exit_script() - except GenericAbort as ga: - if str(ga): - print_warning(str(ga)) + except GenericAbort: abort() except GenericError as ge: + msg = 'Generic Error' if str(ge): - print_error(str(ge)) - else: - print_error('Generic Error?') + msg = str(ge) + print_error(msg) abort() except SystemExit: pass diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index ca4226c5..5069f65c 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -147,24 +147,24 @@ class RecoveryState(): if self.mode == 'clone': # Cloning safety checks if source.is_dir(): - raise GenericAbort('Invalid source "{}"'.format( + raise GenericError('Invalid source "{}"'.format( source.path)) elif not dest.is_dev(): - raise GenericAbort('Invalid destination "{}"'.format( + raise GenericError('Invalid destination "{}"'.format( dest.path)) elif source.size > dest.size: - raise GenericAbort( + raise GenericError( 'Destination is too small, refusing to continue.') else: # Imaging safety checks if not source.is_dev(): - raise GenericAbort('Invalid source "{}"'.format( + raise GenericError('Invalid source "{}"'.format( source.path)) elif not dest.is_dir(): - raise GenericAbort('Invalid destination "{}"'.format( + raise GenericError('Invalid destination "{}"'.format( dest.path)) elif (source.size * 1.2) > dest.size: - raise GenericAbort( + raise GenericError( 'Destination is too small, refusing to continue.') elif dest.fstype.lower() not in RECOMMENDED_FSTYPES: print_error( @@ -174,12 +174,12 @@ class RecoveryState(): ' / '.join(RECOMMENDED_FSTYPES).upper())) print_standard(' ') if not ask('Proceed anyways? (Strongly discouraged)'): - raise GenericAbort('Aborted.') + raise GenericAbort() elif not is_writable_dir(dest): - raise GenericAbort( + raise GenericError( 'Destination is not writable, refusing to continue.') elif not is_writable_filesystem(dest): - raise GenericAbort( + raise GenericError( 'Destination is mounted read-only, refusing to continue.') # Safety checks passed @@ -188,11 +188,7 @@ class RecoveryState(): def self_checks(self): """Run self-checks for each BlockPair object.""" for bp in self.block_pairs: - try: - bp.self_check() - except GenericError as err: - print_error(err) - raise GenericAbort('Aborted.') + bp.self_check() def set_pass_num(self): """Set current pass based on all block-pair's progress.""" @@ -252,7 +248,8 @@ class DevObj(BaseObj): def self_check(self): """Verify that self.path points to a block device.""" if not pathlib.Path(self.path).is_block_device(): - raise GenericError('TODO') + raise GenericError('Path "{}" is not a block device.'.format( + self.path)) def set_details(self): """Set details via lsblk.""" @@ -288,7 +285,8 @@ class DirObj(BaseObj): def self_check(self): """Verify that self.path points to a directory.""" if not pathlib.Path(self.path).is_dir(): - raise GenericError('TODO') + raise GenericError('Path "{}" is not a directory.'.format( + self.path)) def set_details(self): """Set details via findmnt.""" @@ -304,7 +302,8 @@ class ImageObj(BaseObj): def self_check(self): """Verify that self.path points to a file.""" if not pathlib.Path(self.path).is_file(): - raise GenericError('TODO') + raise GenericError('Path "{}" is not an image file.'.format( + self.path)) def set_details(self): """Setup loopback device, set details via lsblk, then detach device.""" @@ -322,11 +321,6 @@ class ImageObj(BaseObj): # Functions -def abort_ddrescue_tui(): - run_program(['losetup', '-D']) - abort() - - def build_outer_panes(source, dest): """Build top and side panes.""" clear_screen() @@ -369,7 +363,7 @@ def create_path_obj(path): elif pathlib.Path(self.path).is_file(): obj = ImageObj(path) else: - raise GenericAbort('TODO') + raise GenericError('Invalid path "{}"'.format(path)) return obj @@ -551,7 +545,7 @@ def menu_clone(source_path, dest_path): # Confirm if not ask('Proceed with clone?'): - abort_ddrescue_tui() + raise GenericAbort() show_safety_check() # Main menu @@ -634,7 +628,7 @@ def menu_image(source_path, dest_path): # Confirm if not ask('Proceed with imaging?'): - abort_ddrescue_tui() + raise GenericAbort() # Main menu build_outer_panes(source, dest) @@ -805,7 +799,7 @@ def menu_select_children(source): elif selection == 'P' and one_or_more_devs_selected: break elif selection == 'Q': - abort_ddrescue_tui() + raise GenericAbort() # Check selection selected_children = [{ @@ -833,8 +827,8 @@ def menu_select_device(title='Which device?', skip_device={}): result = run_program(cmd) json_data = json.loads(result.stdout.decode()) except CalledProcessError: - print_error('Failed to get device details for {}'.format(dev_path)) - abort_ddrescue_tui() + raise GenericError( + 'Failed to get device details for {}'.format(dev_path)) # Build menu dev_options = [] @@ -854,8 +848,7 @@ def menu_select_device(title='Which device?', skip_device={}): 'Disabled': disable_dev}) dev_options = sorted(dev_options, key=itemgetter('Name')) if not dev_options: - print_error('No devices available.') - abort_ddrescue_tui() + raise GenericError('No devices available.') # Show Menu actions = [{'Name': 'Quit', 'Letter': 'Q'}] @@ -868,7 +861,7 @@ def menu_select_device(title='Which device?', skip_device={}): if selection.isnumeric(): return dev_options[int(selection)-1]['Path'] elif selection == 'Q': - abort_ddrescue_tui() + raise GenericAbort() def menu_select_path(skip_device={}): @@ -890,7 +883,7 @@ def menu_select_path(skip_device={}): action_entries=actions) if selection == 'Q': - abort_ddrescue_tui() + raise GenericAbort() elif selection.isnumeric(): index = int(selection) - 1 if path_options[index]['Path']: @@ -928,7 +921,7 @@ def menu_select_path(skip_device={}): if selection.isnumeric(): s_path = vol_options[int(selection)-1]['Path'] elif selection == 'Q': - abort_ddrescue_tui() + raise GenericAbort() elif path_options[index]['Name'] == 'Enter manually': # Manual entry @@ -1007,8 +1000,7 @@ def read_map_file(map_path): try: map_data[m.group('key')] = float(m.group('value')) except ValueError: - print_error('Failed to read map data') - abort_ddrescue_tui() + raise GenericError('Failed to read map data') m = re.match(r'.*current status:\s+(?P.*)', line.strip()) if m: map_data['pass completed'] = bool(m.group('status') == 'finished') @@ -1133,8 +1125,7 @@ def select_dest_path(provided_path=None, skip_device={}): # Check path if not pathlib.Path(dest['Path']).is_dir(): - print_error('Invalid path "{}"'.format(dest['Path'])) - abort_ddrescue_tui() + raise GenericError('Invalid path "{}"'.format(dest['Path'])) # Create ticket folder if ask('Create ticket folder?'): @@ -1144,9 +1135,8 @@ def select_dest_path(provided_path=None, skip_device={}): try: os.makedirs(dest['Path'], exist_ok=True) except OSError: - print_error('Failed to create folder "{}"'.format( - dest['Path'])) - abort_ddrescue_tui() + raise GenericError( + 'Failed to create folder "{}"'.format(dest['Path'])) # Set display name result = run_program(['tput', 'cols']) @@ -1181,8 +1171,7 @@ def select_device(description='device', provided_path=None, dev['Dev Path'] = setup_loopback_device(dev['Path']) dev['Is Image'] = True else: - print_error('Invalid {} "{}"'.format(description, dev['Path'])) - abort_ddrescue_tui() + raise GenericError('Invalid {} "{}"'.format(description, dev['Path'])) # Get device details dev['Details'] = get_device_details(dev['Dev Path']) @@ -1237,8 +1226,7 @@ def setup_loopback_device(source_path): dev_path = out.stdout.decode().strip() sleep(1) except CalledProcessError: - print_error('Failed to setup loopback device for source.') - abort_ddrescue_tui() + raise GenericError('Failed to setup loopback device for source.') else: return dev_path @@ -1272,7 +1260,7 @@ def show_safety_check(): print_warning('This is irreversible and will lead ' 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) if not ask('Asking again to confirm, is this correct?'): - abort_ddrescue_tui() + raise GenericAbort() def show_selection_details(source, dest): From 6cdc4015e741c203dcecb249f31b40d6c0c11261 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 19:34:51 -0600 Subject: [PATCH 066/138] Updated menu_ddrescue() and related sections * RecoveryState is updated before confirmation(s) * New confirmation prompt that supports both cloning and imaging modes * Refactored show_selection_details() to use new objects * Allows resumed state to be detected and prompt switched to "Resume?" * Renamed function show_safety_check() to double_confirm_clone() for clarity --- .bin/Scripts/functions/ddrescue.py | 74 ++++++++++++++++-------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 5069f65c..a99f8085 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -44,6 +44,7 @@ class BlockPair(): self.mode = mode self.name = source.name self.pass_done = [False, False, False] + self.resumed = False self.rescued = 0 self.status = ['Pending', 'Pending', 'Pending'] self.total_size = source.size @@ -64,6 +65,7 @@ class BlockPair(): prefix=source.prefix) if os.path.exists(self.map_path): self.load_map_data() + self.resumed = True def finish_pass(self, pass_num): """Mark pass as done and check if 100% recovered.""" @@ -186,9 +188,12 @@ class RecoveryState(): self.block_pairs.append(BlockPair(source, dest)) def self_checks(self): - """Run self-checks for each BlockPair object.""" + """Run self-checks for each BlockPair and update state values.""" + self.total_size = 0 for bp in self.block_pairs: bp.self_check() + self.resumed |= bp.resumed + self.total_size += bp.size def set_pass_num(self): """Set current pass based on all block-pair's progress.""" @@ -210,10 +215,8 @@ class RecoveryState(): def update_progress(self): """Update overall progress using block_pairs.""" self.rescued = 0 - self.total_size = 0 for bp in self.block_pairs: self.rescued += bp.rescued - self.total_size += bp.size self.status_percent = get_formatted_status( label='Recovered:', data=(self.rescued/self.total_size)*100) self.status_amount = get_formatted_status( @@ -367,6 +370,16 @@ def create_path_obj(path): return obj +def double_confirm_clone(): + """Display warning and get 2nd confirmation from user, returns bool.""" + print_standard('\nSAFETY CHECK') + print_warning('All data will be DELETED from the ' + 'destination device and partition(s) listed above.') + print_warning('This is irreversible and will lead ' + 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) + return ask('Asking again to confirm, is this correct?') + + def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" try: @@ -584,11 +597,22 @@ def menu_ddrescue(source_path, dest_path, run_mode): # TODO select dev or child dev(s) pass + # Update state + state.self_checks() + state.set_pass_num() + state.update_progress() + # Confirmations - # TODO Show selection details - # TODO resume? - # TODO Proceed? (maybe merge with resume? prompt?) - # TODO double-confirm for clones for safety + clear_screen() + show_selection_details(state, source, dest) + prompt = 'Start {}?'.format(state.mode.replace('e', 'ing')) + if state.resumed: + print_info('Map data detected and loaded.') + prompt = prompt.replace('Start', 'Resume') + if not ask(prompt): + raise GenericAbort() + if state.mode == 'clone' and not double_confirm_clone(): + raise GenericAbort() # Main menu build_outer_panes(source, dest) @@ -1252,40 +1276,20 @@ def show_device_details(dev_path): print_standard(line) -def show_safety_check(): - """Display safety check message and get confirmation from user.""" - print_standard('\nSAFETY CHECK') - print_warning('All data will be DELETED from the ' - 'destination device and partition(s) listed above.') - print_warning('This is irreversible and will lead ' - 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) - if not ask('Asking again to confirm, is this correct?'): - raise GenericAbort() - - -def show_selection_details(source, dest): - clear_screen() - +def show_selection_details(state, source, dest): + """Show selection details.""" # Source - print_success('Source device') - if source['Is Image']: - print_standard('Using image file: {}'.format(source['Path'])) - print_standard(' (via loopback device: {})'.format( - source['Dev Path'])) - show_device_details(source['Dev Path']) + print_success('Source') + print_standard(source.report) print_standard(' ') # Destination - if source['Type'] == 'clone': - print_success('Destination device ', end='') + if state.mode == 'clone': + print_success('Destination ', end='') print_error('(ALL DATA WILL BE DELETED)', timestamp=False) - show_device_details(dest['Dev Path']) else: - print_success('Destination path') - print_standard(dest['Path']) - print_info('{:<8}{}'.format('FREE', 'FSTYPE')) - print_standard('{:<8}{}'.format( - dest['Free Space'], dest['Filesystem'])) + print_success('Destination') + print_standard(dest.report) print_standard(' ') From 30b703e025e813f7fdbb30952a5715fecdb911af Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 20:09:48 -0600 Subject: [PATCH 067/138] Updated get_formatted_status() --- .bin/Scripts/functions/ddrescue.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index a99f8085..0dbb24a2 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -72,7 +72,9 @@ class BlockPair(): if map_data['full recovery']: self.pass_done = [True, True, True] self.rescued = self.total_size - self.status[pass_num] = get_formatted_status(100) + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num), + data=100) # Mark future passes as Skipped pass_num += 1 while pass_num <= 2: @@ -474,9 +476,23 @@ def get_size_in_bytes(s): def get_formatted_status(label, data): - """TODO""" - # TODO - pass + """Build status string using provided info, returns str.""" + data_width = SIDE_PANE_WIDTH - len(label) + try: + data_str = '{data:>{data_width}.2f} %'.format( + data=data, + data_width=data_width-2) + except ValueError: + # Assuming non-numeric data + data_str = '{data:>{data_width}}'.format( + data=data, + data_width=data_width) + status = '{label}{s_color}{data_str}{CLEAR}'.format( + label=label, + s_color=get_status_color(data), + data_str=data_str, + **COLORS) + return status def get_status_color(s, t_success=99, t_warn=90): @@ -489,7 +505,7 @@ def get_status_color(s, t_success=99, t_warn=90): # Status is either in lists below or will default to red pass - if s in ('Pending',): + if s in ('Pending',) or str(s)[-2:] in (' b', 'Kb', 'Mb', 'Gb', 'Tb'): color = COLORS['CLEAR'] elif s in ('Skipped', 'Unknown', 'Working'): color = COLORS['YELLOW'] From 53f0b93a5f03ca8b8c65285a72ad55133b52829b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 20:24:34 -0600 Subject: [PATCH 068/138] Misc bugfixes --- .bin/Scripts/functions/ddrescue.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 0dbb24a2..1e2fbcab 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -47,7 +47,7 @@ class BlockPair(): self.resumed = False self.rescued = 0 self.status = ['Pending', 'Pending', 'Pending'] - self.total_size = source.size + self.size = source.size # Set dest paths if self.mode == 'clone': # Cloning @@ -71,7 +71,7 @@ class BlockPair(): """Mark pass as done and check if 100% recovered.""" if map_data['full recovery']: self.pass_done = [True, True, True] - self.rescued = self.total_size + self.rescued = self.size self.status[pass_num] = get_formatted_status( label='Pass {}'.format(pass_num), data=100) @@ -90,10 +90,10 @@ class BlockPair(): def load_map_data(self): """Load data from map file and set progress.""" map_data = read_map_file(self.map_path) - self.rescued = map_data['rescued'] * self.total_size + self.rescued = map_data['rescued'] * self.size if map_data['full recovery']: self.pass_done = [True, True, True] - self.rescued = self.total_size + self.rescued = self.size self.status = ['Skipped', 'Skipped', 'Skipped'] elif map_data['non-tried'] > 0: # Initial pass incomplete @@ -126,10 +126,10 @@ class BlockPair(): """Update progress using map file.""" if os.path.exists(self.map_path): map_data = read_map_file(self.map_path) - self.rescued = map_data['rescued'] * self.total_size + self.rescued = map_data['rescued'] * self.size self.status[pass_num] = get_formatted_status( label='Pass {}'.format(pass_num), - data=(self.rescued/self.total_size)*100) + data=(self.rescued/self.size)*100) class RecoveryState(): @@ -187,7 +187,7 @@ class RecoveryState(): 'Destination is mounted read-only, refusing to continue.') # Safety checks passed - self.block_pairs.append(BlockPair(source, dest)) + self.block_pairs.append(BlockPair(source, dest, self.mode)) def self_checks(self): """Run self-checks for each BlockPair and update state values.""" @@ -314,15 +314,16 @@ class ImageObj(BaseObj): """Setup loopback device, set details via lsblk, then detach device.""" self.type = 'image' self.loop_dev = setup_loopback_device(self.path) - self.details = get_device_details(self.loopdev) + self.details = get_device_details(self.loop_dev) self.details['model'] = 'ImageFile' self.name = self.path[self.path.rfind('/')+1:] self.prefix = '{}_ImageFile'.format( self.details.get('size', 'UNKNOWN')) self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) self.report = get_device_report(self.loop_dev) - self.report = self.report.replace(self.loop_dev, '{Img}') - run_program(['losetup', '--detach', loop_path], check=False) + self.report = self.report.replace( + self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)') + run_program(['losetup', '--detach', self.loop_dev], check=False) # Functions @@ -361,11 +362,11 @@ def build_outer_panes(source, dest): def create_path_obj(path): """Create Dev, Dir, or Image obj based on path given.""" obj = None - if pathlib.Path(self.path).is_block_device(): - obj = Dev(path) - elif pathlib.Path(self.path).is_dir(): + if pathlib.Path(path).is_block_device(): + obj = DevObj(path) + elif pathlib.Path(path).is_dir(): obj = DirObj(path) - elif pathlib.Path(self.path).is_file(): + elif pathlib.Path(path).is_file(): obj = ImageObj(path) else: raise GenericError('Invalid path "{}"'.format(path)) @@ -611,6 +612,7 @@ def menu_ddrescue(source_path, dest_path, run_mode): state.add_block_pair(source, dest) else: # TODO select dev or child dev(s) + state.add_block_pair(source, dest) pass # Update state From 1e195f70fc8a932cffe31a16ceab82afb728b9e9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 26 Jul 2018 20:49:27 -0600 Subject: [PATCH 069/138] Fixed names and started updating build_outer_panes --- .bin/Scripts/functions/ddrescue.py | 43 +++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 1e2fbcab..691df3f0 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -260,7 +260,11 @@ class DevObj(BaseObj): """Set details via lsblk.""" self.type = 'dev' self.details = get_device_details(self.path) - self.name = self.details.get('name', 'UNKNOWN') + self.name = '{name} {size} {model} {serial}'.format( + name=self.details.get('name', 'UNKNOWN'), + size=self.details.get('size', 'UNKNOWN'), + model=self.details.get('model', 'UNKNOWN'), + serial=self.details.get('serial', 'UNKNOWN')) self.model = self.details.get('model', 'UNKNOWN') self.model_size = self.details.get('size', 'UNKNOWN') self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) @@ -298,7 +302,7 @@ class DirObj(BaseObj): self.type = 'dir' self.details = get_dir_details(self.path) self.fstype = self.details.get('fstype', 'UNKNOWN') - self.name = self.path + self.name = self.path + '/' self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) self.report = get_dir_report(self.path) @@ -316,7 +320,9 @@ class ImageObj(BaseObj): self.loop_dev = setup_loopback_device(self.path) self.details = get_device_details(self.loop_dev) self.details['model'] = 'ImageFile' - self.name = self.path[self.path.rfind('/')+1:] + self.name = '{name} {size}'.format( + name=self.path[self.path.rfind('/')+1:], + size=self.details.get('size', 'UNKNOWN')) self.prefix = '{}_ImageFile'.format( self.details.get('size', 'UNKNOWN')) self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) @@ -330,13 +336,25 @@ class ImageObj(BaseObj): def build_outer_panes(source, dest): """Build top and side panes.""" clear_screen() + result = run_program(['tput', 'cols']) + width = int( + (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 # Top panes + source_str = source.name + if len(source_str) > width: + source_str = '{}...'.format(source_str[:width-3]) + dest_str = dest.name + if len(dest_str) > width: + if dest.is_dev(): + dest_str = '{}...'.format(dest_str[:width-3]) + else: + dest_str = '...{}'.format(dest_str[-width+3:]) source_pane = tmux_splitw( '-bdvl', '2', '-PF', '#D', 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( - text=source['Display Name'], + text=source_str, **COLORS)) tmux_splitw( '-t', source_pane, @@ -348,15 +366,16 @@ def build_outer_panes(source, dest): '-t', source_pane, '-dhp', '50', 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( - text=dest['Display Name'], + text=dest_str, **COLORS)) # Side pane - update_progress(source) - tmux_splitw( - '-dhl', '{}'.format(SIDE_PANE_WIDTH), - 'watch', '--color', '--no-title', '--interval', '1', - 'cat', source['Progress Out']) + # TODO FIX IT + #update_progress(source) + #tmux_splitw( + # '-dhl', '{}'.format(SIDE_PANE_WIDTH), + # 'watch', '--color', '--no-title', '--interval', '1', + # 'cat', source['Progress Out']) def create_path_obj(path): @@ -455,8 +474,8 @@ def get_dir_report(dir_path): '--target', dir_path]) for line in result.stdout.decode().strip().splitlines(): if 'FSTYPE' in line: - output.append('{BLUE}{path:<{width}}{line}{CLEAR}'.format( - path=dir_path, + output.append('{BLUE}{label:<{width}}{line}{CLEAR}'.format( + label='PATH', width=width, line=line, **COLORS)) From 7ac91fd312d35ed76face8dcd595f4b3700e92d2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Aug 2018 20:33:35 -0700 Subject: [PATCH 070/138] Adjust pass 1 threshold --- .bin/Scripts/functions/ddrescue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 691df3f0..97bf0b48 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -13,7 +13,7 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES -AUTO_NEXT_PASS_1_THRESHOLD = 90 +AUTO_NEXT_PASS_1_THRESHOLD = 95 AUTO_NEXT_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { '--binary-prefixes': {'Enabled': True, 'Hidden': True}, From 03bdb4b4b72b2a02ddcfe98ae7191f22790d4589 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Aug 2018 21:12:05 -0700 Subject: [PATCH 071/138] Reordered classes and removed old menu functions --- .bin/Scripts/functions/ddrescue.py | 301 +++++++++++------------------ 1 file changed, 112 insertions(+), 189 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 97bf0b48..0e0e19a1 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -37,6 +37,29 @@ USAGE = """ {script_name} clone [source [destination]] # Clases +class BaseObj(): + """Base object used by DevObj, DirObj, and ImageObj.""" + def __init__(self, path): + self.type = 'base' + self.path = os.path.realpath(path) + self.set_details() + + def is_dev(self): + return self.type == 'dev' + + def is_dir(self): + return self.type == 'dir' + + def is_image(self): + return self.type == 'image' + + def self_check(self): + pass + + def set_details(self): + self.details = {} + + class BlockPair(): """Object to track data and methods together for source and dest.""" def __init__(self, source, dest, mode): @@ -132,6 +155,95 @@ class BlockPair(): data=(self.rescued/self.size)*100) +class DevObj(BaseObj): + """Block device object.""" + def self_check(self): + """Verify that self.path points to a block device.""" + if not pathlib.Path(self.path).is_block_device(): + raise GenericError('Path "{}" is not a block device.'.format( + self.path)) + if self.parent: + print_warning('"{}" is a child device.'.format(self.path)) + if ask('Use parent device "{}" instead?'.format(self.parent): + self.path = os.path.realpath(path) + self.set_details() + + def set_details(self): + """Set details via lsblk.""" + self.type = 'dev' + self.details = get_device_details(self.path) + self.name = '{name} {size} {model} {serial}'.format( + name=self.details.get('name', 'UNKNOWN'), + size=self.details.get('size', 'UNKNOWN'), + model=self.details.get('model', 'UNKNOWN'), + serial=self.details.get('serial', 'UNKNOWN')) + self.model = self.details.get('model', 'UNKNOWN') + self.model_size = self.details.get('size', 'UNKNOWN') + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_device_report(self.path) + self.parent = self.details.get('pkname', '') + self.label = self.details.get('label', '') + if not self.label: + # Force empty string in case it's set to None + self.label = '' + self.update_filename_prefix() + + def update_filename_prefix(self): + """Set filename prefix based on details.""" + self.prefix = '{m_size}_{model}'.format( + m_size=self.model_size, + model=self.model) + if self.parent: + # Add child device details + self.prefix += '_{c_num}_{c_size}{sep}{c_label}'.format( + c_num=self.name.replace(self.parent, ''), + c_size=self.details.get('size', 'UNKNOWN'), + sep='_' if self.label else '', + c_label=self.label) + + +class DirObj(BaseObj): + def self_check(self): + """Verify that self.path points to a directory.""" + if not pathlib.Path(self.path).is_dir(): + raise GenericError('Path "{}" is not a directory.'.format( + self.path)) + + def set_details(self): + """Set details via findmnt.""" + self.type = 'dir' + self.details = get_dir_details(self.path) + self.fstype = self.details.get('fstype', 'UNKNOWN') + self.name = self.path + '/' + self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) + self.report = get_dir_report(self.path) + + +class ImageObj(BaseObj): + def self_check(self): + """Verify that self.path points to a file.""" + if not pathlib.Path(self.path).is_file(): + raise GenericError('Path "{}" is not an image file.'.format( + self.path)) + + def set_details(self): + """Setup loopback device, set details via lsblk, then detach device.""" + self.type = 'image' + self.loop_dev = setup_loopback_device(self.path) + self.details = get_device_details(self.loop_dev) + self.details['model'] = 'ImageFile' + self.name = '{name} {size}'.format( + name=self.path[self.path.rfind('/')+1:], + size=self.details.get('size', 'UNKNOWN')) + self.prefix = '{}_ImageFile'.format( + self.details.get('size', 'UNKNOWN')) + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_device_report(self.loop_dev) + self.report = self.report.replace( + self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)') + run_program(['losetup', '--detach', self.loop_dev], check=False) + + class RecoveryState(): """Object to track BlockPair objects and overall state.""" def __init__(self, mode): @@ -225,113 +337,6 @@ class RecoveryState(): label='', data=human_readable_size(self.rescued)) -class BaseObj(): - """Base object used by DevObj, DirObj, and ImageObj.""" - def __init__(self, path): - self.type = 'base' - self.path = os.path.realpath(path) - self.set_details() - - def is_dev(self): - return self.type == 'dev' - - def is_dir(self): - return self.type == 'dir' - - def is_image(self): - return self.type == 'image' - - def self_check(self): - pass - - def set_details(self): - self.details = {} - - -class DevObj(BaseObj): - """Block device object.""" - def self_check(self): - """Verify that self.path points to a block device.""" - if not pathlib.Path(self.path).is_block_device(): - raise GenericError('Path "{}" is not a block device.'.format( - self.path)) - - def set_details(self): - """Set details via lsblk.""" - self.type = 'dev' - self.details = get_device_details(self.path) - self.name = '{name} {size} {model} {serial}'.format( - name=self.details.get('name', 'UNKNOWN'), - size=self.details.get('size', 'UNKNOWN'), - model=self.details.get('model', 'UNKNOWN'), - serial=self.details.get('serial', 'UNKNOWN')) - self.model = self.details.get('model', 'UNKNOWN') - self.model_size = self.details.get('size', 'UNKNOWN') - self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) - self.report = get_device_report(self.path) - self.parent = self.details.get('pkname', '') - self.label = self.details.get('label', '') - if not self.label: - # Force empty string in case it's set to None - self.label = '' - self.update_filename_prefix() - - def update_filename_prefix(self): - """Set filename prefix based on details.""" - self.prefix = '{m_size}_{model}'.format( - m_size=self.model_size, - model=self.model) - if self.parent: - # Add child device details - self.prefix += '_{c_num}_{c_size}{sep}{c_label}'.format( - c_num=self.name.replace(self.parent, ''), - c_size=self.details.get('size', 'UNKNOWN'), - sep='_' if self.label else '', - c_label=self.label) - - -class DirObj(BaseObj): - def self_check(self): - """Verify that self.path points to a directory.""" - if not pathlib.Path(self.path).is_dir(): - raise GenericError('Path "{}" is not a directory.'.format( - self.path)) - - def set_details(self): - """Set details via findmnt.""" - self.type = 'dir' - self.details = get_dir_details(self.path) - self.fstype = self.details.get('fstype', 'UNKNOWN') - self.name = self.path + '/' - self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) - self.report = get_dir_report(self.path) - - -class ImageObj(BaseObj): - def self_check(self): - """Verify that self.path points to a file.""" - if not pathlib.Path(self.path).is_file(): - raise GenericError('Path "{}" is not an image file.'.format( - self.path)) - - def set_details(self): - """Setup loopback device, set details via lsblk, then detach device.""" - self.type = 'image' - self.loop_dev = setup_loopback_device(self.path) - self.details = get_device_details(self.loop_dev) - self.details['model'] = 'ImageFile' - self.name = '{name} {size}'.format( - name=self.path[self.path.rfind('/')+1:], - size=self.details.get('size', 'UNKNOWN')) - self.prefix = '{}_ImageFile'.format( - self.details.get('size', 'UNKNOWN')) - self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) - self.report = get_device_report(self.loop_dev) - self.report = self.report.replace( - self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)') - run_program(['losetup', '--detach', self.loop_dev], check=False) - - # Functions def build_outer_panes(source, dest): """Build top and side panes.""" @@ -565,48 +570,6 @@ def mark_all_passes_pending(source): child[p_num]['Done'] = False -def menu_clone(source_path, dest_path): - """ddrescue cloning menu.""" - - # Set devices - source = select_device('source', source_path) - source['Current Pass'] = 'Pass 1' - source['Pass 1'] = {'Status': 'Pending', 'Done': False} - source['Pass 2'] = {'Status': 'Pending', 'Done': False} - source['Pass 3'] = {'Status': 'Pending', 'Done': False} - source['Recovered Size'] = 0 - source['Started Recovery'] = False - source['Total Size'] = 0 - source['Type'] = 'clone' - dest = select_device( - 'destination', dest_path, - skip_device=source['Details'], allow_image_file=False) - dest_safety_check(source, dest) - - # Show selection details - show_selection_details(source, dest) - - # Set status details - set_dest_image_paths(source, dest) - get_recovery_scope_size(source) - check_dest_paths(source) - resume_from_map(source) - - # Confirm - if not ask('Proceed with clone?'): - raise GenericAbort() - show_safety_check() - - # Main menu - build_outer_panes(source, dest) - menu_main(source, dest) - - # Done - run_program(['losetup', '-D']) - run_program(['tmux', 'kill-window']) - exit_script() - - def menu_ddrescue(source_path, dest_path, run_mode): """ddrescue menu.""" source = None @@ -661,46 +624,6 @@ def menu_ddrescue(source_path, dest_path, run_mode): run_program(['tmux', 'kill-window']) exit_script() -def menu_image(source_path, dest_path): - """ddrescue imaging menu.""" - - # Set devices - source = select_device('source', source_path, allow_image_file=False) - source['Current Pass'] = 'Pass 1' - source['Pass 1'] = {'Status': 'Pending', 'Done': False} - source['Pass 2'] = {'Status': 'Pending', 'Done': False} - source['Pass 3'] = {'Status': 'Pending', 'Done': False} - source['Recovered Size'] = 0 - source['Started Recovery'] = False - source['Total Size'] = 0 - source['Type'] = 'image' - dest = select_dest_path(dest_path, skip_device=source['Details']) - dest_safety_check(source, dest) - - # Select child device(s) - source['Children'] = menu_select_children(source) - set_dest_image_paths(source, dest) - get_recovery_scope_size(source) - check_dest_paths(source) - resume_from_map(source) - - # Show selection details - show_selection_details(source, dest) - - # Confirm - if not ask('Proceed with imaging?'): - raise GenericAbort() - - # Main menu - build_outer_panes(source, dest) - menu_main(source, dest) - - # Done - run_program(['losetup', '-D']) - run_program(['tmux', 'kill-window']) - exit_script() - - def menu_main(source, dest): """Main menu is used to set ddrescue settings.""" title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) From ccf7f0686ed4db5bab1456d92b2a25511ca6e194 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Aug 2018 22:25:01 -0700 Subject: [PATCH 072/138] Updated select_device() to use DevObj() * Also fixed child/parent check(s) * Removed menu_select_device() since code was moved into select_device() --- .bin/Scripts/functions/ddrescue.py | 167 +++++++++-------------------- 1 file changed, 51 insertions(+), 116 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 0e0e19a1..604f3402 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -164,8 +164,8 @@ class DevObj(BaseObj): self.path)) if self.parent: print_warning('"{}" is a child device.'.format(self.path)) - if ask('Use parent device "{}" instead?'.format(self.parent): - self.path = os.path.realpath(path) + if ask('Use parent device "{}" instead?'.format(self.parent)): + self.path = os.path.realpath(self.parent) self.set_details() def set_details(self): @@ -576,17 +576,17 @@ def menu_ddrescue(source_path, dest_path, run_mode): dest = None if source_path: source = create_path_obj(source_path) + else: + source = select_device('source') + source.self_check() if dest_path: dest = create_path_obj(dest_path) - - # Show selection menus (if necessary) - if not source: - source = select_device('source') - if not dest: + else: if run_mode == 'clone': dest = select_device('destination', skip_device=source) else: dest = select_directory() + dest.self_check() # Build BlockPairs state = RecoveryState(run_mode) @@ -796,58 +796,6 @@ def menu_select_children(source): return selected_children -def menu_select_device(title='Which device?', skip_device={}): - """Select block device via a menu, returns dev_path as str.""" - skip_names = [ - skip_device.get('name', None), skip_device.get('pkname', None)] - skip_names = [n for n in skip_names if n] - try: - cmd = ( - 'lsblk', - '--json', - '--nodeps', - '--output-all', - '--paths') - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - except CalledProcessError: - raise GenericError( - 'Failed to get device details for {}'.format(dev_path)) - - # Build menu - dev_options = [] - for dev in json_data['blockdevices']: - # Disable dev if in skip_names - disable_dev = dev['name'] in skip_names or dev['pkname'] in skip_names - - # Append non-matching devices - dev_options.append({ - 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( - name=dev['name'], - tran=dev['tran'] if dev['tran'] else '', - size=dev['size'] if dev['size'] else '', - model=dev['model'] if dev['model'] else '', - serial=dev['serial'] if dev['serial'] else ''), - 'Path': dev['name'], - 'Disabled': disable_dev}) - dev_options = sorted(dev_options, key=itemgetter('Name')) - if not dev_options: - raise GenericError('No devices available.') - - # Show Menu - actions = [{'Name': 'Quit', 'Letter': 'Q'}] - selection = menu_select( - title=title, - main_entries=dev_options, - action_entries=actions, - disabled_label='SOURCE DEVICE') - - if selection.isnumeric(): - return dev_options[int(selection)-1]['Path'] - elif selection == 'Q': - raise GenericAbort() - - def menu_select_path(skip_device={}): """Select path via menu, returns path as str.""" pwd = os.path.realpath(global_vars['Env']['PWD']) @@ -1134,67 +1082,54 @@ def select_dest_path(provided_path=None, skip_device={}): return dest -def select_device(description='device', provided_path=None, - skip_device={}, allow_image_file=True): - """Select device via provided path or menu, return dev as dict.""" - dev = {'Is Dir': False, 'Is Image': False} +def select_device(description='device', skip_device=None): + """Select device via a menu, returns DevObj.""" + cmd = ( + 'lsblk', + '--json', + '--nodeps', + '--output-all', + '--paths') + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + skip_names = [] + if skip_device: + skip_names.append(skip_device.path) + if skip_device.parent: + skip_names.append(skip_device.parent) - # Set path - if provided_path: - dev['Path'] = provided_path - else: - dev['Path'] = menu_select_device( - title='Please select a {}'.format(description), - skip_device=skip_device) - dev['Path'] = os.path.realpath(dev['Path']) + # Build menu + dev_options = [] + for dev in json_data['blockdevices']: + # Disable dev if in skip_names + disabled = dev['name'] in skip_names or dev['pkname'] in skip_names - # Check path - if pathlib.Path(dev['Path']).is_block_device(): - dev['Dev Path'] = dev['Path'] - elif allow_image_file and pathlib.Path(dev['Path']).is_file(): - dev['Dev Path'] = setup_loopback_device(dev['Path']) - dev['Is Image'] = True - else: - raise GenericError('Invalid {} "{}"'.format(description, dev['Path'])) + # Add to options + dev_options.append({ + 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( + name=dev['name'], + tran=dev['tran'] if dev['tran'] else '', + size=dev['size'] if dev['size'] else '', + model=dev['model'] if dev['model'] else '', + serial=dev['serial'] if dev['serial'] else ''), + 'Dev': DevObj(dev['name']), + 'Disabled': disabled}) + dev_options = sorted(dev_options, key=itemgetter('Name')) + if not dev_options: + raise GenericError('No devices available.') - # Get device details - dev['Details'] = get_device_details(dev['Dev Path']) - if 'Children' not in dev: - dev['Children'] = [] + # Show Menu + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + selection = menu_select( + title='Please select the {} device'.format(description), + main_entries=dev_options, + action_entries=actions, + disabled_label='ALREADY SELECTED') - # Check for parent device(s) - while dev['Details']['pkname']: - print_warning('{} "{}" is a child device.'.format( - description.title(), dev['Dev Path'])) - if ask('Use parent device "{}" instead?'.format( - dev['Details']['pkname'])): - # Update dev with parent info - dev['Dev Path'] = dev['Details']['pkname'] - dev['Details'] = get_device_details(dev['Dev Path']) - else: - # Leave alone - break - - # Set display name - if dev['Is Image']: - dev['Display Name'] = dev['Path'] - else: - dev['Display Name'] = '{name} {size} {model}'.format( - **dev['Details']) - result = run_program(['tput', 'cols']) - width = int( - (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 - if len(dev['Display Name']) > width: - if dev['Is Image']: - dev['Display Name'] = '...{}'.format( - dev['Display Name'][-(width-3):]) - else: - dev['Display Name'] = '{}...'.format( - dev['Display Name'][:(width-3)]) - else: - dev['Display Name'] = dev['Display Name'] - - return dev + if selection.isnumeric(): + return dev_options[int(selection)-1]['Dev'] + elif selection == 'Q': + raise GenericAbort() def setup_loopback_device(source_path): From 459b78dcc38833af2d281a6f2f186ca6e318a43d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Aug 2018 23:04:44 -0700 Subject: [PATCH 073/138] Updated select_dest_path(), now select_path() * Moved menu_select_path() code into select_path() * Removed menu_select_path() * Fixed formatting in get_dir_report() --- .bin/Scripts/functions/ddrescue.py | 178 ++++++++++++----------------- 1 file changed, 71 insertions(+), 107 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 604f3402..5d042429 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -281,7 +281,7 @@ class RecoveryState(): dest.path)) elif (source.size * 1.2) > dest.size: raise GenericError( - 'Destination is too small, refusing to continue.') + 'Not enough free space, refusing to continue.') elif dest.fstype.lower() not in RECOMMENDED_FSTYPES: print_error( 'Destination filesystem "{}" is not recommended.'.format( @@ -471,24 +471,25 @@ def get_dir_details(dir_path): def get_dir_report(dir_path): """Build colored dir report using findmnt, returns str.""" + dir_path = dir_path output = [] width = len(dir_path)+1 result = run_program([ 'findmnt', '--output', 'SIZE,AVAIL,USED,FSTYPE,OPTIONS', '--target', dir_path]) - for line in result.stdout.decode().strip().splitlines(): + for line in result.stdout.decode().splitlines(): if 'FSTYPE' in line: output.append('{BLUE}{label:<{width}}{line}{CLEAR}'.format( label='PATH', width=width, - line=line, + line=line.replace('\n',''), **COLORS)) else: output.append('{path:<{width}}{line}'.format( path=dir_path, width=width, - line=line)) + line=line.replace('\n',''))) # Done return '\n'.join(output) @@ -583,9 +584,10 @@ def menu_ddrescue(source_path, dest_path, run_mode): dest = create_path_obj(dest_path) else: if run_mode == 'clone': + x dest = select_device('destination', skip_device=source) else: - dest = select_directory() + dest = select_path(skip_device=source) dest.self_check() # Build BlockPairs @@ -796,78 +798,6 @@ def menu_select_children(source): return selected_children -def menu_select_path(skip_device={}): - """Select path via menu, returns path as str.""" - pwd = os.path.realpath(global_vars['Env']['PWD']) - s_path = None - - # Build Menu - path_options = [ - {'Name': 'Current directory: {}'.format(pwd), 'Path': pwd}, - {'Name': 'Local device', 'Path': None}, - {'Name': 'Enter manually', 'Path': None}] - actions = [{'Name': 'Quit', 'Letter': 'Q'}] - - # Show Menu - selection = menu_select( - title='Please make a selection', - main_entries=path_options, - action_entries=actions) - - if selection == 'Q': - raise GenericAbort() - elif selection.isnumeric(): - index = int(selection) - 1 - if path_options[index]['Path']: - # Current directory - s_path = pwd - - elif path_options[index]['Name'] == 'Local device': - # Local device - local_device = select_device( - skip_device=skip_device, - allow_image_file=False) - - # Mount device volume(s) - report = mount_volumes( - all_devices=False, - device_path=local_device['Dev Path'], - read_write=True) - - # Select volume - vol_options = [] - for k, v in sorted(report.items()): - disabled = v['show_data']['data'] == 'Failed to mount' - if disabled: - name = '{name} (Failed to mount)'.format(**v) - else: - name = '{name} (mounted on "{mount_point}")'.format(**v) - vol_options.append({ - 'Name': name, - 'Path': v['mount_point'], - 'Disabled': disabled}) - selection = menu_select( - title='Please select a volume', - main_entries=vol_options, - action_entries=actions) - if selection.isnumeric(): - s_path = vol_options[int(selection)-1]['Path'] - elif selection == 'Q': - raise GenericAbort() - - elif path_options[index]['Name'] == 'Enter manually': - # Manual entry - while not s_path: - m_path = input('Please enter path: ').strip() - if m_path and pathlib.Path(m_path).is_dir(): - s_path = os.path.realpath(m_path) - elif m_path and pathlib.Path(m_path).is_file(): - print_error('File "{}" exists'.format(m_path)) - else: - print_error('Invalid path "{}"'.format(m_path)) - return s_path - - def menu_settings(source): """Change advanced ddrescue settings.""" title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) @@ -1045,41 +975,75 @@ def run_ddrescue(source, dest, settings): run_program(['tmux', 'kill-pane', '-t', smart_pane]) -def select_dest_path(provided_path=None, skip_device={}): - dest = {'Is Dir': True, 'Is Image': False} +def select_path(skip_device=None): + """Optionally mount local dev and select path, returns DirObj.""" + wd = os.path.realpath(global_vars['Env']['PWD']) + selected_path = None - # Set path - if provided_path: - dest['Path'] = provided_path - else: - dest['Path'] = menu_select_path(skip_device=skip_device) - dest['Path'] = os.path.realpath(dest['Path']) + # Build menu + path_options = [ + {'Name': 'Current directory: {}'.format(wd), 'Path': wd}, + {'Name': 'Local device', 'Path': None}, + {'Name': 'Enter manually', 'Path': None}] + actions = [{'Name': 'Quit', 'Letter': 'Q'}] - # Check path - if not pathlib.Path(dest['Path']).is_dir(): - raise GenericError('Invalid path "{}"'.format(dest['Path'])) + # Show Menu + selection = menu_select( + title='Please make a selection', + main_entries=path_options, + action_entries=actions) - # Create ticket folder - if ask('Create ticket folder?'): - ticket_folder = get_simple_string('Please enter folder name') - dest['Path'] = os.path.join( - dest['Path'], ticket_folder) - try: - os.makedirs(dest['Path'], exist_ok=True) - except OSError: - raise GenericError( - 'Failed to create folder "{}"'.format(dest['Path'])) + if selection == 'Q': + raise GenericAbort() + elif selection.isnumeric(): + index = int(selection) - 1 + if path_options[index]['Path'] == wd: + # Current directory + selected_path = DirObj(wd) - # Set display name - result = run_program(['tput', 'cols']) - width = int( - (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 - if len(dest['Path']) > width: - dest['Display Name'] = '...{}'.format(dest['Path'][-(width-3):]) - else: - dest['Display Name'] = dest['Path'] + elif path_options[index]['Name'] == 'Local device': + # Local device + local_device = select_device( + skip_device=skip_device) - return dest + # Mount device volume(s) + report = mount_volumes( + all_devices=False, + device_path=local_device.path, + read_write=True) + + # Select volume + vol_options = [] + for k, v in sorted(report.items()): + disabled = v['show_data']['data'] == 'Failed to mount' + if disabled: + name = '{name} (Failed to mount)'.format(**v) + else: + name = '{name} (mounted on "{mount_point}")'.format(**v) + vol_options.append({ + 'Name': name, + 'Path': v['mount_point'], + 'Disabled': disabled}) + selection = menu_select( + title='Please select a volume', + main_entries=vol_options, + action_entries=actions) + if selection.isnumeric(): + selected_path = DirObj(vol_options[int(selection)-1]['Path']) + elif selection == 'Q': + raise GenericAbort() + + elif path_options[index]['Name'] == 'Enter manually': + # Manual entry + while not selected_path: + manual_path = input('Please enter path: ').strip() + if manual_path and pathlib.Path(manual_path).is_dir(): + selected_path = DirObj(manual_path) + elif manual_path and pathlib.Path(manual_path).is_file(): + print_error('File "{}" exists'.format(manual_path)) + else: + print_error('Invalid path "{}"'.format(manual_path)) + return selected_path def select_device(description='device', skip_device=None): From 177bf10e2d2ce00c224dbb14a105ba35bc5c4c94 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Aug 2018 23:32:39 -0700 Subject: [PATCH 074/138] Added select_parts() function * This replaced menu_select_children() * Removed menu_select_children() --- .bin/Scripts/functions/ddrescue.py | 141 ++++++++++++++--------------- 1 file changed, 70 insertions(+), 71 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 5d042429..a9b373d3 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -595,9 +595,9 @@ def menu_ddrescue(source_path, dest_path, run_mode): if run_mode == 'clone': state.add_block_pair(source, dest) else: - # TODO select dev or child dev(s) - state.add_block_pair(source, dest) - pass + source_parts = select_parts(source) + for part in source_parts: + state.add_block_pair(part, dest) # Update state state.self_checks() @@ -730,74 +730,6 @@ def menu_main(source, dest): break -def menu_select_children(source): - """Select child device(s) or whole disk, returns list.""" - dev_options = [{ - 'Base Name': '{:<14}(Whole device)'.format(source['Dev Path']), - 'Path': source['Dev Path'], - 'Selected': True}] - for c_details in source['Details'].get('children', []): - dev_options.append({ - 'Base Name': '{:<14}({:>6} {})'.format( - c_details['name'], - c_details['size'], - c_details['fstype'] if c_details['fstype'] else 'Unknown'), - 'Details': c_details, - 'Path': c_details['name'], - 'Selected': False}) - actions = [ - {'Name': 'Proceed', 'Letter': 'P'}, - {'Name': 'Quit', 'Letter': 'Q'}] - - # Skip Menu if there's no children - if len(dev_options) == 1: - return [] - - # Show Menu - while True: - one_or_more_devs_selected = False - # Update entries - for dev in dev_options: - if dev['Selected']: - one_or_more_devs_selected = True - dev['Name'] = '* {}'.format(dev['Base Name']) - else: - dev['Name'] = ' {}'.format(dev['Base Name']) - - selection = menu_select( - title='Please select part(s) to image', - main_entries=dev_options, - action_entries=actions) - - if selection.isnumeric(): - # Toggle selection - index = int(selection) - 1 - dev_options[index]['Selected'] = not dev_options[index]['Selected'] - - # Deselect whole device if child selected (this round) - if index > 0: - dev_options[0]['Selected'] = False - - # Deselect all children if whole device selected - if dev_options[0]['Selected']: - for dev in dev_options[1:]: - dev['Selected'] = False - elif selection == 'P' and one_or_more_devs_selected: - break - elif selection == 'Q': - raise GenericAbort() - - # Check selection - selected_children = [{ - 'Details': d['Details'], - 'Dev Path': d['Path'], - 'Pass 1': {'Status': 'Pending', 'Done': False}, - 'Pass 2': {'Status': 'Pending', 'Done': False}, - 'Pass 3': {'Status': 'Pending', 'Done': False}} for d in dev_options - if d['Selected'] and 'Whole device' not in d['Base Name']] - return selected_children - - def menu_settings(source): """Change advanced ddrescue settings.""" title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) @@ -975,6 +907,73 @@ def run_ddrescue(source, dest, settings): run_program(['tmux', 'kill-pane', '-t', smart_pane]) +def select_parts(source_device): + """Select partition(s) or whole device, returns list of DevObj()s.""" + selected_parts = [] + children = source_device.details.get('children', []) + + if not children: + # No partitions detected, auto-select whole device. + selected_parts = [source_device] + else: + # Build menu + dev_options = [{ + 'Base Name': '{:<14}(Whole device)'.format(source_device.path), + 'Dev': source_device, + 'Selected': True}] + for c_details in children: + dev_options.append({ + 'Base Name': '{:<14}({:>6} {})'.format( + c_details['name'], + c_details['size'], + c_details['fstype'] if c_details['fstype'] else 'Unknown'), + 'Details': c_details, + 'Dev': DevObj(c_details['name']), + 'Selected': False}) + actions = [ + {'Name': 'Proceed', 'Letter': 'P'}, + {'Name': 'Quit', 'Letter': 'Q'}] + + # Show menu + while True: + one_or_more_devs_selected = False + # Update entries + for dev in dev_options: + if dev['Selected']: + one_or_more_devs_selected = True + dev['Name'] = '* {}'.format(dev['Base Name']) + else: + dev['Name'] = ' {}'.format(dev['Base Name']) + + selection = menu_select( + title='Please select part(s) to image', + main_entries=dev_options, + action_entries=actions) + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + dev_options[index]['Selected'] = not dev_options[index]['Selected'] + + # Deselect whole device if child selected (this round) + if index > 0: + dev_options[0]['Selected'] = False + + # Deselect all children if whole device selected + if dev_options[0]['Selected']: + for dev in dev_options[1:]: + dev['Selected'] = False + elif selection == 'P' and one_or_more_devs_selected: + break + elif selection == 'Q': + raise GenericAbort() + + # Build list of selected parts + selected_parts = [d['Dev'] for d in dev_options if d['Selected']] + + return selected_parts + + def select_path(skip_device=None): """Optionally mount local dev and select path, returns DirObj.""" wd = os.path.realpath(global_vars['Env']['PWD']) From c568668fd0971202c50d7e4d501100e6b06d2886 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Aug 2018 23:39:19 -0700 Subject: [PATCH 075/138] Re-added 'Create ticket folder?' section * Only asked if imaging and mounting a local device for the destination. --- .bin/Scripts/functions/ddrescue.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index a9b373d3..14c964c1 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -1004,6 +1004,7 @@ def select_path(skip_device=None): # Local device local_device = select_device( skip_device=skip_device) + s_path = '' # Mount device volume(s) report = mount_volumes( @@ -1028,10 +1029,23 @@ def select_path(skip_device=None): main_entries=vol_options, action_entries=actions) if selection.isnumeric(): - selected_path = DirObj(vol_options[int(selection)-1]['Path']) + s_path = vol_options[int(selection)-1]['Path'] elif selection == 'Q': raise GenericAbort() + # Create folder + if ask('Create ticket folder?'): + ticket_folder = get_simple_string('Please enter folder name') + s_path = os.path.join(s_path, ticket_folder) + try: + os.makedirs(s_path, exist_ok=True) + except OSError: + raise GenericError( + 'Failed to create folder "{}"'.format(s_path)) + + # Create DirObj + selected_path = DirObj(s_path) + elif path_options[index]['Name'] == 'Enter manually': # Manual entry while not selected_path: From d474c8b5d430614375169329f322c4a683b5249e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 1 Aug 2018 23:56:25 -0700 Subject: [PATCH 076/138] Updated build_outer_panes() *BROKEN* * Script broken until update_progress() is fixed --- .bin/Scripts/functions/ddrescue.py | 33 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 14c964c1..99dbf2fc 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -246,11 +246,14 @@ class ImageObj(BaseObj): class RecoveryState(): """Object to track BlockPair objects and overall state.""" - def __init__(self, mode): + def __init__(self, mode, source, dest): + self.mode = mode.lower() + self.source_name = source.name + self.dest_name = dest.name self.block_pairs = [] self.current_pass = 0 self.finished = False - self.mode = mode.lower() + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.rescued = 0 self.resumed = False self.started = False @@ -338,7 +341,7 @@ class RecoveryState(): # Functions -def build_outer_panes(source, dest): +def build_outer_panes(state): """Build top and side panes.""" clear_screen() result = run_program(['tput', 'cols']) @@ -346,12 +349,12 @@ def build_outer_panes(source, dest): (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 # Top panes - source_str = source.name + source_str = state.source_name if len(source_str) > width: source_str = '{}...'.format(source_str[:width-3]) - dest_str = dest.name + dest_str = state.dest_name if len(dest_str) > width: - if dest.is_dev(): + if state.mode == 'clone': dest_str = '{}...'.format(dest_str[:width-3]) else: dest_str = '...{}'.format(dest_str[-width+3:]) @@ -375,12 +378,11 @@ def build_outer_panes(source, dest): **COLORS)) # Side pane - # TODO FIX IT - #update_progress(source) - #tmux_splitw( - # '-dhl', '{}'.format(SIDE_PANE_WIDTH), - # 'watch', '--color', '--no-title', '--interval', '1', - # 'cat', source['Progress Out']) + update_progress(state) + tmux_splitw( + '-dhl', str(SIDE_PANE_WIDTH), + 'watch', '--color', '--no-title', '--interval', '1', + 'cat', state.progress_out) def create_path_obj(path): @@ -591,12 +593,11 @@ def menu_ddrescue(source_path, dest_path, run_mode): dest.self_check() # Build BlockPairs - state = RecoveryState(run_mode) + state = RecoveryState(run_mode, source, dest) if run_mode == 'clone': state.add_block_pair(source, dest) else: - source_parts = select_parts(source) - for part in source_parts: + for part in select_parts(source): state.add_block_pair(part, dest) # Update state @@ -617,7 +618,7 @@ def menu_ddrescue(source_path, dest_path, run_mode): raise GenericAbort() # Main menu - build_outer_panes(source, dest) + build_outer_panes(state) # TODO Fix #menu_main(source, dest) pause('Fake Main Menu... ') From 1a983344c28ef4c2aced823bdf0f53f0b173ce6b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 2 Aug 2018 00:42:14 -0700 Subject: [PATCH 077/138] Updated update_progress() --- .bin/Scripts/functions/ddrescue.py | 222 +++++------------------------ 1 file changed, 37 insertions(+), 185 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 99dbf2fc..362b467f 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -62,10 +62,10 @@ class BaseObj(): class BlockPair(): """Object to track data and methods together for source and dest.""" - def __init__(self, source, dest, mode): - self.source_path = source.path + def __init__(self, mode, source, dest): self.mode = mode - self.name = source.name + self.source = source + self.dest = dest self.pass_done = [False, False, False] self.resumed = False self.rescued = 0 @@ -89,6 +89,11 @@ class BlockPair(): if os.path.exists(self.map_path): self.load_map_data() self.resumed = True + # Fix status strings + for pass_num in [1, 2, 3]: + self.status[pass_num-1] = get_formatted_status( + label='Pass {}'.format(pass_num), + data=self.status[pass_num-1]) def finish_pass(self, pass_num): """Mark pass as done and check if 100% recovered.""" @@ -196,7 +201,7 @@ class DevObj(BaseObj): if self.parent: # Add child device details self.prefix += '_{c_num}_{c_size}{sep}{c_label}'.format( - c_num=self.name.replace(self.parent, ''), + c_num=self.path.replace(self.parent, ''), c_size=self.details.get('size', 'UNKNOWN'), sep='_' if self.label else '', c_label=self.label) @@ -248,8 +253,8 @@ class RecoveryState(): """Object to track BlockPair objects and overall state.""" def __init__(self, mode, source, dest): self.mode = mode.lower() - self.source_name = source.name - self.dest_name = dest.name + self.source = source + self.dest = dest self.block_pairs = [] self.current_pass = 0 self.finished = False @@ -302,7 +307,7 @@ class RecoveryState(): 'Destination is mounted read-only, refusing to continue.') # Safety checks passed - self.block_pairs.append(BlockPair(source, dest, self.mode)) + self.block_pairs.append(BlockPair(self.mode, source, dest)) def self_checks(self): """Run self-checks for each BlockPair and update state values.""" @@ -349,10 +354,10 @@ def build_outer_panes(state): (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 # Top panes - source_str = state.source_name + source_str = state.source.name if len(source_str) > width: source_str = '{}...'.format(source_str[:width-3]) - dest_str = state.dest_name + dest_str = state.dest.name if len(dest_str) > width: if state.mode == 'clone': dest_str = '{}...'.format(dest_str[:width-3]) @@ -586,7 +591,6 @@ def menu_ddrescue(source_path, dest_path, run_mode): dest = create_path_obj(dest_path) else: if run_mode == 'clone': - x dest = select_device('destination', skip_device=source) else: dest = select_path(skip_device=source) @@ -970,7 +974,12 @@ def select_parts(source_device): raise GenericAbort() # Build list of selected parts - selected_parts = [d['Dev'] for d in dev_options if d['Selected']] + for d in dev_options: + if d['Selected']: + d['Dev'].model = source_device.model + d['Dev'].model_size = source_device.model_size + d['Dev'].update_filename_prefix() + selected_parts.append(d['Dev']) return selected_parts @@ -1179,197 +1188,40 @@ def tmux_splitw(*args): return result.stdout.decode().strip() -def update_progress(source, end_run=False): - """Update progress for source dev(s) and update status pane file.""" - current_pass = source['Current Pass'] - pass_complete_for_all_devs = True - total_recovery = True - source['Recovered Size'] = 0 - if current_pass != 'Done': - source[current_pass]['Min Status'] = 100 - try: - current_pass_num = int(current_pass[-1:]) - next_pass_num = current_pass_num + 1 - except ValueError: - # Either Done or undefined? - current_pass_num = -1 - next_pass_num = -1 - if 1 <= next_pass_num <= 3: - next_pass = 'Pass {}'.format(next_pass_num) - else: - next_pass = 'Done' - - # Update children progress - for child in source['Children']: - if current_pass == 'Done': - continue - if os.path.exists(child['Dest Paths']['Map']): - map_data = read_map_file(child['Dest Paths']['Map']) - if child['Dev Path'] == source.get('Current Device', ''): - # Current child device - r_size = map_data['rescued']/100 * child['Size'] - child[current_pass]['Done'] = map_data['pass completed'] - if source['Started Recovery']: - child[current_pass]['Status'] = map_data['rescued'] - child['Recovered Size'] = r_size - - # All child devices - pass_complete_for_all_devs &= child[current_pass]['Done'] - total_recovery &= map_data['full recovery'] - try: - source['Recovered Size'] += child.get('Recovered Size', 0) - source[current_pass]['Min Status'] = min( - source[current_pass]['Min Status'], - child[current_pass]['Status']) - except TypeError: - # Force 0% to disable auto-continue - source[current_pass]['Min Status'] = 0 - else: - # Map missing, assuming this pass hasn't run for this dev yet - pass_complete_for_all_devs = False - total_recovery = False - - # Update source progress - if len(source['Children']) > 0: - # Imaging parts, skip updating source progress - pass - elif os.path.exists(source['Dest Paths']['Map']): - # Cloning/Imaging whole device - map_data = read_map_file(source['Dest Paths']['Map']) - if current_pass != 'Done': - source[current_pass]['Done'] = map_data['pass completed'] - if source['Started Recovery']: - source[current_pass]['Status'] = map_data['rescued'] - try: - source[current_pass]['Min Status'] = min( - source[current_pass]['Min Status'], - source[current_pass]['Status']) - except TypeError: - # Force 0% to disable auto-continue - source[current_pass]['Min Status'] = 0 - pass_complete_for_all_devs &= source[current_pass]['Done'] - source['Recovered Size'] = map_data['rescued']/100 * source['Size'] - total_recovery &= map_data['full recovery'] - else: - # Cloning/Imaging whole device and map missing - pass_complete_for_all_devs = False - total_recovery = False - - # End of pass updates - if end_run: - if total_recovery: - # Sweet! - source['Current Pass'] = 'Done' - source['Recovered Size'] = source['Total Size'] - for p_num in ['Pass 1', 'Pass 2', 'Pass 3']: - if source[p_num]['Status'] == 'Pending': - source[p_num]['Status'] = 'Skipped' - for child in source['Children']: - if child[p_num]['Status'] == 'Pending': - child[p_num]['Status'] = 'Skipped' - elif pass_complete_for_all_devs: - # Ready for next pass? - source['Current Pass'] = next_pass - if current_pass != 'Done': - source[current_pass]['Done'] = True - - # Start building output lines - if 'Progress Out' not in source: - source['Progress Out'] = '{}/progress.out'.format( - global_vars['LogDir']) +def update_progress(state): + """Update progress file for side pane.""" output = [] - if source['Type'] == 'clone': + if state.mode == 'clone': output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) else: output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) output.append('─────────────────────') # Overall progress - recovered_p = (source['Recovered Size'] / source['Total Size']) * 100 - recovered_s = human_readable_size(source['Recovered Size']) output.append('{BLUE}Overall Progress{CLEAR}'.format(**COLORS)) - output.append('Recovered:{s_color}{recovered_p:>9.2f} %{CLEAR}'.format( - s_color=get_status_color(recovered_p), - recovered_p=recovered_p, - **COLORS)) - output.append('{recovered_s:>{width}}'.format( - recovered_s=recovered_s, width=SIDE_PANE_WIDTH)) + output.append(state.status_percent) + output.append(state.status_amount) output.append('─────────────────────') - # Main device - if source['Type'] == 'clone': - output.append('{BLUE}{dev}{CLEAR}'.format( - dev='Image File' if source['Is Image'] else source['Dev Path'], - **COLORS)) - for x in (1, 2, 3): - p_num = 'Pass {}'.format(x) - s_display = source[p_num]['Status'] - try: - s_display = float(s_display) - except ValueError: - # Ignore and leave s_display alone - pass - else: - s_display = '{:0.2f} %'.format(s_display) - output.append('{p_num}{s_color}{s_display:>15}{CLEAR}'.format( - p_num=p_num, - s_color=get_status_color(source[p_num]['Status']), - s_display=s_display, + # Source(s) progress + for bp in state.block_pairs: + if state.source.is_image(): + output.append('{BLUE}Image File{CLEAR}'.format(**COLORS)) + elif state.mode == 'image' and len(state.block_pairs) == 1: + output.append('{BLUE}{source} {YELLOW}(Whole){CLEAR}'.format( + source=bp.source.path, **COLORS)) - else: - # Image mode - if source['Children']: - # Just parts - for child in source['Children']: - output.append('{BLUE}{dev}{CLEAR}'.format( - dev=child['Dev Path'], - **COLORS)) - for x in (1, 2, 3): - p_num = 'Pass {}'.format(x) - s_display = child[p_num]['Status'] - try: - s_display = float(s_display) - except ValueError: - # Ignore and leave s_display alone - pass - else: - s_display = '{:0.2f} %'.format(s_display) - output.append( - '{p_num}{s_color}{s_display:>15}{CLEAR}'.format( - p_num=p_num, - s_color=get_status_color(child[p_num]['Status']), - s_display=s_display, - **COLORS)) - p = (child.get('Recovered Size', 0) / child['Size']) * 100 - output.append('Recovered:{s_color}{p:>9.2f} %{CLEAR}'.format( - s_color=get_status_color(p), p=p, **COLORS)) - output.append(' ') else: - # Whole device - output.append('{BLUE}{dev}{CLEAR} {YELLOW}(Whole){CLEAR}'.format( - dev=source['Dev Path'], + output.append('{BLUE}{source}{CLEAR}'.format( + source=bp.source.path, **COLORS)) - for x in (1, 2, 3): - p_num = 'Pass {}'.format(x) - s_display = source[p_num]['Status'] - try: - s_display = float(s_display) - except ValueError: - # Ignore and leave s_display alone - pass - else: - s_display = '{:0.2f} %'.format(s_display) - output.append( - '{p_num}{s_color}{s_display:>15}{CLEAR}'.format( - p_num=p_num, - s_color=get_status_color(source[p_num]['Status']), - s_display=s_display, - **COLORS)) + output.extend(bp.status) + output.append(' ') # Add line-endings output = ['{}\n'.format(line) for line in output] - with open(source['Progress Out'], 'w') as f: + with open(state.progress_out, 'w') as f: f.writelines(output) From e0a695a6731c33cdbca89d450e60a89ec1da042b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 15 Aug 2018 20:35:09 -0700 Subject: [PATCH 078/138] Enable help flags for aliases --- .bin/Scripts/ddrescue-tui-menu | 10 +++++++--- .bin/Scripts/functions/ddrescue.py | 5 ++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index f96dec36..988ec0fa 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -31,13 +31,17 @@ if __name__ == '__main__': # We'll set the missing paths later pass - # Show usage? + # Show usage + if re.search(r'-*(h|help|\?)', str(sys.argv), re.IGNORECASE): + show_usage(script_name) + exit_script() + + # Start cloning/imaging if run_mode in ('clone', 'image'): menu_ddrescue(source_path, dest_path, run_mode) else: - if not re.search(r'(^$|help|-h|\?)', run_mode, re.IGNORECASE): + if not re.search(r'^-*(h|help\?)', run_mode, re.IGNORECASE): print_error('Invalid mode.') - show_usage(script_name) # Done print_standard('\nDone.') diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 362b467f..3657bb19 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -118,7 +118,7 @@ class BlockPair(): def load_map_data(self): """Load data from map file and set progress.""" map_data = read_map_file(self.map_path) - self.rescued = map_data['rescued'] * self.size + self.rescued = map_data['rescued'] * self.size / 100 if map_data['full recovery']: self.pass_done = [True, True, True] self.rescued = self.size @@ -132,6 +132,9 @@ class BlockPair(): elif map_data['non-scraped'] > 0: self.pass_done = [True, True, False] self.status = ['Skipped', 'Skipped', 'Pending'] + else: + self.pass_done = [True, True, True] + self.status = ['Skipped', 'Skipped', 'Skipped'] def self_check(self): """Self check to abort on bad dest/map combinations.""" From 53a899f9678ca0347e6aa10d5ebdc210ea2679ec Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 15 Aug 2018 21:53:22 -0700 Subject: [PATCH 079/138] Updated menu_main() to use RecoveryState obj * Also fixed rescued size calculations (again) --- .bin/Scripts/functions/ddrescue.py | 92 +++++++++++++++++------------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 3657bb19..cc57fc3c 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -13,8 +13,8 @@ from functions.data import * from operator import itemgetter # STATIC VARIABLES -AUTO_NEXT_PASS_1_THRESHOLD = 95 -AUTO_NEXT_PASS_2_THRESHOLD = 98 +AUTO_PASS_1_THRESHOLD = 95 +AUTO_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { '--binary-prefixes': {'Enabled': True, 'Hidden': True}, '--data-preview': {'Enabled': True, 'Hidden': True}, @@ -118,7 +118,8 @@ class BlockPair(): def load_map_data(self): """Load data from map file and set progress.""" map_data = read_map_file(self.map_path) - self.rescued = map_data['rescued'] * self.size / 100 + self.rescued_percent = map_data['rescued'] + self.rescued = (self.rescued_percent * self.size) / 100 if map_data['full recovery']: self.pass_done = [True, True, True] self.rescued = self.size @@ -157,7 +158,8 @@ class BlockPair(): """Update progress using map file.""" if os.path.exists(self.map_path): map_data = read_map_file(self.map_path) - self.rescued = map_data['rescued'] * self.size + self.rescued_percent = map_data['rescued'] + self.rescued = (self.rescued_percent * self.size) / 100 self.status[pass_num] = get_formatted_status( label='Pass {}'.format(pass_num), data=(self.rescued/self.size)*100) @@ -260,6 +262,8 @@ class RecoveryState(): self.dest = dest self.block_pairs = [] self.current_pass = 0 + self.current_pass_str = '0: Initializing' + self.ddrescue_settings = DDRESCUE_SETTINGS.copy() self.finished = False self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.rescued = 0 @@ -312,6 +316,20 @@ class RecoveryState(): # Safety checks passed self.block_pairs.append(BlockPair(self.mode, source, dest)) + def current_pass_done(self): + """Checks if pass is done for all block-pairs, returns bool.""" + done = True + for bp in self.block_pairs: + done &= bp.get_pass_done(self.current_pass) + return done + + def current_pass_min(self): + """Gets minimum pass rescued percentage, returns float.""" + min_percent = 100 + for bp in self.block_pairs: + min_percent = min(min_percent, bp.rescued) + return min_percent + def self_checks(self): """Run self-checks for each BlockPair and update state values.""" self.total_size = 0 @@ -336,14 +354,23 @@ class RecoveryState(): # Also mark overall recovery as finished if on last pass self.finished = True break + if self.finished: + self.current_pass_str = '- "Done"' + elif pass_num == 0: + self.current_pass_str = '1 "Initial Read"' + elif pass_num == 1: + self.current_pass_str = '2 "Trimming bad areas"' + elif pass_num == 2: + self.current_pass_str = '3 "Scraping bad areas"' def update_progress(self): """Update overall progress using block_pairs.""" self.rescued = 0 for bp in self.block_pairs: self.rescued += bp.rescued + self.rescued_percent = (self.rescued / self.total_size) * 100 self.status_percent = get_formatted_status( - label='Recovered:', data=(self.rescued/self.total_size)*100) + label='Recovered:', data=self.rescued_percent) self.status_amount = get_formatted_status( label='', data=human_readable_size(self.rescued)) @@ -626,9 +653,7 @@ def menu_ddrescue(source_path, dest_path, run_mode): # Main menu build_outer_panes(state) - # TODO Fix - #menu_main(source, dest) - pause('Fake Main Menu... ') + menu_main(state) # Done run_program(['tmux', 'kill-window']) @@ -638,8 +663,6 @@ def menu_main(source, dest): """Main menu is used to set ddrescue settings.""" title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) - if 'Settings' not in source: - source['Settings'] = DDRESCUE_SETTINGS.copy() # Build menu main_options = [ @@ -659,14 +682,6 @@ def menu_main(source, dest): # Show menu while True: - current_pass = source['Current Pass'] - display_pass = '1 "Initial Read"' - if current_pass == 'Pass 2': - display_pass = '2 "Trimming bad areas"' - elif current_pass == 'Pass 3': - display_pass = '3 "Scraping bad areas"' - elif current_pass == 'Done': - display_pass = 'Done' # Update entries for opt in main_options: opt['Name'] = '{} {}'.format( @@ -674,7 +689,7 @@ def menu_main(source, dest): opt['Base Name']) selection = menu_select( - title=title+display_pass, + title=title+state.current_pass_str, main_entries=main_options, action_entries=actions) @@ -685,7 +700,7 @@ def menu_main(source, dest): elif selection == 'S': # Set settings for pass settings = [] - for k, v in source['Settings'].items(): + for k, v in state.ddrescue_settings.items(): if not v['Enabled']: continue if k[-1:] == '=': @@ -707,35 +722,34 @@ def menu_main(source, dest): if 'Auto' not in opt['Base Name']: opt['Enabled'] = False - # Run ddrecue - first_run = True - while auto_run or first_run: - first_run = False - run_ddrescue(source, dest, settings) - update_progress(source, end_run=True) - if current_pass == 'Done': - # "Pass Done" i.e. all passes done + # Run ddrescue + state.started = False + while auto_run or not state.started: + state.started = True + run_ddrescue(state) + if state.finished or not auto_run: break - if not main_options[0]['Enabled']: - # Auto next pass - break - if source[current_pass]['Done']: - min_status = source[current_pass]['Min Status'] - if (current_pass == 'Pass 1' and - min_status < AUTO_NEXT_PASS_1_THRESHOLD): + if state.current_pass_done(): + if (state.current_pass == 0 and + state.current_pass_min() < AUTO_PASS_1_THRESHOLD): auto_run = False - elif (current_pass == 'Pass 2' and - min_status < AUTO_NEXT_PASS_2_THRESHOLD): + if (state.current_pass == 1 and + state.current_pass_min() < AUTO_PASS_2_THRESHOLD): auto_run = False else: auto_run = False # Update current pass for next iteration - current_pass = source['Current Pass'] + state.set_pass_num() elif selection == 'C': menu_settings(source) elif selection == 'Q': - break + if state.rescued_percent < 100: + print_warning('Recovery is less than 100%') + if ask('Are you sure you want to quit?'): + break + else: + break def menu_settings(source): From bb270715c1bad0584d1a71405606024af644759b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 15 Aug 2018 22:38:54 -0700 Subject: [PATCH 080/138] Updated run_ddrescue() to use new objects --- .bin/Scripts/functions/ddrescue.py | 79 ++++++++++++++---------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index cc57fc3c..75375c72 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -65,6 +65,7 @@ class BlockPair(): def __init__(self, mode, source, dest): self.mode = mode self.source = source + self.source_path = source.path self.dest = dest self.pass_done = [False, False, False] self.resumed = False @@ -97,6 +98,7 @@ class BlockPair(): def finish_pass(self, pass_num): """Mark pass as done and check if 100% recovered.""" + map_data = read_map_file(self.map_path) if map_data['full recovery']: self.pass_done = [True, True, True] self.rescued = self.size @@ -111,10 +113,6 @@ class BlockPair(): else: self.pass_done[pass_num] = True - def get_pass_done(self, pass_num): - """Return pass number's done state.""" - return self.pass_done[pass_num] - def load_map_data(self): """Load data from map file and set progress.""" map_data = read_map_file(self.map_path) @@ -128,8 +126,8 @@ class BlockPair(): # Initial pass incomplete pass elif map_data['non-trimmed'] > 0: - self.pass_done[0] = True - self.status[0] = 'Skipped' + self.pass_done = [True, False, False] + self.status = ['Skipped', 'Pending', 'Pending'] elif map_data['non-scraped'] > 0: self.pass_done = [True, True, False] self.status = ['Skipped', 'Skipped', 'Pending'] @@ -259,6 +257,7 @@ class RecoveryState(): def __init__(self, mode, source, dest): self.mode = mode.lower() self.source = source + self.source_path = source.path self.dest = dest self.block_pairs = [] self.current_pass = 0 @@ -320,7 +319,7 @@ class RecoveryState(): """Checks if pass is done for all block-pairs, returns bool.""" done = True for bp in self.block_pairs: - done &= bp.get_pass_done(self.current_pass) + done &= bp.pass_done[self.current_pass] return done def current_pass_min(self): @@ -345,7 +344,7 @@ class RecoveryState(): # Iterate backwards through passes pass_done = True for bp in self.block_pairs: - pass_done &= bp.get_pass_done(pass_num) + pass_done &= bp.pass_done[pass_num] if pass_done: # All block-pairs reported being done # Set to next pass, unless we're on the last pass (2) @@ -806,7 +805,11 @@ def menu_settings(source): def read_map_file(map_path): """Read map file with ddrescuelog and return data as dict.""" map_data = {} - result = run_program(['ddrescuelog', '-t', map_path]) + try: + result = run_program(['ddrescuelog', '-t', map_path]) + except CalledProcessError: + # (Grossly) assuming map_data hasn't been saved yet, return empty dict + return map_data # Parse output for line in result.stdout.decode().splitlines(): @@ -824,7 +827,7 @@ def read_map_file(map_path): # Check if 100% done try: run_program(['ddrescuelog', '-D', map_path]) - except subprocess.CalledProcessError: + except CalledProcessError: map_data['full recovery'] = False else: map_data['full recovery'] = True @@ -832,25 +835,16 @@ def read_map_file(map_path): return map_data -def run_ddrescue(source, dest, settings): +def run_ddrescue(state): """Run ddrescue pass.""" - current_pass = source['Current Pass'] return_code = None - if current_pass == 'Done': + if state.finished: clear_screen() print_warning('Recovery already completed?') pause('Press Enter to return to main menu...') return - # Set device(s) to clone/image - source[current_pass]['Status'] = 'Working' - source['Started Recovery'] = True - source_devs = [source] - if source['Children']: - # Use only selected child devices - source_devs = source['Children'] - # Set heights # NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) result = run_program(['tput', 'lines']) @@ -863,39 +857,35 @@ def run_ddrescue(source, dest, settings): '-bdvl', str(height_smart), '-PF', '#D', 'watch', '--color', '--no-title', '--interval', '300', - 'ddrescue-tui-smart-display', source['Dev Path']) + 'ddrescue-tui-smart-display', state.source_path) - # Start pass for each selected device - for s_dev in source_devs: - if s_dev[current_pass]['Done']: - # Move to next device + # Run pass for each block-pair + for bp in state.block_pairs: + if bp.pass_done[state.current_pass]: + # Skip to next block-pair continue - source['Current Device'] = s_dev['Dev Path'] - s_dev[current_pass]['Status'] = 'Working' - update_progress(source) + bp.status[state.current_pass] = 'Working' + update_progress(state) # Set ddrescue cmd - if source['Type'] == 'clone': - cmd = [ - 'ddrescue', *settings, '--force', s_dev['Dev Path'], - dest['Dev Path'], s_dev['Dest Paths']['Map']] - else: - cmd = [ - 'ddrescue', *settings, s_dev['Dev Path'], - s_dev['Dest Paths']['image'], s_dev['Dest Paths']['Map']] - if current_pass == 'Pass 1': + cmd = [ + 'ddrescue', *settings, + bp.source_path, bp.dest_path, bp.map_path] + if state.mode == 'clone': + cmd.append('--force') + if current_pass == 0: cmd.extend(['--no-trim', '--no-scrape']) - elif current_pass == 'Pass 2': + elif current_pass == 1: # Allow trimming cmd.append('--no-scrape') - elif current_pass == 'Pass 3': + elif current_pass == 2: # Allow trimming and scraping pass # Start ddrescue try: clear_screen() - print_info('Current dev: {}'.format(s_dev['Dev Path'])) + print_info('Current dev: {}'.format(bp.source_path)) ddrescue_proc = popen_program(['./__choose_exit', *cmd]) # ddrescue_proc = popen_program(['./__exit_ok', *cmd]) # ddrescue_proc = popen_program(cmd) @@ -903,10 +893,10 @@ def run_ddrescue(source, dest, settings): try: ddrescue_proc.wait(timeout=10) sleep(2) - update_progress(source) + update_progress(state) break except subprocess.TimeoutExpired: - update_progress(source) + update_progress(state) except KeyboardInterrupt: # Catch user abort pass @@ -921,6 +911,9 @@ def run_ddrescue(source, dest, settings): # i.e. not None and not 0 print_error('Error(s) encountered, see message above.') break + else: + # Mark pass finished + bp.finish_pass(state.current_pass) # Done if str(return_code) != '0': From ca78da4dd4ce95e1f83abf62857de77cfeb2a1d7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 15 Aug 2018 22:49:43 -0700 Subject: [PATCH 081/138] Updated show_selection_details() --- .bin/Scripts/functions/ddrescue.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 75375c72..56a24a4f 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -640,7 +640,7 @@ def menu_ddrescue(source_path, dest_path, run_mode): # Confirmations clear_screen() - show_selection_details(state, source, dest) + show_selection_details(state) prompt = 'Start {}?'.format(state.mode.replace('e', 'ing')) if state.resumed: print_info('Map data detected and loaded.') @@ -1168,11 +1168,11 @@ def show_device_details(dev_path): print_standard(line) -def show_selection_details(state, source, dest): +def show_selection_details(state): """Show selection details.""" # Source print_success('Source') - print_standard(source.report) + print_standard(state.source.report) print_standard(' ') # Destination @@ -1181,7 +1181,7 @@ def show_selection_details(state, source, dest): print_error('(ALL DATA WILL BE DELETED)', timestamp=False) else: print_success('Destination') - print_standard(dest.report) + print_standard(state.dest.report) print_standard(' ') From 5ac05c943ee269264fdb51fabb8b11cac6faf90a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 15 Aug 2018 22:50:05 -0700 Subject: [PATCH 082/138] Removed unused function show_device_details() --- .bin/Scripts/functions/ddrescue.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 56a24a4f..9e8d260c 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -1147,27 +1147,6 @@ def setup_loopback_device(source_path): return dev_path -def show_device_details(dev_path): - """Display device details on screen.""" - cmd = ( - 'lsblk', '--nodeps', - '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', - dev_path) - result = run_program(cmd) - output = result.stdout.decode().splitlines() - print_info(output.pop(0)) - for line in output: - print_standard(line) - - # Children info - cmd = ('lsblk', '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', dev_path) - result = run_program(cmd) - output = result.stdout.decode().splitlines() - print_info(output.pop(0)) - for line in output: - print_standard(line) - - def show_selection_details(state): """Show selection details.""" # Source From 5948d1a62fcb9f5012826f66f1a9a4f35d8385a7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 15 Aug 2018 22:50:47 -0700 Subject: [PATCH 083/138] Fixed menu_main() arguments --- .bin/Scripts/functions/ddrescue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 9e8d260c..e0a31353 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -658,7 +658,7 @@ def menu_ddrescue(source_path, dest_path, run_mode): run_program(['tmux', 'kill-window']) exit_script() -def menu_main(source, dest): +def menu_main(state): """Main menu is used to set ddrescue settings.""" title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) From 8461e581ea13675b3966bb4cd5d21b409079ce12 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 15 Aug 2018 23:24:01 -0700 Subject: [PATCH 084/138] Updated menu_settings() to use RecoveryState --- .bin/Scripts/functions/ddrescue.py | 51 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index e0a31353..c1065d97 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -262,7 +262,7 @@ class RecoveryState(): self.block_pairs = [] self.current_pass = 0 self.current_pass_str = '0: Initializing' - self.ddrescue_settings = DDRESCUE_SETTINGS.copy() + self.settings = DDRESCUE_SETTINGS.copy() self.finished = False self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) self.rescued = 0 @@ -569,7 +569,7 @@ def get_status_color(s, t_success=99, t_warn=90): if s in ('Pending',) or str(s)[-2:] in (' b', 'Kb', 'Mb', 'Gb', 'Tb'): color = COLORS['CLEAR'] - elif s in ('Skipped', 'Unknown', 'Working'): + elif s in ('Skipped', 'Unknown'): color = COLORS['YELLOW'] elif p_recovered >= t_success: color = COLORS['GREEN'] @@ -699,7 +699,7 @@ def menu_main(state): elif selection == 'S': # Set settings for pass settings = [] - for k, v in state.ddrescue_settings.items(): + for k, v in state.settings.items(): if not v['Enabled']: continue if k[-1:] == '=': @@ -741,7 +741,7 @@ def menu_main(state): state.set_pass_num() elif selection == 'C': - menu_settings(source) + menu_settings(state) elif selection == 'Q': if state.rescued_percent < 100: print_warning('Recovery is less than 100%') @@ -751,7 +751,7 @@ def menu_main(state): break -def menu_settings(source): +def menu_settings(state): """Change advanced ddrescue settings.""" title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) title += '{YELLOW}These settings can cause {CLEAR}'.format(**COLORS) @@ -761,7 +761,7 @@ def menu_settings(source): # Build menu settings = [] - for k, v in sorted(source['Settings'].items()): + for k, v in sorted(state.settings.items()): if not v.get('Hidden', False): settings.append({'Base Name': k.replace('=', ''), 'Flag': k}) actions = [{'Name': 'Main Menu', 'Letter': 'M'}] @@ -771,9 +771,9 @@ def menu_settings(source): for s in settings: s['Name'] = '{}{}{}'.format( s['Base Name'], - ' = ' if 'Value' in source['Settings'][s['Flag']] else '', - source['Settings'][s['Flag']].get('Value', '')) - if not source['Settings'][s['Flag']]['Enabled']: + ' = ' if 'Value' in state.settings[s['Flag']] else '', + state.settings[s['Flag']].get('Value', '')) + if not state.settings[s['Flag']]['Enabled']: s['Name'] = '{YELLOW}{name} (Disabled){CLEAR}'.format( name=s['Name'], **COLORS) @@ -784,27 +784,27 @@ def menu_settings(source): if selection.isnumeric(): index = int(selection) - 1 flag = settings[index]['Flag'] - enabled = source['Settings'][flag]['Enabled'] - if 'Value' in source['Settings'][flag]: + enabled = state.settings[flag]['Enabled'] + if 'Value' in state.settings[flag]: answer = choice( choices=['T', 'C'], prompt='Toggle or change value for "{}"'.format(flag)) if answer == 'T': # Toggle - source['Settings'][flag]['Enabled'] = not enabled + state.settings[flag]['Enabled'] = not enabled else: # Update value - source['Settings'][flag]['Value'] = get_simple_string( + state.settings[flag]['Value'] = get_simple_string( prompt='Enter new value') else: - source['Settings'][flag]['Enabled'] = not enabled + state.settings[flag]['Enabled'] = not enabled elif selection == 'M': break def read_map_file(map_path): """Read map file with ddrescuelog and return data as dict.""" - map_data = {} + map_data = {'full recovery': False} try: result = run_program(['ddrescuelog', '-t', map_path]) except CalledProcessError: @@ -864,21 +864,28 @@ def run_ddrescue(state): if bp.pass_done[state.current_pass]: # Skip to next block-pair continue - bp.status[state.current_pass] = 'Working' update_progress(state) # Set ddrescue cmd - cmd = [ - 'ddrescue', *settings, - bp.source_path, bp.dest_path, bp.map_path] + cmd = ['ddrescue', bp.source_path, bp.dest_path, bp.map_path] + for k, v in state.settings.items(): + if not v['Enabled']: + continue + if 'Value' in v: + cmd.append('{}{}{}'.format( + k, + '' if k[-1:] == '=' else ' ', + v['Value'])) + else: + cmd.append(k) if state.mode == 'clone': cmd.append('--force') - if current_pass == 0: + if state.current_pass == 0: cmd.extend(['--no-trim', '--no-scrape']) - elif current_pass == 1: + elif state.current_pass == 1: # Allow trimming cmd.append('--no-scrape') - elif current_pass == 2: + elif state.current_pass == 2: # Allow trimming and scraping pass From 7d30a735fc6cff314d3e1b5eafdafa7e67b8463f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 16 Aug 2018 00:18:25 -0700 Subject: [PATCH 085/138] Fix retry option and settings sections --- .bin/Scripts/functions/ddrescue.py | 71 +++++++++++++----------------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index c1065d97..1858a30f 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -17,15 +17,15 @@ AUTO_PASS_1_THRESHOLD = 95 AUTO_PASS_2_THRESHOLD = 98 DDRESCUE_SETTINGS = { '--binary-prefixes': {'Enabled': True, 'Hidden': True}, - '--data-preview': {'Enabled': True, 'Hidden': True}, + '--data-preview': {'Enabled': True, 'Hidden': True, 'Value': '5'}, '--idirect': {'Enabled': True}, '--odirect': {'Enabled': True}, '--max-read-rate': {'Enabled': False, 'Value': '32MiB'}, '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, '--reopen-on-error': {'Enabled': True}, - '--retry-passes=': {'Enabled': True, 'Value': '0'}, - '--test-mode=': {'Enabled': False, 'Value': 'test.map'}, - '--timeout=': {'Enabled': True, 'Value': '5m'}, + '--retry-passes': {'Enabled': True, 'Value': '0'}, + '--test-mode': {'Enabled': False, 'Value': 'test.map'}, + '--timeout': {'Enabled': True, 'Value': '5m'}, '-vvvv': {'Enabled': True, 'Hidden': True}, } RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] @@ -90,7 +90,10 @@ class BlockPair(): if os.path.exists(self.map_path): self.load_map_data() self.resumed = True - # Fix status strings + fix_status_strings() + + def fix_status_strings(self): + """Format status strings via get_formatted_status().""" for pass_num in [1, 2, 3]: self.status[pass_num-1] = get_formatted_status( label='Pass {}'.format(pass_num), @@ -329,6 +332,15 @@ class RecoveryState(): min_percent = min(min_percent, bp.rescued) return min_percent + def retry_all_passes(self): + """Mark all passes as pending for all block-pairs.""" + self.current_pass = 0 + for bp in self.block_pairs: + bp.pass_done = [False, False, False] + bp.status = ['Pending', 'Pending', 'Pending'] + bp.fix_status_strings() + self.set_pass_num() + def self_checks(self): """Run self-checks for each BlockPair and update state values.""" self.total_size = 0 @@ -595,18 +607,6 @@ def is_writable_filesystem(dir_obj): return 'rw' in dir_obj.details.get('options', '') -def mark_all_passes_pending(source): - """Mark all devs and passes as pending in preparation for retry.""" - source['Current Pass'] = 'Pass 1' - source['Started Recovery'] = False - for p_num in ['Pass 1', 'Pass 2', 'Pass 3']: - source[p_num]['Status'] = 'Pending' - source[p_num]['Done'] = False - for child in source['Children']: - child[p_num]['Status'] = 'Pending' - child[p_num]['Done'] = False - - def menu_ddrescue(source_path, dest_path, run_mode): """ddrescue menu.""" source = None @@ -698,25 +698,22 @@ def menu_main(state): main_options[index]['Enabled'] = not main_options[index]['Enabled'] elif selection == 'S': # Set settings for pass - settings = [] + pass_settings = [] for k, v in state.settings.items(): if not v['Enabled']: continue - if k[-1:] == '=': - settings.append('{}{}'.format(k, v['Value'])) + if 'Value' in v: + pass_settings.append('{}={}'.format(k, v['Value'])) else: - settings.append(k) - if 'Value' in v: - settings.append(v['Value']) + pass_settings.append(k) for opt in main_options: if 'Auto' in opt['Base Name']: auto_run = opt['Enabled'] if 'Retry' in opt['Base Name'] and opt['Enabled']: - settings.extend(['--retrim', '--try-again']) - mark_all_passes_pending(source) - current_pass = 'Pass 1' + pass_settings.extend(['--retrim', '--try-again']) + state.retry_all_passes() if 'Reverse' in opt['Base Name'] and opt['Enabled']: - settings.append('--reverse') + pass_settings.append('--reverse') # Disable for next pass if 'Auto' not in opt['Base Name']: opt['Enabled'] = False @@ -725,7 +722,7 @@ def menu_main(state): state.started = False while auto_run or not state.started: state.started = True - run_ddrescue(state) + run_ddrescue(state, pass_settings) if state.finished or not auto_run: break if state.current_pass_done(): @@ -763,7 +760,7 @@ def menu_settings(state): settings = [] for k, v in sorted(state.settings.items()): if not v.get('Hidden', False): - settings.append({'Base Name': k.replace('=', ''), 'Flag': k}) + settings.append({'Base Name': k, 'Flag': k}) actions = [{'Name': 'Main Menu', 'Letter': 'M'}] # Show menu @@ -835,7 +832,7 @@ def read_map_file(map_path): return map_data -def run_ddrescue(state): +def run_ddrescue(state, pass_settings): """Run ddrescue pass.""" return_code = None @@ -867,17 +864,9 @@ def run_ddrescue(state): update_progress(state) # Set ddrescue cmd - cmd = ['ddrescue', bp.source_path, bp.dest_path, bp.map_path] - for k, v in state.settings.items(): - if not v['Enabled']: - continue - if 'Value' in v: - cmd.append('{}{}{}'.format( - k, - '' if k[-1:] == '=' else ' ', - v['Value'])) - else: - cmd.append(k) + cmd = [ + 'ddrescue', *pass_settings, + bp.source_path, bp.dest_path, bp.map_path] if state.mode == 'clone': cmd.append('--force') if state.current_pass == 0: From afaee53077706ddae7cb73e80dac23ea25d228ad Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 16 Aug 2018 00:43:09 -0700 Subject: [PATCH 086/138] Fixed current_pass updates/progression --- .bin/Scripts/functions/ddrescue.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 1858a30f..484a9f7a 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -90,7 +90,7 @@ class BlockPair(): if os.path.exists(self.map_path): self.load_map_data() self.resumed = True - fix_status_strings() + self.fix_status_strings() def fix_status_strings(self): """Format status strings via get_formatted_status().""" @@ -334,7 +334,7 @@ class RecoveryState(): def retry_all_passes(self): """Mark all passes as pending for all block-pairs.""" - self.current_pass = 0 + self.finished = False for bp in self.block_pairs: bp.pass_done = [False, False, False] bp.status = ['Pending', 'Pending', 'Pending'] @@ -367,11 +367,11 @@ class RecoveryState(): break if self.finished: self.current_pass_str = '- "Done"' - elif pass_num == 0: + elif self.current_pass == 0: self.current_pass_str = '1 "Initial Read"' - elif pass_num == 1: + elif self.current_pass == 1: self.current_pass_str = '2 "Trimming bad areas"' - elif pass_num == 2: + elif self.current_pass == 2: self.current_pass_str = '3 "Scraping bad areas"' def update_progress(self): @@ -724,6 +724,7 @@ def menu_main(state): state.started = True run_ddrescue(state, pass_settings) if state.finished or not auto_run: + state.set_pass_num() break if state.current_pass_done(): if (state.current_pass == 0 and From 2272d133f97924d9ae368c5b17980f9126b44340 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 16 Aug 2018 01:18:06 -0700 Subject: [PATCH 087/138] Fixed update_progress & update_sidepane --- .bin/Scripts/functions/ddrescue.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 484a9f7a..b2880896 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -424,7 +424,7 @@ def build_outer_panes(state): **COLORS)) # Side pane - update_progress(state) + update_sidepane(state) tmux_splitw( '-dhl', str(SIDE_PANE_WIDTH), 'watch', '--color', '--no-title', '--interval', '1', @@ -862,7 +862,7 @@ def run_ddrescue(state, pass_settings): if bp.pass_done[state.current_pass]: # Skip to next block-pair continue - update_progress(state) + update_sidepane(state) # Set ddrescue cmd cmd = [ @@ -887,17 +887,25 @@ def run_ddrescue(state, pass_settings): # ddrescue_proc = popen_program(['./__exit_ok', *cmd]) # ddrescue_proc = popen_program(cmd) while True: + bp.update_progress(state.current_pass) + update_sidepane(state) try: ddrescue_proc.wait(timeout=10) sleep(2) - update_progress(state) + bp.update_progress(state.current_pass) + update_sidepane(state) break except subprocess.TimeoutExpired: - update_progress(state) + # Catch to update bp/sidepane + pass except KeyboardInterrupt: # Catch user abort pass + # Update progress/sidepane again + bp.update_progress(state.current_pass) + update_sidepane(state) + # Was ddrescue aborted? return_code = ddrescue_proc.poll() if return_code is None or return_code is 130: @@ -1174,9 +1182,10 @@ def tmux_splitw(*args): return result.stdout.decode().strip() -def update_progress(state): +def update_sidepane(state): """Update progress file for side pane.""" output = [] + state.update_progress() if state.mode == 'clone': output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) else: @@ -1195,11 +1204,11 @@ def update_progress(state): output.append('{BLUE}Image File{CLEAR}'.format(**COLORS)) elif state.mode == 'image' and len(state.block_pairs) == 1: output.append('{BLUE}{source} {YELLOW}(Whole){CLEAR}'.format( - source=bp.source.path, + source=bp.source_path, **COLORS)) else: output.append('{BLUE}{source}{CLEAR}'.format( - source=bp.source.path, + source=bp.source_path, **COLORS)) output.extend(bp.status) output.append(' ') From 0c8de47893a806529f1e59951f7ab4053b794fdc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 16 Aug 2018 01:49:06 -0700 Subject: [PATCH 088/138] Reworked auto mode and pass status sections --- .bin/Scripts/functions/ddrescue.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index b2880896..13b16fbd 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -106,12 +106,14 @@ class BlockPair(): self.pass_done = [True, True, True] self.rescued = self.size self.status[pass_num] = get_formatted_status( - label='Pass {}'.format(pass_num), + label='Pass {}'.format(pass_num+1), data=100) # Mark future passes as Skipped pass_num += 1 while pass_num <= 2: - self.status[pass_num] = 'Skipped' + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num+1), + data='Skipped') pass_num += 1 else: self.pass_done[pass_num] = True @@ -723,20 +725,18 @@ def menu_main(state): while auto_run or not state.started: state.started = True run_ddrescue(state, pass_settings) - if state.finished or not auto_run: - state.set_pass_num() - break if state.current_pass_done(): if (state.current_pass == 0 and state.current_pass_min() < AUTO_PASS_1_THRESHOLD): auto_run = False - if (state.current_pass == 1 and + elif (state.current_pass == 1 and state.current_pass_min() < AUTO_PASS_2_THRESHOLD): auto_run = False else: auto_run = False - # Update current pass for next iteration state.set_pass_num() + if state.finished: + break elif selection == 'C': menu_settings(state) @@ -919,6 +919,7 @@ def run_ddrescue(state, pass_settings): else: # Mark pass finished bp.finish_pass(state.current_pass) + update_sidepane(state) # Done if str(return_code) != '0': From ee4cea3b0197da065b56c9ef5214d0898739c93c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 16 Aug 2018 01:58:22 -0700 Subject: [PATCH 089/138] Added systemd journal pane --- .bin/Scripts/functions/ddrescue.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 13b16fbd..b967672b 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -847,8 +847,9 @@ def run_ddrescue(state, pass_settings): # NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) result = run_program(['tput', 'lines']) height = int(result.stdout.decode().strip()) - height_smart = int(height * (12 / 33)) - height_ddrescue = height - height_smart + height_smart = int(height * (8 / 33)) + height_journal = int(height * (4 / 33)) + height_ddrescue = height - height_smart - height_journal # Show SMART status smart_pane = tmux_splitw( @@ -857,6 +858,12 @@ def run_ddrescue(state, pass_settings): 'watch', '--color', '--no-title', '--interval', '300', 'ddrescue-tui-smart-display', state.source_path) + # Show systemd journal output + journal_pane = tmux_splitw( + '-dvl', str(height_journal), + '-PF', '#D', + 'journalctl', '-f') + # Run pass for each block-pair for bp in state.block_pairs: if bp.pass_done[state.current_pass]: @@ -926,6 +933,7 @@ def run_ddrescue(state, pass_settings): # Pause on errors pause('Press Enter to return to main menu... ') run_program(['tmux', 'kill-pane', '-t', smart_pane]) + run_program(['tmux', 'kill-pane', '-t', journal_pane]) def select_parts(source_device): From 1d0378dd7bbd2590a7f4908b06dc98815cb82618 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 19 Aug 2018 19:23:24 -0700 Subject: [PATCH 090/138] Training wheels off --- .bin/Scripts/functions/ddrescue.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index b967672b..b890cc59 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -20,7 +20,7 @@ DDRESCUE_SETTINGS = { '--data-preview': {'Enabled': True, 'Hidden': True, 'Value': '5'}, '--idirect': {'Enabled': True}, '--odirect': {'Enabled': True}, - '--max-read-rate': {'Enabled': False, 'Value': '32MiB'}, + '--max-read-rate': {'Enabled': False, 'Value': '1MiB'}, '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, '--reopen-on-error': {'Enabled': True}, '--retry-passes': {'Enabled': True, 'Value': '0'}, @@ -890,9 +890,7 @@ def run_ddrescue(state, pass_settings): try: clear_screen() print_info('Current dev: {}'.format(bp.source_path)) - ddrescue_proc = popen_program(['./__choose_exit', *cmd]) - # ddrescue_proc = popen_program(['./__exit_ok', *cmd]) - # ddrescue_proc = popen_program(cmd) + ddrescue_proc = popen_program(cmd) while True: bp.update_progress(state.current_pass) update_sidepane(state) From 84d7207d90ef19b37721c9cbc996531c92be82c5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 31 Aug 2018 15:32:01 -0600 Subject: [PATCH 091/138] (Re)Add SMART values 196 and 199 * Value is only displayed, no additional aborts --- .bin/Scripts/functions/hw_diags.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 26547d07..dce6f140 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -20,8 +20,10 @@ ATTRIBUTES = { 184: {'Error': 1}, 187: {'Warning': 1}, 188: {'Warning': 1}, + 196: {'Warning': 1, 'Error': 10, 'Ignore': True}, 197: {'Error': 1}, 198: {'Error': 1}, + 199: {'Error': 1, 'Ignore': True}, 201: {'Warning': 1}, }, } From b2e287520c72b77883394204665274807962f11e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 21:32:31 -0600 Subject: [PATCH 092/138] Fix help flags --- .bin/Scripts/ddrescue-tui-menu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu index 988ec0fa..4bec6230 100755 --- a/.bin/Scripts/ddrescue-tui-menu +++ b/.bin/Scripts/ddrescue-tui-menu @@ -32,7 +32,7 @@ if __name__ == '__main__': pass # Show usage - if re.search(r'-*(h|help|\?)', str(sys.argv), re.IGNORECASE): + if re.search(r'-+(h|help)', str(sys.argv), re.IGNORECASE): show_usage(script_name) exit_script() From 7cfdddcfbdee402bc1caedfcb522c65127fa300d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 21:34:14 -0600 Subject: [PATCH 093/138] Fix using image file for clone source --- .bin/Scripts/functions/ddrescue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index b890cc59..2af666a0 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -41,6 +41,7 @@ class BaseObj(): """Base object used by DevObj, DirObj, and ImageObj.""" def __init__(self, path): self.type = 'base' + self.parent = None self.path = os.path.realpath(path) self.set_details() From dbf4559e14b71eeeddf70c2a8f5c3d0ec2c2ac00 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 21:39:32 -0600 Subject: [PATCH 094/138] Adjusted image/map filenames * Partition filenames should include '_pX_' instead of just '_X_' * Trailing whitespace should be removed --- .bin/Scripts/functions/ddrescue.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 2af666a0..31f22446 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -207,13 +207,17 @@ class DevObj(BaseObj): self.prefix = '{m_size}_{model}'.format( m_size=self.model_size, model=self.model) + self.prefix = self.prefix.strip() if self.parent: # Add child device details - self.prefix += '_{c_num}_{c_size}{sep}{c_label}'.format( - c_num=self.path.replace(self.parent, ''), + c_num = self.path.replace(self.parent, '') + self.prefix += '_{c_prefix}{c_num}_{c_size}{sep}{c_label}'.format( + c_prefix='p' if len(c_num) == 1 else '', + c_num=c_num, c_size=self.details.get('size', 'UNKNOWN'), sep='_' if self.label else '', c_label=self.label) + self.prefix = self.prefix.strip() class DirObj(BaseObj): From 86fc23aed80b4317375e573047f072c2161f7852 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 21:42:37 -0600 Subject: [PATCH 095/138] Fix pass number in status pane * It was using the internal 0-2 instead of the display 1-3 --- .bin/Scripts/functions/ddrescue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index 31f22446..e573e255 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -165,7 +165,7 @@ class BlockPair(): self.rescued_percent = map_data['rescued'] self.rescued = (self.rescued_percent * self.size) / 100 self.status[pass_num] = get_formatted_status( - label='Pass {}'.format(pass_num), + label='Pass {}'.format(pass_num+1), data=(self.rescued/self.size)*100) From 9dbfce94d4f31b245bca61f5599cd4342bea0630 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 21:44:28 -0600 Subject: [PATCH 096/138] Fix auto continue logic --- .bin/Scripts/functions/ddrescue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index e573e255..fcdcec17 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -71,6 +71,7 @@ class BlockPair(): self.pass_done = [False, False, False] self.resumed = False self.rescued = 0 + self.rescued_percent = 0 self.status = ['Pending', 'Pending', 'Pending'] self.size = source.size # Set dest paths @@ -336,7 +337,7 @@ class RecoveryState(): """Gets minimum pass rescued percentage, returns float.""" min_percent = 100 for bp in self.block_pairs: - min_percent = min(min_percent, bp.rescued) + min_percent = min(min_percent, bp.rescued_percent) return min_percent def retry_all_passes(self): From bd47f089961eed3dfb69bfad8797efd198ba97f3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 21:45:46 -0600 Subject: [PATCH 097/138] Removed (Whole) label when imaging * If only one partition was selected it would be incorrectly labeled "Whole" * Easier to remove the label than rework the data structures --- .bin/Scripts/functions/ddrescue.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index fcdcec17..d3ca8456 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -1215,10 +1215,6 @@ def update_sidepane(state): for bp in state.block_pairs: if state.source.is_image(): output.append('{BLUE}Image File{CLEAR}'.format(**COLORS)) - elif state.mode == 'image' and len(state.block_pairs) == 1: - output.append('{BLUE}{source} {YELLOW}(Whole){CLEAR}'.format( - source=bp.source_path, - **COLORS)) else: output.append('{BLUE}{source}{CLEAR}'.format( source=bp.source_path, From d35dba75397da4a6cc19e8d44f80e83d96517312 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 22:58:20 -0600 Subject: [PATCH 098/138] Prevent crash when retrying recovery * IMO ddrescue was exiting too quickly to load the map data so I'll assume 0b --- .bin/Scripts/functions/ddrescue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index d3ca8456..ae1194a9 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -163,7 +163,7 @@ class BlockPair(): """Update progress using map file.""" if os.path.exists(self.map_path): map_data = read_map_file(self.map_path) - self.rescued_percent = map_data['rescued'] + self.rescued_percent = map_data.get('rescued', 0) self.rescued = (self.rescued_percent * self.size) / 100 self.status[pass_num] = get_formatted_status( label='Pass {}'.format(pass_num+1), From 240c55f407bc1b18fad05e5d75f448f18b5b6992 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 3 Sep 2018 23:13:17 -0600 Subject: [PATCH 099/138] Remove more whitespace from image/map names --- .bin/Scripts/functions/ddrescue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index ae1194a9..cf518ea0 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -218,7 +218,7 @@ class DevObj(BaseObj): c_size=self.details.get('size', 'UNKNOWN'), sep='_' if self.label else '', c_label=self.label) - self.prefix = self.prefix.strip() + self.prefix = self.prefix.strip().replace(' ', '_') class DirObj(BaseObj): From a35be6878013488a20a8e5c8139b34f5baa6ad03 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Sep 2018 22:22:21 -0600 Subject: [PATCH 100/138] Add net devices to Conky before starting --- .../include/airootfs/etc/skel/.update_conky | 20 ++++++++++--------- .../include/airootfs/etc/skel/.xinitrc | 1 - .../include/airootfs/etc/skel/.zlogin | 6 ++++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.linux_items/include/airootfs/etc/skel/.update_conky b/.linux_items/include/airootfs/etc/skel/.update_conky index f67dc893..79801d8b 100755 --- a/.linux_items/include/airootfs/etc/skel/.update_conky +++ b/.linux_items/include/airootfs/etc/skel/.update_conky @@ -3,14 +3,16 @@ IF_LIST=($(ip l | egrep '^[0-9]+:\s+(eth|en|wl)' | sed -r 's/^[0-9]+:\s+(\w+):.*/\1/' | sort)) # Add interfaces to conkyrc -for i in "${IF_LIST[@]}"; do - if [[ "${i:0:1}" == "e" ]]; then - sed -i -r "s/#Network/Wired:\${alignr}\${addr $i}\n#Network/" ~/.conkyrc - else - sed -i -r "s/#Network/Wireless:\${alignr}\${addr $i}\n#Network/" ~/.conkyrc - fi -done +if fgrep '#Network' $HOME/.conkyrc; then + for i in "${IF_LIST[@]}"; do + if [[ "${i:0:1}" == "e" ]]; then + sed -i -r "s/#Network/Wired:\${alignr}\${addr $i}\n#Network/" $HOME/.conkyrc + else + sed -i -r "s/#Network/Wireless:\${alignr}\${addr $i}\n#Network/" $HOME/.conkyrc + fi + done -# Remove '#Network' line to prevent duplicating lines if this script is re-run -sed -i -r "s/#Network//" ~/.conkyrc + # Remove '#Network' line to prevent duplicating lines if this script is re-run + sed -i -r "s/#Network//" $HOME/.conkyrc +fi diff --git a/.linux_items/include/airootfs/etc/skel/.xinitrc b/.linux_items/include/airootfs/etc/skel/.xinitrc index 78bddfe7..2827fe58 100755 --- a/.linux_items/include/airootfs/etc/skel/.xinitrc +++ b/.linux_items/include/airootfs/etc/skel/.xinitrc @@ -16,5 +16,4 @@ connect-to-network & (sleep 5s && killall dunst) & $HOME/.urxvt_default_res & $HOME/.update_wallpaper & -$HOME/.update_conky & exec openbox-session diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/.linux_items/include/airootfs/etc/skel/.zlogin index 26463919..0898d1e2 100644 --- a/.linux_items/include/airootfs/etc/skel/.zlogin +++ b/.linux_items/include/airootfs/etc/skel/.zlogin @@ -1,9 +1,15 @@ setterm -blank 0 -powerdown 0 if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then + # Update settings if using i3 if fgrep -q "i3" /proc/cmdline; then sed -i -r 's/#(own_window_type override)/\1/' ~/.conkyrc sed -i -r 's/openbox-session/i3/' ~/.xinitrc fi + + # Update Conky + $HOME/.update_conky + + # Start X or HW-diags if ! fgrep -q "nox" /proc/cmdline; then startx >/dev/null else From 5ef7c9b16e61b5610754558d8ee70f911e19af60 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 4 Sep 2018 22:35:39 -0600 Subject: [PATCH 101/138] Updated functions/network.py --- .bin/Scripts/functions/network.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index 0d6beb3a..3f3480f5 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -3,6 +3,7 @@ ## Wizard Kit: Functions - Network import os +import shutil import sys # Init @@ -26,13 +27,8 @@ def connect_to_network(): if is_connected(): return - # LAN - if 'en' in net_ifs: - # Reload the tg3/broadcom driver (known fix for some Dell systems) - try_and_print(message='Reloading drivers...', function=reload_tg3) - # WiFi - if not is_connected() and 'wl' in net_ifs: + if 'wl' in net_ifs: cmd = [ 'nmcli', 'dev', 'wifi', 'connect', WIFI_SSID, @@ -42,6 +38,18 @@ def connect_to_network(): function = run_program, cmd = cmd) + # LAN + if not is_connected(): + # Reload the tg3/broadcom driver (known fix for some Dell systems) + try_and_print(message='Reloading drivers...', function=reload_tg3) + + # Rebuild conkyrc + shutil.copyfile( + '/etc/skel/.conkyrc', + '{HOME}/.conkyrc'.format(**global_vars['Env'])) + cmd = ['{HOME}/.update_conky'.format(**global_vars['Env'])] + run_program(cmd, check=False) + def is_connected(): """Check for a valid private IP.""" devs = psutil.net_if_addrs() From 793581ac22af7116d2cff03076b5db09f64b2eea Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 9 Sep 2018 19:41:46 -0600 Subject: [PATCH 102/138] Rewrote I/O benchmark sections * Displays graph during test and in summary * Reduce test area to speedup the benchmark * Addresses issues #48 & #49 --- .bin/Scripts/functions/hw_diags.py | 216 ++++++++++++++++++++++++++--- 1 file changed, 195 insertions(+), 21 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index dce6f140..f173b6f6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -27,6 +27,30 @@ ATTRIBUTES = { 201: {'Warning': 1}, }, } +IO_VARS = { + 'Block Size': 512*1024, + 'Chunk Size': 16*1024**2, + 'Minimum Dev Size': 8*1024**3, + 'Minimum Test Size': 10*1024**3, + 'Alt Test Size Factor': 0.01, + 'Progress Refresh Rate': 5, + 'Scale 16': [2**(0.6*x)+(16*x) for x in range(1,17)], + 'Scale 32': [2**(0.6*x/2)+(16*x/2) for x in range(1,33)], + 'Threshold Fail': 65*1024**2, + 'Threshold Warn': 135*1024**2, + 'Threshold Great': 750*1024**2, + 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), + 'Graph Horizontal Width': 40, + 'Graph Vertical': ( + '▏', '▎', '▍', '▌', + '▋', '▊', '▉', '█', + '█▏', '█▎', '█▍', '█▌', + '█▋', '█▊', '█▉', '██', + '██▏', '██▎', '██▍', '██▌', + '██▋', '██▊', '██▉', '███', + '███▏', '███▎', '███▍', '███▌', + '███▋', '███▊', '███▉', '████'), + } TESTS = { 'Prime95': { 'Enabled': False, @@ -49,6 +73,45 @@ TESTS = { }, } +def generate_horizontal_graph(rates): + """Generate two-line horizontal graph from rates, returns str.""" + line_top = '' + line_bottom = '' + for r in rates: + step = get_graph_step(r, scale=16) + + # Set color + r_color = COLORS['CLEAR'] + if r < IO_VARS['Threshold Fail']: + r_color = COLORS['RED'] + elif r < IO_VARS['Threshold Warn']: + r_color = COLORS['YELLOW'] + elif r > IO_VARS['Threshold Great']: + r_color = COLORS['GREEN'] + + # Build graph + if step < 8: + line_top += ' ' + line_bottom += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + else: + line_top += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + line_bottom += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) + line_top += COLORS['CLEAR'] + line_bottom += COLORS['CLEAR'] + return '{}\n{}'.format(line_top, line_bottom) + +def get_graph_step(rate, scale=16): + """Get graph step based on rate and scale, returns int.""" + m_rate = rate / (1024**2) + step = 0 + scale_name = 'Scale {}'.format(scale) + for x in range(scale-1, -1, -1): + # Iterate over scale backwards + if m_rate >= IO_VARS[scale_name][x]: + step = x + break + return step + def get_read_rate(s): """Get read rate in bytes/s from dd progress output.""" real_rate = None @@ -254,28 +317,113 @@ def run_iobenchmark(): TESTS['iobenchmark']['Status'][name] = 'Working' update_progress() print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - run_program('tmux split-window -dl 5 {} {} {}'.format( - 'hw-diags-iobenchmark', - '/dev/{}'.format(name), - progress_file).split()) - wait_for_process('dd') + + # Get dev size + cmd = 'sudo lsblk -bdno size /dev/{}'.format(name) + try: + result = run_program(cmd.split()) + dev_size = result.stdout.decode().strip() + dev_size = int(dev_size) + except: + # Failed to get dev size, requires manual testing instead + TESTS['iobenchmark']['Status'][name] = 'Unknown' + continue + if dev_size < IO_VARS['Minimum Dev Size']: + TESTS['iobenchmark']['Status'][name] = 'Unknown' + continue + + # Calculate dd values + ## test_size is the area to be read in bytes + ## If the dev is < 10Gb then it's the whole dev + ## Otherwise it's the smaller of 10Gb and 1% of the dev + ## + ## test_chunks is the number of groups of "Chunk Size" in test_size + ## This number is reduced to a multiple of the graph width in + ## order to allow for the data to be condensed cleanly + ## + ## skip_blocks is the number of "Block Size" groups not tested + ## skip_count is the number of blocks to skip per test_chunk + ## skip_extra is how often to add an additional skip block + ## This is needed to ensure an even testing across the dev + ## This is calculated by using the fractional amount left off + ## of the skip_count variable + test_size = min(IO_VARS['Minimum Test Size'], dev_size) + test_size = max( + test_size, dev_size*IO_VARS['Alt Test Size Factor']) + test_chunks = int(test_size // IO_VARS['Chunk Size']) + test_chunks -= test_chunks % IO_VARS['Graph Horizontal Width'] + test_size = test_chunks * IO_VARS['Chunk Size'] + skip_blocks = int((dev_size - test_size) // IO_VARS['Block Size']) + skip_count = int((skip_blocks / test_chunks) // 1) + skip_extra = 0 + try: + skip_extra = 1 + int(1 / ((skip_blocks / test_chunks) % 1)) + except ZeroDivisionError: + # skip_extra == 0 is fine + pass + + # Open dd progress pane after initializing file + with open(progress_file, 'w') as f: + f.write('') + sleep(1) + cmd = 'tmux split-window -dp 75 -PF #D tail -f {}'.format( + progress_file) + result = run_program(cmd.split()) + bottom_pane = result.stdout.decode().strip() + + # Run dd read tests + offset = 0 + read_rates = [] + for i in range(test_chunks): + i += 1 + s = skip_count + c = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) + if skip_extra and i % skip_extra == 0: + s += 1 + cmd = 'sudo dd bs={b} skip={s} count={c} if=/dev/{n} of={o}'.format( + b=IO_VARS['Block Size'], + s=offset+s, + c=c, + n=name, + o='/dev/null') + result = run_program(cmd.split()) + result_str = result.stderr.decode().replace('\n', '') + read_rates.append(get_read_rate(result_str)) + if i % IO_VARS['Progress Refresh Rate'] == 0: + # Update vertical graph + update_io_progress( + percent=i/test_chunks*100, + rate=read_rates[-1], + progress_file=progress_file) + # Update offset + offset += s + c print_standard('Done', timestamp=False) - # Check results - with open(progress_file, 'r') as f: - text = f.read() - io_stats = text.replace('\r', '\n').split('\n') - try: - io_stats = [get_read_rate(s) for s in io_stats] - io_stats = [float(s/1048576) for s in io_stats if s] - TESTS['iobenchmark']['Results'][name] = 'Read speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( - sum(io_stats) / len(io_stats), - min(io_stats), - max(io_stats)) - TESTS['iobenchmark']['Status'][name] = 'CS' - except: - # Requires manual testing - TESTS['iobenchmark']['Status'][name] = 'NS' + # Close bottom pane + run_program(['tmux', 'kill-pane', '-t', bottom_pane]) + + # Build report + h_graph_rates = [] + pos = 0 + width = int(test_chunks / IO_VARS['Graph Horizontal Width']) + for i in range(IO_VARS['Graph Horizontal Width']): + # Append average rate for WIDTH number of rates to new array + h_graph_rates.append(sum(read_rates[pos:pos+width])/width) + pos += width + report = generate_horizontal_graph(h_graph_rates) + report += '\nRead speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( + sum(read_rates)/len(read_rates)/1024**2, + min(read_rates)/1024**2, + max(read_rates)/1024**2) + TESTS['iobenchmark']['Results'][name] = report + + # Set CS/NS + if min(read_rates) <= IO_VARS['Threshold Fail']: + TESTS['iobenchmark']['Status'][name] = 'NS' + elif min(read_rates) <= IO_VARS['Threshold Warn']: + TESTS['iobenchmark']['Status'][name] = 'Unknown' + else: + TESTS['iobenchmark']['Status'][name] = 'CS' # Move temp file shutil.move(progress_file, '{}/iobenchmark-{}.log'.format( @@ -759,13 +907,38 @@ def show_results(): and io_status not in ['Denied', 'OVERRIDE', 'Skipped']): print_info('Benchmark:') result = TESTS['iobenchmark']['Results'].get(name, '') - print_standard(' {}'.format(result)) + for line in result.split('\n'): + print_standard(' {}'.format(line)) print_standard(' ') # Done pause('Press Enter to return to main menu... ') run_program('tmux kill-pane -a'.split()) +def update_io_progress(percent, rate, progress_file): + """Update I/O progress file.""" + bar_color = COLORS['CLEAR'] + rate_color = COLORS['CLEAR'] + step = get_graph_step(rate, scale=32) + if rate < IO_VARS['Threshold Fail']: + bar_color = COLORS['RED'] + rate_color = COLORS['YELLOW'] + elif rate < IO_VARS['Threshold Warn']: + bar_color = COLORS['YELLOW'] + rate_color = COLORS['YELLOW'] + elif rate > IO_VARS['Threshold Great']: + bar_color = COLORS['GREEN'] + rate_color = COLORS['GREEN'] + line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( + p=percent, + b_color=bar_color, + b=IO_VARS['Graph Vertical'][step], + r_color=rate_color, + r=rate/(1024**2), + c=COLORS['CLEAR']) + with open(progress_file, 'a') as f: + f.write(line) + def update_progress(): """Update progress file.""" if 'Progress Out' not in TESTS: @@ -821,3 +994,4 @@ def update_progress(): if __name__ == '__main__': print("This file is not meant to be called directly.") +# vim: sts=4 sw=4 ts=4 From 8fec6fc5b9a825ffa99dc649aa3c775818bb7b2d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 14:46:04 -0600 Subject: [PATCH 103/138] Save raw I/O read rates in log --- .bin/Scripts/functions/hw_diags.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index f173b6f6..8898c79c 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -135,9 +135,9 @@ def get_smart_details(dev): def get_status_color(s): """Get color based on status, returns str.""" color = COLORS['CLEAR'] - if s in ['Denied', 'NS', 'OVERRIDE', 'Unknown']: + if s in ['Denied', 'NS', 'OVERRIDE']: color = COLORS['RED'] - elif s in ['Aborted', 'Working', 'Skipped']: + elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: color = COLORS['YELLOW'] elif s in ['CS']: color = COLORS['GREEN'] @@ -335,7 +335,7 @@ def run_iobenchmark(): # Calculate dd values ## test_size is the area to be read in bytes ## If the dev is < 10Gb then it's the whole dev - ## Otherwise it's the smaller of 10Gb and 1% of the dev + ## Otherwise it's the larger of 10Gb or 1% of the dev ## ## test_chunks is the number of groups of "Chunk Size" in test_size ## This number is reduced to a multiple of the graph width in @@ -412,9 +412,9 @@ def run_iobenchmark(): pos += width report = generate_horizontal_graph(h_graph_rates) report += '\nRead speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( - sum(read_rates)/len(read_rates)/1024**2, - min(read_rates)/1024**2, - max(read_rates)/1024**2) + sum(read_rates)/len(read_rates)/(1024**2), + min(read_rates)/(1024**2), + max(read_rates)/(1024**2)) TESTS['iobenchmark']['Results'][name] = report # Set CS/NS @@ -425,9 +425,12 @@ def run_iobenchmark(): else: TESTS['iobenchmark']['Status'][name] = 'CS' - # Move temp file - shutil.move(progress_file, '{}/iobenchmark-{}.log'.format( - global_vars['LogDir'], name)) + # Save logs + dest_filename = '{}/iobenchmark-{}.log'.format(global_vars['LogDir'], name) + shutil.move(progress_file, dest_filename) + with open(dest_filename.replace('.', '-raw.'), 'a') as f: + for rate in read_rates: + f.write('{} MB/s\n'.format(rate/(1024**2))) update_progress() # Done From 9f12f2c8560f2449123a380a19caf264a733bdf9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 15:39:28 -0600 Subject: [PATCH 104/138] Added SMART 199/C7 warning/override to HW-Diags --- .bin/Scripts/functions/hw_diags.py | 43 +++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8898c79c..586efbd6 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -132,6 +132,15 @@ def get_smart_details(dev): # Let other sections deal with the missing data return {} +def get_smart_value(smart_data, smart_id): + """Get SMART value from table, returns int or None.""" + value = None + table = smart_data.get('ata_smart_attributes', {}).get('table', []) + for row in table: + if str(row.get('id', '?')) == str(smart_id): + value = row.get('raw', {}).get('value', None) + return value + def get_status_color(s): """Get color based on status, returns str.""" color = COLORS['CLEAR'] @@ -737,19 +746,29 @@ def scan_disks(full_paths=False, only_path=None): data['SMART Support'] = False # Ask for manual overrides if necessary - if not data['Quick Health OK'] and (TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']): + if TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: show_disk_details(data) - print_warning("WARNING: Health can't be confirmed for: {}".format( - '/dev/{}'.format(dev))) - dev_name = data['lsblk']['name'] - print_standard(' ') - if ask('Run tests on this device anyway?'): - TESTS['NVMe/SMART']['Status'][dev_name] = 'OVERRIDE' - else: - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - TESTS['badblocks']['Status'][dev_name] = 'Denied' - TESTS['iobenchmark']['Status'][dev_name] = 'Denied' - print_standard(' ') # In case there's more than one "OVERRIDE" disk + needs_override = False + if not data['Quick Health OK']: + needs_override = True + print_warning( + "WARNING: Health can't be confirmed for: /dev/{}".format(dev)) + if get_smart_value(data['smartctl'], '199'): + # SMART attribute present and it's value is non-zero + needs_override = True + print_warning( + 'WARNING: SMART 199/C7 error detected on /dev/{}'.format(dev)) + print_standard(' (Have you tried swapping the drive cable?)') + if needs_override: + dev_name = data['lsblk']['name'] + print_standard(' ') + if ask('Run tests on this device anyway?'): + TESTS['NVMe/SMART']['Status'][dev_name] = 'OVERRIDE' + else: + TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' + TESTS['badblocks']['Status'][dev_name] = 'Denied' + TESTS['iobenchmark']['Status'][dev_name] = 'Denied' + print_standard(' ') # In case there's more than one "OVERRIDE" disk TESTS['NVMe/SMART']['Devices'] = devs TESTS['badblocks']['Devices'] = devs From 83064d7c9038c30f710d5de949be62276399a80f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 15:54:36 -0600 Subject: [PATCH 105/138] Fix issue #46 --- .bin/Scripts/functions/hw_diags.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 586efbd6..91ebdef2 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -677,6 +677,8 @@ def run_tests(tests): def scan_disks(full_paths=False, only_path=None): """Scan for disks eligible for hardware testing.""" clear_screen() + print_standard(' ') + print_standard('Scanning disks...') # Get eligible disk list cmd = ['lsblk', '-J', '-O'] From 56e354f124b5aeeab8de0115e5156417dba5f8e5 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 16:15:15 -0600 Subject: [PATCH 106/138] Avoid crash described in issue #39 --- .bin/Scripts/functions/hw_diags.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 91ebdef2..49f2960d 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -271,18 +271,21 @@ def run_badblocks(): print_standard('Done', timestamp=False) # Check results - with open(progress_file, 'r') as f: - text = f.read() - TESTS['badblocks']['Results'][name] = text - r = re.search(r'Pass completed.*0/0/0 errors', text) - if r: - TESTS['badblocks']['Status'][name] = 'CS' - else: - TESTS['badblocks']['Status'][name] = 'NS' + if os.path.exists(progress_file): + with open(progress_file, 'r') as f: + text = f.read() + TESTS['badblocks']['Results'][name] = text + r = re.search(r'Pass completed.*0/0/0 errors', text) + if r: + TESTS['badblocks']['Status'][name] = 'CS' + else: + TESTS['badblocks']['Status'][name] = 'NS' - # Move temp file - shutil.move(progress_file, '{}/badblocks-{}.log'.format( - global_vars['LogDir'], name)) + # Move temp file + shutil.move(progress_file, '{}/badblocks-{}.log'.format( + global_vars['LogDir'], name)) + else: + TESTS['badblocks']['Status'][name] = 'NS' update_progress() # Done From 34925a72c0c2b25db4b112cd6a97eede9afbef9c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 17:44:23 -0600 Subject: [PATCH 107/138] Update hostname via reverse DNS lookup Should help differentiate systems --- .../include/airootfs/etc/skel/.update_hostname | 15 +++++++++++++++ .linux_items/include/airootfs/etc/skel/.xinitrc | 1 + 2 files changed, 16 insertions(+) create mode 100755 .linux_items/include/airootfs/etc/skel/.update_hostname diff --git a/.linux_items/include/airootfs/etc/skel/.update_hostname b/.linux_items/include/airootfs/etc/skel/.update_hostname new file mode 100755 index 00000000..da563ab6 --- /dev/null +++ b/.linux_items/include/airootfs/etc/skel/.update_hostname @@ -0,0 +1,15 @@ +#!/bin/bash + +IP="$(ip a show scope global \ + | grep inet \ + | head -1 \ + | sed -r 's#.*inet ([0-9]+.[0-9]+.[0-9]+.[0-9]+.)/.*#\1#')" +HOSTNAME="$(dig +noall +answer +short -x "$IP" \ + | sed 's/\.$//')" + +# Set hostname and renew DHCP lease +sudo hostnamectl set-hostname "${HOSTNAME}" +sudo dhclient -r +sleep 1 +sudo dhclient + diff --git a/.linux_items/include/airootfs/etc/skel/.xinitrc b/.linux_items/include/airootfs/etc/skel/.xinitrc index 2827fe58..573aed1d 100755 --- a/.linux_items/include/airootfs/etc/skel/.xinitrc +++ b/.linux_items/include/airootfs/etc/skel/.xinitrc @@ -12,6 +12,7 @@ conky -d nm-applet & cbatticon & volumeicon & +$HOME/.update_hostname connect-to-network & (sleep 5s && killall dunst) & $HOME/.urxvt_default_res & From deec1746e42637d5c1251452785bfe7c15f4dbba Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 18:20:16 -0600 Subject: [PATCH 108/138] Removed pydf to fix issue #44 --- .linux_items/include/airootfs/etc/skel/.aliases | 1 - .linux_items/packages/live_add | 1 - 2 files changed, 2 deletions(-) diff --git a/.linux_items/include/airootfs/etc/skel/.aliases b/.linux_items/include/airootfs/etc/skel/.aliases index 7566e42c..03d15315 100644 --- a/.linux_items/include/airootfs/etc/skel/.aliases +++ b/.linux_items/include/airootfs/etc/skel/.aliases @@ -4,7 +4,6 @@ alias 7z3='7z a -t7z -mx=3' alias 7z5='7z a -t7z -mx=5' alias 7z7='7z a -t7z -mx=7' alias 7z9='7z a -t7z -mx=9' -alias df='pydf' alias ddrescue='sudo ddrescue --ask --min-read-rate=64k -vvvv' alias diff='colordiff -ur' alias du='du -sch --apparent-size' diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index ade34eb5..a297ca05 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -61,7 +61,6 @@ otf-font-awesome-4 p7zip papirus-icon-theme progsreiserfs -pydf python python-psutil python-requests From d9a9ce0a86e76184ea6d9b19a1b8d2bef6f33e51 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 18:56:39 -0600 Subject: [PATCH 109/138] Allow hw-diags to reattach to the tmux session * The option to kill the current session and start a new was left in place --- .bin/Scripts/hw-diags | 9 ++++++--- .linux_items/packages/live_add | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/hw-diags b/.bin/Scripts/hw-diags index e8d838df..d3a1cb21 100755 --- a/.bin/Scripts/hw-diags +++ b/.bin/Scripts/hw-diags @@ -8,7 +8,7 @@ MENU="hw-diags-menu" function ask() { while :; do - read -p "$1 " -r answer + read -p "$1 [Y/N] " -r answer if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then return 0 elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then @@ -26,7 +26,10 @@ die () { if tmux list-session | grep -q "$SESSION_NAME"; then echo "WARNING: tmux session $SESSION_NAME already exists." echo "" - if ask "Kill current session?"; then + if ask "Connect to current session?"; then + # Do nothing, the command below will attach/connect + echo "" + elif ask "Kill current session and start new session?"; then tmux kill-session -t "$SESSION_NAME" || \ die "Failed to kill session: $SESSION_NAME" else @@ -39,5 +42,5 @@ if tmux list-session | grep -q "$SESSION_NAME"; then fi # Start session -tmux new-session -s "$SESSION_NAME" -n "$WINDOW_NAME" "$MENU" $* +tmux new-session -A -s "$SESSION_NAME" -n "$WINDOW_NAME" "$MENU" $* diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index a297ca05..ea784699 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -76,6 +76,7 @@ spice-vdagent terminus-font testdisk-wip thunar +tigervnc tint2 tk tmux From cb5ff20378637e09c1f52ff68d121eebfcbf5d8d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 20:54:49 -0600 Subject: [PATCH 110/138] Start a VNC server during startup --- .linux_items/include/airootfs/etc/skel/.xinitrc | 1 + .linux_items/include/airootfs/etc/ufw/user.rules | 3 +++ .linux_items/include/airootfs/etc/ufw/user6.rules | 3 +++ Build Linux | 4 ++++ 4 files changed, 11 insertions(+) diff --git a/.linux_items/include/airootfs/etc/skel/.xinitrc b/.linux_items/include/airootfs/etc/skel/.xinitrc index 573aed1d..fefe60f2 100755 --- a/.linux_items/include/airootfs/etc/skel/.xinitrc +++ b/.linux_items/include/airootfs/etc/skel/.xinitrc @@ -17,4 +17,5 @@ connect-to-network & (sleep 5s && killall dunst) & $HOME/.urxvt_default_res & $HOME/.update_wallpaper & +x0vncserver -display :0 -passwordfile $HOME/.vnc/passwd -AlwaysShared & exec openbox-session diff --git a/.linux_items/include/airootfs/etc/ufw/user.rules b/.linux_items/include/airootfs/etc/ufw/user.rules index aa30960c..3dbf5cf4 100644 --- a/.linux_items/include/airootfs/etc/ufw/user.rules +++ b/.linux_items/include/airootfs/etc/ufw/user.rules @@ -21,6 +21,9 @@ -A ufw-user-input -p tcp --dport 22 -j ACCEPT -A ufw-user-input -p udp --dport 22 -j ACCEPT +### tuple ### allow tcp 5900 0.0.0.0/0 any 0.0.0.0/0 VNC - in +-A ufw-user-input -p tcp --dport 5900 -j ACCEPT -m comment --comment 'dapp_VNC' + ### END RULES ### ### LOGGING ### diff --git a/.linux_items/include/airootfs/etc/ufw/user6.rules b/.linux_items/include/airootfs/etc/ufw/user6.rules index 47d96108..13084be4 100644 --- a/.linux_items/include/airootfs/etc/ufw/user6.rules +++ b/.linux_items/include/airootfs/etc/ufw/user6.rules @@ -21,6 +21,9 @@ -A ufw6-user-input -p tcp --dport 22 -j ACCEPT -A ufw6-user-input -p udp --dport 22 -j ACCEPT +### tuple ### allow tcp 5900 ::/0 any ::/0 VNC - in +-A ufw6-user-input -p tcp --dport 5900 -j ACCEPT -m comment --comment 'dapp_VNC' + ### END RULES ### ### LOGGING ### diff --git a/Build Linux b/Build Linux index 6ee0f169..2973174c 100755 --- a/Build Linux +++ b/Build Linux @@ -251,6 +251,10 @@ function update_live_env() { # udevil fix echo "mkdir /media" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + # VNC password + echo "mkdir '/home/$username/.vnc'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + echo "echo '$TECH_PASSWORD' | vncpasswd -f > '/home/$username/.vnc/passwd'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + # Wallpaper mkdir -p "$LIVE_DIR/airootfs/usr/share/wallpaper" cp "$ROOT_DIR/Images/Linux.png" "$LIVE_DIR/airootfs/usr/share/wallpaper/burned.in" From 821cb5cd462e90790f9ac2a728bbd303d2d2446e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Wed, 12 Sep 2018 21:02:24 -0600 Subject: [PATCH 111/138] Super+t URxvt windows auto-connect to tmux session --- .linux_items/include/airootfs/etc/skel/.config/i3/config | 2 +- .linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.linux_items/include/airootfs/etc/skel/.config/i3/config b/.linux_items/include/airootfs/etc/skel/.config/i3/config index de3c6fa8..cf09a2ca 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/i3/config +++ b/.linux_items/include/airootfs/etc/skel/.config/i3/config @@ -73,7 +73,7 @@ bindsym $mod+f exec "thunar ~" bindsym $mod+i exec "hardinfo" bindsym $mod+m exec "urxvt -title 'Mount All Volumes' -e mount-all-volumes gui" bindsym $mod+s exec "urxvt -title 'Hardware Diagnostics' -e hw-diags quick" -bindsym $mod+t exec "urxvt" +bindsym $mod+t exec "urxvt -e zsh -c 'tmux new-session -A -t general; zsh'" bindsym $mod+v exec "urxvt -title 'Hardware Sensors' -e watch -c -n1 -t hw-sensors" bindsym $mod+w exec "firefox" diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml index 43656613..90c7b0e0 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml +++ b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml @@ -329,7 +329,7 @@ - urxvt + urxvt -e zsh -c 'tmux new-session -A -t general; zsh' From 879927c37ca169b5ef45c55eaa63eb5807a45884 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 14 Sep 2018 17:53:35 -0600 Subject: [PATCH 112/138] Add CoreStorage support to mount-all-volumes * Checks for any CoreStorage partitions * If found scans partition with testdisk to find inner volume(s) * If found mapper devices are added with dmsetup * Then the device list is built in mount_volumes() --- .bin/Scripts/functions/data.py | 72 ++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 340dc529..63a19519 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -153,6 +153,67 @@ def cleanup_transfer(dest_path): except Exception: pass +def find_core_storage_volumes(): + """Try to create block devices for any Apple CoreStorage volumes.""" + corestorage_uuid = '53746f72-6167-11aa-aa11-00306543ecac' + dmsetup_cmd_file = '{TmpDir}/dmsetup_command' + + # Get CoreStorage devices + cmd = [ + 'lsblk', '--json', '--list', '--paths', + '--output', 'NAME,PARTTYPE'] + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + devs = json_data.get('blockdevices', []) + devs = [d for d in devs if d.get('parttype', '') == corestorage_uuid] + if devs: + print_standard(' ') + print_standard('Detected CoreStorage partition{}'.format( + '' if len(devs) == 1 else 's')) + print_standard(' Scanning for inner volume(s)....') + + # Search for inner volumes and setup dev mappers + for dev in devs: + dev_path = dev.get('name', '') + if not dev_path: + # Can't setup block device without the dev path + continue + dev_name = re.sub(r'.*/', '', dev_path) + log_path = '{LogDir}/testdisk_{dev_name}.log'.format( + dev_name=dev_name, **global_vars) + + # Run TestDisk + cmd = [ + 'sudo', 'testdisk', + '/logname', log_path, '/debug', '/log', + '/cmd', dev_path, 'partition_none,analyze'] + result = run_program(cmd, check=False) + if result.returncode: + # i.e. return code is non-zero + continue + if not os.path.exists(log_path): + # TestDisk failed to write log + continue + + # Check log for found volumes + cs_vols = {} + with open(log_path, 'r') as f: + for line in f.readlines(): + r = re.match( + r'^.*echo "([^"]+)" . dmsetup create test(\d)$', + line.strip(), + re.IGNORECASE) + if r: + cs_name = 'CoreStorage_{}_{}'.format(dev_name, r.group(2)) + cs_vols[cs_name] = r.group(1) + + # Create mapper device(s) + for name, dm_cmd in sorted(cs_vols.items()): + with open(dmsetup_cmd_file, 'w') as f: + f.write(dm_cmd) + cmd = ['sudo', 'dmsetup', 'create', name, dmsetup_cmd_file] + run_program(cmd, check=False) + def fix_path_sep(path_str): """Replace non-native and duplicate dir separators, returns str.""" return re.sub(r'(\\|/)+', lambda s: os.sep, path_str) @@ -190,14 +251,17 @@ def get_mounted_volumes(): def mount_volumes(all_devices=True, device_path=None, read_write=False): """Mount all detected filesystems.""" report = {} - - # Get list of block devices cmd = [ - 'lsblk', '-J', '-p', - '-o', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE'] + 'lsblk', '--json', '--paths', + '--output', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE'] if not all_devices and device_path: # Only mount volumes for specific device cmd.append(device_path) + else: + # Check for Apple CoreStorage volumes first + find_core_storage_volumes() + + # Get list of block devices result = run_program(cmd) json_data = json.loads(result.stdout.decode()) devs = json_data.get('blockdevices', []) From cdef33d7743ff13998bc11c872d7872f194fcf99 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 12:38:27 -0600 Subject: [PATCH 113/138] cbatticon removed from i3, notifications disabled --- .linux_items/include/airootfs/etc/skel/.config/openbox/autostart | 1 + .linux_items/include/airootfs/etc/skel/.xinitrc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart b/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart index 21fd52e5..6eaa6cc3 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart +++ b/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart @@ -17,3 +17,4 @@ #xfce-mcs-manager & tint2 & +cbatticon --hide-notification & diff --git a/.linux_items/include/airootfs/etc/skel/.xinitrc b/.linux_items/include/airootfs/etc/skel/.xinitrc index fefe60f2..ddc70863 100755 --- a/.linux_items/include/airootfs/etc/skel/.xinitrc +++ b/.linux_items/include/airootfs/etc/skel/.xinitrc @@ -10,7 +10,6 @@ compton --backend xrender --xrender-sync --xrender-sync-fence & sleep 1s conky -d nm-applet & -cbatticon & volumeicon & $HOME/.update_hostname connect-to-network & From 3a801ba72de1be2d56c4c8c383c874a0d89ee66f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 15:08:47 -0600 Subject: [PATCH 114/138] Updated startup settings * Added support for HiDPI devices * Only affects devices with a DPI >= 192 * Most screen objects are doubled in size * Updated .conkyrc * Vertical offset now set to 24 or 48 (to match URxvt) * Removed extra line after network adapters * Updated .xinitrc * Removed dunst logic since `cbatticon` no longer sends notifications * Update .Xresources for all programs before `xrdb -merge` * Update hostname after `connect-to-network` * URxvt windows are now based on a 16:9 ratio (instead of 4:3) * Removed * .update_wallpaper (integrated with .xinitrc) * .urxvt_default_res (integrated with .update_dpi_settings) * Fixes issue #45 --- .../include/airootfs/etc/skel/.Xresources | 3 +- .../include/airootfs/etc/skel/.conkyrc | 9 ++- .../airootfs/etc/skel/.update_dpi_settings | 68 +++++++++++++++++++ .../airootfs/etc/skel/.update_hostname | 1 + .../airootfs/etc/skel/.update_wallpaper | 21 ------ .../airootfs/etc/skel/.urxvt_default_res | 10 --- .../include/airootfs/etc/skel/.wallpaper | 1 + .../include/airootfs/etc/skel/.xinitrc | 7 +- 8 files changed, 79 insertions(+), 41 deletions(-) create mode 100755 .linux_items/include/airootfs/etc/skel/.update_dpi_settings delete mode 100755 .linux_items/include/airootfs/etc/skel/.update_wallpaper delete mode 100755 .linux_items/include/airootfs/etc/skel/.urxvt_default_res create mode 120000 .linux_items/include/airootfs/etc/skel/.wallpaper diff --git a/.linux_items/include/airootfs/etc/skel/.Xresources b/.linux_items/include/airootfs/etc/skel/.Xresources index 68054af5..d659b735 100755 --- a/.linux_items/include/airootfs/etc/skel/.Xresources +++ b/.linux_items/include/airootfs/etc/skel/.Xresources @@ -21,7 +21,7 @@ URxvt*externalBorder: 0 !URxvt.colorIT: #87af5f !URxvt.colorBD: #c5c8c6 !URxvt.colorUL: #87afd7 -URxvt.geometry: 92x16 +URxvt.geometry: 92x16 URxvt.internalBorder: 8 URxvt.shading: 10 URxvt.transparent: true @@ -53,6 +53,7 @@ URxvt.transparent: true *.color15: #ffffff ! fonts +!Xft.dpi: 192 Xft.autohint: 0 Xft.antialias: 1 Xft.hinting: true diff --git a/.linux_items/include/airootfs/etc/skel/.conkyrc b/.linux_items/include/airootfs/etc/skel/.conkyrc index adfd5cbb..af09dd8f 100644 --- a/.linux_items/include/airootfs/etc/skel/.conkyrc +++ b/.linux_items/include/airootfs/etc/skel/.conkyrc @@ -37,7 +37,7 @@ minimum_size 180 0 ### width | height maximum_width 180 gap_x 20 ### left | right -gap_y 45 ### up | down +gap_y 24 ### up | down alignment tr ####################### End Window Settings ### @@ -143,15 +143,14 @@ Uptime:${alignr}${uptime_short} CPU: ${if_match ${cpu cpu0}<10} ${cpu cpu0}\ ${else}${if_match ${cpu cpu0}<100} ${cpu cpu0}\ ${else}${cpu cpu0}${endif}${endif}% Used${alignr}${freq_g} GHz -${cpugraph cpu0 20,180 ${color} ${color}} +${cpugraph cpu0 ${gap_x},${width} ${color} ${color}} RAM: ${mem} Used${alignr}${memmax} -${memgraph 20,180 ${color} ${color}} +${memgraph ${gap_x},${width} ${color} ${color}} Disk I/O: -${diskiograph 20,180 ${color} ${color}} +${diskiograph ${gap_x},${width} ${color} ${color}} Down: ${downspeed}${goto 115}Up:${alignr}${upspeed} #Network - ${alignc}S H O R T C U T K E Y S ${hr} [Super] + d${alignr}HW Diagnostics diff --git a/.linux_items/include/airootfs/etc/skel/.update_dpi_settings b/.linux_items/include/airootfs/etc/skel/.update_dpi_settings new file mode 100755 index 00000000..2287508c --- /dev/null +++ b/.linux_items/include/airootfs/etc/skel/.update_dpi_settings @@ -0,0 +1,68 @@ +#!/bin/env bash +# +## Calculate DPI and adjust display settings if necesary + +REGEX_XRANDR='^.* ([0-9]+)x([0-9]+)\+[0-9]+\+[0-9]+.* ([0-9]+)mm x ([0-9]+)mm.*$' +REGEX_URXVT='(URxvt.geometry:\s+).*' + +# Get screen data +xrandr_str="$(xrandr | grep mm | head -1)" +width_px="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\1/")" +height_px="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\2/")" +width_mm="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\3/")" +height_mm="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\4/")" + +# Convert to in +width_in="$(echo "${width_mm} * 0.03937" | bc)" +height_in="$(echo "${height_mm} * 0.03937" | bc)" + +# Calculate diagonals +diag_px="$(echo "sqrt(${width_px}^2 + ${height_px}^2)" | bc)" +diag_in="$(echo "sqrt(${width_in}^2 + ${height_in}^2)" | bc)" + +# Calculate DPI +dpi="$(echo "${diag_px} / ${diag_in}" | bc 2>/dev/null || True)" +dpi="${dpi:-0}" + +# Calculate URxvt default window size +width_urxvt="$(echo "${width_px} * 112/1280" | bc)" +height_urxvt="$(echo "${height_px} * 33/720" | bc)" +offset_urxvt="24" + +# Update settings if necessary +if [[ "${dpi}" -ge 192 ]]; then + # Conky + sed -i 's/minimum_size 180 0/minimum_size 360 0/' "${HOME}/.conkyrc" + sed -i 's/maximum_width 180/maximum_width 360/' "${HOME}/.conkyrc" + sed -i 's/gap_x 20/gap_x 40/' "${HOME}/.conkyrc" + sed -i 's/gap_y 24/gap_y 48/' "${HOME}/.conkyrc" + + # Fonts + sed -i 's/!Xft.dpi: 192/Xft.dpi: 192/' "${HOME}/.Xresources" + + # GDK + export GDK_SCALE=2 + export GDK_DPI_SCALE=0.5 + + # i3 + sed -i -r 's/(height\s+) 26/\1 52/' "${HOME}/.config/i3/config" + + # Tint2 + sed -i 's/panel_size = 100% 30/panel_size = 100% 60/' \ + "${HOME}/.config/tint2/tint2rc" + sed -i 's/Inconsolata 10/Inconsolata 20/g' \ + "${HOME}/.config/tint2/tint2rc" + sed -i 's/Inconsolata 12/Inconsolata 24/g' \ + "${HOME}/.config/tint2/tint2rc" + sed -i 's/systray_icon_size = 24/systray_icon_size = 48/' \ + "${HOME}/.config/tint2/tint2rc" + + # URxvt + width_urxvt="$(echo "${width_urxvt} / 2" | bc)" + height_urxvt="$(echo "${height_urxvt} / 2" | bc)" + offset_urxvt="$(echo "${offset_urxvt} * 2" | bc)" +fi + +# Update URxvt (Always) +urxvt_geometry="${width_urxvt}x${height_urxvt}+${offset_urxvt}+${offset_urxvt}" +sed -i -r "s/${REGEX_URXVT}/\1${urxvt_geometry}/" "${HOME}/.Xresources" diff --git a/.linux_items/include/airootfs/etc/skel/.update_hostname b/.linux_items/include/airootfs/etc/skel/.update_hostname index da563ab6..3c1bd7c2 100755 --- a/.linux_items/include/airootfs/etc/skel/.update_hostname +++ b/.linux_items/include/airootfs/etc/skel/.update_hostname @@ -5,6 +5,7 @@ IP="$(ip a show scope global \ | head -1 \ | sed -r 's#.*inet ([0-9]+.[0-9]+.[0-9]+.[0-9]+.)/.*#\1#')" HOSTNAME="$(dig +noall +answer +short -x "$IP" \ + | head -1 \ | sed 's/\.$//')" # Set hostname and renew DHCP lease diff --git a/.linux_items/include/airootfs/etc/skel/.update_wallpaper b/.linux_items/include/airootfs/etc/skel/.update_wallpaper deleted file mode 100755 index 7bffa12b..00000000 --- a/.linux_items/include/airootfs/etc/skel/.update_wallpaper +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -BOOT_PATH="/run/archiso/bootmnt/arch/" -BURNED_IN="/usr/share/wallpaper/burned.in" -WALLPAPER="$HOME/.wallpaper.png" - -function link_wall() { - sudo rm "$WALLPAPER" - sudo ln -s "$1" "$WALLPAPER" -} - -# Check for wallpaper -## Checks BOOT_PATH and uses the BURNED_IN file if nothing is found -for f in "$BOOT_PATH"/{Arch,arch}.{jpg,png} "$BURNED_IN"; do - if [[ -f "$f" ]]; then - link_wall "$f" - break - fi -done - -feh --bg-fill "$WALLPAPER" diff --git a/.linux_items/include/airootfs/etc/skel/.urxvt_default_res b/.linux_items/include/airootfs/etc/skel/.urxvt_default_res deleted file mode 100755 index 1e146090..00000000 --- a/.linux_items/include/airootfs/etc/skel/.urxvt_default_res +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -XWIDTH="$(xrandr 2>/dev/null | grep '*' | sed -r 's/^\s+([0-9]+)x.*/\1/')" -XHEIGHT="$(xrandr 2>/dev/null | grep '*' | sed -r 's/^\s+[0-9]+x([0-9]+).*/\1/')" - -WIDTH="$(echo "${XWIDTH}*92/1024" | bc)" -HEIGHT="$(echo "${XHEIGHT}*32/768" | bc)" - -sed -i -r "s/(URxvt.geometry:\s+).*/\1${WIDTH}x${HEIGHT}+24+24/" ~/.Xresources -xrdb -merge ~/.Xresources diff --git a/.linux_items/include/airootfs/etc/skel/.wallpaper b/.linux_items/include/airootfs/etc/skel/.wallpaper new file mode 120000 index 00000000..f2a3d5e1 --- /dev/null +++ b/.linux_items/include/airootfs/etc/skel/.wallpaper @@ -0,0 +1 @@ +/usr/share/wallpaper/burned.in \ No newline at end of file diff --git a/.linux_items/include/airootfs/etc/skel/.xinitrc b/.linux_items/include/airootfs/etc/skel/.xinitrc index ddc70863..abb71f45 100755 --- a/.linux_items/include/airootfs/etc/skel/.xinitrc +++ b/.linux_items/include/airootfs/etc/skel/.xinitrc @@ -1,6 +1,7 @@ #!/bin/sh dbus-update-activation-environment --systemd DISPLAY +$HOME/.update_dpi_settings xrdb -merge $HOME/.Xresources xset s off xset -dpms @@ -11,10 +12,8 @@ sleep 1s conky -d nm-applet & volumeicon & -$HOME/.update_hostname connect-to-network & -(sleep 5s && killall dunst) & -$HOME/.urxvt_default_res & -$HOME/.update_wallpaper & +$HOME/.update_hostname & +feh --bg-fill "$HOME/.wallpaper" & x0vncserver -display :0 -passwordfile $HOME/.vnc/passwd -AlwaysShared & exec openbox-session From d31991a67f16f9b371d239e1cfae2ac9941bce9d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 15:24:59 -0600 Subject: [PATCH 115/138] Always load broadcom before tg3 * Hopefully won't cause any problems. --- .bin/Scripts/functions/network.py | 19 ------------------- .../airootfs/etc/modprobe.d/broadcom.conf | 1 + 2 files changed, 1 insertion(+), 19 deletions(-) create mode 100644 .linux_items/include/airootfs/etc/modprobe.d/broadcom.conf diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index 3f3480f5..d040e343 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -37,18 +37,6 @@ def connect_to_network(): message = 'Connecting to {}...'.format(WIFI_SSID), function = run_program, cmd = cmd) - - # LAN - if not is_connected(): - # Reload the tg3/broadcom driver (known fix for some Dell systems) - try_and_print(message='Reloading drivers...', function=reload_tg3) - - # Rebuild conkyrc - shutil.copyfile( - '/etc/skel/.conkyrc', - '{HOME}/.conkyrc'.format(**global_vars['Env'])) - cmd = ['{HOME}/.update_conky'.format(**global_vars['Env'])] - run_program(cmd, check=False) def is_connected(): """Check for a valid private IP.""" @@ -79,13 +67,6 @@ def speedtest(): output = [(a, float(b), c) for a, b, c in output] return ['{:10}{:6.2f} {}'.format(*line) for line in output] -def reload_tg3(): - """Reload tg3 module as a workaround for some Dell systems.""" - run_program(['sudo', 'modprobe', '-r', 'tg3']) - run_program(['sudo', 'modprobe', 'broadcom']) - run_program(['sudo', 'modprobe', 'tg3']) - sleep(5) - if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.linux_items/include/airootfs/etc/modprobe.d/broadcom.conf b/.linux_items/include/airootfs/etc/modprobe.d/broadcom.conf new file mode 100644 index 00000000..1c85cd7b --- /dev/null +++ b/.linux_items/include/airootfs/etc/modprobe.d/broadcom.conf @@ -0,0 +1 @@ +softdep tg3 pre: broadcom From 1ee4a611b9c76b5d5adeed8f3d5691b43c29f45a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 15:42:40 -0600 Subject: [PATCH 116/138] Hide UFD-only boot options from Linux ISOs * Fixes issue #41 --- .bin/Scripts/build-ufd | 2 ++ .linux_items/include/EFI/boot/refind.conf | 10 +++++----- .linux_items/include/syslinux/wk_pxe.cfg | 2 +- .linux_items/include/syslinux/wk_pxe_extras.cfg | 2 +- .linux_items/include/syslinux/wk_sys.cfg | 2 +- .linux_items/include/syslinux/wk_sys_extras.cfg | 2 +- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index faae7867..c606892b 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -557,7 +557,9 @@ mount "${WINPE_ISO}" /mnt/WinPE -r >> "${LOG_FILE}" 2>&1 echo "Copying Linux files..." rsync ${RSYNC_ARGS} /mnt/Linux/* /mnt/Dest/ >> "${LOG_FILE}" 2>&1 sed -i "s/${ISO_LABEL}/${UFD_LABEL}/" /mnt/Dest/EFI/boot/refind.conf +sed -i "s/#UFD#//" /mnt/Dest/EFI/boot/refind.conf sed -i "s/${ISO_LABEL}/${UFD_LABEL}/" /mnt/Dest/arch/boot/syslinux/*cfg +sed -i "s/#UFD#//" /mnt/Dest/arch/boot/syslinux/*cfg echo "Copying WinPE files..." rsync ${RSYNC_ARGS} /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/Dest/ >> "${LOG_FILE}" 2>&1 diff --git a/.linux_items/include/EFI/boot/refind.conf b/.linux_items/include/EFI/boot/refind.conf index 075f6e46..de83e78c 100644 --- a/.linux_items/include/EFI/boot/refind.conf +++ b/.linux_items/include/EFI/boot/refind.conf @@ -32,8 +32,8 @@ menuentry "Linux" { add_options "nox" } } -menuentry "WindowsPE" { - ostype windows - icon /EFI/boot/icons/wk_win.png - loader /EFI/microsoft/bootx64.efi -} +#UFD#menuentry "WindowsPE" { +#UFD# ostype windows +#UFD# icon /EFI/boot/icons/wk_win.png +#UFD# loader /EFI/microsoft/bootx64.efi +#UFD#} diff --git a/.linux_items/include/syslinux/wk_pxe.cfg b/.linux_items/include/syslinux/wk_pxe.cfg index d5efabb4..92af00e3 100644 --- a/.linux_items/include/syslinux/wk_pxe.cfg +++ b/.linux_items/include/syslinux/wk_pxe.cfg @@ -2,7 +2,7 @@ INCLUDE boot/syslinux/wk_head.cfg MENU BACKGROUND pxelinux.png INCLUDE boot/syslinux/wk_pxe_linux.cfg -INCLUDE boot/syslinux/wk_pxe_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_pxe_winpe.cfg INCLUDE boot/syslinux/wk_pxe_extras_entry.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/include/syslinux/wk_pxe_extras.cfg b/.linux_items/include/syslinux/wk_pxe_extras.cfg index 04cd2ce1..d677b4b0 100644 --- a/.linux_items/include/syslinux/wk_pxe_extras.cfg +++ b/.linux_items/include/syslinux/wk_pxe_extras.cfg @@ -3,7 +3,7 @@ MENU BACKGROUND pxelinux.png INCLUDE boot/syslinux/wk_pxe_linux.cfg INCLUDE boot/syslinux/wk_pxe_linux_extras.cfg -INCLUDE boot/syslinux/wk_pxe_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_pxe_winpe.cfg INCLUDE boot/syslinux/wk_hdt.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/include/syslinux/wk_sys.cfg b/.linux_items/include/syslinux/wk_sys.cfg index beefb77d..0d375cf3 100644 --- a/.linux_items/include/syslinux/wk_sys.cfg +++ b/.linux_items/include/syslinux/wk_sys.cfg @@ -1,7 +1,7 @@ INCLUDE boot/syslinux/wk_head.cfg INCLUDE boot/syslinux/wk_sys_linux.cfg -INCLUDE boot/syslinux/wk_sys_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_sys_winpe.cfg INCLUDE boot/syslinux/wk_sys_extras_entry.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/include/syslinux/wk_sys_extras.cfg b/.linux_items/include/syslinux/wk_sys_extras.cfg index 422bd053..3e5af215 100644 --- a/.linux_items/include/syslinux/wk_sys_extras.cfg +++ b/.linux_items/include/syslinux/wk_sys_extras.cfg @@ -2,7 +2,7 @@ INCLUDE boot/syslinux/wk_head.cfg INCLUDE boot/syslinux/wk_sys_linux.cfg INCLUDE boot/syslinux/wk_sys_linux_extras.cfg -INCLUDE boot/syslinux/wk_sys_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_sys_winpe.cfg INCLUDE boot/syslinux/wk_hdt.cfg INCLUDE boot/syslinux/wk_tail.cfg From 9a093ace9ce763b112cd6517836dfc7fbab9f737 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 15:47:47 -0600 Subject: [PATCH 117/138] Moved 'Scanning disks...' message in hw_diags.py * (Re)fixes issue #46 --- .bin/Scripts/functions/hw_diags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 49f2960d..585bff0b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -647,6 +647,8 @@ def run_tests(tests): # Initialize if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: + print_standard(' ') + print_standard('Scanning disks...') scan_disks() update_progress() @@ -680,8 +682,6 @@ def run_tests(tests): def scan_disks(full_paths=False, only_path=None): """Scan for disks eligible for hardware testing.""" clear_screen() - print_standard(' ') - print_standard('Scanning disks...') # Get eligible disk list cmd = ['lsblk', '-J', '-O'] From da92cee33822f69bbd1e8e238e6eabfb2369d830 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 16:00:24 -0600 Subject: [PATCH 118/138] Fix issue #51 * The curly braces were being interpreted incorrectly by print_standard() --- .bin/Scripts/functions/browsers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index b969a59c..c6d39db9 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -285,6 +285,9 @@ def get_ie_homepages(): homepages.append(main_page) if len(extra_pages) > 0: homepages.extend(extra_pages) + + # Remove all curly braces + homepages = [h.replace('{', '').replace('}', '') for h in homepages] return homepages def get_mozilla_homepages(prefs_path): From 1e21c04a3eff8fbb9d108f98e3c0a8ce37f9b96e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 17:46:05 -0600 Subject: [PATCH 119/138] Address shutdown slowdowns * Unmount filesystem(s) and flush write cache before poweroff/reboot * Also adjusted timouts to maybe fix issue #52 --- .bin/Scripts/wk-power-command | 21 +++++++++++++++++++ .../include/airootfs/etc/oblogout.conf | 6 +++--- Build Linux | 4 ++++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100755 .bin/Scripts/wk-power-command diff --git a/.bin/Scripts/wk-power-command b/.bin/Scripts/wk-power-command new file mode 100755 index 00000000..92af02fb --- /dev/null +++ b/.bin/Scripts/wk-power-command @@ -0,0 +1,21 @@ +#!/bin/bash +# +## Wizard Kit: Wrapper for logout, reboot, & poweroff + +# Unmount filesystems +find /media -maxdepth 1 -mindepth 1 -type d \ + -exec udevil umount "{}" \; + +# Flush write cache +sudo sync + +# Perform requested action +case "${1:-x}" in + poweroff) + sudo systemctl poweroff;; + reboot) + sudo systemctl reboot;; + *) + openbox --exit;; +esac +exit 0 diff --git a/.linux_items/include/airootfs/etc/oblogout.conf b/.linux_items/include/airootfs/etc/oblogout.conf index 4595c766..ea5606ef 100644 --- a/.linux_items/include/airootfs/etc/oblogout.conf +++ b/.linux_items/include/airootfs/etc/oblogout.conf @@ -15,6 +15,6 @@ restart = R logout = L [commands] -shutdown = systemctl poweroff -restart = systemctl reboot -logout = openbox --exit +shutdown = /usr/local/bin/wk-power-command poweroff +restart = /usr/local/bin/wk-power-command reboot +logout = /usr/local/bin/wk-power-command logout diff --git a/Build Linux b/Build Linux index 2973174c..94556c2a 100755 --- a/Build Linux +++ b/Build Linux @@ -219,6 +219,10 @@ function update_live_env() { # Services sed -i -r 's/^(.*pacman-init.*)$/#NOPE#\1/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" sed -i -r 's/^(.*choose-mirror.*)$/#NOPE#\1/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + + # Shutdown stall fix + echo "sed -i -r 's/^.*(DefaultTimeoutStartSec)=.*$/\1=15s/' /etc/systemd/system.conf" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + echo "sed -i -r 's/^.*(DefaultTimeoutStopSec)=.*$/\1=15s/' /etc/systemd/system.conf" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" # SSH mkdir -p "$SKEL_DIR/.ssh" From 35fd50771cc27dd5b92b79171e50ce86b9da1ed3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 17:50:54 -0600 Subject: [PATCH 120/138] Update hw_diags.py systemctl command syntax * Now it matches the wk-power-command style --- .bin/Scripts/functions/hw_diags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 585bff0b..61530e5b 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -224,9 +224,9 @@ def menu_diags(*args): 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split(), check=False, pipe=False) elif selection == 'R': - run_program(['reboot']) + run_program(['systemctl', 'reboot']) elif selection == 'S': - run_program(['poweroff']) + run_program(['systemctl', 'poweroff']) elif selection == 'Q': break From 91f7628081c818849f18ce5463ce6ce261995a3a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 18:17:58 -0600 Subject: [PATCH 121/138] Updated sources * Dropping 2008 --- .bin/Scripts/settings/sources.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 3560b092..5ba2323e 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -48,10 +48,6 @@ SOURCE_URLS = { 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.33.1/aria2-1.33.1-win-32bit-build1.zip', } VCREDIST_SOURCES = { - '2008sp1': { - '32': 'https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x86.exe', - '64': 'https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe', - }, '2010sp1': { '32': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe', '64': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x64.exe', @@ -65,8 +61,8 @@ VCREDIST_SOURCES = { '64': 'https://download.microsoft.com/download/0/5/6/056dcda9-d667-4e27-8001-8a0c6971d6b1/vcredist_x64.exe', }, '2017': { - '32': 'https://download.visualstudio.microsoft.com/download/pr/100349138/88b50ce70017bf10f2d56d60fcba6ab1/VC_redist.x86.exe', - '64': 'https://download.visualstudio.microsoft.com/download/pr/100349091/2cd2dba5748dc95950a5c42c2d2d78e4/VC_redist.x64.exe', + '32': 'https://aka.ms/vs/15/release/vc_redist.x86.exe', + '64': 'https://aka.ms/vs/15/release/vc_redist.x64.exe', }, } NINITE_SOURCES = { From a213ba5d32534fbb1bc220826b5748badda785dc Mon Sep 17 00:00:00 2001 From: Alan Mason <2xShirt@gmail.com> Date: Sat, 15 Sep 2018 21:20:52 -0600 Subject: [PATCH 122/138] Bugfix for mount-all-volumes --- .bin/Scripts/functions/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 63a19519..c8eec384 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -156,7 +156,7 @@ def cleanup_transfer(dest_path): def find_core_storage_volumes(): """Try to create block devices for any Apple CoreStorage volumes.""" corestorage_uuid = '53746f72-6167-11aa-aa11-00306543ecac' - dmsetup_cmd_file = '{TmpDir}/dmsetup_command' + dmsetup_cmd_file = '{TmpDir}/dmsetup_command'.format(**global_vars) # Get CoreStorage devices cmd = [ From 6cdf2a421168bec5d7b9bd72d22d344f931ec658 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 15:41:14 -0600 Subject: [PATCH 123/138] Added authorized_keys file for SSH connections --- .linux_items/authorized_keys | 1 + Build Linux | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .linux_items/authorized_keys diff --git a/.linux_items/authorized_keys b/.linux_items/authorized_keys new file mode 100644 index 00000000..be79388e --- /dev/null +++ b/.linux_items/authorized_keys @@ -0,0 +1 @@ +#Put SSH keys here diff --git a/Build Linux b/Build Linux index 94556c2a..512aa982 100755 --- a/Build Linux +++ b/Build Linux @@ -229,7 +229,8 @@ function update_live_env() { ssh-keygen -b 4096 -C "$username@$hostname" -N "" -f "$SKEL_DIR/.ssh/id_rsa" echo 'rm /root/.ssh/id*' >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" echo 'rm /root/.zlogin' >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" - sed -i -r 's/^(.*PermitRootLogin.*)$/#NOPE#\1/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + sed -i -r 's/^(.*PermitRootLogin.*)$/PermitRootLogin no/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + cp "$ROOT_DIR/.linux_items/authorized_keys" "$SKEL_DIR/.ssh/authorized_keys" # Root user echo "echo 'root:$ROOT_PASSWORD' | chpasswd" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" From a71f7648c29fd659d6c6217f74652119a088132e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 18:45:10 -0600 Subject: [PATCH 124/138] Bugfixes for Build Linux script --- Build Linux | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Build Linux b/Build Linux index 512aa982..af01c928 100755 --- a/Build Linux +++ b/Build Linux @@ -190,9 +190,9 @@ function update_live_env() { # Live packages while read -r p; do - sed -i "/$p/d" "$LIVE_DIR/packages.both" + sed -i "/$p/d" "$LIVE_DIR/packages.x86_64" done < "$ROOT_DIR/.linux_items/packages/live_remove" - cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.both" + cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.x86_64" echo "[custom]" >> "$LIVE_DIR/pacman.conf" echo "SigLevel = Optional TrustAll" >> "$LIVE_DIR/pacman.conf" echo "Server = file://$REPO_DIR" >> "$LIVE_DIR/pacman.conf" @@ -229,7 +229,7 @@ function update_live_env() { ssh-keygen -b 4096 -C "$username@$hostname" -N "" -f "$SKEL_DIR/.ssh/id_rsa" echo 'rm /root/.ssh/id*' >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" echo 'rm /root/.zlogin' >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" - sed -i -r 's/^(.*PermitRootLogin.*)$/PermitRootLogin no/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + sed -r '/.*PermitRootLogin.*/d' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" cp "$ROOT_DIR/.linux_items/authorized_keys" "$SKEL_DIR/.ssh/authorized_keys" # Root user @@ -326,9 +326,11 @@ function build_iso() { # Removing cached (and possibly outdated) custom repo packages for package in $(cat "$ROOT_DIR/.linux_items/packages/aur"); do - if [[ -f /var/cache/pacman/pkg/${package}* ]]; then - rm /var/cache/pacman/pkg/${package}* - fi + for p in /var/cache/pacman/pkg/*${package}*; do + if [[ -f "${p}" ]]; then + rm "${p}" + fi + done done # Build ISO From b79cd5d65afac4fc30d79346c2199c414d61307a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 18:46:02 -0600 Subject: [PATCH 125/138] Updated tool versions * Adjusted bundles * Dropped Office 2013 * Moved to Python 3.7 * Replaced TreeSizeFree with WizTree --- .bin/Scripts/build_kit.ps1 | 12 ++++----- .bin/Scripts/build_pe.ps1 | 19 +++++++------- .bin/Scripts/settings/sources.py | 44 +++++++++++++++----------------- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/.bin/Scripts/build_kit.ps1 b/.bin/Scripts/build_kit.ps1 index dafaa2c8..bba52a66 100644 --- a/.bin/Scripts/build_kit.ps1 +++ b/.bin/Scripts/build_kit.ps1 @@ -82,25 +82,25 @@ if ($MyInvocation.InvocationName -ne ".") { DownloadFile -Path $Path -Name "7z-extra.7z" -Url "https://www.7-zip.org/a/7z1805-extra.7z" # ConEmu - $Url = "https://github.com/Maximus5/ConEmu/releases/download/v18.05.06/ConEmuPack.180506.7z" + $Url = "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z" DownloadFile -Path $Path -Name "ConEmuPack.7z" -Url $Url # Notepad++ - $Url = "https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.7z" + $Url = "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z" DownloadFile -Path $Path -Name "npp.7z" -Url $Url # Python - $Url = "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-win32.zip" + $Url = "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip" DownloadFile -Path $Path -Name "python32.zip" -Url $Url - $Url = "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-amd64.zip" + $Url = "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip" DownloadFile -Path $Path -Name "python64.zip" -Url $Url # Python: psutil $DownloadPage = "https://pypi.org/project/psutil/" - $RegEx = "href=.*-cp36-cp36m-win32.whl" + $RegEx = "href=.*-cp37-cp37m-win32.whl" $Url = FindDynamicUrl $DownloadPage $RegEx DownloadFile -Path $Path -Name "psutil32.whl" -Url $Url - $RegEx = "href=.*-cp36-cp36m-win_amd64.whl" + $RegEx = "href=.*-cp37-cp37m-win_amd64.whl" $Url = FindDynamicUrl $DownloadPage $RegEx DownloadFile -Path $Path -Name "psutil64.whl" -Url $Url diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index 0e19050a..3fb1bcc9 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -136,20 +136,19 @@ if ($MyInvocation.InvocationName -ne ".") { @("bluescreenview32.zip", "http://www.nirsoft.net/utils/bluescreenview.zip"), @("bluescreenview64.zip", "http://www.nirsoft.net/utils/bluescreenview-x64.zip"), # ConEmu - @("ConEmuPack.7z", "https://github.com/Maximus5/ConEmu/releases/download/v18.05.06/ConEmuPack.180506.7z"), + @("ConEmuPack.7z", "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z"), # Fast Copy - @("fastcopy32.zip", "http://ftp.vector.co.jp/69/93/2323/FastCopy341.zip"), - @("fastcopy64.zip", "http://ftp.vector.co.jp/69/93/2323/FastCopy341_x64.zip"), + @("fastcopy.zip", "http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip"), # HWiNFO - @("hwinfo.zip", "http://app.oldfoss.com:81/download/HWiNFO/hwi_582.zip"), + @("hwinfo.zip", "http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip"), # Killer Network Drivers @( "killerinf.zip", ("http://www.killernetworking.com"+(FindDynamicUrl "http://www.killernetworking.com/driver-downloads/item/killer-drivers-inf" "Download Killer-Ethernet").replace('&', '&')) ), # Notepad++ - @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.7z"), - @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.x64.7z"), + @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z"), + @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.x64.7z"), # NT Password Editor @("ntpwed.zip", "http://cdslow.org.ru/files/ntpwedit/ntpwed07.zip"), # Prime95 @@ -159,16 +158,16 @@ if ($MyInvocation.InvocationName -ne ".") { @("produkey32.zip", "http://www.nirsoft.net/utils/produkey.zip"), @("produkey64.zip", "http://www.nirsoft.net/utils/produkey-x64.zip"), # Python - @("python32.zip", "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-win32.zip"), - @("python64.zip", "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-amd64.zip"), + @("python32.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip"), + @("python64.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip"), # Python: psutil @( "psutil64.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win_amd64.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win_amd64.whl") ), @( "psutil32.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win32.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win32.whl") ), # Q-Dir @("qdir32.zip", "https://www.softwareok.com/Download/Q-Dir_Portable.zip"), diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 5ba2323e..903c46fb 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -1,9 +1,10 @@ # Wizard Kit: Settings - Sources SOURCE_URLS = { + 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1801120058/AcroRdrDC1801120058_en_US.exe', + 'AdwCleaner': 'https://downloads.malwarebytes.com/file/adwcleaner', 'AIDA64': 'http://download.aida64.com/aida64engineer597.zip', - 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1801120040/AcroRdrDC1801120040_en_US.exe', - 'AdwCleaner': 'https://toolslib.net/downloads/finish/1-adwcleaner/', + 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0-win-32bit-build1.zip', 'Autoruns': 'https://download.sysinternals.com/files/Autoruns.zip', 'BleachBit': 'https://download.bleachbit.org/BleachBit-2.0-portable.zip', 'BlueScreenView32': 'http://www.nirsoft.net/utils/bluescreenview.zip', @@ -14,38 +15,35 @@ SOURCE_URLS = { 'ERUNT': 'http://www.aumha.org/downloads/erunt.zip', 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.895.x86.zip', 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.895.x64.zip', - 'FastCopy32': 'http://ftp.vector.co.jp/69/93/2323/FastCopy341.zip', - 'FastCopy64': 'http://ftp.vector.co.jp/69/93/2323/FastCopy341_x64.zip', - 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/956394/ublock_origin-1.16.6-an+fx.xpi', - 'HWiNFO': 'http://app.oldfoss.com:81/download/HWiNFO/hwi_582.zip', + 'FastCopy': 'http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip', + 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1056733/ublock_origin-1.16.20-an+fx.xpi', 'HitmanPro32': 'https://dl.surfright.nl/HitmanPro.exe', 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', - 'IOBit_Uninstaller': 'https://portableapps.com/redirect/?a=IObitUninstallerPortable&t=http%3A%2F%2Fdownloads.portableapps.com%2Fportableapps%2Fiobituninstallerportable%2FIObitUninstallerPortable_7.3.0.13.paf.exe', + 'HWiNFO': 'http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip', 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/27656/eng/Intel%20SSD%20Toolbox%20-%20v3.5.2.exe', + 'IOBit_Uninstaller': 'https://portableapps.duckduckgo.com/IObitUninstallerPortable_7.5.0.7.paf.exe', 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', - 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.7z', - 'Office Deployment Tool 2013': 'https://download.microsoft.com/download/6/2/3/6230F7A2-D8A9-478B-AC5C-57091B632FCF/officedeploymenttool_x86_4827-1000.exe', - 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_9326.3600.exe', + 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z', + 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_10810.33603.exe', 'ProduKey32': 'http://www.nirsoft.net/utils/produkey.zip', 'ProduKey64': 'http://www.nirsoft.net/utils/produkey-x64.zip', 'PuTTY': 'https://the.earth.li/~sgtatham/putty/latest/w32/putty.zip', 'RKill': 'https://www.bleepingcomputer.com/download/rkill/dl/10/', + 'Samsung Magician': 'https://s3.ap-northeast-2.amazonaws.com/global.semi.static/SAMSUNG_SSD_v5_2_1_180523/CD0CFAC4675B9E502899B41BE00525C3909ECE3AD57CC1A2FB6B74A766B2A1EA/Samsung_Magician_Installer.zip', 'SDIO Themes': 'http://snappy-driver-installer.org/downloads/SDIO_Themes.zip', 'SDIO Torrent': 'http://snappy-driver-installer.org/downloads/SDIO_Update.torrent', - 'Samsung Magician': 'http://downloadcenter.samsung.com/content/SW/201801/20180123130636806/Samsung_Magician_Installer.exe', 'TDSSKiller': 'https://media.kaspersky.com/utilities/VirusUtilities/EN/tdsskiller.exe', 'TestDisk': 'https://www.cgsecurity.org/testdisk-7.1-WIP.win.zip', - 'TreeSizeFree': 'https://www.jam-software.com/treesize_free/TreeSizeFree-Portable.zip', 'wimlib32': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip', 'wimlib64': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip', 'Winapp2': 'https://github.com/MoscaDotTo/Winapp2/archive/master.zip', - 'XMPlay 7z': 'http://support.xmplay.com/files/16/xmp-7z.zip?v=800962', - 'XMPlay Game': 'http://support.xmplay.com/files/12/xmp-gme.zip?v=515637', - 'XMPlay RAR': 'http://support.xmplay.com/files/16/xmp-rar.zip?v=409646', - 'XMPlay WAModern': 'http://support.xmplay.com/files/10/WAModern.zip?v=207099', - 'XMPlay': 'http://support.xmplay.com/files/20/xmplay383.zip?v=298195', + 'WizTree': 'https://antibody-software.com/files/wiztree_3_26_portable.zip', + 'XMPlay 7z': 'https://support.xmplay.com/files/16/xmp-7z.zip?v=800962', + 'XMPlay Game': 'https://support.xmplay.com/files/12/xmp-gme.zip?v=515637', + 'XMPlay RAR': 'https://support.xmplay.com/files/16/xmp-rar.zip?v=409646', + 'XMPlay WAModern': 'https://support.xmplay.com/files/10/WAModern.zip?v=207099', + 'XMPlay': 'https://support.xmplay.com/files/20/xmplay383.zip?v=298195', 'XYplorerFree': 'https://www.xyplorer.com/download/xyplorer_free_noinstall.zip', - 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.33.1/aria2-1.33.1-win-32bit-build1.zip', } VCREDIST_SOURCES = { '2010sp1': { @@ -67,9 +65,8 @@ VCREDIST_SOURCES = { } NINITE_SOURCES = { 'Bundles': { - 'Runtimes.exe': '.net4.7.1-air-java8-silverlight', - 'Legacy.exe': '.net4.7.1-7zip-air-chrome-firefox-java8-silverlight-vlc', - 'Modern.exe': '.net4.7.1-7zip-air-chrome-classicstart-firefox-java8-silverlight-vlc', + 'Legacy.exe': '.net4.7.2-7zip-chrome-firefox-vlc', + 'Modern.exe': '.net4.7.2-7zip-chrome-classicstart-firefox-vlc', }, 'Audio-Video': { 'AIMP.exe': 'aimp', @@ -94,6 +91,7 @@ NINITE_SOURCES = { 'SugarSync.exe': 'sugarsync', }, 'Communication': { + 'Discord': 'discord', 'Pidgin.exe': 'pidgin', 'Skype.exe': 'skype', 'Trillian.exe': 'trillian', @@ -105,7 +103,6 @@ NINITE_SOURCES = { }, 'Developer': { 'Eclipse.exe': 'eclipse', - 'FileZilla.exe': 'filezilla', 'JDK 8.exe': 'jdk8', 'JDK 8 (x64).exe': 'jdkx8', 'Notepad++.exe': 'notepadplusplus', @@ -149,7 +146,7 @@ NINITE_SOURCES = { }, 'Runtimes': { 'Adobe Air.exe': 'air', - 'dotNET.exe': '.net4.7.1', + 'dotNET.exe': '.net4.7.2', 'Java 8.exe': 'java8', 'Shockwave.exe': 'shockwave', 'Silverlight.exe': 'silverlight', @@ -193,6 +190,7 @@ RST_SOURCES = { 'SetupRST_15.8.exe': 'https://downloadmirror.intel.com/27442/eng/SetupRST.exe', 'SetupRST_15.9.exe': 'https://downloadmirror.intel.com/27400/eng/SetupRST.exe', 'SetupRST_16.0.exe': 'https://downloadmirror.intel.com/27681/eng/SetupRST.exe', + 'SetupRST_16.5.exe': 'https://downloadmirror.intel.com/27984/eng/SetupRST.exe', } From 4e9cd1f1147c3d31389e0dab19074b934129c191 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 19:21:36 -0600 Subject: [PATCH 126/138] Update FastCopy using new installer --- .bin/Scripts/functions/update.py | 33 +++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 6825f9ba..9e1c3e34 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -235,19 +235,34 @@ def update_fastcopy(): remove_from_kit('FastCopy') # Download - download_to_temp('FastCopy32.zip', SOURCE_URLS['FastCopy32']) - download_to_temp('FastCopy64.zip', SOURCE_URLS['FastCopy64']) - - # Extract - extract_temp_to_bin('FastCopy64.zip', 'FastCopy', sz_args=['FastCopy.exe']) + download_to_temp('FastCopy.zip', SOURCE_URLS['FastCopy']) + + # Extract installer + extract_temp_to_bin('FastCopy.zip', 'FastCopy') + _path = r'{}\FastCopy'.format(global_vars['BinDir']) + _installer = 'FastCopy354_installer.exe' + + # Extract 64-bit + cmd = [ + r'{}\{}'.format(_path, _installer), + '/NOSUBDIR', '/DIR={}'.format(_path), + '/EXTRACT64'] + run_program(cmd) shutil.move( r'{}\FastCopy\FastCopy.exe'.format(global_vars['BinDir']), r'{}\FastCopy\FastCopy64.exe'.format(global_vars['BinDir'])) - extract_temp_to_bin('FastCopy32.zip', 'FastCopy', sz_args=[r'-x!setup.exe', r'-x!*.dll']) - + + # Extract 32-bit + cmd = [ + r'{}\{}'.format(_path, _installer), + '/NOSUBDIR', '/DIR={}'.format(_path), + '/EXTRACT32'] + run_program(cmd) + # Cleanup - remove_from_temp('FastCopy32.zip') - remove_from_temp('FastCopy64.zip') + os.remove(r'{}\{}'.format(_path, _installer)) + os.remove(r'{}\setup.exe'.format(_path, _installer)) + remove_from_temp('FastCopy.zip') def update_wimlib(): # Stop running processes From e4bcf88fe5596779efb1fbc05a41450e31f5ea0d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 19:23:36 -0600 Subject: [PATCH 127/138] Update Intel RST (Current Release) --- .bin/Scripts/settings/launchers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 6e011691..0e819553 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -282,8 +282,8 @@ LAUNCHERS = { 'Intel RST (Current Release)': { 'L_TYPE': 'Executable', 'L_PATH': '_Drivers\Intel RST', - 'L_ITEM': 'SetupRST_16.0.exe', - 'L_7ZIP': 'SetupRST_16.0.exe', + 'L_ITEM': 'SetupRST_16.5.exe', + 'L_7ZIP': 'SetupRST_16.5.exe', }, 'Intel RST (Previous Releases)': { 'L_TYPE': 'Folder', From f3885f25d67a5dd9ecd55ca3da780d41b986cf12 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 19:39:05 -0600 Subject: [PATCH 128/138] Update FastCopy using new installer --- .bin/Scripts/build_pe.ps1 | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index 3fb1bcc9..a16edb03 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -254,20 +254,30 @@ if ($MyInvocation.InvocationName -ne ".") { # Fast Copy Write-Host "Extracting: FastCopy" try { + # Extract Installer $ArgumentList = @( - "x", "$Temp\fastcopy64.zip", "-o$Build\bin\amd64\FastCopy", - "-aoa", "-bso0", "-bse0", "-bsp0", - "-x!setup.exe", "-x!*.dll") + "e", "$Temp\fastcopy.zip", "-o$Temp", + "-aoa", "-bso0", "-bse0", "-bsp0") Start-Process -FilePath $SevenZip -ArgumentList $ArgumentList -NoNewWindow -Wait + + # Extract 64-bit $ArgumentList = @( - "e", "$Temp\fastcopy32.zip", "-o$Build\bin\x86\FastCopy", - "-aoa", "-bso0", "-bse0", "-bsp0", - "-x!setup.exe", "-x!*.dll") - Start-Process -FilePath $SevenZip -ArgumentList $ArgumentList -NoNewWindow -Wait + "/NOSUBDIR", "/DIR=$Build\bin\amd64\FastCopy", + "/EXTRACT64") + Start-Process -FilePath "$TEMP\FastCopy354_installer.exe" -ArgumentList $ArgumentList -NoNewWindow -Wait + Remove-Item "$Build\bin\amd64\FastCopy\setup.exe" -Force + + # Extract 32-bit + $ArgumentList = @( + "/NOSUBDIR", "/DIR=$Build\bin\x86\FastCopy", + "/EXTRACT32") + Start-Process -FilePath "$TEMP\FastCopy354_installer.exe" -ArgumentList $ArgumentList -NoNewWindow -Wait + Remove-Item "$Build\bin\x86\FastCopy\setup.exe" -Force } catch { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } + # Killer Network Driver Write-Host "Extracting: Killer Network Driver" From bc572304186bf867f6c1b69313b9f77e0752e4e9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 19:44:41 -0600 Subject: [PATCH 129/138] Replaced TreeSizeFree with WizTree --- .bin/Scripts/functions/update.py | 13 +++++++------ .bin/Scripts/settings/launchers.py | 6 +++--- .bin/Scripts/update_kit.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 9e1c3e34..809f98e0 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -760,22 +760,23 @@ def update_putty(): # Cleanup remove_from_temp('putty.zip') -def update_treesizefree(): +def update_wiztree(): # Stop running processes - kill_process('TreeSizeFree.exe') + for process in ['WizTree.exe', 'WizTree64.exe']: + kill_process(process) # Remove existing folders - remove_from_kit('TreeSizeFree') + remove_from_kit('WizTree') # Download download_to_temp( - 'treesizefree.zip', SOURCE_URLS['TreeSizeFree']) + 'wiztree.zip', SOURCE_URLS['WizTree']) # Extract files - extract_temp_to_cbin('treesizefree.zip', 'TreeSizeFree') + extract_temp_to_cbin('wiztree.zip', 'WizTree') # Cleanup - remove_from_temp('treesizefree.zip') + remove_from_temp('wiztree.zip') def update_xmplay(): # Stop running processes diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 0e819553..e2d74832 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -475,10 +475,10 @@ LAUNCHERS = { 'L_PATH': 'PuTTY', 'L_ITEM': 'PUTTY.EXE', }, - 'TreeSizeFree': { + 'WizTree': { 'L_TYPE': 'Executable', - 'L_PATH': 'TreeSizeFree', - 'L_ITEM': 'TreeSizeFree.exe', + 'L_PATH': 'WizTree', + 'L_ITEM': 'WizTree.exe', 'L_ELEV': 'True', }, 'Update Kit': { diff --git a/.bin/Scripts/update_kit.py b/.bin/Scripts/update_kit.py index 6a5cdf82..7f9b251d 100644 --- a/.bin/Scripts/update_kit.py +++ b/.bin/Scripts/update_kit.py @@ -70,7 +70,7 @@ if __name__ == '__main__': try_and_print(message='FirefoxExtensions...', function=update_firefox_ublock_origin, other_results=other_results, width=40) try_and_print(message='PuTTY...', function=update_putty, other_results=other_results, width=40) try_and_print(message='Notepad++...', function=update_notepadplusplus, other_results=other_results, width=40) - try_and_print(message='TreeSizeFree...', function=update_treesizefree, other_results=other_results, width=40) + try_and_print(message='WizTree...', function=update_wiztree, other_results=other_results, width=40) try_and_print(message='XMPlay...', function=update_xmplay, other_results=other_results, width=40) # Repairs From d502f769ea6154f1d438ad6b46696794374cd494 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 19:58:36 -0600 Subject: [PATCH 130/138] Updated update_samsung_magician() --- .bin/Scripts/functions/update.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 809f98e0..54e64aad 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -489,10 +489,21 @@ def update_samsung_magician(): remove_from_kit('Samsung Magician.exe') # Download - download_generic( - r'{}\_Drivers\Samsung Magician'.format(global_vars['CBinDir']), - 'Samsung Magician.exe', - SOURCE_URLS['Samsung Magician']) + download_to_temp('Samsung Magician.zip', SOURCE_URLS['Samsung Magician']) + + # Extract + extract_generic( + source=r'{}\Samsung Magician.zip'.format(global_vars['TmpDir']), + dest=r'{}\_Drivers'.format(global_vars['CBinDir']), + mode='e') + shutil.move( + r'{}\_Drivers\Samsung_Magician_Installer.exe'.format( + global_vars['CBinDir']), + r'{}\_Drivers\Samsung Magician.exe'.format( + global_vars['CBinDir'])) + + # Cleanup + remove_from_temp('Samsung Magician.zip') def update_sdi_origin(): # Download aria2 From 1e6eb26c779c8e54a3e94fd216ccefe4c541e739 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 20:02:15 -0600 Subject: [PATCH 131/138] Removed Office 2013 sections --- .bin/Scripts/Launch.cmd | 3 +-- .bin/Scripts/functions/update.py | 9 ++++----- .bin/Scripts/settings/launchers.py | 26 -------------------------- 3 files changed, 5 insertions(+), 33 deletions(-) diff --git a/.bin/Scripts/Launch.cmd b/.bin/Scripts/Launch.cmd index 2364dcc8..f7080adb 100644 --- a/.bin/Scripts/Launch.cmd +++ b/.bin/Scripts/Launch.cmd @@ -150,7 +150,6 @@ goto Exit :LaunchOffice call "%bin%\Scripts\init_client_dir.cmd" /Office set "_odt=False" -if %L_PATH% equ 2013 (set "_odt=True") if %L_PATH% equ 2016 (set "_odt=True") if "%_odt%" == "True" ( goto LaunchOfficeODT @@ -493,4 +492,4 @@ goto Exit :Exit popd endlocal -exit /b %errorlevel% \ No newline at end of file +exit /b %errorlevel% diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 54e64aad..3389d3f4 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -587,8 +587,8 @@ def update_office(): if os.path.exists(include_path): shutil.copytree(include_path, dest) - # Download and extract - for year in ['2013', '2016']: + for year in ['2016']: + # Download and extract name = 'odt{}.exe'.format(year) url = 'Office Deployment Tool {}'.format(year) download_to_temp(name, SOURCE_URLS[url]) @@ -602,9 +602,8 @@ def update_office(): r'{}\{}'.format(global_vars['TmpDir'], year), r'{}\_Office\{}'.format(global_vars['CBinDir'], year)) - # Cleanup - remove_from_temp('odt2013.exe') - remove_from_temp('odt2016.exe') + # Cleanup + remove_from_temp('odt{}.exe'.format(year)) def update_classic_start_skin(): # Remove existing folders diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index e2d74832..a125a1f8 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -356,32 +356,6 @@ LAUNCHERS = { 'L_ELEV': 'True', }, }, - r'Installers\Extras\Office\2013': { - 'Home and Business 2013 (x32)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hb_32.xml', - 'L_NCMD': 'True', - }, - 'Home and Business 2013 (x64)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hb_64.xml', - 'L_NCMD': 'True', - }, - 'Home and Student 2013 (x32)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hs_32.xml', - 'L_NCMD': 'True', - }, - 'Home and Student 2013 (x64)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hs_64.xml', - 'L_NCMD': 'True', - }, - }, r'Installers\Extras\Office\2016': { 'Home and Business 2016 (x32)': { 'L_TYPE': 'Office', From 902a6398caf37f3dd73a2af8d24d02e466634cef Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 20:58:25 -0600 Subject: [PATCH 132/138] Bugfix update_samsung_magician() --- .bin/Scripts/functions/update.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 3389d3f4..ab2ffa80 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -492,14 +492,11 @@ def update_samsung_magician(): download_to_temp('Samsung Magician.zip', SOURCE_URLS['Samsung Magician']) # Extract - extract_generic( - source=r'{}\Samsung Magician.zip'.format(global_vars['TmpDir']), - dest=r'{}\_Drivers'.format(global_vars['CBinDir']), - mode='e') + extract_temp_to_cbin('Samsung Magician.zip', '_Drivers\Samsung Magician') shutil.move( - r'{}\_Drivers\Samsung_Magician_Installer.exe'.format( + r'{}\_Drivers\Samsung Magician\Samsung_Magician_Installer.exe'.format( global_vars['CBinDir']), - r'{}\_Drivers\Samsung Magician.exe'.format( + r'{}\_Drivers\Samsung Magician\Samsung Magician.exe'.format( global_vars['CBinDir'])) # Cleanup From 9c0262693777eda0b2c0e3e85dcb00a2162b7e25 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 21:03:49 -0600 Subject: [PATCH 133/138] Updated update_adwcleaner() --- .bin/Scripts/functions/update.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index ab2ffa80..50386673 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -849,11 +849,10 @@ def update_adwcleaner(): remove_from_kit('AdwCleaner') # Download - url = resolve_dynamic_url( - SOURCE_URLS['AdwCleaner'], - 'id="downloadLink"') download_generic( - r'{}\AdwCleaner'.format(global_vars['CBinDir']), 'AdwCleaner.exe', url) + r'{}\AdwCleaner'.format(global_vars['CBinDir']), + 'AdwCleaner.exe', + SOURCE_URLS['AdwCleaner']) def update_kvrt(): # Stop running processes From ae8993821f4cce16f61e25f9273e6a418f6861bc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 21:05:09 -0600 Subject: [PATCH 134/138] Missed a VCR 2008 section --- .cbin/_include/_vcredists/InstallAll.bat | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.cbin/_include/_vcredists/InstallAll.bat b/.cbin/_include/_vcredists/InstallAll.bat index 597cd60e..5d935b34 100644 --- a/.cbin/_include/_vcredists/InstallAll.bat +++ b/.cbin/_include/_vcredists/InstallAll.bat @@ -1,9 +1,6 @@ @echo off setlocal -start "" /wait "2008sp1\x32\vcredist.exe" /qb! /norestart -start "" /wait "2008sp1\x64\vcredist.exe" /qb! /norestart - start "" /wait "2010\x32\vcredist.exe" /passive /norestart start "" /wait "2010\x64\vcredist.exe" /passive /norestart @@ -19,4 +16,4 @@ start "" /wait "2015u3\x64\vcredist.exe" /install /passive /norestart start "" /wait "2017\x32\vcredist.exe" /install /passive /norestart start "" /wait "2017\x64\vcredist.exe" /install /passive /norestart -endlocal \ No newline at end of file +endlocal From 50a503240d63d1e3350f8eddc3c95d0d57489ea3 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 16 Sep 2018 23:08:11 -0600 Subject: [PATCH 135/138] Downgrading Python to 3.6 in WinPE * Python wouldn't load --- .bin/Scripts/build_pe.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index a16edb03..0476e8fe 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -158,16 +158,16 @@ if ($MyInvocation.InvocationName -ne ".") { @("produkey32.zip", "http://www.nirsoft.net/utils/produkey.zip"), @("produkey64.zip", "http://www.nirsoft.net/utils/produkey-x64.zip"), # Python - @("python32.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip"), - @("python64.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip"), + @("python32.zip", "https://www.python.org/ftp/python/3.6.0/python-3.6.0-embed-win32.zip"), + @("python64.zip", "https://www.python.org/ftp/python/3.6.0/python-3.6.0-embed-amd64.zip"), # Python: psutil @( "psutil64.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win_amd64.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win_amd64.whl") ), @( "psutil32.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win32.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win32.whl") ), # Q-Dir @("qdir32.zip", "https://www.softwareok.com/Download/Q-Dir_Portable.zip"), From e3aaa887c53582dd9fadde783173bc54f935c4dd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Sep 2018 14:15:19 -0600 Subject: [PATCH 136/138] Countdown the minutes remaining during Prime95 * Fixes issue #54 --- .bin/Scripts/functions/hw_diags.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 61530e5b..3b8bd354 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -452,7 +452,6 @@ def run_iobenchmark(): def run_mprime(): """Run Prime95 for MPRIME_LIMIT minutes while showing the temps.""" aborted = False - clear_screen() print_log('\nStart Prime95 test') TESTS['Prime95']['Status'] = 'Working' update_progress() @@ -467,12 +466,15 @@ def run_mprime(): # Start test run_program(['apple-fans', 'max']) - print_standard('Running Prime95 for {} minutes'.format(MPRIME_LIMIT)) - print_warning('If running too hot, press CTL+c to abort the test') try: - sleep(int(MPRIME_LIMIT)*60) + for i in range(int(MPRIME_LIMIT)): + clear_screen() + print_standard('Running Prime95 ({} minutes left)'.format( + int(MPRIME_LIMIT)-i)) + print_warning('If running too hot, press CTRL+c to abort the test') + sleep(60) except KeyboardInterrupt: - # Catch CTL+C + # Catch CTRL+C aborted = True # Save "final" temps From 79fc40e57a7596dbbf1d17d4509a6dcae2377848 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Sep 2018 18:50:55 -0600 Subject: [PATCH 137/138] Fixed Python 3.7 dependencies * This re-upgrades Python to 3.7 in WinPE --- .bin/Scripts/build_kit.ps1 | 21 +++++++++++++++++++++ .bin/Scripts/build_pe.ps1 | 31 +++++++++++++++++++++++++++---- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/build_kit.ps1 b/.bin/Scripts/build_kit.ps1 index bba52a66..bd213578 100644 --- a/.bin/Scripts/build_kit.ps1 +++ b/.bin/Scripts/build_kit.ps1 @@ -11,6 +11,7 @@ $Bin = (Get-Item $WD).Parent.FullName $Root = (Get-Item $Bin -Force).Parent.FullName $Temp = "$Bin\tmp" $System32 = "{0}\System32" -f $Env:SystemRoot +$SysWOW64 = "{0}\SysWOW64" -f $Env:SystemRoot Push-Location "$WD" $Host.UI.RawUI.BackgroundColor = "black" $Host.UI.RawUI.ForegroundColor = "white" @@ -112,12 +113,25 @@ if ($MyInvocation.InvocationName -ne ".") { $Url = FindDynamicUrl -SourcePage $DownloadPage -RegEx $RegEx DownloadFile -Path $Path -Name $Name -Url $Url } + + # Visual C++ Runtimes + $Url = "https://aka.ms/vs/15/release/vc_redist.x86.exe" + DownloadFile -Path $Path -Name "vcredist_x86.exe" -Url $Url + $Url = "https://aka.ms/vs/15/release/vc_redist.x64.exe" + DownloadFile -Path $Path -Name "vcredist_x64.exe" -Url $Url ## Bail ## # If errors were encountered during downloads if ($DownloadErrors -gt 0) { Abort } + + ## Install ## + # Visual C++ Runtimes + $ArgumentList = @("/install", "/passive", "/norestart") + Start-Process -FilePath "$Temp\vcredist_x86.exe" -ArgumentList $ArgumentList -Wait + Start-Process -FilePath "$Temp\vcredist_x64.exe" -ArgumentList $ArgumentList -Wait + Remove-Item "$Temp\vcredist*.exe" ## Extract ## # 7-Zip @@ -192,6 +206,13 @@ if ($MyInvocation.InvocationName -ne ".") { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } } + try { + Copy-Item -Path "$System32\vcruntime140.dll" -Destination "$Bin\Python\x64\vcruntime140.dll" -Force + Copy-Item -Path "$SysWOW64\vcruntime140.dll" -Destination "$Bin\Python\x32\vcruntime140.dll" -Force + } + catch { + Write-Host (" ERROR: Failed to copy Visual C++ Runtime DLLs." ) -ForegroundColor "Red" + } Remove-Item "$Temp\python*.zip" Remove-Item "$Temp\*.whl" diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index 0476e8fe..560d570d 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -17,6 +17,7 @@ $Date = Get-Date -UFormat "%Y-%m-%d" $Host.UI.RawUI.BackgroundColor = "Black" $Host.UI.RawUI.ForegroundColor = "White" $HostSystem32 = "{0}\System32" -f $Env:SystemRoot +$HostSysWOW64 = "{0}\SysWOW64" -f $Env:SystemRoot $DISM = "{0}\DISM.exe" -f $Env:DISMRoot #Enable TLS 1.2 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -158,16 +159,16 @@ if ($MyInvocation.InvocationName -ne ".") { @("produkey32.zip", "http://www.nirsoft.net/utils/produkey.zip"), @("produkey64.zip", "http://www.nirsoft.net/utils/produkey-x64.zip"), # Python - @("python32.zip", "https://www.python.org/ftp/python/3.6.0/python-3.6.0-embed-win32.zip"), - @("python64.zip", "https://www.python.org/ftp/python/3.6.0/python-3.6.0-embed-amd64.zip"), + @("python32.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip"), + @("python64.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip"), # Python: psutil @( "psutil64.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win_amd64.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win_amd64.whl") ), @( "psutil32.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win32.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win32.whl") ), # Q-Dir @("qdir32.zip", "https://www.softwareok.com/Download/Q-Dir_Portable.zip"), @@ -177,6 +178,9 @@ if ($MyInvocation.InvocationName -ne ".") { @("testdisk64.zip", "https://www.cgsecurity.org/testdisk-7.1-WIP.win64.zip"), # VirtIO drivers @("virtio-win.iso", "https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/latest-virtio/virtio-win.iso"), + # Visual C++ Runtimes + @("vcredist_x86.exe", "https://aka.ms/vs/15/release/vc_redist.x86.exe"), + @("vcredist_x64.exe", "https://aka.ms/vs/15/release/vc_redist.x64.exe"), # wimlib-imagex @("wimlib32.zip", "https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip"), @("wimlib64.zip", "https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip") @@ -190,6 +194,13 @@ if ($MyInvocation.InvocationName -ne ".") { if ($DownloadErrors -gt 0) { Abort } + + ## Install ## + # Visual C++ Runtimes + Write-Host "Installing: Visual C++ Runtimes" + $ArgumentList = @("/install", "/passive", "/norestart") + Start-Process -FilePath "$Temp\vcredist_x86.exe" -ArgumentList $ArgumentList -Wait + Start-Process -FilePath "$Temp\vcredist_x64.exe" -ArgumentList $ArgumentList -Wait ## Extract ## # 7-Zip @@ -423,6 +434,12 @@ if ($MyInvocation.InvocationName -ne ".") { catch { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } + try { + Copy-Item -Path "$HostSystem32\vcruntime140.dll" -Destination "$Build\bin\amd64\python\vcruntime140.dll" -Force + } + catch { + Write-Host (" ERROR: Failed to copy Visual C++ Runtime DLL." ) -ForegroundColor "Red" + } # Python (x32) Write-Host "Extracting: Python (x32)" @@ -440,6 +457,12 @@ if ($MyInvocation.InvocationName -ne ".") { catch { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } + try { + Copy-Item -Path "$HostSysWOW64\vcruntime140.dll" -Destination "$Build\bin\x86\python\vcruntime140.dll" -Force + } + catch { + Write-Host (" ERROR: Failed to copy Visual C++ Runtime DLL." ) -ForegroundColor "Red" + } # Q-Dir Write-Host "Extracting: Q-Dir" From b34187b86a496998e6a7ef53599b056a718ed63d Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 17 Sep 2018 20:04:47 -0600 Subject: [PATCH 138/138] Use new Firefox 62 method to install uBlock Origin --- .bin/Scripts/functions/browsers.py | 20 +++++++++++++------- .bin/Scripts/functions/setup.py | 29 +++++++++++++++++++++++------ .bin/Scripts/functions/update.py | 14 ++++---------- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index c6d39db9..99086f18 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -46,6 +46,9 @@ UBO_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\cjpalhdl UBO_EXTRA_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin-extra/pgdnlhfefecpicbbihgmbmffkjpaplco?hl=en' UBO_EXTRA_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco' UBO_MOZILLA = 'https://addons.mozilla.org/en-us/firefox/addon/ublock-origin/' +UBO_MOZZILA_PATH = r'{}\Mozilla Firefox\distribution\extensions\ublock_origin.xpi'.format(os.environ.get('PROGRAMFILES')) +UBO_MOZILLA_REG = r'Software\Mozilla\Firefox\Extensions' +UBO_MOZILLA_REG_NAME = 'uBlock0@raymondhill.net' UBO_OPERA = 'https://addons.opera.com/en/extensions/details/ublock/?display=en' SUPPORTED_BROWSERS = { 'Internet Explorer': { @@ -369,14 +372,17 @@ def install_adblock(indent=8, width=32): urls.append(UBO_EXTRA_CHROME) elif browser_data[browser]['base'] == 'mozilla': - # Assume UBO is not installed first and change if it is - urls.append(UBO_MOZILLA) - if browser == 'Mozilla Firefox': - ubo = browser_data[browser]['exe_path'].replace( - 'firefox.exe', - r'distribution\extensions\uBlock0@raymondhill.net') - if os.path.exists(ubo): + # Check for system extensions + try: + with winreg.OpenKey(HKLM, UBO_MOZILLA_REG) as key: + winreg.QueryValueEx(key, UBO_MOZILLA_REG_NAME) + except FileNotFoundError: + urls = [UBO_MOZILLA] + else: + if os.path.exists(UBO_MOZZILA_PATH): urls = ['about:addons'] + else: + urls = [UBO_MOZILLA] elif browser_data[browser]['base'] == 'ie': urls.append(IE_GALLERY) diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index d08692b5..4fca0303 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -5,6 +5,9 @@ from functions.common import * # STATIC VARIABLES HKCU = winreg.HKEY_CURRENT_USER HKLM = winreg.HKEY_LOCAL_MACHINE +MOZILLA_FIREFOX_UBO_PATH = r'{}\{}\ublock_origin.xpi'.format( + os.environ.get('PROGRAMFILES'), + r'Mozilla Firefox\distribution\extensions') OTHER_RESULTS = { 'Error': { 'CalledProcessError': 'Unknown Error', @@ -76,9 +79,6 @@ SETTINGS_EXPLORER_USER = { }, } SETTINGS_GOOGLE_CHROME = { - r'Software\Google\Chrome\Extensions': { - 'WOW64_32': True, - }, r'Software\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm': { 'SZ Items': { 'update_url': 'https://clients2.google.com/service/update2/crx'}, @@ -90,6 +90,19 @@ SETTINGS_GOOGLE_CHROME = { 'WOW64_32': True, }, } +SETTINGS_MOZILLA_FIREFOX_32 = { + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + 'WOW64_32': True, + }, + } +SETTINGS_MOZILLA_FIREFOX_64 = { + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + }, + } VCR_REDISTS = [ {'Name': 'Visual C++ 2008 SP1 x32...', 'Cmd': [r'2008sp1\x32\vcredist.exe', '/qb! /norestart']}, @@ -221,7 +234,7 @@ def install_adobe_reader(): run_program(cmd) def install_chrome_extensions(): - """Update registry to 'install' Google Chrome extensions for all users.""" + """Update registry to install Google Chrome extensions for all users.""" write_registry_settings(SETTINGS_GOOGLE_CHROME, all_users=True) def install_classicstart_skin(): @@ -238,16 +251,20 @@ def install_classicstart_skin(): shutil.copy(source, dest) def install_firefox_extensions(): - """Extract Firefox extensions to installation folder.""" + """Update registry to install Firefox extensions for all users.""" dist_path = r'{PROGRAMFILES}\Mozilla Firefox\distribution\extensions'.format( **global_vars['Env']) source_path = r'{CBinDir}\FirefoxExtensions.7z'.format(**global_vars) if not os.path.exists(source_path): raise FileNotFoundError + + # Update registry + write_registry_settings(SETTINGS_MOZILLA_FIREFOX_32, all_users=True) + write_registry_settings(SETTINGS_MOZILLA_FIREFOX_64, all_users=True) # Extract extension(s) to distribution folder cmd = [ - global_vars['Tools']['SevenZip'], 'x', '-aos', '-bso0', '-bse0', + global_vars['Tools']['SevenZip'], 'e', '-aos', '-bso0', '-bse0', '-p{ArchivePassword}'.format(**global_vars), '-o{dist_path}'.format(dist_path=dist_path), source_path] diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 50386673..9c9cfec0 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -720,16 +720,10 @@ def update_firefox_ublock_origin(): remove_from_kit('FirefoxExtensions') # Download - download_to_temp('ff-uBO.xpi', SOURCE_URLS['Firefox uBO']) - - # Extract files - extract_generic( - r'{}\ff-uBO.xpi'.format(global_vars['TmpDir']), - r'{}\FirefoxExtensions\uBlock0@raymondhill.net'.format( - global_vars['CBinDir'])) - - # Cleanup - remove_from_temp('ff-uBO.xpi') + download_generic( + r'{}\FirefoxExtensions'.format(global_vars['CBinDir']), + 'ublock_origin.xpi', + SOURCE_URLS['Firefox uBO']) def update_notepadplusplus(): # Stop running processes