Merge branch 'datarec' into dev

.bin/Scripts/ddrescue-tui

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+## Wizard Kit: ddrescue TUI Launcher
+
+SESSION_NAME="ddrescue-tui"
+WINDOW_NAME="GNU ddrescue TUI"
+MENU="ddrescue-tui-menu"
+
+function ask() {
+    while :; do
+        read -p "$1 " -r answer
+        if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then
+            return 0
+        elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then
+            return 1
+        fi
+    done
+}
+
+die () {
+    echo "$0:" "$@" >&2
+    exit 1
+}
+
+# Check for running session
+if tmux list-session | grep -q "$SESSION_NAME"; then
+    echo "WARNING: tmux session $SESSION_NAME already exists."
+    echo ""
+    if ask "Kill current session?"; then
+        tmux kill-session -t "$SESSION_NAME" || \
+            die "Failed to kill session: $SESSION_NAME"
+    else
+        echo "Aborted."
+        echo ""
+        echo -n "Press Enter to exit... "
+        read -r
+        exit 0
+    fi
+fi
+
+# Start session
+tmux new-session -s "$SESSION_NAME" -n "$WINDOW_NAME" "$MENU" $*
+

.bin/Scripts/ddrescue-tui-menu

@@ -0,0 +1,63 @@
+#!/bin/python3
+#
+## Wizard Kit: TUI for ddrescue cloning and imaging
+
+import os
+import sys
+
+# Init
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+
+from functions.ddrescue import *
+from functions.hw_diags import *
+init_global_vars()
+
+if __name__ == '__main__':
+    try:
+        # Prep
+        clear_screen()
+        args = list(sys.argv)
+        run_mode = ''
+        source_path = None
+        dest_path = None
+
+        # Parse args
+        try:
+            script_name = os.path.basename(args.pop(0))
+            run_mode = str(args.pop(0)).lower()
+            source_path = args.pop(0)
+            dest_path = args.pop(0)
+        except IndexError:
+            # We'll set the missing paths later
+            pass
+
+        # Show usage
+        if re.search(r'-*(h|help|\?)', str(sys.argv), re.IGNORECASE):
+            show_usage(script_name)
+            exit_script()
+
+        # Start cloning/imaging
+        if run_mode in ('clone', 'image'):
+            menu_ddrescue(source_path, dest_path, run_mode)
+        else:
+            if not re.search(r'^-*(h|help\?)', run_mode, re.IGNORECASE):
+                print_error('Invalid mode.')
+        
+        # Done
+        print_standard('\nDone.')
+        pause("Press Enter to exit...")
+        exit_script()
+    except GenericAbort:
+        abort()
+    except GenericError as ge:
+        msg = 'Generic Error'
+        if str(ge):
+            msg = str(ge)
+        print_error(msg)
+        abort()
+    except SystemExit:
+        pass
+    except:
+        major_exception()
+
+# vim: sts=4 sw=4 ts=4

.bin/Scripts/ddrescue-tui-smart-display

@@ -0,0 +1,39 @@
+#!/bin/python3
+#
+## Wizard Kit: SMART attributes display for ddrescue TUI
+
+import os
+import sys
+import time
+
+# Init
+os.chdir(os.path.dirname(os.path.realpath(__file__)))
+sys.path.append(os.getcwd())
+from functions.hw_diags import *
+#init_global_vars()
+
+if __name__ == '__main__':
+    try:
+        # Prep
+        clear_screen()
+        dev_path = sys.argv[1]
+        devs = scan_disks(True, dev_path)
+        
+        # Warn if SMART unavailable
+        if dev_path not in devs:
+            print_error('SMART data not available')
+            exit_script()
+        
+        # Initial screen
+        dev = devs[dev_path]
+        clear_screen()
+        show_disk_details(dev, only_attributes=True)
+        
+        # Done
+        exit_script()
+    except SystemExit:
+        pass
+    except:
+        major_exception()
+
+# vim: sts=4 sw=4 ts=4

.bin/Scripts/echo-and-hold

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+## Wizard Kit: "echo" text to screen and "hold" by waiting for user input
+
+function usage {
+    echo "Usage: $(basename "$0") \"text\""
+    echo "  e.g. $(basename "$0") \"Some text to show\""
+}
+
+echo -en "$@" && read -r __dont_care
+exit 0
+

.bin/Scripts/functions/common.py

@@ -197,6 +197,30 @@ def extract_item(item, filter='', silent=False):
         if not silent:
             print_warning('WARNING: Errors encountered while exctracting data')
 
+def get_process(name=None):
+    """Get process by name, returns psutil.Process obj."""
+    proc = None
+    if not name:
+        raise GenericError
+
+    for p in psutil.process_iter():
+        try:
+            if p.name() == name:
+                proc = p
+        except psutil._exceptions.NoSuchProcess:
+            # Process finished during iteration? Going to ignore
+            pass
+    return proc
+
+def get_simple_string(prompt='Enter string'):
+    """Get string from user (minimal allowed character set) and return as str."""
+    simple_string = None
+    while simple_string is None:
+        _input = input('{}: '.format(prompt))
+        if re.match(r"^(\w|-| |\.|')+$", _input, re.ASCII):
+            simple_string = _input.strip()
+    return simple_string
+
 def get_ticket_number():
     """Get TicketNumber from user, save in LogDir, and return as str."""
     if not ENABLED_TICKET_NUMBERS:
@@ -213,15 +237,6 @@ def get_ticket_number():
                 f.write(ticket_number)
     return ticket_number
 
-def get_simple_string(prompt='Enter string'):
-    """Get string from user (only alphanumeric/space chars) and return as str."""
-    simple_string = None
-    while simple_string is None:
-        _input = input('{}: '.format(prompt))
-        if re.match(r'^(\w|-| )+$', _input, re.ASCII):
-            simple_string = _input.strip()
-    return simple_string
-
 def human_readable_size(size, decimals=0):
     """Convert size in bytes to a human-readable format and return a str."""
     # Prep string formatting

.bin/Scripts/functions/data.py

@@ -187,7 +187,7 @@ def get_mounted_volumes():
         mounted_volumes.extend(item.get('children', []))
     return {item['source']: item for item in mounted_volumes}
 
-def mount_all_volumes():
+def mount_volumes(all_devices=True, device_path=None, read_write=False):
     """Mount all detected filesystems."""
     report = {}
 
@@ -195,6 +195,9 @@ def mount_all_volumes():
     cmd = [
         'lsblk', '-J', '-p',
         '-o', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE']
+    if not all_devices and device_path:
+        # Only mount volumes for specific device
+        cmd.append(device_path)
     result = run_program(cmd)
     json_data = json.loads(result.stdout.decode())
     devs = json_data.get('blockdevices', [])
@@ -233,8 +236,11 @@ def mount_all_volumes():
                 vol_data['show_data']['warning'] = True
             else:
                 # Mount volume
+                cmd = ['udevil', 'mount',
+                    '-o', 'rw' if read_write else 'ro',
+                    vol_path]
                 try:
-                    run_program(['udevil', 'mount', '-o', 'ro', vol_path])
+                    run_program(cmd)
                 except subprocess.CalledProcessError:
                     vol_data['show_data']['data'] = 'Failed to mount'
                     vol_data['show_data']['error'] = True
@@ -242,11 +248,16 @@ def mount_all_volumes():
             mounted_volumes = get_mounted_volumes()
 
             # Format pretty result string
-            if vol_data['show_data']['data'] != 'Failed to mount':
+            if vol_data['show_data']['data'] == 'Failed to mount':
+                vol_data['mount_point'] = None
+            else:
                 size_used = human_readable_size(
                     mounted_volumes[vol_path]['used'])
                 size_avail = human_readable_size(
                     mounted_volumes[vol_path]['avail'])
+                vol_data['size_avail'] = size_avail
+                vol_data['size_used'] = size_used
+                vol_data['mount_point'] = mounted_volumes[vol_path]['target']
                 vol_data['show_data']['data'] = 'Mounted on {}'.format(
                     mounted_volumes[vol_path]['target'])
                 vol_data['show_data']['data'] = '{:40} ({} used, {} free)'.format(

.bin/Scripts/functions/hw_diags.py

@@ -1,6 +1,7 @@
 # Wizard Kit: Functions - HW Diagnostics
 
 import json
+import time
 
 from functions.common import *
 
@@ -56,7 +57,9 @@ def get_read_rate(s):
 
 def get_smart_details(dev):
     """Get SMART data for dev if possible, returns dict."""
-    cmd = 'sudo smartctl --all --json /dev/{}'.format(dev).split()
+    cmd = 'sudo smartctl --all --json {}{}'.format(
+        '' if '/dev/' in dev else '/dev/',
+        dev).split()
     result = run_program(cmd, check=False)
     try:
         return json.loads(result.stdout.decode())
@@ -509,12 +512,17 @@ def run_tests(tests):
                 global_vars['LogFile']))
             pause('Press Enter to exit...')
 
-def scan_disks():
+def scan_disks(full_paths=False, only_path=None):
     """Scan for disks eligible for hardware testing."""
     clear_screen()
 
     # Get eligible disk list
-    result = run_program(['lsblk', '-J', '-O'])
+    cmd = ['lsblk', '-J', '-O']
+    if full_paths:
+        cmd.append('-p')
+    if only_path:
+        cmd.append(only_path)
+    result = run_program(cmd)
     json_data = json.loads(result.stdout.decode())
     devs = {}
     for d in json_data.get('blockdevices', []):
@@ -536,13 +544,18 @@ def scan_disks():
     for dev, data in devs.items():
         # Get SMART attributes
         run_program(
-            cmd = 'sudo smartctl -s on /dev/{}'.format(dev).split(),
+            cmd = 'sudo smartctl -s on {}{}'.format(
+              '' if full_paths else '/dev/',
+              dev).split(),
             check = False)
         data['smartctl'] = get_smart_details(dev)
     
         # Get NVMe attributes
         if data['lsblk']['tran'] == 'nvme':
             cmd = 'sudo nvme smart-log /dev/{} -o json'.format(dev).split()
+            cmd = 'sudo nvme smart-log {}{} -o json'.format(
+                '' if full_paths else '/dev/',
+                dev).split()
             result = run_program(cmd, check=False)
             try:
                 data['nvme-cli'] = json.loads(result.stdout.decode())
@@ -588,19 +601,23 @@ def scan_disks():
     TESTS['NVMe/SMART']['Devices'] = devs
     TESTS['badblocks']['Devices'] = devs
     TESTS['iobenchmark']['Devices'] = devs
+    return devs
 
-def show_disk_details(dev):
+def show_disk_details(dev, only_attributes=False):
     """Display disk details."""
     dev_name = dev['lsblk']['name']
-    # Device description
+    if not only_attributes:
-    print_info('Device: /dev/{}'.format(dev['lsblk']['name']))
+      # Device description
-    print_standard(' {:>4} ({}) {} {}'.format(
+      print_info('Device: {}{}'.format(
-        str(dev['lsblk'].get('size', '???b')).strip(),
+          '' if '/dev/' in dev['lsblk']['name'] else '/dev/',
-        str(dev['lsblk'].get('tran', '???')).strip().upper().replace(
+          dev['lsblk']['name']))
-            'NVME', 'NVMe'),
+      print_standard(' {:>4} ({}) {} {}'.format(
-        str(dev['lsblk'].get('model', 'Unknown Model')).strip(),
+          str(dev['lsblk'].get('size', '???b')).strip(),
-        str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(),
+          str(dev['lsblk'].get('tran', '???')).strip().upper().replace(
-        ))
+              'NVME', 'NVMe'),
+          str(dev['lsblk'].get('model', 'Unknown Model')).strip(),
+          str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(),
+          ))
 
     # Warnings
     if dev.get('NVMe Disk', False):
@@ -615,7 +632,12 @@ def show_disk_details(dev):
 
     # Attributes
     if dev.get('NVMe Disk', False):
-        print_info('Attributes:')
+        if only_attributes:
+            print_info('SMART Attributes:', end='')
+            print_warning('             Updated: {}'.format(
+                time.strftime('%Y-%m-%d %H:%M %Z')))
+        else:
+            print_info('Attributes:')
         for attrib, threshold in sorted(ATTRIBUTES['NVMe'].items()):
             if attrib in dev['nvme-cli']:
                 print_standard(
@@ -636,7 +658,12 @@ def show_disk_details(dev):
                     print_success(raw_str, timestamp=False)
     elif dev['smartctl'].get('ata_smart_attributes', None):
         # SMART attributes
-        print_info('Attributes:')
+        if only_attributes:
+            print_info('SMART Attributes:', end='')
+            print_warning('             Updated: {}'.format(
+                time.strftime('%Y-%m-%d %H:%M %Z')))
+        else:
+            print_info('Attributes:')
         s_table = dev['smartctl'].get('ata_smart_attributes', {}).get(
             'table', {})
         s_table = {a.get('id', 'Unknown'): a for a in s_table}

.bin/Scripts/hw-diags

@@ -24,7 +24,7 @@ die () {
 
 # Check for running session
 if tmux list-session | grep -q "$SESSION_NAME"; then
-    echo "WARNING: hw-diags tmux session already exists."
+    echo "WARNING: tmux session $SESSION_NAME already exists."
     echo ""
     if ask "Kill current session?"; then
         tmux kill-session -t "$SESSION_NAME" || \

.bin/Scripts/mount-all-volumes

@@ -18,7 +18,7 @@ if __name__ == '__main__':
         print_standard('{}: Volume mount tool'.format(KIT_NAME_FULL))
 
         # Mount volumes
-        report = mount_all_volumes()
+        report = mount_volumes(all_devices=True)
 
         # Print report
         print_info('\nResults')

.linux_items/include/airootfs/etc/skel/.aliases

@@ -5,6 +5,7 @@ alias 7z5='7z a -t7z -mx=5'
 alias 7z7='7z a -t7z -mx=7'
 alias 7z9='7z a -t7z -mx=9'
 alias df='pydf'
+alias ddrescue='sudo ddrescue --ask --min-read-rate=64k -vvvv'
 alias diff='colordiff -ur'
 alias du='du -sch --apparent-size'
 alias fix-perms='find -type d -exec chmod 755 "{}" \; && find -type f -exec chmod 644 "{}" \;'
@@ -36,3 +37,5 @@ alias srsz='sudo rsync -avhzPS --stats --exclude-from="$HOME/.rsync_exclusions"'
 alias testdisk='sudo testdisk'
 alias umount='sudo umount'
 alias unmount='sudo umount'
+alias wkclone='sudo ddrescue-tui clone'
+alias wkimage='sudo ddrescue-tui image'