2Shirt/WizardKit
2Shirt/WizardKit
1
0
Fork 0
Code Issues 2 Pull requests Projects Releases 26 Packages Wiki Activity Actions

Updated ddrescue sections

Browse source
This commit is contained in:
2Shirt 2019-06-04 20:00:11 -06:00
parent 248e321438
commit 52e4415b43
Signed by: 2Shirt
GPG key ID: 152FAC923B0E132C
2 changed files with 303 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

534
.bin/Scripts/functions/ddrescue.py
View file

@ -1,25 +1,25 @@
# Wizard Kit: Functions - ddrescue-tui # pylint: disable=no-name-in-module,too-many-lines,wildcard-import
# vim: sts=2 sw=2 ts=2
'''Wizard Kit: Functions - ddrescue-tui'''
import datetime import datetime
import pathlib import pathlib
import psutil
import pytz
import re import re
import signal
import stat import stat
import time import time
from operator import itemgetter
from collections import OrderedDict import pytz
from functions.data import * from functions.data import *
from functions.hw_diags import * from functions.hw_diags import *
from functions.json import * from functions.json import *
from functions.tmux import * from functions.tmux import *
from operator import itemgetter
from settings.ddrescue import * from settings.ddrescue import *
# Clases # Clases
class BaseObj(): class BaseObj():
# pylint: disable=missing-docstring
"""Base object used by DevObj, DirObj, and ImageObj.""" """Base object used by DevObj, DirObj, and ImageObj."""
def __init__(self, path): def __init__(self, path):
self.type = 'base' self.type = 'base'
@ -44,6 +44,7 @@ class BaseObj():
class BlockPair(): class BlockPair():
# pylint: disable=too-many-instance-attributes
"""Object to track data and methods together for source and dest.""" """Object to track data and methods together for source and dest."""
def __init__(self, mode, source, dest): def __init__(self, mode, source, dest):
self.mode = mode self.mode = mode
@ -60,9 +61,10 @@ class BlockPair():
if self.mode == 'clone': if self.mode == 'clone':
# Cloning # Cloning
self.dest_path = dest.path self.dest_path = dest.path
self.map_path = '{pwd}/Clone_{prefix}.map'.format( self.map_path = '{cwd}/Clone_{prefix}.map'.format(
pwd=os.path.realpath(global_vars['Env']['PWD']), cwd=os.path.realpath(os.getcwd()),
prefix=source.prefix) prefix=source.prefix,
)
else: else:
# Imaging # Imaging
self.dest_path = '{path}/{prefix}.dd'.format( self.dest_path = '{path}/{prefix}.dd'.format(
@ -105,19 +107,19 @@ class BlockPair():
def load_map_data(self): def load_map_data(self):
"""Load data from map file and set progress.""" """Load data from map file and set progress."""
map_data = read_map_file(self.map_path) map_data = read_map_file(self.map_path)
self.rescued_percent = map_data['rescued'] self.rescued = map_data.get('rescued', 0)
self.rescued = (self.rescued_percent * self.size) / 100 self.rescued_percent = (self.rescued / self.size) * 100
if map_data['full recovery']: if map_data['full recovery']:
self.pass_done = [True, True, True] self.pass_done = [True, True, True]
self.rescued = self.size self.rescued = self.size
self.status = ['Skipped', 'Skipped', 'Skipped'] self.status = ['Skipped', 'Skipped', 'Skipped']
elif map_data['non-tried'] > 0: elif map_data.get('non-tried', 0) > 0:
# Initial pass incomplete # Initial pass incomplete
pass pass
elif map_data['non-trimmed'] > 0: elif map_data.get('non-trimmed', 0) > 0:
self.pass_done = [True, False, False] self.pass_done = [True, False, False]
self.status = ['Skipped', 'Pending', 'Pending'] self.status = ['Skipped', 'Pending', 'Pending']
elif map_data['non-scraped'] > 0: elif map_data.get('non-scraped', 0) > 0:
self.pass_done = [True, True, False] self.pass_done = [True, True, False]
self.status = ['Skipped', 'Skipped', 'Pending'] self.status = ['Skipped', 'Skipped', 'Pending']
else: else:
@ -145,14 +147,15 @@ class BlockPair():
"""Update progress using map file.""" """Update progress using map file."""
if os.path.exists(self.map_path): if os.path.exists(self.map_path):
map_data = read_map_file(self.map_path) map_data = read_map_file(self.map_path)
self.rescued_percent = map_data.get('rescued', 0) self.rescued = map_data.get('rescued', 0)
self.rescued = (self.rescued_percent * self.size) / 100 self.rescued_percent = (self.rescued / self.size) * 100
self.status[pass_num] = get_formatted_status( self.status[pass_num] = get_formatted_status(
label='Pass {}'.format(pass_num+1), label='Pass {}'.format(pass_num+1),
data=(self.rescued/self.size)*100) data=(self.rescued/self.size)*100)
class DevObj(BaseObj): class DevObj(BaseObj):
# pylint: disable=too-many-instance-attributes
"""Block device object.""" """Block device object."""
def self_check(self): def self_check(self):
"""Verify that self.path points to a block device.""" """Verify that self.path points to a block device."""
@ -186,6 +189,7 @@ class DevObj(BaseObj):
self.update_filename_prefix() self.update_filename_prefix()
def update_filename_prefix(self): def update_filename_prefix(self):
# pylint: disable=attribute-defined-outside-init
"""Set filename prefix based on details.""" """Set filename prefix based on details."""
self.prefix = '{m_size}_{model}'.format( self.prefix = '{m_size}_{model}'.format(
m_size=self.model_size, m_size=self.model_size,
@ -205,6 +209,7 @@ class DevObj(BaseObj):
class DirObj(BaseObj): class DirObj(BaseObj):
"""Directory object."""
def self_check(self): def self_check(self):
"""Verify that self.path points to a directory.""" """Verify that self.path points to a directory."""
if not pathlib.Path(self.path).is_dir(): if not pathlib.Path(self.path).is_dir():
@ -222,6 +227,7 @@ class DirObj(BaseObj):
class ImageObj(BaseObj): class ImageObj(BaseObj):
"""Image file object."""
def self_check(self): def self_check(self):
"""Verify that self.path points to a file.""" """Verify that self.path points to a file."""
if not pathlib.Path(self.path).is_file(): if not pathlib.Path(self.path).is_file():
@ -243,10 +249,11 @@ class ImageObj(BaseObj):
self.report = get_device_report(self.loop_dev) self.report = get_device_report(self.loop_dev)
self.report = self.report.replace( self.report = self.report.replace(
self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)') self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)')
run_program(['losetup', '--detach', self.loop_dev], check=False) run_program(['sudo', 'losetup', '--detach', self.loop_dev], check=False)
class RecoveryState(): class RecoveryState():
# pylint: disable=too-many-instance-attributes
"""Object to track BlockPair objects and overall state.""" """Object to track BlockPair objects and overall state."""
def __init__(self, mode, source, dest): def __init__(self, mode, source, dest):
self.mode = mode.lower() self.mode = mode.lower()
@ -270,6 +277,7 @@ class RecoveryState():
if mode not in ('clone', 'image'): if mode not in ('clone', 'image'):
raise GenericError('Unsupported mode') raise GenericError('Unsupported mode')
self.get_smart_source() self.get_smart_source()
self.set_working_dir()
def add_block_pair(self, source, dest): def add_block_pair(self, source, dest):
"""Run safety checks and append new BlockPair to internal list.""" """Run safety checks and append new BlockPair to internal list."""
@ -314,20 +322,134 @@ class RecoveryState():
# Safety checks passed # Safety checks passed
self.block_pairs.append(BlockPair(self.mode, source, dest)) self.block_pairs.append(BlockPair(self.mode, source, dest))
def build_outer_panes(self):
"""Build top and side panes."""
clear_screen()
# Top
self.panes['Source'] = tmux_split_window(
behind=True, vertical=True, lines=2,
text='{BLUE}Source{CLEAR}'.format(**COLORS))
# Started
self.panes['Started'] = tmux_split_window(
lines=SIDE_PANE_WIDTH, target_pane=self.panes['Source'],
text='{BLUE}Started{CLEAR}\n{s}'.format(
s=time.strftime("%Y-%m-%d %H:%M %Z"),
**COLORS))
# Destination
self.panes['Destination'] = tmux_split_window(
percent=50, target_pane=self.panes['Source'],
text='{BLUE}Destination{CLEAR}'.format(**COLORS))
# Progress
update_sidepane(self)
self.panes['Progress'] = tmux_split_window(
lines=SIDE_PANE_WIDTH, watch=self.progress_out)
def current_pass_done(self): def current_pass_done(self):
"""Checks if pass is done for all block-pairs, returns bool.""" """Checks if pass is done for all block-pairs, returns bool."""
done = True done = True
for bp in self.block_pairs: for b_pair in self.block_pairs:
done = done and bp.pass_done[self.current_pass] done = done and b_pair.pass_done[self.current_pass]
return done return done
def current_pass_min(self): def current_pass_min(self):
"""Gets minimum pass rescued percentage, returns float.""" """Gets minimum pass rescued percentage, returns float."""
min_percent = 100 min_percent = 100
for bp in self.block_pairs: for b_pair in self.block_pairs:
min_percent = min(min_percent, bp.rescued_percent) min_percent = min(min_percent, b_pair.rescued_percent)
return min_percent return min_percent
def fix_tmux_panes(self, forced=False):
# pylint: disable=too-many-branches,too-many-locals
"""Fix pane sizes if the winodw has been resized."""
needs_fixed = False
# Check layout
for pane, pane_data in TMUX_LAYOUT.items():
if not pane_data.get('Check'):
# Not concerned with the size of this pane
continue
# Get target
target = None
if pane != 'Current':
if pane not in self.panes:
# Skip missing panes
continue
else:
target = self.panes[pane]
# Check pane size
size_x, size_y = tmux_get_pane_size(pane_id=target)
if pane_data.get('x', False) and pane_data['x'] != size_x:
needs_fixed = True
if pane_data.get('y', False) and pane_data['y'] != size_y:
needs_fixed = True
# Bail?
if not needs_fixed and not forced:
return
# Remove Destination pane (temporarily)
tmux_kill_pane(self.panes['Destination'])
# Update layout
for pane, pane_data in TMUX_LAYOUT.items():
# Get target
target = None
if pane != 'Current':
if pane not in self.panes:
# Skip missing panes
continue
else:
target = self.panes[pane]
# Resize pane
tmux_resize_pane(pane_id=target, **pane_data)
# Calc Source/Destination pane sizes
width, height = tmux_get_pane_size()
width = int(width / 2) - 1
# Update Source string
source_str = self.source.name
if len(source_str) > width:
source_str = '{}...'.format(source_str[:width-3])
# Update Destination string
dest_str = self.dest.name
if len(dest_str) > width:
if self.mode == 'clone':
dest_str = '{}...'.format(dest_str[:width-3])
else:
dest_str = '...{}'.format(dest_str[-width+3:])
# Rebuild Source/Destination panes
tmux_update_pane(
pane_id=self.panes['Source'],
text='{BLUE}Source{CLEAR}\n{s}'.format(
s=source_str, **COLORS))
self.panes['Destination'] = tmux_split_window(
percent=50, target_pane=self.panes['Source'],
text='{BLUE}Destination{CLEAR}\n{s}'.format(
s=dest_str, **COLORS))
if 'SMART' in self.panes:
# Calc SMART/ddrescue/Journal panes sizes
ratio = [12, 22, 4]
width, height = tmux_get_pane_size(pane_id=self.panes['Progress'])
height -= 2
total = sum(ratio)
p_ratio = [int((x/total) * height) for x in ratio]
p_ratio[1] = height - p_ratio[0] - p_ratio[2]
# Resize SMART/Journal panes
tmux_resize_pane(self.panes['SMART'], y=ratio[0])
tmux_resize_pane(y=ratio[1])
tmux_resize_pane(self.panes['Journal'], y=ratio[2])
def get_smart_source(self): def get_smart_source(self):
"""Get source for SMART dispay.""" """Get source for SMART dispay."""
disk_path = self.source.path disk_path = self.source.path
@ -339,18 +461,15 @@ class RecoveryState():
def retry_all_passes(self): def retry_all_passes(self):
"""Mark all passes as pending for all block-pairs.""" """Mark all passes as pending for all block-pairs."""
self.finished = False self.finished = False
for bp in self.block_pairs: for b_pair in self.block_pairs:
bp.pass_done = [False, False, False] b_pair.pass_done = [False, False, False]
bp.status = ['Pending', 'Pending', 'Pending'] b_pair.status = ['Pending', 'Pending', 'Pending']
bp.fix_status_strings() b_pair.fix_status_strings()
self.set_pass_num() self.set_pass_num()
def self_checks(self): def self_checks(self):
"""Run self-checks and update state values.""" """Run self-checks and update state values."""
cmd = ['findmnt', '--json', '--target', os.getcwd()] cmd = ['findmnt', '--json', '--target', os.getcwd()]
map_allowed_fstypes = RECOMMENDED_FSTYPES.copy()
map_allowed_fstypes.extend(['cifs', 'ext2', 'vfat'])
map_allowed_fstypes.sort()
json_data = get_json_from_command(cmd) json_data = get_json_from_command(cmd)
# Abort if json_data is empty # Abort if json_data is empty
@ -361,24 +480,24 @@ class RecoveryState():
# Avoid saving map to non-persistent filesystem # Avoid saving map to non-persistent filesystem
fstype = json_data.get( fstype = json_data.get(
'filesystems', [{}])[0].get( 'filesystems', [{}])[0].get(
'fstype', 'unknown') 'fstype', 'unknown')
if fstype not in map_allowed_fstypes: if fstype not in RECOMMENDED_MAP_FSTYPES:
print_error( print_error(
"Map isn't being saved to a recommended filesystem ({})".format( "Map isn't being saved to a recommended filesystem ({})".format(
fstype.upper())) fstype.upper()))
print_info('Recommended types are: {}'.format( print_info('Recommended types are: {}'.format(
' / '.join(map_allowed_fstypes).upper())) ' / '.join(RECOMMENDED_MAP_FSTYPES).upper()))
print_standard(' ') print_standard(' ')
if not ask('Proceed anyways? (Strongly discouraged)'): if not ask('Proceed anyways? (Strongly discouraged)'):
raise GenericAbort() raise GenericAbort()
# Run BlockPair self checks and get total size # Run BlockPair self checks and get total size
self.total_size = 0 self.total_size = 0
for bp in self.block_pairs: for b_pair in self.block_pairs:
bp.self_check() b_pair.self_check()
if bp.resumed: if b_pair.resumed:
self.resumed = True self.resumed = True
self.total_size += bp.size self.total_size += b_pair.size
def set_pass_num(self): def set_pass_num(self):
"""Set current pass based on all block-pair's progress.""" """Set current pass based on all block-pair's progress."""
@ -386,8 +505,8 @@ class RecoveryState():
for pass_num in (2, 1, 0): for pass_num in (2, 1, 0):
# Iterate backwards through passes # Iterate backwards through passes
pass_done = True pass_done = True
for bp in self.block_pairs: for b_pair in self.block_pairs:
pass_done = pass_done and bp.pass_done[pass_num] pass_done = pass_done and b_pair.pass_done[pass_num]
if pass_done: if pass_done:
# All block-pairs reported being done # All block-pairs reported being done
# Set to next pass, unless we're on the last pass (2) # Set to next pass, unless we're on the last pass (2)
@ -405,6 +524,34 @@ class RecoveryState():
elif self.current_pass == 2: elif self.current_pass == 2:
self.current_pass_str = '3 "Scraping bad areas"' self.current_pass_str = '3 "Scraping bad areas"'
def set_working_dir(self):
# pylint: disable=no-self-use
"""Set working dir to MAP_DIR if possible.
NOTE: This is to help ensure the map file
is saved to non-volatile storage."""
map_dir = '{}/{}'.format(MAP_DIR, global_vars['Date-Time'])
# Mount backup shares
mount_backup_shares(read_write=True)
# Get MAP_DIR filesystem type
# NOTE: If the backup share fails to mount then this will
# likely be the type of /
cmd = [
'findmnt',
'--noheadings',
'--target', MAP_DIR,
'--output', 'FSTYPE',
]
result = run_program(cmd, check=False, encoding='utf-8', errors='ingnore')
map_dir_type = result.stdout.strip().lower()
# Change working dir if map_dir_type is acceptable
if map_dir_type in RECOMMENDED_MAP_FSTYPES:
os.makedirs(map_dir, exist_ok=True)
os.chdir(map_dir)
def update_etoc(self): def update_etoc(self):
"""Search ddrescue output for the current EToC, returns str.""" """Search ddrescue output for the current EToC, returns str."""
now = datetime.datetime.now(tz=self.timezone) now = datetime.datetime.now(tz=self.timezone)
@ -414,7 +561,7 @@ class RecoveryState():
# Just set to N/A (NOTE: this overrules the refresh rate below) # Just set to N/A (NOTE: this overrules the refresh rate below)
self.etoc = 'N/A' self.etoc = 'N/A'
return return
elif 'In Progress' not in self.status: if 'In Progress' not in self.status:
# Don't update when EToC is hidden # Don't update when EToC is hidden
return return
if now.second % ETOC_REFRESH_RATE != 0: if now.second % ETOC_REFRESH_RATE != 0:
@ -428,13 +575,14 @@ class RecoveryState():
# Capture main tmux pane # Capture main tmux pane
try: try:
text = tmux_capture_pane() text = tmux_capture_pane()
except Exception: except Exception: # pylint: disable=broad-except
# Ignore # Ignore
pass pass
# Search for EToC delta # Search for EToC delta
matches = re.findall(r'remaining time:.*$', text, re.MULTILINE) matches = re.findall(r'remaining time:.*$', text, re.MULTILINE)
if matches: if matches:
# pylint: disable=invalid-name
r = REGEX_REMAINING_TIME.search(matches[-1]) r = REGEX_REMAINING_TIME.search(matches[-1])
if r.group('na'): if r.group('na'):
self.etoc = 'N/A' self.etoc = 'N/A'
@ -451,7 +599,7 @@ class RecoveryState():
minutes=int(minutes), minutes=int(minutes),
seconds=int(seconds), seconds=int(seconds),
) )
except Exception: except Exception: # pylint: disable=broad-except
# Ignore and leave as raw string # Ignore and leave as raw string
pass pass
@ -461,15 +609,16 @@ class RecoveryState():
now = datetime.datetime.now(tz=self.timezone) now = datetime.datetime.now(tz=self.timezone)
_etoc = now + etoc_delta _etoc = now + etoc_delta
self.etoc = _etoc.strftime('%Y-%m-%d %H:%M %Z') self.etoc = _etoc.strftime('%Y-%m-%d %H:%M %Z')
except Exception: except Exception: # pylint: disable=broad-except
# Ignore and leave as current string # Ignore and leave as current string
pass pass
def update_progress(self): def update_progress(self):
# pylint: disable=attribute-defined-outside-init
"""Update overall progress using block_pairs.""" """Update overall progress using block_pairs."""
self.rescued = 0 self.rescued = 0
for bp in self.block_pairs: for b_pair in self.block_pairs:
self.rescued += bp.rescued self.rescued += b_pair.rescued
self.rescued_percent = (self.rescued / self.total_size) * 100 self.rescued_percent = (self.rescued / self.total_size) * 100
self.status_percent = get_formatted_status( self.status_percent = get_formatted_status(
label='Recovered:', data=self.rescued_percent) label='Recovered:', data=self.rescued_percent)
@ -478,26 +627,6 @@ class RecoveryState():
# Functions # Functions
def build_outer_panes(state):
"""Build top and side panes."""
state.panes['Source'] = tmux_split_window(
behind=True, vertical=True, lines=2,
text='{BLUE}Source{CLEAR}'.format(**COLORS))
state.panes['Started'] = tmux_split_window(
lines=SIDE_PANE_WIDTH, target_pane=state.panes['Source'],
text='{BLUE}Started{CLEAR}\n{s}'.format(
s=time.strftime("%Y-%m-%d %H:%M %Z"),
**COLORS))
state.panes['Destination'] = tmux_split_window(
percent=50, target_pane=state.panes['Source'],
text='{BLUE}Destination{CLEAR}'.format(**COLORS))
# Side pane
update_sidepane(state)
state.panes['Progress'] = tmux_split_window(
lines=SIDE_PANE_WIDTH, watch=state.progress_out)
def create_path_obj(path): def create_path_obj(path):
"""Create Dev, Dir, or Image obj based on path given.""" """Create Dev, Dir, or Image obj based on path given."""
obj = None obj = None
@ -515,101 +644,16 @@ def create_path_obj(path):
def double_confirm_clone(): def double_confirm_clone():
"""Display warning and get 2nd confirmation, returns bool.""" """Display warning and get 2nd confirmation, returns bool."""
print_standard('\nSAFETY CHECK') print_standard('\nSAFETY CHECK')
print_warning('All data will be DELETED from the ' print_warning(
'destination device and partition(s) listed above.') 'All data will be DELETED from the '
print_warning('This is irreversible and will lead ' 'destination device and partition(s) listed above.'
'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) )
print_warning(
'This is irreversible and will lead to {CLEAR}{RED}DATA LOSS.'.format(
**COLORS))
return ask('Asking again to confirm, is this correct?') return ask('Asking again to confirm, is this correct?')
def fix_tmux_panes(state, forced=False):
"""Fix pane sizes if the winodw has been resized."""
needs_fixed = False
# Check layout
for k, v in TMUX_LAYOUT.items():
if not v.get('Check'):
# Not concerned with the size of this pane
continue
# Get target
target = None
if k != 'Current':
if k not in state.panes:
# Skip missing panes
continue
else:
target = state.panes[k]
# Check pane size
x, y = tmux_get_pane_size(pane_id=target)
if v.get('x', False) and v['x'] != x:
needs_fixed = True
if v.get('y', False) and v['y'] != y:
needs_fixed = True
# Bail?
if not needs_fixed and not forced:
return
# Remove Destination pane (temporarily)
tmux_kill_pane(state.panes['Destination'])
# Update layout
for k, v in TMUX_LAYOUT.items():
# Get target
target = None
if k != 'Current':
if k not in state.panes:
# Skip missing panes
continue
else:
target = state.panes[k]
# Resize pane
tmux_resize_pane(pane_id=target, **v)
# Calc Source/Destination pane sizes
width, height = tmux_get_pane_size()
width = int(width / 2) - 1
# Update Source string
source_str = state.source.name
if len(source_str) > width:
source_str = '{}...'.format(source_str[:width-3])
# Update Destination string
dest_str = state.dest.name
if len(dest_str) > width:
if state.mode == 'clone':
dest_str = '{}...'.format(dest_str[:width-3])
else:
dest_str = '...{}'.format(dest_str[-width+3:])
# Rebuild Source/Destination panes
tmux_update_pane(
pane_id=state.panes['Source'],
text='{BLUE}Source{CLEAR}\n{s}'.format(
s=source_str, **COLORS))
state.panes['Destination'] = tmux_split_window(
percent=50, target_pane=state.panes['Source'],
text='{BLUE}Destination{CLEAR}\n{s}'.format(
s=dest_str, **COLORS))
if 'SMART' in state.panes:
# Calc SMART/ddrescue/Journal panes sizes
ratio = [12, 22, 4]
width, height = tmux_get_pane_size(pane_id=state.panes['Progress'])
height -= 2
total = sum(ratio)
p_ratio = [int((x/total) * height) for x in ratio]
p_ratio[1] = height - p_ratio[0] - p_ratio[2]
# Resize SMART/Journal panes
tmux_resize_pane(state.panes['SMART'], y=ratio[0])
tmux_resize_pane(y=ratio[1])
tmux_resize_pane(state.panes['Journal'], y=ratio[2])
def get_device_details(dev_path): def get_device_details(dev_path):
"""Get device details via lsblk, returns JSON dict.""" """Get device details via lsblk, returns JSON dict."""
cmd = ['lsblk', '--json', '--output-all', '--paths', dev_path] cmd = ['lsblk', '--json', '--output-all', '--paths', dev_path]
@ -678,22 +722,22 @@ def get_dir_report(dir_path):
output.append('{BLUE}{label:<{width}}{line}{CLEAR}'.format( output.append('{BLUE}{label:<{width}}{line}{CLEAR}'.format(
label='PATH', label='PATH',
width=width, width=width,
line=line.replace('\n',''), line=line.replace('\n', ''),
**COLORS)) **COLORS))
else: else:
output.append('{path:<{width}}{line}'.format( output.append('{path:<{width}}{line}'.format(
path=dir_path, path=dir_path,
width=width, width=width,
line=line.replace('\n',''))) line=line.replace('\n', '')))
# Done # Done
return '\n'.join(output) return '\n'.join(output)
def get_size_in_bytes(s): def get_size_in_bytes(size):
"""Convert size string from lsblk string to bytes, returns int.""" """Convert size string from lsblk string to bytes, returns int."""
s = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', s, re.IGNORECASE) size = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', size, re.IGNORECASE)
return convert_to_bytes(s) return convert_to_bytes(size)
def get_formatted_status(label, data): def get_formatted_status(label, data):
@ -701,13 +745,15 @@ def get_formatted_status(label, data):
data_width = SIDE_PANE_WIDTH - len(label) data_width = SIDE_PANE_WIDTH - len(label)
try: try:
data_str = '{data:>{data_width}.2f} %'.format( data_str = '{data:>{data_width}.2f} %'.format(
data=data, data=data,
data_width=data_width-2) data_width=data_width-2,
)
except ValueError: except ValueError:
# Assuming non-numeric data # Assuming non-numeric data
data_str = '{data:>{data_width}}'.format( data_str = '{data:>{data_width}}'.format(
data=data, data=data,
data_width=data_width) data_width=data_width,
)
status = '{label}{s_color}{data_str}{CLEAR}'.format( status = '{label}{s_color}{data_str}{CLEAR}'.format(
label=label, label=label,
s_color=get_status_color(data), s_color=get_status_color(data),
@ -716,19 +762,19 @@ def get_formatted_status(label, data):
return status return status
def get_status_color(s, t_success=99, t_warn=90): def get_status_color(status, t_success=99, t_warn=90):
"""Get color based on status, returns str.""" """Get color based on status, returns str."""
color = COLORS['CLEAR'] color = COLORS['CLEAR']
p_recovered = -1 p_recovered = -1
try: try:
p_recovered = float(s) p_recovered = float(status)
except ValueError: except ValueError:
# Status is either in lists below or will default to red # Status is either in lists below or will default to red
pass pass
if s in ('Pending',) or str(s)[-2:] in (' b', 'Kb', 'Mb', 'Gb', 'Tb'): if status == 'Pending' or str(status)[-2:] in (' b', 'Kb', 'Mb', 'Gb', 'Tb'):
color = COLORS['CLEAR'] color = COLORS['CLEAR']
elif s in ('Skipped', 'Unknown'): elif status in ('Skipped', 'Unknown'):
color = COLORS['YELLOW'] color = COLORS['YELLOW']
elif p_recovered >= t_success: elif p_recovered >= t_success:
color = COLORS['GREEN'] color = COLORS['GREEN']
@ -755,6 +801,7 @@ def is_writable_filesystem(dir_obj):
def menu_ddrescue(source_path, dest_path, run_mode): def menu_ddrescue(source_path, dest_path, run_mode):
# pylint: disable=too-many-branches
"""ddrescue menu.""" """ddrescue menu."""
source = None source = None
dest = None dest = None
@ -798,9 +845,8 @@ def menu_ddrescue(source_path, dest_path, run_mode):
raise GenericAbort() raise GenericAbort()
# Main menu # Main menu
clear_screen() state.build_outer_panes()
build_outer_panes(state) state.fix_tmux_panes(forced=True)
fix_tmux_panes(state, forced=True)
menu_main(state) menu_main(state)
# Done # Done
@ -809,6 +855,7 @@ def menu_ddrescue(source_path, dest_path, run_mode):
def menu_main(state): def menu_main(state):
# pylint: disable=too-many-branches,too-many-statements
"""Main menu is used to set ddrescue settings.""" """Main menu is used to set ddrescue settings."""
checkmark = '*' checkmark = '*'
if 'DISPLAY' in global_vars['Env']: if 'DISPLAY' in global_vars['Env']:
@ -819,16 +866,15 @@ def menu_main(state):
# Build menu # Build menu
main_options = [ main_options = [
{'Base Name': 'Auto continue (if recovery % over threshold)', {'Base Name': 'Auto continue (if recovery % over threshold)',
'Enabled': True}, 'Enabled': True},
{'Base Name': 'Retry (mark non-rescued sectors "non-tried")', {'Base Name': 'Retry (mark non-rescued sectors "non-tried")',
'Enabled': False}, 'Enabled': False},
{'Base Name': 'Reverse direction', 'Enabled': False}, {'Base Name': 'Reverse direction', 'Enabled': False},
] ]
actions = [ actions = [
{'Name': 'Start', 'Letter': 'S'}, {'Name': 'Start', 'Letter': 'S'},
{'Name': 'Change settings {YELLOW}(experts only){CLEAR}'.format( {'Name': 'Change settings {YELLOW}(experts only){CLEAR}'.format(**COLORS),
**COLORS), 'Letter': 'C'},
'Letter': 'C'},
{'Name': 'Quit', 'Letter': 'Q', 'CRLF': True}, {'Name': 'Quit', 'Letter': 'Q', 'CRLF': True},
] ]
@ -859,13 +905,13 @@ def menu_main(state):
elif selection == 'S': elif selection == 'S':
# Set settings for pass # Set settings for pass
pass_settings = [] pass_settings = []
for k, v in state.settings.items(): for option, option_data in state.settings.items():
if not v['Enabled']: if not option_data['Enabled']:
continue continue
if 'Value' in v: if 'Value' in option_data:
pass_settings.append('{}={}'.format(k, v['Value'])) pass_settings.append('{}={}'.format(option, option_data['Value']))
else: else:
pass_settings.append(k) pass_settings.append(option)
for opt in main_options: for opt in main_options:
if 'Auto' in opt['Base Name']: if 'Auto' in opt['Base Name']:
auto_run = opt['Enabled'] auto_run = opt['Enabled']
@ -888,7 +934,7 @@ def menu_main(state):
state.current_pass_min() < AUTO_PASS_1_THRESHOLD): state.current_pass_min() < AUTO_PASS_1_THRESHOLD):
auto_run = False auto_run = False
elif (state.current_pass == 1 and elif (state.current_pass == 1 and
state.current_pass_min() < AUTO_PASS_2_THRESHOLD): state.current_pass_min() < AUTO_PASS_2_THRESHOLD):
auto_run = False auto_run = False
else: else:
auto_run = False auto_run = False
@ -917,13 +963,15 @@ def menu_settings(state):
# Build menu # Build menu
settings = [] settings = []
for k, v in sorted(state.settings.items()): for option, option_data in sorted(state.settings.items()):
if not v.get('Hidden', False): if not option_data.get('Hidden', False):
settings.append({'Base Name': k, 'Flag': k}) settings.append({'Base Name': option, 'Flag': option})
actions = [{'Name': 'Main Menu', 'Letter': 'M'}] actions = [{'Name': 'Main Menu', 'Letter': 'M'}]
# Show menu # Show menu
while True: while True:
# pylint: disable=invalid-name
# TODO: Clean up and/or replace with new menu-select function
for s in settings: for s in settings:
s['Name'] = '{}{}{}'.format( s['Name'] = '{}{}{}'.format(
s['Base Name'], s['Base Name'],
@ -960,25 +1008,27 @@ def menu_settings(state):
def read_map_file(map_path): def read_map_file(map_path):
"""Read map file with ddrescuelog and return data as dict.""" """Read map file with ddrescuelog and return data as dict."""
map_data = {'full recovery': False} cmd = [
'ddrescuelog',
'--binary-prefixes',
'--show-status',
map_path,
]
map_data = {'full recovery': False, 'pass completed': False}
try: try:
result = run_program(['ddrescuelog', '-t', map_path]) result = run_program(cmd, encoding='utf-8', errors='ignore')
except CalledProcessError: except CalledProcessError:
# (Grossly) assuming map_data hasn't been saved yet, return empty dict # (Grossly) assuming map_data hasn't been saved yet, return empty dict
return map_data return map_data
# Parse output # Parse output
for line in result.stdout.decode().splitlines(): for line in result.stdout.splitlines():
m = re.match( line = line.strip()
r'^\s*(?P<key>\S+):.*\(\s*(?P<value>\d+\.?\d*)%.*', line.strip()) _r = REGEX_DDRESCUE_LOG.search(line)
if m: if _r:
try: map_data[_r.group('key')] = convert_to_bytes('{size} {unit}B'.format(
map_data[m.group('key')] = float(m.group('value')) **_r.groupdict()))
except ValueError: map_data['pass completed'] = 'current status: finished' in line
raise GenericError('Failed to read map data')
m = re.match(r'.*current status:\s+(?P<status>.*)', line.strip())
if m:
map_data['pass completed'] = bool(m.group('status') == 'finished')
# Check if 100% done # Check if 100% done
try: try:
@ -992,6 +1042,7 @@ def read_map_file(map_path):
def run_ddrescue(state, pass_settings): def run_ddrescue(state, pass_settings):
# pylint: disable=too-many-branches,too-many-statements
"""Run ddrescue pass.""" """Run ddrescue pass."""
return_code = -1 return_code = -1
aborted = False aborted = False
@ -1006,8 +1057,8 @@ def run_ddrescue(state, pass_settings):
# Create SMART monitor pane # Create SMART monitor pane
state.smart_out = '{}/smart_{}.out'.format( state.smart_out = '{}/smart_{}.out'.format(
global_vars['TmpDir'], state.smart_source.name) global_vars['TmpDir'], state.smart_source.name)
with open(state.smart_out, 'w') as f: with open(state.smart_out, 'w') as _f:
f.write('Initializing...') _f.write('Initializing...')
state.panes['SMART'] = tmux_split_window( state.panes['SMART'] = tmux_split_window(
behind=True, lines=12, vertical=True, watch=state.smart_out) behind=True, lines=12, vertical=True, watch=state.smart_out)
@ -1017,19 +1068,19 @@ def run_ddrescue(state, pass_settings):
command=['sudo', 'journalctl', '-f']) command=['sudo', 'journalctl', '-f'])
# Fix layout # Fix layout
fix_tmux_panes(state, forced=True) state.fix_tmux_panes(forced=True)
# Run pass for each block-pair # Run pass for each block-pair
for bp in state.block_pairs: for b_pair in state.block_pairs:
if bp.pass_done[state.current_pass]: if b_pair.pass_done[state.current_pass]:
# Skip to next block-pair # Skip to next block-pair
continue continue
update_sidepane(state) update_sidepane(state)
# Set ddrescue cmd # Set ddrescue cmd
cmd = [ cmd = [
'ddrescue', *pass_settings, 'sudo', 'ddrescue', *pass_settings,
bp.source_path, bp.dest_path, bp.map_path] b_pair.source_path, b_pair.dest_path, b_pair.map_path]
if state.mode == 'clone': if state.mode == 'clone':
cmd.append('--force') cmd.append('--force')
if state.current_pass == 0: if state.current_pass == 0:
@ -1044,36 +1095,36 @@ def run_ddrescue(state, pass_settings):
# Start ddrescue # Start ddrescue
try: try:
clear_screen() clear_screen()
print_info('Current dev: {}'.format(bp.source_path)) print_info('Current dev: {}'.format(b_pair.source_path))
ddrescue_proc = popen_program(cmd) ddrescue_proc = popen_program(cmd)
i = 0 i = 0
while True: while True:
# Update SMART display (every 30 seconds) # Update SMART display (every 30 seconds)
if i % 30 == 0: if i % 30 == 0:
state.smart_source.get_smart_details() state.smart_source.get_smart_details()
with open(state.smart_out, 'w') as f: with open(state.smart_out, 'w') as _f:
report = state.smart_source.generate_attribute_report( report = state.smart_source.generate_attribute_report(
timestamp=True) timestamp=True)
for line in report: for line in report:
f.write('{}\n'.format(line)) _f.write('{}\n'.format(line))
i += 1 i += 1
# Update progress # Update progress
bp.update_progress(state.current_pass) b_pair.update_progress(state.current_pass)
update_sidepane(state) update_sidepane(state)
# Fix panes # Fix panes
fix_tmux_panes(state) state.fix_tmux_panes()
# Check if ddrescue has finished # Check if ddrescue has finished
try: try:
ddrescue_proc.wait(timeout=1) ddrescue_proc.wait(timeout=1)
sleep(2) sleep(2)
bp.update_progress(state.current_pass) b_pair.update_progress(state.current_pass)
update_sidepane(state) update_sidepane(state)
break break
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
# Catch to update smart/bp/sidepane # Catch to update smart/b_pair/sidepane
pass pass
except KeyboardInterrupt: except KeyboardInterrupt:
@ -1082,7 +1133,7 @@ def run_ddrescue(state, pass_settings):
ddrescue_proc.wait(timeout=10) ddrescue_proc.wait(timeout=10)
# Update progress/sidepane again # Update progress/sidepane again
bp.update_progress(state.current_pass) b_pair.update_progress(state.current_pass)
update_sidepane(state) update_sidepane(state)
# Was ddrescue aborted? # Was ddrescue aborted?
@ -1104,7 +1155,7 @@ def run_ddrescue(state, pass_settings):
break break
else: else:
# Mark pass finished # Mark pass finished
bp.finish_pass(state.current_pass) b_pair.finish_pass(state.current_pass)
update_sidepane(state) update_sidepane(state)
# Done # Done
@ -1120,6 +1171,8 @@ def run_ddrescue(state, pass_settings):
def select_parts(source_device): def select_parts(source_device):
# pylint: disable=too-many-branches
# TODO: Clean up and/or replace with new menu-select function
"""Select partition(s) or whole device, returns list of DevObj()s.""" """Select partition(s) or whole device, returns list of DevObj()s."""
selected_parts = [] selected_parts = []
children = source_device.details.get('children', []) children = source_device.details.get('children', [])
@ -1181,24 +1234,26 @@ def select_parts(source_device):
raise GenericAbort() raise GenericAbort()
# Build list of selected parts # Build list of selected parts
for d in dev_options: for _d in dev_options:
if d['Selected']: if _d['Selected']:
d['Dev'].model = source_device.model _d['Dev'].model = source_device.model
d['Dev'].model_size = source_device.model_size _d['Dev'].model_size = source_device.model_size
d['Dev'].update_filename_prefix() _d['Dev'].update_filename_prefix()
selected_parts.append(d['Dev']) selected_parts.append(_d['Dev'])
return selected_parts return selected_parts
def select_path(skip_device=None): def select_path(skip_device=None):
# pylint: disable=too-many-branches,too-many-locals
# TODO: Clean up and/or replace with new menu-select function
"""Optionally mount local dev and select path, returns DirObj.""" """Optionally mount local dev and select path, returns DirObj."""
wd = os.path.realpath(global_vars['Env']['PWD']) work_dir = os.path.realpath(global_vars['Env']['PWD'])
selected_path = None selected_path = None
# Build menu # Build menu
path_options = [ path_options = [
{'Name': 'Current directory: {}'.format(wd), 'Path': wd}, {'Name': 'Current directory: {}'.format(work_dir), 'Path': work_dir},
{'Name': 'Local device', 'Path': None}, {'Name': 'Local device', 'Path': None},
{'Name': 'Enter manually', 'Path': None}] {'Name': 'Enter manually', 'Path': None}]
actions = [{'Name': 'Quit', 'Letter': 'Q'}] actions = [{'Name': 'Quit', 'Letter': 'Q'}]
@ -1213,9 +1268,9 @@ def select_path(skip_device=None):
raise GenericAbort() raise GenericAbort()
elif selection.isnumeric(): elif selection.isnumeric():
index = int(selection) - 1 index = int(selection) - 1
if path_options[index]['Path'] == wd: if path_options[index]['Path'] == work_dir:
# Current directory # Current directory
selected_path = DirObj(wd) selected_path = DirObj(work_dir)
elif path_options[index]['Name'] == 'Local device': elif path_options[index]['Name'] == 'Local device':
# Local device # Local device
@ -1231,15 +1286,15 @@ def select_path(skip_device=None):
# Select volume # Select volume
vol_options = [] vol_options = []
for k, v in sorted(report.items()): for _k, _v in sorted(report.items()):
disabled = v['show_data']['data'] == 'Failed to mount' disabled = _v['show_data']['data'] == 'Failed to mount'
if disabled: if disabled:
name = '{name} (Failed to mount)'.format(**v) name = '{name} (Failed to mount)'.format(**_v)
else: else:
name = '{name} (mounted on "{mount_point}")'.format(**v) name = '{name} (mounted on "{mount_point}")'.format(**_v)
vol_options.append({ vol_options.append({
'Name': name, 'Name': name,
'Path': v['mount_point'], 'Path': _v['mount_point'],
'Disabled': disabled}) 'Disabled': disabled})
selection = menu_select( selection = menu_select(
title='Please select a volume', title='Please select a volume',
@ -1314,15 +1369,17 @@ def select_device(description='device', skip_device=None):
action_entries=actions, action_entries=actions,
disabled_label='ALREADY SELECTED') disabled_label='ALREADY SELECTED')
if selection == 'Q':
raise GenericAbort()
if selection.isnumeric(): if selection.isnumeric():
return dev_options[int(selection)-1]['Dev'] return dev_options[int(selection)-1]['Dev']
elif selection == 'Q':
raise GenericAbort()
def setup_loopback_device(source_path): def setup_loopback_device(source_path):
"""Setup loopback device for source_path, returns dev_path as str.""" """Setup loopback device for source_path, returns dev_path as str."""
cmd = ( cmd = (
'sudo',
'losetup', 'losetup',
'--find', '--find',
'--partscan', '--partscan',
@ -1356,6 +1413,7 @@ def show_selection_details(state):
def show_usage(script_name): def show_usage(script_name):
"""Show usage."""
print_info('Usage:') print_info('Usage:')
print_standard(USAGE.format(script_name=script_name)) print_standard(USAGE.format(script_name=script_name))
pause() pause()
@ -1379,14 +1437,14 @@ def update_sidepane(state):
output.append('─────────────────────') output.append('─────────────────────')
# Source(s) progress # Source(s) progress
for bp in state.block_pairs: for b_pair in state.block_pairs:
if state.source.is_image(): if state.source.is_image():
output.append('{BLUE}Image File{CLEAR}'.format(**COLORS)) output.append('{BLUE}Image File{CLEAR}'.format(**COLORS))
else: else:
output.append('{BLUE}{source}{CLEAR}'.format( output.append('{BLUE}{source}{CLEAR}'.format(
source=bp.source_path, source=b_pair.source_path,
**COLORS)) **COLORS))
output.extend(bp.status) output.extend(b_pair.status)
output.append(' ') output.append(' ')
# EToC # EToC
@ -1405,11 +1463,9 @@ def update_sidepane(state):
# Add line-endings # Add line-endings
output = ['{}\n'.format(line) for line in output] output = ['{}\n'.format(line) for line in output]
with open(state.progress_out, 'w') as f: with open(state.progress_out, 'w') as _f:
f.writelines(output) _f.writelines(output)
if __name__ == '__main__': if __name__ == '__main__':
print("This file is not meant to be called directly.") print("This file is not meant to be called directly.")
# vim: sts=2 sw=2 ts=2

8
.bin/Scripts/settings/ddrescue.py
View file

@ -5,7 +5,9 @@ import re
from collections import OrderedDict from collections import OrderedDict
# General # General
MAP_DIR = '/Backups/ddrescue-tui'
RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs']
RECOMMENDED_MAP_FSTYPES = ['cifs', 'ext2', 'ext3', 'ext4', 'vfat', 'xfs']
USAGE = """ {script_name} clone [source [destination]] USAGE = """ {script_name} clone [source [destination]]
{script_name} image [source [destination]] {script_name} image [source [destination]]
(e.g. {script_name} clone /dev/sda /dev/sdb) (e.g. {script_name} clone /dev/sda /dev/sdb)
@ -36,6 +38,12 @@ DDRESCUE_SETTINGS = {
'-vvvv': {'Enabled': True, 'Hidden': True, }, '-vvvv': {'Enabled': True, 'Hidden': True, },
} }
ETOC_REFRESH_RATE = 30 # in seconds ETOC_REFRESH_RATE = 30 # in seconds
REGEX_DDRESCUE_LOG = re.compile(
r'^\s*(?P<key>\S+):\s+'
r'(?P<size>\d+)\s+'
r'(?P<unit>[PTGMKB])i?B?',
re.IGNORECASE,
)
REGEX_REMAINING_TIME = re.compile( REGEX_REMAINING_TIME = re.compile(
r'remaining time:' r'remaining time:'
r'\s*((?P<days>\d+)d)?' r'\s*((?P<days>\d+)d)?'