Merge branch 'new-ddrescue-arguments' into dev

scripts/wk/cfg/ddrescue.py

@@ -16,22 +16,30 @@ TMUX_LAYOUT = OrderedDict({
 # ddrescue
 AUTO_PASS_THRESHOLDS = {
   # NOTE: The scrape key is set to infinity to force a break
-  'read': 95,
+  'read-skip':  50,
-  'trim': 98,
+  'read-full':  95,
-  'scrape': float('inf'),
+  'trim':       98,
+  'scrape':     float('inf'),
   }
+DDRESCUE_MAP_TEMPLATE = '''# Mapfile. Created by {name}
+0x0 ? 1
+0x0 {size:#x} ?
+'''
 DDRESCUE_SETTINGS = {
   'Default': {
     '--binary-prefixes':  {'Selected': True,                'Hidden': True, },
+    '--complete-only':    {'Selected': True,                'Hidden': True, },
     '--data-preview':     {'Selected': True,  'Value': '5', 'Hidden': True, },
     '--idirect':          {'Selected': True,                                },
     '--odirect':          {'Selected': True,                                },
+    '--input-position':   {'Selected': False, 'Value': '0',                 },
     '--max-error-rate':   {'Selected': True,  'Value': '100MiB',            },
     '--max-read-rate':    {'Selected': False, 'Value': '1MiB',              },
     '--min-read-rate':    {'Selected': True,  'Value': '64KiB',             },
-    '--reopen-on-error':  {'Selected': True,                                },
+    '--reopen-on-error':  {'Selected': False,                               },
     '--retry-passes':     {'Selected': True,  'Value': '0',                 },
     '--reverse':          {'Selected': False,                               },
+    '--skip-size':        {'Selected': True,  'Value': '0.001,0.05',        },  # Percentages of source size
     '--test-mode':        {'Selected': False, 'Value': 'test.map',          },
     '--timeout':          {'Selected': True,  'Value': '30m',               },
     '-vvvv':              {'Selected': True,                'Hidden': True, },
@@ -39,16 +47,19 @@ DDRESCUE_SETTINGS = {
   'Fast': {
     '--max-error-rate':   {'Selected': True,  'Value': '32MiB',             },
     '--min-read-rate':    {'Selected': True,  'Value': '1MiB',              },
-    '--reopen-on-error':  {'Selected': False,                               },
     '--timeout':          {'Selected': True,  'Value': '5m',                },
     },
   'Safe': {
     '--max-read-rate':    {'Selected': True,  'Value': '64MiB',             },
     '--min-read-rate':    {'Selected': True,  'Value': '1KiB',              },
-    '--reopen-on-error':  {'Selected': True,                                },
     '--timeout':          {'Selected': False, 'Value': '30m',               },
     },
   }
+DDRESCUE_SPECIFIC_PASS_SETTINGS = {
+  'read-skip':  ['--no-scrape', '--no-trim', '--cpass=1,2'],
+  'read-full':  ['--no-scrape', '--no-trim'],
+  'trim':       ['--no-scrape'],
+  }
 DRIVE_POWEROFF_TIMEOUT = 90
 PARTITION_TYPES = {
   'GPT': {

scripts/wk/hw/ddrescue.py

@@ -22,7 +22,11 @@ import psutil
 import pytz
 
 from wk import cfg, debug, exe, io, log, net, std, tmux
-from wk.cfg.ddrescue import DDRESCUE_SETTINGS
+from wk.cfg.ddrescue import (
+  DDRESCUE_MAP_TEMPLATE,
+  DDRESCUE_SETTINGS,
+  DDRESCUE_SPECIFIC_PASS_SETTINGS,
+  )
 from wk.hw import obj as hw_obj
 
 
@@ -67,6 +71,7 @@ DDRESCUE_LOG_REGEX = re.compile(
   r'.*\(\s*(?P<percent>\d+\.?\d*)%\)$',
   re.IGNORECASE,
   )
+INITIAL_SKIP_MIN = 64 * 1024 # This is ddrescue's minimum accepted value
 REGEX_REMAINING_TIME = re.compile(
   r'remaining time:'
   r'\s*((?P<days>\d+)d)?'
@@ -104,11 +109,11 @@ SETTING_PRESETS = (
   'Safe',
   )
 STATUS_COLORS = {
-  'Passed': 'GREEN',
+  'Passed':   'GREEN',
-  'Aborted': 'YELLOW',
+  'Aborted':  'YELLOW',
-  'Skipped': 'YELLOW',
+  'Skipped':  'YELLOW',
-  'Working': 'YELLOW',
+  'Working':  'YELLOW',
-  'ERROR': 'RED',
+  'ERROR':    'RED',
   }
 TIMEZONE = pytz.timezone(cfg.main.LINUX_TIME_ZONE)
 
@@ -116,29 +121,28 @@ TIMEZONE = pytz.timezone(cfg.main.LINUX_TIME_ZONE)
 # Classes
 class BlockPair():
   """Object for tracking source to dest recovery data."""
+  # pylint: disable=too-many-instance-attributes
   def __init__(self, source, destination, model, working_dir):
     """Initialize BlockPair()
 
     NOTE: source should be a wk.hw.obj.Disk() object
           and destination should be a pathlib.Path() object.
     """
+    self.sector_size = source.details.get('phy-sec', 512)
     self.source = source.path
     self.destination = destination
     self.map_data = {}
     self.map_path = None
     self.size = source.details['size']
     self.status = OrderedDict({
-      'read': 'Pending',
+      'read-skip':  'Pending',
-      'trim': 'Pending',
+      'read-full':  'Pending',
-      'scrape': 'Pending',
+      'trim':       'Pending',
+      'scrape':     'Pending',
       })
-    self.view_proc = 'Disabled'
+    self.view_map = 'DISPLAY' in os.environ or 'WAYLAND_DISPLAY' in os.environ
-    if 'DISPLAY' in os.environ or 'WAYLAND_DISPLAY' in os.environ:
-      # Enable opening ddrescueview during recovery
-      self.view_proc = None
 
-
+    # Set map path
-    # Set map file
     # e.g. '(Clone|Image)_Model[_p#]_Size[_Label].map'
     map_name = model if model else 'None'
     if source.details['bus'] == 'Image':
@@ -147,7 +151,7 @@ class BlockPair():
       part_num = re.sub(r"^.*?(\d+)$", r"\1", source.path.name)
       map_name += f'_p{part_num}'
     size_str = std.bytes_to_string(
-      size=source.details["size"],
+      size=self.size,
       use_binary=False,
       )
     map_name += f'_{size_str.replace(" ", "")}'
@@ -163,7 +167,17 @@ class BlockPair():
     else:
       # Cloning
       self.map_path = pathlib.Path(f'{working_dir}/Clone_{map_name}.map')
-    self.map_path.touch()
+
+    # Create map file if needed
+    # NOTE: We need to set the domain size for --complete-only to work
+    if not self.map_path.exists():
+      self.map_path.write_text(
+        data=DDRESCUE_MAP_TEMPLATE.format(
+          name=cfg.main.KIT_NAME_FULL,
+          size=self.size,
+          ),
+        encoding='utf-8',
+      )
 
     # Set initial status
     self.set_initial_status()
@@ -296,10 +310,9 @@ class BlockPair():
 
     # Mark future passes as skipped if applicable
     if percent == 100:
-      if pass_name == 'read':
+      status_keys = list(self.status.keys())
-        self.status['trim'] = 'Skipped'
+      for i in status_keys[status_keys.index(pass_name)+1:]:
-      if pass_name in ('read', 'trim'):
+        self.status[status_keys[i]] = 'Skipped'
-        self.status['scrape'] = 'Skipped'
 
 
 class State():
@@ -1202,22 +1215,38 @@ def build_block_pair_report(block_pairs, settings):
   return report
 
 
-def build_ddrescue_cmd(block_pair, pass_name, settings):
+def build_ddrescue_cmd(block_pair, pass_name, settings_menu):
   """Build ddrescue cmd using passed details, returns list."""
   cmd = ['sudo', 'ddrescue']
   if (block_pair.destination.is_block_device()
       or block_pair.destination.is_char_device()):
     cmd.append('--force')
-  if pass_name == 'read':
+  cmd.extend(DDRESCUE_SPECIFIC_PASS_SETTINGS.get(pass_name, []))
-    cmd.extend(['--no-trim', '--no-scrape'])
+
-  elif pass_name == 'trim':
+  # Fix domain size based on starting position
-    # Allow trimming
+  domain_size = block_pair.size
-    cmd.append('--no-scrape')
+  if settings_menu.options['--input-position']['Selected']:
-  elif pass_name == 'scrape':
+    settings_menu.options['--reverse']['Selected'] = False
-    # Allow trimming and scraping
+    input_position = std.string_to_bytes(
-    pass
+      settings_menu.options['--input-position']['Value'],
-  cmd.extend(settings)
+      )
-  cmd.append(f'--size={block_pair.size}')
+    domain_size -= input_position
+  cmd.append(f'--size={domain_size}')
+
+  # Determine skip sizes
+  if settings_menu.options['--skip-size']['Selected']:
+    skip_sizes = settings_menu.options['--skip-size']['Value'].split(',')
+    skip_sizes = [float(s) for s in skip_sizes]
+    initial_skip = max(INITIAL_SKIP_MIN, int(block_pair.size * skip_sizes[0]))
+    max_skip = min(int(block_pair.size * skip_sizes[1]), domain_size)
+    max_skip = max(INITIAL_SKIP_MIN, max_skip)
+    cmd.append(f'--skip-size={initial_skip},{max_skip}')
+  cmd.extend(get_ddrescue_settings(settings_menu))
+
+  # Add source physical sector size (if possible)
+  cmd.append(f'--sector-size={block_pair.sector_size}')
+
+  # Add block pair and map file
   if PLATFORM == 'Darwin':
     # Use Raw disks if possible
     for dev in (block_pair.source, block_pair.destination):
@@ -1569,6 +1598,8 @@ def get_ddrescue_settings(settings_menu):
 
   # Check menu selections
   for name, details in settings_menu.options.items():
+    if name == '--skip-size':
+      continue
     if details['Selected']:
       if 'Value' in details:
         settings.append(f'{name}={details["Value"]}')
@@ -2017,8 +2048,13 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True):
     LOG.info('ddrescue cmd: %s', cmd)
     return
 
-  # Start ddrescue
+  # Start ddrescue and ddrescueview (if enabled)
   proc = exe.popen_program(cmd)
+  if block_pair.view_map:
+    exe.popen_program(
+      ['ddrescueview', '-r', '5s', block_pair.map_path],
+      pipe=True,
+      )
 
   # ddrescue loop
   _i = 0
@@ -2035,15 +2071,6 @@ def run_ddrescue(state, block_pair, pass_name, settings, dry_run=True):
         std.print_error(warning_message)
         break
 
-      # Open ddrescueview
-      ## NOTE: This needs to be started a bit into the recovery since it needs
-      ##       a non-zero map file to read
-      if not block_pair.view_proc and _i > 1:
-        block_pair.view_proc = exe.popen_program(
-          ['ddrescueview', '-r', '5s', block_pair.map_path],
-          pipe=True,
-          )
-
     if _i % 60 == 0:
       # Clear ddrescue pane
       tmux.clear_pane()
@@ -2130,7 +2157,6 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True):
     if 'Retry' in name and details['Selected']:
       details['Selected'] = False
       state.retry_all_passes()
-  settings = get_ddrescue_settings(settings_menu)
 
   # Start SMART/Journal
   state.panes['SMART'] = tmux.split_window(
@@ -2143,7 +2169,7 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True):
       )
 
   # Run pass(es)
-  for pass_name in ('read', 'trim', 'scrape'):
+  for pass_name in ('read-skip', 'read-full', 'trim', 'scrape'):
     abort = False
 
     # Skip to next pass
@@ -2158,7 +2184,7 @@ def run_recovery(state, main_menu, settings_menu, dry_run=True):
         attempted_recovery = True
         state.mark_started()
         try:
-          run_ddrescue(state, pair, pass_name, settings, dry_run=dry_run)
+          run_ddrescue(state, pair, pass_name, settings_menu, dry_run=dry_run)
         except (FileNotFoundError, KeyboardInterrupt, std.GenericAbort):
           is_missing_source_or_destination(state)
           abort = True