From dbe4a342cc00baaf3b3da915034e5c7dc3b0a018 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 26 Aug 2023 14:30:22 -0700 Subject: [PATCH 1/4] Fix source_parts usage Addresses issue #221 --- scripts/wk/clone/block_pair.py | 18 +++++++++++++----- scripts/wk/clone/state.py | 4 ++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/wk/clone/block_pair.py b/scripts/wk/clone/block_pair.py index 5e330a51..d9d5b2b5 100644 --- a/scripts/wk/clone/block_pair.py +++ b/scripts/wk/clone/block_pair.py @@ -244,7 +244,7 @@ class BlockPair(): # Functions -def add_clone_block_pairs(state) -> None: +def add_clone_block_pairs(state) -> list[hw_disk.Disk]: """Add device to device block pairs and set settings if necessary.""" source_sep = get_partition_separator(state.source.path.name) dest_sep = get_partition_separator(state.destination.path.name) @@ -255,6 +255,7 @@ def add_clone_block_pairs(state) -> None: # Add pairs from previous run if settings['Partition Mapping']: + source_parts = [] for part_map in settings['Partition Mapping']: bp_source = hw_disk.Disk( f'{state.source.path}{source_sep}{part_map[0]}', @@ -262,8 +263,9 @@ def add_clone_block_pairs(state) -> None: bp_dest = pathlib.Path( f'{state.destination.path}{dest_sep}{part_map[1]}', ) + source_parts.append(bp_source) state.add_block_pair(bp_source, bp_dest) - return + return source_parts # Add pairs from selection source_parts = menus.select_disk_parts('Clone', state.source) @@ -271,7 +273,7 @@ def add_clone_block_pairs(state) -> None: # Whole disk (or single partition via args), skip settings bp_dest = state.destination.path state.add_block_pair(state.source, bp_dest) - return + return source_parts # New run, use new settings file settings['Needs Format'] = True @@ -306,13 +308,19 @@ def add_clone_block_pairs(state) -> None: # Save settings state.save_settings(settings) + # Done + return source_parts -def add_image_block_pairs(state) -> None: + +def add_image_block_pairs(state) -> list[hw_disk.Disk]: """Add device to image file block pairs.""" source_parts = menus.select_disk_parts(state.mode, state.source) for part in source_parts: state.add_block_pair(part, state.destination) + # Done + return source_parts + def build_block_pair_report(block_pairs, settings) -> list: """Build block pair report, returns list.""" @@ -378,7 +386,7 @@ def build_sfdisk_partition_line(table_type, dev_path, size, details) -> str: # Set dest type if re.match(r'^0x\w+$', source_type): - # Both source and dest are MBR + # Source is a MBR type source_table_type = 'MBR' if table_type == 'MBR': dest_type = source_type.replace('0x', '').lower() diff --git a/scripts/wk/clone/state.py b/scripts/wk/clone/state.py index bb0d6c26..f0dd9c66 100644 --- a/scripts/wk/clone/state.py +++ b/scripts/wk/clone/state.py @@ -378,9 +378,9 @@ class State(): # Add block pairs if self.mode == 'Clone': - add_clone_block_pairs(self) + source_parts = add_clone_block_pairs(self) else: - add_image_block_pairs(self) + source_parts = add_image_block_pairs(self) # Update SMART data ## TODO: Verify if needed From f5681a93d8d7fd3f83bacb05f2cd7925fc2d6518 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 26 Aug 2023 16:54:53 -0700 Subject: [PATCH 2/4] Add finalization menu to ddrescue-tui Addresses issue #220 --- scripts/wk/clone/ddrescue.py | 97 +++++++++++++++++++++++++++++------- scripts/wk/clone/state.py | 33 ++++++------ 2 files changed, 99 insertions(+), 31 deletions(-) diff --git a/scripts/wk/clone/ddrescue.py b/scripts/wk/clone/ddrescue.py index f019e0de..fa9c570a 100644 --- a/scripts/wk/clone/ddrescue.py +++ b/scripts/wk/clone/ddrescue.py @@ -17,7 +17,7 @@ from docopt import docopt from wk import cfg, exe, io, log, std from wk.cfg.ddrescue import DDRESCUE_SPECIFIC_PASS_SETTINGS from wk.clone import menus -from wk.clone.state import State +from wk.clone.state import State, mark_non_recovered_as_non_tried from wk.hw import disk as hw_disk from wk.hw.smart import ( check_attributes, @@ -177,10 +177,68 @@ def get_ddrescue_settings(settings_menu) -> list: def finalize_recovery(state: State, dry_run: bool = True) -> None: - """Show recovery finalization options.""" - zero_fill_destination(state, dry_run=dry_run) - if state.mode == 'Clone': + """Show recovery finalization options and run selected functions.""" + options = ( + 'Zero-fill Extra Space', + ) + toggles = ( + 'Relocate Backup GPT', + 'Zero-fill Gaps', + ) + + # Get destination size + dest_size = -1 + if hasattr(state.destination, 'size'): + # hw_disk.Disk + dest_size = state.destination.size + if hasattr(state.destination, 'stat'): + # pathlib.Path + dest_size = state.destination.stat().st_size + + # Run checks to disable item(s) + whole_disk = bool( + len(state.block_pairs) == 1 + and not state.source.parent + ) + disable_gpt_toggle = not bool( + ## Breakdown of below tests: + ## Only offer this option when cloning a whole, non-child device + ## where the source is using a GUID_Partition_Table + ## and the source is smaller than the destination + whole_disk + and str(state.source.raw_details.get('pttype', 'Unknown')).lower() == 'gpt' + and state.source.size < dest_size + ) + + # Build menu + menu = cli.Menu(title=ansi.color_string('ddrescue TUI: Finalization', 'GREEN')) + menu.separator = ' ' + menu.add_action('Start') + menu.add_action('Quit') + for name in options: + menu.add_option(name, details={'Selected': True}) + for name in toggles: + details = {'Selected': True} + if 'GPT' in name and disable_gpt_toggle: + details['Disabled'] = True + details['Selected'] = False + menu.add_toggle(name, details) + + # Show menu + selection = menu.advanced_select() + if 'Quit' in selection: + return + + # Run functions + if menu.toggles['Relocate Backup GPT']['Selected']: relocate_backup_gpt(state, dry_run=dry_run) + if menu.toggles['Zero-fill Gaps']['Selected']: + zero_fill_gaps( + state, + dest_size=dest_size, + dry_run=dry_run, + extend_to_end=menu.options['Zero-fill Extra Space']['Selected'], + ) def is_missing_source_or_destination(state) -> bool: @@ -587,25 +645,28 @@ def source_or_destination_changed(state) -> bool: return changed -def zero_fill_destination(state: State, dry_run: bool = True) -> None: - """Zero-fill any gaps and space on destination beyond the source size.""" +def zero_fill_gaps( + state: State, + dest_size: int, + dry_run: bool = True, + extend_to_end: bool = False, + ) -> None: + """Zero-fill any gaps on the destination.""" + #fake_settings_menu = menus.settings(state.mode) full_disk_clone = False - larger_destination = False + larger_destination = state.source.size < dest_size percent_recovered = state.get_percent_recovered() if state.mode == 'Clone' and len(state.block_pairs) == 1: full_disk_clone = True # Bail early - if not ( - (percent_recovered < 100 - or (full_disk_clone and state.source.size < state.destination.size)) - and cli.ask('Fill gaps with zeros?')): + if percent_recovered == 100 and not (larger_destination and extend_to_end): return for block_pair in state.block_pairs: - destination_size = block_pair.size + domain_size = block_pair.size if (full_disk_clone and state.source.size < state.destination.size): - destination_size = state.destination.size + domain_size = dest_size larger_destination = True # Prep zero-fill map file @@ -613,11 +674,13 @@ def zero_fill_destination(state: State, dry_run: bool = True) -> None: f'{block_pair.map_path.stem}_zero-fill', ) io.copy_file(block_pair.map_path, zero_map_path, overwrite=True) - if larger_destination: + mark_non_recovered_as_non_tried(zero_map_path) + if full_disk_clone and larger_destination and extend_to_end: + # Extend domain size with open(zero_map_path, 'a', encoding='utf-8') as f: f.write( - f'{hex(block_pair.size)} ' - f'{hex(destination_size - block_pair.size)} ?' + f'\n{hex(block_pair.size)} ' + f'{hex(domain_size - block_pair.size)} ?' ) # Build cmd @@ -625,7 +688,7 @@ def zero_fill_destination(state: State, dry_run: bool = True) -> None: 'sudo', 'ddrescue', '--force', - f'--size={destination_size}', + f'--size={domain_size}', '--binary-prefixes', '--complete-only', '--data-preview=5', diff --git a/scripts/wk/clone/state.py b/scripts/wk/clone/state.py index f0dd9c66..f9badfce 100644 --- a/scripts/wk/clone/state.py +++ b/scripts/wk/clone/state.py @@ -497,29 +497,16 @@ class State(): def retry_all_passes(self) -> None: """Prep block_pairs for a retry recovery attempt.""" - bad_statuses = ('*', '/', '-') LOG.warning('Updating block_pairs for retry') # Update all block_pairs for pair in self.block_pairs: - map_data = [] + mark_non_recovered_as_non_tried(pair.map_path) # Reset status strings for name in pair.status.keys(): pair.status[name] = 'Pending' - # Mark all non-trimmed, non-scraped, and bad areas as non-tried - with open(pair.map_path, 'r', encoding='utf-8') as _f: - for line in _f.readlines(): - line = line.strip() - if line.startswith('0x') and line.endswith(bad_statuses): - line = f'{line[:-1]}?' - map_data.append(line) - - # Save updated map - with open(pair.map_path, 'w', encoding='utf-8') as _f: - _f.write('\n'.join(map_data)) - # Reinitialize status pair.set_initial_status() @@ -1009,6 +996,24 @@ def get_working_dir(mode, destination, force_local=False) -> pathlib.Path: return working_dir +def mark_non_recovered_as_non_tried(map_path: pathlib.Path) -> None: + """Read map file and mark all non-recovered sections as non-tried.""" + bad_statuses = ('*', '/', '-') + map_data = [] + + # Mark all non-trimmed, non-scraped, and bad areas as non-tried + with open(map_path, 'r', encoding='utf-8') as _f: + for line in _f.readlines(): + line = line.strip() + if line.startswith('0x') and line.endswith(bad_statuses): + line = f'{line[:-1]}?' + map_data.append(line) + + # Save updated map + with open(map_path, 'w', encoding='utf-8') as _f: + _f.write('\n'.join(map_data)) + + def select_disk_obj(label:str, disk_menu: cli.Menu, disk_path: str) -> hw_disk.Disk: """Get disk based on path or menu selection, returns Disk.""" if not disk_path: From d933ec64152d5e48aae29f372785faa6abdde842 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 26 Aug 2023 16:55:25 -0700 Subject: [PATCH 3/4] Ignore some unused function arguments --- scripts/wk/hw/diags.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/wk/hw/diags.py b/scripts/wk/hw/diags.py index 5e032a41..b08bf5eb 100644 --- a/scripts/wk/hw/diags.py +++ b/scripts/wk/hw/diags.py @@ -352,6 +352,7 @@ def cpu_tests_end(state) -> None: def cpu_test_cooling(state, test_object, test_mode=False) -> None: """CPU cooling test via sensor data assessment.""" + _ = test_mode LOG.info('CPU Test (Cooling)') # Bail early @@ -516,6 +517,7 @@ def cpu_test_sysbench(state, test_object, test_mode=False) -> None: def disk_attribute_check(state, test_objects, test_mode=False) -> None: """Disk attribute check.""" + _ = test_mode LOG.info('Disk Attribute Check') for test in test_objects: disk_smart_status_check(test.dev, mid_run=False) @@ -593,6 +595,7 @@ def disk_io_benchmark( def disk_self_test(state, test_objects, test_mode=False) -> None: """Disk self-test if available.""" + _ = test_mode LOG.info('Disk Self-Test(s)') aborted = False threads = [] From 5147a4105fe8ab84930e6d8083a66e71e78a56bb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sat, 26 Aug 2023 16:57:33 -0700 Subject: [PATCH 4/4] Simplify finalization menu Addresses issue #220 --- scripts/wk/clone/ddrescue.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/scripts/wk/clone/ddrescue.py b/scripts/wk/clone/ddrescue.py index fa9c570a..3f31cb6d 100644 --- a/scripts/wk/clone/ddrescue.py +++ b/scripts/wk/clone/ddrescue.py @@ -179,11 +179,9 @@ def get_ddrescue_settings(settings_menu) -> list: def finalize_recovery(state: State, dry_run: bool = True) -> None: """Show recovery finalization options and run selected functions.""" options = ( - 'Zero-fill Extra Space', - ) - toggles = ( 'Relocate Backup GPT', 'Zero-fill Gaps', + 'Zero-fill Extra Space', ) # Get destination size @@ -200,7 +198,7 @@ def finalize_recovery(state: State, dry_run: bool = True) -> None: len(state.block_pairs) == 1 and not state.source.parent ) - disable_gpt_toggle = not bool( + disable_gpt_option = not bool( ## Breakdown of below tests: ## Only offer this option when cloning a whole, non-child device ## where the source is using a GUID_Partition_Table @@ -216,13 +214,11 @@ def finalize_recovery(state: State, dry_run: bool = True) -> None: menu.add_action('Start') menu.add_action('Quit') for name in options: - menu.add_option(name, details={'Selected': True}) - for name in toggles: details = {'Selected': True} - if 'GPT' in name and disable_gpt_toggle: + if 'GPT' in name and disable_gpt_option: details['Disabled'] = True details['Selected'] = False - menu.add_toggle(name, details) + menu.add_option(name, details) # Show menu selection = menu.advanced_select() @@ -230,9 +226,9 @@ def finalize_recovery(state: State, dry_run: bool = True) -> None: return # Run functions - if menu.toggles['Relocate Backup GPT']['Selected']: + if menu.options['Relocate Backup GPT']['Selected']: relocate_backup_gpt(state, dry_run=dry_run) - if menu.toggles['Zero-fill Gaps']['Selected']: + if menu.options['Zero-fill Gaps']['Selected']: zero_fill_gaps( state, dest_size=dest_size,