From a22e6f552f5c3b31db5f5969889e5d63dfa3a68c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 19:01:03 -0700 Subject: [PATCH 01/61] Start build-ufd rewrite in python --- .bin/Scripts/{build-ufd => build-ufd.bash} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .bin/Scripts/{build-ufd => build-ufd.bash} (100%) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd.bash similarity index 100% rename from .bin/Scripts/build-ufd rename to .bin/Scripts/build-ufd.bash From 639a338cca8101afad48d51e057a2a15090b1da8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 19:03:56 -0700 Subject: [PATCH 02/61] Initial Python layout * Going to use docopt for argument handling * Script will be run as user using sudo where needed * Tentatively dropping tmux usage --- .bin/Scripts/build-ufd | 89 ++++++++++++++++++++++++++++++++ .bin/Scripts/functions/common.py | 51 +++++++++--------- .linux_items/packages/live_add | 1 + 3 files changed, 114 insertions(+), 27 deletions(-) create mode 100755 .bin/Scripts/build-ufd diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd new file mode 100755 index 00000000..d73b306c --- /dev/null +++ b/.bin/Scripts/build-ufd @@ -0,0 +1,89 @@ +#!/bin/env python3 +# +## Wizard Kit: UFD build tool + +# Init +import os +import sys + +sys.path.append(os.path.dirname(os.path.realpath(__file__))) + +from docopt import docopt +from functions.common import * + +set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars['Env'])) + + +# STATIC VARIABLES +DOCSTRING = '''Build UFD. + +Usage: + build-ufd [options] --ufd-device PATH --linux-iso PATH + [--linux-minimal-iso PATH] + [--main-kit PATH] + [--winpe-iso PATH] + [--extra-dir PATH] + build-ufd (-h | --help) + +Options: + -e PATH, --extra-dir PATH + -k PATH, --main-kit PATH + -l PATH, --linux-iso PATH + -m PATH, --linux-minimal-iso PATH + -u PATH, --ufd-device PATH + -w PATH, --winpe-iso PATH + + -d --debug Enable debug mode + -h --help Show this page + -v --verbose Enable verbose mode + -M --use-mbr Use real MBR instead of GPT w/ Protective MBR + -F --force Bypass all confirmation messages. USE WITH EXTREME CAUTION! + -U --update Don't format device, just update +''' +ISO_LABEL = '{}_LINUX'.format(KIT_NAME_SHORT) +UFD_LABEL = '{}_UFD'.format(KIT_NAME_SHORT) + + +# Functions +def get_full_path(item): + """Get full path to item, returns str.""" + #TODO + pass + + +def is_block_device(item): + """Verify item is a block device, returns bool.""" + #TODO + pass + + +def is_valid_main_kit(path): + """Verify path contains the main kit, returns bool.""" + + +# Main section +if __name__ == '__main__': + args = docopt(DOCSTRING) + + # Verify selections + + # Show selections + + # Double-check if formating device + + # Format and partition device + + # Mount sources + + # Copy sources + + # Update boot entries + + # Install syslinux + + # Hide items + + # Unmount sources + + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 502c985b..bbc2529c 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -879,25 +879,19 @@ def make_tmp_dirs(): def set_common_vars(): """Set common variables.""" - global_vars['Date'] = time.strftime("%Y-%m-%d") - global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") - global_vars['Env'] = os.environ.copy() + global_vars['Date'] = time.strftime("%Y-%m-%d") + global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") + global_vars['Env'] = os.environ.copy() global_vars['ArchivePassword'] = ARCHIVE_PASSWORD - global_vars['BinDir'] = r'{BaseDir}\.bin'.format( - **global_vars) - global_vars['CBinDir'] = r'{BaseDir}\.cbin'.format( - **global_vars) - global_vars['ClientDir'] = r'{SYSTEMDRIVE}\{prefix}'.format( - prefix=KIT_NAME_SHORT, **global_vars['Env']) - global_vars['BackupDir'] = r'{ClientDir}\Backups'.format( - **global_vars) - global_vars['LogDir'] = r'{ClientDir}\Logs\{Date}'.format( - **global_vars) - global_vars['QuarantineDir'] = r'{ClientDir}\Quarantine'.format( - **global_vars) - global_vars['TmpDir'] = r'{BinDir}\tmp'.format( - **global_vars) + global_vars['BinDir'] = r'{BaseDir}\.bin'.format(**global_vars) + global_vars['CBinDir'] = r'{BaseDir}\.cbin'.format(**global_vars) + global_vars['ClientDir'] = r'{SYSTEMDRIVE}\{prefix}'.format( + prefix=KIT_NAME_SHORT, **global_vars['Env']) + global_vars['BackupDir'] = r'{ClientDir}\Backups'.format(**global_vars) + global_vars['LogDir'] = r'{ClientDir}\Logs\{Date}'.format(**global_vars) + global_vars['QuarantineDir'] = r'{ClientDir}\Quarantine'.format(**global_vars) + global_vars['TmpDir'] = r'{BinDir}\tmp'.format(**global_vars) def set_linux_vars(): @@ -905,12 +899,12 @@ def set_linux_vars(): These assume we're running under a WK-Linux build.""" result = run_program(['mktemp', '-d']) - global_vars['TmpDir'] = result.stdout.decode().strip() - global_vars['Date'] = time.strftime("%Y-%m-%d") - global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") - global_vars['Env'] = os.environ.copy() - global_vars['BinDir'] = '/usr/local/bin' - global_vars['LogDir'] = global_vars['TmpDir'] + global_vars['TmpDir'] = result.stdout.decode().strip() + global_vars['Date'] = time.strftime("%Y-%m-%d") + global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") + global_vars['Env'] = os.environ.copy() + global_vars['BinDir'] = '/usr/local/bin' + global_vars['LogDir'] = '{}/Logs',format(global_vars['Env']['HOME']) global_vars['Tools'] = { 'wimlib-imagex': 'wimlib-imagex', 'SevenZip': '7z', @@ -919,10 +913,13 @@ def set_linux_vars(): def set_log_file(log_name): """Sets global var LogFile and creates path as needed.""" - folder_path = '{}{}{}'.format( - global_vars['LogDir'], - os.sep, - KIT_NAME_FULL) + if psutil.LINUX: + folder_path = global_vars['LogDir'] + else: + folder_path = '{}{}{}'.format( + global_vars['LogDir'], + os.sep, + KIT_NAME_FULL) log_file = '{}{}{}'.format( folder_path, os.sep, diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index 668c10c0..57f77bc1 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -31,6 +31,7 @@ networkmanager p7zip progsreiserfs python +python-docopt python-psutil python-requests reiserfsprogs From 575126eaa2ce91e1f1f34750258b92ea851e7216 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 20:51:54 -0700 Subject: [PATCH 03/61] Split build-ufd file to match other scripts --- .bin/Scripts/build-ufd | 109 +++++++++++++++------------------- .bin/Scripts/functions/ufd.py | 27 +++++++++ .bin/Scripts/settings/ufd.py | 80 +++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 62 deletions(-) create mode 100644 .bin/Scripts/functions/ufd.py create mode 100644 .bin/Scripts/settings/ufd.py diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index d73b306c..b7324dd1 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -2,88 +2,73 @@ # ## Wizard Kit: UFD build tool -# Init import os import sys +# Init sys.path.append(os.path.dirname(os.path.realpath(__file__))) - from docopt import docopt -from functions.common import * - +from functions.ufd import * +from settings.ufd import * +init_global_vars() set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars['Env'])) -# STATIC VARIABLES -DOCSTRING = '''Build UFD. - -Usage: - build-ufd [options] --ufd-device PATH --linux-iso PATH - [--linux-minimal-iso PATH] - [--main-kit PATH] - [--winpe-iso PATH] - [--extra-dir PATH] - build-ufd (-h | --help) - -Options: - -e PATH, --extra-dir PATH - -k PATH, --main-kit PATH - -l PATH, --linux-iso PATH - -m PATH, --linux-minimal-iso PATH - -u PATH, --ufd-device PATH - -w PATH, --winpe-iso PATH - - -d --debug Enable debug mode - -h --help Show this page - -v --verbose Enable verbose mode - -M --use-mbr Use real MBR instead of GPT w/ Protective MBR - -F --force Bypass all confirmation messages. USE WITH EXTREME CAUTION! - -U --update Don't format device, just update -''' -ISO_LABEL = '{}_LINUX'.format(KIT_NAME_SHORT) -UFD_LABEL = '{}_UFD'.format(KIT_NAME_SHORT) - - -# Functions -def get_full_path(item): - """Get full path to item, returns str.""" - #TODO - pass - - -def is_block_device(item): - """Verify item is a block device, returns bool.""" - #TODO - pass - - -def is_valid_main_kit(path): - """Verify path contains the main kit, returns bool.""" - - # Main section if __name__ == '__main__': - args = docopt(DOCSTRING) + try: + args = docopt(DOCSTRING) + sources = () - # Verify selections + # Verify selections + ## UFD + try: + ufd_dev = get_full_path(args['--ufd-device']) + if not is_block_device(ufd_dev): + print_error('Invalid UFD device: {}'.format(ufd_dev)) + abort() + except Exception: + # TODO Catch FileNotFound exception and abort accordingly + raise + ## Sources + for label, source in UFD_SOURCES: + if args[source]: + try: + sources.append((label, get_full_path(source))) + except Exception: + # TODO Catch FileNotFound exception and abort accordingly + raise - # Show selections + # Show selections + # TODO FIXME + print_standard('UFD: {}'.format(ufd_dev)) + print_standard('Sources:') + for s in sources: + print_standard(' {}: {}'.format(*s)) - # Double-check if formating device + # Double-check if formating device - # Format and partition device + # Format and partition device - # Mount sources + # Copy sources - # Copy sources + # Update boot entries - # Update boot entries + # Install syslinux - # Install syslinux + # Hide items - # Hide items + # Unmount sources - # Unmount sources + # Done + if not args['--force']: + print_standard('\nDone.') + pause('Press Enter to exit...') + exit_script() + except SystemExit: + pass + except: + major_exception() # vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py new file mode 100644 index 00000000..654f1f27 --- /dev/null +++ b/.bin/Scripts/functions/ufd.py @@ -0,0 +1,27 @@ +# Wizard Kit: Functions - UFD + +from functions.common import * + + +def get_full_path(item): + """Get full path to item, returns str.""" + #TODO + pass + + +def is_block_device(item): + """Verify item is a block device, returns bool.""" + #TODO + pass + + +def is_valid_main_kit(path): + """Verify path contains the main kit, returns bool.""" + #TODO + pass + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py new file mode 100644 index 00000000..b7f552eb --- /dev/null +++ b/.bin/Scripts/settings/ufd.py @@ -0,0 +1,80 @@ +# Wizard Kit: Settings - UFD + +from settings.main import * + +# General +DOCSTRING = '''WizardKit: Build UFD + +Usage: + build-ufd [options] --ufd-device PATH --linux-iso PATH + [--linux-minimal-iso PATH] + [--main-kit PATH] + [--winpe-iso PATH] + [--extra-dir PATH] + build-ufd (-h | --help) + +Options: + -e PATH, --extra-dir PATH + -k PATH, --main-kit PATH + -l PATH, --linux-iso PATH + -m PATH, --linux-minimal-iso PATH + -u PATH, --ufd-device PATH + -w PATH, --winpe-iso PATH + + -d --debug Enable debug mode + -h --help Show this page + -v --verbose Enable verbose mode + -M --use-mbr Use real MBR instead of GPT w/ Protective MBR + -F --force Bypass all confirmation messages. USE WITH EXTREME CAUTION! + -U --update Don't format device, just update +''' +ISO_LABEL = '{}_LINUX'.format(KIT_NAME_SHORT) +UFD_LABEL = '{}_UFD'.format(KIT_NAME_SHORT) +UFD_SOURCES = ( + # NOTE: Using tuple of tuples to ensure copy order + ('Linux', '--linux-iso'), + ('Linux (Minimal)', '--linux-minimal-iso'), + ('WinPE', '--winpe-iso'), + ('Main Kit', '--main-kit'), + ('Extras', '--extra-dir'), + ) + +# Definitions: Boot entries +## NOTE: if key path exists uncomment #value# lines +BOOT_ENTRIES = { + 'arch_minimal': 'MINIMAL', + 'sources/boot.wim': 'WINPE', + } + +# Definitions: Sources and Destinations +## NOTES: Paths are relative to the root of the ISO/UFD +## Sources use rsync's trailing slash syntax +ITEMS_LINUX_FULL = ( + ('/arch', '/'), + ('/isolinux', '/'), + ('/EFI/boot', '/EFI/'), + ('/EFI/memtest86', '/EFI/'), + ) +ITEMS_LINUX_MINIMAL = ( + ('/arch/boot/archiso.img', '/arch_minimal/'), + ('/arch/boot/vmlinuz', '/arch_minimal/'), + ('/arch/pkglist.x86_64.txt', '/arch_minimal/'), + ('/arch/x86_64', '/arch_minimal/'), + ) +ITEMS_WINPE = ( + ('/bootmgr', '/'), + ('/bootmgr.efi', '/'), + ('/en_us', '/'), + ('/Boot/', '/boot/'), + ('/EFI/Boot/', '/EFI/Microsoft/'), + ('/EFI/Microsoft/', '/EFI/Microsoft/'), + ('/Boot/BCD', '/sources/'), + ('/Boot/boot.sdi', '/sources/'), + ('/bootmgr', '/sources/'), + ('/sources/boot.wim', '/sources/'), + ) + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 From c4d00a2073cc2580715c6bf0c8c4b562ea3cca06 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 22:35:09 -0700 Subject: [PATCH 04/61] Fix typos --- .bin/Scripts/build-ufd | 2 +- .bin/Scripts/functions/common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index b7324dd1..a5577907 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -11,7 +11,7 @@ from docopt import docopt from functions.ufd import * from settings.ufd import * init_global_vars() -set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars['Env'])) +set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) # Main section diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index bbc2529c..b6008d27 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -904,7 +904,7 @@ def set_linux_vars(): global_vars['Date-Time'] = time.strftime("%Y-%m-%d_%H%M_%z") global_vars['Env'] = os.environ.copy() global_vars['BinDir'] = '/usr/local/bin' - global_vars['LogDir'] = '{}/Logs',format(global_vars['Env']['HOME']) + global_vars['LogDir'] = '{}/Logs'.format(global_vars['Env']['HOME']) global_vars['Tools'] = { 'wimlib-imagex': 'wimlib-imagex', 'SevenZip': '7z', From cd0a9456cb20cb862140409d49f72c3dbdde59ee Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 22:41:41 -0700 Subject: [PATCH 05/61] Use pathlib.Path to resolve paths * is_block_device() is no longer needed * (pathlib.Path provides that functionality) --- .bin/Scripts/functions/ufd.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 654f1f27..441b4770 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -1,24 +1,21 @@ # Wizard Kit: Functions - UFD +import pathlib from functions.common import * def get_full_path(item): - """Get full path to item, returns str.""" - #TODO - pass + """Get full path to item, returns pathlib.Path obj.""" + path_obj = pathlib.Path(item).resolve() + if not path_obj.exists(): + raise FileNotFoundError(path_obj) + + return path_obj -def is_block_device(item): - """Verify item is a block device, returns bool.""" - #TODO - pass - - -def is_valid_main_kit(path): - """Verify path contains the main kit, returns bool.""" - #TODO - pass +def is_valid_main_kit(path_obj): + """Verify PathObj contains the main kit, returns bool.""" + return path_obj.is_dir() and path_obj.joinpath('.bin').exists() if __name__ == '__main__': From b6c6fc9aa9f9c8e0aa503cc7b06792dfb5de8bc0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 22:44:15 -0700 Subject: [PATCH 06/61] Find paths case-insensitively --- .bin/Scripts/functions/ufd.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 441b4770..9aeb1b3d 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -4,6 +4,33 @@ import pathlib from functions.common import * +def case_insensitive_search(path, item): + """Search path for item case insensitively, returns str.""" + if os.path.exists('{}/{}'.format(path, item)): + # Easy mode + return '{}/{}'.format(path, item) + + # Check all items in dir + for entry in os.scandir(path): + if re.match(entry.name, item, re.IGNORECASE): + return '{}/{}'.format(path, entry.name) + + # If we get here the item wasn't found + raise FileNotFoundError('{}/{}'.format(path, item)) + + +def find_source_item(source_dir, item): + """Find item relative to source dir, returns str.""" + path = source_dir + if item.startswith('/'): + item = item[1:] + + for part in item.split('/'): + path = case_insensitive_search(path, part) + + return path + + def get_full_path(item): """Get full path to item, returns pathlib.Path obj.""" path_obj = pathlib.Path(item).resolve() From 50cb765108c5cb84bd22c63a314564b24d3e9dce Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 23:26:22 -0700 Subject: [PATCH 07/61] Merged find_source_item() and get_full_path() * Also fixed regex match (it wasn't matching the whole string) --- .bin/Scripts/functions/ufd.py | 49 +++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 9aeb1b3d..e6a13c47 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -6,37 +6,48 @@ from functions.common import * def case_insensitive_search(path, item): """Search path for item case insensitively, returns str.""" + regex_match = '^{}$'.format(item) + real_path = '' + + # Quick check first if os.path.exists('{}/{}'.format(path, item)): - # Easy mode - return '{}/{}'.format(path, item) + real_path = '{}{}{}'.format( + path, + '' if path == '/' else '/', + item, + ) # Check all items in dir for entry in os.scandir(path): - if re.match(entry.name, item, re.IGNORECASE): - return '{}/{}'.format(path, entry.name) + if re.match(regex_match, entry.name, re.IGNORECASE): + real_path = '{}{}{}'.format( + path, + '' if path == '/' else '/', + entry.name, + ) - # If we get here the item wasn't found - raise FileNotFoundError('{}/{}'.format(path, item)) + # Done + if real_path: + return real_path + else: + raise FileNotFoundError('{}/{}'.format(path, item)) -def find_source_item(source_dir, item): - """Find item relative to source dir, returns str.""" - path = source_dir - if item.startswith('/'): - item = item[1:] +def find_path(path): + """Find path case-insensitively, returns pathlib.Path obj.""" + parts = pathlib.Path(path).resolve().relative_to('/').parts + real_path = '/' - for part in item.split('/'): - path = case_insensitive_search(path, part) + # Fix case + for part in parts: + real_path = case_insensitive_search(real_path, part) - return path - - -def get_full_path(item): - """Get full path to item, returns pathlib.Path obj.""" - path_obj = pathlib.Path(item).resolve() + # Raise error if path doesn't exist + path_obj = pathlib.Path(real_path) if not path_obj.exists(): raise FileNotFoundError(path_obj) + # Done return path_obj From e420074c833f39c3157a7804a8d0d3bc97a2f8e9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 23:35:00 -0700 Subject: [PATCH 08/61] Use correct errors in find_path() * FileNotFoundError is accurate since we're just looking for a path --- .bin/Scripts/functions/ufd.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index e6a13c47..83dd1852 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -40,7 +40,11 @@ def find_path(path): # Fix case for part in parts: - real_path = case_insensitive_search(real_path, part) + try: + real_path = case_insensitive_search(real_path, part) + except NotADirectoryError: + # Reclassify error + raise FileNotFoundError(path) # Raise error if path doesn't exist path_obj = pathlib.Path(real_path) From b166172d10faabff877f72094edc7be5746423a1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 7 Apr 2019 23:45:08 -0700 Subject: [PATCH 09/61] Only perform case-insensitive search if needed --- .bin/Scripts/functions/ufd.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 83dd1852..f0ce6ad3 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -35,10 +35,15 @@ def case_insensitive_search(path, item): def find_path(path): """Find path case-insensitively, returns pathlib.Path obj.""" - parts = pathlib.Path(path).resolve().relative_to('/').parts - real_path = '/' + path_obj = pathlib.Path(path).resolve() + + # Quick check first + if path_obj.exists(): + return path_obj # Fix case + parts = path_obj.relative_to('/').parts + real_path = '/' for part in parts: try: real_path = case_insensitive_search(real_path, part) From 4c33c110b758d7fbb9b9ef0c58e8d1a7d6a37d7e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Apr 2019 00:35:53 -0700 Subject: [PATCH 10/61] Fixed source validation --- .bin/Scripts/build-ufd | 39 +++++++++++++++++++++-------------- .bin/Scripts/functions/ufd.py | 16 +++++++++++--- .bin/Scripts/settings/ufd.py | 28 ++++++++++++------------- 3 files changed, 50 insertions(+), 33 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index a5577907..9c048dea 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -7,6 +7,7 @@ import sys # Init sys.path.append(os.path.dirname(os.path.realpath(__file__))) +from collections import OrderedDict from docopt import docopt from functions.ufd import * from settings.ufd import * @@ -18,33 +19,39 @@ set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) if __name__ == '__main__': try: args = docopt(DOCSTRING) - sources = () + sources = OrderedDict() # Verify selections ## UFD try: - ufd_dev = get_full_path(args['--ufd-device']) - if not is_block_device(ufd_dev): - print_error('Invalid UFD device: {}'.format(ufd_dev)) - abort() - except Exception: - # TODO Catch FileNotFound exception and abort accordingly - raise + ufd_dev = find_path(args['--ufd-device']) + except FileNotFoundError: + print_error('ERROR: UFD device not found: {}'.format( + args['--ufd-device'])) + abort() + if not is_valid_path(ufd_dev, 'UFD'): + print_error('ERROR: Invalid UFD device: {}'.format(ufd_dev)) + abort() ## Sources - for label, source in UFD_SOURCES: - if args[source]: + for label, data in UFD_SOURCES.items(): + s_path = args[data['Arg']] + if s_path: try: - sources.append((label, get_full_path(source))) - except Exception: - # TODO Catch FileNotFound exception and abort accordingly - raise + s_path_obj = find_path(s_path) + except FileNotFoundError: + print_error('ERROR: {} not found: {}'.format(label, s_path)) + abort() + if not is_valid_path(s_path_obj, data['Type']): + print_error('ERROR: Invalid {} source: {}'.format(label, s_path)) + abort() + sources[label] = s_path_obj # Show selections # TODO FIXME print_standard('UFD: {}'.format(ufd_dev)) print_standard('Sources:') - for s in sources: - print_standard(' {}: {}'.format(*s)) + for label, s_path in sources.items(): + print_standard(' {:<16} {}'.format(label+':', s_path)) # Double-check if formating device diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index f0ce6ad3..ae1bc37d 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -60,9 +60,19 @@ def find_path(path): return path_obj -def is_valid_main_kit(path_obj): - """Verify PathObj contains the main kit, returns bool.""" - return path_obj.is_dir() and path_obj.joinpath('.bin').exists() +def is_valid_path(path_obj, path_type): + """Verify path_obj is valid by type, returns bool.""" + valid_path = False + if path_type == 'DIR': + valid_path = path_obj.is_dir() + elif path_type == 'KIT': + valid_path = path_obj.is_dir() and path_obj.joinpath('.bin').exists() + elif path_type == 'ISO': + valid_path = path_obj.is_file() and path_obj.suffix.lower() == '.iso' + elif path_type == 'UFD': + valid_path = path_obj.is_block_device() + + return valid_path if __name__ == '__main__': diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index b7f552eb..50fc6934 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -1,25 +1,26 @@ # Wizard Kit: Settings - UFD +from collections import OrderedDict from settings.main import * # General DOCSTRING = '''WizardKit: Build UFD Usage: - build-ufd [options] --ufd-device PATH --linux-iso PATH - [--linux-minimal-iso PATH] + build-ufd [options] --ufd-device PATH --linux PATH + [--linux-minimal PATH] [--main-kit PATH] - [--winpe-iso PATH] + [--winpe PATH] [--extra-dir PATH] build-ufd (-h | --help) Options: -e PATH, --extra-dir PATH -k PATH, --main-kit PATH - -l PATH, --linux-iso PATH - -m PATH, --linux-minimal-iso PATH + -l PATH, --linux PATH + -m PATH, --linux-minimal PATH -u PATH, --ufd-device PATH - -w PATH, --winpe-iso PATH + -w PATH, --winpe PATH -d --debug Enable debug mode -h --help Show this page @@ -30,14 +31,13 @@ Options: ''' ISO_LABEL = '{}_LINUX'.format(KIT_NAME_SHORT) UFD_LABEL = '{}_UFD'.format(KIT_NAME_SHORT) -UFD_SOURCES = ( - # NOTE: Using tuple of tuples to ensure copy order - ('Linux', '--linux-iso'), - ('Linux (Minimal)', '--linux-minimal-iso'), - ('WinPE', '--winpe-iso'), - ('Main Kit', '--main-kit'), - ('Extras', '--extra-dir'), - ) +UFD_SOURCES = ({ + 'Linux': {'Arg': '--linux', 'Type': 'ISO'}, + 'Linux (Minimal)': {'Arg': '--linux-minimal', 'Type': 'ISO'}, + 'WinPE': {'Arg': '--winpe', 'Type': 'ISO'}, + 'Main Kit': {'Arg': '--main-kit', 'Type': 'KIT'}, + 'Extra Dir': {'Arg': '--extra-dir', 'Type': 'DIR'}, + }) # Definitions: Boot entries ## NOTE: if key path exists uncomment #value# lines From 5fe37b62b288bda89c7084f9fa3f4d85fea15bc6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Apr 2019 02:58:07 -0700 Subject: [PATCH 11/61] Adjusted source/dest items --- .bin/Scripts/settings/ufd.py | 56 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index 50fc6934..4f99018c 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -49,30 +49,38 @@ BOOT_ENTRIES = { # Definitions: Sources and Destinations ## NOTES: Paths are relative to the root of the ISO/UFD ## Sources use rsync's trailing slash syntax -ITEMS_LINUX_FULL = ( - ('/arch', '/'), - ('/isolinux', '/'), - ('/EFI/boot', '/EFI/'), - ('/EFI/memtest86', '/EFI/'), - ) -ITEMS_LINUX_MINIMAL = ( - ('/arch/boot/archiso.img', '/arch_minimal/'), - ('/arch/boot/vmlinuz', '/arch_minimal/'), - ('/arch/pkglist.x86_64.txt', '/arch_minimal/'), - ('/arch/x86_64', '/arch_minimal/'), - ) -ITEMS_WINPE = ( - ('/bootmgr', '/'), - ('/bootmgr.efi', '/'), - ('/en_us', '/'), - ('/Boot/', '/boot/'), - ('/EFI/Boot/', '/EFI/Microsoft/'), - ('/EFI/Microsoft/', '/EFI/Microsoft/'), - ('/Boot/BCD', '/sources/'), - ('/Boot/boot.sdi', '/sources/'), - ('/bootmgr', '/sources/'), - ('/sources/boot.wim', '/sources/'), - ) +ITEMS = { + 'Extra Dir': ( + ('/', '/'), + ), + 'Linux': ( + ('/arch', '/'), + ('/isolinux', '/'), + ('/EFI/boot', '/EFI/'), + ('/EFI/memtest86', '/EFI/'), + ), + 'Linux (Minimal)': ( + ('/arch/boot/archiso.img', '/arch_minimal/'), + ('/arch/boot/vmlinuz', '/arch_minimal/'), + ('/arch/pkglist.x86_64.txt', '/arch_minimal/'), + ('/arch/x86_64', '/arch_minimal/'), + ), + 'Main Kit': ( + ('/', '/{}/'.format(KIT_NAME_FULL)), + ), + 'WinPE': ( + ('/bootmgr', '/'), + ('/bootmgr.efi', '/'), + ('/en_us', '/'), + ('/Boot/', '/boot/'), + ('/EFI/Boot/', '/EFI/Microsoft/'), + ('/EFI/Microsoft/', '/EFI/Microsoft/'), + ('/Boot/BCD', '/sources/'), + ('/Boot/boot.sdi', '/sources/'), + ('/bootmgr', '/sources/'), + ('/sources/boot.wim', '/sources/'), + ), + } if __name__ == '__main__': print("This file is not meant to be called directly.") From 8cdb834499670a73059008d7f736d48b01291239 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Apr 2019 02:59:03 -0700 Subject: [PATCH 12/61] Initial, incomplete version of recursive_copy() --- .bin/Scripts/functions/ufd.py | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index ae1bc37d..92782d12 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -75,6 +75,46 @@ def is_valid_path(path_obj, path_type): return valid_path +def recursive_copy(source, dest, overwrite=True): + """Copy source to dest recursively. + + NOTE: This uses rsync style source/dest syntax. + If the source has a trailing slash then it's contents are copied, + otherwise the source itself is copied. + + Examples assuming "ExDir/ExFile.txt" exists: + recursive_copy("ExDir", "Dest/") results in "Dest/ExDir/ExFile.txt" + recursive_copy("ExDir/", "Dest/") results in "Dest/ExFile.txt" + + NOTE 2: dest does not use find_path because it might not exist. + """ + copy_contents = source.endswith('/') + source = find_path(source) + dest = pathlib.Path(dest) + + if copy_contents: + for item in os.scandir(source): + recursive_copy(item.path, dest, overwrite=overwrite) + elif source.is_dir(): + if not dest.exists(): + shutil.copytree(source, dest.joinpath('source.name')) + elif not dest.is_dir(): + raise GenericError('Unexpected item in dest: {}'.format(dest)) + else: + # Dest exists and is a dir + for item in os.scandir(source): + recursive_copy( + item.path, + dest.joinpath(source.name), + overwrite=overwrite, + ) + elif source.is_file(): + # TODO FIXME + if dest.exists(): + # TODO FIXME + pass + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 415f56863d89b9f54946232526a24d536a7be217 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Apr 2019 20:26:41 -0700 Subject: [PATCH 13/61] Finished recursive_copy() --- .bin/Scripts/functions/ufd.py | 49 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 92782d12..3671a262 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -90,29 +90,38 @@ def recursive_copy(source, dest, overwrite=True): """ copy_contents = source.endswith('/') source = find_path(source) - dest = pathlib.Path(dest) + dest = pathlib.Path(dest).resolve().joinpath(source.name) + os.makedirs(dest.parent, exist_ok=True) - if copy_contents: - for item in os.scandir(source): - recursive_copy(item.path, dest, overwrite=overwrite) - elif source.is_dir(): - if not dest.exists(): - shutil.copytree(source, dest.joinpath('source.name')) - elif not dest.is_dir(): - raise GenericError('Unexpected item in dest: {}'.format(dest)) - else: - # Dest exists and is a dir + if source.is_dir(): + if copy_contents: + # Trailing slash syntax for item in os.scandir(source): - recursive_copy( - item.path, - dest.joinpath(source.name), - overwrite=overwrite, - ) + recursive_copy(item.path, dest.parent, overwrite=overwrite) + elif not dest.exists(): + # No conflict, copying whole tree (no merging needed) + shutil.copytree(source, dest) + elif not dest.is_dir(): + # Refusing to replace file with dir + raise GenericError('Refusing to replace file with dir: {}'.format(dest)) + else: + # Dest exists and is a dir, merge dirs + for item in os.scandir(source): + recursive_copy(item.path, dest, overwrite=overwrite) elif source.is_file(): - # TODO FIXME - if dest.exists(): - # TODO FIXME - pass + if not dest.exists(): + # No conflict, copying file + shutil.copy2(source, dest) + elif not dest.is_file(): + # Refusing to replace dir with file + raise GenericError('Refusing to replace dir with file: {}'.format(dest)) + elif overwrite: + # Dest file exists, deleting and replacing file + os.remove(dest) + shutil.copy2(source, dest) + else: + # Refusing to delete file when overwrite=False + raise GenericError('Refusing to delete file: {}'.format(dest)) if __name__ == '__main__': From 1a8b6705e01a1c9bb71dd654eff764f7c2bfede9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 8 Apr 2019 20:57:35 -0700 Subject: [PATCH 14/61] Show selections before starting work --- .bin/Scripts/build-ufd | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 9c048dea..2f7f7451 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -47,6 +47,60 @@ if __name__ == '__main__': sources[label] = s_path_obj # Show selections + ## Header + clear_screen() + print_success(KIT_NAME_FULL) + print_standard('UFD Build Tool') + print_standard(' ') + ## Sources + print_info('Sources') + for label in UFD_SOURCES.keys(): + if label in sources: + print_standard(' {label:<18} {path}'.format( + label=label+':', + path=sources[label], + )) + else: + print_standard(' {label:<18} {YELLOW}Not Specified{CLEAR}'.format( + label=label+':', + **COLORS, + )) + ## Destination + print_standard(' ') + print_info('Destination') + cmd = [ + 'lsblk', '--nodeps', '--noheadings', + '--output', 'NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL', + ] + result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') + print_standard(result.stdout.strip()) + cmd = [ + 'lsblk', '--noheadings', + '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', + ] + result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') + for line in result.stdout.splitlines()[1:]: + print_standard(line) + if args['--use-mbr'] and not args['--update']: + print_warning('Formatting using legacy MBR') + print_standard(' ') + if not ask('Is the above information correct?'): + abort() + ## Safety check + print_standard(' ') + print_warning('SAFETY CHECK') + print_standard( + 'All data will be DELETED from the disk and partition(s) listed above.') + print_standard( + 'This is irreversible and will lead to {RED}DATA LOSS.{CLEAR}'.format( + **COLORS)) + if not ask('Asking again to confirm, is this correct?'): + abort() + + print_standard(' ') + print_success("It's go-time!") + exit_script(1) + # TODO FIXME print_standard('UFD: {}'.format(ufd_dev)) print_standard('Sources:') From d33f78960d81d4db3d551f61384fe06a6df2ccf4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 11 Apr 2019 20:50:21 -0700 Subject: [PATCH 15/61] Reduced imports --- .bin/Scripts/build-ufd | 1 + .bin/Scripts/functions/ufd.py | 16 +++++++++------- .bin/Scripts/settings/ufd.py | 10 +++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 2f7f7451..a2f47a7e 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -9,6 +9,7 @@ import sys sys.path.append(os.path.dirname(os.path.realpath(__file__))) from collections import OrderedDict from docopt import docopt +from functions.common import * from functions.ufd import * from settings.ufd import * init_global_vars() diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 3671a262..e8fb164f 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -1,7 +1,9 @@ # Wizard Kit: Functions - UFD +import os +import re +import shutil import pathlib -from functions.common import * def case_insensitive_search(path, item): @@ -27,11 +29,11 @@ def case_insensitive_search(path, item): ) # Done - if real_path: - return real_path - else: + if not real_path: raise FileNotFoundError('{}/{}'.format(path, item)) + return real_path + def find_path(path): """Find path case-insensitively, returns pathlib.Path obj.""" @@ -103,7 +105,7 @@ def recursive_copy(source, dest, overwrite=True): shutil.copytree(source, dest) elif not dest.is_dir(): # Refusing to replace file with dir - raise GenericError('Refusing to replace file with dir: {}'.format(dest)) + raise FileExistsError('Refusing to replace file: {}'.format(dest)) else: # Dest exists and is a dir, merge dirs for item in os.scandir(source): @@ -114,14 +116,14 @@ def recursive_copy(source, dest, overwrite=True): shutil.copy2(source, dest) elif not dest.is_file(): # Refusing to replace dir with file - raise GenericError('Refusing to replace dir with file: {}'.format(dest)) + raise FileExistsError('Refusing to replace dir: {}'.format(dest)) elif overwrite: # Dest file exists, deleting and replacing file os.remove(dest) shutil.copy2(source, dest) else: # Refusing to delete file when overwrite=False - raise GenericError('Refusing to delete file: {}'.format(dest)) + raise FileExistsError('Refusing to delete file: {}'.format(dest)) if __name__ == '__main__': diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index 4f99018c..13ecd4e9 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -1,7 +1,9 @@ -# Wizard Kit: Settings - UFD +'''Wizard Kit: Settings - UFD''' +# pylint: disable=C0326,E0611 +# vim: sts=2 sw=2 ts=2 from collections import OrderedDict -from settings.main import * +from settings.main import KIT_NAME_FULL,KIT_NAME_SHORT # General DOCSTRING = '''WizardKit: Build UFD @@ -31,7 +33,7 @@ Options: ''' ISO_LABEL = '{}_LINUX'.format(KIT_NAME_SHORT) UFD_LABEL = '{}_UFD'.format(KIT_NAME_SHORT) -UFD_SOURCES = ({ +UFD_SOURCES = OrderedDict({ 'Linux': {'Arg': '--linux', 'Type': 'ISO'}, 'Linux (Minimal)': {'Arg': '--linux-minimal', 'Type': 'ISO'}, 'WinPE': {'Arg': '--winpe', 'Type': 'ISO'}, @@ -84,5 +86,3 @@ ITEMS = { if __name__ == '__main__': print("This file is not meant to be called directly.") - -# vim: sts=2 sw=2 ts=2 From 0883b099fd7c83aa56d11553940bec15c2bbc6a7 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 11 Apr 2019 22:24:25 -0700 Subject: [PATCH 16/61] Skip safety check if upgrading --- .bin/Scripts/build-ufd | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index a2f47a7e..05458c6b 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -82,32 +82,29 @@ if __name__ == '__main__': result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') for line in result.stdout.splitlines()[1:]: print_standard(line) - if args['--use-mbr'] and not args['--update']: + if args['--update']: + print_warning('Updating kit in-place') + elif args['--use-mbr']: print_warning('Formatting using legacy MBR') print_standard(' ') if not ask('Is the above information correct?'): abort() ## Safety check - print_standard(' ') - print_warning('SAFETY CHECK') - print_standard( - 'All data will be DELETED from the disk and partition(s) listed above.') - print_standard( - 'This is irreversible and will lead to {RED}DATA LOSS.{CLEAR}'.format( - **COLORS)) - if not ask('Asking again to confirm, is this correct?'): - abort() + if not args['--update']: + print_standard(' ') + print_warning('SAFETY CHECK') + print_standard( + 'All data will be DELETED from the disk and partition(s) listed above.') + print_standard( + 'This is irreversible and will lead to {RED}DATA LOSS.{CLEAR}'.format( + **COLORS)) + if not ask('Asking again to confirm, is this correct?'): + abort() print_standard(' ') print_success("It's go-time!") exit_script(1) - # TODO FIXME - print_standard('UFD: {}'.format(ufd_dev)) - print_standard('Sources:') - for label, s_path in sources.items(): - print_standard(' {:<16} {}'.format(label+':', s_path)) - # Double-check if formating device # Format and partition device From b2528b90f8abc4a35672ff707c4e561e4d925c55 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 11 Apr 2019 22:25:13 -0700 Subject: [PATCH 17/61] Fix docopt handling * Show usage if invalid arguments are used --- .bin/Scripts/build-ufd | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 05458c6b..726efa4d 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -20,6 +20,13 @@ set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) if __name__ == '__main__': try: args = docopt(DOCSTRING) + except SystemExit as err: + # Catch docopt exits + print(err) + exit_script() + except: + major_exception() + try: sources = OrderedDict() # Verify selections From 5a1f7b1829a20f8e46fd669341064cb9108ce916 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 15:58:09 -0700 Subject: [PATCH 18/61] Silence build-ufd init --- .bin/Scripts/build-ufd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 726efa4d..c4fcb26a 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -12,7 +12,7 @@ from docopt import docopt from functions.common import * from functions.ufd import * from settings.ufd import * -init_global_vars() +init_global_vars(silent=True) set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) From d62a647fa9cfe90dc14aaa9f9d609c72cd1bf8b8 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 16:46:50 -0700 Subject: [PATCH 19/61] Avoid setting LogDir to /root/Logs under su/sudo --- .bin/Scripts/build-ufd | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index c4fcb26a..d83eddf2 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -13,11 +13,36 @@ from functions.common import * from functions.ufd import * from settings.ufd import * init_global_vars(silent=True) -set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) + +# Functions +def get_user_home(user): + """Get path to user's home dir, returns str.""" + home_dir = None + cmd = ['getent', 'passwd', user] + result = run_program(cmd, encoding='utf-8', errors='ignore', check=False) + try: + home_dir = result.stdout.split(':')[5] + except Exception: + # Just use HOME from ENV (or '/root' if that fails) + home_dir = os.environ.get('HOME', '/root') + + return home_dir + + +def get_user_name(): + """Get real user name, returns str.""" + user = None + if 'SUDO_USER' in os.environ: + user = os.environ.get('SUDO_USER') + else: + user = os.environ.get('USER') + + return user # Main section if __name__ == '__main__': + # Docopt try: args = docopt(DOCSTRING) except SystemExit as err: @@ -26,10 +51,15 @@ if __name__ == '__main__': exit_script() except: major_exception() + try: - sources = OrderedDict() + # Set log + global_vars['LogDir'] = '{}/Logs'.format( + get_user_home(get_user_name())) + set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) # Verify selections + sources = OrderedDict() ## UFD try: ufd_dev = find_path(args['--ufd-device']) From fde9be6b3ff16cdd9c709ab5929f1cc3f6e3c928 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 16:48:34 -0700 Subject: [PATCH 20/61] Updated get_user_name() --- .bin/Scripts/build-ufd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index d83eddf2..610a1e66 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -33,9 +33,9 @@ def get_user_name(): """Get real user name, returns str.""" user = None if 'SUDO_USER' in os.environ: - user = os.environ.get('SUDO_USER') + user = os.environ.get('SUDO_USER', 'Unknown') else: - user = os.environ.get('USER') + user = os.environ.get('USER', 'Unknown') return user From 6734460d42d553551b2986c159859255e243509c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 17:01:53 -0700 Subject: [PATCH 21/61] Added option to skip pause in abort() * Also use exit_script(1) --- .bin/Scripts/build-ufd | 12 ++++++------ .bin/Scripts/functions/common.py | 9 +++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 610a1e66..5542a0ac 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -66,10 +66,10 @@ if __name__ == '__main__': except FileNotFoundError: print_error('ERROR: UFD device not found: {}'.format( args['--ufd-device'])) - abort() + abort(False) if not is_valid_path(ufd_dev, 'UFD'): print_error('ERROR: Invalid UFD device: {}'.format(ufd_dev)) - abort() + abort(False) ## Sources for label, data in UFD_SOURCES.items(): s_path = args[data['Arg']] @@ -78,10 +78,10 @@ if __name__ == '__main__': s_path_obj = find_path(s_path) except FileNotFoundError: print_error('ERROR: {} not found: {}'.format(label, s_path)) - abort() + abort(False) if not is_valid_path(s_path_obj, data['Type']): print_error('ERROR: Invalid {} source: {}'.format(label, s_path)) - abort() + abort(False) sources[label] = s_path_obj # Show selections @@ -125,7 +125,7 @@ if __name__ == '__main__': print_warning('Formatting using legacy MBR') print_standard(' ') if not ask('Is the above information correct?'): - abort() + abort(False) ## Safety check if not args['--update']: print_standard(' ') @@ -136,7 +136,7 @@ if __name__ == '__main__': 'This is irreversible and will lead to {RED}DATA LOSS.{CLEAR}'.format( **COLORS)) if not ask('Asking again to confirm, is this correct?'): - abort() + abort(False) print_standard(' ') print_success("It's go-time!") diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index b6008d27..fe75f1ef 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -90,12 +90,13 @@ class SecureBootUnknownError(Exception): # General functions -def abort(): +def abort(show_prompt=True): """Abort script.""" print_warning('Aborted.') - sleep(1) - pause(prompt='Press Enter to exit... ') - exit_script() + if show_prompt: + sleep(timeout) + pause(prompt='Press Enter to exit... ') + exit_script(1) def ask(prompt='Kotaero!'): From cc1a318e3dece2dab58c85dcdd5ec7387da86ecc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 17:03:04 -0700 Subject: [PATCH 22/61] Abort if not running as root --- .bin/Scripts/build-ufd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 5542a0ac..38b6f8d6 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -42,6 +42,11 @@ def get_user_name(): # Main section if __name__ == '__main__': + # Check if running as root + if os.geteuid() != 0: + print_error('ERROR: This script is meant to be run as root.') + abort(False) + # Docopt try: args = docopt(DOCSTRING) From cf1d11eb789280ddf9eb44aac2cb37363a8460be Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 17:16:13 -0700 Subject: [PATCH 23/61] Always show header --- .bin/Scripts/build-ufd | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 38b6f8d6..592a6143 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -42,6 +42,19 @@ def get_user_name(): # Main section if __name__ == '__main__': + # Set log + try: + global_vars['LogDir'] = '{}/Logs'.format( + get_user_home(get_user_name())) + set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) + except: + major_exception() + + # Header + print_success(KIT_NAME_FULL) + print_standard('UFD Build Tool') + print_standard(' ') + # Check if running as root if os.geteuid() != 0: print_error('ERROR: This script is meant to be run as root.') @@ -58,11 +71,6 @@ if __name__ == '__main__': major_exception() try: - # Set log - global_vars['LogDir'] = '{}/Logs'.format( - get_user_home(get_user_name())) - set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) - # Verify selections sources = OrderedDict() ## UFD @@ -90,11 +98,6 @@ if __name__ == '__main__': sources[label] = s_path_obj # Show selections - ## Header - clear_screen() - print_success(KIT_NAME_FULL) - print_standard('UFD Build Tool') - print_standard(' ') ## Sources print_info('Sources') for label in UFD_SOURCES.keys(): @@ -145,7 +148,7 @@ if __name__ == '__main__': print_standard(' ') print_success("It's go-time!") - exit_script(1) + exit_script() # Double-check if formating device From 1b312658e949a93cfafbdc9dd4b3772671678af6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 17:17:11 -0700 Subject: [PATCH 24/61] Fixed return code handling --- .bin/Scripts/build-ufd | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 592a6143..3b688e17 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -42,6 +42,7 @@ def get_user_name(): # Main section if __name__ == '__main__': + return_code = 0 # Set log try: global_vars['LogDir'] = '{}/Logs'.format( @@ -169,10 +170,11 @@ if __name__ == '__main__': print_standard('\nDone.') pause('Press Enter to exit...') exit_script() - except SystemExit: - pass + except SystemExit as e: + return_code = e.code except: major_exception() + exit_script(return_code) # vim: sts=2 sw=2 ts=2 From 0e6be3ad08620de0042beffa8ca3694ab63b9534 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 17:57:46 -0700 Subject: [PATCH 25/61] Better SystemExit handling --- .bin/Scripts/build-ufd | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 3b688e17..cfb19124 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -42,7 +42,6 @@ def get_user_name(): # Main section if __name__ == '__main__': - return_code = 0 # Set log try: global_vars['LogDir'] = '{}/Logs'.format( @@ -170,11 +169,9 @@ if __name__ == '__main__': print_standard('\nDone.') pause('Press Enter to exit...') exit_script() - except SystemExit as e: - return_code = e.code + except SystemExit as sys_exit: + exit_script(sys_exit.code) except: major_exception() - exit_script(return_code) - # vim: sts=2 sw=2 ts=2 From 164fd4c646d6df829ec4c646cdd367bbea7f3ecc Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 18:09:36 -0700 Subject: [PATCH 26/61] Fixed SystemExit calls from docopt --- .bin/Scripts/build-ufd | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index cfb19124..15e4e680 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -63,10 +63,9 @@ if __name__ == '__main__': # Docopt try: args = docopt(DOCSTRING) - except SystemExit as err: + except SystemExit as sys_exit: # Catch docopt exits - print(err) - exit_script() + exit_script(sys_exit.code) except: major_exception() From 3310e3d2537f49c9d6d655f64cc17a1300e240c9 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 18:43:08 -0700 Subject: [PATCH 27/61] Moved most logic to functions.ufd --- .bin/Scripts/build-ufd | 108 ++-------------------------- .bin/Scripts/functions/ufd.py | 131 ++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 102 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 15e4e680..887932a7 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -7,39 +7,12 @@ import sys # Init sys.path.append(os.path.dirname(os.path.realpath(__file__))) -from collections import OrderedDict from docopt import docopt from functions.common import * from functions.ufd import * from settings.ufd import * init_global_vars(silent=True) -# Functions -def get_user_home(user): - """Get path to user's home dir, returns str.""" - home_dir = None - cmd = ['getent', 'passwd', user] - result = run_program(cmd, encoding='utf-8', errors='ignore', check=False) - try: - home_dir = result.stdout.split(':')[5] - except Exception: - # Just use HOME from ENV (or '/root' if that fails) - home_dir = os.environ.get('HOME', '/root') - - return home_dir - - -def get_user_name(): - """Get real user name, returns str.""" - user = None - if 'SUDO_USER' in os.environ: - user = os.environ.get('SUDO_USER', 'Unknown') - else: - user = os.environ.get('USER', 'Unknown') - - return user - - # Main section if __name__ == '__main__': # Set log @@ -56,7 +29,7 @@ if __name__ == '__main__': print_standard(' ') # Check if running as root - if os.geteuid() != 0: + if not running_as_root(): print_error('ERROR: This script is meant to be run as root.') abort(False) @@ -71,81 +44,12 @@ if __name__ == '__main__': try: # Verify selections - sources = OrderedDict() - ## UFD - try: - ufd_dev = find_path(args['--ufd-device']) - except FileNotFoundError: - print_error('ERROR: UFD device not found: {}'.format( - args['--ufd-device'])) - abort(False) - if not is_valid_path(ufd_dev, 'UFD'): - print_error('ERROR: Invalid UFD device: {}'.format(ufd_dev)) - abort(False) - ## Sources - for label, data in UFD_SOURCES.items(): - s_path = args[data['Arg']] - if s_path: - try: - s_path_obj = find_path(s_path) - except FileNotFoundError: - print_error('ERROR: {} not found: {}'.format(label, s_path)) - abort(False) - if not is_valid_path(s_path_obj, data['Type']): - print_error('ERROR: Invalid {} source: {}'.format(label, s_path)) - abort(False) - sources[label] = s_path_obj + ufd_dev = verify_ufd(args['--ufd-device']) + sources = verify_sources(args, UFD_SOURCES) + show_selections(args, sources, ufd_dev, UFD_SOURCES) + confirm_selections(args) - # Show selections - ## Sources - print_info('Sources') - for label in UFD_SOURCES.keys(): - if label in sources: - print_standard(' {label:<18} {path}'.format( - label=label+':', - path=sources[label], - )) - else: - print_standard(' {label:<18} {YELLOW}Not Specified{CLEAR}'.format( - label=label+':', - **COLORS, - )) - ## Destination - print_standard(' ') - print_info('Destination') - cmd = [ - 'lsblk', '--nodeps', '--noheadings', - '--output', 'NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL', - ] - result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') - print_standard(result.stdout.strip()) - cmd = [ - 'lsblk', '--noheadings', - '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', - ] - result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') - for line in result.stdout.splitlines()[1:]: - print_standard(line) - if args['--update']: - print_warning('Updating kit in-place') - elif args['--use-mbr']: - print_warning('Formatting using legacy MBR') - print_standard(' ') - if not ask('Is the above information correct?'): - abort(False) - ## Safety check - if not args['--update']: - print_standard(' ') - print_warning('SAFETY CHECK') - print_standard( - 'All data will be DELETED from the disk and partition(s) listed above.') - print_standard( - 'This is irreversible and will lead to {RED}DATA LOSS.{CLEAR}'.format( - **COLORS)) - if not ask('Asking again to confirm, is this correct?'): - abort(False) - - print_standard(' ') + # TODO: DELETEME print_success("It's go-time!") exit_script() diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index e8fb164f..7c7a9b48 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -4,6 +4,8 @@ import os import re import shutil import pathlib +from collections import OrderedDict +from functions.common import * def case_insensitive_search(path, item): @@ -35,6 +37,25 @@ def case_insensitive_search(path, item): return real_path +def confirm_selections(args): + """Ask tech to confirm selections, twice if necessary.""" + if not ask('Is the above information correct?'): + abort(False) + ## Safety check + if not args['--update']: + print_standard(' ') + print_warning('SAFETY CHECK') + print_standard( + 'All data will be DELETED from the disk and partition(s) listed above.') + print_standard( + 'This is irreversible and will lead to {RED}DATA LOSS.{CLEAR}'.format( + **COLORS)) + if not ask('Asking again to confirm, is this correct?'): + abort(False) + + print_standard(' ') + + def find_path(path): """Find path case-insensitively, returns pathlib.Path obj.""" path_obj = pathlib.Path(path).resolve() @@ -62,6 +83,31 @@ def find_path(path): return path_obj +def get_user_home(user): + """Get path to user's home dir, returns str.""" + home_dir = None + cmd = ['getent', 'passwd', user] + result = run_program(cmd, encoding='utf-8', errors='ignore', check=False) + try: + home_dir = result.stdout.split(':')[5] + except Exception: + # Just use HOME from ENV (or '/root' if that fails) + home_dir = os.environ.get('HOME', '/root') + + return home_dir + + +def get_user_name(): + """Get real user name, returns str.""" + user = None + if 'SUDO_USER' in os.environ: + user = os.environ.get('SUDO_USER', 'Unknown') + else: + user = os.environ.get('USER', 'Unknown') + + return user + + def is_valid_path(path_obj, path_type): """Verify path_obj is valid by type, returns bool.""" valid_path = False @@ -126,6 +172,91 @@ def recursive_copy(source, dest, overwrite=True): raise FileExistsError('Refusing to delete file: {}'.format(dest)) +def running_as_root(): + """Check if running with effective UID of 0, returns bool.""" + return os.geteuid() == 0 + + +def show_selections(args, sources, ufd_dev, ufd_sources): + """Show selections including non-specified options.""" + + # Sources + print_info('Sources') + for label in ufd_sources.keys(): + if label in sources: + print_standard(' {label:<18} {path}'.format( + label=label+':', + path=sources[label], + )) + else: + print_standard(' {label:<18} {YELLOW}Not Specified{CLEAR}'.format( + label=label+':', + **COLORS, + )) + print_standard(' ') + + # Destination + print_info('Destination') + cmd = [ + 'lsblk', '--nodeps', '--noheadings', + '--output', 'NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL', + ] + result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') + print_standard(result.stdout.strip()) + cmd = [ + 'lsblk', '--noheadings', + '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', + ] + result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') + for line in result.stdout.splitlines()[1:]: + print_standard(line) + + # Notes + if args['--update']: + print_warning('Updating kit in-place') + elif args['--use-mbr']: + print_warning('Formatting using legacy MBR') + print_standard(' ') + + +def verify_sources(args, ufd_sources): + """Check all sources and abort if necessary, returns dict.""" + sources = OrderedDict() + + for label, data in ufd_sources.items(): + s_path = args[data['Arg']] + if s_path: + try: + s_path_obj = find_path(s_path) + except FileNotFoundError: + print_error('ERROR: {} not found: {}'.format(label, s_path)) + abort(False) + if not is_valid_path(s_path_obj, data['Type']): + print_error('ERROR: Invalid {} source: {}'.format(label, s_path)) + abort(False) + sources[label] = s_path_obj + + return sources + + +def verify_ufd(dev_path): + """Check that dev_path is a valid UFD, returns pathlib.Path obj.""" + ufd_dev = None + + try: + ufd_dev = find_path(dev_path) + except FileNotFoundError: + print_error('ERROR: UFD device not found: {}'.format( + args['--ufd-device'])) + abort(False) + + if not is_valid_path(ufd_dev, 'UFD'): + print_error('ERROR: Invalid UFD device: {}'.format(ufd_dev)) + abort(False) + + return ufd_dev + + if __name__ == '__main__': print("This file is not meant to be called directly.") From 72cc33cb7ed75a61bdb27b05d5cce271d82863ed Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 18:48:54 -0700 Subject: [PATCH 28/61] Only show destination device, not all devices --- .bin/Scripts/functions/ufd.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 7c7a9b48..7753d88a 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -200,12 +200,14 @@ def show_selections(args, sources, ufd_dev, ufd_sources): cmd = [ 'lsblk', '--nodeps', '--noheadings', '--output', 'NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL', + ufd_dev, ] result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') print_standard(result.stdout.strip()) cmd = [ 'lsblk', '--noheadings', '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', + ufd_dev, ] result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') for line in result.stdout.splitlines()[1:]: @@ -246,8 +248,7 @@ def verify_ufd(dev_path): try: ufd_dev = find_path(dev_path) except FileNotFoundError: - print_error('ERROR: UFD device not found: {}'.format( - args['--ufd-device'])) + print_error('ERROR: UFD device not found: {}'.format(dev_path)) abort(False) if not is_valid_path(ufd_dev, 'UFD'): From e098d40c2fb1d06bd9712ca317e756ac0d72f061 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Sun, 14 Apr 2019 18:49:32 -0700 Subject: [PATCH 29/61] Added pylint exceptions --- .bin/Scripts/build-ufd | 13 +++++++------ .bin/Scripts/functions/ufd.py | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 887932a7..72967920 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -1,6 +1,8 @@ #!/bin/env python3 # -## Wizard Kit: UFD build tool +# pylint: disable=no-name-in-module,wildcard-import +# vim: sts=2 sw=2 ts=2 +"""Wizard Kit: UFD build tool""" import os import sys @@ -15,12 +17,13 @@ init_global_vars(silent=True) # Main section if __name__ == '__main__': + # pylint: disable=invalid-name # Set log try: global_vars['LogDir'] = '{}/Logs'.format( get_user_home(get_user_name())) set_log_file('Build UFD ({Date-Time}).log'.format(**global_vars)) - except: + except: # pylint: disable=bare-except major_exception() # Header @@ -39,7 +42,7 @@ if __name__ == '__main__': except SystemExit as sys_exit: # Catch docopt exits exit_script(sys_exit.code) - except: + except: # pylint: disable=bare-except major_exception() try: @@ -74,7 +77,5 @@ if __name__ == '__main__': exit_script() except SystemExit as sys_exit: exit_script(sys_exit.code) - except: + except: # pylint: disable=bare-except major_exception() - -# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 7753d88a..1d9d9970 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -1,4 +1,6 @@ -# Wizard Kit: Functions - UFD +"""Wizard Kit: Functions - UFD""" +# pylint: disable=broad-except,wildcard-import +# vim: sts=2 sw=2 ts=2 import os import re @@ -260,5 +262,3 @@ def verify_ufd(dev_path): if __name__ == '__main__': print("This file is not meant to be called directly.") - -# vim: sts=2 sw=2 ts=2 From 2230ea1eeab4634155b6dde5983c0e6e49a69ceb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 20:30:39 -0700 Subject: [PATCH 30/61] Added prep_device() * Skipped if --update is used --- .bin/Scripts/build-ufd | 4 +-- .bin/Scripts/functions/ufd.py | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 72967920..1b29b91b 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -56,9 +56,9 @@ if __name__ == '__main__': print_success("It's go-time!") exit_script() - # Double-check if formating device - # Format and partition device + if not args['--update']: + prep_device(ufd_dev, UFD_LABEL, use_mbr=args['--use-mbr']) # Copy sources diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 1d9d9970..902d3b14 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -125,6 +125,59 @@ def is_valid_path(path_obj, path_type): return valid_path +def prep_device(dev_path, label, use_mbr=False): + """Format device in preparation for applying the WizardKit components + + This is done is four steps: + 1. Zero-out first 64MB (this deletes the partition table and/or bootloader) + 2. Create a new partition table (GPT by default, optionally MBR) + 3. Set boot flag + 4. Format partition (FAT32, 4K aligned) + """ + # Zero-out first 64MB + cmd = 'dd bs=4M count=16 if=/dev/zero of={}'.format(dev_path).split() + try_and_print( + message='Zeroing first 64MB...', + function=run_program, + cmd=cmd, + ) + + # Create partition table + cmd = 'parted {} --script -- mklabel {} primary fat32 4MiB {}'.format( + dev_path, + 'msdos' if use_mbr else 'gpt', + '-1s' if use_mbr else '-4MiB', + ).split() + try_and_print( + message='Creating partition table...', + function=run_program, + cmd=cmd, + ) + + # Set boot flag + cmd = 'parted {} set 1 {} on'.format( + dev_path, + 'boot' if use_mbr else 'legacy_boot', + ).split() + try_and_print( + message='Setting boot flag...', + function=run_program, + cmd=cmd, + ) + + # Format partition + cmd = [ + 'mkfs.vfat', '-F', '32', + '-n', label, + '{}1'.format(dev_path), + ] + try_and_print( + message='Formatting partition...', + function=run_program, + cmd=cmd, + ) + + def recursive_copy(source, dest, overwrite=True): """Copy source to dest recursively. From c8944e5a14b6beb87aa722b1a30a05867ff2f2af Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 21:27:27 -0700 Subject: [PATCH 31/61] Fixed prep_device() --- .bin/Scripts/build-ufd | 8 ++++---- .bin/Scripts/functions/ufd.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 1b29b91b..3c0abbe6 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -52,14 +52,14 @@ if __name__ == '__main__': show_selections(args, sources, ufd_dev, UFD_SOURCES) confirm_selections(args) - # TODO: DELETEME - print_success("It's go-time!") - exit_script() - # Format and partition device if not args['--update']: prep_device(ufd_dev, UFD_LABEL, use_mbr=args['--use-mbr']) + # TODO: DELETEME + print_success("It's go-time!") + exit_script() + # Copy sources # Update boot entries diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 902d3b14..34940e1f 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -143,7 +143,7 @@ def prep_device(dev_path, label, use_mbr=False): ) # Create partition table - cmd = 'parted {} --script -- mklabel {} primary fat32 4MiB {}'.format( + cmd = 'parted {} --script -- mklabel {} mkpart primary fat32 4MiB {}'.format( dev_path, 'msdos' if use_mbr else 'gpt', '-1s' if use_mbr else '-4MiB', @@ -165,11 +165,23 @@ def prep_device(dev_path, label, use_mbr=False): cmd=cmd, ) + # Find partition + cmd = [ + 'lsblk', + '--list', + '--noheadings', + '--output', 'name', + '--paths', + dev_path, + ] + result = run_program(cmd, encoding='utf-8', errors='ignore') + part_path = result.stdout.splitlines()[-1].strip() + # Format partition cmd = [ 'mkfs.vfat', '-F', '32', '-n', label, - '{}1'.format(dev_path), + part_path, ] try_and_print( message='Formatting partition...', From af757c5582514c5ee02a7395b9ce14a598b4cfff Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 21:28:53 -0700 Subject: [PATCH 32/61] Show full device paths --- .bin/Scripts/functions/ufd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 34940e1f..93e0644c 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -265,14 +265,14 @@ def show_selections(args, sources, ufd_dev, ufd_sources): # Destination print_info('Destination') cmd = [ - 'lsblk', '--nodeps', '--noheadings', - '--output', 'NAME,TRAN,SIZE,VENDOR,MODEL,SERIAL', + 'lsblk', '--nodeps', '--noheadings', '--paths', + '--output', 'NAME,FSTYPE,TRAN,SIZE,VENDOR,MODEL,SERIAL', ufd_dev, ] result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') print_standard(result.stdout.strip()) cmd = [ - 'lsblk', '--noheadings', + 'lsblk', '--noheadings', '--paths', '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', ufd_dev, ] From 807c94e2cef8d3207b57571183c49f2d5b91e433 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 21:29:10 -0700 Subject: [PATCH 33/61] Add argument for Linux (dGPU) --- .bin/Scripts/settings/ufd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index 13ecd4e9..c754d2b9 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -18,6 +18,7 @@ Usage: Options: -e PATH, --extra-dir PATH + -g PATH, --linux-dgpu PATH -k PATH, --main-kit PATH -l PATH, --linux PATH -m PATH, --linux-minimal PATH @@ -35,6 +36,7 @@ ISO_LABEL = '{}_LINUX'.format(KIT_NAME_SHORT) UFD_LABEL = '{}_UFD'.format(KIT_NAME_SHORT) UFD_SOURCES = OrderedDict({ 'Linux': {'Arg': '--linux', 'Type': 'ISO'}, + 'Linux (dGPU)': {'Arg': '--linux-dgpu', 'Type': 'ISO'}, 'Linux (Minimal)': {'Arg': '--linux-minimal', 'Type': 'ISO'}, 'WinPE': {'Arg': '--winpe', 'Type': 'ISO'}, 'Main Kit': {'Arg': '--main-kit', 'Type': 'KIT'}, From 996b01bb340aafcc91684d7ea577c6d98935c175 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 21:30:44 -0700 Subject: [PATCH 34/61] Add Linux dGPU items --- .bin/Scripts/settings/ufd.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index c754d2b9..2aa9b5e2 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -63,6 +63,12 @@ ITEMS = { ('/EFI/boot', '/EFI/'), ('/EFI/memtest86', '/EFI/'), ), + 'Linux (dGPU)': ( + ('/arch/boot/archiso.img', '/dgpu/'), + ('/arch/boot/vmlinuz', '/dgpu/'), + ('/arch/pkglist.x86_64.txt', '/dgpu/'), + ('/arch/x86_64', '/dgpu/'), + ), 'Linux (Minimal)': ( ('/arch/boot/archiso.img', '/arch_minimal/'), ('/arch/boot/vmlinuz', '/arch_minimal/'), From d769b1ad4ee79c9894cc20985b413cf4dbe05004 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 22:05:48 -0700 Subject: [PATCH 35/61] Set overwrite=False by default --- .bin/Scripts/functions/ufd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 93e0644c..06bb3e45 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -190,7 +190,7 @@ def prep_device(dev_path, label, use_mbr=False): ) -def recursive_copy(source, dest, overwrite=True): +def recursive_copy(source, dest, overwrite=False): """Copy source to dest recursively. NOTE: This uses rsync style source/dest syntax. From 7ee04060d80d7176cae7a66e08f77ce4821d9481 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 22:37:58 -0700 Subject: [PATCH 36/61] Added dGPU to BOOT_ENTRIES --- .bin/Scripts/settings/ufd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index 2aa9b5e2..b09f84fe 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -46,7 +46,8 @@ UFD_SOURCES = OrderedDict({ # Definitions: Boot entries ## NOTE: if key path exists uncomment #value# lines BOOT_ENTRIES = { - 'arch_minimal': 'MINIMAL', + 'arch_minimal': 'MINIMAL', + 'dgpu': 'DGPU', 'sources/boot.wim': 'WINPE', } From 26aca0df9f87d8d5a6ffbb23e69cb661c85dfccb Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 22:38:54 -0700 Subject: [PATCH 37/61] Added mount and unmount sections --- .bin/Scripts/build-ufd | 12 +++++++----- .bin/Scripts/functions/ufd.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 3c0abbe6..7571eae3 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -56,9 +56,13 @@ if __name__ == '__main__': if not args['--update']: prep_device(ufd_dev, UFD_LABEL, use_mbr=args['--use-mbr']) - # TODO: DELETEME - print_success("It's go-time!") - exit_script() + # Mount UFD + try_and_print( + message='Mounting UFD...', + function=mount, + mount_source=ufd_dev, + mount_point='/mnt/UFD', + ) # Copy sources @@ -68,8 +72,6 @@ if __name__ == '__main__': # Hide items - # Unmount sources - # Done if not args['--force']: print_standard('\nDone.') diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 06bb3e45..96ce2d0a 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -125,6 +125,19 @@ def is_valid_path(path_obj, path_type): return valid_path +def mount(mount_source, mount_point): + """Mount mount_source on mount_point.""" + os.makedirs(mount_point, exist_ok=True) + cmd = ['mount', mount_source, mount_point] + run_program(cmd) + + +def unmount(mount_point): + """Unmount mount_point.""" + cmd = ['umount', mount_point] + run_program(cmd) + + def prep_device(dev_path, label, use_mbr=False): """Format device in preparation for applying the WizardKit components From fc9de6126996556c4cf672e29f0d1905466802dd Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 22:40:20 -0700 Subject: [PATCH 38/61] Added copy_source() --- .bin/Scripts/build-ufd | 8 ++++++++ .bin/Scripts/functions/ufd.py | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 7571eae3..fbe1bdff 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -65,6 +65,14 @@ if __name__ == '__main__': ) # Copy sources + for s_label, s_path in sources.items(): + try_and_print( + message='Copying {} files...'.format(s_label), + function=copy_source, + source=s_path, + items=ITEMS[s_label], + overwrite=True, + ) # Update boot entries diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 96ce2d0a..344c273d 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -58,6 +58,17 @@ def confirm_selections(args): print_standard(' ') +def copy_source(source, items, overwrite=False): + """Mount source and copy items to /mnt/UFD.""" + os.makedirs('/mnt/Source', exist_ok=True) + mount(source, '/mnt/Source') + for i_source, i_dest in items: + i_source = '/mnt/Source{}'.format(i_source) + i_dest = '/mnt/UFD{}'.format(i_dest) + recursive_copy(i_source, i_dest, overwrite=overwrite) + unmount('/mnt/Source') + + def find_path(path): """Find path case-insensitively, returns pathlib.Path obj.""" path_obj = pathlib.Path(path).resolve() From 95f4877862e8edcc63895820bfed4a9992dccb2f Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 23:04:37 -0700 Subject: [PATCH 39/61] Reordered functions --- .bin/Scripts/functions/ufd.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 344c273d..9a17ea64 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -60,7 +60,6 @@ def confirm_selections(args): def copy_source(source, items, overwrite=False): """Mount source and copy items to /mnt/UFD.""" - os.makedirs('/mnt/Source', exist_ok=True) mount(source, '/mnt/Source') for i_source, i_dest in items: i_source = '/mnt/Source{}'.format(i_source) @@ -143,12 +142,6 @@ def mount(mount_source, mount_point): run_program(cmd) -def unmount(mount_point): - """Unmount mount_point.""" - cmd = ['umount', mount_point] - run_program(cmd) - - def prep_device(dev_path, label, use_mbr=False): """Format device in preparation for applying the WizardKit components @@ -312,6 +305,12 @@ def show_selections(args, sources, ufd_dev, ufd_sources): print_standard(' ') +def unmount(mount_point): + """Unmount mount_point.""" + cmd = ['umount', mount_point] + run_program(cmd) + + def verify_sources(args, ufd_sources): """Check all sources and abort if necessary, returns dict.""" sources = OrderedDict() From c16e97e49a4f72dffcec2138137abbe38deaf3fa Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 23:07:33 -0700 Subject: [PATCH 40/61] Added boot entry sections --- .bin/Scripts/build-ufd | 6 ++++++ .bin/Scripts/functions/ufd.py | 29 +++++++++++++++++++++++++++++ .bin/Scripts/settings/ufd.py | 13 +++++++++---- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index fbe1bdff..e21d876c 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -75,6 +75,12 @@ if __name__ == '__main__': ) # Update boot entries + try_and_print( + message='Enabling boot entries...', + function=enable_boot_entries, + boot_entries=BOOT_ENTRIES, + boot_files=BOOT_FILES, + ) # Install syslinux diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 9a17ea64..dbce5bb6 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -68,6 +68,35 @@ def copy_source(source, items, overwrite=False): unmount('/mnt/Source') +def enable_boot_entries(boot_entries, boot_files): + """Enable boot entries if related paths exist.""" + configs = [] + + # Find config files + for c_path, c_ext in boot_files.items(): + c_path = find_path(c_path) + for item in os.scandir(c_path): + if item.name.lower().endswith(c_ext.lower()): + configs.append(item.path) + + # Uncomment found entries + for b_path, b_comment in boot_entries: + try: + find_path('/mnt/UFD{}'.format(b_path)) + except (FileNotFoundError, NotADirectoryError): + # Entry not found, continue to next entry + continue + + # Update config files + cmd = [ + 'sed', + '--in-place', + '"s/#{}#//"'.format(b_comment), + *configs, + ] + run_program(cmd) + + def find_path(path): """Find path case-insensitively, returns pathlib.Path obj.""" path_obj = pathlib.Path(path).resolve() diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index b09f84fe..dc7693c4 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -44,11 +44,16 @@ UFD_SOURCES = OrderedDict({ }) # Definitions: Boot entries -## NOTE: if key path exists uncomment #value# lines BOOT_ENTRIES = { - 'arch_minimal': 'MINIMAL', - 'dgpu': 'DGPU', - 'sources/boot.wim': 'WINPE', + # Path to check: Comment to remove + '/arch_minimal': 'UFD-MINIMAL', + '/dgpu': 'UFD-DGPU', + '/sources/boot.wim': 'UFD-WINPE', + } +BOOT_FILES = { + # Directory: extension + '/arch/boot/syslinux': 'cfg', + '/EFI/boot': 'conf', } # Definitions: Sources and Destinations From c66b2facbf37bafc00f801f2c7e93133f70d0a1b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 23:12:27 -0700 Subject: [PATCH 41/61] Skip confirm_selections if --force is used --- .bin/Scripts/build-ufd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index e21d876c..4ad726e1 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -50,7 +50,8 @@ if __name__ == '__main__': ufd_dev = verify_ufd(args['--ufd-device']) sources = verify_sources(args, UFD_SOURCES) show_selections(args, sources, ufd_dev, UFD_SOURCES) - confirm_selections(args) + if not args['--force']: + confirm_selections(args) # Format and partition device if not args['--update']: From 4115f3cfe178648b5e4f637d204e04c50d4309b2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 23:16:57 -0700 Subject: [PATCH 42/61] Removed unused args --- .bin/Scripts/settings/ufd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index dc7693c4..5123a8a1 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -17,17 +17,15 @@ Usage: build-ufd (-h | --help) Options: + -d PATH, --linux-dgpu PATH -e PATH, --extra-dir PATH - -g PATH, --linux-dgpu PATH -k PATH, --main-kit PATH -l PATH, --linux PATH -m PATH, --linux-minimal PATH -u PATH, --ufd-device PATH -w PATH, --winpe PATH - -d --debug Enable debug mode -h --help Show this page - -v --verbose Enable verbose mode -M --use-mbr Use real MBR instead of GPT w/ Protective MBR -F --force Bypass all confirmation messages. USE WITH EXTREME CAUTION! -U --update Don't format device, just update From 89ac824d828a7adf7f4c90bda0d10c27201cf795 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 23:44:42 -0700 Subject: [PATCH 43/61] Expanded boot entries sections * Update Arch labels from ISO_LABEL to UFD_LABEL * Added Linux (Minimal) to syslinux * Added Linux (Minimal) and Linux (dGPU) entries to rEFInd * Removed Linux (i3) entries * I guess this is a hidden feature now... --- .bin/Scripts/build-ufd | 6 +++-- .bin/Scripts/functions/ufd.py | 18 ++++++++++--- .linux_items/include/EFI/boot/icons/dgpu.png | Bin 0 -> 3086 bytes .linux_items/include/EFI/boot/refind.conf | 23 +++++++++++++--- .../include/syslinux/wk_sys_linux.cfg | 25 +++++++++--------- 5 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 .linux_items/include/EFI/boot/icons/dgpu.png diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 4ad726e1..646d96dd 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -77,10 +77,12 @@ if __name__ == '__main__': # Update boot entries try_and_print( - message='Enabling boot entries...', - function=enable_boot_entries, + message='Updating boot entries...', + function=update_boot_entries, boot_entries=BOOT_ENTRIES, boot_files=BOOT_FILES, + iso_label=ISO_LABEL, + ufd_label=UFD_LABEL, ) # Install syslinux diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index dbce5bb6..f6083da0 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -68,8 +68,8 @@ def copy_source(source, items, overwrite=False): unmount('/mnt/Source') -def enable_boot_entries(boot_entries, boot_files): - """Enable boot entries if related paths exist.""" +def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): + """Update boot files for UFD usage""" configs = [] # Find config files @@ -79,7 +79,17 @@ def enable_boot_entries(boot_entries, boot_files): if item.name.lower().endswith(c_ext.lower()): configs.append(item.path) - # Uncomment found entries + # Update Linux labels + cmd = [ + 'sed', + '--in-place', + '--regexp-extended', + 's/{}/{}/'.format(iso_label, ufd_label), + *configs, + ] + run_program(cmd) + + # Uncomment extra entries if present for b_path, b_comment in boot_entries: try: find_path('/mnt/UFD{}'.format(b_path)) @@ -87,7 +97,7 @@ def enable_boot_entries(boot_entries, boot_files): # Entry not found, continue to next entry continue - # Update config files + # Entry found, update config files cmd = [ 'sed', '--in-place', diff --git a/.linux_items/include/EFI/boot/icons/dgpu.png b/.linux_items/include/EFI/boot/icons/dgpu.png new file mode 100644 index 0000000000000000000000000000000000000000..e9770abd1ba2487b1e3d08e9a643004f94d019c3 GIT binary patch literal 3086 zcmV+p4Ds`cP)Tvy7{wCK`mp%wa=cJ5>U=$&R3OXpGM z&Y8LAo7|+G%$>*nxBwb z4BQ{8JP-;11=u3fGGVnf$j6AY0JjCm05D&s+Kt+$ewm&ekjOF4Y_FfA=D zBHiK;kZolF+T~0R6#=CA!3pMevuxQicI?=}qD6}^O%p{?Fin#mfBaE6MngpacL?)k zi(eo?6otaVLdwd@n3$MAQIzS=gW(Xgh0wJ8G$V}w5&+=(>#wJ#riS$NbS5V!|IhSb zUA%ZvQ~-=fBS1J+L`Uf9r=Mocnl+ea=suUTXU`^HmL@H}ysE0I$j!|S%Y1;7Cr=8S zgGm4)kjLX;-@bk1l6#XP|YeZoJ#;{kZ+p@$+q z=iIq-l$4Yt#q&EJfWpGUh`v8OJj}xnKb$nr?`QxrGBWV_d|_AjdU|@;ym@nS)qtD~ zz`S|$!v1bULjzS+RmoQcb}|4Sk0<0?-{0R)MMVXjot;UU295^c(q`X4zrV-hp|`h} z{rmUR)YO#JY2au8d_Et#uH*On(KLYP-%jc;~%`_cBYW9+} zs$!aE*dvE*)*k>Jm>|MIp+aiy+O@1$v4UG}xrMZ}wCN^Xzu(WVzy3;ldpnJdjp(`_ zlk4Am@4XZi6>;sg*Wz}&al75PTrLd5z%UGST}RV2Mn^|!Yir}cfddQ=56cJ;td(Y2tFZICbh2pMU;2 zpMLsj*ypZVwThymB37iRR?%eUWtOLcWM-QC?1fSKg=dU@=z$5^vw%@zG! z(=_pVy&O1jfVQ@_8LM@R7A@kAJMLi9rcHP}o;jO7+r2b5H&aiju1R$W>Zo7?& ziVFOGe@w4qnkH#!X&gOzl>YvH($mwq{r1~uob40KdjQzJeLFopJraPOJ9n~i<3?K& zU$h!Td3ibQ?d{G5prWFJB}ebTuAnCf!d+)uM`~YNU zXR~|vZbygxnbh6g%>xfSko*9=^2#eQYpP9DwYIjhbLUPpO|$j7w)X|BU%#G(3l~bq zgYy2my1KYcF0s8Dc<9ifkOMKCWPWXJEp>HuvAm2p)xgc0H;cNCPF&2dudk1_`C^p; z)~#D73w}WbzM7hvxZj6u0OaTAla-YvooA|xc=ztzLfngO06hBWqfyPkNk+Z)+G~uA zj5rH`8*jW(hJRpUVuCNe_(GWbu?2v6^X75&)mO_`A9&-9H-w6Wwg9kU!v@*#1B{Q4 z$Fkd3TmaUuUoWeDrfKIeFmV7-Rh5j4495eYC<^bt|GqN;aJgKL2f#24A=eU&IRGw~ zE2`yJl1&2x147IB`O{Z|R5#aRFxJ$h6+A5<22^2sNi1wda!olRwBr85Bd;DZljIba3|^7Hd~_uY5nRx_{-fZu-mjlsb|>G}*1+;GDUG&VMp zlarIY004ad{r55(xC{g?my6okT8fK{Z5>5zD*%TM9g=Q`0Ab6PEi^SXv2x|gU_9$BrG7i$>0%ufP7<)|td{1HkY1vuDp9>G&mYzujpP*OeO%A3iL{ z3kK<%Z@#g8{lK^{-SN^(FG=SCP1EACdT87M`0A^#=r+_S-^adBSZUA9y<&8X8DVO-)Jwe*XDqwr<^O*X0st`}km`rl!)^ z*qC(7%xap(#*G{8+#fJ;$^gNdn3!PKu3bq5fN7d++O)~`%^?%23=pjD?rzG<%VRd7 z#5BzrkC^1Oio%W^JA~OQXga$U{jF(lZ%0uS%F4>-{N|XZNlHozr%s)sv$K;kXU_1; zFTYH$C>l$OW)bb04WwALLj?j$r4_D z^;O(%cSMGmrishtqNk^a=H_NT`Q($Zm#oRm%%r%um^<&hGpaR@P19s>aF8dSc!K`^ z{&*$b9{>%QjIa(=>cOA9Zzg?BBotieKC7^|EW%E(!_?@OV6^s*0+r)4=)tewv$`*}HeI2uXDj z$g#}#M*;vA09OI^K%t1`165T?PfsT+D+`~`$Kc=~nx@&ZE}^0*(}&ybM%Q%=!-#1? zG77Z<{{TiU03s>`2A<+$Q2{UvgOQPuxRk7ejTU$rypJ#Ol1L(e3V5yX-vKN{Dv@*+ z_zQ3W7zd0<+W|}~Y-qrCnVt!!?UsN3*&;xw7M8$?=Ycc8UuC)`g#HP%SXKDRxd4C_ zI!vGkI1UtuRx6O2RzI)}IAYZRe3uLT;gkV_zvB)n4`c%O14Tf-gh7a^-oJr^K!X*n zFIxG&Hrv}DE&ze=xtQu9^jgPN){%nXeF{k|!wT`E*5L!jnCkh{Lzy250DL};3{{R3007*qoM6N<$f`+B;Bme*a literal 0 HcmV?d00001 diff --git a/.linux_items/include/EFI/boot/refind.conf b/.linux_items/include/EFI/boot/refind.conf index 4ab3be7b..62a639f7 100644 --- a/.linux_items/include/EFI/boot/refind.conf +++ b/.linux_items/include/EFI/boot/refind.conf @@ -26,15 +26,30 @@ menuentry "Linux" { initrd /arch/boot/amd_ucode.img initrd /arch/boot/x86_64/archiso.img options "archisobasedir=arch archisolabel=%ARCHISO_LABEL% copytoram loglevel=3" - submenuentry "Linux (i3)" { - add_options "i3" - } submenuentry "Linux (CLI)" { - add_options "loglevel=4 nomodeset nox" + add_options "nox" } + #UFD-MINIMAL#submenuentry "Linux (Minimal)" { + #UFD-MINIMAL# loader /arch_minimal/vmlinuz + #UFD-MINIMAL# initrd + #UFD-MINIMAL# initrd /arch/boot/intel_ucode.img + #UFD-MINIMAL# initrd /arch/boot/amd_ucode.img + #UFD-MINIMAL# initrd /arch_minimal/archiso.img + #UFD-MINIMAL# options + #UFD-MINIMAL# options "archisobasedir=arch_minimal archisolabel=%ARCHISO_LABEL% copytoram loglevel=3" + #UFD-MINIMAL#} } #UFD-WINPE#menuentry "WindowsPE" { #UFD-WINPE# ostype windows #UFD-WINPE# icon /EFI/boot/icons/wk_win.png #UFD-WINPE# loader /EFI/microsoft/bootx64.efi #UFD-WINPE#} +#UFD-DGPU#menuentry "Mac dGPU Disable Tool" { +#UFD-DGPU# icon /EFI/boot/icons/dgpu.png +#UFD-DGPU# loader /dgpu/vmlinuz +#UFD-DGPU# initrd /arch/boot/intel_ucode.img +#UFD-DGPU# initrd /arch/boot/amd_ucode.img +#UFD-DGPU# initrd /dgpu/archiso.img +#UFD-DGPU# options "archisobasedir=dgpu archisolabel=%ARCHISO_LABEL% nomodeset" +#UFD-DGPU#} + diff --git a/.linux_items/include/syslinux/wk_sys_linux.cfg b/.linux_items/include/syslinux/wk_sys_linux.cfg index d4319c00..b6a9370c 100644 --- a/.linux_items/include/syslinux/wk_sys_linux.cfg +++ b/.linux_items/include/syslinux/wk_sys_linux.cfg @@ -8,17 +8,6 @@ LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% copytoram loglevel=3 -LABEL wk_linux_i3 -TEXT HELP -A live Linux environment (i3) - * HW diagnostics, file-based backups, data recovery, etc -ENDTEXT -MENU LABEL Linux (i3) -LINUX boot/x86_64/vmlinuz -INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% copytoram loglevel=3 i3 -SYSAPPEND 3 - LABEL wk_linux_cli TEXT HELP A live Linux environment (CLI) @@ -27,5 +16,17 @@ ENDTEXT MENU LABEL Linux (CLI) LINUX boot/x86_64/vmlinuz INITRD boot/intel_ucode.img,boot/amd_ucode.img,boot/x86_64/archiso.img -APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% copytoram nox nomodeset +APPEND archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% copytoram nox SYSAPPEND 3 + +#UFD-MINIMAL#LABEL wk_linux_minimal +#UFD-MINIMAL#TEXT HELP +#UFD-MINIMAL#A live Linux environment (Minimal) +#UFD-MINIMAL# * HW diagnostics, file-based backups, data recovery, etc +#UFD-MINIMAL#ENDTEXT +#UFD-MINIMAL#MENU LABEL Linux (Minimal) +#UFD-MINIMAL#LINUX ../arch_minimal/vmlinuz +#UFD-MINIMAL#INITRD boot/intel_ucode.img,boot/amd_ucode.img,../arch_minimal/archiso.img +#UFD-MINIMAL#APPEND archisobasedir=arch_minimal archisolabel=%ARCHISO_LABEL% copytoram loglevel=3 +#UFD-MINIMAL#SYSAPPEND 3 + From 9652aa4623565503134ea2f3c238625a3aa43a0a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Mon, 15 Apr 2019 23:57:46 -0700 Subject: [PATCH 44/61] Added install Syslinux section --- .bin/Scripts/build-ufd | 6 ++++++ .bin/Scripts/functions/ufd.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 646d96dd..1fbb1f46 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -86,6 +86,12 @@ if __name__ == '__main__': ) # Install syslinux + try_and_print( + message='Installing Syslinux...', + function=install_syslinux, + ufd_dev=ufd_dev, + use_mbr=args['--use-mbr'], + ) # Hide items diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index f6083da0..5b289a73 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -159,6 +159,20 @@ def get_user_name(): return user +def install_syslinux(ufd_dev, use_mbr): + """Install Syslinux to UFD.""" + cmd = [ + 'dd', + 'bs=440', + 'count=1', + 'if=/usr/lib/syslinux/bios/{}.bin'.format( + 'mbr' if use_mbr else 'gptmbr', + ), + 'of={}'.format(ufd_dev), + ] + run_program(cmd) + + def is_valid_path(path_obj, path_type): """Verify path_obj is valid by type, returns bool.""" valid_path = False From 4215a2fd78103a5a3e6c36b8a1bda8959cf26f5e Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Apr 2019 00:14:46 -0700 Subject: [PATCH 45/61] Added find_first_partition() --- .bin/Scripts/functions/ufd.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 5b289a73..274b50c1 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -107,6 +107,22 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): run_program(cmd) +def find_first_partition(dev_path): + """Find path to first partition of dev, returns str.""" + cmd = [ + 'lsblk', + '--list', + '--noheadings', + '--output', 'name', + '--paths', + dev_path, + ] + result = run_program(cmd, encoding='utf-8', errors='ignore') + part_path = result.stdout.splitlines()[-1].strip() + + return part_path + + def find_path(path): """Find path case-insensitively, returns pathlib.Path obj.""" path_obj = pathlib.Path(path).resolve() @@ -235,23 +251,11 @@ def prep_device(dev_path, label, use_mbr=False): cmd=cmd, ) - # Find partition - cmd = [ - 'lsblk', - '--list', - '--noheadings', - '--output', 'name', - '--paths', - dev_path, - ] - result = run_program(cmd, encoding='utf-8', errors='ignore') - part_path = result.stdout.splitlines()[-1].strip() - # Format partition cmd = [ 'mkfs.vfat', '-F', '32', '-n', label, - part_path, + find_first_partition(dev_path), ] try_and_print( message='Formatting partition...', From a9461311bc97075d083160d5bf1bccc669483330 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Apr 2019 00:15:33 -0700 Subject: [PATCH 46/61] Unmount UFD before installing Syslinux --- .bin/Scripts/build-ufd | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 1fbb1f46..27f0b1af 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -85,6 +85,13 @@ if __name__ == '__main__': ufd_label=UFD_LABEL, ) + # Unmount UFD + try_and_print( + message='Unmounting UFD...', + function=mount, + mount_point='/mnt/UFD', + ) + # Install syslinux try_and_print( message='Installing Syslinux...', From 949efa4de60d604222944ac4394a63099a40a5c0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Apr 2019 00:16:05 -0700 Subject: [PATCH 47/61] Added hide items sections --- .bin/Scripts/build-ufd | 6 ++++++ .bin/Scripts/functions/ufd.py | 14 ++++++++++++++ .bin/Scripts/settings/ufd.py | 15 +++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 27f0b1af..21363677 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -101,6 +101,12 @@ if __name__ == '__main__': ) # Hide items + try_and_print( + message='Hiding items...', + function=hide_items, + ufd_dev=ufd_dev, + items=ITEMS_HIDDEN, + ) # Done if not args['--force']: diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 274b50c1..de4994cb 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -175,6 +175,20 @@ def get_user_name(): return user +def hide_items(ufd_dev, items): + """Set FAT32 hidden flag for items.""" + # pylint: disable=invalid-name + with open('/root/.mtoolsrc', 'w') as f: + f.write('drive U: file="{}"\n'.format( + find_first_partition(ufd_dev))) + f.write('mtools_skip_check=1\n') + + # Hide items + for item in items: + cmd = ['yes | mattrib +h "U:/{}"'.format(item)] + run_program(cmd, check=False, shell=True) + + def install_syslinux(ufd_dev, use_mbr): """Install Syslinux to UFD.""" cmd = [ diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index 5123a8a1..db4a2127 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -95,6 +95,21 @@ ITEMS = { ('/sources/boot.wim', '/sources/'), ), } +ITEMS_HIDDEN = ( + # Linux (all versions) + 'arch', + 'arch_minimal', + 'dgpu', + 'EFI', + 'isolinux', + # WinPE + 'boot', + 'bootmgr', + 'bootmgr.efi', + 'en-us', + 'images', + 'sources', + ) if __name__ == '__main__': print("This file is not meant to be called directly.") From cd4ea7586208bb36e7fda7e4ad08a1b2e18767c2 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Apr 2019 00:23:58 -0700 Subject: [PATCH 48/61] Remove the arch dir when updating the UFD --- .bin/Scripts/build-ufd | 11 +++++++++-- .bin/Scripts/functions/ufd.py | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 21363677..2cb06ac1 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -53,8 +53,15 @@ if __name__ == '__main__': if not args['--force']: confirm_selections(args) - # Format and partition device - if not args['--update']: + # Prep UFD + if args['--update']: + # Remove arch folder + try_and_print( + message='Removing Linux...', + function=remove_arch, + ) + else: + # Format and partition prep_device(ufd_dev, UFD_LABEL, use_mbr=args['--use-mbr']) # Mount UFD diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index de4994cb..f771c1da 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -327,6 +327,14 @@ def recursive_copy(source, dest, overwrite=False): raise FileExistsError('Refusing to delete file: {}'.format(dest)) +def remove_arch(): + """Remove arch dir from UFD. + + This ensures a clean installation to the UFD and resets the boot files + """ + shutil.rmtree(find_path('/mnt/UFD/arch')) + + def running_as_root(): """Check if running with effective UID of 0, returns bool.""" return os.geteuid() == 0 From 427243c9339a4c2b461e543a31f300fa73712529 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Tue, 16 Apr 2019 00:33:47 -0700 Subject: [PATCH 49/61] Reordered functions --- .bin/Scripts/functions/ufd.py | 78 +++++++++++++++++------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index f771c1da..51b1ad7c 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -68,45 +68,6 @@ def copy_source(source, items, overwrite=False): unmount('/mnt/Source') -def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): - """Update boot files for UFD usage""" - configs = [] - - # Find config files - for c_path, c_ext in boot_files.items(): - c_path = find_path(c_path) - for item in os.scandir(c_path): - if item.name.lower().endswith(c_ext.lower()): - configs.append(item.path) - - # Update Linux labels - cmd = [ - 'sed', - '--in-place', - '--regexp-extended', - 's/{}/{}/'.format(iso_label, ufd_label), - *configs, - ] - run_program(cmd) - - # Uncomment extra entries if present - for b_path, b_comment in boot_entries: - try: - find_path('/mnt/UFD{}'.format(b_path)) - except (FileNotFoundError, NotADirectoryError): - # Entry not found, continue to next entry - continue - - # Entry found, update config files - cmd = [ - 'sed', - '--in-place', - '"s/#{}#//"'.format(b_comment), - *configs, - ] - run_program(cmd) - - def find_first_partition(dev_path): """Find path to first partition of dev, returns str.""" cmd = [ @@ -390,6 +351,45 @@ def unmount(mount_point): run_program(cmd) +def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): + """Update boot files for UFD usage""" + configs = [] + + # Find config files + for c_path, c_ext in boot_files.items(): + c_path = find_path(c_path) + for item in os.scandir(c_path): + if item.name.lower().endswith(c_ext.lower()): + configs.append(item.path) + + # Update Linux labels + cmd = [ + 'sed', + '--in-place', + '--regexp-extended', + 's/{}/{}/'.format(iso_label, ufd_label), + *configs, + ] + run_program(cmd) + + # Uncomment extra entries if present + for b_path, b_comment in boot_entries: + try: + find_path('/mnt/UFD{}'.format(b_path)) + except (FileNotFoundError, NotADirectoryError): + # Entry not found, continue to next entry + continue + + # Entry found, update config files + cmd = [ + 'sed', + '--in-place', + '"s/#{}#//"'.format(b_comment), + *configs, + ] + run_program(cmd) + + def verify_sources(args, ufd_sources): """Check all sources and abort if necessary, returns dict.""" sources = OrderedDict() From 21a587ee0f95872e353fd6db70be1bf327faa91c Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 19:24:39 -0700 Subject: [PATCH 50/61] Bugfix sleep() --- .bin/Scripts/functions/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index fe75f1ef..28834660 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -94,7 +94,7 @@ def abort(show_prompt=True): """Abort script.""" print_warning('Aborted.') if show_prompt: - sleep(timeout) + sleep(1) pause(prompt='Press Enter to exit... ') exit_script(1) From e07c019fb2c422db772b1ced224a87793ebd2d3b Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 19:25:13 -0700 Subject: [PATCH 51/61] Bugfix: Call unmount() not mount() --- .bin/Scripts/build-ufd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 2cb06ac1..5dd15a7c 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -95,7 +95,7 @@ if __name__ == '__main__': # Unmount UFD try_and_print( message='Unmounting UFD...', - function=mount, + function=unmount, mount_point='/mnt/UFD', ) From 050bef7f85fb7aeb10e2c1407d71bd7f6837b141 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 19:26:28 -0700 Subject: [PATCH 52/61] Skip FileNotFoundError in copy_source() * Avoids crash when copying some WinPE ISOs --- .bin/Scripts/functions/ufd.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 51b1ad7c..1bbe0b53 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -64,7 +64,11 @@ def copy_source(source, items, overwrite=False): for i_source, i_dest in items: i_source = '/mnt/Source{}'.format(i_source) i_dest = '/mnt/UFD{}'.format(i_dest) - recursive_copy(i_source, i_dest, overwrite=overwrite) + try: + recursive_copy(i_source, i_dest, overwrite=overwrite) + except FileNotFoundError: + # Going to assume (hope) that this is fine + pass unmount('/mnt/Source') From 153cda7cf8f6652924bfd588fe6e9bb98b2ad9f4 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 19:27:41 -0700 Subject: [PATCH 53/61] Fixed update_boot_entries() --- .bin/Scripts/functions/ufd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 1bbe0b53..be341b59 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -361,7 +361,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): # Find config files for c_path, c_ext in boot_files.items(): - c_path = find_path(c_path) + c_path = find_path('/mnt/UFD{}'.format(c_path)) for item in os.scandir(c_path): if item.name.lower().endswith(c_ext.lower()): configs.append(item.path) @@ -377,7 +377,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): run_program(cmd) # Uncomment extra entries if present - for b_path, b_comment in boot_entries: + for b_path, b_comment in boot_entries.items(): try: find_path('/mnt/UFD{}'.format(b_path)) except (FileNotFoundError, NotADirectoryError): From f062f3ce7168e7de14357efc2f5c14a97d8b48c6 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 19:28:15 -0700 Subject: [PATCH 54/61] Fixed source ITEMS --- .bin/Scripts/settings/ufd.py | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.bin/Scripts/settings/ufd.py b/.bin/Scripts/settings/ufd.py index db4a2127..94cf333e 100644 --- a/.bin/Scripts/settings/ufd.py +++ b/.bin/Scripts/settings/ufd.py @@ -59,40 +59,40 @@ BOOT_FILES = { ## Sources use rsync's trailing slash syntax ITEMS = { 'Extra Dir': ( - ('/', '/'), + ('/', '/'), ), 'Linux': ( - ('/arch', '/'), - ('/isolinux', '/'), - ('/EFI/boot', '/EFI/'), - ('/EFI/memtest86', '/EFI/'), + ('/arch', '/'), + ('/isolinux', '/'), + ('/EFI/boot', '/EFI/'), + ('/EFI/memtest86', '/EFI/'), ), 'Linux (dGPU)': ( - ('/arch/boot/archiso.img', '/dgpu/'), - ('/arch/boot/vmlinuz', '/dgpu/'), - ('/arch/pkglist.x86_64.txt', '/dgpu/'), - ('/arch/x86_64', '/dgpu/'), + ('/arch/boot/x86_64/archiso.img', '/dgpu/'), + ('/arch/boot/x86_64/vmlinuz', '/dgpu/'), + ('/arch/pkglist.x86_64.txt', '/dgpu/'), + ('/arch/x86_64', '/dgpu/'), ), 'Linux (Minimal)': ( - ('/arch/boot/archiso.img', '/arch_minimal/'), - ('/arch/boot/vmlinuz', '/arch_minimal/'), - ('/arch/pkglist.x86_64.txt', '/arch_minimal/'), - ('/arch/x86_64', '/arch_minimal/'), + ('/arch/boot/x86_64/archiso.img', '/arch_minimal/'), + ('/arch/boot/x86_64/vmlinuz', '/arch_minimal/'), + ('/arch/pkglist.x86_64.txt', '/arch_minimal/'), + ('/arch/x86_64', '/arch_minimal/'), ), 'Main Kit': ( - ('/', '/{}/'.format(KIT_NAME_FULL)), + ('/', '/{}/'.format(KIT_NAME_FULL)), ), 'WinPE': ( - ('/bootmgr', '/'), - ('/bootmgr.efi', '/'), - ('/en_us', '/'), - ('/Boot/', '/boot/'), - ('/EFI/Boot/', '/EFI/Microsoft/'), - ('/EFI/Microsoft/', '/EFI/Microsoft/'), - ('/Boot/BCD', '/sources/'), - ('/Boot/boot.sdi', '/sources/'), - ('/bootmgr', '/sources/'), - ('/sources/boot.wim', '/sources/'), + ('/bootmgr', '/'), + ('/bootmgr.efi', '/'), + ('/en_us', '/'), + ('/Boot/', '/boot/'), + ('/EFI/Boot/', '/EFI/Microsoft/'), + ('/EFI/Microsoft/', '/EFI/Microsoft/'), + ('/Boot/BCD', '/sources/'), + ('/Boot/boot.sdi', '/sources/'), + ('/bootmgr', '/sources/'), + ('/sources/boot.wim', '/sources/'), ), } ITEMS_HIDDEN = ( From b05057d743bb313bc4c3c4ef2841e2d3c7f6a1b1 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 20:00:37 -0700 Subject: [PATCH 55/61] Ignore errors when uncommenting boot entries * Allows using older ISOs which lack newer boot entries * Not a great idea, but it won't crash --- .bin/Scripts/functions/ufd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index be341b59..c1296d44 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -391,7 +391,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): '"s/#{}#//"'.format(b_comment), *configs, ] - run_program(cmd) + run_program(cmd, check=False) def verify_sources(args, ufd_sources): From 27eb7975ef321bfcc8d5e326c41660dfb4737a53 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 20:51:14 -0700 Subject: [PATCH 56/61] Fixed SYSLINUX installation --- .bin/Scripts/build-ufd | 13 ++++++++++--- .bin/Scripts/functions/ufd.py | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 5dd15a7c..4d0072c4 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -92,6 +92,13 @@ if __name__ == '__main__': ufd_label=UFD_LABEL, ) + # Install syslinux (to partition) + try_and_print( + message='Syslinux (partition)...', + function=install_syslinux_to_partition, + partition=find_first_partition(ufd_dev), + ) + # Unmount UFD try_and_print( message='Unmounting UFD...', @@ -99,10 +106,10 @@ if __name__ == '__main__': mount_point='/mnt/UFD', ) - # Install syslinux + # Install syslinux (to device) try_and_print( - message='Installing Syslinux...', - function=install_syslinux, + message='Syslinux (device)...', + function=install_syslinux_to_dev, ufd_dev=ufd_dev, use_mbr=args['--use-mbr'], ) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index c1296d44..897334f6 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -154,8 +154,8 @@ def hide_items(ufd_dev, items): run_program(cmd, check=False, shell=True) -def install_syslinux(ufd_dev, use_mbr): - """Install Syslinux to UFD.""" +def install_syslinux_to_dev(ufd_dev, use_mbr): + """Install Syslinux to UFD (dev).""" cmd = [ 'dd', 'bs=440', @@ -168,6 +168,18 @@ def install_syslinux(ufd_dev, use_mbr): run_program(cmd) +def install_syslinux_to_partition(partition): + """Install Syslinux to UFD (partition).""" + cmd = [ + 'syslinux', + '--install', + '--directory', + '/arch/boot/syslinux/', + partition, + ] + run_program(cmd) + + def is_valid_path(path_obj, path_type): """Verify path_obj is valid by type, returns bool.""" valid_path = False From 72c0e7515655e2c4bca25bb2646528db90e62841 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 20:52:11 -0700 Subject: [PATCH 57/61] Fix mounting UFD --- .bin/Scripts/build-ufd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 4d0072c4..4ed2212a 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -68,7 +68,7 @@ if __name__ == '__main__': try_and_print( message='Mounting UFD...', function=mount, - mount_source=ufd_dev, + mount_source=find_first_partition(ufd_dev), mount_point='/mnt/UFD', ) From 19799bb1f4594ec554ceb1d85b6a30ba843f4326 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Thu, 18 Apr 2019 20:52:47 -0700 Subject: [PATCH 58/61] Adjusting formatting --- .bin/Scripts/build-ufd | 17 ++++++++++++++++- .bin/Scripts/functions/ufd.py | 6 +++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 4ed2212a..fff63363 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -54,9 +54,11 @@ if __name__ == '__main__': confirm_selections(args) # Prep UFD + print_info('Prep UFD') if args['--update']: # Remove arch folder try_and_print( + indent=2, message='Removing Linux...', function=remove_arch, ) @@ -66,6 +68,7 @@ if __name__ == '__main__': # Mount UFD try_and_print( + indent=2, message='Mounting UFD...', function=mount, mount_source=find_first_partition(ufd_dev), @@ -73,9 +76,12 @@ if __name__ == '__main__': ) # Copy sources + print_standard(' ') + print_info('Copy Sources') for s_label, s_path in sources.items(): try_and_print( - message='Copying {} files...'.format(s_label), + indent=2, + message='Copying {}...'.format(s_label), function=copy_source, source=s_path, items=ITEMS[s_label], @@ -83,7 +89,10 @@ if __name__ == '__main__': ) # Update boot entries + + print_info('Boot Setup') try_and_print( + indent=2, message='Updating boot entries...', function=update_boot_entries, boot_entries=BOOT_ENTRIES, @@ -94,6 +103,7 @@ if __name__ == '__main__': # Install syslinux (to partition) try_and_print( + indent=2, message='Syslinux (partition)...', function=install_syslinux_to_partition, partition=find_first_partition(ufd_dev), @@ -101,6 +111,7 @@ if __name__ == '__main__': # Unmount UFD try_and_print( + indent=2, message='Unmounting UFD...', function=unmount, mount_point='/mnt/UFD', @@ -108,6 +119,7 @@ if __name__ == '__main__': # Install syslinux (to device) try_and_print( + indent=2, message='Syslinux (device)...', function=install_syslinux_to_dev, ufd_dev=ufd_dev, @@ -115,7 +127,10 @@ if __name__ == '__main__': ) # Hide items + print_standard(' ') + print_info('Final Touches') try_and_print( + indent=2, message='Hiding items...', function=hide_items, ufd_dev=ufd_dev, diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 897334f6..c54646b7 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -202,7 +202,7 @@ def mount(mount_source, mount_point): run_program(cmd) -def prep_device(dev_path, label, use_mbr=False): +def prep_device(dev_path, label, use_mbr=False, indent=2): """Format device in preparation for applying the WizardKit components This is done is four steps: @@ -214,6 +214,7 @@ def prep_device(dev_path, label, use_mbr=False): # Zero-out first 64MB cmd = 'dd bs=4M count=16 if=/dev/zero of={}'.format(dev_path).split() try_and_print( + indent=indent, message='Zeroing first 64MB...', function=run_program, cmd=cmd, @@ -226,6 +227,7 @@ def prep_device(dev_path, label, use_mbr=False): '-1s' if use_mbr else '-4MiB', ).split() try_and_print( + indent=indent, message='Creating partition table...', function=run_program, cmd=cmd, @@ -237,6 +239,7 @@ def prep_device(dev_path, label, use_mbr=False): 'boot' if use_mbr else 'legacy_boot', ).split() try_and_print( + indent=indent, message='Setting boot flag...', function=run_program, cmd=cmd, @@ -249,6 +252,7 @@ def prep_device(dev_path, label, use_mbr=False): find_first_partition(dev_path), ] try_and_print( + indent=indent, message='Formatting partition...', function=run_program, cmd=cmd, From e2d788e26563a0e4f1745c06e6e295e3eb310686 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 19 Apr 2019 15:23:21 -0700 Subject: [PATCH 59/61] Fixed upate_boot_entries() --- .bin/Scripts/functions/ufd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index c54646b7..96b74b51 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -404,7 +404,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): cmd = [ 'sed', '--in-place', - '"s/#{}#//"'.format(b_comment), + '"s/#UFD-{}#//"'.format(b_comment), *configs, ] run_program(cmd, check=False) From 731a2818053c6224086beec8476940625ea09e2a Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 19 Apr 2019 16:33:12 -0700 Subject: [PATCH 60/61] More bugfixes --- .bin/Scripts/build-ufd | 2 +- .bin/Scripts/functions/ufd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index fff63363..bb0813a8 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -89,7 +89,7 @@ if __name__ == '__main__': ) # Update boot entries - + print_standard(' ') print_info('Boot Setup') try_and_print( indent=2, diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 96b74b51..95a55b64 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -404,7 +404,7 @@ def update_boot_entries(boot_entries, boot_files, iso_label, ufd_label): cmd = [ 'sed', '--in-place', - '"s/#UFD-{}#//"'.format(b_comment), + 's/#{}#//'.format(b_comment), *configs, ] run_program(cmd, check=False) From a83dcdd06a7ab293120b4d5e9ae1ea587a7d3dc0 Mon Sep 17 00:00:00 2001 From: 2Shirt <2xShirt@gmail.com> Date: Fri, 19 Apr 2019 18:14:10 -0700 Subject: [PATCH 61/61] Handle non-ISO source items --- .bin/Scripts/functions/ufd.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/ufd.py b/.bin/Scripts/functions/ufd.py index 95a55b64..598985d3 100644 --- a/.bin/Scripts/functions/ufd.py +++ b/.bin/Scripts/functions/ufd.py @@ -59,17 +59,29 @@ def confirm_selections(args): def copy_source(source, items, overwrite=False): - """Mount source and copy items to /mnt/UFD.""" - mount(source, '/mnt/Source') + """Copy source items to /mnt/UFD.""" + is_iso = source.name.lower().endswith('.iso') + + # Mount source if necessary + if is_iso: + mount(source, '/mnt/Source') + + # Copy items for i_source, i_dest in items: - i_source = '/mnt/Source{}'.format(i_source) + i_source = '{}{}'.format( + '/mnt/Source' if is_iso else source, + i_source, + ) i_dest = '/mnt/UFD{}'.format(i_dest) try: recursive_copy(i_source, i_dest, overwrite=overwrite) except FileNotFoundError: # Going to assume (hope) that this is fine pass - unmount('/mnt/Source') + + # Unmount source if necessary + if is_iso: + unmount('/mnt/Source') def find_first_partition(dev_path):