From 2cce7c34d6d2a199b4bb957983c36c34bf228f56 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 12 Jan 2018 15:22:07 -0700 Subject: [PATCH 01/32] Fixed copying main.py to settings --- Build Linux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build Linux b/Build Linux index aa7a51f0..84062426 100755 --- a/Build Linux +++ b/Build Linux @@ -110,7 +110,7 @@ function copy_live_env() { rsync -aI "$ROOT_DIR/.linux_items/include/" "$LIVE_DIR/" mkdir -p "$LIVE_DIR/airootfs/usr/local/bin" rsync -aI "$ROOT_DIR/.bin/Scripts/" "$LIVE_DIR/airootfs/usr/local/bin/" - cp -a "$BUILD_DIR/main.py" "$LIVE_DIR/airootfs/usr/local/bin/" + cp -a "$BUILD_DIR/main.py" "$LIVE_DIR/airootfs/usr/local/bin/settings/" } function run_elevated() { From 07f6179dab81bfb39c9b3579b8d53b2f8a6e6219 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 12 Jan 2018 15:23:17 -0700 Subject: [PATCH 02/32] i3-gaps has been added to the community repo --- .linux_items/packages/aur | 1 - 1 file changed, 1 deletion(-) diff --git a/.linux_items/packages/aur b/.linux_items/packages/aur index 9de0855e..917357bf 100644 --- a/.linux_items/packages/aur +++ b/.linux_items/packages/aur @@ -2,7 +2,6 @@ aic94xx-firmware bash-pipes gtk-theme-arc-git hfsprogs -i3-gaps i3lock-fancy-git mprime-bin nvme-cli From f3d8daf8e20317281321508076d81e8b187e5180 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 12 Jan 2018 16:12:29 -0700 Subject: [PATCH 03/32] Use HTTPS for git clones during build --- Build Linux | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Build Linux b/Build Linux index 84062426..ff3119fe 100755 --- a/Build Linux +++ b/Build Linux @@ -184,12 +184,12 @@ function update_live_env() { sed -i -r "s/_+/$KIT_NAME_FULL Linux Environment/" "$LIVE_DIR/airootfs/etc/motd" # Oh My ZSH - git clone --depth=1 git://github.com/robbyrussell/oh-my-zsh.git "$SKEL_DIR/.oh-my-zsh" + git clone --depth=1 https://github.com/robbyrussell/oh-my-zsh.git "$SKEL_DIR/.oh-my-zsh" rm -Rf "$SKEL_DIR/.oh-my-zsh/.git" curl -o "$SKEL_DIR/.oh-my-zsh/themes/lean.zsh-theme" https://raw.githubusercontent.com/miekg/lean/master/prompt_lean_setup # Openbox theme - git clone --depth=1 git@github.com:addy-dclxvi/Openbox-Theme-Collections.git "$TEMP_DIR/ob-themes" + git clone --depth=1 https://github.com/addy-dclxvi/Openbox-Theme-Collections.git "$TEMP_DIR/ob-themes" mkdir -p "$LIVE_DIR/airootfs/usr/share/themes" cp -a "$TEMP_DIR/ob-themes/Triste-Orange" "$LIVE_DIR/airootfs/usr/share/themes/" From 5c21f8569c39ea9435405e57923d071a42d98084 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 12 Jan 2018 16:13:02 -0700 Subject: [PATCH 04/32] wimlib was added to the community repo --- .linux_items/packages/aur | 1 - 1 file changed, 1 deletion(-) diff --git a/.linux_items/packages/aur b/.linux_items/packages/aur index 917357bf..dd625627 100644 --- a/.linux_items/packages/aur +++ b/.linux_items/packages/aur @@ -12,4 +12,3 @@ smartmontools-svn testdisk-wip ttf-font-awesome wd719x-firmware -wimlib From 10cf7a15758d8b4716f02f1b29fd56d9a0b3c567 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Mon, 15 Jan 2018 12:57:27 -0700 Subject: [PATCH 05/32] Updated hw-sensors * Filter out non-temp sensors * Adjusted formatting * Partially addresses issue #9 --- .bin/Scripts/hw-sensors | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.bin/Scripts/hw-sensors b/.bin/Scripts/hw-sensors index bfaddc7e..36655911 100755 --- a/.bin/Scripts/hw-sensors +++ b/.bin/Scripts/hw-sensors @@ -61,12 +61,9 @@ def get_feature_string(chip, feature): skipname = len(feature.name)+1 # skip common prefix data = {} - if feature.type == sensors.feature.INTRUSION: - vals = [sensors.get_value(chip, sf.number) for sf in sfs] - # short path for INTRUSION to demonstrate type usage - status = "alarm" if int(vals[0]) == 1 else "normal" - print_standard(' {:18} {}'.format(label, status)) - return + if feature.type != sensors.feature.TEMP: + # Skip non-temperature sensors + return None for sf in sfs: name = sf.name[skipname:].decode("utf-8").strip() @@ -94,7 +91,7 @@ def get_feature_string(chip, feature): list_data.append('{}: {}'.format(item, data.pop(item))) list_data.extend( ['{}: {}'.format(k, v) for k, v in sorted(data.items())]) - data_str = '{:18} {} ({})'.format( + data_str = '{:18} {} {}'.format( label, main_temp, ', '.join(list_data)) else: list_data.extend(sorted(data.items())) @@ -119,10 +116,13 @@ if __name__ == '__main__': chip_name = '{} ({})'.format( sensors.chip_snprintf_name(chip), sensors.get_adapter_name(chip.bus)) - chip_temps[chip_name] = [chip_name] - for feature in sensors.FeatureIterator(chip): - chip_temps[chip_name].append(get_feature_string(chip, feature)) - chip_temps[chip_name].append('') + chip_feats = [get_feature_string(chip, feature) + for feature in sensors.FeatureIterator(chip)] + # Strip empty/None items + chip_feats = [f for f in chip_feats if f] + + if chip_feats: + chip_temps[chip_name] = [chip_name, *chip_feats, ''] # Sort chips sensor_temps = [] From 0e87bf55a0f4e1a10720d9cadec43344f37e4514 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Mon, 15 Jan 2018 13:45:58 -0700 Subject: [PATCH 06/32] Enable HW-Diags for more devices * Skip all removable WizardKit devices * Fixes issue #8 --- .bin/Scripts/functions/hw_diags.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 8b2c164e..99a6a3ed 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -410,10 +410,18 @@ def scan_disks(): json_data = json.loads(result.stdout.decode()) devs = {} for d in json_data.get('blockdevices', []): - if d['type'] == 'disk' and d['hotplug'] == '0': - devs[d['name']] = {'lsblk': d} - TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' - TESTS['badblocks']['Status'][d['name']] = 'Pending' + if d['type'] == 'disk': + if d['hotplug'] == '0': + devs[d['name']] = {'lsblk': d} + TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' + TESTS['badblocks']['Status'][d['name']] = 'Pending' + else: + # Skip WizardKit devices + wk_label = '{}_LINUX'.format(KIT_NAME_SHORT) + if wk_label not in [c.get('label', '') for c in d['children']]: + devs[d['name']] = {'lsblk': d} + TESTS['NVMe/SMART']['Status'][d['name']] = 'Pending' + TESTS['badblocks']['Status'][d['name']] = 'Pending' for dev, data in devs.items(): # Get SMART attributes From 480ac48aae7907786268294a9f9c56322e87bebc Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Mon, 15 Jan 2018 14:13:35 -0700 Subject: [PATCH 07/32] Removed '+' from positive temps in hw-sensors --- .bin/Scripts/hw-sensors | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/hw-sensors b/.bin/Scripts/hw-sensors index 36655911..2a1a46e0 100755 --- a/.bin/Scripts/hw-sensors +++ b/.bin/Scripts/hw-sensors @@ -51,7 +51,7 @@ def color_temp(temp): color = COLORS['CLEAR'] return '{color}{prefix}{temp:2.0f}°C{CLEAR}'.format( color = color, - prefix = '+' if temp>0 else '-', + prefix = '-' if temp < 0 else '', temp = temp, **COLORS) @@ -78,7 +78,7 @@ def get_feature_string(chip, feature): data[name] = ' {}°C'.format(val) else: data[name] = '{}{:2.0f}°C'.format( - '+' if temp>0 else '-', + '-' if temp < 0 else '', temp) else: data[name] = color_temp(val) From 3bc45e983203f3a89b76bc07c9fb1781188a2ece Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Mon, 15 Jan 2018 14:16:59 -0700 Subject: [PATCH 08/32] Fix issue #7 by switching to Font Awesome v4 --- .../airootfs/etc/skel/.config/i3status/config | 14 +++++++------- .linux_items/packages/aur | 2 +- .linux_items/packages/live | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.linux_items/include/airootfs/etc/skel/.config/i3status/config b/.linux_items/include/airootfs/etc/skel/.config/i3status/config index d783b408..d36294a4 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/i3status/config +++ b/.linux_items/include/airootfs/etc/skel/.config/i3status/config @@ -21,7 +21,7 @@ order += "tztime local" #order += "tztime utc" cpu_usage { - format = ". %usage" + format = " %usage" max_threshold = 90 #format_above_threshold = " %usage" degraded_threshold = 75 @@ -29,19 +29,19 @@ cpu_usage { } wireless _first_ { - format_up = ". (%quality at %essid) %ip" - format_down = ". Down" + format_up = " (%quality at %essid) %ip" + format_down = " Down" } ethernet _first_ { # if you use %speed, i3status requires root privileges - format_up = ". %ip" - format_down = ". Down" + format_up = " %ip" + format_down = " Down" } battery all { integer_battery_capacity = true - format = "%status. %percentage" + format = "%status %percentage" format_down = "" status_chr = "" status_bat = "" @@ -53,7 +53,7 @@ battery all { } volume master { - format = ". %volume" + format = " %volume" format_muted = " muted" device = "pulse" } diff --git a/.linux_items/packages/aur b/.linux_items/packages/aur index 9de0855e..13aef81f 100644 --- a/.linux_items/packages/aur +++ b/.linux_items/packages/aur @@ -11,6 +11,6 @@ papirus-icon-theme pasystray smartmontools-svn testdisk-wip -ttf-font-awesome +ttf-font-awesome-4 wd719x-firmware wimlib diff --git a/.linux_items/packages/live b/.linux_items/packages/live index efc1e1d8..892b0bf2 100644 --- a/.linux_items/packages/live +++ b/.linux_items/packages/live @@ -50,6 +50,7 @@ networkmanager nvme-cli oblogout openbox-patched +otf-font-awesome-4 p7zip papirus-icon-theme pasystray @@ -75,7 +76,7 @@ tint2 tk tmux tree -ttf-font-awesome +ttf-font-awesome-4 ttf-inconsolata udevil udisks2 From 5a1f9f096113331ad40a7b298f39cebc1a9799f4 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Mon, 15 Jan 2018 14:28:31 -0700 Subject: [PATCH 09/32] Added Cleanup CBS launcher --- .bin/Scripts/settings/launchers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index f3e6b0a2..7d52cd51 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -441,6 +441,12 @@ LAUNCHERS = { }, }, r'Misc': { + 'Cleanup CBS Temp Files': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'cbs_fix.py', + 'L_ELEV': 'True', + }, 'ConEmu (as ADMIN)': { 'L_TYPE': 'Executable', 'L_PATH': 'ConEmu', From fbd7f47749c0f76f445fd6d6b855c5f6d9f30c08 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Mon, 15 Jan 2018 17:30:47 -0700 Subject: [PATCH 10/32] Added build-ufd script * Includes safety checks to avoid unintentional data loss --- .bin/Scripts/build-ufd | 201 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100755 .bin/Scripts/build-ufd diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd new file mode 100755 index 00000000..a3508beb --- /dev/null +++ b/.bin/Scripts/build-ufd @@ -0,0 +1,201 @@ +#!/bin/bash +# +## Wizard Kit: UFD Build Tool + +DEST_DEV="$1" +DEST_PARTITION="${DEST_DEV}1" +MAIN_PY="/usr/local/bin/settings/main.py" +WD=$(pwd) +EXTRA_DIR="${WD}/Extras" +MAIN_KIT="$(dirname $(find $WD -type d -name '.bin') 2>/dev/null)" +LINUX_ISO="$(find $WD -type f -iname '*Linux*iso' | head -1)" +WINPE_ISO="$(find $WD -type f -iname '*WinPE*iso' | head -1)" +if [ "$2" == "--silent" ]; then + SILENT="True" +fi + +# COLORS +CLEAR="\e[0m" +RED="\e[31m" +GREEN="\e[32m" +YELLOW="\e[33m" +BLUE="\e[34m" + +# Functions +function ask() { + if [ "$SILENT" == "True" ]; then + echo -e "$1 Yes ${BLUE}(Silent)${CLEAR}" + return 0 + fi + while :; do + read -p "$1 " -r answer + if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then + return 0 + elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then + return 1 + fi + done +} + +die () { + echo "$0:" "$@" >&2 + echo "" + read -p "Press Enter to exit... " -r + exit 1 +} + +# Load main.py settings +if [ ! -f "$MAIN_PY" ]; then + echo -e "${RED}ERROR${CLEAR}: $MAIN_PY not found." + die "Aborted." +fi +while read line; do + if echo "$line" | egrep -q "^\w+='"; then + line="$(echo "$line" | sed -r 's/[\r\n]+//')" + eval "$line" + fi +done < "$MAIN_PY" +if [ -z ${KIT_NAME_FULL+x} ]; then + # KIT_NAME_FULL is not set, assume main.py missing or malformatted + echo -e "${RED}ERROR${CLEAR}: failed to load settings from $MAIN_PY" + die "Aborted." +fi +UFD_LABEL="${KIT_NAME_SHORT}_LINUX" + +# Check if root +if [[ "$EUID" -ne 0 ]]; then + echo -e "${RED}ERROR${CLEAR}: This script must be run as root." + die "Aborted." +fi + +# Check if in tmux +if ! tmux list-session | grep -q "build-ufd"; then + # Reload in tmux + tmux new-session -s "build-ufd" "$0" $* + exit 0 +fi + +# Header +echo -e "${GREEN}$KIT_NAME_FULL${CLEAR}: UFD Build Tool" +echo "" + +# Dest and Sources Check +abort="False" +if [ ! -b "$DEST_DEV" ]; then + echo -e "${RED}ERROR${CLEAR}: Device $DEST_DEV not found." + abort="True" +fi +if [ ! -f "$LINUX_ISO" ]; then + echo -e "${RED}ERROR${CLEAR}: Linux ISO not found." + abort="True" +fi +if [ ! -f "$WINPE_ISO" ]; then + echo -e "${RED}ERROR${CLEAR}: WinPE ISO not found." + abort="True" +fi +if [ ! -d "$MAIN_KIT" ]; then + echo -e "${RED}ERROR${CLEAR}: Wizard Kit directory not found." + abort="True" +fi +if [ ! -d "$EXTRA_DIR" ]; then + # Warn but don't abort + echo -e "${YELLOW}WARNING${CLEAR}: $EXTRA_DIR not found." + echo "" + EXTRA_DIR='__None__' +fi +if [ "$abort" == "True" ]; then + echo "" + die "Aborted." +fi + +# Display Job Settings +echo -e "${BLUE}Sources${CLEAR}" +echo "Main Kit: $MAIN_KIT" +echo "Linux ISO: $LINUX_ISO" +echo "WinPE ISO: $WINPE_ISO" +echo "Extras: $EXTRA_DIR" +echo "" +echo -e "${BLUE}Destination${CLEAR}" +lsblk -n -o NAME,LABEL,SIZE,MODEL,SERIAL $DEST_DEV + +# Ask before starting job +echo "" +if ask "Is the above information correct?"; then + echo "" + echo -e "${YELLOW}SAFETY CHECK${CLEAR}" + echo "All data will be DELETED from the disk and partition(s) listed above." + echo -e "This is irreversible and will lead to ${RED}DATA LOSS.${CLEAR}" + if ! ask "Asking again to confirm, is this correct?"; then + die "Aborted." + fi +else + die "Aborted." +fi + +# Start Build +echo "" +echo -e "${GREEN}Building Kit${CLEAR}" + +# Format +echo "Formatting drive..." +parted "$DEST_DEV" -s -- mklabel msdos mkpart primary fat32 1MiB -1s +parted "$DEST_DEV" set 1 boot on +mkfs.vfat -F 32 -n "$UFD_LABEL" "$DEST_PARTITION" >/dev/null 2>&1 + +# Mount sources and dest +echo "Mounting sources and destination..." +mkdir /mnt/{Dest,Linux,WinPE} -p +mount $DEST_PARTITION /mnt +mount "$LINUX_ISO" /mnt/Linux -r +mount "$WINPE_ISO" /mnt/WinPE -r + +# Copy files +echo "Copying Linux files..." +tmux split-window -l 15 cp -Lrv /mnt/Linux/* /mnt/ + +echo "Copying WinPE files..." +tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/ +tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Microsoft /mnt/EFI/ +tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Boot/* /mnt/EFI/Microsoft/ +tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot/{BCD,boot.sdi},bootmgr} /mnt/sources/ + +echo "Copying Main Kit..." +tmux split-window -l 15 cp -Lrv "$MAIN_KIT" /mnt/ +if [ "$EXTRA_DIR" != "__None__" ]; then + echo "Copying Extra files..." + tmux split-window -l 15 cp -Lrv "$EXTRA_DIR"/* /mnt/ +fi + +# Install syslinux +echo "Copying Syslinux files..." +tmux split-window -l 15 cp -Lrv /usr/lib/syslinux/bios/*.c32 /mnt/arch/boot/syslinux/ +syslinux --install -d /arch/boot/syslinux/ $DEST_PARTITION >/dev/null 2>&1 + +echo "Unmounting destination..." +umount /mnt +sync + +echo "Installing Syslinux MBR..." +dd bs=440 count=1 if=/usr/lib/syslinux/bios/mbr.bin of=$DEST_DEV >/dev/null 2>&1 +sync + +# Cleanup +echo "Hiding boot files..." +echo "drive s: file=\"$DEST_PARTITION\"" > /root/.mtoolsrc +echo 'mtools_skip_check=1' >> /root/.mtoolsrc +for item in BOOT{,MGR,MGR.EFI} EFI EN-US ISOLINUX LOADER SOURCES SYSLINUX.CFG; do + yes | mattrib +h S:/$item >/dev/null 2>&1 +done +sync + +# Unmount Sources +echo "Unmounting sources..." +umount /mnt/* -R + +# Done +echo "" +echo "Done." +echo "" +read -p "Press Enter to exit..." -r +exit 0 + From f7f3f0d53c2cc3c0e7fd3f23fa71fc13650a67ad Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Wed, 17 Jan 2018 17:22:45 -0700 Subject: [PATCH 11/32] Updated info.py Should fix #10 --- .bin/Scripts/functions/info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/Scripts/functions/info.py b/.bin/Scripts/functions/info.py index 3c882333..066ebbd7 100644 --- a/.bin/Scripts/functions/info.py +++ b/.bin/Scripts/functions/info.py @@ -460,8 +460,8 @@ def show_user_data_summary(indent=8, width=32): indent = ' ' * indent, width = width, folder = folder, - size = folders[folder]['Size'], - path = folders[folder]['Path'])) + size = folders[folder].get('Size', 'Unknown'), + path = folders[folder].get('Path', 'Unknown'))) if __name__ == '__main__': print("This file is not meant to be called directly.") From f0ae207890f43a9ebd039352af7eed1f88777761 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Thu, 25 Jan 2018 19:57:21 -0700 Subject: [PATCH 12/32] Improved source scanning for user data transfers * Fixes recursion bug when Windows.old folders are present * Combined logic for file/folder sources and WIM sources * Code uses proper folder separators for the running OS * (e.g. '\' for Windows and '/' for the rest) --- .bin/Scripts/functions/common.py | 4 + .bin/Scripts/functions/data.py | 246 +++++++++++++++---------------- 2 files changed, 127 insertions(+), 123 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 437591cb..7ebd81b4 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -749,6 +749,10 @@ def set_linux_vars(): global_vars['Env'] = os.environ.copy() global_vars['BinDir'] = '/usr/local/bin' global_vars['LogDir'] = global_vars['TmpDir'] + global_vars['Tools'] = { + 'wimlib-imagex': 'wimlib-imagex', + 'SevenZip': '7z', + } if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 414c2fba..3f8c4aa2 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -17,6 +17,11 @@ class LocalDisk(): # Should always be true return True +class SourceItem(): + def __init__(self, name, path): + self.name = name + self.path = path + # Regex REGEX_EXCL_ITEMS = re.compile( r'^(\.(AppleDB|AppleDesktop|AppleDouble' @@ -29,7 +34,7 @@ REGEX_EXCL_ITEMS = re.compile( r'|Thumbs\.db)$', re.IGNORECASE) REGEX_EXCL_ROOT_ITEMS = re.compile( - r'^\\?(boot(mgr|nxt)$|Config.msi' + r'^(boot(mgr|nxt)$|Config.msi' r'|(eula|globdata|install|vc_?red)' r'|.*.sys$|System Volume Information|RECYCLER?|\$Recycle\.bin' r'|\$?Win(dows(.old.*|\.~BT|)$|RE_)|\$GetCurrent|Windows10Upgrade' @@ -37,7 +42,7 @@ REGEX_EXCL_ROOT_ITEMS = re.compile( r'|.*\.(esd|swm|wim|dd|map|dmg|image)$)', re.IGNORECASE) REGEX_INCL_ROOT_ITEMS = re.compile( - r'^\\?(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' + r'^(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' r'|Media|Music|Pic(ture|)s?|Vid(eo|)s?)' r'|{prefix}(-?Info|-?Transfer|)' r'|(ProgramData|Recovery|Temp.*|Users)$' @@ -48,7 +53,7 @@ REGEX_WIM_FILE = re.compile( r'\.wim$', re.IGNORECASE) REGEX_WINDOWS_OLD = re.compile( - r'^\\Win(dows|)\.old', + r'^Win(dows|)\.old', re.IGNORECASE) # STATIC VARIABLES @@ -334,150 +339,151 @@ def run_wimextract(source, items, dest): '--nullglob'] run_program(cmd) -def scan_source(source_obj, dest_path): - """Scan source for files/folders to transfer.""" - selected_items = [] - +def list_source_items(source_obj, rel_path=None): + items = [] + rel_path = '{}{}'.format(os.sep, rel_path) if rel_path else '' if source_obj.is_dir(): - # File-Based - print_standard('Scanning source (folder): {}'.format(source_obj.path)) - selected_items = scan_source_path(source_obj.path, dest_path) + source_path = '{}{}'.format(source_obj.path, rel_path) + items = [SourceItem(name=item.name, path=item.path) + for item in os.scandir(source_path)] else: - # Image-Based - if REGEX_WIM_FILE.search(source_obj.name): - print_standard('Scanning source (image): {}'.format( - source_obj.path)) - selected_items = scan_source_wim(source_obj.path, dest_path) - else: - print_error('ERROR: Unsupported image: {}'.format( - source_obj.path)) - raise GenericError - - return selected_items + # Prep wimlib-imagex + if psutil.WINDOWS: + extract_item('wimlib', silent=True) + cmd = [ + global_vars['Tools']['wimlib-imagex'], 'dir', + source_obj.path, '1'] + if rel_path: + cmd.append('--path={}'.format(rel_path)) -def scan_source_path(source_path, dest_path, rel_path=None, interactive=True): - """Scan source folder for files/folders to transfer, returns list. - - This will scan the root and (recursively) any Windows.old folders.""" - rel_path = '\\' + rel_path if rel_path else '' - if rel_path: - dest_path = dest_path + rel_path - selected_items = [] - win_olds = [] + # Get item list + try: + items = run_program(cmd) + except subprocess.CalledProcessError: + print_error('ERROR: Failed to get file list.') + raise - # Root items - root_items = [] - for item in os.scandir(source_path): - if REGEX_INCL_ROOT_ITEMS.search(item.name): - root_items.append(item.path) - elif not REGEX_EXCL_ROOT_ITEMS.search(item.name): - if (not interactive - or ask('Copy: "{}{}" ?'.format(rel_path, item.name))): - root_items.append(item.path) - if REGEX_WINDOWS_OLD.search(item.name): - win_olds.append(item) - if root_items: - selected_items.append({ - 'Message': '{}Root Items...'.format(rel_path), - 'Items': root_items.copy(), - 'Destination': dest_path}) + # Strip non-root items + items = [re.sub(r'(\\|/)', os.sep, i.strip()) + for i in items.stdout.decode('utf-8', 'ignore').splitlines()] + if rel_path: + items = [i.replace(rel_path, '') for i in items] + items = [i for i in items + if i.count(os.sep) == 1 and i.strip() != os.sep] + items = [SourceItem(name=i[1:], path=rel_path+i) for i in items] - # Fonts - if os.path.exists(r'{}\Windows\Fonts'.format(source_path)): - selected_items.append({ - 'Message': '{}Fonts...'.format(rel_path), - 'Items': [r'{}\Windows\Fonts'.format(rel_path)], - 'Destination': r'{}\Windows'.format(dest_path)}) - - # Registry - registry_items = [] - for folder in ['config', 'OEM']: - folder = r'Windows\System32\{}'.format(folder) - folder = os.path.join(source_path, folder) - if os.path.exists(folder): - registry_items.append(folder) - if registry_items: - selected_items.append({ - 'Message': '{}Registry...'.format(rel_path), - 'Items': registry_items.copy(), - 'Destination': r'{}\Windows\System32'.format(dest_path)}) - - # Windows.old(s) - for old in win_olds: - selected_items.append( - scan_source_path( - old.path, dest_path, rel_path=old.name, interactive=False)) - # Done - return selected_items + return items -def scan_source_wim(source_wim, dest_path, rel_path=None, interactive=True): - """Scan source WIM file for files/folders to transfer, returns list. +def scan_source(source_obj, dest_path, rel_path='', interactive=True): + """Scan source for files/folders to transfer, returns list. This will scan the root and (recursively) any Windows.old folders.""" - rel_path = '\\' + rel_path if rel_path else '' selected_items = [] win_olds = [] - # Scan source - extract_item('wimlib', silent=True) - cmd = [ - global_vars['Tools']['wimlib-imagex'], 'dir', - source_wim, '1'] - try: - file_list = run_program(cmd) - except subprocess.CalledProcessError: - print_error('ERROR: Failed to get file list.') - raise - # Root Items - file_list = [i.strip() - for i in file_list.stdout.decode('utf-8', 'ignore').splitlines() - if i.count('\\') == 1 and i.strip() != '\\'] root_items = [] - if rel_path: - file_list = [i.replace(rel_path, '') for i in file_list] - for item in file_list: - if REGEX_INCL_ROOT_ITEMS.search(item): - root_items.append(item) - elif not REGEX_EXCL_ROOT_ITEMS.search(item): - if (not interactive - or ask('Extract: "{}{}" ?'.format(rel_path, item))): - root_items.append('{}{}'.format(rel_path, item)) - if REGEX_WINDOWS_OLD.search(item): + item_list = list_source_items(source_obj, rel_path) + for item in item_list: + if REGEX_INCL_ROOT_ITEMS.search(item.name): + print_success('Auto-Selected: {}'.format(item.path)) + root_items.append('{}'.format(item.path)) + elif not REGEX_EXCL_ROOT_ITEMS.search(item.name): + if not interactive: + print_success('Auto-Selected: {}'.format(item.path)) + root_items.append('{}'.format(item.path)) + elif ask('Extract: "{}{}{}" ?'.format( + rel_path, + os.sep if rel_path else '', + item.name)): + root_items.append('{}'.format(item.path)) + if REGEX_WINDOWS_OLD.search(item.name): + item.name = '{}{}{}'.format( + rel_path, + os.sep if rel_path else '', + item.name) win_olds.append(item) if root_items: selected_items.append({ - 'Message': '{}Root Items...'.format(rel_path), + 'Message': '{}{}Root Items...'.format( + rel_path, + ' ' if rel_path else ''), 'Items': root_items.copy(), 'Destination': dest_path}) # Fonts - if wim_contains(source_wim, r'{}Windows\Fonts'.format(rel_path)): + font_obj = get_source_item_obj(source_obj, rel_path, 'Windows/Fonts') + if font_obj: selected_items.append({ - 'Message': '{}Fonts...'.format(rel_path), - 'Items': [r'{}\Windows\Fonts'.format(rel_path)], + 'Message': '{}{}Fonts...'.format( + rel_path, + ' ' if rel_path else ''), + 'Items': [font_obj.path], 'Destination': dest_path}) # Registry registry_items = [] for folder in ['config', 'OEM']: - folder = r'{}Windows\System32\{}'.format(rel_path, folder) - if wim_contains(source_wim, folder): - registry_items.append(folder) + folder_obj = get_source_item_obj( + source_obj, rel_path, 'Windows/System32/{}'.format(folder)) + if folder_obj: + registry_items.append(folder_obj.path) if registry_items: selected_items.append({ - 'Message': '{}Registry...'.format(rel_path), + 'Message': '{}{}Registry...'.format( + rel_path, + ' ' if rel_path else ''), 'Items': registry_items.copy(), 'Destination': dest_path}) # Windows.old(s) for old in win_olds: - scan_source_wim(source_wim, dest_path, rel_path=old, interactive=False) + selected_items.extend(scan_source( + source_obj, + dest_path, + rel_path=old.name, + interactive=False)) # Done return selected_items +def get_source_item_obj(source_obj, rel_path, item_path): + item_obj = None + item_path = re.sub(r'(\\|/)', os.sep, item_path) + if source_obj.is_dir(): + item_obj = SourceItem( + name = item_path, + path = '{}{}{}{}{}'.format( + source_obj.path, + os.sep, + rel_path, + os.sep if rel_path else '', + item_path)) + else: + # Assuming WIM file + if psutil.WINDOWS: + extract_item('wimlib', silent=True) + cmd = [ + global_vars['Tools']['wimlib-imagex'], 'dir', + source_obj.path, '1', + '--path={}'.format(item_path), + '--one-file-only'] + try: + run_program(cmd) + except subprocess.CalledProcessError: + # function will return None below + pass + else: + item_obj = SourceItem( + name = item_path, + path = '{}{}{}{}'.format( + os.sep, + rel_path, + os.sep if rel_path else '', + item_path)) + return item_obj + def select_destination(folder_path, prompt='Select destination'): """Select destination drive, returns path as string.""" disk = select_volume(prompt) @@ -623,6 +629,14 @@ def select_source(ticket_number): pause("Press Enter to exit...") exit_script() + # Sanity check + if selected_source.is_file(): + # Image-Based + if not REGEX_WIM_FILE.search(selected_source.name): + print_error('ERROR: Unsupported image: {}'.format( + selected_source.path)) + raise GenericError + # Done return selected_source @@ -716,19 +730,5 @@ def umount_network_share(server): print_info('Umounted {Name}'.format(**server)) server['Mounted'] = False -def wim_contains(source_path, file_path): - """Check if the WIM contains a file or folder.""" - _cmd = [ - global_vars['Tools']['wimlib-imagex'], 'dir', - source_path, '1', - '--path={}'.format(file_path), - '--one-file-only'] - try: - run_program(_cmd) - except subprocess.CalledProcessError: - return False - else: - return True - if __name__ == '__main__': print("This file is not meant to be called directly.") From a7451f86baa7195b8e2fbacced6410d11a9a5af0 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 26 Jan 2018 18:32:33 -0700 Subject: [PATCH 13/32] Added choice() function * Provides a simlilar interface to the Windows choice command. --- .bin/Scripts/functions/common.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 7ebd81b4..214a7d05 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -93,6 +93,34 @@ def ask(prompt='Kotaero!'): print_log(message=message) return answer +def choice(choices, prompt='Kotaero!'): + """Prompt the user with a choice question, log answer, and returns str.""" + answer = None + choices = [str(c) for c in choices] + choices_short = {c[:1].upper(): c for c in choices} + prompt = '{} [{}]: '.format(prompt, '/'.join(choices)) + regex = '^({}|{})$'.format( + '|'.join([c[:1] for c in choices]), + '|'.join(choices)) + + # Get user's choice + while answer is None: + tmp = input(prompt) + if re.search(regex, tmp, re.IGNORECASE): + answer = tmp + + # Log result + message = '{prompt}{answer_text}'.format( + prompt = prompt, + answer_text = 'Yes' if answer else 'No') + print_log(message=message) + + # Fix answer formatting to match provided values + answer = choices_short[answer[:1].upper()] + + # Done + return answer + def clear_screen(): """Simple wrapper for cls/clear.""" if psutil.WINDOWS: From 30dcd8758ea37088d08f9ffb14a033e4618fa310 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 26 Jan 2018 18:35:05 -0700 Subject: [PATCH 14/32] Added a "Yes to all" option to User Data Transfer --- .bin/Scripts/functions/data.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 3f8c4aa2..8cb1783f 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -393,11 +393,19 @@ def scan_source(source_obj, dest_path, rel_path='', interactive=True): if not interactive: print_success('Auto-Selected: {}'.format(item.path)) root_items.append('{}'.format(item.path)) - elif ask('Extract: "{}{}{}" ?'.format( - rel_path, - os.sep if rel_path else '', - item.name)): - root_items.append('{}'.format(item.path)) + else: + prompt = 'Transfer: "{}{}{}" ?'.format( + rel_path, + os.sep if rel_path else '', + item.name) + choices = ['Yes', 'No', 'All', 'Quit'] + answer = choice(prompt=prompt, choices=choices) + if answer == 'Quit': + abort() + elif answer == 'All': + interactive = False + if answer in ['Yes', 'All']: + root_items.append('{}'.format(item.path)) if REGEX_WINDOWS_OLD.search(item.name): item.name = '{}{}{}'.format( rel_path, From d96256fd87b8f4e928d8b3155b14d56170cd59dc Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sat, 27 Jan 2018 12:45:37 -0700 Subject: [PATCH 15/32] Add reboot/shutdown entries when not in X --- .bin/Scripts/functions/hw_diags.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 99a6a3ed..e75849b8 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -83,6 +83,13 @@ def menu_diags(*args): {'Letter': 'Q', 'Name': 'Quit', 'CRLF': True}, ] + # CLI-mode actions + if 'DISPLAY' not in global_vars['Env']: + actions.extend([ + {'Letter': 'R', 'Name': 'Reboot', 'CRLF': True}, + {'Letter': 'S', 'Name': 'Shutdown'}, + ]) + # Quick disk check if 'quick' in args: run_tests(['Quick', 'NVMe/SMART']) @@ -118,6 +125,10 @@ def menu_diags(*args): run_program( 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split(), check=False, pipe=False) + elif selection == 'R': + run_program(['reboot']) + elif selection == 'S': + run_program(['poweroff']) elif selection == 'Q': break From 3d8af3cb0ad8caf9bc0d1fc94340c6359a2f5970 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sat, 27 Jan 2018 12:48:11 -0700 Subject: [PATCH 16/32] Switching to mprime over mprime-bin * Source url has been broken for a while now * mprime doesn't take too long to compile --- .linux_items/packages/aur | 2 +- .linux_items/packages/dependencies | 1 + .linux_items/packages/live | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.linux_items/packages/aur b/.linux_items/packages/aur index ee0aa3b7..ae607f33 100644 --- a/.linux_items/packages/aur +++ b/.linux_items/packages/aur @@ -3,7 +3,7 @@ bash-pipes gtk-theme-arc-git hfsprogs i3lock-fancy-git -mprime-bin +mprime nvme-cli openbox-patched papirus-icon-theme diff --git a/.linux_items/packages/dependencies b/.linux_items/packages/dependencies index 564053e8..5cd71ab9 100644 --- a/.linux_items/packages/dependencies +++ b/.linux_items/packages/dependencies @@ -4,6 +4,7 @@ base-devel curl dos2unix git +hwloc libewf openssh p7zip diff --git a/.linux_items/packages/live b/.linux_items/packages/live index 892b0bf2..a6798b79 100644 --- a/.linux_items/packages/live +++ b/.linux_items/packages/live @@ -41,7 +41,7 @@ mdadm mediainfo mesa-demos mkvtoolnix-cli -mprime-bin +mprime mpv mupdf ncdu From 1cd35712539f4c4702846f21c7d10b4d18ef5108 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sat, 27 Jan 2018 15:19:04 -0700 Subject: [PATCH 17/32] Using safer bash config for Build Linux script --- Build Linux | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Build Linux b/Build Linux index ff3119fe..bd0a0f1d 100755 --- a/Build Linux +++ b/Build Linux @@ -2,6 +2,11 @@ # ## Wizard Kit: Live Linux Build Tool +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail + # Prep DATE="$(date +%F)" DATETIME="$(date +%F_%H%M)" @@ -21,7 +26,7 @@ elif which vim >/dev/null 2>&1; then else EDITOR=vi fi -if which sudo >/dev/null 2>&1; then +if [ ! -z ${SUDO_USER+x} ]; then REAL_USER="$SUDO_USER" fi @@ -243,15 +248,15 @@ function update_repo() { # Archive current files if [[ -d "$REPO_DIR" ]]; then - mkdir "$BUILD_DIR/Archive" 2>/dev/null + mkdir -p "$BUILD_DIR/Archive" 2>/dev/null archive="$BUILD_DIR/Archive/$(date "+%F_%H%M%S")" mv -bv "$REPO_DIR" "$archive" fi sleep 1s # Build custom repo packages - mkdir "$REPO_DIR" 2>/dev/null - mkdir "$TEMP_DIR" 2>/dev/null + mkdir -p "$REPO_DIR" 2>/dev/null + mkdir -p "$TEMP_DIR" 2>/dev/null pushd "$TEMP_DIR" >/dev/null while read -r p; do echo "Building: $p" @@ -325,32 +330,38 @@ function build_full() { } # Check input -case $1 in +case ${1:-} in -b|--build-full) build_full + echo Done ;; -f|--fix-perms) fix_kit_permissions + echo Done ;; -i|--install-deps) install_deps + echo Done ;; -o|--build-iso) load_settings build_iso + echo Done ;; -p|--prep-live-env) load_settings copy_live_env update_live_env + echo Done ;; -u|--update-repo) update_repo + echo Done ;; *) From 8e05cb0e4025c9b6938ac34fd7c3caaf821a2db3 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sat, 27 Jan 2018 15:52:49 -0700 Subject: [PATCH 18/32] Remove stale AUR packages from cache during build --- Build Linux | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Build Linux b/Build Linux index bd0a0f1d..a0600d74 100755 --- a/Build Linux +++ b/Build Linux @@ -296,6 +296,13 @@ function build_iso() { chown root:root "$LIVE_DIR" -R chmod 700 "$LIVE_DIR/airootfs/etc/skel/.ssh" chmod 600 "$LIVE_DIR/airootfs/etc/skel/.ssh/id_rsa" + + # Removing cached (and possibly outdated) custom repo packages + for package in $(cat "$ROOT_DIR/.linux_items/packages/aur"); do + if [[ -f /var/cache/pacman/pkg/${package}* ]]; then + rm /var/cache/pacman/pkg/${package}* + fi + done # Build ISO prefix="${KIT_NAME_SHORT}-Linux" From c24554720fbb484b31af6f2027b426e75228d5f2 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sun, 28 Jan 2018 17:48:42 -0700 Subject: [PATCH 19/32] Add option to upload crash details * Disabled by default, enabled via main.py ENABLE_UPLOAD_DATA * Upload destination set via main.py CRASH_SERVER variable --- .bin/Scripts/functions/common.py | 51 ++++++++- .bin/Scripts/settings/main.py | 172 ++++++++++++++++--------------- 2 files changed, 139 insertions(+), 84 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 214a7d05..f668306b 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -255,7 +255,20 @@ def major_exception(): print_warning(SUPPORT_MESSAGE) print(traceback.format_exc()) print_log(traceback.format_exc()) - sleep(30) + try: + upload_crash_details() + except GenericAbort: + # User declined upload + print_warning('Upload: Aborted') + sleep(30) + except GenericError: + # No log file or uploading disabled + sleep(30) + except: + print_error('Upload: NS') + sleep(30) + else: + print_success('Upload: CS') pause('Press Enter to exit...') exit_script(1) @@ -515,6 +528,42 @@ def try_and_print(message='Trying...', else: return {'CS': not bool(err), 'Error': err, 'Out': out} +def upload_crash_details(): + if not ENABLED_UPLOAD_DATA: + raise GenericError + + import requests + if 'LogFile' in global_vars and global_vars['LogFile']: + if ask('Upload crash details to {}?'.format(CRASH_SERVER['Name'])): + with open(global_vars['LogFile']) as f: + data = '''{} +############################# +Runtime Details: + +sys.argv: {} + +global_vars: {}'''.format(f.read(), sys.argv, global_vars) + filename = global_vars.get('LogFile', 'Unknown') + filename = re.sub(r'.*(\\|/)', '', filename) + filename += '.txt' + url = '{}/Crash_{}__{}'.format( + CRASH_SERVER['Url'], + global_vars.get('Date-Time', 'Unknown Date-Time'), + filename) + r = requests.put(url, data=data, + headers = {'X-Requested-With': 'XMLHttpRequest'}, + auth = (CRASH_SERVER['User'], CRASH_SERVER['Pass'])) + # Raise exception if upload NS + if not r.ok: + raise Exception + else: + # User said no + raise GenericAbort + else: + # No LogFile defined (or invalid LogFile) + raise GenericError + + def upload_data(path, file): """Add CLIENT_INFO_SERVER to authorized connections and upload file.""" if not ENABLED_UPLOAD_DATA: diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index d48b0ba0..7da23dce 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -1,83 +1,89 @@ -# Wizard Kit: Settings - Main / Branding - -# Features -ENABLED_UPLOAD_DATA = False - -# STATIC VARIABLES (also used by BASH and BATCH files) -## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH -# Main Kit -ARCHIVE_PASSWORD='Abracadabra' -KIT_NAME_FULL='Wizard Kit' -KIT_NAME_SHORT='WK' -SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' -# Live Linux -DIAG_SHARE='/srv/ClientInfo' -DIAG_USER='wkdiag' -MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags -ROOT_PASSWORD='Abracadabra' -SKIP_UPLOAD='False' -TECH_PASSWORD='Abracadabra' -# Server IP addresses -DIAG_SERVER='10.0.0.10' -OFFICE_SERVER_IP='10.0.0.10' -QUICKBOOKS_SERVER_IP='10.0.0.10' -# Time Zones -LINUX_TIME_ZONE='America/Los_Angeles' # See 'timedatectl list-timezones' for valid values -WINDOWS_TIME_ZONE='Pacific Standard Time' # See 'tzutil /l' for valid values -# WiFi -WIFI_SSID='SomeWifi' -WIFI_PASSWORD='Abracadabra' - -# SERVER VARIABLES -## NOTE: Windows can only use one user per server. This means that if -## one server serves multiple shares then you have to use the same -## user/password for all of those shares. -BACKUP_SERVERS = [ - { 'IP': '10.0.0.10', - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'Backups', - 'User': 'restore', - 'Pass': 'Abracadabra', - }, - { 'IP': '10.0.0.11', - 'Name': 'ServerTwo', - 'Mounted': False, - 'Share': 'Backups', - 'User': 'restore', - 'Pass': 'Abracadabra', - }, -] -CLIENT_INFO_SERVER = { - 'IP': '10.0.0.10', - 'RegEntry': r'0x10001,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', - 'Share': '/srv/ClientInfo', - 'User': 'upload', -} -OFFICE_SERVER = { - 'IP': OFFICE_SERVER_IP, - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'Office', - 'User': 'restore', - 'Pass': 'Abracadabra', -} -QUICKBOOKS_SERVER = { - 'IP': QUICKBOOKS_SERVER_IP, - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'QuickBooks', - 'User': 'restore', - 'Pass': 'Abracadabra', -} -WINDOWS_SERVER = { - 'IP': '10.0.0.10', - 'Name': 'ServerOne', - 'Mounted': False, - 'Share': 'Windows', - 'User': 'restore', - 'Pass': 'Abracadabra', -} - -if __name__ == '__main__': - print("This file is not meant to be called directly.") +# Wizard Kit: Settings - Main / Branding + +# Features +ENABLED_UPLOAD_DATA = False + +# STATIC VARIABLES (also used by BASH and BATCH files) +## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH +# Main Kit +ARCHIVE_PASSWORD='Abracadabra' +KIT_NAME_FULL='Wizard Kit' +KIT_NAME_SHORT='WK' +SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' +# Live Linux +DIAG_SHARE='/srv/ClientInfo' +DIAG_USER='wkdiag' +MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags +ROOT_PASSWORD='Abracadabra' +SKIP_UPLOAD='False' +TECH_PASSWORD='Abracadabra' +# Server IP addresses +DIAG_SERVER='10.0.0.10' +OFFICE_SERVER_IP='10.0.0.10' +QUICKBOOKS_SERVER_IP='10.0.0.10' +# Time Zones +LINUX_TIME_ZONE='America/Los_Angeles' # See 'timedatectl list-timezones' for valid values +WINDOWS_TIME_ZONE='Pacific Standard Time' # See 'tzutil /l' for valid values +# WiFi +WIFI_SSID='SomeWifi' +WIFI_PASSWORD='Abracadabra' + +# SERVER VARIABLES +## NOTE: Windows can only use one user per server. This means that if +## one server serves multiple shares then you have to use the same +## user/password for all of those shares. +BACKUP_SERVERS = [ + { 'IP': '10.0.0.10', + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'Backups', + 'User': 'restore', + 'Pass': 'Abracadabra', + }, + { 'IP': '10.0.0.11', + 'Name': 'ServerTwo', + 'Mounted': False, + 'Share': 'Backups', + 'User': 'restore', + 'Pass': 'Abracadabra', + }, +] +CLIENT_INFO_SERVER = { + 'IP': '10.0.0.10', + 'RegEntry': r'0x10001,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + 'Share': '/srv/ClientInfo', + 'User': 'upload', +} +CRASH_SERVER = { + 'Name': 'CrashServer', + 'Url': '', + 'User': '', + 'Pass': '', +} +OFFICE_SERVER = { + 'IP': OFFICE_SERVER_IP, + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'Office', + 'User': 'restore', + 'Pass': 'Abracadabra', +} +QUICKBOOKS_SERVER = { + 'IP': QUICKBOOKS_SERVER_IP, + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'QuickBooks', + 'User': 'restore', + 'Pass': 'Abracadabra', +} +WINDOWS_SERVER = { + 'IP': '10.0.0.10', + 'Name': 'ServerOne', + 'Mounted': False, + 'Share': 'Windows', + 'User': 'restore', + 'Pass': 'Abracadabra', +} + +if __name__ == '__main__': + print("This file is not meant to be called directly.") From 012da0082123453737bc575a38bc3f58c89963a1 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sun, 28 Jan 2018 17:55:40 -0700 Subject: [PATCH 20/32] Removed ClientInfo upload code --- .bin/Scripts/functions/common.py | 54 ------------------------------ .bin/Scripts/system_checklist.py | 8 ----- .bin/Scripts/system_diagnostics.py | 8 ----- 3 files changed, 70 deletions(-) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index f668306b..195b57c4 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -563,60 +563,6 @@ global_vars: {}'''.format(f.read(), sys.argv, global_vars) # No LogFile defined (or invalid LogFile) raise GenericError - -def upload_data(path, file): - """Add CLIENT_INFO_SERVER to authorized connections and upload file.""" - if not ENABLED_UPLOAD_DATA: - raise GenericError('Feature disabled.') - - extract_item('PuTTY', filter='wizkit.ppk psftp.exe', silent=True) - - # Authorize connection to the server - winreg.CreateKey(HKCU, r'Software\SimonTatham\PuTTY\SshHostKeys') - with winreg.OpenKey(HKCU, r'Software\SimonTatham\PuTTY\SshHostKeys', - access=winreg.KEY_WRITE) as key: - winreg.SetValueEx(key, - 'rsa2@22:{IP}'.format(**CLIENT_INFO_SERVER), 0, - winreg.REG_SZ, CLIENT_INFO_SERVER['RegEntry']) - - # Write batch file - with open(r'{}\psftp.batch'.format(global_vars['TmpDir']), - 'w', encoding='ascii') as f: - f.write('lcd "{path}"\n'.format(path=path)) - f.write('cd "{Share}"\n'.format(**CLIENT_INFO_SERVER)) - f.write('mkdir {TicketNumber}\n'.format(**global_vars)) - f.write('cd {TicketNumber}\n'.format(**global_vars)) - f.write('put "{file}"\n'.format(file=file)) - - # Upload Info - cmd = [ - global_vars['Tools']['PuTTY-PSFTP'], - '-noagent', - '-i', r'{BinDir}\PuTTY\wizkit.ppk'.format(**global_vars), - '{User}@{IP}'.format(**CLIENT_INFO_SERVER), - '-b', r'{TmpDir}\psftp.batch'.format(**global_vars)] - run_program(cmd) - -def upload_info(): - """Upload compressed Info file to the NAS as set in settings.main.py.""" - if not ENABLED_UPLOAD_DATA: - raise GenericError('Feature disabled.') - - path = '{ClientDir}'.format(**global_vars) - file = 'Info_{Date-Time}.7z'.format(**global_vars) - upload_data(path, file) - -def compress_info(): - """Compress ClientDir info folders with 7-Zip for upload_info().""" - path = '{ClientDir}'.format(**global_vars) - file = 'Info_{Date-Time}.7z'.format(**global_vars) - _cmd = [ - global_vars['Tools']['SevenZip'], - 'a', '-t7z', '-mx=9', '-bso0', '-bse0', - r'{}\{}'.format(path, file), - r'{ClientDir}\Info'.format(**global_vars)] - run_program(_cmd) - def wait_for_process(name, poll_rate=3): """Wait for process by name.""" running = True diff --git a/.bin/Scripts/system_checklist.py b/.bin/Scripts/system_checklist.py index 4d4d34f8..9de65c8a 100644 --- a/.bin/Scripts/system_checklist.py +++ b/.bin/Scripts/system_checklist.py @@ -80,14 +80,6 @@ if __name__ == '__main__': try_and_print(message='Installed RAM:', function=show_installed_ram, ns='Unknown', silent_function=False) - # Upload info - if ENABLED_UPLOAD_DATA: - print_info('Finalizing') - try_and_print(message='Compressing Info...', - function=compress_info, cs='Done') - try_and_print(message='Uploading to NAS...', - function=upload_info, cs='Done') - # Play audio, show devices, open Windows updates, and open Activation popen_program(['mmc', 'devmgmt.msc']) run_hwinfo_sensors() diff --git a/.bin/Scripts/system_diagnostics.py b/.bin/Scripts/system_diagnostics.py index ad4b423e..0d10f144 100644 --- a/.bin/Scripts/system_diagnostics.py +++ b/.bin/Scripts/system_diagnostics.py @@ -106,14 +106,6 @@ if __name__ == '__main__': except Exception: print_error(' Unknown error.') - # Upload info - if ENABLED_UPLOAD_DATA: - print_info('Finalizing') - try_and_print(message='Compressing Info...', - function=compress_info, cs='Done') - try_and_print(message='Uploading to NAS...', - function=upload_info, cs='Done') - # Done print_standard('\nDone.') pause('Press Enter to exit...') From 3ac2d8d1d4176bcf1b4e4b92803f987563b0b9c1 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sun, 28 Jan 2018 18:19:57 -0700 Subject: [PATCH 21/32] Bugfix: Setting Windows time should work again * Was using the wrong variable name --- .bin/Scripts/functions/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index 41ca3340..bd40f9e5 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -192,7 +192,7 @@ def config_explorer_user(): def update_clock(): """Set Timezone and sync clock.""" - run_program(['tzutil' ,'/s', TIME_ZONE], check=False) + run_program(['tzutil' ,'/s', WINDOWS_TIME_ZONE], check=False) run_program(['net', 'stop', 'w32ime'], check=False) run_program( ['w32tm', '/config', '/syncfromflags:manual', From 314bc24d098aac5a24c04b341a9578f926bf6247 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sun, 28 Jan 2018 18:22:52 -0700 Subject: [PATCH 22/32] Removed PuTTY-PSTFP from tools list * Only used for uploading client info (which was removed) --- .bin/Scripts/settings/tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/.bin/Scripts/settings/tools.py b/.bin/Scripts/settings/tools.py index f8fe7737..bb2cda60 100644 --- a/.bin/Scripts/settings/tools.py +++ b/.bin/Scripts/settings/tools.py @@ -35,8 +35,6 @@ TOOLS = { 'ProduKey': { '32': r'ProduKey\ProduKey.exe', '64': r'ProduKey\ProduKey64.exe'}, - 'PuTTY-PSFTP': { - '32': r'PuTTY\PSFTP.EXE'}, 'RKill': { '32': r'RKill\RKill.exe'}, 'SevenZip': { From 16ea653ca4c352c2090c4f9d6631afc423487e66 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Sun, 28 Jan 2018 18:23:59 -0700 Subject: [PATCH 23/32] Removed unused main.py settings --- .bin/Scripts/settings/main.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 7da23dce..24ae4ebc 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -11,14 +11,10 @@ KIT_NAME_FULL='Wizard Kit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux -DIAG_SHARE='/srv/ClientInfo' -DIAG_USER='wkdiag' MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags ROOT_PASSWORD='Abracadabra' -SKIP_UPLOAD='False' TECH_PASSWORD='Abracadabra' # Server IP addresses -DIAG_SERVER='10.0.0.10' OFFICE_SERVER_IP='10.0.0.10' QUICKBOOKS_SERVER_IP='10.0.0.10' # Time Zones @@ -48,12 +44,6 @@ BACKUP_SERVERS = [ 'Pass': 'Abracadabra', }, ] -CLIENT_INFO_SERVER = { - 'IP': '10.0.0.10', - 'RegEntry': r'0x10001,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', - 'Share': '/srv/ClientInfo', - 'User': 'upload', -} CRASH_SERVER = { 'Name': 'CrashServer', 'Url': '', From fbf956c65160ee5ccd755198c4cd3711f90a75da Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Tue, 30 Jan 2018 13:33:42 -0700 Subject: [PATCH 24/32] Finished build-ufd script --- .bin/Scripts/build-ufd | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index a3508beb..71b7a01f 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -28,7 +28,7 @@ function ask() { return 0 fi while :; do - read -p "$1 " -r answer + read -p "$1 [Y/N] " -r answer if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then return 0 elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then @@ -145,34 +145,34 @@ mkfs.vfat -F 32 -n "$UFD_LABEL" "$DEST_PARTITION" >/dev/null 2>&1 # Mount sources and dest echo "Mounting sources and destination..." mkdir /mnt/{Dest,Linux,WinPE} -p -mount $DEST_PARTITION /mnt +mount $DEST_PARTITION /mnt/Dest mount "$LINUX_ISO" /mnt/Linux -r mount "$WINPE_ISO" /mnt/WinPE -r # Copy files echo "Copying Linux files..." -tmux split-window -l 15 cp -Lrv /mnt/Linux/* /mnt/ +tmux split-window -l 15 cp -Lrv /mnt/Linux/* /mnt/Dest/ echo "Copying WinPE files..." -tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/ -tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Microsoft /mnt/EFI/ -tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Boot/* /mnt/EFI/Microsoft/ -tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot/{BCD,boot.sdi},bootmgr} /mnt/sources/ +tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/Dest/ +tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Microsoft /mnt/Dest/EFI/ +tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Boot/* /mnt/Dest/EFI/Microsoft/ +tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot/{BCD,boot.sdi},bootmgr} /mnt/Dest/sources/ echo "Copying Main Kit..." -tmux split-window -l 15 cp -Lrv "$MAIN_KIT" /mnt/ +tmux split-window -l 15 cp -Lrv "$MAIN_KIT" /mnt/Dest/ if [ "$EXTRA_DIR" != "__None__" ]; then echo "Copying Extra files..." - tmux split-window -l 15 cp -Lrv "$EXTRA_DIR"/* /mnt/ + tmux split-window -l 15 cp -Lrv "$EXTRA_DIR"/* /mnt/Dest/ fi # Install syslinux echo "Copying Syslinux files..." -tmux split-window -l 15 cp -Lrv /usr/lib/syslinux/bios/*.c32 /mnt/arch/boot/syslinux/ +tmux split-window -l 15 cp -Lrv /usr/lib/syslinux/bios/*.c32 /mnt/Dest/arch/boot/syslinux/ syslinux --install -d /arch/boot/syslinux/ $DEST_PARTITION >/dev/null 2>&1 echo "Unmounting destination..." -umount /mnt +umount /mnt/Dest sync echo "Installing Syslinux MBR..." @@ -190,7 +190,7 @@ sync # Unmount Sources echo "Unmounting sources..." -umount /mnt/* -R +umount /mnt/{Linux,WinPE} -R # Done echo "" From e0db7560e6abe1c940051cd0dc4a4cede4299802 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Tue, 30 Jan 2018 15:23:46 -0700 Subject: [PATCH 25/32] Better server share mounting * Added ability to mount read-only or read-write * Allows the same main.py file to be used for all build scripts. --- .bin/Scripts/functions/data.py | 25 +++++++++++++++++++------ .bin/Scripts/functions/windows_setup.py | 7 +++++-- .bin/Scripts/functions/winpe_menus.py | 2 +- .bin/Scripts/settings/main.py | 10 ++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 8cb1783f..e2e76d7c 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -239,7 +239,7 @@ def mount_all_volumes(): line.append(info) return report -def mount_backup_shares(): +def mount_backup_shares(read_write=False): """Mount the backup shares unless labeled as already mounted.""" if psutil.LINUX: mounted_data = get_mounted_data() @@ -258,12 +258,22 @@ def mount_backup_shares(): print_warning(mounted_str) continue - mount_network_share(server) + mount_network_share(server, read_write) -def mount_network_share(server): +def mount_network_share(server, read_write=False): """Mount a network share defined by server.""" + if read_write: + username = server['RW-User'] + password = server['RW-Pass'] + else: + username = server['User'] + password = server['Pass'] if psutil.WINDOWS: - cmd = r'net use \\{IP}\{Share} /user:{User} {Pass}'.format(**server) + cmd = r'net use \\{ip}\{share} /user:{username} {password}'.format( + ip = server['IP'], + share = server['Share'], + username = username, + password = password) cmd = cmd.split(' ') warning = r'Failed to mount \\{Name}\{Share}, {IP} unreachable.'.format( **server) @@ -278,7 +288,10 @@ def mount_network_share(server): 'sudo', 'mount', '//{IP}/{Share}'.format(**server), '/Backups/{Name}'.format(**server), - '-o', 'username={User},password={Pass}'.format(**server)] + '-o', '{}username={},password={}'.format( + '' if read_write else 'ro,', + username, + password)] warning = 'Failed to mount /Backups/{Name}, {IP} unreachable.'.format( **server) error = 'Failed to mount /Backups/{Name}'.format(**server) @@ -514,7 +527,7 @@ def select_source(ticket_number): local_sources = [] remote_sources = [] sources = [] - mount_backup_shares() + mount_backup_shares(read_write=False) # Check for ticket folders on servers for server in BACKUP_SERVERS: diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index 5519e7cc..9efa0123 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -154,9 +154,12 @@ def format_mbr(disk): run_diskpart(script) def mount_windows_share(): - """Mount the Windows images share unless labeled as already mounted.""" + """Mount the Windows images share unless labeled as already mounted.""" if not WINDOWS_SERVER['Mounted']: - mount_network_share(WINDOWS_SERVER) + # Mounting read-write in case a backup was done in the same session + # and the server was left mounted read-write. This avoids throwing an + # error by trying to mount the same server with multiple credentials. + mount_network_share(WINDOWS_SERVER, read_write=True) def select_windows_version(): actions = [ diff --git a/.bin/Scripts/functions/winpe_menus.py b/.bin/Scripts/functions/winpe_menus.py index 1b8a546e..99bc3f8f 100644 --- a/.bin/Scripts/functions/winpe_menus.py +++ b/.bin/Scripts/functions/winpe_menus.py @@ -80,7 +80,7 @@ def menu_backup(): ticket_number = get_ticket_number() # Mount backup shares - mount_backup_shares() + mount_backup_shares(read_write=True) # Select destination destination = select_backup_destination(auto_select=False) diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index 24ae4ebc..86b0fcee 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -35,6 +35,8 @@ BACKUP_SERVERS = [ 'Share': 'Backups', 'User': 'restore', 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', }, { 'IP': '10.0.0.11', 'Name': 'ServerTwo', @@ -42,6 +44,8 @@ BACKUP_SERVERS = [ 'Share': 'Backups', 'User': 'restore', 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', }, ] CRASH_SERVER = { @@ -57,6 +61,8 @@ OFFICE_SERVER = { 'Share': 'Office', 'User': 'restore', 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', } QUICKBOOKS_SERVER = { 'IP': QUICKBOOKS_SERVER_IP, @@ -65,6 +71,8 @@ QUICKBOOKS_SERVER = { 'Share': 'QuickBooks', 'User': 'restore', 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', } WINDOWS_SERVER = { 'IP': '10.0.0.10', @@ -73,6 +81,8 @@ WINDOWS_SERVER = { 'Share': 'Windows', 'User': 'restore', 'Pass': 'Abracadabra', + 'RW-User': 'backup', + 'RW-Pass': 'Abracadabra', } if __name__ == '__main__': From 89c343943f677d226351e09f9c95756708cbfdef Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Tue, 30 Jan 2018 16:00:59 -0700 Subject: [PATCH 26/32] Replaced remaining show_info calls with show_data --- .bin/Scripts/cbs_fix.py | 2 +- .bin/Scripts/functions/common.py | 3 --- .bin/Scripts/user_data_transfer.py | 6 +++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.bin/Scripts/cbs_fix.py b/.bin/Scripts/cbs_fix.py index 22b6c49c..a3b40a8d 100644 --- a/.bin/Scripts/cbs_fix.py +++ b/.bin/Scripts/cbs_fix.py @@ -23,7 +23,7 @@ if __name__ == '__main__': # Show details print_info('{}: CBS Cleanup Tool\n'.format(KIT_NAME_FULL)) - show_info('Backup / Temp path:', dest) + show_data('Backup / Temp path:', dest) print_standard('\n') if (not ask('Proceed with CBS cleanup?')): abort() diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 195b57c4..dbbcda61 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -447,9 +447,6 @@ def show_data(message='~Some message~', data='~Some data~', indent=8, width=32, else: print_standard(message) -def show_info(message='~Some message~', info='~Some info~', indent=8, width=32): - show_data(message=message, data=info, indent=indent, width=width) - def sleep(seconds=2): """Wait for a while.""" time.sleep(seconds) diff --git a/.bin/Scripts/user_data_transfer.py b/.bin/Scripts/user_data_transfer.py index d3da5e46..8616137e 100644 --- a/.bin/Scripts/user_data_transfer.py +++ b/.bin/Scripts/user_data_transfer.py @@ -28,9 +28,9 @@ if __name__ == '__main__': # Transfer clear_screen() print_info('Transfer Details:\n') - show_info('Ticket:', ticket_number) - show_info('Source:', source.path) - show_info('Destination:', dest) + show_data('Ticket:', ticket_number) + show_data('Source:', source.path) + show_data('Destination:', dest) if (not ask('Proceed with transfer?')): umount_backup_shares() From 96c6997a44803bd17995b026186b8ed4ebe3d971 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Tue, 30 Jan 2018 16:50:28 -0700 Subject: [PATCH 27/32] Update function docstrings --- .bin/Scripts/functions/backup.py | 10 +++++++++- .bin/Scripts/functions/common.py | 7 +++++++ .bin/Scripts/functions/data.py | 8 ++++++-- .bin/Scripts/functions/disk.py | 13 +++++++++++++ .bin/Scripts/functions/hw_diags.py | 11 +++++++++++ .bin/Scripts/functions/network.py | 2 ++ .bin/Scripts/functions/update.py | 11 +++++++++++ .bin/Scripts/functions/windows_setup.py | 5 +++++ .bin/Scripts/functions/winpe_menus.py | 4 ++++ 9 files changed, 68 insertions(+), 3 deletions(-) diff --git a/.bin/Scripts/functions/backup.py b/.bin/Scripts/functions/backup.py index acb930a1..fe0935fb 100644 --- a/.bin/Scripts/functions/backup.py +++ b/.bin/Scripts/functions/backup.py @@ -11,6 +11,7 @@ REGEX_BAD_PATH_NAMES = re.compile( re.IGNORECASE) def backup_partition(disk, par): + """Create a backup image of a partition.""" if par.get('Image Exists', False) or par['Number'] in disk['Bad Partitions']: raise GenericAbort @@ -28,9 +29,15 @@ def backup_partition(disk, par): run_program(cmd) def fix_path(path): + """Replace invalid filename characters with underscores.""" return REGEX_BAD_PATH_NAMES.sub('_', path) def prep_disk_for_backup(destination, disk, ticket_number): + """Gather details about the disk and its partitions. + + This includes partitions that can't be backed up, + whether backups already exist on the BACKUP_SERVER, + partition names/sizes/used space, etc.""" disk['Clobber Risk'] = [] width = len(str(len(disk['Partitions']))) @@ -102,7 +109,7 @@ def prep_disk_for_backup(destination, disk, ticket_number): disk['Backup Warnings'] = warnings def select_backup_destination(auto_select=True): - # Build menu + """Select a backup destination from a menu, returns server dict.""" destinations = [s for s in BACKUP_SERVERS if s['Mounted']] actions = [ {'Name': 'Main Menu', 'Letter': 'M'}, @@ -136,6 +143,7 @@ def select_backup_destination(auto_select=True): return destinations[int(selection)-1] def verify_wim_backup(partition): + """Verify WIM integrity.""" if not os.path.exists(partition['Image Path']): raise PathNotFoundError cmd = [ diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index dbbcda61..189e8bf5 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -399,6 +399,7 @@ def print_warning(*args, **kwargs): print_standard(*args, color=COLORS['YELLOW'], **kwargs) def print_log(message='', end='\n', timestamp=True): + """Writes message to a log if LogFile is set.""" time_str = time.strftime("%Y-%m-%d %H%M%z: ") if timestamp else '' if 'LogFile' in global_vars and global_vars['LogFile']: with open(global_vars['LogFile'], 'a', encoding='utf-8') as f: @@ -526,6 +527,9 @@ def try_and_print(message='Trying...', return {'CS': not bool(err), 'Error': err, 'Out': out} def upload_crash_details(): + """Upload log and runtime data to the CRASH_SERVER. + + Intended for uploading to a public Nextcloud share.""" if not ENABLED_UPLOAD_DATA: raise GenericError @@ -762,6 +766,9 @@ def set_common_vars(): **global_vars) def set_linux_vars(): + """Set common variables in a Linux environment. + + These assume we're running under a WK-Linux build.""" result = run_program(['mktemp', '-d']) global_vars['TmpDir'] = result.stdout.decode().strip() global_vars['Date'] = time.strftime("%Y-%m-%d") diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index e2e76d7c..964eb965 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -353,6 +353,7 @@ def run_wimextract(source, items, dest): run_program(cmd) def list_source_items(source_obj, rel_path=None): + """List items in a dir or WIM, returns a list of SourceItem objects.""" items = [] rel_path = '{}{}'.format(os.sep, rel_path) if rel_path else '' if source_obj.is_dir(): @@ -470,6 +471,7 @@ def scan_source(source_obj, dest_path, rel_path='', interactive=True): return selected_items def get_source_item_obj(source_obj, rel_path, item_path): + """Check if the item exists and return a SourceItem object if it does.""" item_obj = None item_path = re.sub(r'(\\|/)', os.sep, item_path) if source_obj.is_dir(): @@ -481,6 +483,8 @@ def get_source_item_obj(source_obj, rel_path, item_path): rel_path, os.sep if rel_path else '', item_path)) + if not os.path.exists(item_obj.path): + item_obj = None else: # Assuming WIM file if psutil.WINDOWS: @@ -734,12 +738,12 @@ def transfer_source(source_obj, dest_path, selected_items): raise GenericError def umount_backup_shares(): - """Unnount the backup shares regardless of current status.""" + """Unmount the backup shares regardless of current status.""" for server in BACKUP_SERVERS: umount_network_share(server) def umount_network_share(server): - """Unnount a network share defined by server.""" + """Unmount a network share defined by server.""" cmd = r'net use \\{IP}\{Share} /delete'.format(**server) cmd = cmd.split(' ') try: diff --git a/.bin/Scripts/functions/disk.py b/.bin/Scripts/functions/disk.py index 3864352a..2d7cf0bb 100644 --- a/.bin/Scripts/functions/disk.py +++ b/.bin/Scripts/functions/disk.py @@ -12,6 +12,7 @@ REGEX_DISK_MBR = re.compile(r'Disk ID: [A-Z0-9]+', re.IGNORECASE) REGEX_DISK_RAW = re.compile(r'Disk ID: 00000000', re.IGNORECASE) def assign_volume_letters(): + """Assign a volume letter to all available volumes.""" remove_volume_letters() # Write script @@ -24,6 +25,7 @@ def assign_volume_letters(): run_diskpart(script) def get_boot_mode(): + """Check if the boot mode was UEFI or legacy.""" boot_mode = 'Legacy' try: reg_key = winreg.OpenKey( @@ -37,6 +39,7 @@ def get_boot_mode(): return boot_mode def get_disk_details(disk): + """Get disk details using DiskPart.""" details = {} script = [ 'select disk {}'.format(disk['Number']), @@ -61,6 +64,7 @@ def get_disk_details(disk): return details def get_disks(): + """Get list of attached disks using DiskPart.""" disks = [] try: @@ -79,6 +83,7 @@ def get_disks(): return disks def get_partition_details(disk, partition): + """Get partition details using DiskPart and fsutil.""" details = {} script = [ 'select disk {}'.format(disk['Number']), @@ -157,6 +162,7 @@ def get_partition_details(disk, partition): return details def get_partitions(disk): + """Get list of partition using DiskPart.""" partitions = [] script = [ 'select disk {}'.format(disk['Number']), @@ -179,6 +185,7 @@ def get_partitions(disk): return partitions def get_table_type(disk): + """Get disk partition table type using DiskPart.""" part_type = 'Unknown' script = [ 'select disk {}'.format(disk['Number']), @@ -200,6 +207,7 @@ def get_table_type(disk): return part_type def get_volumes(): + """Get list of volumes using DiskPart.""" vols = [] try: result = run_diskpart(['list volume']) @@ -214,9 +222,11 @@ def get_volumes(): return vols def is_bad_partition(par): + """Check if the partition is accessible.""" return 'Letter' not in par or REGEX_BAD_PARTITION.search(par['FileSystem']) def prep_disk_for_formatting(disk=None): + """Gather details about the disk and its partitions.""" disk['Format Warnings'] = '\n' width = len(str(len(disk['Partitions']))) @@ -261,6 +271,7 @@ def prep_disk_for_formatting(disk=None): partition['Display String'] = display def reassign_volume_letter(letter, new_letter='I'): + """Assign a new letter to a volume using DiskPart.""" if not letter: # Ignore return None @@ -276,6 +287,7 @@ def reassign_volume_letter(letter, new_letter='I'): return new_letter def remove_volume_letters(keep=None): + """Remove all assigned volume letters using DiskPart.""" if not keep: keep = '' @@ -292,6 +304,7 @@ def remove_volume_letters(keep=None): pass def run_diskpart(script): + """Run DiskPart script.""" tempfile = r'{}\diskpart.script'.format(global_vars['Env']['TMP']) # Write script diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index e75849b8..ac9d5a26 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -42,6 +42,7 @@ TESTS = { } def get_smart_details(dev): + """Get SMART data for dev if possible, returns dict.""" cmd = 'sudo smartctl --all --json /dev/{}'.format(dev).split() result = run_program(cmd, check=False) try: @@ -51,6 +52,7 @@ def get_smart_details(dev): return {} def get_status_color(s): + """Get color based on status, returns str.""" color = COLORS['CLEAR'] if s in ['Denied', 'NS', 'OVERRIDE', 'Unknown']: color = COLORS['RED'] @@ -61,6 +63,7 @@ def get_status_color(s): return color def menu_diags(*args): + """Main HW-Diagnostic menu.""" diag_modes = [ {'Name': 'All tests', 'Tests': ['Prime95', 'NVMe/SMART', 'badblocks']}, @@ -133,6 +136,7 @@ def menu_diags(*args): break def run_badblocks(): + """Run a read-only test for all detected disks.""" aborted = False clear_screen() print_log('\nStart badblocks test(s)\n') @@ -191,6 +195,7 @@ def run_badblocks(): pass def run_mprime(): + """Run Prime95 for MPRIME_LIMIT minutes while showing the temps.""" aborted = False clear_screen() print_log('\nStart Prime95 test') @@ -282,6 +287,7 @@ def run_mprime(): run_program('tmux kill-pane -a'.split()) def run_nvme_smart(): + """Run the built-in NVMe or SMART test for all detected disks.""" aborted = False clear_screen() print_log('\nStart NVMe/SMART test(s)\n') @@ -376,6 +382,7 @@ def run_nvme_smart(): run_program('tmux kill-pane -a'.split(), check=False) def run_tests(tests): + """Run selected hardware test(s).""" print_log('Starting Hardware Diagnostics') print_log('\nRunning tests: {}'.format(', '.join(tests))) # Enable selected tests @@ -414,6 +421,7 @@ def run_tests(tests): pause('Press Enter to exit...') def scan_disks(): + """Scan for disks eligible for hardware testing.""" clear_screen() # Get eligible disk list @@ -489,6 +497,7 @@ def scan_disks(): TESTS['badblocks']['Devices'] = devs def show_disk_details(dev): + """Display disk details.""" dev_name = dev['lsblk']['name'] # Device description print_info('Device: /dev/{}'.format(dev['lsblk']['name'])) @@ -566,6 +575,7 @@ def show_disk_details(dev): print_success(raw_str, timestamp=False) def show_results(): + """Show results for selected test(s).""" clear_screen() print_log('\n───────────────────────────') print_standard('Hardware Diagnostic Results') @@ -629,6 +639,7 @@ def show_results(): run_program('tmux kill-pane -a'.split()) def update_progress(): + """Update progress file.""" if 'Progress Out' not in TESTS: TESTS['Progress Out'] = '{}/progress.out'.format(global_vars['LogDir']) output = [] diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index 1f987248..0d6beb3a 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -54,6 +54,7 @@ def is_connected(): return False def show_valid_addresses(): + """Show all valid private IP addresses assigned to the system.""" devs = psutil.net_if_addrs() for dev, families in sorted(devs.items()): for family in families: @@ -62,6 +63,7 @@ def show_valid_addresses(): show_data(message=dev, data=family.address) def speedtest(): + """Run a network speedtest using speedtest-cli.""" result = run_program(['speedtest-cli', '--simple']) output = [line.strip() for line in result.stdout.decode().splitlines() if line.strip()] diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 813e6df2..afaa59bd 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -9,6 +9,7 @@ from settings.music import * from settings.sources import * def compress_and_remove_item(item): + """Compress and delete an item unless an error is encountered.""" try: compress_item(item) except: @@ -17,6 +18,7 @@ def compress_and_remove_item(item): remove_item(item.path) def compress_item(item): + """Compress an item in a 7-Zip archive using the ARCHIVE_PASSWORD.""" # Prep prev_dir = os.getcwd() dest = '{}.7z'.format(item.path) @@ -58,9 +60,11 @@ def download_generic(out_dir, out_name, source_url): raise GenericError('Failed to download file.') def download_to_temp(out_name, source_url): + """Download a file to the TmpDir.""" download_generic(global_vars['TmpDir'], out_name, source_url) def extract_generic(source, dest, mode='x', sz_args=[]): + """Extract a file to a destination.""" cmd = [ global_vars['Tools']['SevenZip'], mode, source, r'-o{}'.format(dest), @@ -70,11 +74,13 @@ def extract_generic(source, dest, mode='x', sz_args=[]): run_program(cmd) def extract_temp_to_bin(source, item, mode='x', sz_args=[]): + """Extract a file to the .bin folder.""" source = r'{}\{}'.format(global_vars['TmpDir'], source) dest = r'{}\{}'.format(global_vars['BinDir'], item) extract_generic(source, dest, mode, sz_args) def extract_temp_to_cbin(source, item, mode='x', sz_args=[]): + """Extract a file to the .cbin folder.""" source = r'{}\{}'.format(global_vars['TmpDir'], source) dest = r'{}\{}'.format(global_vars['CBinDir'], item) include_path = r'{}\_include\{}'.format(global_vars['CBinDir'], item) @@ -83,6 +89,7 @@ def extract_temp_to_cbin(source, item, mode='x', sz_args=[]): extract_generic(source, dest, mode, sz_args) def generate_launcher(section, name, options): + """Generate a launcher script.""" # Prep dest = r'{}\{}'.format(global_vars['BaseDir'], section) if section == '(Root)': @@ -119,6 +126,7 @@ def generate_launcher(section, name, options): f.write('\n'.join(out_text)) def remove_item(item_path): + """Delete a file or folder.""" if os.path.exists(item_path): if os.path.isdir(item_path): shutil.rmtree(item_path, ignore_errors=True) @@ -126,6 +134,7 @@ def remove_item(item_path): os.remove(item_path) def remove_from_kit(item): + """Delete a file or folder from the .bin/.cbin folders.""" item_locations = [] for p in [global_vars['BinDir'], global_vars['CBinDir']]: item_locations.append(r'{}\{}'.format(p, item)) @@ -134,6 +143,7 @@ def remove_from_kit(item): remove_item(item_path) def remove_from_temp(item): + """Delete a file or folder from the TmpDir folder.""" item_path = r'{}\{}'.format(global_vars['TmpDir'], item) remove_item(item_path) @@ -159,6 +169,7 @@ def resolve_dynamic_url(source_url, regex): return url def scan_for_net_installers(server, family_name, min_year): + """Scan network shares for installers.""" if not server['Mounted']: mount_network_share(server) diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index 9efa0123..0749838d 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -162,6 +162,7 @@ def mount_windows_share(): mount_network_share(WINDOWS_SERVER, read_write=True) def select_windows_version(): + """Select Windows version from a menu, returns dict.""" actions = [ {'Name': 'Main Menu', 'Letter': 'M'}, ] @@ -178,6 +179,7 @@ def select_windows_version(): raise GenericAbort def setup_windows(windows_image, windows_version): + """Apply a Windows image to W:\""" cmd = [ global_vars['Tools']['wimlib-imagex'], 'apply', @@ -189,6 +191,7 @@ def setup_windows(windows_image, windows_version): run_program(cmd) def setup_windows_re(windows_version, windows_letter='W', tools_letter='T'): + """Setup the WinRE partition.""" win = r'{}:\Windows'.format(windows_letter) winre = r'{}\System32\Recovery\WinRE.wim'.format(win) dest = r'{}:\Recovery\WindowsRE'.format(tools_letter) @@ -206,6 +209,7 @@ def setup_windows_re(windows_version, windows_letter='W', tools_letter='T'): run_program(cmd) def update_boot_partition(system_letter='S', windows_letter='W', mode='ALL'): + """Setup the Windows boot partition.""" cmd = [ r'{}\Windows\System32\bcdboot.exe'.format( global_vars['Env']['SYSTEMDRIVE']), @@ -215,6 +219,7 @@ def update_boot_partition(system_letter='S', windows_letter='W', mode='ALL'): run_program(cmd) def wim_contains_image(filename, imagename): + """Check if an ESD/WIM contains the specified image, returns bool.""" cmd = [ global_vars['Tools']['wimlib-imagex'], 'info', diff --git a/.bin/Scripts/functions/winpe_menus.py b/.bin/Scripts/functions/winpe_menus.py index 99bc3f8f..bdbfc11c 100644 --- a/.bin/Scripts/functions/winpe_menus.py +++ b/.bin/Scripts/functions/winpe_menus.py @@ -51,6 +51,7 @@ PE_TOOLS = { } def check_pe_tools(): + """Fix tool paths for WinPE layout.""" for k in PE_TOOLS.keys(): PE_TOOLS[k]['Path'] = r'{}\{}'.format( global_vars['BinDir'], PE_TOOLS[k]['Path']) @@ -203,6 +204,7 @@ def menu_backup(): pause('\nPress Enter to return to main menu... ') def menu_root(): + """Main WinPE menu.""" check_pe_tools() menus = [ {'Name': 'Create Backups', 'Menu': menu_backup}, @@ -381,6 +383,7 @@ def menu_setup(): pause('\nPress Enter to return to main menu... ') def menu_tools(): + """Tool launcher menu.""" tools = [{'Name': k} for k in sorted(PE_TOOLS.keys())] actions = [{'Name': 'Main Menu', 'Letter': 'M'},] set_title(KIT_NAME_FULL) @@ -409,6 +412,7 @@ def menu_tools(): break def select_minidump_path(): + """Select BSOD minidump path from a menu.""" dumps = [] # Assign volume letters first From 7bebe745a11b064b02701e51b65cc70ec9dbe06b Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Tue, 30 Jan 2018 17:10:54 -0700 Subject: [PATCH 28/32] Update build_pe.ps1 --- .bin/Scripts/build_pe.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index 142bc88c..61d43bf4 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -10,7 +10,7 @@ $Host.UI.RawUI.WindowTitle = "Wizard Kit: Windows PE Build Tool" $WD = $(Split-Path $MyInvocation.MyCommand.Path) $Bin = (Get-Item $WD -Force).Parent.FullName $Root = (Get-Item $Bin -Force).Parent.FullName -$Build = "$Root\BUILD" +$Build = "$Root\BUILD_PE" $LogDir = "$Build\Logs" $Temp = "$Build\Temp" $Date = Get-Date -UFormat "%Y-%m-%d" From 44acb9b755fc9b23ee2eccc9f0cf902cf9251ad2 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Tue, 30 Jan 2018 17:28:34 -0700 Subject: [PATCH 29/32] Update windows_setup.py --- .bin/Scripts/functions/windows_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index 0749838d..bf21069a 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -179,7 +179,7 @@ def select_windows_version(): raise GenericAbort def setup_windows(windows_image, windows_version): - """Apply a Windows image to W:\""" + """Apply a Windows image to W:""" cmd = [ global_vars['Tools']['wimlib-imagex'], 'apply', From 9feff50913c6ebcdb237bb8cd00e864876470733 Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 2 Feb 2018 16:34:22 -0700 Subject: [PATCH 30/32] Added syslinux dependency --- .linux_items/packages/dependencies | 1 + 1 file changed, 1 insertion(+) diff --git a/.linux_items/packages/dependencies b/.linux_items/packages/dependencies index 5cd71ab9..02b90822 100644 --- a/.linux_items/packages/dependencies +++ b/.linux_items/packages/dependencies @@ -11,3 +11,4 @@ p7zip progsreiserfs refind-efi rsync +syslinux From bd8151a8cb3fe7a20d711f7001c259bd7622d19c Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 2 Feb 2018 16:37:13 -0700 Subject: [PATCH 31/32] Improved safety checks for build-ufd --- .bin/Scripts/build-ufd | 76 +++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 71b7a01f..3cce38df 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -2,16 +2,25 @@ # ## Wizard Kit: UFD Build Tool +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail + DEST_DEV="$1" DEST_PARTITION="${DEST_DEV}1" MAIN_PY="/usr/local/bin/settings/main.py" +LOG_FILE="${HOME}/build-ufd_${DEST_DEV##*/}_$(date +%Y-%m-%d_%H%M_%z).log" +RSYNC_ARGS="-hrtuvS --modify-window=1 --progress" WD=$(pwd) EXTRA_DIR="${WD}/Extras" -MAIN_KIT="$(dirname $(find $WD -type d -name '.bin') 2>/dev/null)" -LINUX_ISO="$(find $WD -type f -iname '*Linux*iso' | head -1)" -WINPE_ISO="$(find $WD -type f -iname '*WinPE*iso' | head -1)" -if [ "$2" == "--silent" ]; then +MAIN_KIT="$(dirname $(find $WD -type d -name '.bin' || true) 2>/dev/null || true)" +LINUX_ISO="$((find $WD -maxdepth 1 -type f -iname '*Linux*iso' 2>/dev/null || echo "__Missing__") | sort -r | head -1)" +WINPE_ISO="$((find $WD -maxdepth 1 -type f -iname '*WinPE*amd64*iso' 2>/dev/null || echo "__Missing__") | sort -r | head -1)" +if [ "${2:-}" == "--silent" ]; then SILENT="True" +else + SILENT="False" fi # COLORS @@ -23,12 +32,12 @@ BLUE="\e[34m" # Functions function ask() { - if [ "$SILENT" == "True" ]; then - echo -e "$1 Yes ${BLUE}(Silent)${CLEAR}" + if [[ "${SILENT}" == "True" ]]; then + echo -e "${1:-} Yes ${BLUE}(Silent)${CLEAR}" return 0 fi while :; do - read -p "$1 [Y/N] " -r answer + read -p "${1:-} [Y/N] " -r answer if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then return 0 elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then @@ -38,7 +47,7 @@ function ask() { } die () { - echo "$0:" "$@" >&2 + echo "$@" >&2 echo "" read -p "Press Enter to exit... " -r exit 1 @@ -69,9 +78,9 @@ if [[ "$EUID" -ne 0 ]]; then fi # Check if in tmux -if ! tmux list-session | grep -q "build-ufd"; then +if ! tmux list-session 2>/dev/null | grep -q "build-ufd"; then # Reload in tmux - tmux new-session -s "build-ufd" "$0" $* + tmux new-session -s "build-ufd" "${0:-}" $* exit 0 fi @@ -135,62 +144,67 @@ fi # Start Build echo "" echo -e "${GREEN}Building Kit${CLEAR}" +touch "$LOG_FILE" +tmux split-window -dl 10 tail -f "$LOG_FILE" # Format echo "Formatting drive..." -parted "$DEST_DEV" -s -- mklabel msdos mkpart primary fat32 1MiB -1s -parted "$DEST_DEV" set 1 boot on -mkfs.vfat -F 32 -n "$UFD_LABEL" "$DEST_PARTITION" >/dev/null 2>&1 +parted "$DEST_DEV" -s -- mklabel msdos mkpart primary fat32 1MiB -1s >> "$LOG_FILE" 2>&1 +parted "$DEST_DEV" set 1 boot on >> "$LOG_FILE" 2>&1 +mkfs.vfat -F 32 -n "$UFD_LABEL" "$DEST_PARTITION" >> "$LOG_FILE" 2>&1 # Mount sources and dest echo "Mounting sources and destination..." -mkdir /mnt/{Dest,Linux,WinPE} -p -mount $DEST_PARTITION /mnt/Dest -mount "$LINUX_ISO" /mnt/Linux -r -mount "$WINPE_ISO" /mnt/WinPE -r +mkdir /mnt/{Dest,Linux,WinPE} -p >> "$LOG_FILE" 2>&1 +mount $DEST_PARTITION /mnt/Dest >> "$LOG_FILE" 2>&1 +mount "$LINUX_ISO" /mnt/Linux -r >> "$LOG_FILE" 2>&1 +mount "$WINPE_ISO" /mnt/WinPE -r >> "$LOG_FILE" 2>&1 # Copy files echo "Copying Linux files..." -tmux split-window -l 15 cp -Lrv /mnt/Linux/* /mnt/Dest/ +rsync ${RSYNC_ARGS} /mnt/Linux/* /mnt/Dest/ >> "$LOG_FILE" 2>&1 echo "Copying WinPE files..." -tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/Dest/ -tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Microsoft /mnt/Dest/EFI/ -tmux split-window -l 15 cp -Lrv /mnt/WinPE/EFI/Boot/* /mnt/Dest/EFI/Microsoft/ -tmux split-window -l 15 cp -Lrv /mnt/WinPE/{Boot/{BCD,boot.sdi},bootmgr} /mnt/Dest/sources/ +rsync ${RSYNC_ARGS} /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/Dest/ >> "$LOG_FILE" 2>&1 +rsync ${RSYNC_ARGS} /mnt/WinPE/EFI/Microsoft /mnt/Dest/EFI/ >> "$LOG_FILE" 2>&1 +rsync ${RSYNC_ARGS} /mnt/WinPE/EFI/Boot/* /mnt/Dest/EFI/Microsoft/ >> "$LOG_FILE" 2>&1 +rsync ${RSYNC_ARGS} /mnt/WinPE/{Boot/{BCD,boot.sdi},bootmgr} /mnt/Dest/sources/ >> "$LOG_FILE" 2>&1 echo "Copying Main Kit..." -tmux split-window -l 15 cp -Lrv "$MAIN_KIT" /mnt/Dest/ +rsync ${RSYNC_ARGS} "$MAIN_KIT/" "/mnt/Dest/$KIT_NAME_FULL/" >> "$LOG_FILE" 2>&1 if [ "$EXTRA_DIR" != "__None__" ]; then echo "Copying Extra files..." - tmux split-window -l 15 cp -Lrv "$EXTRA_DIR"/* /mnt/Dest/ + rsync ${RSYNC_ARGS} "$EXTRA_DIR"/* /mnt/Dest/ >> "$LOG_FILE" 2>&1 fi # Install syslinux echo "Copying Syslinux files..." -tmux split-window -l 15 cp -Lrv /usr/lib/syslinux/bios/*.c32 /mnt/Dest/arch/boot/syslinux/ -syslinux --install -d /arch/boot/syslinux/ $DEST_PARTITION >/dev/null 2>&1 +rsync ${RSYNC_ARGS} /usr/lib/syslinux/bios/*.c32 /mnt/Dest/arch/boot/syslinux/ >> "$LOG_FILE" 2>&1 +syslinux --install -d /arch/boot/syslinux/ $DEST_PARTITION >> "$LOG_FILE" 2>&1 echo "Unmounting destination..." -umount /mnt/Dest +umount /mnt/Dest >> "$LOG_FILE" 2>&1 sync echo "Installing Syslinux MBR..." -dd bs=440 count=1 if=/usr/lib/syslinux/bios/mbr.bin of=$DEST_DEV >/dev/null 2>&1 +dd bs=440 count=1 if=/usr/lib/syslinux/bios/mbr.bin of=$DEST_DEV >> "$LOG_FILE" 2>&1 sync # Cleanup echo "Hiding boot files..." echo "drive s: file=\"$DEST_PARTITION\"" > /root/.mtoolsrc echo 'mtools_skip_check=1' >> /root/.mtoolsrc -for item in BOOT{,MGR,MGR.EFI} EFI EN-US ISOLINUX LOADER SOURCES SYSLINUX.CFG; do - yes | mattrib +h S:/$item >/dev/null 2>&1 +for item in boot{,mgr,mgr.efi} efi en-us images isolinux sources "$KIT_NAME_FULL"/{.bin,.cbin}; do + yes | mattrib +h "S:/$item" >> "$LOG_FILE" 2>&1 || true done sync # Unmount Sources echo "Unmounting sources..." -umount /mnt/{Linux,WinPE} -R +umount /mnt/{Linux,WinPE} -R >> "$LOG_FILE" 2>&1 + +# Close progress pane +pkill -f "tail.*$LOG_FILE" # Done echo "" From 15be408a20aa5e082a8ca674c268c73bca3774ec Mon Sep 17 00:00:00 2001 From: 2Shirt <1923621+2Shirt@users.noreply.github.com> Date: Fri, 2 Feb 2018 18:13:23 -0700 Subject: [PATCH 32/32] Update README.md --- README.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 973220f7..8eaf3c34 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A collection of scripts to help technicians service Windows systems. ### Build Requirements ### * PowerShell 3.0 or newer1 -* 6 Gb disk space +* 10 Gb disk space ### Initial Setup ### @@ -49,19 +49,39 @@ A collection of scripts to help technicians service Windows systems. ### Initial Setup ### * Replace artwork as desired -* Run `Build_Linux` which will do the following: - * Install missing dependancies with pacman - * Open `main.py` in nano for configuration - * Build the local repo for the AUR packages - * Build the live Linux environment (exported as an ISO file) +* Install Arch Linux in a virtual machine ([VirtualBox](https://www.virtualbox.org/) is a good option for Windows systems). + * See the [installation guide](https://wiki.archlinux.org/index.php/Installation_guide) for details. +* Add a standard user to the Arch Linux installation. + * See the [wiki page](https://wiki.archlinux.org/index.php/Users_and_groups#User_management) for details. +* Install git # `pacman -Syu git` +* _(Recommended)_ Install and configure `sudo` + * See the [wiki page](https://wiki.archlinux.org/index.php/Sudo) for details. +* Login to the user added above +* Download the Github repo $ `git clone https://github.com/2Shirt/WizardKit.git` +* Run the build script + * $ `cd WizardKit` + * $ `./Build\ Linux -b` + * The build script does the following: + * Installs missing dependencies via `pacman` + * Opens `main.py` in `nano` for configuration + * Downloads, builds, and adds AUR packages to a local repo + * Builds the Live Linux ISO + +### Notes ### + +* The WinPE boot options require files to be copied from a completed WinPE build. + * This is done below for the Combined UFD ## Windows PE ## ### Build Requirements ### * Windows Assessment and Deployment Kit for Windows 10 + * Deployment Tools + * Windows Preinstallation Environment (Windows PE) + * _All other features are not required_ * PowerShell 3.0 or newer -* 2 Gb disk space +* 8 Gb disk space ### Initial Setup ### @@ -72,5 +92,56 @@ A collection of scripts to help technicians service Windows systems. * Download all tools * Build both 32-bit & 64-bit PE images (exported as ISO files) +## Combined Wizard Kit ## + +### Build Requirements ### + +* 64-bit system or virtual machine +* 4 Gb RAM +* 8 Gb USB flash drive _(16 Gb or larger recommended)_ + +### Overview ### + +There's a `build-ufd` script which does the following: + +* Checks for the presence if the Linux ISO and the (64-bit) WinPE ISO. +* Formats the selected UFD using FAT32. + * All data will be deleted from the UFD resulting in **DATA LOSS**. +* Copies the required files from the Linux ISO, WinPE ISO, and Main Kit folder to the UFD. +* Installs Syslinux to the UFD making it bootable on legacy systems. +* Sets the boot files/folders to be hidden under Windows. + +### Setup ### + +* Boot to a Live Linux ISO built following the instructions above. + * You can apply it to a UFD using [rufus](https://rufus.akeo.ie/) for physical systems. + * Virtual machines should be able to use the Linux ISO directly. +* Put the Linux ISO, the WinPE ISO, and the Main Kit folder _(usually "OUT_KIT")_ in the same directory. + * "OUT_KIT" will be renamed on the UFD using `$KIT_NAME_FULL` + * `$KIT_NAME_FULL` defaults to "Wizard Kit" but can be changed in `main.py` + * "OUT_KIT" can be renamed in the source folder. + * The script searched for the ".bin" folder and uses it's parent folder as the Main Kit source. + * Additional files/folders can be included by putting them in a folder named "Extras". + * These files/folders will be copied to the root of the UFD. + * To include images for the WinPE Setup section, put the files in "Extras/images". + * WinPE Setup will recognize ESD, WIM, and SWM2 images. + * The filenames should be "Win7", "Win8", or "Win10" + * The final layout should be similar to this: _(assuming it's mounted to "/Sources")_ + * **(Required)** `/Sources/OUT_KIT` + * **(Required)** `/Sources/WK-Linux-2018-01-01-x86_64.iso` + * **(Required)** `/Sources/WK-WinPE-2018-01-01-amd64.iso` + * _(Optional)_ `/Sources/Extras/Essential Windows Updates` + * _(Optional)_ `/Sources/Extras/images/Win7.wim` + * _(Optional)_ `/Sources/Extras/images/Win8.wim` + * _(Optional)_ `/Sources/Extras/images/Win10.esd` +* Connect the UFD but don't mount it. +* Mount the device, or connect to the share, with the ISOs and Main Kit folder. +* $ `cd /Sources` _(replace with real path to source files)_ +* Get the device name of the UFD. + * You can use $ `lsblk --fs` or $ `inxi -Dxx` to help. +* $ `sudo build-ufd /dev/sdX` _(replace `/dev/sdX` with the desired device)_ + * **2nd Warning**: All data will be erased from the UFD resulting in **DATA LOSS**. + ## Notes ## 1. PowerShell 6.0 on Windows 7 is not supported by the build script. +2. See [wimlib-imagex](https://wimlib.net/) for details about split WIM images.