diff --git a/.bin/Scripts/Launch.cmd b/.bin/Scripts/Launch.cmd index 18c304f2..faea56f8 100644 --- a/.bin/Scripts/Launch.cmd +++ b/.bin/Scripts/Launch.cmd @@ -263,6 +263,7 @@ call :ExtractOrFindPath || goto ErrorProgramNotFound set "script=%_path%\%L_ITEM%" rem Verify +"%PYTHON%" --version >nul || goto ErrorPythonUnsupported if not exist "%script%" goto ErrorScriptNotFound rem Run @@ -435,6 +436,16 @@ echo ERROR: Office version not supported by this script. start "" "explorer.exe" "%client_dir%\Office" goto Abort +:ErrorPythonUnsupported +rem The Windows installation lacks Windows update KB2999226 needed to run Python +echo. +echo ERROR: Failed to run Python, try installing Windows update KB2999226. +echo NOTE: That update is from October 2015 so this system is SEVERELY outdated +if exist "%bin%\..\Installers\Extras\Windows Updates" ( + start "" "explorer.exe" "%bin%\..\Installers\Extras\Windows Updates" +) +goto Abort + :ErrorQuickBooksSourceNotFound echo. echo ERROR: QuickBooks source "%L_ITEM%" not found. @@ -486,7 +497,7 @@ echo Press any key to exit... pause>nul rem reset color and reset errorlevel to 0 rem NOTE: This is done to avoid causing a ErrorLaunchCMD in the launcher.cmd -color +color 07 goto Exit :: Cleanup and exit :: diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index 1253472c..5f272d7d 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -143,11 +143,11 @@ function help () { [[ "${__usage+x}" ]] || read -r -d '' __usage <<-'EOF' || true # exits non-zero when EOF encountered OPTIONS: -u --ufd-device [arg] Device to which the kit will be applied - -m --main-kit [arg] Path to the Main Kit -l --linux-iso [arg] Path to the Linux ISO - -w --winpe-iso [arg] Path to the WinPE ISO -e --extra-dir [arg] Path to the Extra folder (optional) + -m --main-kit [arg] Path to the Main Kit (optional) + -w --winpe-iso [arg] Path to the WinPE ISO (optional) -h --help This page ADVANCED: @@ -390,15 +390,11 @@ if [[ "${arg_h:?}" == 1 ]]; then else # Print warning line [[ "${arg_u:-}" ]] || echo " -u or --ufd-device is required" - [[ "${arg_m:-}" ]] || echo " -m or --main-kit is required" [[ "${arg_l:-}" ]] || echo " -l or --linux-iso is required" - [[ "${arg_w:-}" ]] || echo " -w or --winpe-iso is required" # Bail if necessary [[ "${arg_u:-}" ]] || help "${__usage_example}" [[ "${arg_l:-}" ]] || help "${__usage_example}" - [[ "${arg_m:-}" ]] || help "${__usage_example}" - [[ "${arg_w:-}" ]] || help "${__usage_example}" fi [[ "${LOG_LEVEL:-}" ]] || emergency "Cannot continue without LOG_LEVEL. " @@ -492,26 +488,30 @@ echo "" # Verify sources [[ -b "${DEST_DEV}" ]] || abort "${DEST_DEV} is not a valid device." -[[ -d "${MAIN_KIT}/.bin" ]] || abort "Invalid Main Kit, ${MAIN_KIT}/.bin not found." [[ -e "${LINUX_ISO}" ]] || abort "Linux ISO not found." -[[ -e "${WINPE_ISO}" ]] || abort "WinPE ISO not found." +if [[ ! -z "${arg_m:-}" ]]; then + [[ -d "${MAIN_KIT}/.bin" ]] || abort "Invalid Main Kit, ${MAIN_KIT}/.bin not found." +fi +if [[ ! -z "${arg_w:-}" ]]; then + [[ -e "${WINPE_ISO}" ]] || abort "WinPE ISO not found." +fi if [[ ! -z "${arg_e:-}" ]]; then [[ -d "${EXTRA_DIR}" ]] || abort "Extra Dir not found." fi # Print Info -echo -e "${BLUE}Sources${CLEAR}" -echo "Main Kit: ${MAIN_KIT}" -echo "Linux ISO: ${LINUX_ISO}" -echo "WinPE ISO: ${WINPE_ISO}" -echo "Extra Dir: ${EXTRA_DIR:-(Not Specified)}" -echo "" -echo -e "${BLUE}Destination${CLEAR}" -lsblk -n -o NAME,LABEL,SIZE,MODEL,SERIAL "${DEST_DEV}" +echo -e "${BLUE}Sources${CLEAR}" | tee -a "${LOG_FILE}" +echo "Main Kit: ${MAIN_KIT}" | tee -a "${LOG_FILE}" +echo "Linux ISO: ${LINUX_ISO}" | tee -a "${LOG_FILE}" +echo "WinPE ISO: ${WINPE_ISO}" | tee -a "${LOG_FILE}" +echo "Extra Dir: ${EXTRA_DIR:-(Not Specified)}" | tee -a "${LOG_FILE}" +echo "" | tee -a "${LOG_FILE}" +echo -e "${BLUE}Destination${CLEAR}" | tee -a "${LOG_FILE}" +lsblk -n -o NAME,LABEL,SIZE,MODEL,SERIAL "${DEST_DEV}" | tee -a "${LOG_FILE}" if [[ "${USE_MBR}" == "True" ]]; then - echo -e "${YELLOW}Formatting using legacy MBR${CLEAR}" + echo -e "${YELLOW}Formatting using legacy MBR${CLEAR}" | tee -a "${LOG_FILE}" fi -echo "" +echo "" | tee -a "${LOG_FILE}" # Ask before starting job echo "" @@ -528,8 +528,8 @@ else fi # Start Build -echo "" -echo -e "${GREEN}Building Kit${CLEAR}" +echo "" | tee -a "${LOG_FILE}" +echo -e "${GREEN}Building Kit${CLEAR}" | tee -a "${LOG_FILE}" touch "${LOG_FILE}" tmux split-window -dl 10 tail -f "${LOG_FILE}" @@ -537,7 +537,7 @@ tmux split-window -dl 10 tail -f "${LOG_FILE}" dd bs=4M count=16 if=/dev/zero of="${DEST_DEV}" >> "${LOG_FILE}" 2>&1 # Format -echo "Formatting drive..." +echo "Formatting drive..." | tee -a "${LOG_FILE}" if [[ "${USE_MBR}" == "True" ]]; then parted "${DEST_DEV}" --script -- mklabel msdos mkpart primary fat32 4MiB -1s >> "${LOG_FILE}" 2>&1 parted "${DEST_DEV}" set 1 boot on >> "${LOG_FILE}" 2>&1 @@ -550,45 +550,94 @@ fi mkfs.vfat -F 32 -n "${UFD_LABEL}" "${DEST_PAR}" >> "${LOG_FILE}" 2>&1 # Mount sources and dest -echo "Mounting sources and destination..." +echo "Mounting sources and destination..." | tee -a "${LOG_FILE}" mkdir /mnt/{Dest,Linux,WinPE} -p >> "${LOG_FILE}" 2>&1 mount ${DEST_PAR} /mnt/Dest >> "${LOG_FILE}" 2>&1 mount "${LINUX_ISO}" /mnt/Linux -r >> "${LOG_FILE}" 2>&1 -mount "${WINPE_ISO}" /mnt/WinPE -r >> "${LOG_FILE}" 2>&1 +if [[ ! -z "${arg_w:-}" ]]; then + mount "${WINPE_ISO}" /mnt/WinPE -r >> "${LOG_FILE}" 2>&1 +fi + +# Find WinPE source +w_boot="$(find /mnt/WinPE -iwholename "/mnt/WinPE/Boot")" +w_boot_bcd="$(find /mnt/WinPE -iwholename "/mnt/WinPE/Boot/BCD")" +w_boot_sdi="$(find /mnt/WinPE -iwholename "/mnt/WinPE/Boot/boot.sdi")" +w_bootmgr="$(find /mnt/WinPE -iwholename "/mnt/WinPE/bootmgr")" +w_bootmgr_efi="$(find /mnt/WinPE -iwholename "/mnt/WinPE/bootmgr.efi")" +w_efi_boot="$(find /mnt/WinPE -iwholename "/mnt/WinPE/EFI/Boot")" +w_efi_microsoft="$(find /mnt/WinPE -iwholename "/mnt/WinPE/EFI/Microsoft")" +w_en_us="$(find /mnt/WinPE -iwholename "/mnt/WinPE/en-us")" +w_sources="$(find /mnt/WinPE -iwholename "/mnt/WinPE/sources")" # Copy files -echo "Copying Linux files..." +echo "Copying Linux files..." | tee -a "${LOG_FILE}" rsync ${RSYNC_ARGS} /mnt/Linux/* /mnt/Dest/ >> "${LOG_FILE}" 2>&1 sed -i "s/${ISO_LABEL}/${UFD_LABEL}/" /mnt/Dest/EFI/boot/refind.conf -sed -i "s/#UFD#//" /mnt/Dest/EFI/boot/refind.conf sed -i "s/${ISO_LABEL}/${UFD_LABEL}/" /mnt/Dest/arch/boot/syslinux/*cfg -sed -i "s/#UFD#//" /mnt/Dest/arch/boot/syslinux/*cfg -echo "Copying WinPE files..." -rsync ${RSYNC_ARGS} /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/Dest/ >> "${LOG_FILE}" 2>&1 -rsync ${RSYNC_ARGS} /mnt/WinPE/EFI/Microsoft /mnt/Dest/EFI/ >> "${LOG_FILE}" 2>&1 -rsync ${RSYNC_ARGS} /mnt/WinPE/EFI/Boot/* /mnt/Dest/EFI/Microsoft/ >> "${LOG_FILE}" 2>&1 -rsync ${RSYNC_ARGS} /mnt/WinPE/{Boot/{BCD,boot.sdi},bootmgr} /mnt/Dest/sources/ >> "${LOG_FILE}" 2>&1 +echo "Copying WinPE files..." | tee -a "${LOG_FILE}" +if [[ ! -z "${arg_w:-}" ]]; then + if [[ ! -z "${w_bootmgr:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_bootmgr}" /mnt/Dest/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_bootmgr_efi:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_bootmgr_efi}" /mnt/Dest/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_en_us:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_en_us}" /mnt/Dest/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_boot:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_boot}"/* /mnt/Dest/Boot/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_efi_boot:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_efi_boot}"/* /mnt/Dest/EFI/Microsoft/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_efi_microsoft:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_efi_microsoft}"/* /mnt/Dest/EFI/Microsoft/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_boot_bcd:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_boot_bcd}" /mnt/Dest/sources/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_boot_sdi:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_boot_sdi}" /mnt/Dest/sources/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_bootmgr:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_bootmgr}" /mnt/Dest/sources/ >> "${LOG_FILE}" 2>&1 + fi + if [[ ! -z "${w_sources:-}" ]]; then + rsync ${RSYNC_ARGS} "${w_sources}"/* /mnt/Dest/sources/ >> "${LOG_FILE}" 2>&1 + fi -echo "Copying Main Kit..." -rsync ${RSYNC_ARGS} "${MAIN_KIT}/" "/mnt/Dest/${KIT_NAME_FULL}/" >> "${LOG_FILE}" 2>&1 + # Uncomment boot entries + sed -i "s/#UFD-WINPE#//" /mnt/Dest/EFI/boot/refind.conf + sed -i "s/#UFD-WINPE#//" /mnt/Dest/arch/boot/syslinux/*cfg +fi + +echo "Copying Main Kit..." | tee -a "${LOG_FILE}" +if [[ ! -z "${arg_m:-}" ]]; then + rsync ${RSYNC_ARGS} \ + "${MAIN_KIT}/" \ + "/mnt/Dest/${KIT_NAME_FULL}/" >> "${LOG_FILE}" 2>&1 +fi if [[ ! -z "${EXTRA_DIR:-}" ]]; then - echo "Copying Extra files..." - rsync ${RSYNC_ARGS} "${EXTRA_DIR}"/ /mnt/Dest/ >> "${LOG_FILE}" 2>&1 + echo "Copying Extra files..." | tee -a "${LOG_FILE}" + rsync ${RSYNC_ARGS} \ + "${EXTRA_DIR}"/ \ + /mnt/Dest/ >> "${LOG_FILE}" 2>&1 fi # Install syslinux -echo "Copying Syslinux files..." +echo "Copying Syslinux files..." | tee -a "${LOG_FILE}" rsync ${RSYNC_ARGS} /usr/lib/syslinux/bios/*.c32 /mnt/Dest/arch/boot/syslinux/ >> "${LOG_FILE}" 2>&1 syslinux --install -d /arch/boot/syslinux/ ${DEST_PAR} >> "${LOG_FILE}" 2>&1 -echo "Unmounting destination..." +echo "Unmounting destination..." | tee -a "${LOG_FILE}" umount /mnt/Dest >> "${LOG_FILE}" 2>&1 rmdir /mnt/Dest >> "${LOG_FILE}" 2>&1 sync -echo "Installing Syslinux MBR..." +echo "Installing Syslinux MBR..." | tee -a "${LOG_FILE}" if [[ "${USE_MBR}" == "True" ]]; then dd bs=440 count=1 if=/usr/lib/syslinux/bios/mbr.bin of=${DEST_DEV} >> "${LOG_FILE}" 2>&1 else @@ -597,7 +646,7 @@ fi sync # Cleanup -echo "Hiding boot files..." +echo "Hiding boot files..." | tee -a "${LOG_FILE}" echo "drive s: file=\"${DEST_PAR}\"" > /root/.mtoolsrc echo 'mtools_skip_check=1' >> /root/.mtoolsrc for item in arch Boot bootmgr{,.efi} EFI en-us images isolinux sources "${KIT_NAME_FULL}"/{.bin,.cbin}; do @@ -606,7 +655,7 @@ done sync # Unmount Sources -echo "Unmounting sources..." +echo "Unmounting sources..." | tee -a "${LOG_FILE}" for d in Linux WinPE; do umount "/mnt/${d}" >> "${LOG_FILE}" 2>&1 || true rmdir "/mnt/${d}" >> "${LOG_FILE}" 2>&1 || true @@ -616,7 +665,7 @@ done pkill -f "tail.*${LOG_FILE}" # Done -echo "" -echo "Done." +echo "" | tee -a "${LOG_FILE}" +echo "Done." | tee -a "${LOG_FILE}" echo "" exit 0 diff --git a/.bin/Scripts/build_kit.ps1 b/.bin/Scripts/build_kit.ps1 index 08a050c9..76e69966 100644 --- a/.bin/Scripts/build_kit.ps1 +++ b/.bin/Scripts/build_kit.ps1 @@ -79,15 +79,15 @@ if ($MyInvocation.InvocationName -ne ".") { $Path = $Temp # 7-Zip - DownloadFile -Path $Path -Name "7z-installer.msi" -Url "https://www.7-zip.org/a/7z1806.msi" - DownloadFile -Path $Path -Name "7z-extra.7z" -Url "https://www.7-zip.org/a/7z1806-extra.7z" + DownloadFile -Path $Path -Name "7z-installer.msi" -Url "https://www.7-zip.org/a/7z1900.msi" + DownloadFile -Path $Path -Name "7z-extra.7z" -Url "https://www.7-zip.org/a/7z1900-extra.7z" # ConEmu - $Url = "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z" + $Url = "https://github.com/Maximus5/ConEmu/releases/download/v19.03.10/ConEmuPack.190310.7z" DownloadFile -Path $Path -Name "ConEmuPack.7z" -Url $Url # Notepad++ - $Url = "https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z" + $Url = "https://notepad-plus-plus.org/repository/7.x/7.6.4/npp.7.6.4.bin.minimalist.7z" DownloadFile -Path $Path -Name "npp.7z" -Url $Url # Python diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index 00405a12..e783e39e 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -131,25 +131,25 @@ if ($MyInvocation.InvocationName -ne ".") { ## Download Tools ## $ToolSources = @( # 7-Zip - @("7z-installer.msi", "https://www.7-zip.org/a/7z1806.msi"), - @("7z-extra.7z", "https://www.7-zip.org/a/7z1806-extra.7z"), + @("7z-installer.msi", "https://www.7-zip.org/a/7z1900.msi"), + @("7z-extra.7z", "https://www.7-zip.org/a/7z1900-extra.7z"), # Blue Screen View @("bluescreenview32.zip", "http://www.nirsoft.net/utils/bluescreenview.zip"), @("bluescreenview64.zip", "http://www.nirsoft.net/utils/bluescreenview-x64.zip"), # ConEmu - @("ConEmuPack.7z", "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z"), + @("ConEmuPack.7z", "https://github.com/Maximus5/ConEmu/releases/download/v19.03.10/ConEmuPack.190310.7z"), # Fast Copy - @("fastcopy.zip", "http://ftp.vector.co.jp/70/93/2323/FastCopy361_installer.exe"), + @("fastcopy.zip", "http://ftp.vector.co.jp/71/31/2323/FastCopy363_installer.exe"), # HWiNFO - @("hwinfo.zip", "https://www.fosshub.com/HWiNFO.html?dwl=hwi_600.zip"), + @("hwinfo.zip", "http://files2.majorgeeks.com/377527622c5325acc1cb937fb149d0de922320c0/systeminfo/hwi_602.zip"), # Killer Network Drivers @( "killerinf.zip", ("https://www.killernetworking.com"+(FindDynamicUrl "https://www.killernetworking.com/killersupport/category/other-downloads" "Download Killer-Ethernet").replace('&', '&')) ), # Notepad++ - @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z"), - @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.x64.7z"), + @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.6.4/npp.7.6.4.bin.minimalist.7z"), + @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.6.4/npp.7.6.4.bin.minimalist.x64.7z"), # NT Password Editor @("ntpwed.zip", "http://cdslow.org.ru/files/ntpwedit/ntpwed07.zip"), # Prime95 diff --git a/.bin/Scripts/debug/hw_diags.py b/.bin/Scripts/debug/hw_diags.py new file mode 100644 index 00000000..87a35990 --- /dev/null +++ b/.bin/Scripts/debug/hw_diags.py @@ -0,0 +1,183 @@ +# Wizard Kit: Debug - HW Diagnostics + +import base64 +import requests + +from functions.common import * + +def debug_report_cpu(cpu_obj): + """Generate report for CpuObj, returns list.""" + report = [] + + # CPU Info + report.append('CPU: {}'.format(cpu_obj.name)) + report.append('lscpu:') + for k, v in sorted(cpu_obj.lscpu.items()): + report.append(' {}: {}'.format(k, v)) + + # Tests + report.append('Tests:') + for k, v in cpu_obj.tests.items(): + report.extend(debug_report_test(v, k)) + + # Done + return report + + +def debug_report_disk(disk_obj): + """Generate report for DiskObj, returns list.""" + report = [] + expand = [ + 'lsblk', + 'nvme_attributes', + 'nvme_smart_notes', + 'smart_attributes', + 'smart_self_test', + 'smartctl', + ] + skip = [ + 'add_nvme_smart_note', + 'calc_io_dd_values', + 'check_attributes', + 'check_smart_self_test', + 'description', + 'disable_test', + 'generate_attribute_report', + 'generate_disk_report', + 'get_details', + 'get_size', + 'get_smart_details', + 'name', + 'safety_check', + 'tests', + ] + + # Disk Info + report.append('Disk: {} {}'.format( + disk_obj.name, disk_obj.description)) + for a in dir(disk_obj): + if a.startswith('_') or a in skip: + continue + if a in expand: + report.append('{}:'.format(a)) + attr = getattr(disk_obj, a) + try: + for k, v in sorted(attr.items()): + report.append(' {}: {}'.format(k, v)) + except Exception: + # Ignore + pass + else: + report.append('{}: {}'.format(a, getattr(disk_obj, a))) + + # Tests + report.append('Tests:') + for k, v in disk_obj.tests.items(): + report.extend(debug_report_test(v, k)) + + # Done + return report + + +def debug_report_state(state): + """Generate report for State, returns list.""" + report = [] + + # Devs + report.append('CPU: {}'.format(state.cpu)) + report.append('Disks: {}'.format(state.disks)) + + # Settings + report.append('Progress Out: {}'.format(state.progress_out)) + report.append('Quick Mode: {}'.format(state.quick_mode)) + + # Tests + report.append('Tests:') + for k, v in state.tests.items(): + report.append(' {}:'.format(k)) + for k2, v2 in sorted(v.items()): + report.append(' {}: {}'.format(k2, v2)) + + # tmux + if hasattr(state, 'tmux_layout'): + report.append('tmux Layout:') + for k, v in state.tmux_layout.items(): + report.append(' {}: {}'.format(k, str(v))) + if hasattr(state, 'panes'): + report.append('tmux Panes:') + for k, v in state.panes.items(): + report.append(' {}: {}'.format(k, str(v))) + + # Done + return report + + +def debug_report_test(test_obj, test_name): + """Generate report for TestObj, returns list.""" + report = [' {}:'.format(test_name)] + skip = ['update_status'] + + # Attributes + for a in [a for a in dir(test_obj) if not a.startswith('_')]: + if a in skip: + continue + report.append(' {}: {}'.format(a, getattr(test_obj, a))) + + # Done + return report + + +def save_debug_reports(state, global_vars): + """Save debug reports if possible.""" + debug_dest = '{}/debug'.format(global_vars['LogDir']) + os.makedirs(debug_dest, exist_ok=True) + + # State + with open('{}/state.report'.format(debug_dest), 'a') as f: + for line in debug_report_state(state): + f.write('{}\n'.format(line)) + + # CPU + with open('{}/cpu.report'.format(debug_dest), 'a') as f: + for line in debug_report_cpu(state.cpu): + f.write('{}\n'.format(line)) + + # Disk(s) + for disk in state.disks: + with open('{}/disk_{}.report'.format(debug_dest, disk.name), 'a') as f: + for line in debug_report_disk(disk): + f.write('{}\n'.format(line)) + + +def upload_logdir(global_vars): + """Upload compressed LogDir to CRASH_SERVER.""" + source = global_vars['LogDir'] + source = source[source.rfind('/')+1:] + dest = '{}.txz'.format(source) + data = None + + # Compress LogDir + os.chdir('{}/..'.format(global_vars['LogDir'])) + cmd = ['tar', 'caf', dest, source] + run_program(cmd) + + # Read file + with open(dest, 'rb') as f: + data = f.read() + + # Upload data + url = '{}/Crash_{}.txz'.format(CRASH_SERVER['Url'], source) + r = requests.put( + url, + data=data, + headers={'X-Requested-With': 'XMLHttpRequest'}, + auth=(CRASH_SERVER['User'], CRASH_SERVER['Pass'])) + # Raise exception if upload NS + if not r.ok: + raise GenericError + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index 34c0252f..dcb0ed2f 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -2,6 +2,7 @@ from functions.common import * from operator import itemgetter +from settings.browsers import * # Define other_results for later try_and_print @@ -17,95 +18,6 @@ other_results = { } -# STATIC VARIABLES -DEFAULT_HOMEPAGE = 'https://www.google.com/' -IE_GALLERY = 'https://www.microsoft.com/en-us/iegallery' -MOZILLA_PREFS = { - 'browser.search.defaultenginename': '"Google"', - 'browser.search.defaultenginename.US': '"Google"', - 'browser.search.geoSpecificDefaults': 'false', - 'browser.startup.homepage': '"{}"'.format(DEFAULT_HOMEPAGE), - 'extensions.ui.lastCategory': '"addons://list/extension"', - } -UBO_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin/cjpalhdlnbpafiamejdnhcphjbkeiagm?hl=en' -UBO_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm' -UBO_EXTRA_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin-extra/pgdnlhfefecpicbbihgmbmffkjpaplco?hl=en' -UBO_EXTRA_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco' -UBO_MOZILLA = 'https://addons.mozilla.org/en-us/firefox/addon/ublock-origin/' -UBO_MOZZILA_PATH = r'{}\Mozilla Firefox\distribution\extensions\ublock_origin.xpi'.format(os.environ.get('PROGRAMFILES')) -UBO_MOZILLA_REG = r'Software\Mozilla\Firefox\Extensions' -UBO_MOZILLA_REG_NAME = 'uBlock0@raymondhill.net' -UBO_OPERA = 'https://addons.opera.com/en/extensions/details/ublock/?display=en' -SUPPORTED_BROWSERS = { - 'Internet Explorer': { - 'base': 'ie', - 'exe_name': 'iexplore.exe', - 'rel_install_path': 'Internet Explorer', - 'user_data_path': r'{USERPROFILE}\Favorites', - }, - 'Google Chrome': { - 'base': 'chromium', - 'exe_name': 'chrome.exe', - 'rel_install_path': r'Google\Chrome\Application', - 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome\User Data', - }, - 'Google Chrome Canary': { - 'base': 'chromium', - 'exe_name': 'chrome.exe', - 'rel_install_path': r'Google\Chrome SxS\Application', - 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome SxS\User Data', - }, - 'Mozilla Firefox': { - 'base': 'mozilla', - 'exe_name': 'firefox.exe', - 'rel_install_path': 'Mozilla Firefox', - 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', - }, - 'Mozilla Firefox Dev': { - 'base': 'mozilla', - 'exe_name': 'firefox.exe', - 'rel_install_path': 'Firefox Developer Edition', - 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', - }, - 'Opera': { - 'base': 'chromium', - 'exe_name': 'launcher.exe', - 'rel_install_path': 'Opera', - 'user_data_path': r'{APPDATA}\Opera Software\Opera Stable', - }, - 'Opera Beta': { - 'base': 'chromium', - 'exe_name': 'launcher.exe', - 'rel_install_path': 'Opera beta', - 'user_data_path': r'{APPDATA}\Opera Software\Opera Next', - }, - 'Opera Dev': { - 'base': 'chromium', - 'exe_name': 'launcher.exe', - 'rel_install_path': 'Opera developer', - 'user_data_path': r'{APPDATA}\Opera Software\Opera Developer', - }, - } - - -# Regex -REGEX_BACKUP = re.compile( - r'\.\w*bak.*', - re.IGNORECASE) -REGEX_CHROMIUM_PROFILE = re.compile( - r'^(Default|Profile)', - re.IGNORECASE) -REGEX_CHROMIUM_ITEMS = re.compile( - r'^(Bookmarks|Cookies|Favicons|Google Profile' - r'|History|Login Data|Top Sites|TransportSecurity' - r'|Visited Links|Web Data)', - re.IGNORECASE) -REGEX_MOZILLA = re.compile( - r'^(bookmarkbackups|(cookies|formhistory|places).sqlite' - r'|key3.db|logins.json|persdict.dat)$', - re.IGNORECASE) - - def archive_all_users(): """Create backups for all browsers for all users.""" users_root = r'{}\Users'.format(global_vars['Env']['SYSTEMDRIVE']) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index 2bbb4291..502c985b 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -27,11 +27,12 @@ global_vars = {} # STATIC VARIABLES COLORS = { 'CLEAR': '\033[0m', - 'RED': '\033[31m', + 'RED': '\033[31m', 'ORANGE': '\033[31;1m', 'GREEN': '\033[32m', 'YELLOW': '\033[33m', 'BLUE': '\033[34m', + 'PURPLE': '\033[35m', 'CYAN': '\033[36m', } try: @@ -114,6 +115,14 @@ def ask(prompt='Kotaero!'): return answer +def beep(repeat=1): + """Play system bell with optional repeat.""" + for i in range(repeat): + # Print bell char + print('\a') + sleep(0.5) + + def choice(choices, prompt='Kotaero!'): """Prompt the user with a choice question, returns str.""" answer = None @@ -428,6 +437,9 @@ def ping(addr='google.com'): def popen_program(cmd, pipe=False, minimized=False, shell=False, **kwargs): """Run program and return a subprocess.Popen object.""" cmd_kwargs = {'args': cmd, 'shell': shell} + for kw in ('encoding', 'errors'): + if kw in kwargs: + cmd_kwargs[kw] = kwargs[kw] if minimized: startupinfo = subprocess.STARTUPINFO() @@ -497,6 +509,9 @@ def run_program(cmd, check=True, pipe=True, shell=False, **kwargs): cmd = ' '.join(cmd) cmd_kwargs = {'args': cmd, 'check': check, 'shell': shell} + for kw in ('encoding', 'errors'): + if kw in kwargs: + cmd_kwargs[kw] = kwargs[kw] if pipe: cmd_kwargs.update({ diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index 7dcee3cf..eb59d030 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -1,103 +1,10 @@ # Wizard Kit: Functions - Data import ctypes -import json -from functions.common import * +from functions.json import * from operator import itemgetter - - -# STATIC VARIABLES -FAST_COPY_EXCLUDES = [ - r'\*.esd', - r'\*.swm', - r'\*.wim', - r'\*.dd', - r'\*.dd.tgz', - r'\*.dd.txz', - r'\*.map', - r'\*.dmg', - r'\*.image', - r'$RECYCLE.BIN', - r'$Recycle.Bin', - r'.AppleDB', - r'.AppleDesktop', - r'.AppleDouble', - r'.com.apple.timemachine.supported', - r'.dbfseventsd', - r'.DocumentRevisions-V100*', - r'.DS_Store', - r'.fseventsd', - r'.PKInstallSandboxManager', - r'.Spotlight*', - r'.SymAV*', - r'.symSchedScanLockxz', - r'.TemporaryItems', - r'.Trash*', - r'.vol', - r'.VolumeIcon.icns', - r'desktop.ini', - r'Desktop?DB', - r'Desktop?DF', - r'hiberfil.sys', - r'lost+found', - r'Network?Trash?Folder', - r'pagefile.sys', - r'Recycled', - r'RECYCLER', - r'System?Volume?Information', - r'Temporary?Items', - r'Thumbs.db', - ] -FAST_COPY_ARGS = [ - '/cmd=noexist_only', - '/utf8', - '/skip_empty_dir', - '/linkdest', - '/no_ui', - '/auto_close', - '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), - ] -# Code borrowed from: https://stackoverflow.com/a/29075319 -SEM_NORMAL = ctypes.c_uint() -SEM_FAILCRITICALERRORS = 1 -SEM_NOOPENFILEERRORBOX = 0x8000 -SEM_FAIL = SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS - - -# Regex -REGEX_EXCL_ITEMS = re.compile( - r'^(\.(AppleDB|AppleDesktop|AppleDouble' - r'|com\.apple\.timemachine\.supported|dbfseventsd' - r'|DocumentRevisions-V100.*|DS_Store|fseventsd|PKInstallSandboxManager' - r'|Spotlight.*|SymAV.*|symSchedScanLockxz|TemporaryItems|Trash.*' - r'|vol|VolumeIcon\.icns)|desktop\.(ini|.*DB|.*DF)' - r'|(hiberfil|pagefile)\.sys|lost\+found|Network\.*Trash\.*Folder' - r'|Recycle[dr]|System\.*Volume\.*Information|Temporary\.*Items' - r'|Thumbs\.db)$', - re.IGNORECASE) -REGEX_EXCL_ROOT_ITEMS = re.compile( - r'^(boot(mgr|nxt)$|Config.msi' - r'|(eula|globdata|install|vc_?red)' - r'|.*.sys$|System Volume Information|RECYCLER?|\$Recycle\.bin' - r'|\$?Win(dows(.old.*|\. BT|)$|RE_)|\$GetCurrent|Windows10Upgrade' - r'|PerfLogs|Program Files|SYSTEM.SAV' - r'|.*\.(esd|swm|wim|dd|map|dmg|image)$)', - re.IGNORECASE) -REGEX_INCL_ROOT_ITEMS = re.compile( - r'^(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' - r'|Media|Music|Pic(ture|)s?|Vid(eo|)s?)' - r'|{prefix}(-?Info|-?Transfer|)' - r'|(ProgramData|Recovery|Temp.*|Users)$' - r'|.*\.(log|txt|rtf|qb\w*|avi|m4a|m4v|mp4|mkv|jpg|png|tiff?)$)' - r''.format(prefix=KIT_NAME_SHORT), - re.IGNORECASE) -REGEX_WIM_FILE = re.compile( - r'\.wim$', - re.IGNORECASE) -REGEX_WINDOWS_OLD = re.compile( - r'^Win(dows|)\.old', - re.IGNORECASE) +from settings.data import * # Classes @@ -170,8 +77,7 @@ def find_core_storage_volumes(device_path=None): '--output', 'NAME,PARTTYPE'] if device_path: cmd.append(device_path) - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) + json_data = get_json_from_command(cmd) devs = json_data.get('blockdevices', []) devs = [d for d in devs if d.get('parttype', '') == corestorage_uuid] if devs: @@ -251,8 +157,7 @@ def get_mounted_volumes(): 'devtmpfs,hugetlbfs,mqueue,proc,pstore,securityfs,sysfs,tmpfs' ), '-o', 'SOURCE,TARGET,FSTYPE,LABEL,SIZE,AVAIL,USED'] - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) + json_data = get_json_from_command(cmd) mounted_volumes = [] for item in json_data.get('filesystems', []): mounted_volumes.append(item) @@ -277,8 +182,7 @@ def mount_volumes( find_core_storage_volumes(device_path) # Get list of block devices - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) + json_data = get_json_from_command(cmd) devs = json_data.get('blockdevices', []) # Get list of volumes diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py index f6c30e0a..545d08e0 100644 --- a/.bin/Scripts/functions/ddrescue.py +++ b/.bin/Scripts/functions/ddrescue.py @@ -1,8 +1,9 @@ -# Wizard Kit: Functions - ddrescue +# Wizard Kit: Functions - ddrescue-tui -import json +import datetime import pathlib import psutil +import pytz import re import signal import stat @@ -11,37 +12,10 @@ import time from collections import OrderedDict from functions.data import * from functions.hw_diags import * +from functions.json import * from functions.tmux import * from operator import itemgetter - - -# STATIC VARIABLES -AUTO_PASS_1_THRESHOLD = 95 -AUTO_PASS_2_THRESHOLD = 98 -DDRESCUE_SETTINGS = { - '--binary-prefixes': {'Enabled': True, 'Hidden': True}, - '--data-preview': {'Enabled': True, 'Hidden': True, 'Value': '5'}, - '--idirect': {'Enabled': True}, - '--odirect': {'Enabled': True}, - '--max-read-rate': {'Enabled': False, 'Value': '1MiB'}, - '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, - '--reopen-on-error': {'Enabled': True}, - '--retry-passes': {'Enabled': True, 'Value': '0'}, - '--test-mode': {'Enabled': False, 'Value': 'test.map'}, - '--timeout': {'Enabled': True, 'Value': '5m'}, - '-vvvv': {'Enabled': True, 'Hidden': True}, - } -RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] -SIDE_PANE_WIDTH = 21 -TMUX_LAYOUT = OrderedDict({ - 'Source': {'y': 2, 'Check': True}, - 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, - 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, -}) -USAGE = """ {script_name} clone [source [destination]] - {script_name} image [source [destination]] - (e.g. {script_name} clone /dev/sda /dev/sdb) -""" +from settings.ddrescue import * # Clases @@ -282,6 +256,7 @@ class RecoveryState(): self.block_pairs = [] self.current_pass = 0 self.current_pass_str = '0: Initializing' + self.etoc = '' self.settings = DDRESCUE_SETTINGS.copy() self.finished = False self.panes = {} @@ -289,6 +264,8 @@ class RecoveryState(): self.rescued = 0 self.resumed = False self.started = False + self.status = 'Inactive' + self.timezone = pytz.timezone(LINUX_TIME_ZONE) self.total_size = 0 if mode not in ('clone', 'image'): raise GenericError('Unsupported mode') @@ -374,15 +351,14 @@ class RecoveryState(): map_allowed_fstypes = RECOMMENDED_FSTYPES.copy() map_allowed_fstypes.extend(['cifs', 'ext2', 'vfat']) map_allowed_fstypes.sort() - json_data = {} + json_data = get_json_from_command(cmd) - # Avoid saving map to non-persistent filesystem - try: - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) - except Exception: + # Abort if json_data is empty + if not json_data: print_error('ERROR: Failed to verify map path') raise GenericAbort() + + # Avoid saving map to non-persistent filesystem fstype = json_data.get( 'filesystems', [{}])[0].get( 'fstype', 'unknown') @@ -428,6 +404,66 @@ class RecoveryState(): elif self.current_pass == 2: self.current_pass_str = '3 "Scraping bad areas"' + def update_etoc(self): + """Search ddrescue output for the current EToC, returns str.""" + now = datetime.datetime.now(tz=self.timezone) + + # Bail early + if 'NEEDS ATTENTION' in self.status: + # Just set to N/A (NOTE: this overrules the refresh rate below) + self.etoc = 'N/A' + return + elif 'In Progress' not in self.status: + # Don't update when EToC is hidden + return + if now.second % ETOC_REFRESH_RATE != 0: + # Limit updates based on settings/ddrescue.py + return + + self.etoc = 'Unknown' + etoc_delta = None + text = '' + + # Capture main tmux pane + try: + text = tmux_capture_pane() + except Exception: + # Ignore + pass + + # Search for EToC delta + matches = re.findall(r'remaining time:.*$', text, re.MULTILINE) + if matches: + r = REGEX_REMAINING_TIME.search(matches[-1]) + if r.group('na'): + self.etoc = 'N/A' + else: + self.etoc = r.string + days = r.group('days') if r.group('days') else 0 + hours = r.group('hours') if r.group('hours') else 0 + minutes = r.group('minutes') if r.group('minutes') else 0 + seconds = r.group('seconds') if r.group('seconds') else 0 + try: + etoc_delta = datetime.timedelta( + days=int(days), + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds), + ) + except Exception: + # Ignore and leave as raw string + pass + + # Calc finish time if EToC delta found + if etoc_delta: + try: + now = datetime.datetime.now(tz=self.timezone) + _etoc = now + etoc_delta + self.etoc = _etoc.strftime('%Y-%m-%d %H:%M %Z') + except Exception: + # Ignore and leave as current string + pass + def update_progress(self): """Update overall progress using block_pairs.""" self.rescued = 0 @@ -575,21 +611,11 @@ def fix_tmux_panes(state, forced=False): def get_device_details(dev_path): """Get device details via lsblk, returns JSON dict.""" - try: - cmd = ( - 'lsblk', - '--json', - '--output-all', - '--paths', - dev_path) - result = run_program(cmd) - except CalledProcessError: - # Return empty dict and let calling section deal with the issue - return {} + cmd = ['lsblk', '--json', '--output-all', '--paths', dev_path] + json_data = get_json_from_command(cmd) - json_data = json.loads(result.stdout.decode()) # Just return the first device (there should only be one) - return json_data['blockdevices'][0] + return json_data.get('blockdevices', [{}])[0] def get_device_report(dev_path): @@ -622,17 +648,19 @@ def get_device_report(dev_path): def get_dir_details(dir_path): """Get dir details via findmnt, returns JSON dict.""" - try: - result = run_program([ - 'findmnt', '-J', - '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', - '-T', dir_path]) - json_data = json.loads(result.stdout.decode()) - except Exception: + cmd = [ + 'findmnt', '-J', + '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', + '-T', dir_path, + ] + json_data = get_json_from_command(cmd) + + # Raise exception if json_data is empty + if not json_data: raise GenericError( - 'Failed to get directory details for "{}".'.format(self.path)) - else: - return json_data['filesystems'][0] + 'Failed to get directory details for "{}".'.format(dir_path)) + + return json_data.get('filesystems', [{}])[0] def get_dir_report(dir_path): @@ -805,6 +833,13 @@ def menu_main(state): # Show menu while True: + # Update status + if state.finished: + state.status = ' Finished' + else: + state.status = ' Inactive' + update_sidepane(state) + # Update entries for opt in main_options: opt['Name'] = '[{}] {}'.format( @@ -959,6 +994,7 @@ def run_ddrescue(state, pass_settings): """Run ddrescue pass.""" return_code = -1 aborted = False + state.status = ' In Progress' if state.finished: clear_screen() @@ -1073,6 +1109,9 @@ def run_ddrescue(state, pass_settings): # Done if str(return_code) != '0': # Pause on errors + state.status = ' {YELLOW}NEEDS ATTENTION{CLEAR}'.format(**COLORS) + state.status = state.status.replace('33m', '33;5m') + update_sidepane(state) pause('Press Enter to return to main menu... ') # Cleanup @@ -1238,14 +1277,8 @@ def select_path(skip_device=None): def select_device(description='device', skip_device=None): """Select device via a menu, returns DevObj.""" - cmd = ( - 'lsblk', - '--json', - '--nodeps', - '--output-all', - '--paths') - result = run_program(cmd) - json_data = json.loads(result.stdout.decode()) + cmd = ['lsblk', '--json', '--nodeps', '--output-all', '--paths'] + json_data = get_json_from_command(cmd) skip_names = [] if skip_device: skip_names.append(skip_device.path) @@ -1254,7 +1287,7 @@ def select_device(description='device', skip_device=None): # Build menu dev_options = [] - for dev in json_data['blockdevices']: + for dev in json_data.get('blockdevices', []): # Disable dev if in skip_names disabled = dev['name'] in skip_names or dev['pkname'] in skip_names @@ -1335,6 +1368,7 @@ def update_sidepane(state): output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) else: output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) + output.append(state.status) output.append('─────────────────────') # Overall progress @@ -1354,6 +1388,19 @@ def update_sidepane(state): output.extend(bp.status) output.append(' ') + # EToC + if re.search(r'(In Progress|NEEDS ATTENTION)', state.status): + if not output[-1].strip(): + # Last line is empty + output.pop() + output.append('─────────────────────') + output.append('{BLUE}Estimated Pass Finish{CLEAR}'.format(**COLORS)) + state.update_etoc() + if 'N/A' in state.etoc.upper(): + output.append('{YELLOW}N/A{CLEAR}'.format(**COLORS)) + else: + output.append(state.etoc) + # Add line-endings output = ['{}\n'.format(line) for line in output] diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index b1ed5869..f5235111 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1,95 +1,22 @@ # Wizard Kit: Functions - HW Diagnostics -import json import re import time from collections import OrderedDict +from functions.json import * from functions.sensors import * from functions.threading import * from functions.tmux import * +from settings.hw_diags import * +if DEBUG_MODE: + from debug.hw_diags import * -# STATIC VARIABLES -ATTRIBUTES = { - 'NVMe': { - 'critical_warning': {'Error': 1, 'Critical': True}, - 'media_errors': {'Error': 1, 'Critical': True}, - 'power_on_hours': {'Warning': 12000, 'Error': 26298, 'Ignore': True}, - 'unsafe_shutdowns': {'Warning': 1}, - }, - 'SMART': { - 5: {'Hex': '05', 'Error': 1, 'Critical': True}, - 9: {'Hex': '09', 'Warning': 12000, 'Error': 26298, 'Ignore': True}, - 10: {'Hex': '0A', 'Error': 1}, - 184: {'Hex': 'B8', 'Error': 1}, - 187: {'Hex': 'BB', 'Error': 1}, - 188: {'Hex': 'BC', 'Error': 1}, - 196: {'Hex': 'C4', 'Error': 1}, - 197: {'Hex': 'C5', 'Error': 1, 'Critical': True}, - 198: {'Hex': 'C6', 'Error': 1, 'Critical': True}, - 199: {'Hex': 'C7', 'Error': 1, 'Ignore': True}, - 201: {'Hex': 'C9', 'Error': 1}, - }, - } -HW_OVERRIDES_FORCED = HW_OVERRIDES_FORCED and not HW_OVERRIDES_LIMITED -IO_VARS = { - 'Block Size': 512*1024, - 'Chunk Size': 32*1024**2, - 'Minimum Test Size': 10*1024**3, - 'Alt Test Size Factor': 0.01, - 'Progress Refresh Rate': 5, - 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], - 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], - 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], - 'Threshold Graph Fail': 65*1024**2, - 'Threshold Graph Warn': 135*1024**2, - 'Threshold Graph Great': 750*1024**2, - 'Threshold HDD Min': 50*1024**2, - 'Threshold HDD High Avg': 75*1024**2, - 'Threshold HDD Low Avg': 65*1024**2, - 'Threshold SSD Min': 90*1024**2, - 'Threshold SSD High Avg': 135*1024**2, - 'Threshold SSD Low Avg': 100*1024**2, - 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), - 'Graph Horizontal Width': 40, - 'Graph Vertical': ( - '▏', '▎', '▍', '▌', - '▋', '▊', '▉', '█', - '█▏', '█▎', '█▍', '█▌', - '█▋', '█▊', '█▉', '██', - '██▏', '██▎', '██▍', '██▌', - '██▋', '██▊', '██▉', '███', - '███▏', '███▎', '███▍', '███▌', - '███▋', '███▊', '███▉', '████'), - } -KEY_NVME = 'nvme_smart_health_information_log' -KEY_SMART = 'ata_smart_attributes' -QUICK_LABEL = '{YELLOW}(Quick){CLEAR}'.format(**COLORS) -SIDE_PANE_WIDTH = 20 -STATUSES = { - 'RED': ['Denied', 'ERROR', 'NS', 'TimedOut'], - 'YELLOW': ['Aborted', 'N/A', 'OVERRIDE', 'Unknown', 'Working'], - 'GREEN': ['CS'], -} -TESTS_CPU = ['Prime95'] -TESTS_DISK = [ - 'I/O Benchmark', - 'NVMe / SMART', - 'badblocks', - ] -TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}'.format(**COLORS) -TMUX_LAYOUT = OrderedDict({ - 'Top': {'y': 2, 'Check': True}, - 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, - 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, - # Testing panes - 'Prime95': {'y': 11, 'Check': False}, - 'Temps': {'y': 1000, 'Check': False}, - 'SMART': {'y': 3, 'Check': True}, - 'badblocks': {'y': 5, 'Check': True}, - 'I/O Benchmark': {'y': 1000, 'Check': False}, -}) +# Fix settings +OVERRIDES_FORCED = OVERRIDES_FORCED and not OVERRIDES_LIMITED +QUICK_LABEL = QUICK_LABEL.format(**COLORS) +TOP_PANE_TEXT = TOP_PANE_TEXT.format(**COLORS) # Regex @@ -113,15 +40,10 @@ class CpuObj(): def get_details(self): """Get CPU details from lscpu.""" cmd = ['lscpu', '--json'] - try: - result = run_program(cmd, check=False) - json_data = json.loads(result.stdout.decode()) - except Exception: - # Ignore and leave self.lscpu empty - return - for line in json_data.get('lscpu', []): - _field = line.get('field', None).replace(':', '') - _data = line.get('data', None) + json_data = get_json_from_command(cmd) + for line in json_data.get('lscpu', [{}]): + _field = line.get('field', '').replace(':', '') + _data = line.get('data', '') if not _field and not _data: # Skip print_warning(_field, _data) @@ -145,18 +67,21 @@ class CpuObj(): class DiskObj(): """Object for tracking disk specific data.""" def __init__(self, disk_path): + self.attr_type = 'UNKNOWN' self.disk_ok = True self.labels = [] self.lsblk = {} self.name = re.sub(r'^.*/(.*)', r'\1', disk_path) self.nvme_attributes = {} + self.nvme_smart_notes = {} + self.override_disabled = False self.path = disk_path self.smart_attributes = {} - self.smart_timeout = False self.smart_self_test = {} self.smartctl = {} self.tests = OrderedDict() self.get_details() + self.get_size() # Try enabling SMART run_program(['sudo', 'smartctl', '--smart=on', self.path], check=False) @@ -166,39 +91,39 @@ class DiskObj(): self.description = '{size} ({tran}) {model} {serial}'.format( **self.lsblk) - def calc_io_dd_values(self): - """Calcualte I/O benchmark dd values.""" - # Get real disk size - cmd = ['lsblk', - '--bytes', '--nodeps', '--noheadings', - '--output', 'size', self.path] - result = run_program(cmd) - self.size_bytes = int(result.stdout.decode().strip()) + def add_nvme_smart_note(self, note): + """Add note that will be included in the NVMe / SMART report.""" + # A dict is used to avoid duplicate notes + self.nvme_smart_notes[note] = None - # dd calculations - ## The minimum dev size is 'Graph Horizontal Width' * 'Chunk Size' - ## (e.g. 1.25 GB for a width of 40 and a chunk size of 32MB) - ## If the device is smaller than the minimum dd_chunks would be set - ## to zero which would cause a divide by zero error. - ## If the device is below the minimum size an Exception will be raised - ## - ## dd_size is the area to be read in bytes - ## If the dev is < 10Gb then it's the whole dev - ## Otherwise it's the larger of 10Gb or 1% of the dev - ## - ## dd_chunks is the number of groups of "Chunk Size" in self.dd_size - ## This number is reduced to a multiple of the graph width in - ## order to allow for the data to be condensed cleanly - ## - ## dd_chunk_blocks is the chunk size in number of blocks - ## (e.g. 64 if block size is 512KB and chunk size is 32MB - ## - ## dd_skip_blocks is the number of "Block Size" groups not tested - ## dd_skip_count is the number of blocks to skip per self.dd_chunk - ## dd_skip_extra is how often to add an additional skip block - ## This is needed to ensure an even testing across the dev - ## This is calculated by using the fractional amount left off - ## of the dd_skip_count variable + def calc_io_dd_values(self): + """Calcualte I/O benchmark dd values. + + Calculations + The minimum dev size is 'Graph Horizontal Width' * 'Chunk Size' + (e.g. 1.25 GB for a width of 40 and a chunk size of 32MB) + If the device is smaller than the minimum dd_chunks would be set + to zero which would cause a divide by zero error. + If the device is below the minimum size an Exception will be raised + + dd_size is the area to be read in bytes + If the dev is < 10Gb then it's the whole dev + Otherwise it's the larger of 10Gb or 1% of the dev + + dd_chunks is the number of groups of "Chunk Size" in self.dd_size + This number is reduced to a multiple of the graph width in + order to allow for the data to be condensed cleanly + + dd_chunk_blocks is the chunk size in number of blocks + (e.g. 64 if block size is 512KB and chunk size is 32MB + + dd_skip_blocks is the number of "Block Size" groups not tested + dd_skip_count is the number of blocks to skip per self.dd_chunk + dd_skip_extra is how often to add an additional skip block + This is needed to ensure an even testing across the dev + This is calculated by using the fractional amount left off + of the dd_skip_count variable + """ self.dd_size = min(IO_VARS['Minimum Test Size'], self.size_bytes) self.dd_size = max( self.dd_size, @@ -220,61 +145,72 @@ class DiskObj(): # self.dd_skip_extra == 0 is fine pass - def check_attributes(self, silent=False): - """Check NVMe / SMART attributes for errors.""" - override_disabled = False + def check_attributes(self): + """Check NVMe / SMART attributes for errors, returns bool.""" + attr_type = self.attr_type + disk_ok = True + + # Get updated attributes + self.get_smart_details() + + # Check attributes if self.nvme_attributes: - attr_type = 'NVMe' + self.add_nvme_smart_note( + ' {YELLOW}NVMe disk support is still experimental{CLEAR}'.format( + **COLORS)) items = self.nvme_attributes.items() elif self.smart_attributes: - attr_type = 'SMART' items = self.smart_attributes.items() for k, v in items: if k in ATTRIBUTES[attr_type]: - if 'Error' not in ATTRIBUTES[attr_type][k]: - # Only worried about error thresholds + if not ATTRIBUTES[attr_type][k]['Error']: + # Informational attribute, skip continue - if ATTRIBUTES[attr_type][k].get('Ignore', False): + if ATTRIBUTES[attr_type][k]['Ignore']: # Attribute is non-failing, skip continue if v['raw'] >= ATTRIBUTES[attr_type][k]['Error']: - self.disk_ok = False + if (ATTRIBUTES[attr_type][k]['Maximum'] + and v['raw'] >= ATTRIBUTES[attr_type][k]['Maximum']): + # Non-standard value, skip + continue + else: + disk_ok = False - # Disable override if necessary - override_disabled |= ATTRIBUTES[attr_type][k].get( - 'Critical', False) + # Disable override if necessary + self.override_disabled |= ATTRIBUTES[attr_type][k].get( + 'Critical', False) # SMART overall assessment ## NOTE: Only fail drives if the overall value exists and reports failed if not self.smartctl.get('smart_status', {}).get('passed', True): - self.disk_ok = False - override_disabled = True + disk_ok = False + self.override_disabled = True + self.add_nvme_smart_note( + ' {RED}SMART overall self-assessment: Failed{CLEAR}'.format(**COLORS)) - # Print errors - if not silent: - if self.disk_ok: - # 199/C7 warning - if self.smart_attributes.get(199, {}).get('raw', 0) > 0: - print_warning('199/C7 error detected') - print_standard(' (Have you tried swapping the disk cable?)') - else: - # Override? - show_report( - self.generate_attribute_report(description=True), - log_report=True) - print_warning(' {} error(s) detected.'.format(attr_type)) - if override_disabled: - print_standard('Tests disabled for this device') - pause() - elif not (len(self.tests) == 3 and HW_OVERRIDES_LIMITED): - if HW_OVERRIDES_FORCED or ask('Run tests on this device anyway?'): - self.disk_ok = True - if 'NVMe / SMART' in self.tests: - self.disable_test('NVMe / SMART', 'OVERRIDE') - if not self.nvme_attributes and self.smart_attributes: - # Re-enable for SMART short-tests - self.tests['NVMe / SMART'].disabled = False - print_standard(' ') + # Done + return disk_ok + + def check_smart_self_test(self, silent=False): + """Check if a SMART self-test is currently running, returns bool.""" + msg = 'SMART self-test in progress' + test_running = 'remaining_percent' in self.smart_self_test.get('status', '') + + if test_running: + # Ask to abort + if not silent: + print_warning('WARNING: {}'.format(msg)) + print_standard(' ') + if ask('Abort HW Diagnostics?'): + raise GenericAbort('Bail') + + # Add warning note + self.add_nvme_smart_note( + ' {YELLOW}WARNING: {msg}{CLEAR}'.format(msg=msg, **COLORS)) + + # Done + return test_running def disable_test(self, name, status): """Disable test by name and update status.""" @@ -283,32 +219,21 @@ class DiskObj(): self.tests[name].disabled = True def generate_attribute_report( - self, description=False, short_test=False, timestamp=False): + self, description=False, timestamp=False): """Generate NVMe / SMART report, returns list.""" + attr_type = self.attr_type report = [] if description: report.append('{BLUE}Device ({name}){CLEAR}'.format( name=self.name, **COLORS)) report.append(' {}'.format(self.description)) - # Warnings - if self.nvme_attributes: - attr_type = 'NVMe' - report.append( - ' {YELLOW}NVMe disk support is still experimental{CLEAR}'.format( - **COLORS)) - elif self.smart_attributes: - attr_type = 'SMART' - else: - # No attribute data available, return short report + # Skip attributes if they don't exist + if not (self.nvme_attributes or self.smart_attributes): report.append( ' {YELLOW}No NVMe or SMART data available{CLEAR}'.format( **COLORS)) return report - if not self.smartctl.get('smart_status', {}).get('passed', True): - report.append( - ' {RED}SMART overall self-assessment: Failed{CLEAR}'.format( - **COLORS)) # Attributes report.append('{BLUE}{a} Attributes{YELLOW}{u:>23} {t}{CLEAR}'.format( @@ -337,10 +262,12 @@ class DiskObj(): n=v['name'][:28]) # Set color - for _t, _c in [['Warning', 'YELLOW'], ['Error', 'RED']]: - if _t in ATTRIBUTES[attr_type][k]: + for _t, _c in ATTRIBUTE_COLORS: + if ATTRIBUTES[attr_type][k][_t]: if v['raw'] >= ATTRIBUTES[attr_type][k][_t]: _color = COLORS[_c] + if _t == 'Maximum': + _note = '(invalid?)' # 199/C7 warning if str(k) == '199' and v['raw'] > 0: @@ -356,30 +283,21 @@ class DiskObj(): # Add line to report report.append(_line) - # SMART short-test - if short_test: - report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) - report.append(' {}'.format( - self.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) - if self.smart_timeout: - report.append(' {YELLOW}Timed out{CLEAR}'.format(**COLORS)) - # Done return report def generate_disk_report(self): """Generate disk report with data from all tests.""" report = [] - report.append('{BLUE}Device ({name}){CLEAR}'.format( - name=self.name, **COLORS)) - report.append(' {}'.format(self.description)) # Attributes - if 'NVMe / SMART' not in self.tests: - report.extend(self.generate_attribute_report()) - elif not self.tests['NVMe / SMART'].report: - report.extend(self.generate_attribute_report()) + report.extend(self.generate_attribute_report(description=True)) + + # Notes + if self.nvme_smart_notes: + report.append('{BLUE}{attr_type} Notes{CLEAR}'.format( + attr_type=self.attr_type, **COLORS)) + report.extend(sorted(self.nvme_smart_notes.keys())) # Tests for test in self.tests.values(): @@ -390,26 +308,25 @@ class DiskObj(): def get_details(self): """Get data from lsblk.""" cmd = ['lsblk', '--json', '--output-all', '--paths', self.path] - try: - result = run_program(cmd, check=False) - json_data = json.loads(result.stdout.decode()) - self.lsblk = json_data['blockdevices'][0] - except Exception: - # Leave self.lsblk empty - pass + json_data = get_json_from_command(cmd) + self.lsblk = json_data.get('blockdevices', [{}])[0] # Set necessary details self.lsblk['model'] = self.lsblk.get('model', 'Unknown Model') self.lsblk['name'] = self.lsblk.get('name', self.path) + self.lsblk['phy-sec'] = self.lsblk.get('phy-sec', -1) self.lsblk['rota'] = self.lsblk.get('rota', True) self.lsblk['serial'] = self.lsblk.get('serial', 'Unknown Serial') self.lsblk['size'] = self.lsblk.get('size', '???b') self.lsblk['tran'] = self.lsblk.get('tran', '???') - # Ensure certain attributes are strings + # Ensure certain attributes types for attr in ['model', 'name', 'rota', 'serial', 'size', 'tran']: if not isinstance(self.lsblk[attr], str): self.lsblk[attr] = str(self.lsblk[attr]) + for attr in ['phy-sec']: + if not isinstance(self.lsblk[attr], int): + self.lsblk[attr] = int(self.lsblk[attr]) self.lsblk['tran'] = self.lsblk['tran'].upper().replace('NVME', 'NVMe') # Build list of labels @@ -418,18 +335,26 @@ class DiskObj(): self.labels.append(disk.get('partlabel', '')) self.labels = [str(label) for label in self.labels if label] + def get_size(self): + """Get real disk size.""" + cmd = ['lsblk', + '--bytes', '--nodeps', '--noheadings', + '--output', 'size', self.path] + try: + result = run_program(cmd) + self.size_bytes = int(result.stdout.decode().strip()) + except Exception: + # Setting to impossible value for now + self.size_bytes = -1 + def get_smart_details(self): """Get data from smartctl.""" cmd = ['sudo', 'smartctl', '--all', '--json', self.path] - try: - result = run_program(cmd, check=False) - self.smartctl = json.loads(result.stdout.decode()) - except Exception: - # Leave self.smartctl empty - pass + self.smartctl = get_json_from_command(cmd) # Check for attributes if KEY_NVME in self.smartctl: + self.attr_type = 'NVMe' self.nvme_attributes = {} for k, v in self.smartctl[KEY_NVME].items(): try: @@ -442,6 +367,7 @@ class DiskObj(): # TODO: Limit this check pass elif KEY_SMART in self.smartctl: + self.attr_type = 'SMART' for a in self.smartctl[KEY_SMART].get('table', {}): try: _id = int(a.get('id', -1)) @@ -471,48 +397,58 @@ class DiskObj(): def safety_check(self, silent=False): """Run safety checks and disable tests if necessary.""" + test_running = False if self.nvme_attributes or self.smart_attributes: - self.check_attributes(silent) + disk_ok = self.check_attributes() + test_running = self.check_smart_self_test(silent) - # Check if a self-test is currently running - if 'remaining_percent' in self.smart_self_test.get('status', ''): - _msg = 'SMART self-test in progress, all tests disabled' - - # Ask to abort - if not silent: - print_warning('WARNING: {}'.format(_msg)) - print_standard(' ') - if ask('Abort HW Diagnostics?'): - exit_script() - - # Add warning to report - if 'NVMe / SMART' in self.tests: - self.tests['NVMe / SMART'].report = self.generate_attribute_report() - self.tests['NVMe / SMART'].report.append( - '{YELLOW}WARNING: {msg}{CLEAR}'.format(msg=_msg, **COLORS)) - - # Disable all tests for this disk - for t in self.tests.keys(): - self.disable_test(t, 'Denied') + # Show errors (unless a SMART self-test is running) + if not (silent or test_running): + if disk_ok: + # 199/C7 warning + if self.smart_attributes.get(199, {}).get('raw', 0) > 0: + print_warning('199/C7 error detected') + print_standard(' (Have you tried swapping the disk cable?)') + else: + # Override? + show_report( + self.generate_attribute_report(description=True), + log_report=True) + print_warning(' {} error(s) detected.'.format(self.attr_type)) + if self.override_disabled: + print_standard('Tests disabled for this device') + pause() + elif not (len(self.tests) == 3 and OVERRIDES_LIMITED): + if OVERRIDES_FORCED or ask('Run tests on this device anyway?'): + disk_ok = True + if 'NVMe / SMART' in self.tests: + self.disable_test('NVMe / SMART', 'OVERRIDE') + if not self.nvme_attributes and self.smart_attributes: + # Re-enable for SMART short-tests + self.tests['NVMe / SMART'].disabled = False + print_standard(' ') else: # No NVMe/SMART details self.disable_test('NVMe / SMART', 'N/A') if silent: - self.disk_ok = HW_OVERRIDES_FORCED + disk_ok = OVERRIDES_FORCED else: - print_info('Device ({})'.format(self.name)) - print_standard(' {}'.format(self.description)) - print_warning(' No NVMe or SMART data available') - self.disk_ok = HW_OVERRIDES_FORCED or ask( - 'Run tests on this device anyway?') + show_report( + self.generate_attribute_report(description=True), + log_report=True) + disk_ok = OVERRIDES_FORCED or ask('Run tests on this device anyway?') print_standard(' ') - if not self.disk_ok: - if 'NVMe / SMART' in self.tests: - # NOTE: This will not overwrite the existing status if set - self.disable_test('NVMe / SMART', 'NS') - if not self.tests['NVMe / SMART'].report: - self.tests['NVMe / SMART'].report = self.generate_attribute_report() + + # Disable tests if necessary (statuses won't be overwritten) + if test_running: + if not silent: + # silent is only True in quick_mode + self.disable_test('NVMe / SMART', 'Denied') + for t in ['badblocks', 'I/O Benchmark']: + self.disable_test(t, 'Denied') + elif not disk_ok: + self.disable_test('NVMe / SMART', 'NS') for t in ['badblocks', 'I/O Benchmark']: self.disable_test(t, 'Denied') @@ -569,9 +505,8 @@ class State(): # Add block devices cmd = ['lsblk', '--json', '--nodeps', '--paths'] - result = run_program(cmd, check=False) - json_data = json.loads(result.stdout.decode()) - for disk in json_data['blockdevices']: + json_data = get_json_from_command(cmd) + for disk in json_data.get('blockdevices', []): skip_disk = False disk_obj = DiskObj(disk['name']) @@ -909,6 +844,8 @@ def run_audio_test(): def run_badblocks_test(state, test): """Run a read-only surface scan with badblocks.""" + dev = test.dev + # Bail early if test.disabled: return @@ -926,7 +863,7 @@ def run_badblocks_test(state, test): break # Prep - print_log('Starting badblocks test for {}'.format(test.dev.path)) + print_log('Starting badblocks test for {}'.format(dev.path)) test.started = True test.update_status() update_progress_pane(state) @@ -935,23 +872,30 @@ def run_badblocks_test(state, test): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, test.dev.description)) + TOP_PANE_TEXT, dev.description)) # Create monitor pane test.badblocks_out = '{}/badblocks_{}.out'.format( - global_vars['LogDir'], test.dev.name) + global_vars['LogDir'], dev.name) state.panes['badblocks'] = tmux_split_window( lines=5, vertical=True, watch=test.badblocks_out, watch_cmd='tail') # Show disk details clear_screen() - show_report(test.dev.generate_attribute_report()) + show_report(dev.generate_attribute_report()) print_standard(' ') + # Set read block size + if dev.lsblk['phy-sec'] == 4096 or dev.size_bytes >= BADBLOCKS_LARGE_DISK: + block_size = '4096' + else: + # Use default value + block_size = '1024' + # Start badblocks print_standard('Running badblocks test...') test.badblocks_proc = popen_program( - ['sudo', 'badblocks', '-sv', '-e', '1', test.dev.path], + ['sudo', 'badblocks', '-sv', '-b', block_size, '-e', '1', dev.path], pipe=True, bufsize=1) test.badblocks_nbsr = NonBlockingStreamReader(test.badblocks_proc.stderr) test.badblocks_stderr = '' @@ -981,7 +925,8 @@ def run_badblocks_test(state, test): else: test.report.append(' {YELLOW}{line}{CLEAR}'.format( line=line, **COLORS)) - test.failed = True + if not test.aborted: + test.failed = True if test.aborted: test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) test.update_status('Aborted') @@ -989,7 +934,7 @@ def run_badblocks_test(state, test): # Disable other drive tests if necessary if not test.passed: - test.dev.disable_test('I/O Benchmark', 'Denied') + dev.disable_test('I/O Benchmark', 'Denied') # Update status if test.failed: @@ -1044,7 +989,12 @@ def run_hw_tests(state): _disk_tests_enabled |= state.tests[k]['Enabled'] if _disk_tests_enabled: for disk in state.disks: - disk.safety_check(silent=state.quick_mode) + try: + disk.safety_check(silent=state.quick_mode) + except GenericAbort: + tmux_kill_pane(*state.panes.values()) + state.panes.clear() + return # Run tests ## Because state.tests is an OrderedDict and the disks were added @@ -1059,6 +1009,13 @@ def run_hw_tests(state): # No devices available v['Objects'].append(TestObj(dev=None, label='')) v['Objects'][-1].update_status('N/A') + # Recheck attributes + if state.tests['NVMe / SMART']['Enabled']: + for test_obj in state.tests['NVMe / SMART']['Objects']: + if test_obj.dev is not None: + # dev == None for the 'N/A' lines set above + run_nvme_smart_tests(state, test_obj, update_mode=True) + except GenericAbort: # Cleanup tmux_kill_pane(*state.panes.values()) @@ -1094,12 +1051,14 @@ def run_hw_tests(state): def run_io_benchmark(state, test): """Run a read-only I/O benchmark using dd.""" + dev = test.dev + # Bail early if test.disabled: return # Prep - print_log('Starting I/O benchmark test for {}'.format(test.dev.path)) + print_log('Starting I/O benchmark test for {}'.format(dev.path)) test.started = True test.update_status() update_progress_pane(state) @@ -1108,12 +1067,12 @@ def run_io_benchmark(state, test): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, test.dev.description)) + TOP_PANE_TEXT, dev.description)) state.tmux_layout['Current'] = {'y': 15, 'Check': True} # Create monitor pane test.io_benchmark_out = '{}/io_benchmark_{}.out'.format( - global_vars['LogDir'], test.dev.name) + global_vars['LogDir'], dev.name) state.panes['io_benchmark'] = tmux_split_window( percent=75, vertical=True, watch=test.io_benchmark_out, watch_cmd='tail') @@ -1121,7 +1080,7 @@ def run_io_benchmark(state, test): # Show disk details clear_screen() - show_report(test.dev.generate_attribute_report()) + show_report(dev.generate_attribute_report()) print_standard(' ') # Start I/O Benchmark @@ -1129,23 +1088,23 @@ def run_io_benchmark(state, test): try: test.merged_rates = [] test.read_rates = [] - test.dev.calc_io_dd_values() + dev.calc_io_dd_values() # Run dd read tests offset = 0 - for i in range(test.dev.dd_chunks): + for i in range(dev.dd_chunks): # Build cmd i += 1 - skip = test.dev.dd_skip_count - if test.dev.dd_skip_extra and i % test.dev.dd_skip_extra == 0: + skip = dev.dd_skip_count + if dev.dd_skip_extra and i % dev.dd_skip_extra == 0: skip += 1 cmd = [ 'sudo', 'dd', 'bs={}'.format(IO_VARS['Block Size']), 'skip={}'.format(offset+skip), - 'count={}'.format(test.dev.dd_chunk_blocks), + 'count={}'.format(dev.dd_chunk_blocks), 'iflag=direct', - 'if={}'.format(test.dev.path), + 'if={}'.format(dev.path), 'of=/dev/null'] # Run cmd and get read rate @@ -1159,12 +1118,12 @@ def run_io_benchmark(state, test): # Show progress if i % IO_VARS['Progress Refresh Rate'] == 0: update_io_progress( - percent=(i/test.dev.dd_chunks)*100, + percent=(i/dev.dd_chunks)*100, rate=cur_rate, progress_file=test.io_benchmark_out) # Update offset - offset += test.dev.dd_chunk_blocks + skip + offset += dev.dd_chunk_blocks + skip except DeviceTooSmallError: # Device too small, skipping test @@ -1190,7 +1149,7 @@ def run_io_benchmark(state, test): else: # Merge rates for horizontal graph offset = 0 - width = int(test.dev.dd_chunks / IO_VARS['Graph Horizontal Width']) + width = int(dev.dd_chunks / IO_VARS['Graph Horizontal Width']) for i in range(IO_VARS['Graph Horizontal Width']): test.merged_rates.append( sum(test.read_rates[offset:offset+width])/width) @@ -1211,7 +1170,7 @@ def run_io_benchmark(state, test): test.report.append(avg_min_max) # Compare read speeds to thresholds - if test.dev.lsblk['rota']: + if dev.lsblk['rota']: # Use HDD scale thresh_min = IO_VARS['Threshold HDD Min'] thresh_high_avg = IO_VARS['Threshold HDD High Avg'] @@ -1252,6 +1211,8 @@ def run_keyboard_test(): def run_mprime_test(state, test): """Test CPU with Prime95 and track temps.""" + dev = test.dev + # Bail early if test.disabled: return @@ -1262,11 +1223,12 @@ def run_mprime_test(state, test): test.update_status() update_progress_pane(state) test.sensor_data = get_sensor_data() + test.thermal_abort = False # Update tmux layout tmux_update_pane( state.panes['Top'], - text='{}\n{}'.format(TOP_PANE_TEXT, test.dev.name)) + text='{}\n{}'.format(TOP_PANE_TEXT, dev.name)) # Start live sensor monitor test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir']) @@ -1297,12 +1259,12 @@ def run_mprime_test(state, test): # Stress CPU print_log('Starting Prime95') test.abort_msg = 'If running too hot, press CTRL+c to abort the test' - run_program(['apple-fans', 'max']) + run_program(['apple-fans', 'max'], check=False) tmux_update_pane( state.panes['Prime95'], command=['hw-diags-prime95', global_vars['TmpDir']], working_dir=global_vars['TmpDir']) - time_limit = int(MPRIME_LIMIT) * 60 + time_limit = MPRIME_LIMIT * 60 try: for i in range(time_limit): clear_screen() @@ -1319,15 +1281,19 @@ def run_mprime_test(state, test): # Not using print wrappers to avoid flooding the log print(_status_str) print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS)) - update_sensor_data(test.sensor_data) + update_sensor_data(test.sensor_data, THERMAL_LIMIT) # Wait sleep(1) - except KeyboardInterrupt: - # Catch CTRL+C + except (KeyboardInterrupt, ThermalLimitReachedError) as err: + # CTRL+c pressed or thermal limit reached test.aborted = True - test.update_status('Aborted') - print_warning('\nAborted.') + if isinstance(err, KeyboardInterrupt): + test.update_status('Aborted') + elif isinstance(err, ThermalLimitReachedError): + test.failed = True + test.thermal_abort = True + test.update_status('NS') update_progress_pane(state) # Restart live monitor @@ -1341,7 +1307,7 @@ def run_mprime_test(state, test): tmux_kill_pane(state.panes.pop('Prime95', None)) # Get cooldown temp - run_program(['apple-fans', 'auto']) + run_program(['apple-fans', 'auto'], check=False) clear_screen() try_and_print( message='Letting CPU cooldown for bit...', indent=0, @@ -1428,6 +1394,16 @@ def run_mprime_test(state, test): test.sensor_data, 'Idle', 'Max', 'Cooldown', core_only=True): test.report.append(' {}'.format(line)) + # Add abort message(s) + if test.aborted: + test.report.append( + ' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + if test.thermal_abort: + test.report.append( + ' {RED}CPU reached temperature limit of {temp}°C{CLEAR}'.format( + temp=THERMAL_LIMIT, + **COLORS)) + # Done update_progress_pane(state) @@ -1447,15 +1423,19 @@ def run_network_test(): pause('Press Enter to return to main menu... ') -def run_nvme_smart_tests(state, test): - """Run NVMe or SMART test for test.dev.""" +def run_nvme_smart_tests(state, test, update_mode=False): + """Run NVMe or SMART test for test.dev. + + Update mode is used to refresh the attributes and recheck them. + (i.e. no self-test and don't disable other tests)""" + dev = test.dev + # Bail early if test.disabled: return # Prep - print_log('Starting NVMe/SMART test for {}'.format(test.dev.path)) - _include_short_test = False + print_log('Starting NVMe/SMART test for {}'.format(dev.path)) test.started = True test.update_status() update_progress_pane(state) @@ -1464,122 +1444,123 @@ def run_nvme_smart_tests(state, test): tmux_update_pane( state.panes['Top'], text='{}\n{}'.format( - TOP_PANE_TEXT, test.dev.description)) + TOP_PANE_TEXT, dev.description)) - # NVMe - if test.dev.nvme_attributes: - # NOTE: Pass/Fail is just the attribute check - if test.dev.disk_ok: + # SMART short self-test + if dev.smart_attributes and not (state.quick_mode or update_mode): + run_smart_short_test(state, test) + + # Attribute check + dev.check_attributes() + + # Check results + if dev.nvme_attributes or state.quick_mode: + if dev.disk_ok: test.passed = True test.update_status('CS') else: - # NOTE: Other test(s) should've been disabled by DiskObj.safety_check() test.failed = True test.update_status('NS') - - # SMART - elif test.dev.smart_attributes: - # NOTE: Pass/Fail based on both attributes and SMART short self-test - if not (test.dev.disk_ok or 'OVERRIDE' in test.status): + elif dev.smart_attributes: + if dev.disk_ok and dev.self_test_passed and 'OVERRIDE' not in test.status: + test.passed = True + test.update_status('CS') + elif test.aborted: + test.update_status('Aborted') + raise GenericAbort('Aborted') + elif dev.self_test_timed_out: + test.failed = True + test.update_status('TimedOut') + elif dev.override_disabled or 'OVERRIDE' not in test.status: + # override_disabled is set to True if one or more critical attributes + # have exceeded the Error threshold. This overrules an override. test.failed = True test.update_status('NS') - elif state.quick_mode: - if test.dev.disk_ok: - test.passed = True - test.update_status('CS') - else: - test.failed = True - test.update_status('NS') - else: - # Prep - test.timeout = test.dev.smart_self_test['polling_minutes'].get( - 'short', 5) - test.timeout = int(test.timeout) + 5 - _include_short_test = True - _self_test_started = False - _self_test_finished = False + else: + # This dev lacks both NVMe and SMART data. This test should've been + # disabled during the safety_check(). + pass - # Create monitor pane - test.smart_out = '{}/smart_{}.out'.format( - global_vars['LogDir'], test.dev.name) - with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n Starting...') - state.panes['SMART'] = tmux_split_window( - lines=3, vertical=True, watch=test.smart_out) - - # Show attributes - clear_screen() - show_report(test.dev.generate_attribute_report()) - print_standard(' ') - - # Start short test - print_standard('Running self-test...') - cmd = ['sudo', 'smartctl', '--test=short', test.dev.path] - run_program(cmd, check=False) - - # Monitor progress - try: - for i in range(int(test.timeout*60/5)): - sleep(5) - - # Update SMART data - test.dev.get_smart_details() - - if _self_test_started: - # Update progress file - with open(test.smart_out, 'w') as f: - f.write('SMART self-test status:\n {}'.format( - test.dev.smart_self_test['status'].get( - 'string', 'UNKNOWN').capitalize())) - - # Check if test has finished - if 'remaining_percent' not in test.dev.smart_self_test['status']: - _self_test_finished = True - break - - else: - # Check if test has started - if 'remaining_percent' in test.dev.smart_self_test['status']: - _self_test_started = True - - except KeyboardInterrupt: - test.aborted = True - test.report = test.dev.generate_attribute_report() - test.report.append('{BLUE}SMART Short self-test{CLEAR}'.format( - **COLORS)) - test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) - test.update_status('Aborted') - raise GenericAbort('Aborted') - - # Check if timed out - if _self_test_finished: - if test.dev.smart_self_test['status'].get('passed', False): - if 'OVERRIDE' not in test.status: - test.passed = True - test.update_status('CS') - else: - test.failed = True - test.update_status('NS') - else: - test.dev.smart_timeout = True - test.update_status('TimedOut') - - # Disable other drive tests if necessary - if test.failed: - for t in ['badblocks', 'I/O Benchmark']: - test.dev.disable_test(t, 'Denied') - - # Cleanup - tmux_kill_pane(state.panes.pop('SMART', None)) - - # Save report - test.report = test.dev.generate_attribute_report( - short_test=_include_short_test) + # Disable other disk tests if necessary + if test.failed and not update_mode: + for t in ['badblocks', 'I/O Benchmark']: + dev.disable_test(t, 'Denied') # Done update_progress_pane(state) +def run_smart_short_test(state, test): + """Run SMART short self-test for test.dev.""" + dev = test.dev + dev.self_test_started = False + dev.self_test_finished = False + dev.self_test_passed = False + dev.self_test_timed_out = False + test.timeout = dev.smart_self_test['polling_minutes'].get('short', 5) + test.timeout = int(test.timeout) + 5 + + # Create monitor pane + test.smart_out = '{}/smart_{}.out'.format(global_vars['LogDir'], dev.name) + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n Starting...') + state.panes['SMART'] = tmux_split_window( + lines=3, vertical=True, watch=test.smart_out) + + # Show attributes + clear_screen() + show_report(dev.generate_attribute_report()) + print_standard(' ') + + # Start short test + print_standard('Running self-test...') + cmd = ['sudo', 'smartctl', '--test=short', dev.path] + run_program(cmd, check=False) + + # Monitor progress + try: + for i in range(int(test.timeout*60/5)): + sleep(5) + + # Update SMART data + dev.get_smart_details() + + if dev.self_test_started: + # Update progress file + with open(test.smart_out, 'w') as f: + f.write('SMART self-test status:\n {}'.format( + dev.smart_self_test['status'].get( + 'string', 'UNKNOWN').capitalize())) + + # Check if test has finished + if 'remaining_percent' not in dev.smart_self_test['status']: + dev.self_test_finished = True + break + + else: + # Check if test has started + if 'remaining_percent' in dev.smart_self_test['status']: + dev.self_test_started = True + except KeyboardInterrupt: + # Will be handled in run_nvme_smart_tests() + test.aborted = True + + # Save report + test.report.append('{BLUE}SMART Short self-test{CLEAR}'.format(**COLORS)) + test.report.append(' {}'.format( + dev.smart_self_test['status'].get('string', 'UNKNOWN').capitalize())) + if dev.self_test_finished: + dev.self_test_passed = dev.smart_self_test['status'].get('passed', False) + elif test.aborted: + test.report.append(' {YELLOW}Aborted{CLEAR}'.format(**COLORS)) + else: + dev.self_test_timed_out = True + test.report.append(' {YELLOW}Timed out{CLEAR}'.format(**COLORS)) + + # Cleanup + tmux_kill_pane(state.panes.pop('SMART', None)) + + def secret_screensaver(screensaver=None): """Show screensaver.""" if screensaver == 'matrix': diff --git a/.bin/Scripts/functions/info.py b/.bin/Scripts/functions/info.py index ad71bce7..84d92663 100644 --- a/.bin/Scripts/functions/info.py +++ b/.bin/Scripts/functions/info.py @@ -3,57 +3,7 @@ from borrowed import knownpaths from functions.activation import * from operator import itemgetter - - -# STATIC VARIABLES -REG_PROFILE_LIST = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList' -REG_SHELL_FOLDERS = r'Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders' -TMP_HIVE_PATH = 'TEMP_HIVE_MOUNT' -EXTRA_FOLDERS = [ - 'Dropbox', - 'Google Drive', - 'OneDrive', - 'SkyDrive', -] -SHELL_FOLDERS = { - #GUIDs from: https://msdn.microsoft.com/en-us/library/windows/desktop/dd378457(v=vs.85).aspx - 'Desktop': ( - '{B4BFCC3A-DB2C-424C-B029-7FE99A87C641}', - ), - 'Documents': ( - 'Personal', - '{FDD39AD0-238F-46AF-ADB4-6C85480369C7}', - ), - 'Downloads': ( - '{374DE290-123F-4565-9164-39C4925E467B}', - ), - 'Favorites': ( - '{1777F761-68AD-4D8A-87BD-30B759FA33DD}', - ), - 'Music': ( - 'My Music', - '{4BD8D571-6D19-48D3-BE97-422220080E43}', - ), - 'Pictures': ( - 'My Pictures', - '{33E28130-4E1E-4676-835A-98395C3BC3BB}', - ), - 'Videos': ( - 'My Video', - '{18989B1D-99B5-455B-841C-AB7C74E4DDFC}', - ), -} - - -# Regex -REGEX_OFFICE = re.compile( - r'(Microsoft (Office\s+' - r'(365|Enterprise|Home|Pro(\s|fessional)' - r'|Single|Small|Standard|Starter|Ultimate|system)' - r'|Works[-\s\d]+\d)' - r'|(Libre|Open|Star)\s*Office' - r'|WordPerfect|Gnumeric|Abiword)', - re.IGNORECASE) +from settings.info import * def backup_file_list(): diff --git a/.bin/Scripts/functions/json.py b/.bin/Scripts/functions/json.py new file mode 100644 index 00000000..b4527c0d --- /dev/null +++ b/.bin/Scripts/functions/json.py @@ -0,0 +1,32 @@ +# Wizard Kit: Functions - JSON + +import json + +from functions.common import * + +def get_json_from_command(cmd, ignore_errors=True): + """Capture JSON content from cmd output, returns dict. + + If the data can't be decoded then either an exception is raised + or an empty dict is returned depending on ignore_errors. + """ + errors = 'strict' + json_data = {} + + if ignore_errors: + errors = 'ignore' + + try: + result = run_program(cmd, encoding='utf-8', errors=errors) + json_data = json.loads(result.stdout) + except (subprocess.CalledProcessError, json.decoder.JSONDecodeError): + if not ignore_errors: + raise + + return json_data + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/functions/sensors.py b/.bin/Scripts/functions/sensors.py index 5d2f2fae..993306bd 100644 --- a/.bin/Scripts/functions/sensors.py +++ b/.bin/Scripts/functions/sensors.py @@ -4,19 +4,12 @@ import json import re from functions.tmux import * +from settings.sensors import * -# STATIC VARIABLES -TEMP_LIMITS = { - 'GREEN': 60, - 'YELLOW': 70, - 'ORANGE': 80, - 'RED': 90, - } - - -# REGEX -REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') +# Error Classes +class ThermalLimitReachedError(Exception): + pass def clear_temps(sensor_data): @@ -112,7 +105,7 @@ def get_raw_sensor_data(): # Get raw data try: result = run_program(cmd) - result = result.stdout.decode().splitlines() + result = result.stdout.decode('utf-8', errors='ignore').splitlines() except subprocess.CalledProcessError: # Assuming no sensors available, set to empty list result = [] @@ -214,7 +207,7 @@ def save_average_temp(sensor_data, temp_label, seconds=10): _data[temp_label] = sum(_data['Temps']) / len(_data['Temps']) -def update_sensor_data(sensor_data): +def update_sensor_data(sensor_data, thermal_limit=None): """Read sensors and update existing sensor_data, returns dict.""" json_data = get_raw_sensor_data() for _section, _adapters in sensor_data.items(): @@ -230,6 +223,11 @@ def update_sensor_data(sensor_data): # Dumb workound for Dell sensors with changing source names pass + # Check if thermal limit reached + if thermal_limit and _section == 'CoreTemps': + if max(_data['Current'], _data['Max']) >= thermal_limit: + raise ThermalLimitReachedError('CoreTemps reached limit') + def join_columns(column1, column2, width=55): return '{:<{}}{}'.format( diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index f51bf40b..dc6c53a4 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -1,134 +1,10 @@ # Wizard Kit: Functions - Setup from functions.update import * +from settings.setup import * -# STATIC VARIABLES -HKU = winreg.HKEY_USERS -HKCR = winreg.HKEY_CLASSES_ROOT -HKCU = winreg.HKEY_CURRENT_USER -HKLM = winreg.HKEY_LOCAL_MACHINE -MOZILLA_FIREFOX_UBO_PATH = r'{}\{}\ublock_origin.xpi'.format( - os.environ.get('PROGRAMFILES'), - r'Mozilla Firefox\distribution\extensions') -OTHER_RESULTS = { - 'Error': { - 'CalledProcessError': 'Unknown Error', - 'FileNotFoundError': 'File not found', - }, - 'Warning': {}} -SETTINGS_CLASSIC_START = { - r'Software\IvoSoft\ClassicShell\Settings': {}, - r'Software\IvoSoft\ClassicStartMenu': { - 'DWORD Items': {'ShowedStyle2': 1}, - }, - r'Software\IvoSoft\ClassicStartMenu\MRU': {}, - r'Software\IvoSoft\ClassicStartMenu\Settings': { - 'DWORD Items': {'SkipMetro': 1}, - 'SZ Items': { - 'MenuStyle': 'Win7', - 'RecentPrograms': 'Recent', - }, - }, - } -SETTINGS_EXPLORER_SYSTEM = { - # Disable Location Tracking - r'Software\Microsoft\Windows NT\CurrentVersion\Sensor\Overrides\{BFA794E4-F964-4FDB-90F6-51056BFE4B44}': { - 'DWORD Items': {'SensorPermissionState': 0}, - }, - r'System\CurrentControlSet\Services\lfsvc\Service\Configuration': { - 'Status': {'Value': 0}, - }, - # Disable Telemetry - r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { - 'DWORD Items': {'AllowTelemetry': 0}, - }, - r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { - 'DWORD Items': {'AllowTelemetry': 0}, - 'WOW64_32': True, - }, - r'Software\Policies\Microsoft\Windows\DataCollection': { - 'DWORD Items': {'AllowTelemetry': 0}, - }, - # Disable Wi-Fi Sense - r'Software\Microsoft\PolicyManager\default\WiFi\AllowWiFiHotSpotReporting': { - 'DWORD Items': {'Value': 0}, - }, - r'Software\Microsoft\PolicyManager\default\WiFi\AllowAutoConnectToWiFiSenseHotspots': { - 'DWORD Items': {'Value': 0}, - }, - } -SETTINGS_EXPLORER_USER = { - # Disable silently installed apps - r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { - 'DWORD Items': {'SilentInstalledAppsEnabled': 0}, - }, - # Disable Tips and Tricks - r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { - 'DWORD Items': {'SoftLandingEnabled ': 0}, - }, - # Hide People bar - r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced\People': { - 'DWORD Items': {'PeopleBand': 0}, - }, - # Hide Search button / box - r'Software\Microsoft\Windows\CurrentVersion\Search': { - 'DWORD Items': {'SearchboxTaskbarMode': 0}, - }, - # Change default Explorer view to "Computer" - r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced': { - 'DWORD Items': {'LaunchTo': 1}, - }, - } -SETTINGS_GOOGLE_CHROME = { - r'Software\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm': { - 'SZ Items': { - 'update_url': 'https://clients2.google.com/service/update2/crx'}, - 'WOW64_32': True, - }, - r'Software\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco': { - 'SZ Items': { - 'update_url': 'https://clients2.google.com/service/update2/crx'}, - 'WOW64_32': True, - }, - } -SETTINGS_MOZILLA_FIREFOX_32 = { - r'Software\Mozilla\Firefox\Extensions': { - 'SZ Items': { - 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, - 'WOW64_32': True, - }, - } -SETTINGS_MOZILLA_FIREFOX_64 = { - r'Software\Mozilla\Firefox\Extensions': { - 'SZ Items': { - 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, - }, - } -VCR_REDISTS = [ - {'Name': 'Visual C++ 2010 x32...', - 'Cmd': [r'2010sp1\x32\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2010 x64...', - 'Cmd': [r'2010sp1\x64\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2012 Update 4 x32...', - 'Cmd': [r'2012u4\x32\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2012 Update 4 x64...', - 'Cmd': [r'2012u4\x64\vcredist.exe', '/passive', '/norestart']}, - {'Name': 'Visual C++ 2013 x32...', - 'Cmd': [r'2013\x32\vcredist.exe', '/install', - '/passive', '/norestart']}, - {'Name': 'Visual C++ 2013 x64...', - 'Cmd': [r'2013\x64\vcredist.exe', '/install', - '/passive', '/norestart']}, - {'Name': 'Visual C++ 2017 x32...', - 'Cmd': [r'2017\x32\vcredist.exe', '/install', - '/passive', '/norestart']}, - {'Name': 'Visual C++ 2017 x64...', - 'Cmd': [r'2017\x64\vcredist.exe', '/install', - '/passive', '/norestart']}, - ] - - +# Configuration def config_classicstart(): """Configure ClassicStart.""" # User level, not system level @@ -192,6 +68,11 @@ def config_explorer_user(): write_registry_settings(SETTINGS_EXPLORER_USER, all_users=False) +def config_windows_updates(): + """Configure Windows updates.""" + write_registry_settings(SETTINGS_WINDOWS_UPDATES, all_users=True) + + def disable_windows_telemetry(): """Disable Windows 10 telemetry settings with O&O ShutUp10.""" extract_item('ShutUp10', silent=True) @@ -345,6 +226,13 @@ def open_windows_updates(): popen_program(['control', '/name', 'Microsoft.WindowsUpdate']) +def restart_explorer(): + """Restart Explorer.""" + kill_process('explorer.exe') + sleep(2) + kill_process('explorer.exe') + + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/sw_diags.py b/.bin/Scripts/functions/sw_diags.py index 100e5ad5..1b965766 100644 --- a/.bin/Scripts/functions/sw_diags.py +++ b/.bin/Scripts/functions/sw_diags.py @@ -1,31 +1,9 @@ -# Wizard Kit: Functions - Diagnostics +# Wizard Kit: Functions - SW Diagnostics import ctypes from functions.common import * - - -# STATIC VARIABLES -AUTORUNS_SETTINGS = { - r'Software\Sysinternals\AutoRuns': { - 'checkvirustotal': 1, - 'EulaAccepted': 1, - 'shownomicrosoft': 1, - 'shownowindows': 1, - 'showonlyvirustotal': 1, - 'submitvirustotal': 0, - 'verifysignatures': 1, - }, - r'Software\Sysinternals\AutoRuns\SigCheck': { - 'EulaAccepted': 1, - }, - r'Software\Sysinternals\AutoRuns\Streams': { - 'EulaAccepted': 1, - }, - r'Software\Sysinternals\AutoRuns\VirusTotal': { - 'VirusTotalTermsAccepted': 1, - }, - } +from settings.sw_diags import * def check_connection(): diff --git a/.bin/Scripts/functions/tmux.py b/.bin/Scripts/functions/tmux.py index e2d1b333..81522268 100644 --- a/.bin/Scripts/functions/tmux.py +++ b/.bin/Scripts/functions/tmux.py @@ -10,6 +10,21 @@ def create_file(filepath): f.write('') +def tmux_capture_pane(pane_id=None): + """Capture text from target, or current, pane, returns str.""" + cmd = ['tmux', 'capture-pane', '-p'] + if pane_id: + cmd.extend(['-t', pane_id]) + text = '' + + # Capture + result = run_program(cmd, check=False, encoding='utf-8', errors='ignore') + text = result.stdout + + # Done + return str(text) + + def tmux_get_pane_size(pane_id=None): """Get target, or current, pane size, returns tuple.""" x = -1 diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py old mode 100644 new mode 100755 index adb26c6d..40bc3a27 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -67,6 +67,22 @@ def download_to_temp(out_name, source_url): download_generic(global_vars['TmpDir'], out_name, source_url) +def download_windows_updates(): + """Download stand alone Windows Update installers.""" + # Prep + dest = r'{}\Installers\Extras\Windows Updates'.format( + global_vars['BaseDir']) + + + # Download + for kb, v in WINDOWS_UPDATE_SOURCES.items(): + for winver, v2 in v.items(): + for arch, url in v2.items(): + name = 'KB{}-Windows{}-x{}.msu'.format(kb, winver, arch) + if not os.path.exists(r'{}\{}'.format(dest, name)): + download_generic(dest, name, url) + + def extract_generic(source, dest, mode='x', sz_args=[]): """Extract a file to a destination.""" cmd = [ @@ -827,8 +843,13 @@ def update_wiztree(): # Extract files extract_temp_to_cbin('wiztree.zip', 'WizTree') + ## NOTE: It's double-zipped for some reason? + extract_generic( + r'{}\WizTree\wiztree'.format(global_vars['CBinDir']), + r'{}\WizTree'.format(global_vars['CBinDir'])) # Cleanup + remove_item(r'{}\WizTree\wiztree'.format(global_vars['CBinDir'])) remove_from_temp('wiztree.zip') diff --git a/.bin/Scripts/functions/windows_setup.py b/.bin/Scripts/functions/windows_setup.py index e790a4b6..eabaac40 100644 --- a/.bin/Scripts/functions/windows_setup.py +++ b/.bin/Scripts/functions/windows_setup.py @@ -2,39 +2,7 @@ from functions.data import * from functions.disk import * - - -# STATIC VARIABLES -WINDOWS_VERSIONS = [ - {'Name': 'Windows 7 Home Basic', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 HOMEBASIC'}, - {'Name': 'Windows 7 Home Premium', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 HOMEPREMIUM'}, - {'Name': 'Windows 7 Professional', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 PROFESSIONAL'}, - {'Name': 'Windows 7 Ultimate', - 'Image File': 'Win7', - 'Image Name': 'Windows 7 ULTIMATE'}, - - {'Name': 'Windows 8.1', - 'Image File': 'Win8', - 'Image Name': 'Windows 8.1', - 'CRLF': True}, - {'Name': 'Windows 8.1 Pro', - 'Image File': 'Win8', - 'Image Name': 'Windows 8.1 Pro'}, - - {'Name': 'Windows 10 Home', - 'Image File': 'Win10', - 'Image Name': 'Windows 10 Home', - 'CRLF': True}, - {'Name': 'Windows 10 Pro', - 'Image File': 'Win10', - 'Image Name': 'Windows 10 Pro'}, - ] +from settings.windows_setup import * def find_windows_image(windows_version): diff --git a/.bin/Scripts/functions/winpe_menus.py b/.bin/Scripts/functions/winpe_menus.py index 22df7449..08622e4e 100644 --- a/.bin/Scripts/functions/winpe_menus.py +++ b/.bin/Scripts/functions/winpe_menus.py @@ -3,53 +3,7 @@ from functions.backup import * from functions.disk import * from functions.windows_setup import * - - -# STATIC VARIABLES -FAST_COPY_PE_ARGS = [ - '/cmd=noexist_only', - '/utf8', - '/skip_empty_dir', - '/linkdest', - '/no_ui', - '/auto_close', - '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), - ] -PE_TOOLS = { - 'BlueScreenView': { - 'Path': r'BlueScreenView\BlueScreenView.exe', - }, - 'FastCopy': { - 'Path': r'FastCopy\FastCopy.exe', - 'Args': FAST_COPY_PE_ARGS, - }, - 'HWiNFO': { - 'Path': r'HWiNFO\HWiNFO.exe', - }, - 'NT Password Editor': { - 'Path': r'NT Password Editor\ntpwedit.exe', - }, - 'Notepad++': { - 'Path': r'NotepadPlusPlus\NotepadPlusPlus.exe', - }, - 'PhotoRec': { - 'Path': r'TestDisk\photorec_win.exe', - 'Args': ['-new_console:n'], - }, - 'Prime95': { - 'Path': r'Prime95\prime95.exe', - }, - 'ProduKey': { - 'Path': r'ProduKey\ProduKey.exe', - }, - 'Q-Dir': { - 'Path': r'Q-Dir\Q-Dir.exe', - }, - 'TestDisk': { - 'Path': r'TestDisk\testdisk_win.exe', - 'Args': ['-new_console:n'], - }, - } +from settings.winpe import * def check_pe_tools(): diff --git a/.bin/Scripts/hw-diags-menu b/.bin/Scripts/hw-diags-menu index 1c3cfc42..fa7f0cd8 100755 --- a/.bin/Scripts/hw-diags-menu +++ b/.bin/Scripts/hw-diags-menu @@ -26,8 +26,37 @@ if __name__ == '__main__': # Normal exit pass except: + # Cleanup tmux_kill_all_panes() - major_exception() + + if DEBUG_MODE: + # Custom major exception + print_standard(' ') + print_error('Major exception') + print_warning(SUPPORT_MESSAGE) + print(traceback.format_exc()) + print_log(traceback.format_exc()) + + # Save debug reports and upload data + try_and_print( + message='Saving debug reports...', + function=save_debug_reports, + state=state, global_vars=global_vars) + question = 'Upload crash details to {}?'.format(CRASH_SERVER['Name']) + if ENABLED_UPLOAD_DATA and ask(question): + try_and_print( + message='Uploading Data...', + function=upload_logdir, + global_vars=global_vars) + + # Done + sleep(10) + pause('Press Enter to exit...') + exit_script(1) + + else: + # "Normal" major exception + major_exception() # Done tmux_kill_all_panes() diff --git a/.bin/Scripts/new_system_setup.py b/.bin/Scripts/new_system_setup.py index 6ab88973..170cf323 100644 --- a/.bin/Scripts/new_system_setup.py +++ b/.bin/Scripts/new_system_setup.py @@ -93,19 +93,25 @@ if __name__ == '__main__': if global_vars['OS']['Version'] == '10': try_and_print(message='ClassicStart...', function=config_classicstart, cs='Done') - try_and_print(message='Explorer...', + try_and_print(message='Explorer (user)...', function=config_explorer_user, cs='Done') # Configure system print_info('Configuring system') if global_vars['OS']['Version'] == '10': - try_and_print(message='Explorer...', + try_and_print(message='Explorer (system)...', function=config_explorer_system, cs='Done') try_and_print(message='Disabling telemetry...', function=disable_windows_telemetry, cs='Done') + try_and_print(message='Windows Updates...', + function=config_windows_updates, cs='Done') try_and_print(message='Updating Clock...', function=update_clock, cs='Done') + # Restart Explorer + try_and_print(message='Restarting Explorer...', + function=restart_explorer, cs='Done') + # Summary print_info('Summary') try_and_print(message='Operating System:', diff --git a/.bin/Scripts/settings/browsers.py b/.bin/Scripts/settings/browsers.py new file mode 100644 index 00000000..dc394342 --- /dev/null +++ b/.bin/Scripts/settings/browsers.py @@ -0,0 +1,99 @@ +# Wizard Kit: Settings - Browsers + +import os +import re + +# General +DEFAULT_HOMEPAGE = 'https://www.google.com/' +IE_GALLERY = 'https://www.microsoft.com/en-us/iegallery' +MOZILLA_PREFS = { + 'browser.search.defaultenginename': '"Google"', + 'browser.search.defaultenginename.US': '"Google"', + 'browser.search.geoSpecificDefaults': 'false', + 'browser.startup.homepage': '"{}"'.format(DEFAULT_HOMEPAGE), + 'extensions.ui.lastCategory': '"addons://list/extension"', + } +SUPPORTED_BROWSERS = { + 'Internet Explorer': { + 'base': 'ie', + 'exe_name': 'iexplore.exe', + 'rel_install_path': 'Internet Explorer', + 'user_data_path': r'{USERPROFILE}\Favorites', + }, + 'Google Chrome': { + 'base': 'chromium', + 'exe_name': 'chrome.exe', + 'rel_install_path': r'Google\Chrome\Application', + 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome\User Data', + }, + 'Google Chrome Canary': { + 'base': 'chromium', + 'exe_name': 'chrome.exe', + 'rel_install_path': r'Google\Chrome SxS\Application', + 'user_data_path': r'{LOCALAPPDATA}\Google\Chrome SxS\User Data', + }, + 'Mozilla Firefox': { + 'base': 'mozilla', + 'exe_name': 'firefox.exe', + 'rel_install_path': 'Mozilla Firefox', + 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', + }, + 'Mozilla Firefox Dev': { + 'base': 'mozilla', + 'exe_name': 'firefox.exe', + 'rel_install_path': 'Firefox Developer Edition', + 'user_data_path': r'{APPDATA}\Mozilla\Firefox\Profiles', + }, + 'Opera': { + 'base': 'chromium', + 'exe_name': 'launcher.exe', + 'rel_install_path': 'Opera', + 'user_data_path': r'{APPDATA}\Opera Software\Opera Stable', + }, + 'Opera Beta': { + 'base': 'chromium', + 'exe_name': 'launcher.exe', + 'rel_install_path': 'Opera beta', + 'user_data_path': r'{APPDATA}\Opera Software\Opera Next', + }, + 'Opera Dev': { + 'base': 'chromium', + 'exe_name': 'launcher.exe', + 'rel_install_path': 'Opera developer', + 'user_data_path': r'{APPDATA}\Opera Software\Opera Developer', + }, + } + +# Regex +REGEX_BACKUP = re.compile( + r'\.\w*bak.*', + re.IGNORECASE) +REGEX_CHROMIUM_PROFILE = re.compile( + r'^(Default|Profile)', + re.IGNORECASE) +REGEX_CHROMIUM_ITEMS = re.compile( + r'^(Bookmarks|Cookies|Favicons|Google Profile' + r'|History|Login Data|Top Sites|TransportSecurity' + r'|Visited Links|Web Data)', + re.IGNORECASE) +REGEX_MOZILLA = re.compile( + r'^(bookmarkbackups|(cookies|formhistory|places).sqlite' + r'|key3.db|logins.json|persdict.dat)$', + re.IGNORECASE) + +# uBlock Origin +UBO_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin/cjpalhdlnbpafiamejdnhcphjbkeiagm?hl=en' +UBO_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm' +UBO_EXTRA_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin-extra/pgdnlhfefecpicbbihgmbmffkjpaplco?hl=en' +UBO_EXTRA_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco' +UBO_MOZILLA = 'https://addons.mozilla.org/en-us/firefox/addon/ublock-origin/' +UBO_MOZZILA_PATH = r'{}\Mozilla Firefox\distribution\extensions\ublock_origin.xpi'.format(os.environ.get('PROGRAMFILES')) +UBO_MOZILLA_REG = r'Software\Mozilla\Firefox\Extensions' +UBO_MOZILLA_REG_NAME = 'uBlock0@raymondhill.net' +UBO_OPERA = 'https://addons.opera.com/en/extensions/details/ublock/?display=en' + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 tw=0 diff --git a/.bin/Scripts/settings/data.py b/.bin/Scripts/settings/data.py new file mode 100644 index 00000000..4dd29d03 --- /dev/null +++ b/.bin/Scripts/settings/data.py @@ -0,0 +1,105 @@ +# Wizard Kit: Settings - Data + +import ctypes +import re + +from settings.main import * + +# FastCopy +FAST_COPY_EXCLUDES = [ + r'\*.esd', + r'\*.swm', + r'\*.wim', + r'\*.dd', + r'\*.dd.tgz', + r'\*.dd.txz', + r'\*.map', + r'\*.dmg', + r'\*.image', + r'$RECYCLE.BIN', + r'$Recycle.Bin', + r'.AppleDB', + r'.AppleDesktop', + r'.AppleDouble', + r'.com.apple.timemachine.supported', + r'.dbfseventsd', + r'.DocumentRevisions-V100*', + r'.DS_Store', + r'.fseventsd', + r'.PKInstallSandboxManager', + r'.Spotlight*', + r'.SymAV*', + r'.symSchedScanLockxz', + r'.TemporaryItems', + r'.Trash*', + r'.vol', + r'.VolumeIcon.icns', + r'desktop.ini', + r'Desktop?DB', + r'Desktop?DF', + r'hiberfil.sys', + r'lost+found', + r'Network?Trash?Folder', + r'pagefile.sys', + r'Recycled', + r'RECYCLER', + r'System?Volume?Information', + r'Temporary?Items', + r'Thumbs.db', + ] +FAST_COPY_ARGS = [ + '/cmd=noexist_only', + '/utf8', + '/skip_empty_dir', + '/linkdest', + '/no_ui', + '/auto_close', + '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), + ] + +# Regex +REGEX_EXCL_ITEMS = re.compile( + r'^(\.(AppleDB|AppleDesktop|AppleDouble' + r'|com\.apple\.timemachine\.supported|dbfseventsd' + r'|DocumentRevisions-V100.*|DS_Store|fseventsd|PKInstallSandboxManager' + r'|Spotlight.*|SymAV.*|symSchedScanLockxz|TemporaryItems|Trash.*' + r'|vol|VolumeIcon\.icns)|desktop\.(ini|.*DB|.*DF)' + r'|(hiberfil|pagefile)\.sys|lost\+found|Network\.*Trash\.*Folder' + r'|Recycle[dr]|System\.*Volume\.*Information|Temporary\.*Items' + r'|Thumbs\.db)$', + re.IGNORECASE) +REGEX_EXCL_ROOT_ITEMS = re.compile( + r'^(boot(mgr|nxt)$|Config.msi' + r'|(eula|globdata|install|vc_?red)' + r'|.*.sys$|System Volume Information|RECYCLER?|\$Recycle\.bin' + r'|\$?Win(dows(.old.*|\. BT|)$|RE_)|\$GetCurrent|Windows10Upgrade' + r'|PerfLogs|Program Files|SYSTEM.SAV' + r'|.*\.(esd|swm|wim|dd|map|dmg|image)$)', + re.IGNORECASE) +REGEX_INCL_ROOT_ITEMS = re.compile( + r'^(AdwCleaner|(My\s*|)(Doc(uments?( and Settings|)|s?)|Downloads' + r'|Media|Music|Pic(ture|)s?|Vid(eo|)s?)' + r'|{prefix}(-?Info|-?Transfer|)' + r'|(ProgramData|Recovery|Temp.*|Users)$' + r'|.*\.(log|txt|rtf|qb\w*|avi|m4a|m4v|mp4|mkv|jpg|png|tiff?)$)' + r''.format(prefix=KIT_NAME_SHORT), + re.IGNORECASE) +REGEX_WIM_FILE = re.compile( + r'\.wim$', + re.IGNORECASE) +REGEX_WINDOWS_OLD = re.compile( + r'^Win(dows|)\.old', + re.IGNORECASE) + +# Thread error modes +## Code borrowed from: https://stackoverflow.com/a/29075319 +SEM_NORMAL = ctypes.c_uint() +SEM_FAILCRITICALERRORS = 1 +SEM_NOOPENFILEERRORBOX = 0x8000 +SEM_FAIL = SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/ddrescue.py b/.bin/Scripts/settings/ddrescue.py new file mode 100644 index 00000000..675019ca --- /dev/null +++ b/.bin/Scripts/settings/ddrescue.py @@ -0,0 +1,53 @@ +# Wizard Kit: Settings - ddrescue-tui + +import re + +from collections import OrderedDict + +# General +RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] +USAGE = """ {script_name} clone [source [destination]] + {script_name} image [source [destination]] + (e.g. {script_name} clone /dev/sda /dev/sdb) +""" + +# Layout +SIDE_PANE_WIDTH = 21 +TMUX_LAYOUT = OrderedDict({ + 'Source': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, +}) + +# ddrescue +AUTO_PASS_1_THRESHOLD = 95 +AUTO_PASS_2_THRESHOLD = 98 +DDRESCUE_SETTINGS = { + '--binary-prefixes': {'Enabled': True, 'Hidden': True, }, + '--data-preview': {'Enabled': True, 'Value': '5', 'Hidden': True, }, + '--idirect': {'Enabled': True, }, + '--odirect': {'Enabled': True, }, + '--max-read-rate': {'Enabled': False, 'Value': '1MiB', }, + '--min-read-rate': {'Enabled': True, 'Value': '64KiB', }, + '--reopen-on-error': {'Enabled': True, }, + '--retry-passes': {'Enabled': True, 'Value': '0', }, + '--test-mode': {'Enabled': False, 'Value': 'test.map', }, + '--timeout': {'Enabled': True, 'Value': '5m', }, + '-vvvv': {'Enabled': True, 'Hidden': True, }, + } +ETOC_REFRESH_RATE = 30 # in seconds +REGEX_REMAINING_TIME = re.compile( + r'remaining time:' + r'\s*((?P\d+)d)?' + r'\s*((?P\d+)h)?' + r'\s*((?P\d+)m)?' + r'\s*((?P\d+)s)?' + r'\s*(?Pn/a)?', + re.IGNORECASE + ) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/hw_diags.py b/.bin/Scripts/settings/hw_diags.py new file mode 100644 index 00000000..048f489b --- /dev/null +++ b/.bin/Scripts/settings/hw_diags.py @@ -0,0 +1,112 @@ +# Wizard Kit: Settings - HW Diagnostics + +from collections import OrderedDict + +# General +DEBUG_MODE = False +OVERRIDES_FORCED = False +OVERRIDES_LIMITED = True # If True this disables OVERRIDE_FORCED +STATUSES = { + 'RED': ['Denied', 'ERROR', 'NS', 'TimedOut'], + 'YELLOW': ['Aborted', 'N/A', 'OVERRIDE', 'Unknown', 'Working'], + 'GREEN': ['CS'], +} +TESTS_CPU = ['Prime95'] +TESTS_DISK = [ + 'I/O Benchmark', + 'NVMe / SMART', + 'badblocks', + ] + +# Layout +## NOTE: Colors will be applied in functions/hw_diags.py +QUICK_LABEL = '{YELLOW}(Quick){CLEAR}' +SIDE_PANE_WIDTH = 20 +TOP_PANE_TEXT = '{GREEN}Hardware Diagnostics{CLEAR}' +TMUX_LAYOUT = OrderedDict({ + 'Top': {'y': 2, 'Check': True}, + 'Started': {'x': SIDE_PANE_WIDTH, 'Check': True}, + 'Progress': {'x': SIDE_PANE_WIDTH, 'Check': True}, + # Testing panes + 'Prime95': {'y': 11, 'Check': False}, + 'Temps': {'y': 1000, 'Check': False}, + 'SMART': {'y': 3, 'Check': True}, + 'badblocks': {'y': 5, 'Check': True}, + 'I/O Benchmark': {'y': 1000, 'Check': False}, +}) + +# Tests: badblocks +## NOTE: Force 4K read block size for disks >= to 3TB +BADBLOCKS_LARGE_DISK = 3*1024**4 + +# Tests: I/O Benchmark +IO_VARS = { + 'Block Size': 512*1024, + 'Chunk Size': 32*1024**2, + 'Minimum Test Size': 10*1024**3, + 'Alt Test Size Factor': 0.01, + 'Progress Refresh Rate': 5, + 'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)], + 'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)], + 'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)], + 'Threshold Graph Fail': 65*1024**2, + 'Threshold Graph Warn': 135*1024**2, + 'Threshold Graph Great': 750*1024**2, + 'Threshold HDD Min': 50*1024**2, + 'Threshold HDD High Avg': 75*1024**2, + 'Threshold HDD Low Avg': 65*1024**2, + 'Threshold SSD Min': 90*1024**2, + 'Threshold SSD High Avg': 135*1024**2, + 'Threshold SSD Low Avg': 100*1024**2, + 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), + 'Graph Horizontal Width': 40, + 'Graph Vertical': ( + '▏', '▎', '▍', '▌', + '▋', '▊', '▉', '█', + '█▏', '█▎', '█▍', '█▌', + '█▋', '█▊', '█▉', '██', + '██▏', '██▎', '██▍', '██▌', + '██▋', '██▊', '██▉', '███', + '███▏', '███▎', '███▍', '███▌', + '███▋', '███▊', '███▉', '████'), + } + +# Tests: NVMe/SMART +ATTRIBUTES = { + 'NVMe': { + 'critical_warning': {'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'media_errors': {'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 'power_on_hours': {'Critical': False, 'Ignore': True, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 'unsafe_shutdowns': {'Critical': False, 'Ignore': True, 'Warning': 1, 'Error': None, 'Maximum': None, }, + }, + 'SMART': { + 5: {'Hex': '05', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 9: {'Hex': '09', 'Critical': False, 'Ignore': True, 'Warning': 17532, 'Error': 26298, 'Maximum': None, }, + 10: {'Hex': '10', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 184: {'Hex': 'B8', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 187: {'Hex': 'BB', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 188: {'Hex': 'BC', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 196: {'Hex': 'C4', 'Critical': False, 'Ignore': False, 'Warning': 1, 'Error': 10, 'Maximum': 10000, }, + 197: {'Hex': 'C5', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 198: {'Hex': 'C6', 'Critical': True, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 199: {'Hex': 'C7', 'Critical': False, 'Ignore': True, 'Warning': None, 'Error': 1, 'Maximum': None, }, + 201: {'Hex': 'C9', 'Critical': False, 'Ignore': False, 'Warning': None, 'Error': 1, 'Maximum': 10000, }, + }, + } +ATTRIBUTE_COLORS = ( + ('Error', 'RED'), + ('Maximum', 'PURPLE'), + ('Warning', 'YELLOW'), + ) +KEY_NVME = 'nvme_smart_health_information_log' +KEY_SMART = 'ata_smart_attributes' + +# Tests: Prime95 +MPRIME_LIMIT = 7 # of minutes to run Prime95 +THERMAL_LIMIT = 95 # Abort temperature in Celsius + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 tw=0 diff --git a/.bin/Scripts/settings/info.py b/.bin/Scripts/settings/info.py new file mode 100644 index 00000000..bb1f1421 --- /dev/null +++ b/.bin/Scripts/settings/info.py @@ -0,0 +1,58 @@ +# Wizard Kit: Settings - Information + +import re + +# General +REG_PROFILE_LIST = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList' +REG_SHELL_FOLDERS = r'Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders' +TMP_HIVE_PATH = 'TEMP_HIVE_MOUNT' +EXTRA_FOLDERS = [ + 'Dropbox', + 'Google Drive', + 'OneDrive', + 'SkyDrive', +] +SHELL_FOLDERS = { + #GUIDs from: https://msdn.microsoft.com/en-us/library/windows/desktop/dd378457(v=vs.85).aspx + 'Desktop': ( + '{B4BFCC3A-DB2C-424C-B029-7FE99A87C641}', + ), + 'Documents': ( + 'Personal', + '{FDD39AD0-238F-46AF-ADB4-6C85480369C7}', + ), + 'Downloads': ( + '{374DE290-123F-4565-9164-39C4925E467B}', + ), + 'Favorites': ( + '{1777F761-68AD-4D8A-87BD-30B759FA33DD}', + ), + 'Music': ( + 'My Music', + '{4BD8D571-6D19-48D3-BE97-422220080E43}', + ), + 'Pictures': ( + 'My Pictures', + '{33E28130-4E1E-4676-835A-98395C3BC3BB}', + ), + 'Videos': ( + 'My Video', + '{18989B1D-99B5-455B-841C-AB7C74E4DDFC}', + ), +} + +# Regex +REGEX_OFFICE = re.compile( + r'(Microsoft (Office\s+' + r'(365|Enterprise|Home|Pro(\s|fessional)' + r'|Single|Small|Standard|Starter|Ultimate|system)' + r'|Works[-\s\d]+\d)' + r'|(Libre|Open|Star)\s*Office' + r'|WordPerfect|Gnumeric|Abiword)', + re.IGNORECASE) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 2e88f058..73a70923 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -8,6 +8,12 @@ LAUNCHERS = { 'L_ITEM': 'activate.py', 'L_ELEV': 'True', }, + 'New System Setup': { + 'L_TYPE': 'PyScript', + 'L_PATH': 'Scripts', + 'L_ITEM': 'new_system_setup.py', + 'L_ELEV': 'True', + }, 'System Checklist': { 'L_TYPE': 'PyScript', 'L_PATH': 'Scripts', @@ -282,8 +288,8 @@ LAUNCHERS = { 'Intel RST (Current Release)': { 'L_TYPE': 'Executable', 'L_PATH': '_Drivers\Intel RST', - 'L_ITEM': 'SetupRST_16.8.exe', - 'L_7ZIP': 'SetupRST_16.8.exe', + 'L_ITEM': 'SetupRST_17.2.exe', + 'L_7ZIP': 'SetupRST_17.2.exe', }, 'Intel RST (Previous Releases)': { 'L_TYPE': 'Folder', @@ -585,6 +591,7 @@ LAUNCHERS = { }, } + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/main.py b/.bin/Scripts/settings/main.py index da83e830..99011cf9 100644 --- a/.bin/Scripts/settings/main.py +++ b/.bin/Scripts/settings/main.py @@ -4,8 +4,6 @@ ENABLED_OPEN_LOGS = False ENABLED_TICKET_NUMBERS = False ENABLED_UPLOAD_DATA = False -HW_OVERRIDES_FORCED = False -HW_OVERRIDES_LIMITED = True # If True this disables HW_OVERRIDE_FORCED # STATIC VARIABLES (also used by BASH and BATCH files) ## NOTE: There are no spaces around the = for easier parsing in BASH and BATCH @@ -15,7 +13,6 @@ KIT_NAME_FULL='WizardKit' KIT_NAME_SHORT='WK' SUPPORT_MESSAGE='Please let 2Shirt know by opening an issue on GitHub' # Live Linux -MPRIME_LIMIT='7' # of minutes to run Prime95 during hw-diags ROOT_PASSWORD='Abracadabra' TECH_PASSWORD='Abracadabra' # Server IP addresses @@ -89,6 +86,7 @@ WINDOWS_SERVER = { 'RW-Pass': 'Abracadabra', } + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/music.py b/.bin/Scripts/settings/music.py index 37f5d178..63163a40 100644 --- a/.bin/Scripts/settings/music.py +++ b/.bin/Scripts/settings/music.py @@ -66,6 +66,7 @@ MUSIC_SNES = [ 'zamn' ] + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/partition_uids.py b/.bin/Scripts/settings/partition_uids.py index f4b21df0..29817b2c 100644 --- a/.bin/Scripts/settings/partition_uids.py +++ b/.bin/Scripts/settings/partition_uids.py @@ -319,6 +319,7 @@ PARTITION_UIDS = { 'BC13C2FF-59E6-4262-A352-B275FD6F7172': {'OS': 'Freedesktop', 'Description': 'Extended Boot Partition ($BOOT)'}, } + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/sensors.py b/.bin/Scripts/settings/sensors.py new file mode 100644 index 00000000..cdfdf708 --- /dev/null +++ b/.bin/Scripts/settings/sensors.py @@ -0,0 +1,21 @@ +# Wizard Kit: Settings - Sensors + +import re + +# General +TEMP_LIMITS = { + 'GREEN': 60, + 'YELLOW': 70, + 'ORANGE': 80, + 'RED': 90, + } + + +# Regex +REGEX_COLORS = re.compile(r'\033\[\d+;?1?m') + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/setup.py b/.bin/Scripts/settings/setup.py new file mode 100644 index 00000000..232e3a0d --- /dev/null +++ b/.bin/Scripts/settings/setup.py @@ -0,0 +1,161 @@ +# Wizard Kit: Settings - Setup + +import os +import winreg + +# General +HKU = winreg.HKEY_USERS +HKCR = winreg.HKEY_CLASSES_ROOT +HKCU = winreg.HKEY_CURRENT_USER +HKLM = winreg.HKEY_LOCAL_MACHINE +OTHER_RESULTS = { + 'Error': { + 'CalledProcessError': 'Unknown Error', + 'FileNotFoundError': 'File not found', + }, + 'Warning': {}, + } + +# Browsers +MOZILLA_FIREFOX_UBO_PATH = r'{}\{}\ublock_origin.xpi'.format( + os.environ.get('PROGRAMFILES'), + r'Mozilla Firefox\distribution\extensions') +SETTINGS_GOOGLE_CHROME = { + r'Software\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm': { + 'SZ Items': { + 'update_url': 'https://clients2.google.com/service/update2/crx'}, + 'WOW64_32': True, + }, + r'Software\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco': { + 'SZ Items': { + 'update_url': 'https://clients2.google.com/service/update2/crx'}, + 'WOW64_32': True, + }, + } +SETTINGS_MOZILLA_FIREFOX_32 = { + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + 'WOW64_32': True, + }, + } +SETTINGS_MOZILLA_FIREFOX_64 = { + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + }, + } + +# Classic Start +SETTINGS_CLASSIC_START = { + r'Software\IvoSoft\ClassicShell\Settings': {}, + r'Software\IvoSoft\ClassicStartMenu': { + 'DWORD Items': {'ShowedStyle2': 1}, + }, + r'Software\IvoSoft\ClassicStartMenu\MRU': {}, + r'Software\IvoSoft\ClassicStartMenu\Settings': { + 'DWORD Items': {'SkipMetro': 1}, + 'SZ Items': { + 'MenuStyle': 'Win7', + 'RecentPrograms': 'Recent', + }, + }, + } + +# Explorer +SETTINGS_EXPLORER_SYSTEM = { + # Disable Location Tracking + r'Software\Microsoft\Windows NT\CurrentVersion\Sensor\Overrides\{BFA794E4-F964-4FDB-90F6-51056BFE4B44}': { + 'DWORD Items': {'SensorPermissionState': 0}, + }, + r'System\CurrentControlSet\Services\lfsvc\Service\Configuration': { + 'Status': {'Value': 0}, + }, + # Disable Telemetry + r'SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { + # Using SOFTWARE in all caps to avoid collision with 32-bit setting below + 'DWORD Items': {'AllowTelemetry': 0}, + }, + r'Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection': { + 'DWORD Items': {'AllowTelemetry': 0}, + 'WOW64_32': True, + }, + r'Software\Policies\Microsoft\Windows\DataCollection': { + 'DWORD Items': {'AllowTelemetry': 0}, + }, + # Disable Wi-Fi Sense + r'Software\Microsoft\PolicyManager\default\WiFi\AllowWiFiHotSpotReporting': { + 'DWORD Items': {'Value': 0}, + }, + r'Software\Microsoft\PolicyManager\default\WiFi\AllowAutoConnectToWiFiSenseHotspots': { + 'DWORD Items': {'Value': 0}, + }, + } +SETTINGS_EXPLORER_USER = { + # Disable features + r'Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager': { + 'DWORD Items': { + # Silently installed apps + 'SilentInstalledAppsEnabled': 0, + # Tips and Tricks + 'SoftLandingEnabled ': 0, + 'SubscribedContent-338389Enabled': 0, + }, + }, + # File Explorer + r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced': { + 'DWORD Items': { + # Change default Explorer view to "Computer" + 'LaunchTo': 1, + }, + }, + # Hide People bar + r'Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced\People': { + 'DWORD Items': {'PeopleBand': 0}, + }, + # Hide Search button / box + r'Software\Microsoft\Windows\CurrentVersion\Search': { + 'DWORD Items': {'SearchboxTaskbarMode': 0}, + }, + } + +# Visual C++ Runtimes +VCR_REDISTS = [ + {'Name': 'Visual C++ 2010 x32...', + 'Cmd': [r'2010sp1\x32\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2010 x64...', + 'Cmd': [r'2010sp1\x64\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2012 Update 4 x32...', + 'Cmd': [r'2012u4\x32\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2012 Update 4 x64...', + 'Cmd': [r'2012u4\x64\vcredist.exe', '/passive', '/norestart']}, + {'Name': 'Visual C++ 2013 x32...', + 'Cmd': [r'2013\x32\vcredist.exe', '/install', + '/passive', '/norestart']}, + {'Name': 'Visual C++ 2013 x64...', + 'Cmd': [r'2013\x64\vcredist.exe', '/install', + '/passive', '/norestart']}, + {'Name': 'Visual C++ 2017 x32...', + 'Cmd': [r'2017\x32\vcredist.exe', '/install', + '/passive', '/norestart']}, + {'Name': 'Visual C++ 2017 x64...', + 'Cmd': [r'2017\x64\vcredist.exe', '/install', + '/passive', '/norestart']}, + ] + +# Windows Updates +SETTINGS_WINDOWS_UPDATES = { + r'Software\Microsoft\WindowsUpdate\UX\Settings': { + 'DWORD Items': { + # Set to non-targeted readiness level + 'BranchReadinessLevel': 32, + 'DeferFeatureUpdatesPeriodInDays': 60, + }, + } + } + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 8a35895e..e6cf5c3a 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -1,7 +1,7 @@ # Wizard Kit: Settings - Sources SOURCE_URLS = { - 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1901020069/AcroRdrDC1901020069_en_US.exe', + 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1901020098/AcroRdrDC1901020098_en_US.exe', 'AdwCleaner': 'https://downloads.malwarebytes.com/file/adwcleaner', 'AIDA64': 'http://download.aida64.com/aida64engineer599.zip', 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0-win-32bit-build1.zip', @@ -13,21 +13,21 @@ SOURCE_URLS = { 'ClassicStartSkin': 'http://www.classicshell.net/forum/download/file.php?id=3001&sid=9a195960d98fd754867dcb63d9315335', 'Du': 'https://download.sysinternals.com/files/DU.zip', 'ERUNT': 'http://www.aumha.org/downloads/erunt.zip', - 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.924.x86.zip', - 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.924.x64.zip', - 'FastCopy': 'http://ftp.vector.co.jp/70/93/2323/FastCopy361_installer.exe', - 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1166954/ublock_origin-1.17.4-an+fx.xpi', + 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.935.x86.en-US.zip', + 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.935.x64.en-US.zip', + 'FastCopy': 'http://ftp.vector.co.jp/71/31/2323/FastCopy363_installer.exe', + 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1709472/ublock_origin-1.18.6-an+fx.xpi', 'HitmanPro32': 'https://dl.surfright.nl/HitmanPro.exe', 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', - 'HWiNFO': 'http://files2.majorgeeks.com/caae8849cf31a8d77c51283b720e60e49ce1dc78/systeminfo/hwi_600.zip', - 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/28447/eng/Intel%20SSD%20Toolbox%20-%20v3.5.8.exe', + 'HWiNFO': 'http://files2.majorgeeks.com/377527622c5325acc1cb937fb149d0de922320c0/systeminfo/hwi_602.zip', + 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/28593/eng/Intel%20SSD%20Toolbox%20-%20v3.5.9.exe', 'IOBit_Uninstaller': r'https://portableapps.com/redirect/?a=IObitUninstallerPortable&s=s&d=pa&f=IObitUninstallerPortable_7.5.0.7.paf.exe', 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', 'Macs Fan Control': 'https://www.crystalidea.com/downloads/macsfancontrol_setup.exe', 'NirCmd32': 'https://www.nirsoft.net/utils/nircmd.zip', 'NirCmd64': 'https://www.nirsoft.net/utils/nircmd-x64.zip', - 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.6.2/npp.7.6.2.bin.minimalist.7z', - 'Office Deployment Tool': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_11107-33602.exe', + 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.6.4/npp.7.6.4.bin.minimalist.7z', + 'Office Deployment Tool': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_11509-33604.exe', 'ProduKey32': 'http://www.nirsoft.net/utils/produkey.zip', 'ProduKey64': 'http://www.nirsoft.net/utils/produkey-x64.zip', 'PuTTY': 'https://the.earth.li/~sgtatham/putty/latest/w32/putty.zip', @@ -40,7 +40,7 @@ SOURCE_URLS = { 'wimlib32': 'https://wimlib.net/downloads/wimlib-1.13.0-windows-i686-bin.zip', 'wimlib64': 'https://wimlib.net/downloads/wimlib-1.13.0-windows-x86_64-bin.zip', 'Winapp2': 'https://github.com/MoscaDotTo/Winapp2/archive/master.zip', - 'WizTree': 'https://antibody-software.com/files/wiztree_3_26_portable.zip', + 'WizTree': 'https://antibody-software.com/files/wiztree_3_28_portable.zip', 'XMPlay 7z': 'https://support.xmplay.com/files/16/xmp-7z.zip?v=800962', 'XMPlay Game': 'https://support.xmplay.com/files/12/xmp-gme.zip?v=515637', 'XMPlay RAR': 'https://support.xmplay.com/files/16/xmp-rar.zip?v=409646', @@ -94,7 +94,7 @@ NINITE_SOURCES = { 'SugarSync.exe': 'sugarsync', }, 'Communication': { - 'Discord': 'discord', + 'Discord.exe': 'discord', 'Pidgin.exe': 'pidgin', 'Skype.exe': 'skype', 'Trillian.exe': 'trillian', @@ -106,8 +106,6 @@ NINITE_SOURCES = { }, 'Developer': { 'Eclipse.exe': 'eclipse', - 'JDK 8.exe': 'jdk8', - 'JDK 8 (x64).exe': 'jdkx8', 'Notepad++.exe': 'notepadplusplus', 'PuTTY.exe': 'putty', 'Python 2.exe': 'python', @@ -150,7 +148,6 @@ NINITE_SOURCES = { 'Runtimes': { 'Adobe Air.exe': 'air', 'dotNET.exe': '.net4.7.2', - 'Java 8.exe': 'java8', 'Shockwave.exe': 'shockwave', 'Silverlight.exe': 'silverlight', }, @@ -191,12 +188,31 @@ RST_SOURCES = { 'SetupRST_14.0.exe': 'https://downloadmirror.intel.com/25091/eng/SetupRST.exe', 'SetupRST_14.8.exe': 'https://downloadmirror.intel.com/26759/eng/setuprst.exe', 'SetupRST_15.8.exe': 'https://downloadmirror.intel.com/27442/eng/SetupRST.exe', - #SetupRST_15.9.exe : Deprecated by Intel + 'SetupRST_15.9.exe': 'https://downloadmirror.intel.com/28656/eng/SetupRST.exe', #SetupRST_16.0.exe : Deprecated by Intel #SetupRST_16.5.exe : Deprecated by Intel #SetupRST_16.7.exe : Deprecated by Intel - 'SetupRST_16.8.exe': 'https://downloadmirror.intel.com/28400/eng/SetupRST.exe', + 'SetupRST_16.8.exe': 'https://downloadmirror.intel.com/28653/eng/SetupRST.exe', + 'SetupRST_17.2.exe': 'https://downloadmirror.intel.com/28650/eng/SetupRST.exe', } +WINDOWS_UPDATE_SOURCES = { + '2999226': { + # https://support.microsoft.com/en-us/help/2999226/update-for-universal-c-runtime-in-windows + '7': { + '32': 'https://download.microsoft.com/download/4/F/E/4FE73868-5EDD-4B47-8B33-CE1BB7B2B16A/Windows6.1-KB2999226-x86.msu', + '64': 'https://download.microsoft.com/download/1/1/5/11565A9A-EA09-4F0A-A57E-520D5D138140/Windows6.1-KB2999226-x64.msu', + }, + '8': { + '32': 'https://download.microsoft.com/download/1/E/8/1E8AFE90-5217-464D-9292-7D0B95A56CE4/Windows8-RT-KB2999226-x86.msu', + '64': 'https://download.microsoft.com/download/A/C/1/AC15393F-A6E6-469B-B222-C44B3BB6ECCC/Windows8-RT-KB2999226-x64.msu', + }, + '8.1': { + '32': 'https://download.microsoft.com/download/E/4/6/E4694323-8290-4A08-82DB-81F2EB9452C2/Windows8.1-KB2999226-x86.msu', + '64': 'https://download.microsoft.com/download/9/6/F/96FD0525-3DDF-423D-8845-5F92F4A6883E/Windows8.1-KB2999226-x64.msu', + }, + }, + } + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/sw_diags.py b/.bin/Scripts/settings/sw_diags.py new file mode 100644 index 00000000..00892943 --- /dev/null +++ b/.bin/Scripts/settings/sw_diags.py @@ -0,0 +1,29 @@ +# Wizard Kit: Settings - SW Diagnostics + +# General +AUTORUNS_SETTINGS = { + r'Software\Sysinternals\AutoRuns': { + 'checkvirustotal': 1, + 'EulaAccepted': 1, + 'shownomicrosoft': 1, + 'shownowindows': 1, + 'showonlyvirustotal': 1, + 'submitvirustotal': 0, + 'verifysignatures': 1, + }, + r'Software\Sysinternals\AutoRuns\SigCheck': { + 'EulaAccepted': 1, + }, + r'Software\Sysinternals\AutoRuns\Streams': { + 'EulaAccepted': 1, + }, + r'Software\Sysinternals\AutoRuns\VirusTotal': { + 'VirusTotalTermsAccepted': 1, + }, + } + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/tools.py b/.bin/Scripts/settings/tools.py index 0ee74dec..dd3812e1 100644 --- a/.bin/Scripts/settings/tools.py +++ b/.bin/Scripts/settings/tools.py @@ -52,6 +52,7 @@ TOOLS = { '32': r'XMPlay\xmplay.exe'}, } + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/windows_builds.py b/.bin/Scripts/settings/windows_builds.py index 1b451daf..db538bf2 100644 --- a/.bin/Scripts/settings/windows_builds.py +++ b/.bin/Scripts/settings/windows_builds.py @@ -209,6 +209,7 @@ WINDOWS_BUILDS = { '18855': ('10', None, '20H1', None, 'preview build'), } + if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/settings/windows_setup.py b/.bin/Scripts/settings/windows_setup.py new file mode 100644 index 00000000..2f01c506 --- /dev/null +++ b/.bin/Scripts/settings/windows_setup.py @@ -0,0 +1,39 @@ +# Wizard Kit: Settings - Windows Setup + +# General +WINDOWS_VERSIONS = [ + {'Name': 'Windows 7 Home Basic', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 HOMEBASIC'}, + {'Name': 'Windows 7 Home Premium', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 HOMEPREMIUM'}, + {'Name': 'Windows 7 Professional', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 PROFESSIONAL'}, + {'Name': 'Windows 7 Ultimate', + 'Image File': 'Win7', + 'Image Name': 'Windows 7 ULTIMATE'}, + + {'Name': 'Windows 8.1', + 'Image File': 'Win8', + 'Image Name': 'Windows 8.1', + 'CRLF': True}, + {'Name': 'Windows 8.1 Pro', + 'Image File': 'Win8', + 'Image Name': 'Windows 8.1 Pro'}, + + {'Name': 'Windows 10 Home', + 'Image File': 'Win10', + 'Image Name': 'Windows 10 Home', + 'CRLF': True}, + {'Name': 'Windows 10 Pro', + 'Image File': 'Win10', + 'Image Name': 'Windows 10 Pro'}, + ] + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/settings/winpe.py b/.bin/Scripts/settings/winpe.py new file mode 100644 index 00000000..2d52353b --- /dev/null +++ b/.bin/Scripts/settings/winpe.py @@ -0,0 +1,57 @@ +# Wizard Kit: Settings - WinPE + +from settings.data import * + +# FastCopy +FAST_COPY_PE_ARGS = [ + '/cmd=noexist_only', + '/utf8', + '/skip_empty_dir', + '/linkdest', + '/no_ui', + '/auto_close', + '/exclude={}'.format(';'.join(FAST_COPY_EXCLUDES)), + ] + +# General +PE_TOOLS = { + 'BlueScreenView': { + 'Path': r'BlueScreenView\BlueScreenView.exe', + }, + 'FastCopy': { + 'Path': r'FastCopy\FastCopy.exe', + 'Args': FAST_COPY_PE_ARGS, + }, + 'HWiNFO': { + 'Path': r'HWiNFO\HWiNFO.exe', + }, + 'NT Password Editor': { + 'Path': r'NT Password Editor\ntpwedit.exe', + }, + 'Notepad++': { + 'Path': r'NotepadPlusPlus\NotepadPlusPlus.exe', + }, + 'PhotoRec': { + 'Path': r'TestDisk\photorec_win.exe', + 'Args': ['-new_console:n'], + }, + 'Prime95': { + 'Path': r'Prime95\prime95.exe', + }, + 'ProduKey': { + 'Path': r'ProduKey\ProduKey.exe', + }, + 'Q-Dir': { + 'Path': r'Q-Dir\Q-Dir.exe', + }, + 'TestDisk': { + 'Path': r'TestDisk\testdisk_win.exe', + 'Args': ['-new_console:n'], + }, + } + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=2 sw=2 ts=2 diff --git a/.bin/Scripts/system_checklist.py b/.bin/Scripts/system_checklist.py index 5900ef00..d092c8da 100644 --- a/.bin/Scripts/system_checklist.py +++ b/.bin/Scripts/system_checklist.py @@ -43,9 +43,15 @@ if __name__ == '__main__': if global_vars['OS']['Version'] == '10': try_and_print(message='Explorer...', function=config_explorer_system, cs='Done') + try_and_print(message='Windows Updates...', + function=config_windows_updates, cs='Done') try_and_print(message='Updating Clock...', function=update_clock, cs='Done') + # Restart Explorer + try_and_print(message='Restarting Explorer...', + function=restart_explorer, cs='Done') + # Cleanup print_info('Cleanup') try_and_print(message='AdwCleaner...', diff --git a/.bin/Scripts/update_kit.py b/.bin/Scripts/update_kit.py index 18bbb69e..15b3162e 100644 --- a/.bin/Scripts/update_kit.py +++ b/.bin/Scripts/update_kit.py @@ -60,6 +60,7 @@ if __name__ == '__main__': try_and_print(message='Macs Fan Control...', function=update_macs_fan_control, other_results=other_results, width=40) try_and_print(message='MS Office...', function=update_office, other_results=other_results, width=40) try_and_print(message='Visual C++ Runtimes...', function=update_vcredists, other_results=other_results, width=40) + try_and_print(message='Windows Updates...', function=download_windows_updates, other_results=other_results, width=40) update_all_ninite(other_results=other_results, width=40) # Misc diff --git a/.bin/Scripts/user_checklist.py b/.bin/Scripts/user_checklist.py index 91e5915a..e74d26f9 100644 --- a/.bin/Scripts/user_checklist.py +++ b/.bin/Scripts/user_checklist.py @@ -71,6 +71,10 @@ if __name__ == '__main__': if not answer_config_browsers: print_warning(' Skipped') + # Restart Explorer + try_and_print(message='Restarting Explorer...', + function=restart_explorer, cs='Done') + # Run speedtest popen_program(['start', '', 'https://fast.com'], shell=True) diff --git a/.linux_items/include/EFI/boot/refind.conf b/.linux_items/include/EFI/boot/refind.conf index 1406ac73..4ab3be7b 100644 --- a/.linux_items/include/EFI/boot/refind.conf +++ b/.linux_items/include/EFI/boot/refind.conf @@ -33,8 +33,8 @@ menuentry "Linux" { add_options "loglevel=4 nomodeset nox" } } -#UFD#menuentry "WindowsPE" { -#UFD# ostype windows -#UFD# icon /EFI/boot/icons/wk_win.png -#UFD# loader /EFI/microsoft/bootx64.efi -#UFD#} +#UFD-WINPE#menuentry "WindowsPE" { +#UFD-WINPE# ostype windows +#UFD-WINPE# icon /EFI/boot/icons/wk_win.png +#UFD-WINPE# loader /EFI/microsoft/bootx64.efi +#UFD-WINPE#} diff --git a/.linux_items/include/syslinux/wk_pxe.cfg b/.linux_items/include/syslinux/wk_pxe.cfg index f8a78220..10112666 100644 --- a/.linux_items/include/syslinux/wk_pxe.cfg +++ b/.linux_items/include/syslinux/wk_pxe.cfg @@ -2,7 +2,7 @@ INCLUDE boot/syslinux/wk_head.cfg MENU BACKGROUND pxelinux.png INCLUDE boot/syslinux/wk_pxe_linux.cfg -#UFD#INCLUDE boot/syslinux/wk_pxe_winpe.cfg +#UFD-WINPE#INCLUDE boot/syslinux/wk_pxe_winpe.cfg #DISABLED_UPSTREAM_BUG#INCLUDE boot/syslinux/wk_hdt.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/include/syslinux/wk_sys.cfg b/.linux_items/include/syslinux/wk_sys.cfg index 55a47e7f..f90e4406 100644 --- a/.linux_items/include/syslinux/wk_sys.cfg +++ b/.linux_items/include/syslinux/wk_sys.cfg @@ -1,7 +1,7 @@ INCLUDE boot/syslinux/wk_head.cfg INCLUDE boot/syslinux/wk_sys_linux.cfg -#UFD#INCLUDE boot/syslinux/wk_sys_winpe.cfg +#UFD-WINPE#INCLUDE boot/syslinux/wk_sys_winpe.cfg #DISABLED_UPSTREAM_BUG#INCLUDE boot/syslinux/wk_hdt.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/packages/aur b/.linux_items/packages/aur index 9588b129..60272aea 100644 --- a/.linux_items/packages/aur +++ b/.linux_items/packages/aur @@ -2,6 +2,7 @@ aic94xx-firmware bash-pipes hfsprogs i3lock-fancy-git +ldmtool macbook12-spi-driver-dkms mprime openbox-patched diff --git a/.linux_items/packages/dependencies b/.linux_items/packages/dependencies index edf99bb6..4558e9eb 100644 --- a/.linux_items/packages/dependencies +++ b/.linux_items/packages/dependencies @@ -4,8 +4,10 @@ base-devel curl dos2unix git +gtk-doc hwloc imlib2 +json-glib lhasa libbsd libewf diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index 85ea5489..668c10c0 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -16,6 +16,7 @@ e2fsprogs hexedit hfsprogs htop +ldmtool ldns lha libewf diff --git a/Build Linux b/Build Linux index 4ccd1729..8ed965b5 100755 --- a/Build Linux +++ b/Build Linux @@ -170,6 +170,9 @@ function update_live_env() { rsync -aI "/usr/share/refind/icons/" "$LIVE_DIR/EFI/boot/icons/" --exclude "/usr/share/refind/icons/svg" sed -i "s/%ARCHISO_LABEL%/${label}/" "$LIVE_DIR/EFI/boot/refind.conf" + # Customize_airootfs.sh + sed -i -r 's/set -e -u/set -o errexit\nset -o errtrace\nset -o nounset\nset -o pipefail/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + # Memtest86 mkdir -p "$LIVE_DIR/EFI/memtest86/Benchmark" mkdir -p "$TEMP_DIR/memtest86"