diff --git a/.bin/Scripts/Launch.cmd b/.bin/Scripts/Launch.cmd index 2364dcc8..f7080adb 100644 --- a/.bin/Scripts/Launch.cmd +++ b/.bin/Scripts/Launch.cmd @@ -150,7 +150,6 @@ goto Exit :LaunchOffice call "%bin%\Scripts\init_client_dir.cmd" /Office set "_odt=False" -if %L_PATH% equ 2013 (set "_odt=True") if %L_PATH% equ 2016 (set "_odt=True") if "%_odt%" == "True" ( goto LaunchOfficeODT @@ -493,4 +492,4 @@ goto Exit :Exit popd endlocal -exit /b %errorlevel% \ No newline at end of file +exit /b %errorlevel% diff --git a/.bin/Scripts/build-ufd b/.bin/Scripts/build-ufd index faae7867..c606892b 100755 --- a/.bin/Scripts/build-ufd +++ b/.bin/Scripts/build-ufd @@ -557,7 +557,9 @@ mount "${WINPE_ISO}" /mnt/WinPE -r >> "${LOG_FILE}" 2>&1 echo "Copying Linux files..." rsync ${RSYNC_ARGS} /mnt/Linux/* /mnt/Dest/ >> "${LOG_FILE}" 2>&1 sed -i "s/${ISO_LABEL}/${UFD_LABEL}/" /mnt/Dest/EFI/boot/refind.conf +sed -i "s/#UFD#//" /mnt/Dest/EFI/boot/refind.conf sed -i "s/${ISO_LABEL}/${UFD_LABEL}/" /mnt/Dest/arch/boot/syslinux/*cfg +sed -i "s/#UFD#//" /mnt/Dest/arch/boot/syslinux/*cfg echo "Copying WinPE files..." rsync ${RSYNC_ARGS} /mnt/WinPE/{Boot,bootmgr{,.efi},en-us,sources} /mnt/Dest/ >> "${LOG_FILE}" 2>&1 diff --git a/.bin/Scripts/build_kit.ps1 b/.bin/Scripts/build_kit.ps1 index dafaa2c8..bd213578 100644 --- a/.bin/Scripts/build_kit.ps1 +++ b/.bin/Scripts/build_kit.ps1 @@ -11,6 +11,7 @@ $Bin = (Get-Item $WD).Parent.FullName $Root = (Get-Item $Bin -Force).Parent.FullName $Temp = "$Bin\tmp" $System32 = "{0}\System32" -f $Env:SystemRoot +$SysWOW64 = "{0}\SysWOW64" -f $Env:SystemRoot Push-Location "$WD" $Host.UI.RawUI.BackgroundColor = "black" $Host.UI.RawUI.ForegroundColor = "white" @@ -82,25 +83,25 @@ if ($MyInvocation.InvocationName -ne ".") { DownloadFile -Path $Path -Name "7z-extra.7z" -Url "https://www.7-zip.org/a/7z1805-extra.7z" # ConEmu - $Url = "https://github.com/Maximus5/ConEmu/releases/download/v18.05.06/ConEmuPack.180506.7z" + $Url = "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z" DownloadFile -Path $Path -Name "ConEmuPack.7z" -Url $Url # Notepad++ - $Url = "https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.7z" + $Url = "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z" DownloadFile -Path $Path -Name "npp.7z" -Url $Url # Python - $Url = "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-win32.zip" + $Url = "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip" DownloadFile -Path $Path -Name "python32.zip" -Url $Url - $Url = "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-amd64.zip" + $Url = "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip" DownloadFile -Path $Path -Name "python64.zip" -Url $Url # Python: psutil $DownloadPage = "https://pypi.org/project/psutil/" - $RegEx = "href=.*-cp36-cp36m-win32.whl" + $RegEx = "href=.*-cp37-cp37m-win32.whl" $Url = FindDynamicUrl $DownloadPage $RegEx DownloadFile -Path $Path -Name "psutil32.whl" -Url $Url - $RegEx = "href=.*-cp36-cp36m-win_amd64.whl" + $RegEx = "href=.*-cp37-cp37m-win_amd64.whl" $Url = FindDynamicUrl $DownloadPage $RegEx DownloadFile -Path $Path -Name "psutil64.whl" -Url $Url @@ -112,12 +113,25 @@ if ($MyInvocation.InvocationName -ne ".") { $Url = FindDynamicUrl -SourcePage $DownloadPage -RegEx $RegEx DownloadFile -Path $Path -Name $Name -Url $Url } + + # Visual C++ Runtimes + $Url = "https://aka.ms/vs/15/release/vc_redist.x86.exe" + DownloadFile -Path $Path -Name "vcredist_x86.exe" -Url $Url + $Url = "https://aka.ms/vs/15/release/vc_redist.x64.exe" + DownloadFile -Path $Path -Name "vcredist_x64.exe" -Url $Url ## Bail ## # If errors were encountered during downloads if ($DownloadErrors -gt 0) { Abort } + + ## Install ## + # Visual C++ Runtimes + $ArgumentList = @("/install", "/passive", "/norestart") + Start-Process -FilePath "$Temp\vcredist_x86.exe" -ArgumentList $ArgumentList -Wait + Start-Process -FilePath "$Temp\vcredist_x64.exe" -ArgumentList $ArgumentList -Wait + Remove-Item "$Temp\vcredist*.exe" ## Extract ## # 7-Zip @@ -192,6 +206,13 @@ if ($MyInvocation.InvocationName -ne ".") { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } } + try { + Copy-Item -Path "$System32\vcruntime140.dll" -Destination "$Bin\Python\x64\vcruntime140.dll" -Force + Copy-Item -Path "$SysWOW64\vcruntime140.dll" -Destination "$Bin\Python\x32\vcruntime140.dll" -Force + } + catch { + Write-Host (" ERROR: Failed to copy Visual C++ Runtime DLLs." ) -ForegroundColor "Red" + } Remove-Item "$Temp\python*.zip" Remove-Item "$Temp\*.whl" diff --git a/.bin/Scripts/build_pe.ps1 b/.bin/Scripts/build_pe.ps1 index 0e19050a..560d570d 100644 --- a/.bin/Scripts/build_pe.ps1 +++ b/.bin/Scripts/build_pe.ps1 @@ -17,6 +17,7 @@ $Date = Get-Date -UFormat "%Y-%m-%d" $Host.UI.RawUI.BackgroundColor = "Black" $Host.UI.RawUI.ForegroundColor = "White" $HostSystem32 = "{0}\System32" -f $Env:SystemRoot +$HostSysWOW64 = "{0}\SysWOW64" -f $Env:SystemRoot $DISM = "{0}\DISM.exe" -f $Env:DISMRoot #Enable TLS 1.2 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -136,20 +137,19 @@ if ($MyInvocation.InvocationName -ne ".") { @("bluescreenview32.zip", "http://www.nirsoft.net/utils/bluescreenview.zip"), @("bluescreenview64.zip", "http://www.nirsoft.net/utils/bluescreenview-x64.zip"), # ConEmu - @("ConEmuPack.7z", "https://github.com/Maximus5/ConEmu/releases/download/v18.05.06/ConEmuPack.180506.7z"), + @("ConEmuPack.7z", "https://github.com/Maximus5/ConEmu/releases/download/v18.06.26/ConEmuPack.180626.7z"), # Fast Copy - @("fastcopy32.zip", "http://ftp.vector.co.jp/69/93/2323/FastCopy341.zip"), - @("fastcopy64.zip", "http://ftp.vector.co.jp/69/93/2323/FastCopy341_x64.zip"), + @("fastcopy.zip", "http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip"), # HWiNFO - @("hwinfo.zip", "http://app.oldfoss.com:81/download/HWiNFO/hwi_582.zip"), + @("hwinfo.zip", "http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip"), # Killer Network Drivers @( "killerinf.zip", ("http://www.killernetworking.com"+(FindDynamicUrl "http://www.killernetworking.com/driver-downloads/item/killer-drivers-inf" "Download Killer-Ethernet").replace('&', '&')) ), # Notepad++ - @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.7z"), - @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.x64.7z"), + @("npp_x86.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z"), + @("npp_amd64.7z", "https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.x64.7z"), # NT Password Editor @("ntpwed.zip", "http://cdslow.org.ru/files/ntpwedit/ntpwed07.zip"), # Prime95 @@ -159,16 +159,16 @@ if ($MyInvocation.InvocationName -ne ".") { @("produkey32.zip", "http://www.nirsoft.net/utils/produkey.zip"), @("produkey64.zip", "http://www.nirsoft.net/utils/produkey-x64.zip"), # Python - @("python32.zip", "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-win32.zip"), - @("python64.zip", "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-amd64.zip"), + @("python32.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-win32.zip"), + @("python64.zip", "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip"), # Python: psutil @( "psutil64.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win_amd64.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win_amd64.whl") ), @( "psutil32.whl", - (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp36-cp36m-win32.whl") + (FindDynamicUrl "https://pypi.org/project/psutil/" "href=.*-cp37-cp37m-win32.whl") ), # Q-Dir @("qdir32.zip", "https://www.softwareok.com/Download/Q-Dir_Portable.zip"), @@ -178,6 +178,9 @@ if ($MyInvocation.InvocationName -ne ".") { @("testdisk64.zip", "https://www.cgsecurity.org/testdisk-7.1-WIP.win64.zip"), # VirtIO drivers @("virtio-win.iso", "https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/latest-virtio/virtio-win.iso"), + # Visual C++ Runtimes + @("vcredist_x86.exe", "https://aka.ms/vs/15/release/vc_redist.x86.exe"), + @("vcredist_x64.exe", "https://aka.ms/vs/15/release/vc_redist.x64.exe"), # wimlib-imagex @("wimlib32.zip", "https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip"), @("wimlib64.zip", "https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip") @@ -191,6 +194,13 @@ if ($MyInvocation.InvocationName -ne ".") { if ($DownloadErrors -gt 0) { Abort } + + ## Install ## + # Visual C++ Runtimes + Write-Host "Installing: Visual C++ Runtimes" + $ArgumentList = @("/install", "/passive", "/norestart") + Start-Process -FilePath "$Temp\vcredist_x86.exe" -ArgumentList $ArgumentList -Wait + Start-Process -FilePath "$Temp\vcredist_x64.exe" -ArgumentList $ArgumentList -Wait ## Extract ## # 7-Zip @@ -255,20 +265,30 @@ if ($MyInvocation.InvocationName -ne ".") { # Fast Copy Write-Host "Extracting: FastCopy" try { + # Extract Installer $ArgumentList = @( - "x", "$Temp\fastcopy64.zip", "-o$Build\bin\amd64\FastCopy", - "-aoa", "-bso0", "-bse0", "-bsp0", - "-x!setup.exe", "-x!*.dll") + "e", "$Temp\fastcopy.zip", "-o$Temp", + "-aoa", "-bso0", "-bse0", "-bsp0") Start-Process -FilePath $SevenZip -ArgumentList $ArgumentList -NoNewWindow -Wait + + # Extract 64-bit $ArgumentList = @( - "e", "$Temp\fastcopy32.zip", "-o$Build\bin\x86\FastCopy", - "-aoa", "-bso0", "-bse0", "-bsp0", - "-x!setup.exe", "-x!*.dll") - Start-Process -FilePath $SevenZip -ArgumentList $ArgumentList -NoNewWindow -Wait + "/NOSUBDIR", "/DIR=$Build\bin\amd64\FastCopy", + "/EXTRACT64") + Start-Process -FilePath "$TEMP\FastCopy354_installer.exe" -ArgumentList $ArgumentList -NoNewWindow -Wait + Remove-Item "$Build\bin\amd64\FastCopy\setup.exe" -Force + + # Extract 32-bit + $ArgumentList = @( + "/NOSUBDIR", "/DIR=$Build\bin\x86\FastCopy", + "/EXTRACT32") + Start-Process -FilePath "$TEMP\FastCopy354_installer.exe" -ArgumentList $ArgumentList -NoNewWindow -Wait + Remove-Item "$Build\bin\x86\FastCopy\setup.exe" -Force } catch { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } + # Killer Network Driver Write-Host "Extracting: Killer Network Driver" @@ -414,6 +434,12 @@ if ($MyInvocation.InvocationName -ne ".") { catch { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } + try { + Copy-Item -Path "$HostSystem32\vcruntime140.dll" -Destination "$Build\bin\amd64\python\vcruntime140.dll" -Force + } + catch { + Write-Host (" ERROR: Failed to copy Visual C++ Runtime DLL." ) -ForegroundColor "Red" + } # Python (x32) Write-Host "Extracting: Python (x32)" @@ -431,6 +457,12 @@ if ($MyInvocation.InvocationName -ne ".") { catch { Write-Host (" ERROR: Failed to extract files." ) -ForegroundColor "Red" } + try { + Copy-Item -Path "$HostSysWOW64\vcruntime140.dll" -Destination "$Build\bin\x86\python\vcruntime140.dll" -Force + } + catch { + Write-Host (" ERROR: Failed to copy Visual C++ Runtime DLL." ) -ForegroundColor "Red" + } # Q-Dir Write-Host "Extracting: Q-Dir" diff --git a/.bin/Scripts/ddrescue-tui b/.bin/Scripts/ddrescue-tui new file mode 100755 index 00000000..d83b3a7e --- /dev/null +++ b/.bin/Scripts/ddrescue-tui @@ -0,0 +1,43 @@ +#!/bin/bash +# +## Wizard Kit: ddrescue TUI Launcher + +SESSION_NAME="ddrescue-tui" +WINDOW_NAME="GNU ddrescue TUI" +MENU="ddrescue-tui-menu" + +function ask() { + while :; do + read -p "$1 " -r answer + if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then + return 0 + elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then + return 1 + fi + done +} + +die () { + echo "$0:" "$@" >&2 + exit 1 +} + +# Check for running session +if tmux list-session | grep -q "$SESSION_NAME"; then + echo "WARNING: tmux session $SESSION_NAME already exists." + echo "" + if ask "Kill current session?"; then + tmux kill-session -t "$SESSION_NAME" || \ + die "Failed to kill session: $SESSION_NAME" + else + echo "Aborted." + echo "" + echo -n "Press Enter to exit... " + read -r + exit 0 + fi +fi + +# Start session +tmux new-session -s "$SESSION_NAME" -n "$WINDOW_NAME" "$MENU" $* + diff --git a/.bin/Scripts/ddrescue-tui-menu b/.bin/Scripts/ddrescue-tui-menu new file mode 100755 index 00000000..4bec6230 --- /dev/null +++ b/.bin/Scripts/ddrescue-tui-menu @@ -0,0 +1,63 @@ +#!/bin/python3 +# +## Wizard Kit: TUI for ddrescue cloning and imaging + +import os +import sys + +# Init +sys.path.append(os.path.dirname(os.path.realpath(__file__))) + +from functions.ddrescue import * +from functions.hw_diags import * +init_global_vars() + +if __name__ == '__main__': + try: + # Prep + clear_screen() + args = list(sys.argv) + run_mode = '' + source_path = None + dest_path = None + + # Parse args + try: + script_name = os.path.basename(args.pop(0)) + run_mode = str(args.pop(0)).lower() + source_path = args.pop(0) + dest_path = args.pop(0) + except IndexError: + # We'll set the missing paths later + pass + + # Show usage + if re.search(r'-+(h|help)', str(sys.argv), re.IGNORECASE): + show_usage(script_name) + exit_script() + + # Start cloning/imaging + if run_mode in ('clone', 'image'): + menu_ddrescue(source_path, dest_path, run_mode) + else: + if not re.search(r'^-*(h|help\?)', run_mode, re.IGNORECASE): + print_error('Invalid mode.') + + # Done + print_standard('\nDone.') + pause("Press Enter to exit...") + exit_script() + except GenericAbort: + abort() + except GenericError as ge: + msg = 'Generic Error' + if str(ge): + msg = str(ge) + print_error(msg) + abort() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/ddrescue-tui-smart-display b/.bin/Scripts/ddrescue-tui-smart-display new file mode 100755 index 00000000..285229d6 --- /dev/null +++ b/.bin/Scripts/ddrescue-tui-smart-display @@ -0,0 +1,39 @@ +#!/bin/python3 +# +## Wizard Kit: SMART attributes display for ddrescue TUI + +import os +import sys +import time + +# Init +os.chdir(os.path.dirname(os.path.realpath(__file__))) +sys.path.append(os.getcwd()) +from functions.hw_diags import * +#init_global_vars() + +if __name__ == '__main__': + try: + # Prep + clear_screen() + dev_path = sys.argv[1] + devs = scan_disks(True, dev_path) + + # Warn if SMART unavailable + if dev_path not in devs: + print_error('SMART data not available') + exit_script() + + # Initial screen + dev = devs[dev_path] + clear_screen() + show_disk_details(dev, only_attributes=True) + + # Done + exit_script() + except SystemExit: + pass + except: + major_exception() + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/echo-and-hold b/.bin/Scripts/echo-and-hold new file mode 100755 index 00000000..97c69830 --- /dev/null +++ b/.bin/Scripts/echo-and-hold @@ -0,0 +1,12 @@ +#!/bin/bash +# +## Wizard Kit: "echo" text to screen and "hold" by waiting for user input + +function usage { + echo "Usage: $(basename "$0") \"text\"" + echo " e.g. $(basename "$0") \"Some text to show\"" +} + +echo -en "$@" && read -r __dont_care +exit 0 + diff --git a/.bin/Scripts/functions/browsers.py b/.bin/Scripts/functions/browsers.py index b969a59c..99086f18 100644 --- a/.bin/Scripts/functions/browsers.py +++ b/.bin/Scripts/functions/browsers.py @@ -46,6 +46,9 @@ UBO_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\cjpalhdl UBO_EXTRA_CHROME = 'https://chrome.google.com/webstore/detail/ublock-origin-extra/pgdnlhfefecpicbbihgmbmffkjpaplco?hl=en' UBO_EXTRA_CHROME_REG = r'Software\Wow6432Node\Google\Chrome\Extensions\pgdnlhfefecpicbbihgmbmffkjpaplco' UBO_MOZILLA = 'https://addons.mozilla.org/en-us/firefox/addon/ublock-origin/' +UBO_MOZZILA_PATH = r'{}\Mozilla Firefox\distribution\extensions\ublock_origin.xpi'.format(os.environ.get('PROGRAMFILES')) +UBO_MOZILLA_REG = r'Software\Mozilla\Firefox\Extensions' +UBO_MOZILLA_REG_NAME = 'uBlock0@raymondhill.net' UBO_OPERA = 'https://addons.opera.com/en/extensions/details/ublock/?display=en' SUPPORTED_BROWSERS = { 'Internet Explorer': { @@ -285,6 +288,9 @@ def get_ie_homepages(): homepages.append(main_page) if len(extra_pages) > 0: homepages.extend(extra_pages) + + # Remove all curly braces + homepages = [h.replace('{', '').replace('}', '') for h in homepages] return homepages def get_mozilla_homepages(prefs_path): @@ -366,14 +372,17 @@ def install_adblock(indent=8, width=32): urls.append(UBO_EXTRA_CHROME) elif browser_data[browser]['base'] == 'mozilla': - # Assume UBO is not installed first and change if it is - urls.append(UBO_MOZILLA) - if browser == 'Mozilla Firefox': - ubo = browser_data[browser]['exe_path'].replace( - 'firefox.exe', - r'distribution\extensions\uBlock0@raymondhill.net') - if os.path.exists(ubo): + # Check for system extensions + try: + with winreg.OpenKey(HKLM, UBO_MOZILLA_REG) as key: + winreg.QueryValueEx(key, UBO_MOZILLA_REG_NAME) + except FileNotFoundError: + urls = [UBO_MOZILLA] + else: + if os.path.exists(UBO_MOZZILA_PATH): urls = ['about:addons'] + else: + urls = [UBO_MOZILLA] elif browser_data[browser]['base'] == 'ie': urls.append(IE_GALLERY) diff --git a/.bin/Scripts/functions/common.py b/.bin/Scripts/functions/common.py index f5a2ef95..dc427157 100644 --- a/.bin/Scripts/functions/common.py +++ b/.bin/Scripts/functions/common.py @@ -197,6 +197,30 @@ def extract_item(item, filter='', silent=False): if not silent: print_warning('WARNING: Errors encountered while exctracting data') +def get_process(name=None): + """Get process by name, returns psutil.Process obj.""" + proc = None + if not name: + raise GenericError + + for p in psutil.process_iter(): + try: + if p.name() == name: + proc = p + except psutil._exceptions.NoSuchProcess: + # Process finished during iteration? Going to ignore + pass + return proc + +def get_simple_string(prompt='Enter string'): + """Get string from user (minimal allowed character set) and return as str.""" + simple_string = None + while simple_string is None: + _input = input('{}: '.format(prompt)) + if re.match(r"^(\w|-| |\.|')+$", _input, re.ASCII): + simple_string = _input.strip() + return simple_string + def get_ticket_number(): """Get TicketNumber from user, save in LogDir, and return as str.""" if not ENABLED_TICKET_NUMBERS: @@ -213,15 +237,6 @@ def get_ticket_number(): f.write(ticket_number) return ticket_number -def get_simple_string(prompt='Enter string'): - """Get string from user (only alphanumeric/space chars) and return as str.""" - simple_string = None - while simple_string is None: - _input = input('{}: '.format(prompt)) - if re.match(r'^(\w|-| )+$', _input, re.ASCII): - simple_string = _input.strip() - return simple_string - def human_readable_size(size, decimals=0): """Convert size in bytes to a human-readable format and return a str.""" # Prep string formatting @@ -234,6 +249,8 @@ def human_readable_size(size, decimals=0): size = int(size) except ValueError: size = convert_to_bytes(size) + except TypeError: + size = -1 # Verify we have a valid size if size < 0: diff --git a/.bin/Scripts/functions/data.py b/.bin/Scripts/functions/data.py index ad4fe4f1..c8eec384 100644 --- a/.bin/Scripts/functions/data.py +++ b/.bin/Scripts/functions/data.py @@ -153,6 +153,67 @@ def cleanup_transfer(dest_path): except Exception: pass +def find_core_storage_volumes(): + """Try to create block devices for any Apple CoreStorage volumes.""" + corestorage_uuid = '53746f72-6167-11aa-aa11-00306543ecac' + dmsetup_cmd_file = '{TmpDir}/dmsetup_command'.format(**global_vars) + + # Get CoreStorage devices + cmd = [ + 'lsblk', '--json', '--list', '--paths', + '--output', 'NAME,PARTTYPE'] + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + devs = json_data.get('blockdevices', []) + devs = [d for d in devs if d.get('parttype', '') == corestorage_uuid] + if devs: + print_standard(' ') + print_standard('Detected CoreStorage partition{}'.format( + '' if len(devs) == 1 else 's')) + print_standard(' Scanning for inner volume(s)....') + + # Search for inner volumes and setup dev mappers + for dev in devs: + dev_path = dev.get('name', '') + if not dev_path: + # Can't setup block device without the dev path + continue + dev_name = re.sub(r'.*/', '', dev_path) + log_path = '{LogDir}/testdisk_{dev_name}.log'.format( + dev_name=dev_name, **global_vars) + + # Run TestDisk + cmd = [ + 'sudo', 'testdisk', + '/logname', log_path, '/debug', '/log', + '/cmd', dev_path, 'partition_none,analyze'] + result = run_program(cmd, check=False) + if result.returncode: + # i.e. return code is non-zero + continue + if not os.path.exists(log_path): + # TestDisk failed to write log + continue + + # Check log for found volumes + cs_vols = {} + with open(log_path, 'r') as f: + for line in f.readlines(): + r = re.match( + r'^.*echo "([^"]+)" . dmsetup create test(\d)$', + line.strip(), + re.IGNORECASE) + if r: + cs_name = 'CoreStorage_{}_{}'.format(dev_name, r.group(2)) + cs_vols[cs_name] = r.group(1) + + # Create mapper device(s) + for name, dm_cmd in sorted(cs_vols.items()): + with open(dmsetup_cmd_file, 'w') as f: + f.write(dm_cmd) + cmd = ['sudo', 'dmsetup', 'create', name, dmsetup_cmd_file] + run_program(cmd, check=False) + def fix_path_sep(path_str): """Replace non-native and duplicate dir separators, returns str.""" return re.sub(r'(\\|/)+', lambda s: os.sep, path_str) @@ -187,14 +248,20 @@ def get_mounted_volumes(): mounted_volumes.extend(item.get('children', [])) return {item['source']: item for item in mounted_volumes} -def mount_all_volumes(): +def mount_volumes(all_devices=True, device_path=None, read_write=False): """Mount all detected filesystems.""" report = {} + cmd = [ + 'lsblk', '--json', '--paths', + '--output', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE'] + if not all_devices and device_path: + # Only mount volumes for specific device + cmd.append(device_path) + else: + # Check for Apple CoreStorage volumes first + find_core_storage_volumes() # Get list of block devices - cmd = [ - 'lsblk', '-J', '-p', - '-o', 'NAME,FSTYPE,LABEL,UUID,PARTTYPE,TYPE,SIZE'] result = run_program(cmd) json_data = json.loads(result.stdout.decode()) devs = json_data.get('blockdevices', []) @@ -233,8 +300,11 @@ def mount_all_volumes(): vol_data['show_data']['warning'] = True else: # Mount volume + cmd = ['udevil', 'mount', + '-o', 'rw' if read_write else 'ro', + vol_path] try: - run_program(['udevil', 'mount', '-o', 'ro', vol_path]) + run_program(cmd) except subprocess.CalledProcessError: vol_data['show_data']['data'] = 'Failed to mount' vol_data['show_data']['error'] = True @@ -242,11 +312,16 @@ def mount_all_volumes(): mounted_volumes = get_mounted_volumes() # Format pretty result string - if vol_data['show_data']['data'] != 'Failed to mount': + if vol_data['show_data']['data'] == 'Failed to mount': + vol_data['mount_point'] = None + else: size_used = human_readable_size( mounted_volumes[vol_path]['used']) size_avail = human_readable_size( mounted_volumes[vol_path]['avail']) + vol_data['size_avail'] = size_avail + vol_data['size_used'] = size_used + vol_data['mount_point'] = mounted_volumes[vol_path]['target'] vol_data['show_data']['data'] = 'Mounted on {}'.format( mounted_volumes[vol_path]['target']) vol_data['show_data']['data'] = '{:40} ({} used, {} free)'.format( diff --git a/.bin/Scripts/functions/ddrescue.py b/.bin/Scripts/functions/ddrescue.py new file mode 100644 index 00000000..cf518ea0 --- /dev/null +++ b/.bin/Scripts/functions/ddrescue.py @@ -0,0 +1,1235 @@ +# Wizard Kit: Functions - ddrescue + +import json +import pathlib +import psutil +import re +import signal +import stat +import time + +from functions.common import * +from functions.data import * +from operator import itemgetter + +# STATIC VARIABLES +AUTO_PASS_1_THRESHOLD = 95 +AUTO_PASS_2_THRESHOLD = 98 +DDRESCUE_SETTINGS = { + '--binary-prefixes': {'Enabled': True, 'Hidden': True}, + '--data-preview': {'Enabled': True, 'Hidden': True, 'Value': '5'}, + '--idirect': {'Enabled': True}, + '--odirect': {'Enabled': True}, + '--max-read-rate': {'Enabled': False, 'Value': '1MiB'}, + '--min-read-rate': {'Enabled': True, 'Value': '64KiB'}, + '--reopen-on-error': {'Enabled': True}, + '--retry-passes': {'Enabled': True, 'Value': '0'}, + '--test-mode': {'Enabled': False, 'Value': 'test.map'}, + '--timeout': {'Enabled': True, 'Value': '5m'}, + '-vvvv': {'Enabled': True, 'Hidden': True}, + } +RECOMMENDED_FSTYPES = ['ext3', 'ext4', 'xfs'] +SIDE_PANE_WIDTH = 21 +USAGE = """ {script_name} clone [source [destination]] + {script_name} image [source [destination]] + (e.g. {script_name} clone /dev/sda /dev/sdb) +""" + + +# Clases +class BaseObj(): + """Base object used by DevObj, DirObj, and ImageObj.""" + def __init__(self, path): + self.type = 'base' + self.parent = None + self.path = os.path.realpath(path) + self.set_details() + + def is_dev(self): + return self.type == 'dev' + + def is_dir(self): + return self.type == 'dir' + + def is_image(self): + return self.type == 'image' + + def self_check(self): + pass + + def set_details(self): + self.details = {} + + +class BlockPair(): + """Object to track data and methods together for source and dest.""" + def __init__(self, mode, source, dest): + self.mode = mode + self.source = source + self.source_path = source.path + self.dest = dest + self.pass_done = [False, False, False] + self.resumed = False + self.rescued = 0 + self.rescued_percent = 0 + self.status = ['Pending', 'Pending', 'Pending'] + self.size = source.size + # Set dest paths + if self.mode == 'clone': + # Cloning + self.dest_path = dest.path + self.map_path = '{pwd}/Clone_{prefix}.map'.format( + pwd=os.path.realpath(global_vars['Env']['PWD']), + prefix=source.prefix) + else: + # Imaging + self.dest_path = '{path}/{prefix}.dd'.format( + path=dest.path, + prefix=source.prefix) + self.map_path = '{path}/{prefix}.map'.format( + path=dest.path, + prefix=source.prefix) + if os.path.exists(self.map_path): + self.load_map_data() + self.resumed = True + self.fix_status_strings() + + def fix_status_strings(self): + """Format status strings via get_formatted_status().""" + for pass_num in [1, 2, 3]: + self.status[pass_num-1] = get_formatted_status( + label='Pass {}'.format(pass_num), + data=self.status[pass_num-1]) + + def finish_pass(self, pass_num): + """Mark pass as done and check if 100% recovered.""" + map_data = read_map_file(self.map_path) + if map_data['full recovery']: + self.pass_done = [True, True, True] + self.rescued = self.size + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num+1), + data=100) + # Mark future passes as Skipped + pass_num += 1 + while pass_num <= 2: + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num+1), + data='Skipped') + pass_num += 1 + else: + self.pass_done[pass_num] = True + + def load_map_data(self): + """Load data from map file and set progress.""" + map_data = read_map_file(self.map_path) + self.rescued_percent = map_data['rescued'] + self.rescued = (self.rescued_percent * self.size) / 100 + if map_data['full recovery']: + self.pass_done = [True, True, True] + self.rescued = self.size + self.status = ['Skipped', 'Skipped', 'Skipped'] + elif map_data['non-tried'] > 0: + # Initial pass incomplete + pass + elif map_data['non-trimmed'] > 0: + self.pass_done = [True, False, False] + self.status = ['Skipped', 'Pending', 'Pending'] + elif map_data['non-scraped'] > 0: + self.pass_done = [True, True, False] + self.status = ['Skipped', 'Skipped', 'Pending'] + else: + self.pass_done = [True, True, True] + self.status = ['Skipped', 'Skipped', 'Skipped'] + + def self_check(self): + """Self check to abort on bad dest/map combinations.""" + dest_exists = os.path.exists(self.dest_path) + map_exists = os.path.exists(self.map_path) + if self.mode == 'image': + if dest_exists and not map_exists: + raise GenericError( + 'Detected image "{}" but not the matching map'.format( + self.dest_path)) + elif not dest_exists and map_exists: + raise GenericError( + 'Detected map "{}" but not the matching image'.format( + self.map_path)) + elif not dest_exists: + raise GenericError('Destination device "{}" missing'.format( + self.dest_path)) + + def update_progress(self, pass_num): + """Update progress using map file.""" + if os.path.exists(self.map_path): + map_data = read_map_file(self.map_path) + self.rescued_percent = map_data.get('rescued', 0) + self.rescued = (self.rescued_percent * self.size) / 100 + self.status[pass_num] = get_formatted_status( + label='Pass {}'.format(pass_num+1), + data=(self.rescued/self.size)*100) + + +class DevObj(BaseObj): + """Block device object.""" + def self_check(self): + """Verify that self.path points to a block device.""" + if not pathlib.Path(self.path).is_block_device(): + raise GenericError('Path "{}" is not a block device.'.format( + self.path)) + if self.parent: + print_warning('"{}" is a child device.'.format(self.path)) + if ask('Use parent device "{}" instead?'.format(self.parent)): + self.path = os.path.realpath(self.parent) + self.set_details() + + def set_details(self): + """Set details via lsblk.""" + self.type = 'dev' + self.details = get_device_details(self.path) + self.name = '{name} {size} {model} {serial}'.format( + name=self.details.get('name', 'UNKNOWN'), + size=self.details.get('size', 'UNKNOWN'), + model=self.details.get('model', 'UNKNOWN'), + serial=self.details.get('serial', 'UNKNOWN')) + self.model = self.details.get('model', 'UNKNOWN') + self.model_size = self.details.get('size', 'UNKNOWN') + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_device_report(self.path) + self.parent = self.details.get('pkname', '') + self.label = self.details.get('label', '') + if not self.label: + # Force empty string in case it's set to None + self.label = '' + self.update_filename_prefix() + + def update_filename_prefix(self): + """Set filename prefix based on details.""" + self.prefix = '{m_size}_{model}'.format( + m_size=self.model_size, + model=self.model) + self.prefix = self.prefix.strip() + if self.parent: + # Add child device details + c_num = self.path.replace(self.parent, '') + self.prefix += '_{c_prefix}{c_num}_{c_size}{sep}{c_label}'.format( + c_prefix='p' if len(c_num) == 1 else '', + c_num=c_num, + c_size=self.details.get('size', 'UNKNOWN'), + sep='_' if self.label else '', + c_label=self.label) + self.prefix = self.prefix.strip().replace(' ', '_') + + +class DirObj(BaseObj): + def self_check(self): + """Verify that self.path points to a directory.""" + if not pathlib.Path(self.path).is_dir(): + raise GenericError('Path "{}" is not a directory.'.format( + self.path)) + + def set_details(self): + """Set details via findmnt.""" + self.type = 'dir' + self.details = get_dir_details(self.path) + self.fstype = self.details.get('fstype', 'UNKNOWN') + self.name = self.path + '/' + self.size = get_size_in_bytes(self.details.get('avail', 'UNKNOWN')) + self.report = get_dir_report(self.path) + + +class ImageObj(BaseObj): + def self_check(self): + """Verify that self.path points to a file.""" + if not pathlib.Path(self.path).is_file(): + raise GenericError('Path "{}" is not an image file.'.format( + self.path)) + + def set_details(self): + """Setup loopback device, set details via lsblk, then detach device.""" + self.type = 'image' + self.loop_dev = setup_loopback_device(self.path) + self.details = get_device_details(self.loop_dev) + self.details['model'] = 'ImageFile' + self.name = '{name} {size}'.format( + name=self.path[self.path.rfind('/')+1:], + size=self.details.get('size', 'UNKNOWN')) + self.prefix = '{}_ImageFile'.format( + self.details.get('size', 'UNKNOWN')) + self.size = get_size_in_bytes(self.details.get('size', 'UNKNOWN')) + self.report = get_device_report(self.loop_dev) + self.report = self.report.replace( + self.loop_dev[self.loop_dev.rfind('/')+1:], '(Img)') + run_program(['losetup', '--detach', self.loop_dev], check=False) + + +class RecoveryState(): + """Object to track BlockPair objects and overall state.""" + def __init__(self, mode, source, dest): + self.mode = mode.lower() + self.source = source + self.source_path = source.path + self.dest = dest + self.block_pairs = [] + self.current_pass = 0 + self.current_pass_str = '0: Initializing' + self.settings = DDRESCUE_SETTINGS.copy() + self.finished = False + self.progress_out = '{}/progress.out'.format(global_vars['LogDir']) + self.rescued = 0 + self.resumed = False + self.started = False + self.total_size = 0 + if mode not in ('clone', 'image'): + raise GenericError('Unsupported mode') + + def add_block_pair(self, source, dest): + """Run safety checks and append new BlockPair to internal list.""" + if self.mode == 'clone': + # Cloning safety checks + if source.is_dir(): + raise GenericError('Invalid source "{}"'.format( + source.path)) + elif not dest.is_dev(): + raise GenericError('Invalid destination "{}"'.format( + dest.path)) + elif source.size > dest.size: + raise GenericError( + 'Destination is too small, refusing to continue.') + else: + # Imaging safety checks + if not source.is_dev(): + raise GenericError('Invalid source "{}"'.format( + source.path)) + elif not dest.is_dir(): + raise GenericError('Invalid destination "{}"'.format( + dest.path)) + elif (source.size * 1.2) > dest.size: + raise GenericError( + 'Not enough free space, refusing to continue.') + elif dest.fstype.lower() not in RECOMMENDED_FSTYPES: + print_error( + 'Destination filesystem "{}" is not recommended.'.format( + dest.fstype.upper())) + print_info('Recommended types are: {}'.format( + ' / '.join(RECOMMENDED_FSTYPES).upper())) + print_standard(' ') + if not ask('Proceed anyways? (Strongly discouraged)'): + raise GenericAbort() + elif not is_writable_dir(dest): + raise GenericError( + 'Destination is not writable, refusing to continue.') + elif not is_writable_filesystem(dest): + raise GenericError( + 'Destination is mounted read-only, refusing to continue.') + + # Safety checks passed + self.block_pairs.append(BlockPair(self.mode, source, dest)) + + def current_pass_done(self): + """Checks if pass is done for all block-pairs, returns bool.""" + done = True + for bp in self.block_pairs: + done &= bp.pass_done[self.current_pass] + return done + + def current_pass_min(self): + """Gets minimum pass rescued percentage, returns float.""" + min_percent = 100 + for bp in self.block_pairs: + min_percent = min(min_percent, bp.rescued_percent) + return min_percent + + def retry_all_passes(self): + """Mark all passes as pending for all block-pairs.""" + self.finished = False + for bp in self.block_pairs: + bp.pass_done = [False, False, False] + bp.status = ['Pending', 'Pending', 'Pending'] + bp.fix_status_strings() + self.set_pass_num() + + def self_checks(self): + """Run self-checks for each BlockPair and update state values.""" + self.total_size = 0 + for bp in self.block_pairs: + bp.self_check() + self.resumed |= bp.resumed + self.total_size += bp.size + + def set_pass_num(self): + """Set current pass based on all block-pair's progress.""" + self.current_pass = 0 + for pass_num in (2, 1, 0): + # Iterate backwards through passes + pass_done = True + for bp in self.block_pairs: + pass_done &= bp.pass_done[pass_num] + if pass_done: + # All block-pairs reported being done + # Set to next pass, unless we're on the last pass (2) + self.current_pass = min(2, pass_num + 1) + if pass_num == 2: + # Also mark overall recovery as finished if on last pass + self.finished = True + break + if self.finished: + self.current_pass_str = '- "Done"' + elif self.current_pass == 0: + self.current_pass_str = '1 "Initial Read"' + elif self.current_pass == 1: + self.current_pass_str = '2 "Trimming bad areas"' + elif self.current_pass == 2: + self.current_pass_str = '3 "Scraping bad areas"' + + def update_progress(self): + """Update overall progress using block_pairs.""" + self.rescued = 0 + for bp in self.block_pairs: + self.rescued += bp.rescued + self.rescued_percent = (self.rescued / self.total_size) * 100 + self.status_percent = get_formatted_status( + label='Recovered:', data=self.rescued_percent) + self.status_amount = get_formatted_status( + label='', data=human_readable_size(self.rescued)) + + +# Functions +def build_outer_panes(state): + """Build top and side panes.""" + clear_screen() + result = run_program(['tput', 'cols']) + width = int( + (int(result.stdout.decode().strip()) - SIDE_PANE_WIDTH) / 2) - 2 + + # Top panes + source_str = state.source.name + if len(source_str) > width: + source_str = '{}...'.format(source_str[:width-3]) + dest_str = state.dest.name + if len(dest_str) > width: + if state.mode == 'clone': + dest_str = '{}...'.format(dest_str[:width-3]) + else: + dest_str = '...{}'.format(dest_str[-width+3:]) + source_pane = tmux_splitw( + '-bdvl', '2', + '-PF', '#D', + 'echo-and-hold "{BLUE}Source{CLEAR}\n{text}"'.format( + text=source_str, + **COLORS)) + tmux_splitw( + '-t', source_pane, + '-dhl', '{}'.format(SIDE_PANE_WIDTH), + 'echo-and-hold "{BLUE}Started{CLEAR}\n{text}"'.format( + text=time.strftime("%Y-%m-%d %H:%M %Z"), + **COLORS)) + tmux_splitw( + '-t', source_pane, + '-dhp', '50', + 'echo-and-hold "{BLUE}Destination{CLEAR}\n{text}"'.format( + text=dest_str, + **COLORS)) + + # Side pane + update_sidepane(state) + tmux_splitw( + '-dhl', str(SIDE_PANE_WIDTH), + 'watch', '--color', '--no-title', '--interval', '1', + 'cat', state.progress_out) + + +def create_path_obj(path): + """Create Dev, Dir, or Image obj based on path given.""" + obj = None + if pathlib.Path(path).is_block_device(): + obj = DevObj(path) + elif pathlib.Path(path).is_dir(): + obj = DirObj(path) + elif pathlib.Path(path).is_file(): + obj = ImageObj(path) + else: + raise GenericError('Invalid path "{}"'.format(path)) + return obj + + +def double_confirm_clone(): + """Display warning and get 2nd confirmation from user, returns bool.""" + print_standard('\nSAFETY CHECK') + print_warning('All data will be DELETED from the ' + 'destination device and partition(s) listed above.') + print_warning('This is irreversible and will lead ' + 'to {CLEAR}{RED}DATA LOSS.'.format(**COLORS)) + return ask('Asking again to confirm, is this correct?') + + +def get_device_details(dev_path): + """Get device details via lsblk, returns JSON dict.""" + try: + cmd = ( + 'lsblk', + '--json', + '--output-all', + '--paths', + dev_path) + result = run_program(cmd) + except CalledProcessError: + # Return empty dict and let calling section deal with the issue + return {} + + json_data = json.loads(result.stdout.decode()) + # Just return the first device (there should only be one) + return json_data['blockdevices'][0] + + +def get_device_report(dev_path): + """Build colored device report using lsblk, returns str.""" + result = run_program([ + 'lsblk', '--nodeps', + '--output', 'NAME,TRAN,TYPE,SIZE,VENDOR,MODEL,SERIAL', + dev_path]) + lines = result.stdout.decode().strip().splitlines() + lines.append('') + + # FS details (if any) + result = run_program([ + 'lsblk', + '--output', 'NAME,SIZE,FSTYPE,LABEL,MOUNTPOINT', + dev_path]) + lines.extend(result.stdout.decode().strip().splitlines()) + + # Color label lines + output = [] + for line in lines: + if line[0:4] == 'NAME': + output.append('{BLUE}{line}{CLEAR}'.format(line=line, **COLORS)) + else: + output.append(line) + + # Done + return '\n'.join(output) + + +def get_dir_details(dir_path): + """Get dir details via findmnt, returns JSON dict.""" + try: + result = run_program([ + 'findmnt', '-J', + '-o', 'SOURCE,TARGET,FSTYPE,OPTIONS,SIZE,AVAIL,USED', + '-T', dir_path]) + json_data = json.loads(result.stdout.decode()) + except Exception: + raise GenericError( + 'Failed to get directory details for "{}".'.format(self.path)) + else: + return json_data['filesystems'][0] + + +def get_dir_report(dir_path): + """Build colored dir report using findmnt, returns str.""" + dir_path = dir_path + output = [] + width = len(dir_path)+1 + result = run_program([ + 'findmnt', + '--output', 'SIZE,AVAIL,USED,FSTYPE,OPTIONS', + '--target', dir_path]) + for line in result.stdout.decode().splitlines(): + if 'FSTYPE' in line: + output.append('{BLUE}{label:<{width}}{line}{CLEAR}'.format( + label='PATH', + width=width, + line=line.replace('\n',''), + **COLORS)) + else: + output.append('{path:<{width}}{line}'.format( + path=dir_path, + width=width, + line=line.replace('\n',''))) + + # Done + return '\n'.join(output) + + +def get_size_in_bytes(s): + """Convert size string from lsblk string to bytes, returns int.""" + s = re.sub(r'(\d+\.?\d*)\s*([KMGTB])B?', r'\1 \2B', s, re.IGNORECASE) + return convert_to_bytes(s) + + +def get_formatted_status(label, data): + """Build status string using provided info, returns str.""" + data_width = SIDE_PANE_WIDTH - len(label) + try: + data_str = '{data:>{data_width}.2f} %'.format( + data=data, + data_width=data_width-2) + except ValueError: + # Assuming non-numeric data + data_str = '{data:>{data_width}}'.format( + data=data, + data_width=data_width) + status = '{label}{s_color}{data_str}{CLEAR}'.format( + label=label, + s_color=get_status_color(data), + data_str=data_str, + **COLORS) + return status + + +def get_status_color(s, t_success=99, t_warn=90): + """Get color based on status, returns str.""" + color = COLORS['CLEAR'] + p_recovered = -1 + try: + p_recovered = float(s) + except ValueError: + # Status is either in lists below or will default to red + pass + + if s in ('Pending',) or str(s)[-2:] in (' b', 'Kb', 'Mb', 'Gb', 'Tb'): + color = COLORS['CLEAR'] + elif s in ('Skipped', 'Unknown'): + color = COLORS['YELLOW'] + elif p_recovered >= t_success: + color = COLORS['GREEN'] + elif p_recovered >= t_warn: + color = COLORS['YELLOW'] + else: + color = COLORS['RED'] + return color + + +def is_writable_dir(dir_obj): + """Check if we have read-write-execute permissions, returns bool.""" + is_ok = True + path_st_mode = os.stat(dir_obj.path).st_mode + is_ok == is_ok and path_st_mode & stat.S_IRUSR + is_ok == is_ok and path_st_mode & stat.S_IWUSR + is_ok == is_ok and path_st_mode & stat.S_IXUSR + return is_ok + + +def is_writable_filesystem(dir_obj): + """Check if filesystem is mounted read-write, returns bool.""" + return 'rw' in dir_obj.details.get('options', '') + + +def menu_ddrescue(source_path, dest_path, run_mode): + """ddrescue menu.""" + source = None + dest = None + if source_path: + source = create_path_obj(source_path) + else: + source = select_device('source') + source.self_check() + if dest_path: + dest = create_path_obj(dest_path) + else: + if run_mode == 'clone': + dest = select_device('destination', skip_device=source) + else: + dest = select_path(skip_device=source) + dest.self_check() + + # Build BlockPairs + state = RecoveryState(run_mode, source, dest) + if run_mode == 'clone': + state.add_block_pair(source, dest) + else: + for part in select_parts(source): + state.add_block_pair(part, dest) + + # Update state + state.self_checks() + state.set_pass_num() + state.update_progress() + + # Confirmations + clear_screen() + show_selection_details(state) + prompt = 'Start {}?'.format(state.mode.replace('e', 'ing')) + if state.resumed: + print_info('Map data detected and loaded.') + prompt = prompt.replace('Start', 'Resume') + if not ask(prompt): + raise GenericAbort() + if state.mode == 'clone' and not double_confirm_clone(): + raise GenericAbort() + + # Main menu + build_outer_panes(state) + menu_main(state) + + # Done + run_program(['tmux', 'kill-window']) + exit_script() + +def menu_main(state): + """Main menu is used to set ddrescue settings.""" + title = '{GREEN}ddrescue TUI: Main Menu{CLEAR}\n\n'.format(**COLORS) + title += '{BLUE}Current pass: {CLEAR}'.format(**COLORS) + + # Build menu + main_options = [ + {'Base Name': 'Auto continue (if recovery % over threshold)', + 'Enabled': True}, + {'Base Name': 'Retry (mark non-rescued sectors "non-tried")', + 'Enabled': False}, + {'Base Name': 'Reverse direction', 'Enabled': False}, + ] + actions = [ + {'Name': 'Start', 'Letter': 'S'}, + {'Name': 'Change settings {YELLOW}(experts only){CLEAR}'.format( + **COLORS), + 'Letter': 'C'}, + {'Name': 'Quit', 'Letter': 'Q', 'CRLF': True}, + ] + + # Show menu + while True: + # Update entries + for opt in main_options: + opt['Name'] = '{} {}'.format( + '[✓]' if opt['Enabled'] else '[ ]', + opt['Base Name']) + + selection = menu_select( + title=title+state.current_pass_str, + main_entries=main_options, + action_entries=actions) + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + main_options[index]['Enabled'] = not main_options[index]['Enabled'] + elif selection == 'S': + # Set settings for pass + pass_settings = [] + for k, v in state.settings.items(): + if not v['Enabled']: + continue + if 'Value' in v: + pass_settings.append('{}={}'.format(k, v['Value'])) + else: + pass_settings.append(k) + for opt in main_options: + if 'Auto' in opt['Base Name']: + auto_run = opt['Enabled'] + if 'Retry' in opt['Base Name'] and opt['Enabled']: + pass_settings.extend(['--retrim', '--try-again']) + state.retry_all_passes() + if 'Reverse' in opt['Base Name'] and opt['Enabled']: + pass_settings.append('--reverse') + # Disable for next pass + if 'Auto' not in opt['Base Name']: + opt['Enabled'] = False + + # Run ddrescue + state.started = False + while auto_run or not state.started: + state.started = True + run_ddrescue(state, pass_settings) + if state.current_pass_done(): + if (state.current_pass == 0 and + state.current_pass_min() < AUTO_PASS_1_THRESHOLD): + auto_run = False + elif (state.current_pass == 1 and + state.current_pass_min() < AUTO_PASS_2_THRESHOLD): + auto_run = False + else: + auto_run = False + state.set_pass_num() + if state.finished: + break + + elif selection == 'C': + menu_settings(state) + elif selection == 'Q': + if state.rescued_percent < 100: + print_warning('Recovery is less than 100%') + if ask('Are you sure you want to quit?'): + break + else: + break + + +def menu_settings(state): + """Change advanced ddrescue settings.""" + title = '{GREEN}ddrescue TUI: Expert Settings{CLEAR}\n\n'.format(**COLORS) + title += '{YELLOW}These settings can cause {CLEAR}'.format(**COLORS) + title += '{RED}MAJOR DAMAGE{CLEAR}{YELLOW} to drives{CLEAR}\n'.format( + **COLORS) + title += 'Please read the manual before making any changes' + + # Build menu + settings = [] + for k, v in sorted(state.settings.items()): + if not v.get('Hidden', False): + settings.append({'Base Name': k, 'Flag': k}) + actions = [{'Name': 'Main Menu', 'Letter': 'M'}] + + # Show menu + while True: + for s in settings: + s['Name'] = '{}{}{}'.format( + s['Base Name'], + ' = ' if 'Value' in state.settings[s['Flag']] else '', + state.settings[s['Flag']].get('Value', '')) + if not state.settings[s['Flag']]['Enabled']: + s['Name'] = '{YELLOW}{name} (Disabled){CLEAR}'.format( + name=s['Name'], + **COLORS) + selection = menu_select( + title=title, + main_entries=settings, + action_entries=actions) + if selection.isnumeric(): + index = int(selection) - 1 + flag = settings[index]['Flag'] + enabled = state.settings[flag]['Enabled'] + if 'Value' in state.settings[flag]: + answer = choice( + choices=['T', 'C'], + prompt='Toggle or change value for "{}"'.format(flag)) + if answer == 'T': + # Toggle + state.settings[flag]['Enabled'] = not enabled + else: + # Update value + state.settings[flag]['Value'] = get_simple_string( + prompt='Enter new value') + else: + state.settings[flag]['Enabled'] = not enabled + elif selection == 'M': + break + + +def read_map_file(map_path): + """Read map file with ddrescuelog and return data as dict.""" + map_data = {'full recovery': False} + try: + result = run_program(['ddrescuelog', '-t', map_path]) + except CalledProcessError: + # (Grossly) assuming map_data hasn't been saved yet, return empty dict + return map_data + + # Parse output + for line in result.stdout.decode().splitlines(): + m = re.match( + r'^\s*(?P\S+):.*\(\s*(?P\d+\.?\d*)%.*', line.strip()) + if m: + try: + map_data[m.group('key')] = float(m.group('value')) + except ValueError: + raise GenericError('Failed to read map data') + m = re.match(r'.*current status:\s+(?P.*)', line.strip()) + if m: + map_data['pass completed'] = bool(m.group('status') == 'finished') + + # Check if 100% done + try: + run_program(['ddrescuelog', '-D', map_path]) + except CalledProcessError: + map_data['full recovery'] = False + else: + map_data['full recovery'] = True + + return map_data + + +def run_ddrescue(state, pass_settings): + """Run ddrescue pass.""" + return_code = None + + if state.finished: + clear_screen() + print_warning('Recovery already completed?') + pause('Press Enter to return to main menu...') + return + + # Set heights + # NOTE: 12/33 is based on min heights for SMART/ddrescue panes (12+22+1sep) + result = run_program(['tput', 'lines']) + height = int(result.stdout.decode().strip()) + height_smart = int(height * (8 / 33)) + height_journal = int(height * (4 / 33)) + height_ddrescue = height - height_smart - height_journal + + # Show SMART status + smart_pane = tmux_splitw( + '-bdvl', str(height_smart), + '-PF', '#D', + 'watch', '--color', '--no-title', '--interval', '300', + 'ddrescue-tui-smart-display', state.source_path) + + # Show systemd journal output + journal_pane = tmux_splitw( + '-dvl', str(height_journal), + '-PF', '#D', + 'journalctl', '-f') + + # Run pass for each block-pair + for bp in state.block_pairs: + if bp.pass_done[state.current_pass]: + # Skip to next block-pair + continue + update_sidepane(state) + + # Set ddrescue cmd + cmd = [ + 'ddrescue', *pass_settings, + bp.source_path, bp.dest_path, bp.map_path] + if state.mode == 'clone': + cmd.append('--force') + if state.current_pass == 0: + cmd.extend(['--no-trim', '--no-scrape']) + elif state.current_pass == 1: + # Allow trimming + cmd.append('--no-scrape') + elif state.current_pass == 2: + # Allow trimming and scraping + pass + + # Start ddrescue + try: + clear_screen() + print_info('Current dev: {}'.format(bp.source_path)) + ddrescue_proc = popen_program(cmd) + while True: + bp.update_progress(state.current_pass) + update_sidepane(state) + try: + ddrescue_proc.wait(timeout=10) + sleep(2) + bp.update_progress(state.current_pass) + update_sidepane(state) + break + except subprocess.TimeoutExpired: + # Catch to update bp/sidepane + pass + except KeyboardInterrupt: + # Catch user abort + pass + + # Update progress/sidepane again + bp.update_progress(state.current_pass) + update_sidepane(state) + + # Was ddrescue aborted? + return_code = ddrescue_proc.poll() + if return_code is None or return_code is 130: + clear_screen() + print_warning('Aborted') + break + elif return_code: + # i.e. not None and not 0 + print_error('Error(s) encountered, see message above.') + break + else: + # Mark pass finished + bp.finish_pass(state.current_pass) + update_sidepane(state) + + # Done + if str(return_code) != '0': + # Pause on errors + pause('Press Enter to return to main menu... ') + run_program(['tmux', 'kill-pane', '-t', smart_pane]) + run_program(['tmux', 'kill-pane', '-t', journal_pane]) + + +def select_parts(source_device): + """Select partition(s) or whole device, returns list of DevObj()s.""" + selected_parts = [] + children = source_device.details.get('children', []) + + if not children: + # No partitions detected, auto-select whole device. + selected_parts = [source_device] + else: + # Build menu + dev_options = [{ + 'Base Name': '{:<14}(Whole device)'.format(source_device.path), + 'Dev': source_device, + 'Selected': True}] + for c_details in children: + dev_options.append({ + 'Base Name': '{:<14}({:>6} {})'.format( + c_details['name'], + c_details['size'], + c_details['fstype'] if c_details['fstype'] else 'Unknown'), + 'Details': c_details, + 'Dev': DevObj(c_details['name']), + 'Selected': False}) + actions = [ + {'Name': 'Proceed', 'Letter': 'P'}, + {'Name': 'Quit', 'Letter': 'Q'}] + + # Show menu + while True: + one_or_more_devs_selected = False + # Update entries + for dev in dev_options: + if dev['Selected']: + one_or_more_devs_selected = True + dev['Name'] = '* {}'.format(dev['Base Name']) + else: + dev['Name'] = ' {}'.format(dev['Base Name']) + + selection = menu_select( + title='Please select part(s) to image', + main_entries=dev_options, + action_entries=actions) + + if selection.isnumeric(): + # Toggle selection + index = int(selection) - 1 + dev_options[index]['Selected'] = not dev_options[index]['Selected'] + + # Deselect whole device if child selected (this round) + if index > 0: + dev_options[0]['Selected'] = False + + # Deselect all children if whole device selected + if dev_options[0]['Selected']: + for dev in dev_options[1:]: + dev['Selected'] = False + elif selection == 'P' and one_or_more_devs_selected: + break + elif selection == 'Q': + raise GenericAbort() + + # Build list of selected parts + for d in dev_options: + if d['Selected']: + d['Dev'].model = source_device.model + d['Dev'].model_size = source_device.model_size + d['Dev'].update_filename_prefix() + selected_parts.append(d['Dev']) + + return selected_parts + + +def select_path(skip_device=None): + """Optionally mount local dev and select path, returns DirObj.""" + wd = os.path.realpath(global_vars['Env']['PWD']) + selected_path = None + + # Build menu + path_options = [ + {'Name': 'Current directory: {}'.format(wd), 'Path': wd}, + {'Name': 'Local device', 'Path': None}, + {'Name': 'Enter manually', 'Path': None}] + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + + # Show Menu + selection = menu_select( + title='Please make a selection', + main_entries=path_options, + action_entries=actions) + + if selection == 'Q': + raise GenericAbort() + elif selection.isnumeric(): + index = int(selection) - 1 + if path_options[index]['Path'] == wd: + # Current directory + selected_path = DirObj(wd) + + elif path_options[index]['Name'] == 'Local device': + # Local device + local_device = select_device( + skip_device=skip_device) + s_path = '' + + # Mount device volume(s) + report = mount_volumes( + all_devices=False, + device_path=local_device.path, + read_write=True) + + # Select volume + vol_options = [] + for k, v in sorted(report.items()): + disabled = v['show_data']['data'] == 'Failed to mount' + if disabled: + name = '{name} (Failed to mount)'.format(**v) + else: + name = '{name} (mounted on "{mount_point}")'.format(**v) + vol_options.append({ + 'Name': name, + 'Path': v['mount_point'], + 'Disabled': disabled}) + selection = menu_select( + title='Please select a volume', + main_entries=vol_options, + action_entries=actions) + if selection.isnumeric(): + s_path = vol_options[int(selection)-1]['Path'] + elif selection == 'Q': + raise GenericAbort() + + # Create folder + if ask('Create ticket folder?'): + ticket_folder = get_simple_string('Please enter folder name') + s_path = os.path.join(s_path, ticket_folder) + try: + os.makedirs(s_path, exist_ok=True) + except OSError: + raise GenericError( + 'Failed to create folder "{}"'.format(s_path)) + + # Create DirObj + selected_path = DirObj(s_path) + + elif path_options[index]['Name'] == 'Enter manually': + # Manual entry + while not selected_path: + manual_path = input('Please enter path: ').strip() + if manual_path and pathlib.Path(manual_path).is_dir(): + selected_path = DirObj(manual_path) + elif manual_path and pathlib.Path(manual_path).is_file(): + print_error('File "{}" exists'.format(manual_path)) + else: + print_error('Invalid path "{}"'.format(manual_path)) + return selected_path + + +def select_device(description='device', skip_device=None): + """Select device via a menu, returns DevObj.""" + cmd = ( + 'lsblk', + '--json', + '--nodeps', + '--output-all', + '--paths') + result = run_program(cmd) + json_data = json.loads(result.stdout.decode()) + skip_names = [] + if skip_device: + skip_names.append(skip_device.path) + if skip_device.parent: + skip_names.append(skip_device.parent) + + # Build menu + dev_options = [] + for dev in json_data['blockdevices']: + # Disable dev if in skip_names + disabled = dev['name'] in skip_names or dev['pkname'] in skip_names + + # Add to options + dev_options.append({ + 'Name': '{name:12} {tran:5} {size:6} {model} {serial}'.format( + name=dev['name'], + tran=dev['tran'] if dev['tran'] else '', + size=dev['size'] if dev['size'] else '', + model=dev['model'] if dev['model'] else '', + serial=dev['serial'] if dev['serial'] else ''), + 'Dev': DevObj(dev['name']), + 'Disabled': disabled}) + dev_options = sorted(dev_options, key=itemgetter('Name')) + if not dev_options: + raise GenericError('No devices available.') + + # Show Menu + actions = [{'Name': 'Quit', 'Letter': 'Q'}] + selection = menu_select( + title='Please select the {} device'.format(description), + main_entries=dev_options, + action_entries=actions, + disabled_label='ALREADY SELECTED') + + if selection.isnumeric(): + return dev_options[int(selection)-1]['Dev'] + elif selection == 'Q': + raise GenericAbort() + + +def setup_loopback_device(source_path): + """Setup a loopback device for source_path, returns dev_path as str.""" + cmd = ( + 'losetup', + '--find', + '--partscan', + '--show', + source_path) + try: + out = run_program(cmd, check=True) + dev_path = out.stdout.decode().strip() + sleep(1) + except CalledProcessError: + raise GenericError('Failed to setup loopback device for source.') + else: + return dev_path + + +def show_selection_details(state): + """Show selection details.""" + # Source + print_success('Source') + print_standard(state.source.report) + print_standard(' ') + + # Destination + if state.mode == 'clone': + print_success('Destination ', end='') + print_error('(ALL DATA WILL BE DELETED)', timestamp=False) + else: + print_success('Destination') + print_standard(state.dest.report) + print_standard(' ') + + +def show_usage(script_name): + print_info('Usage:') + print_standard(USAGE.format(script_name=script_name)) + pause() + + +def tmux_splitw(*args): + """Run tmux split-window command and return output as str.""" + cmd = ['tmux', 'split-window', *args] + result = run_program(cmd) + return result.stdout.decode().strip() + + +def update_sidepane(state): + """Update progress file for side pane.""" + output = [] + state.update_progress() + if state.mode == 'clone': + output.append(' {BLUE}Cloning Status{CLEAR}'.format(**COLORS)) + else: + output.append(' {BLUE}Imaging Status{CLEAR}'.format(**COLORS)) + output.append('─────────────────────') + + # Overall progress + output.append('{BLUE}Overall Progress{CLEAR}'.format(**COLORS)) + output.append(state.status_percent) + output.append(state.status_amount) + output.append('─────────────────────') + + # Source(s) progress + for bp in state.block_pairs: + if state.source.is_image(): + output.append('{BLUE}Image File{CLEAR}'.format(**COLORS)) + else: + output.append('{BLUE}{source}{CLEAR}'.format( + source=bp.source_path, + **COLORS)) + output.extend(bp.status) + output.append(' ') + + # Add line-endings + output = ['{}\n'.format(line) for line in output] + + with open(state.progress_out, 'w') as f: + f.writelines(output) + + +if __name__ == '__main__': + print("This file is not meant to be called directly.") + +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/functions/hw_diags.py b/.bin/Scripts/functions/hw_diags.py index 63942ed2..3b8bd354 100644 --- a/.bin/Scripts/functions/hw_diags.py +++ b/.bin/Scripts/functions/hw_diags.py @@ -1,6 +1,7 @@ # Wizard Kit: Functions - HW Diagnostics import json +import time from functions.common import * @@ -19,11 +20,37 @@ ATTRIBUTES = { 184: {'Error': 1}, 187: {'Warning': 1}, 188: {'Warning': 1}, + 196: {'Warning': 1, 'Error': 10, 'Ignore': True}, 197: {'Error': 1}, 198: {'Error': 1}, + 199: {'Error': 1, 'Ignore': True}, 201: {'Warning': 1}, }, } +IO_VARS = { + 'Block Size': 512*1024, + 'Chunk Size': 16*1024**2, + 'Minimum Dev Size': 8*1024**3, + 'Minimum Test Size': 10*1024**3, + 'Alt Test Size Factor': 0.01, + 'Progress Refresh Rate': 5, + 'Scale 16': [2**(0.6*x)+(16*x) for x in range(1,17)], + 'Scale 32': [2**(0.6*x/2)+(16*x/2) for x in range(1,33)], + 'Threshold Fail': 65*1024**2, + 'Threshold Warn': 135*1024**2, + 'Threshold Great': 750*1024**2, + 'Graph Horizontal': ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'), + 'Graph Horizontal Width': 40, + 'Graph Vertical': ( + '▏', '▎', '▍', '▌', + '▋', '▊', '▉', '█', + '█▏', '█▎', '█▍', '█▌', + '█▋', '█▊', '█▉', '██', + '██▏', '██▎', '██▍', '██▌', + '██▋', '██▊', '██▉', '███', + '███▏', '███▎', '███▍', '███▌', + '███▋', '███▊', '███▉', '████'), + } TESTS = { 'Prime95': { 'Enabled': False, @@ -46,6 +73,45 @@ TESTS = { }, } +def generate_horizontal_graph(rates): + """Generate two-line horizontal graph from rates, returns str.""" + line_top = '' + line_bottom = '' + for r in rates: + step = get_graph_step(r, scale=16) + + # Set color + r_color = COLORS['CLEAR'] + if r < IO_VARS['Threshold Fail']: + r_color = COLORS['RED'] + elif r < IO_VARS['Threshold Warn']: + r_color = COLORS['YELLOW'] + elif r > IO_VARS['Threshold Great']: + r_color = COLORS['GREEN'] + + # Build graph + if step < 8: + line_top += ' ' + line_bottom += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step]) + else: + line_top += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][step-8]) + line_bottom += '{}{}'.format(r_color, IO_VARS['Graph Horizontal'][-1]) + line_top += COLORS['CLEAR'] + line_bottom += COLORS['CLEAR'] + return '{}\n{}'.format(line_top, line_bottom) + +def get_graph_step(rate, scale=16): + """Get graph step based on rate and scale, returns int.""" + m_rate = rate / (1024**2) + step = 0 + scale_name = 'Scale {}'.format(scale) + for x in range(scale-1, -1, -1): + # Iterate over scale backwards + if m_rate >= IO_VARS[scale_name][x]: + step = x + break + return step + def get_read_rate(s): """Get read rate in bytes/s from dd progress output.""" real_rate = None @@ -56,7 +122,9 @@ def get_read_rate(s): def get_smart_details(dev): """Get SMART data for dev if possible, returns dict.""" - cmd = 'sudo smartctl --all --json /dev/{}'.format(dev).split() + cmd = 'sudo smartctl --all --json {}{}'.format( + '' if '/dev/' in dev else '/dev/', + dev).split() result = run_program(cmd, check=False) try: return json.loads(result.stdout.decode()) @@ -64,12 +132,21 @@ def get_smart_details(dev): # Let other sections deal with the missing data return {} +def get_smart_value(smart_data, smart_id): + """Get SMART value from table, returns int or None.""" + value = None + table = smart_data.get('ata_smart_attributes', {}).get('table', []) + for row in table: + if str(row.get('id', '?')) == str(smart_id): + value = row.get('raw', {}).get('value', None) + return value + def get_status_color(s): """Get color based on status, returns str.""" color = COLORS['CLEAR'] - if s in ['Denied', 'NS', 'OVERRIDE', 'Unknown']: + if s in ['Denied', 'NS', 'OVERRIDE']: color = COLORS['RED'] - elif s in ['Aborted', 'Working', 'Skipped']: + elif s in ['Aborted', 'Unknown', 'Working', 'Skipped']: color = COLORS['YELLOW'] elif s in ['CS']: color = COLORS['GREEN'] @@ -147,9 +224,9 @@ def menu_diags(*args): 'pipes -t 0 -t 1 -t 2 -t 3 -p 5 -R -r 4000'.split(), check=False, pipe=False) elif selection == 'R': - run_program(['reboot']) + run_program(['systemctl', 'reboot']) elif selection == 'S': - run_program(['poweroff']) + run_program(['systemctl', 'poweroff']) elif selection == 'Q': break @@ -194,18 +271,21 @@ def run_badblocks(): print_standard('Done', timestamp=False) # Check results - with open(progress_file, 'r') as f: - text = f.read() - TESTS['badblocks']['Results'][name] = text - r = re.search(r'Pass completed.*0/0/0 errors', text) - if r: - TESTS['badblocks']['Status'][name] = 'CS' - else: - TESTS['badblocks']['Status'][name] = 'NS' + if os.path.exists(progress_file): + with open(progress_file, 'r') as f: + text = f.read() + TESTS['badblocks']['Results'][name] = text + r = re.search(r'Pass completed.*0/0/0 errors', text) + if r: + TESTS['badblocks']['Status'][name] = 'CS' + else: + TESTS['badblocks']['Status'][name] = 'NS' - # Move temp file - shutil.move(progress_file, '{}/badblocks-{}.log'.format( - global_vars['LogDir'], name)) + # Move temp file + shutil.move(progress_file, '{}/badblocks-{}.log'.format( + global_vars['LogDir'], name)) + else: + TESTS['badblocks']['Status'][name] = 'NS' update_progress() # Done @@ -249,32 +329,120 @@ def run_iobenchmark(): TESTS['iobenchmark']['Status'][name] = 'Working' update_progress() print_standard(' /dev/{:11} '.format(name+'...'), end='', flush=True) - run_program('tmux split-window -dl 5 {} {} {}'.format( - 'hw-diags-iobenchmark', - '/dev/{}'.format(name), - progress_file).split()) - wait_for_process('dd') + + # Get dev size + cmd = 'sudo lsblk -bdno size /dev/{}'.format(name) + try: + result = run_program(cmd.split()) + dev_size = result.stdout.decode().strip() + dev_size = int(dev_size) + except: + # Failed to get dev size, requires manual testing instead + TESTS['iobenchmark']['Status'][name] = 'Unknown' + continue + if dev_size < IO_VARS['Minimum Dev Size']: + TESTS['iobenchmark']['Status'][name] = 'Unknown' + continue + + # Calculate dd values + ## test_size is the area to be read in bytes + ## If the dev is < 10Gb then it's the whole dev + ## Otherwise it's the larger of 10Gb or 1% of the dev + ## + ## test_chunks is the number of groups of "Chunk Size" in test_size + ## This number is reduced to a multiple of the graph width in + ## order to allow for the data to be condensed cleanly + ## + ## skip_blocks is the number of "Block Size" groups not tested + ## skip_count is the number of blocks to skip per test_chunk + ## skip_extra is how often to add an additional skip block + ## This is needed to ensure an even testing across the dev + ## This is calculated by using the fractional amount left off + ## of the skip_count variable + test_size = min(IO_VARS['Minimum Test Size'], dev_size) + test_size = max( + test_size, dev_size*IO_VARS['Alt Test Size Factor']) + test_chunks = int(test_size // IO_VARS['Chunk Size']) + test_chunks -= test_chunks % IO_VARS['Graph Horizontal Width'] + test_size = test_chunks * IO_VARS['Chunk Size'] + skip_blocks = int((dev_size - test_size) // IO_VARS['Block Size']) + skip_count = int((skip_blocks / test_chunks) // 1) + skip_extra = 0 + try: + skip_extra = 1 + int(1 / ((skip_blocks / test_chunks) % 1)) + except ZeroDivisionError: + # skip_extra == 0 is fine + pass + + # Open dd progress pane after initializing file + with open(progress_file, 'w') as f: + f.write('') + sleep(1) + cmd = 'tmux split-window -dp 75 -PF #D tail -f {}'.format( + progress_file) + result = run_program(cmd.split()) + bottom_pane = result.stdout.decode().strip() + + # Run dd read tests + offset = 0 + read_rates = [] + for i in range(test_chunks): + i += 1 + s = skip_count + c = int(IO_VARS['Chunk Size'] / IO_VARS['Block Size']) + if skip_extra and i % skip_extra == 0: + s += 1 + cmd = 'sudo dd bs={b} skip={s} count={c} if=/dev/{n} of={o}'.format( + b=IO_VARS['Block Size'], + s=offset+s, + c=c, + n=name, + o='/dev/null') + result = run_program(cmd.split()) + result_str = result.stderr.decode().replace('\n', '') + read_rates.append(get_read_rate(result_str)) + if i % IO_VARS['Progress Refresh Rate'] == 0: + # Update vertical graph + update_io_progress( + percent=i/test_chunks*100, + rate=read_rates[-1], + progress_file=progress_file) + # Update offset + offset += s + c print_standard('Done', timestamp=False) - # Check results - with open(progress_file, 'r') as f: - text = f.read() - io_stats = text.replace('\r', '\n').split('\n') - try: - io_stats = [get_read_rate(s) for s in io_stats] - io_stats = [float(s/1048576) for s in io_stats if s] - TESTS['iobenchmark']['Results'][name] = 'Read speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( - sum(io_stats) / len(io_stats), - min(io_stats), - max(io_stats)) - TESTS['iobenchmark']['Status'][name] = 'CS' - except: - # Requires manual testing - TESTS['iobenchmark']['Status'][name] = 'NS' + # Close bottom pane + run_program(['tmux', 'kill-pane', '-t', bottom_pane]) - # Move temp file - shutil.move(progress_file, '{}/iobenchmark-{}.log'.format( - global_vars['LogDir'], name)) + # Build report + h_graph_rates = [] + pos = 0 + width = int(test_chunks / IO_VARS['Graph Horizontal Width']) + for i in range(IO_VARS['Graph Horizontal Width']): + # Append average rate for WIDTH number of rates to new array + h_graph_rates.append(sum(read_rates[pos:pos+width])/width) + pos += width + report = generate_horizontal_graph(h_graph_rates) + report += '\nRead speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format( + sum(read_rates)/len(read_rates)/(1024**2), + min(read_rates)/(1024**2), + max(read_rates)/(1024**2)) + TESTS['iobenchmark']['Results'][name] = report + + # Set CS/NS + if min(read_rates) <= IO_VARS['Threshold Fail']: + TESTS['iobenchmark']['Status'][name] = 'NS' + elif min(read_rates) <= IO_VARS['Threshold Warn']: + TESTS['iobenchmark']['Status'][name] = 'Unknown' + else: + TESTS['iobenchmark']['Status'][name] = 'CS' + + # Save logs + dest_filename = '{}/iobenchmark-{}.log'.format(global_vars['LogDir'], name) + shutil.move(progress_file, dest_filename) + with open(dest_filename.replace('.', '-raw.'), 'a') as f: + for rate in read_rates: + f.write('{} MB/s\n'.format(rate/(1024**2))) update_progress() # Done @@ -284,7 +452,6 @@ def run_iobenchmark(): def run_mprime(): """Run Prime95 for MPRIME_LIMIT minutes while showing the temps.""" aborted = False - clear_screen() print_log('\nStart Prime95 test') TESTS['Prime95']['Status'] = 'Working' update_progress() @@ -299,12 +466,15 @@ def run_mprime(): # Start test run_program(['apple-fans', 'max']) - print_standard('Running Prime95 for {} minutes'.format(MPRIME_LIMIT)) - print_warning('If running too hot, press CTL+c to abort the test') try: - sleep(int(MPRIME_LIMIT)*60) + for i in range(int(MPRIME_LIMIT)): + clear_screen() + print_standard('Running Prime95 ({} minutes left)'.format( + int(MPRIME_LIMIT)-i)) + print_warning('If running too hot, press CTRL+c to abort the test') + sleep(60) except KeyboardInterrupt: - # Catch CTL+C + # Catch CTRL+C aborted = True # Save "final" temps @@ -479,6 +649,8 @@ def run_tests(tests): # Initialize if TESTS['NVMe/SMART']['Enabled'] or TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: + print_standard(' ') + print_standard('Scanning disks...') scan_disks() update_progress() @@ -509,12 +681,17 @@ def run_tests(tests): global_vars['LogFile'])) pause('Press Enter to exit...') -def scan_disks(): +def scan_disks(full_paths=False, only_path=None): """Scan for disks eligible for hardware testing.""" clear_screen() # Get eligible disk list - result = run_program(['lsblk', '-J', '-O']) + cmd = ['lsblk', '-J', '-O'] + if full_paths: + cmd.append('-p') + if only_path: + cmd.append(only_path) + result = run_program(cmd) json_data = json.loads(result.stdout.decode()) devs = {} for d in json_data.get('blockdevices', []): @@ -536,13 +713,18 @@ def scan_disks(): for dev, data in devs.items(): # Get SMART attributes run_program( - cmd = 'sudo smartctl -s on /dev/{}'.format(dev).split(), + cmd = 'sudo smartctl -s on {}{}'.format( + '' if full_paths else '/dev/', + dev).split(), check = False) data['smartctl'] = get_smart_details(dev) # Get NVMe attributes if data['lsblk']['tran'] == 'nvme': cmd = 'sudo nvme smart-log /dev/{} -o json'.format(dev).split() + cmd = 'sudo nvme smart-log {}{} -o json'.format( + '' if full_paths else '/dev/', + dev).split() result = run_program(cmd, check=False) try: data['nvme-cli'] = json.loads(result.stdout.decode()) @@ -571,36 +753,50 @@ def scan_disks(): data['SMART Support'] = False # Ask for manual overrides if necessary - if not data['Quick Health OK'] and (TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']): + if TESTS['badblocks']['Enabled'] or TESTS['iobenchmark']['Enabled']: show_disk_details(data) - print_warning("WARNING: Health can't be confirmed for: {}".format( - '/dev/{}'.format(dev))) - dev_name = data['lsblk']['name'] - print_standard(' ') - if ask('Run tests on this device anyway?'): - TESTS['NVMe/SMART']['Status'][dev_name] = 'OVERRIDE' - else: - TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' - TESTS['badblocks']['Status'][dev_name] = 'Denied' - TESTS['iobenchmark']['Status'][dev_name] = 'Denied' - print_standard(' ') # In case there's more than one "OVERRIDE" disk + needs_override = False + if not data['Quick Health OK']: + needs_override = True + print_warning( + "WARNING: Health can't be confirmed for: /dev/{}".format(dev)) + if get_smart_value(data['smartctl'], '199'): + # SMART attribute present and it's value is non-zero + needs_override = True + print_warning( + 'WARNING: SMART 199/C7 error detected on /dev/{}'.format(dev)) + print_standard(' (Have you tried swapping the drive cable?)') + if needs_override: + dev_name = data['lsblk']['name'] + print_standard(' ') + if ask('Run tests on this device anyway?'): + TESTS['NVMe/SMART']['Status'][dev_name] = 'OVERRIDE' + else: + TESTS['NVMe/SMART']['Status'][dev_name] = 'NS' + TESTS['badblocks']['Status'][dev_name] = 'Denied' + TESTS['iobenchmark']['Status'][dev_name] = 'Denied' + print_standard(' ') # In case there's more than one "OVERRIDE" disk TESTS['NVMe/SMART']['Devices'] = devs TESTS['badblocks']['Devices'] = devs TESTS['iobenchmark']['Devices'] = devs + return devs -def show_disk_details(dev): +def show_disk_details(dev, only_attributes=False): """Display disk details.""" dev_name = dev['lsblk']['name'] - # Device description - print_info('Device: /dev/{}'.format(dev['lsblk']['name'])) - print_standard(' {:>4} ({}) {} {}'.format( - str(dev['lsblk'].get('size', '???b')).strip(), - str(dev['lsblk'].get('tran', '???')).strip().upper().replace( - 'NVME', 'NVMe'), - str(dev['lsblk'].get('model', 'Unknown Model')).strip(), - str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(), - )) + if not only_attributes: + # Device description + print_info('Device: {}{}'.format( + '' if '/dev/' in dev['lsblk']['name'] else '/dev/', + dev['lsblk']['name'])) + print_standard(' {:>4} ({}) {} {}'.format( + str(dev['lsblk'].get('size', '???b')).strip(), + str(dev['lsblk'].get('tran', '???')).strip().upper().replace( + 'NVME', 'NVMe'), + str(dev['lsblk'].get('model', 'Unknown Model')).strip(), + str(dev['lsblk'].get('serial', 'Unknown Serial')).strip(), + )) # Warnings if dev.get('NVMe Disk', False): @@ -615,7 +811,12 @@ def show_disk_details(dev): # Attributes if dev.get('NVMe Disk', False): - print_info('Attributes:') + if only_attributes: + print_info('SMART Attributes:', end='') + print_warning(' Updated: {}'.format( + time.strftime('%Y-%m-%d %H:%M %Z'))) + else: + print_info('Attributes:') for attrib, threshold in sorted(ATTRIBUTES['NVMe'].items()): if attrib in dev['nvme-cli']: print_standard( @@ -636,7 +837,12 @@ def show_disk_details(dev): print_success(raw_str, timestamp=False) elif dev['smartctl'].get('ata_smart_attributes', None): # SMART attributes - print_info('Attributes:') + if only_attributes: + print_info('SMART Attributes:', end='') + print_warning(' Updated: {}'.format( + time.strftime('%Y-%m-%d %H:%M %Z'))) + else: + print_info('Attributes:') s_table = dev['smartctl'].get('ata_smart_attributes', {}).get( 'table', {}) s_table = {a.get('id', 'Unknown'): a for a in s_table} @@ -730,13 +936,38 @@ def show_results(): and io_status not in ['Denied', 'OVERRIDE', 'Skipped']): print_info('Benchmark:') result = TESTS['iobenchmark']['Results'].get(name, '') - print_standard(' {}'.format(result)) + for line in result.split('\n'): + print_standard(' {}'.format(line)) print_standard(' ') # Done pause('Press Enter to return to main menu... ') run_program('tmux kill-pane -a'.split()) +def update_io_progress(percent, rate, progress_file): + """Update I/O progress file.""" + bar_color = COLORS['CLEAR'] + rate_color = COLORS['CLEAR'] + step = get_graph_step(rate, scale=32) + if rate < IO_VARS['Threshold Fail']: + bar_color = COLORS['RED'] + rate_color = COLORS['YELLOW'] + elif rate < IO_VARS['Threshold Warn']: + bar_color = COLORS['YELLOW'] + rate_color = COLORS['YELLOW'] + elif rate > IO_VARS['Threshold Great']: + bar_color = COLORS['GREEN'] + rate_color = COLORS['GREEN'] + line = ' {p:5.1f}% {b_color}{b:<4} {r_color}{r:6.1f} Mb/s{c}\n'.format( + p=percent, + b_color=bar_color, + b=IO_VARS['Graph Vertical'][step], + r_color=rate_color, + r=rate/(1024**2), + c=COLORS['CLEAR']) + with open(progress_file, 'a') as f: + f.write(line) + def update_progress(): """Update progress file.""" if 'Progress Out' not in TESTS: @@ -792,3 +1023,4 @@ def update_progress(): if __name__ == '__main__': print("This file is not meant to be called directly.") +# vim: sts=4 sw=4 ts=4 diff --git a/.bin/Scripts/functions/network.py b/.bin/Scripts/functions/network.py index 0d6beb3a..d040e343 100644 --- a/.bin/Scripts/functions/network.py +++ b/.bin/Scripts/functions/network.py @@ -3,6 +3,7 @@ ## Wizard Kit: Functions - Network import os +import shutil import sys # Init @@ -26,13 +27,8 @@ def connect_to_network(): if is_connected(): return - # LAN - if 'en' in net_ifs: - # Reload the tg3/broadcom driver (known fix for some Dell systems) - try_and_print(message='Reloading drivers...', function=reload_tg3) - # WiFi - if not is_connected() and 'wl' in net_ifs: + if 'wl' in net_ifs: cmd = [ 'nmcli', 'dev', 'wifi', 'connect', WIFI_SSID, @@ -41,7 +37,7 @@ def connect_to_network(): message = 'Connecting to {}...'.format(WIFI_SSID), function = run_program, cmd = cmd) - + def is_connected(): """Check for a valid private IP.""" devs = psutil.net_if_addrs() @@ -71,13 +67,6 @@ def speedtest(): output = [(a, float(b), c) for a, b, c in output] return ['{:10}{:6.2f} {}'.format(*line) for line in output] -def reload_tg3(): - """Reload tg3 module as a workaround for some Dell systems.""" - run_program(['sudo', 'modprobe', '-r', 'tg3']) - run_program(['sudo', 'modprobe', 'broadcom']) - run_program(['sudo', 'modprobe', 'tg3']) - sleep(5) - if __name__ == '__main__': print("This file is not meant to be called directly.") diff --git a/.bin/Scripts/functions/setup.py b/.bin/Scripts/functions/setup.py index d08692b5..4fca0303 100644 --- a/.bin/Scripts/functions/setup.py +++ b/.bin/Scripts/functions/setup.py @@ -5,6 +5,9 @@ from functions.common import * # STATIC VARIABLES HKCU = winreg.HKEY_CURRENT_USER HKLM = winreg.HKEY_LOCAL_MACHINE +MOZILLA_FIREFOX_UBO_PATH = r'{}\{}\ublock_origin.xpi'.format( + os.environ.get('PROGRAMFILES'), + r'Mozilla Firefox\distribution\extensions') OTHER_RESULTS = { 'Error': { 'CalledProcessError': 'Unknown Error', @@ -76,9 +79,6 @@ SETTINGS_EXPLORER_USER = { }, } SETTINGS_GOOGLE_CHROME = { - r'Software\Google\Chrome\Extensions': { - 'WOW64_32': True, - }, r'Software\Google\Chrome\Extensions\cjpalhdlnbpafiamejdnhcphjbkeiagm': { 'SZ Items': { 'update_url': 'https://clients2.google.com/service/update2/crx'}, @@ -90,6 +90,19 @@ SETTINGS_GOOGLE_CHROME = { 'WOW64_32': True, }, } +SETTINGS_MOZILLA_FIREFOX_32 = { + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + 'WOW64_32': True, + }, + } +SETTINGS_MOZILLA_FIREFOX_64 = { + r'Software\Mozilla\Firefox\Extensions': { + 'SZ Items': { + 'uBlock0@raymondhill.net': MOZILLA_FIREFOX_UBO_PATH}, + }, + } VCR_REDISTS = [ {'Name': 'Visual C++ 2008 SP1 x32...', 'Cmd': [r'2008sp1\x32\vcredist.exe', '/qb! /norestart']}, @@ -221,7 +234,7 @@ def install_adobe_reader(): run_program(cmd) def install_chrome_extensions(): - """Update registry to 'install' Google Chrome extensions for all users.""" + """Update registry to install Google Chrome extensions for all users.""" write_registry_settings(SETTINGS_GOOGLE_CHROME, all_users=True) def install_classicstart_skin(): @@ -238,16 +251,20 @@ def install_classicstart_skin(): shutil.copy(source, dest) def install_firefox_extensions(): - """Extract Firefox extensions to installation folder.""" + """Update registry to install Firefox extensions for all users.""" dist_path = r'{PROGRAMFILES}\Mozilla Firefox\distribution\extensions'.format( **global_vars['Env']) source_path = r'{CBinDir}\FirefoxExtensions.7z'.format(**global_vars) if not os.path.exists(source_path): raise FileNotFoundError + + # Update registry + write_registry_settings(SETTINGS_MOZILLA_FIREFOX_32, all_users=True) + write_registry_settings(SETTINGS_MOZILLA_FIREFOX_64, all_users=True) # Extract extension(s) to distribution folder cmd = [ - global_vars['Tools']['SevenZip'], 'x', '-aos', '-bso0', '-bse0', + global_vars['Tools']['SevenZip'], 'e', '-aos', '-bso0', '-bse0', '-p{ArchivePassword}'.format(**global_vars), '-o{dist_path}'.format(dist_path=dist_path), source_path] diff --git a/.bin/Scripts/functions/update.py b/.bin/Scripts/functions/update.py index 6825f9ba..9c9cfec0 100644 --- a/.bin/Scripts/functions/update.py +++ b/.bin/Scripts/functions/update.py @@ -235,19 +235,34 @@ def update_fastcopy(): remove_from_kit('FastCopy') # Download - download_to_temp('FastCopy32.zip', SOURCE_URLS['FastCopy32']) - download_to_temp('FastCopy64.zip', SOURCE_URLS['FastCopy64']) - - # Extract - extract_temp_to_bin('FastCopy64.zip', 'FastCopy', sz_args=['FastCopy.exe']) + download_to_temp('FastCopy.zip', SOURCE_URLS['FastCopy']) + + # Extract installer + extract_temp_to_bin('FastCopy.zip', 'FastCopy') + _path = r'{}\FastCopy'.format(global_vars['BinDir']) + _installer = 'FastCopy354_installer.exe' + + # Extract 64-bit + cmd = [ + r'{}\{}'.format(_path, _installer), + '/NOSUBDIR', '/DIR={}'.format(_path), + '/EXTRACT64'] + run_program(cmd) shutil.move( r'{}\FastCopy\FastCopy.exe'.format(global_vars['BinDir']), r'{}\FastCopy\FastCopy64.exe'.format(global_vars['BinDir'])) - extract_temp_to_bin('FastCopy32.zip', 'FastCopy', sz_args=[r'-x!setup.exe', r'-x!*.dll']) - + + # Extract 32-bit + cmd = [ + r'{}\{}'.format(_path, _installer), + '/NOSUBDIR', '/DIR={}'.format(_path), + '/EXTRACT32'] + run_program(cmd) + # Cleanup - remove_from_temp('FastCopy32.zip') - remove_from_temp('FastCopy64.zip') + os.remove(r'{}\{}'.format(_path, _installer)) + os.remove(r'{}\setup.exe'.format(_path, _installer)) + remove_from_temp('FastCopy.zip') def update_wimlib(): # Stop running processes @@ -474,10 +489,18 @@ def update_samsung_magician(): remove_from_kit('Samsung Magician.exe') # Download - download_generic( - r'{}\_Drivers\Samsung Magician'.format(global_vars['CBinDir']), - 'Samsung Magician.exe', - SOURCE_URLS['Samsung Magician']) + download_to_temp('Samsung Magician.zip', SOURCE_URLS['Samsung Magician']) + + # Extract + extract_temp_to_cbin('Samsung Magician.zip', '_Drivers\Samsung Magician') + shutil.move( + r'{}\_Drivers\Samsung Magician\Samsung_Magician_Installer.exe'.format( + global_vars['CBinDir']), + r'{}\_Drivers\Samsung Magician\Samsung Magician.exe'.format( + global_vars['CBinDir'])) + + # Cleanup + remove_from_temp('Samsung Magician.zip') def update_sdi_origin(): # Download aria2 @@ -561,8 +584,8 @@ def update_office(): if os.path.exists(include_path): shutil.copytree(include_path, dest) - # Download and extract - for year in ['2013', '2016']: + for year in ['2016']: + # Download and extract name = 'odt{}.exe'.format(year) url = 'Office Deployment Tool {}'.format(year) download_to_temp(name, SOURCE_URLS[url]) @@ -576,9 +599,8 @@ def update_office(): r'{}\{}'.format(global_vars['TmpDir'], year), r'{}\_Office\{}'.format(global_vars['CBinDir'], year)) - # Cleanup - remove_from_temp('odt2013.exe') - remove_from_temp('odt2016.exe') + # Cleanup + remove_from_temp('odt{}.exe'.format(year)) def update_classic_start_skin(): # Remove existing folders @@ -698,16 +720,10 @@ def update_firefox_ublock_origin(): remove_from_kit('FirefoxExtensions') # Download - download_to_temp('ff-uBO.xpi', SOURCE_URLS['Firefox uBO']) - - # Extract files - extract_generic( - r'{}\ff-uBO.xpi'.format(global_vars['TmpDir']), - r'{}\FirefoxExtensions\uBlock0@raymondhill.net'.format( - global_vars['CBinDir'])) - - # Cleanup - remove_from_temp('ff-uBO.xpi') + download_generic( + r'{}\FirefoxExtensions'.format(global_vars['CBinDir']), + 'ublock_origin.xpi', + SOURCE_URLS['Firefox uBO']) def update_notepadplusplus(): # Stop running processes @@ -745,22 +761,23 @@ def update_putty(): # Cleanup remove_from_temp('putty.zip') -def update_treesizefree(): +def update_wiztree(): # Stop running processes - kill_process('TreeSizeFree.exe') + for process in ['WizTree.exe', 'WizTree64.exe']: + kill_process(process) # Remove existing folders - remove_from_kit('TreeSizeFree') + remove_from_kit('WizTree') # Download download_to_temp( - 'treesizefree.zip', SOURCE_URLS['TreeSizeFree']) + 'wiztree.zip', SOURCE_URLS['WizTree']) # Extract files - extract_temp_to_cbin('treesizefree.zip', 'TreeSizeFree') + extract_temp_to_cbin('wiztree.zip', 'WizTree') # Cleanup - remove_from_temp('treesizefree.zip') + remove_from_temp('wiztree.zip') def update_xmplay(): # Stop running processes @@ -826,11 +843,10 @@ def update_adwcleaner(): remove_from_kit('AdwCleaner') # Download - url = resolve_dynamic_url( - SOURCE_URLS['AdwCleaner'], - 'id="downloadLink"') download_generic( - r'{}\AdwCleaner'.format(global_vars['CBinDir']), 'AdwCleaner.exe', url) + r'{}\AdwCleaner'.format(global_vars['CBinDir']), + 'AdwCleaner.exe', + SOURCE_URLS['AdwCleaner']) def update_kvrt(): # Stop running processes diff --git a/.bin/Scripts/hw-diags b/.bin/Scripts/hw-diags index c1d7d3ca..d3a1cb21 100755 --- a/.bin/Scripts/hw-diags +++ b/.bin/Scripts/hw-diags @@ -8,7 +8,7 @@ MENU="hw-diags-menu" function ask() { while :; do - read -p "$1 " -r answer + read -p "$1 [Y/N] " -r answer if echo "$answer" | egrep -iq '^(y|yes|sure)$'; then return 0 elif echo "$answer" | egrep -iq '^(n|no|nope)$'; then @@ -24,9 +24,12 @@ die () { # Check for running session if tmux list-session | grep -q "$SESSION_NAME"; then - echo "WARNING: hw-diags tmux session already exists." + echo "WARNING: tmux session $SESSION_NAME already exists." echo "" - if ask "Kill current session?"; then + if ask "Connect to current session?"; then + # Do nothing, the command below will attach/connect + echo "" + elif ask "Kill current session and start new session?"; then tmux kill-session -t "$SESSION_NAME" || \ die "Failed to kill session: $SESSION_NAME" else @@ -39,5 +42,5 @@ if tmux list-session | grep -q "$SESSION_NAME"; then fi # Start session -tmux new-session -s "$SESSION_NAME" -n "$WINDOW_NAME" "$MENU" $* +tmux new-session -A -s "$SESSION_NAME" -n "$WINDOW_NAME" "$MENU" $* diff --git a/.bin/Scripts/mount-all-volumes b/.bin/Scripts/mount-all-volumes index d743d656..9e5de0ea 100755 --- a/.bin/Scripts/mount-all-volumes +++ b/.bin/Scripts/mount-all-volumes @@ -18,7 +18,7 @@ if __name__ == '__main__': print_standard('{}: Volume mount tool'.format(KIT_NAME_FULL)) # Mount volumes - report = mount_all_volumes() + report = mount_volumes(all_devices=True) # Print report print_info('\nResults') diff --git a/.bin/Scripts/mount-raw-image b/.bin/Scripts/mount-raw-image index e738c445..6a183859 100755 --- a/.bin/Scripts/mount-raw-image +++ b/.bin/Scripts/mount-raw-image @@ -24,7 +24,7 @@ if [[ -f "${1:-}" ]]; then done else # losetup did not detect partitions, attempt whole image - udevil mount -o to "${LOOPDEV}" || true + udevil mount -o ro "${LOOPDEV}" || true fi else usage diff --git a/.bin/Scripts/settings/launchers.py b/.bin/Scripts/settings/launchers.py index 6e011691..a125a1f8 100644 --- a/.bin/Scripts/settings/launchers.py +++ b/.bin/Scripts/settings/launchers.py @@ -282,8 +282,8 @@ LAUNCHERS = { 'Intel RST (Current Release)': { 'L_TYPE': 'Executable', 'L_PATH': '_Drivers\Intel RST', - 'L_ITEM': 'SetupRST_16.0.exe', - 'L_7ZIP': 'SetupRST_16.0.exe', + 'L_ITEM': 'SetupRST_16.5.exe', + 'L_7ZIP': 'SetupRST_16.5.exe', }, 'Intel RST (Previous Releases)': { 'L_TYPE': 'Folder', @@ -356,32 +356,6 @@ LAUNCHERS = { 'L_ELEV': 'True', }, }, - r'Installers\Extras\Office\2013': { - 'Home and Business 2013 (x32)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hb_32.xml', - 'L_NCMD': 'True', - }, - 'Home and Business 2013 (x64)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hb_64.xml', - 'L_NCMD': 'True', - }, - 'Home and Student 2013 (x32)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hs_32.xml', - 'L_NCMD': 'True', - }, - 'Home and Student 2013 (x64)': { - 'L_TYPE': 'Office', - 'L_PATH': '2013', - 'L_ITEM': 'hs_64.xml', - 'L_NCMD': 'True', - }, - }, r'Installers\Extras\Office\2016': { 'Home and Business 2016 (x32)': { 'L_TYPE': 'Office', @@ -475,10 +449,10 @@ LAUNCHERS = { 'L_PATH': 'PuTTY', 'L_ITEM': 'PUTTY.EXE', }, - 'TreeSizeFree': { + 'WizTree': { 'L_TYPE': 'Executable', - 'L_PATH': 'TreeSizeFree', - 'L_ITEM': 'TreeSizeFree.exe', + 'L_PATH': 'WizTree', + 'L_ITEM': 'WizTree.exe', 'L_ELEV': 'True', }, 'Update Kit': { diff --git a/.bin/Scripts/settings/sources.py b/.bin/Scripts/settings/sources.py index 3560b092..903c46fb 100644 --- a/.bin/Scripts/settings/sources.py +++ b/.bin/Scripts/settings/sources.py @@ -1,9 +1,10 @@ # Wizard Kit: Settings - Sources SOURCE_URLS = { + 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1801120058/AcroRdrDC1801120058_en_US.exe', + 'AdwCleaner': 'https://downloads.malwarebytes.com/file/adwcleaner', 'AIDA64': 'http://download.aida64.com/aida64engineer597.zip', - 'Adobe Reader DC': 'http://ardownload.adobe.com/pub/adobe/reader/win/AcrobatDC/1801120040/AcroRdrDC1801120040_en_US.exe', - 'AdwCleaner': 'https://toolslib.net/downloads/finish/1-adwcleaner/', + 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0-win-32bit-build1.zip', 'Autoruns': 'https://download.sysinternals.com/files/Autoruns.zip', 'BleachBit': 'https://download.bleachbit.org/BleachBit-2.0-portable.zip', 'BlueScreenView32': 'http://www.nirsoft.net/utils/bluescreenview.zip', @@ -14,44 +15,37 @@ SOURCE_URLS = { 'ERUNT': 'http://www.aumha.org/downloads/erunt.zip', 'Everything32': 'https://www.voidtools.com/Everything-1.4.1.895.x86.zip', 'Everything64': 'https://www.voidtools.com/Everything-1.4.1.895.x64.zip', - 'FastCopy32': 'http://ftp.vector.co.jp/69/93/2323/FastCopy341.zip', - 'FastCopy64': 'http://ftp.vector.co.jp/69/93/2323/FastCopy341_x64.zip', - 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/956394/ublock_origin-1.16.6-an+fx.xpi', - 'HWiNFO': 'http://app.oldfoss.com:81/download/HWiNFO/hwi_582.zip', + 'FastCopy': 'http://ftp.vector.co.jp/70/64/2323/FastCopy354_installer.zip', + 'Firefox uBO': 'https://addons.mozilla.org/firefox/downloads/file/1056733/ublock_origin-1.16.20-an+fx.xpi', 'HitmanPro32': 'https://dl.surfright.nl/HitmanPro.exe', 'HitmanPro64': 'https://dl.surfright.nl/HitmanPro_x64.exe', - 'IOBit_Uninstaller': 'https://portableapps.com/redirect/?a=IObitUninstallerPortable&t=http%3A%2F%2Fdownloads.portableapps.com%2Fportableapps%2Fiobituninstallerportable%2FIObitUninstallerPortable_7.3.0.13.paf.exe', + 'HWiNFO': 'http://app.oldfoss.com:81/download/HWiNFO/hwi_588.zip', 'Intel SSD Toolbox': r'https://downloadmirror.intel.com/27656/eng/Intel%20SSD%20Toolbox%20-%20v3.5.2.exe', + 'IOBit_Uninstaller': 'https://portableapps.duckduckgo.com/IObitUninstallerPortable_7.5.0.7.paf.exe', 'KVRT': 'http://devbuilds.kaspersky-labs.com/devbuilds/KVRT/latest/full/KVRT.exe', - 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.6/npp.7.5.6.bin.minimalist.7z', - 'Office Deployment Tool 2013': 'https://download.microsoft.com/download/6/2/3/6230F7A2-D8A9-478B-AC5C-57091B632FCF/officedeploymenttool_x86_4827-1000.exe', - 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_9326.3600.exe', + 'NotepadPlusPlus': 'https://notepad-plus-plus.org/repository/7.x/7.5.8/npp.7.5.8.bin.minimalist.7z', + 'Office Deployment Tool 2016': 'https://download.microsoft.com/download/2/7/A/27AF1BE6-DD20-4CB4-B154-EBAB8A7D4A7E/officedeploymenttool_10810.33603.exe', 'ProduKey32': 'http://www.nirsoft.net/utils/produkey.zip', 'ProduKey64': 'http://www.nirsoft.net/utils/produkey-x64.zip', 'PuTTY': 'https://the.earth.li/~sgtatham/putty/latest/w32/putty.zip', 'RKill': 'https://www.bleepingcomputer.com/download/rkill/dl/10/', + 'Samsung Magician': 'https://s3.ap-northeast-2.amazonaws.com/global.semi.static/SAMSUNG_SSD_v5_2_1_180523/CD0CFAC4675B9E502899B41BE00525C3909ECE3AD57CC1A2FB6B74A766B2A1EA/Samsung_Magician_Installer.zip', 'SDIO Themes': 'http://snappy-driver-installer.org/downloads/SDIO_Themes.zip', 'SDIO Torrent': 'http://snappy-driver-installer.org/downloads/SDIO_Update.torrent', - 'Samsung Magician': 'http://downloadcenter.samsung.com/content/SW/201801/20180123130636806/Samsung_Magician_Installer.exe', 'TDSSKiller': 'https://media.kaspersky.com/utilities/VirusUtilities/EN/tdsskiller.exe', 'TestDisk': 'https://www.cgsecurity.org/testdisk-7.1-WIP.win.zip', - 'TreeSizeFree': 'https://www.jam-software.com/treesize_free/TreeSizeFree-Portable.zip', 'wimlib32': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-i686-bin.zip', 'wimlib64': 'https://wimlib.net/downloads/wimlib-1.12.0-windows-x86_64-bin.zip', 'Winapp2': 'https://github.com/MoscaDotTo/Winapp2/archive/master.zip', - 'XMPlay 7z': 'http://support.xmplay.com/files/16/xmp-7z.zip?v=800962', - 'XMPlay Game': 'http://support.xmplay.com/files/12/xmp-gme.zip?v=515637', - 'XMPlay RAR': 'http://support.xmplay.com/files/16/xmp-rar.zip?v=409646', - 'XMPlay WAModern': 'http://support.xmplay.com/files/10/WAModern.zip?v=207099', - 'XMPlay': 'http://support.xmplay.com/files/20/xmplay383.zip?v=298195', + 'WizTree': 'https://antibody-software.com/files/wiztree_3_26_portable.zip', + 'XMPlay 7z': 'https://support.xmplay.com/files/16/xmp-7z.zip?v=800962', + 'XMPlay Game': 'https://support.xmplay.com/files/12/xmp-gme.zip?v=515637', + 'XMPlay RAR': 'https://support.xmplay.com/files/16/xmp-rar.zip?v=409646', + 'XMPlay WAModern': 'https://support.xmplay.com/files/10/WAModern.zip?v=207099', + 'XMPlay': 'https://support.xmplay.com/files/20/xmplay383.zip?v=298195', 'XYplorerFree': 'https://www.xyplorer.com/download/xyplorer_free_noinstall.zip', - 'aria2': 'https://github.com/aria2/aria2/releases/download/release-1.33.1/aria2-1.33.1-win-32bit-build1.zip', } VCREDIST_SOURCES = { - '2008sp1': { - '32': 'https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x86.exe', - '64': 'https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe', - }, '2010sp1': { '32': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe', '64': 'https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x64.exe', @@ -65,15 +59,14 @@ VCREDIST_SOURCES = { '64': 'https://download.microsoft.com/download/0/5/6/056dcda9-d667-4e27-8001-8a0c6971d6b1/vcredist_x64.exe', }, '2017': { - '32': 'https://download.visualstudio.microsoft.com/download/pr/100349138/88b50ce70017bf10f2d56d60fcba6ab1/VC_redist.x86.exe', - '64': 'https://download.visualstudio.microsoft.com/download/pr/100349091/2cd2dba5748dc95950a5c42c2d2d78e4/VC_redist.x64.exe', + '32': 'https://aka.ms/vs/15/release/vc_redist.x86.exe', + '64': 'https://aka.ms/vs/15/release/vc_redist.x64.exe', }, } NINITE_SOURCES = { 'Bundles': { - 'Runtimes.exe': '.net4.7.1-air-java8-silverlight', - 'Legacy.exe': '.net4.7.1-7zip-air-chrome-firefox-java8-silverlight-vlc', - 'Modern.exe': '.net4.7.1-7zip-air-chrome-classicstart-firefox-java8-silverlight-vlc', + 'Legacy.exe': '.net4.7.2-7zip-chrome-firefox-vlc', + 'Modern.exe': '.net4.7.2-7zip-chrome-classicstart-firefox-vlc', }, 'Audio-Video': { 'AIMP.exe': 'aimp', @@ -98,6 +91,7 @@ NINITE_SOURCES = { 'SugarSync.exe': 'sugarsync', }, 'Communication': { + 'Discord': 'discord', 'Pidgin.exe': 'pidgin', 'Skype.exe': 'skype', 'Trillian.exe': 'trillian', @@ -109,7 +103,6 @@ NINITE_SOURCES = { }, 'Developer': { 'Eclipse.exe': 'eclipse', - 'FileZilla.exe': 'filezilla', 'JDK 8.exe': 'jdk8', 'JDK 8 (x64).exe': 'jdkx8', 'Notepad++.exe': 'notepadplusplus', @@ -153,7 +146,7 @@ NINITE_SOURCES = { }, 'Runtimes': { 'Adobe Air.exe': 'air', - 'dotNET.exe': '.net4.7.1', + 'dotNET.exe': '.net4.7.2', 'Java 8.exe': 'java8', 'Shockwave.exe': 'shockwave', 'Silverlight.exe': 'silverlight', @@ -197,6 +190,7 @@ RST_SOURCES = { 'SetupRST_15.8.exe': 'https://downloadmirror.intel.com/27442/eng/SetupRST.exe', 'SetupRST_15.9.exe': 'https://downloadmirror.intel.com/27400/eng/SetupRST.exe', 'SetupRST_16.0.exe': 'https://downloadmirror.intel.com/27681/eng/SetupRST.exe', + 'SetupRST_16.5.exe': 'https://downloadmirror.intel.com/27984/eng/SetupRST.exe', } diff --git a/.bin/Scripts/update_kit.py b/.bin/Scripts/update_kit.py index 6a5cdf82..7f9b251d 100644 --- a/.bin/Scripts/update_kit.py +++ b/.bin/Scripts/update_kit.py @@ -70,7 +70,7 @@ if __name__ == '__main__': try_and_print(message='FirefoxExtensions...', function=update_firefox_ublock_origin, other_results=other_results, width=40) try_and_print(message='PuTTY...', function=update_putty, other_results=other_results, width=40) try_and_print(message='Notepad++...', function=update_notepadplusplus, other_results=other_results, width=40) - try_and_print(message='TreeSizeFree...', function=update_treesizefree, other_results=other_results, width=40) + try_and_print(message='WizTree...', function=update_wiztree, other_results=other_results, width=40) try_and_print(message='XMPlay...', function=update_xmplay, other_results=other_results, width=40) # Repairs diff --git a/.bin/Scripts/wk-power-command b/.bin/Scripts/wk-power-command new file mode 100755 index 00000000..92af02fb --- /dev/null +++ b/.bin/Scripts/wk-power-command @@ -0,0 +1,21 @@ +#!/bin/bash +# +## Wizard Kit: Wrapper for logout, reboot, & poweroff + +# Unmount filesystems +find /media -maxdepth 1 -mindepth 1 -type d \ + -exec udevil umount "{}" \; + +# Flush write cache +sudo sync + +# Perform requested action +case "${1:-x}" in + poweroff) + sudo systemctl poweroff;; + reboot) + sudo systemctl reboot;; + *) + openbox --exit;; +esac +exit 0 diff --git a/.cbin/_include/_vcredists/InstallAll.bat b/.cbin/_include/_vcredists/InstallAll.bat index 597cd60e..5d935b34 100644 --- a/.cbin/_include/_vcredists/InstallAll.bat +++ b/.cbin/_include/_vcredists/InstallAll.bat @@ -1,9 +1,6 @@ @echo off setlocal -start "" /wait "2008sp1\x32\vcredist.exe" /qb! /norestart -start "" /wait "2008sp1\x64\vcredist.exe" /qb! /norestart - start "" /wait "2010\x32\vcredist.exe" /passive /norestart start "" /wait "2010\x64\vcredist.exe" /passive /norestart @@ -19,4 +16,4 @@ start "" /wait "2015u3\x64\vcredist.exe" /install /passive /norestart start "" /wait "2017\x32\vcredist.exe" /install /passive /norestart start "" /wait "2017\x64\vcredist.exe" /install /passive /norestart -endlocal \ No newline at end of file +endlocal diff --git a/.linux_items/authorized_keys b/.linux_items/authorized_keys new file mode 100644 index 00000000..be79388e --- /dev/null +++ b/.linux_items/authorized_keys @@ -0,0 +1 @@ +#Put SSH keys here diff --git a/.linux_items/include/EFI/boot/refind.conf b/.linux_items/include/EFI/boot/refind.conf index 075f6e46..de83e78c 100644 --- a/.linux_items/include/EFI/boot/refind.conf +++ b/.linux_items/include/EFI/boot/refind.conf @@ -32,8 +32,8 @@ menuentry "Linux" { add_options "nox" } } -menuentry "WindowsPE" { - ostype windows - icon /EFI/boot/icons/wk_win.png - loader /EFI/microsoft/bootx64.efi -} +#UFD#menuentry "WindowsPE" { +#UFD# ostype windows +#UFD# icon /EFI/boot/icons/wk_win.png +#UFD# loader /EFI/microsoft/bootx64.efi +#UFD#} diff --git a/.linux_items/include/airootfs/etc/modprobe.d/broadcom.conf b/.linux_items/include/airootfs/etc/modprobe.d/broadcom.conf new file mode 100644 index 00000000..1c85cd7b --- /dev/null +++ b/.linux_items/include/airootfs/etc/modprobe.d/broadcom.conf @@ -0,0 +1 @@ +softdep tg3 pre: broadcom diff --git a/.linux_items/include/airootfs/etc/oblogout.conf b/.linux_items/include/airootfs/etc/oblogout.conf index 4595c766..ea5606ef 100644 --- a/.linux_items/include/airootfs/etc/oblogout.conf +++ b/.linux_items/include/airootfs/etc/oblogout.conf @@ -15,6 +15,6 @@ restart = R logout = L [commands] -shutdown = systemctl poweroff -restart = systemctl reboot -logout = openbox --exit +shutdown = /usr/local/bin/wk-power-command poweroff +restart = /usr/local/bin/wk-power-command reboot +logout = /usr/local/bin/wk-power-command logout diff --git a/.linux_items/include/airootfs/etc/pydfrc b/.linux_items/include/airootfs/etc/pydfrc new file mode 100644 index 00000000..100b1f05 --- /dev/null +++ b/.linux_items/include/airootfs/etc/pydfrc @@ -0,0 +1,24 @@ +normal_colour = 'default' +header_colour = 'blue' +local_fs_colour = 'default' +remote_fs_colour = 'green' +special_fs_colour = 'yellow' +readonly_fs_colour = 'cyan' +filled_fs_colour = 'red' +full_fs_colour = 'on_red', 'green', 'blink' +sizeformat = "-h" +column_separator = ' ' +column_separator_colour = 'none' +stretch_screen = 0.3 +FILL_THRESH = 75.0 +FULL_THRESH = 85.0 +format = [ + ('fs', 10, "l"), ('size', 5, "r"), + ('used', 5, "r"), ('avail', 5, "r"), ('perc', 5, "r"), + ('bar', 0.1, "l"), ('on', 11, "l") + ] +barchar = '#' +bar_fillchar = '.' +hidebinds = True +mountfile = ['/etc/mtab', '/etc/mnttab', '/proc/mounts'] + diff --git a/.linux_items/include/airootfs/etc/skel/.Xresources b/.linux_items/include/airootfs/etc/skel/.Xresources index 68054af5..d659b735 100755 --- a/.linux_items/include/airootfs/etc/skel/.Xresources +++ b/.linux_items/include/airootfs/etc/skel/.Xresources @@ -21,7 +21,7 @@ URxvt*externalBorder: 0 !URxvt.colorIT: #87af5f !URxvt.colorBD: #c5c8c6 !URxvt.colorUL: #87afd7 -URxvt.geometry: 92x16 +URxvt.geometry: 92x16 URxvt.internalBorder: 8 URxvt.shading: 10 URxvt.transparent: true @@ -53,6 +53,7 @@ URxvt.transparent: true *.color15: #ffffff ! fonts +!Xft.dpi: 192 Xft.autohint: 0 Xft.antialias: 1 Xft.hinting: true diff --git a/.linux_items/include/airootfs/etc/skel/.aliases b/.linux_items/include/airootfs/etc/skel/.aliases index d20b59b3..03d15315 100644 --- a/.linux_items/include/airootfs/etc/skel/.aliases +++ b/.linux_items/include/airootfs/etc/skel/.aliases @@ -4,9 +4,11 @@ alias 7z3='7z a -t7z -mx=3' alias 7z5='7z a -t7z -mx=5' alias 7z7='7z a -t7z -mx=7' alias 7z9='7z a -t7z -mx=9' +alias ddrescue='sudo ddrescue --ask --min-read-rate=64k -vvvv' alias diff='colordiff -ur' alias du='du -sch --apparent-size' alias fix-perms='find -type d -exec chmod 755 "{}" \; && find -type f -exec chmod 644 "{}" \;' +alias hexedit='hexedit --color' alias hw-info='sudo hw-info | less -S' alias inxi='echo -e "\e[33mWARNING: inxi is being replaced and will be removed in a future WizardKit release\e[0m"; echo -e " \e[32mReplacements include:\e[0m 'hw-drive-info', 'hw-info', & 'hw-sensors'"; echo ""; inxi' alias less='less -S' @@ -34,3 +36,5 @@ alias srsz='sudo rsync -avhzPS --stats --exclude-from="$HOME/.rsync_exclusions"' alias testdisk='sudo testdisk' alias umount='sudo umount' alias unmount='sudo umount' +alias wkclone='sudo ddrescue-tui clone' +alias wkimage='sudo ddrescue-tui image' diff --git a/.linux_items/include/airootfs/etc/skel/.config/i3/config b/.linux_items/include/airootfs/etc/skel/.config/i3/config index de3c6fa8..cf09a2ca 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/i3/config +++ b/.linux_items/include/airootfs/etc/skel/.config/i3/config @@ -73,7 +73,7 @@ bindsym $mod+f exec "thunar ~" bindsym $mod+i exec "hardinfo" bindsym $mod+m exec "urxvt -title 'Mount All Volumes' -e mount-all-volumes gui" bindsym $mod+s exec "urxvt -title 'Hardware Diagnostics' -e hw-diags quick" -bindsym $mod+t exec "urxvt" +bindsym $mod+t exec "urxvt -e zsh -c 'tmux new-session -A -t general; zsh'" bindsym $mod+v exec "urxvt -title 'Hardware Sensors' -e watch -c -n1 -t hw-sensors" bindsym $mod+w exec "firefox" diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart b/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart index 21fd52e5..6eaa6cc3 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart +++ b/.linux_items/include/airootfs/etc/skel/.config/openbox/autostart @@ -17,3 +17,4 @@ #xfce-mcs-manager & tint2 & +cbatticon --hide-notification & diff --git a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml index 43656613..90c7b0e0 100644 --- a/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml +++ b/.linux_items/include/airootfs/etc/skel/.config/openbox/rc.xml @@ -329,7 +329,7 @@ - urxvt + urxvt -e zsh -c 'tmux new-session -A -t general; zsh' diff --git a/.linux_items/include/airootfs/etc/skel/.conkyrc b/.linux_items/include/airootfs/etc/skel/.conkyrc index adfd5cbb..af09dd8f 100644 --- a/.linux_items/include/airootfs/etc/skel/.conkyrc +++ b/.linux_items/include/airootfs/etc/skel/.conkyrc @@ -37,7 +37,7 @@ minimum_size 180 0 ### width | height maximum_width 180 gap_x 20 ### left | right -gap_y 45 ### up | down +gap_y 24 ### up | down alignment tr ####################### End Window Settings ### @@ -143,15 +143,14 @@ Uptime:${alignr}${uptime_short} CPU: ${if_match ${cpu cpu0}<10} ${cpu cpu0}\ ${else}${if_match ${cpu cpu0}<100} ${cpu cpu0}\ ${else}${cpu cpu0}${endif}${endif}% Used${alignr}${freq_g} GHz -${cpugraph cpu0 20,180 ${color} ${color}} +${cpugraph cpu0 ${gap_x},${width} ${color} ${color}} RAM: ${mem} Used${alignr}${memmax} -${memgraph 20,180 ${color} ${color}} +${memgraph ${gap_x},${width} ${color} ${color}} Disk I/O: -${diskiograph 20,180 ${color} ${color}} +${diskiograph ${gap_x},${width} ${color} ${color}} Down: ${downspeed}${goto 115}Up:${alignr}${upspeed} #Network - ${alignc}S H O R T C U T K E Y S ${hr} [Super] + d${alignr}HW Diagnostics diff --git a/.linux_items/include/airootfs/etc/skel/.update_conky b/.linux_items/include/airootfs/etc/skel/.update_conky index f67dc893..79801d8b 100755 --- a/.linux_items/include/airootfs/etc/skel/.update_conky +++ b/.linux_items/include/airootfs/etc/skel/.update_conky @@ -3,14 +3,16 @@ IF_LIST=($(ip l | egrep '^[0-9]+:\s+(eth|en|wl)' | sed -r 's/^[0-9]+:\s+(\w+):.*/\1/' | sort)) # Add interfaces to conkyrc -for i in "${IF_LIST[@]}"; do - if [[ "${i:0:1}" == "e" ]]; then - sed -i -r "s/#Network/Wired:\${alignr}\${addr $i}\n#Network/" ~/.conkyrc - else - sed -i -r "s/#Network/Wireless:\${alignr}\${addr $i}\n#Network/" ~/.conkyrc - fi -done +if fgrep '#Network' $HOME/.conkyrc; then + for i in "${IF_LIST[@]}"; do + if [[ "${i:0:1}" == "e" ]]; then + sed -i -r "s/#Network/Wired:\${alignr}\${addr $i}\n#Network/" $HOME/.conkyrc + else + sed -i -r "s/#Network/Wireless:\${alignr}\${addr $i}\n#Network/" $HOME/.conkyrc + fi + done -# Remove '#Network' line to prevent duplicating lines if this script is re-run -sed -i -r "s/#Network//" ~/.conkyrc + # Remove '#Network' line to prevent duplicating lines if this script is re-run + sed -i -r "s/#Network//" $HOME/.conkyrc +fi diff --git a/.linux_items/include/airootfs/etc/skel/.update_dpi_settings b/.linux_items/include/airootfs/etc/skel/.update_dpi_settings new file mode 100755 index 00000000..2287508c --- /dev/null +++ b/.linux_items/include/airootfs/etc/skel/.update_dpi_settings @@ -0,0 +1,68 @@ +#!/bin/env bash +# +## Calculate DPI and adjust display settings if necesary + +REGEX_XRANDR='^.* ([0-9]+)x([0-9]+)\+[0-9]+\+[0-9]+.* ([0-9]+)mm x ([0-9]+)mm.*$' +REGEX_URXVT='(URxvt.geometry:\s+).*' + +# Get screen data +xrandr_str="$(xrandr | grep mm | head -1)" +width_px="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\1/")" +height_px="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\2/")" +width_mm="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\3/")" +height_mm="$(echo "${xrandr_str}" | sed -r "s/${REGEX_XRANDR}/\4/")" + +# Convert to in +width_in="$(echo "${width_mm} * 0.03937" | bc)" +height_in="$(echo "${height_mm} * 0.03937" | bc)" + +# Calculate diagonals +diag_px="$(echo "sqrt(${width_px}^2 + ${height_px}^2)" | bc)" +diag_in="$(echo "sqrt(${width_in}^2 + ${height_in}^2)" | bc)" + +# Calculate DPI +dpi="$(echo "${diag_px} / ${diag_in}" | bc 2>/dev/null || True)" +dpi="${dpi:-0}" + +# Calculate URxvt default window size +width_urxvt="$(echo "${width_px} * 112/1280" | bc)" +height_urxvt="$(echo "${height_px} * 33/720" | bc)" +offset_urxvt="24" + +# Update settings if necessary +if [[ "${dpi}" -ge 192 ]]; then + # Conky + sed -i 's/minimum_size 180 0/minimum_size 360 0/' "${HOME}/.conkyrc" + sed -i 's/maximum_width 180/maximum_width 360/' "${HOME}/.conkyrc" + sed -i 's/gap_x 20/gap_x 40/' "${HOME}/.conkyrc" + sed -i 's/gap_y 24/gap_y 48/' "${HOME}/.conkyrc" + + # Fonts + sed -i 's/!Xft.dpi: 192/Xft.dpi: 192/' "${HOME}/.Xresources" + + # GDK + export GDK_SCALE=2 + export GDK_DPI_SCALE=0.5 + + # i3 + sed -i -r 's/(height\s+) 26/\1 52/' "${HOME}/.config/i3/config" + + # Tint2 + sed -i 's/panel_size = 100% 30/panel_size = 100% 60/' \ + "${HOME}/.config/tint2/tint2rc" + sed -i 's/Inconsolata 10/Inconsolata 20/g' \ + "${HOME}/.config/tint2/tint2rc" + sed -i 's/Inconsolata 12/Inconsolata 24/g' \ + "${HOME}/.config/tint2/tint2rc" + sed -i 's/systray_icon_size = 24/systray_icon_size = 48/' \ + "${HOME}/.config/tint2/tint2rc" + + # URxvt + width_urxvt="$(echo "${width_urxvt} / 2" | bc)" + height_urxvt="$(echo "${height_urxvt} / 2" | bc)" + offset_urxvt="$(echo "${offset_urxvt} * 2" | bc)" +fi + +# Update URxvt (Always) +urxvt_geometry="${width_urxvt}x${height_urxvt}+${offset_urxvt}+${offset_urxvt}" +sed -i -r "s/${REGEX_URXVT}/\1${urxvt_geometry}/" "${HOME}/.Xresources" diff --git a/.linux_items/include/airootfs/etc/skel/.update_hostname b/.linux_items/include/airootfs/etc/skel/.update_hostname new file mode 100755 index 00000000..3c1bd7c2 --- /dev/null +++ b/.linux_items/include/airootfs/etc/skel/.update_hostname @@ -0,0 +1,16 @@ +#!/bin/bash + +IP="$(ip a show scope global \ + | grep inet \ + | head -1 \ + | sed -r 's#.*inet ([0-9]+.[0-9]+.[0-9]+.[0-9]+.)/.*#\1#')" +HOSTNAME="$(dig +noall +answer +short -x "$IP" \ + | head -1 \ + | sed 's/\.$//')" + +# Set hostname and renew DHCP lease +sudo hostnamectl set-hostname "${HOSTNAME}" +sudo dhclient -r +sleep 1 +sudo dhclient + diff --git a/.linux_items/include/airootfs/etc/skel/.update_wallpaper b/.linux_items/include/airootfs/etc/skel/.update_wallpaper deleted file mode 100755 index 7bffa12b..00000000 --- a/.linux_items/include/airootfs/etc/skel/.update_wallpaper +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -BOOT_PATH="/run/archiso/bootmnt/arch/" -BURNED_IN="/usr/share/wallpaper/burned.in" -WALLPAPER="$HOME/.wallpaper.png" - -function link_wall() { - sudo rm "$WALLPAPER" - sudo ln -s "$1" "$WALLPAPER" -} - -# Check for wallpaper -## Checks BOOT_PATH and uses the BURNED_IN file if nothing is found -for f in "$BOOT_PATH"/{Arch,arch}.{jpg,png} "$BURNED_IN"; do - if [[ -f "$f" ]]; then - link_wall "$f" - break - fi -done - -feh --bg-fill "$WALLPAPER" diff --git a/.linux_items/include/airootfs/etc/skel/.urxvt_default_res b/.linux_items/include/airootfs/etc/skel/.urxvt_default_res deleted file mode 100755 index 1e146090..00000000 --- a/.linux_items/include/airootfs/etc/skel/.urxvt_default_res +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -XWIDTH="$(xrandr 2>/dev/null | grep '*' | sed -r 's/^\s+([0-9]+)x.*/\1/')" -XHEIGHT="$(xrandr 2>/dev/null | grep '*' | sed -r 's/^\s+[0-9]+x([0-9]+).*/\1/')" - -WIDTH="$(echo "${XWIDTH}*92/1024" | bc)" -HEIGHT="$(echo "${XHEIGHT}*32/768" | bc)" - -sed -i -r "s/(URxvt.geometry:\s+).*/\1${WIDTH}x${HEIGHT}+24+24/" ~/.Xresources -xrdb -merge ~/.Xresources diff --git a/.linux_items/include/airootfs/etc/skel/.wallpaper b/.linux_items/include/airootfs/etc/skel/.wallpaper new file mode 120000 index 00000000..f2a3d5e1 --- /dev/null +++ b/.linux_items/include/airootfs/etc/skel/.wallpaper @@ -0,0 +1 @@ +/usr/share/wallpaper/burned.in \ No newline at end of file diff --git a/.linux_items/include/airootfs/etc/skel/.xinitrc b/.linux_items/include/airootfs/etc/skel/.xinitrc index 78bddfe7..abb71f45 100755 --- a/.linux_items/include/airootfs/etc/skel/.xinitrc +++ b/.linux_items/include/airootfs/etc/skel/.xinitrc @@ -1,6 +1,7 @@ #!/bin/sh dbus-update-activation-environment --systemd DISPLAY +$HOME/.update_dpi_settings xrdb -merge $HOME/.Xresources xset s off xset -dpms @@ -10,11 +11,9 @@ compton --backend xrender --xrender-sync --xrender-sync-fence & sleep 1s conky -d nm-applet & -cbatticon & volumeicon & connect-to-network & -(sleep 5s && killall dunst) & -$HOME/.urxvt_default_res & -$HOME/.update_wallpaper & -$HOME/.update_conky & +$HOME/.update_hostname & +feh --bg-fill "$HOME/.wallpaper" & +x0vncserver -display :0 -passwordfile $HOME/.vnc/passwd -AlwaysShared & exec openbox-session diff --git a/.linux_items/include/airootfs/etc/skel/.zlogin b/.linux_items/include/airootfs/etc/skel/.zlogin index 26463919..0898d1e2 100644 --- a/.linux_items/include/airootfs/etc/skel/.zlogin +++ b/.linux_items/include/airootfs/etc/skel/.zlogin @@ -1,9 +1,15 @@ setterm -blank 0 -powerdown 0 if [ "$(fgconsole 2>/dev/null)" -eq "1" ]; then + # Update settings if using i3 if fgrep -q "i3" /proc/cmdline; then sed -i -r 's/#(own_window_type override)/\1/' ~/.conkyrc sed -i -r 's/openbox-session/i3/' ~/.xinitrc fi + + # Update Conky + $HOME/.update_conky + + # Start X or HW-diags if ! fgrep -q "nox" /proc/cmdline; then startx >/dev/null else diff --git a/.linux_items/include/airootfs/etc/ufw/user.rules b/.linux_items/include/airootfs/etc/ufw/user.rules index aa30960c..3dbf5cf4 100644 --- a/.linux_items/include/airootfs/etc/ufw/user.rules +++ b/.linux_items/include/airootfs/etc/ufw/user.rules @@ -21,6 +21,9 @@ -A ufw-user-input -p tcp --dport 22 -j ACCEPT -A ufw-user-input -p udp --dport 22 -j ACCEPT +### tuple ### allow tcp 5900 0.0.0.0/0 any 0.0.0.0/0 VNC - in +-A ufw-user-input -p tcp --dport 5900 -j ACCEPT -m comment --comment 'dapp_VNC' + ### END RULES ### ### LOGGING ### diff --git a/.linux_items/include/airootfs/etc/ufw/user6.rules b/.linux_items/include/airootfs/etc/ufw/user6.rules index 47d96108..13084be4 100644 --- a/.linux_items/include/airootfs/etc/ufw/user6.rules +++ b/.linux_items/include/airootfs/etc/ufw/user6.rules @@ -21,6 +21,9 @@ -A ufw6-user-input -p tcp --dport 22 -j ACCEPT -A ufw6-user-input -p udp --dport 22 -j ACCEPT +### tuple ### allow tcp 5900 ::/0 any ::/0 VNC - in +-A ufw6-user-input -p tcp --dport 5900 -j ACCEPT -m comment --comment 'dapp_VNC' + ### END RULES ### ### LOGGING ### diff --git a/.linux_items/include/syslinux/wk_pxe.cfg b/.linux_items/include/syslinux/wk_pxe.cfg index d5efabb4..92af00e3 100644 --- a/.linux_items/include/syslinux/wk_pxe.cfg +++ b/.linux_items/include/syslinux/wk_pxe.cfg @@ -2,7 +2,7 @@ INCLUDE boot/syslinux/wk_head.cfg MENU BACKGROUND pxelinux.png INCLUDE boot/syslinux/wk_pxe_linux.cfg -INCLUDE boot/syslinux/wk_pxe_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_pxe_winpe.cfg INCLUDE boot/syslinux/wk_pxe_extras_entry.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/include/syslinux/wk_pxe_extras.cfg b/.linux_items/include/syslinux/wk_pxe_extras.cfg index 04cd2ce1..d677b4b0 100644 --- a/.linux_items/include/syslinux/wk_pxe_extras.cfg +++ b/.linux_items/include/syslinux/wk_pxe_extras.cfg @@ -3,7 +3,7 @@ MENU BACKGROUND pxelinux.png INCLUDE boot/syslinux/wk_pxe_linux.cfg INCLUDE boot/syslinux/wk_pxe_linux_extras.cfg -INCLUDE boot/syslinux/wk_pxe_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_pxe_winpe.cfg INCLUDE boot/syslinux/wk_hdt.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/include/syslinux/wk_sys.cfg b/.linux_items/include/syslinux/wk_sys.cfg index beefb77d..0d375cf3 100644 --- a/.linux_items/include/syslinux/wk_sys.cfg +++ b/.linux_items/include/syslinux/wk_sys.cfg @@ -1,7 +1,7 @@ INCLUDE boot/syslinux/wk_head.cfg INCLUDE boot/syslinux/wk_sys_linux.cfg -INCLUDE boot/syslinux/wk_sys_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_sys_winpe.cfg INCLUDE boot/syslinux/wk_sys_extras_entry.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/include/syslinux/wk_sys_extras.cfg b/.linux_items/include/syslinux/wk_sys_extras.cfg index 422bd053..3e5af215 100644 --- a/.linux_items/include/syslinux/wk_sys_extras.cfg +++ b/.linux_items/include/syslinux/wk_sys_extras.cfg @@ -2,7 +2,7 @@ INCLUDE boot/syslinux/wk_head.cfg INCLUDE boot/syslinux/wk_sys_linux.cfg INCLUDE boot/syslinux/wk_sys_linux_extras.cfg -INCLUDE boot/syslinux/wk_sys_winpe.cfg +#UFD#INCLUDE boot/syslinux/wk_sys_winpe.cfg INCLUDE boot/syslinux/wk_hdt.cfg INCLUDE boot/syslinux/wk_tail.cfg diff --git a/.linux_items/packages/live_add b/.linux_items/packages/live_add index a297ca05..ea784699 100644 --- a/.linux_items/packages/live_add +++ b/.linux_items/packages/live_add @@ -76,6 +76,7 @@ spice-vdagent terminus-font testdisk-wip thunar +tigervnc tint2 tk tmux diff --git a/Build Linux b/Build Linux index 6ee0f169..af01c928 100755 --- a/Build Linux +++ b/Build Linux @@ -190,9 +190,9 @@ function update_live_env() { # Live packages while read -r p; do - sed -i "/$p/d" "$LIVE_DIR/packages.both" + sed -i "/$p/d" "$LIVE_DIR/packages.x86_64" done < "$ROOT_DIR/.linux_items/packages/live_remove" - cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.both" + cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.x86_64" echo "[custom]" >> "$LIVE_DIR/pacman.conf" echo "SigLevel = Optional TrustAll" >> "$LIVE_DIR/pacman.conf" echo "Server = file://$REPO_DIR" >> "$LIVE_DIR/pacman.conf" @@ -219,13 +219,18 @@ function update_live_env() { # Services sed -i -r 's/^(.*pacman-init.*)$/#NOPE#\1/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" sed -i -r 's/^(.*choose-mirror.*)$/#NOPE#\1/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + + # Shutdown stall fix + echo "sed -i -r 's/^.*(DefaultTimeoutStartSec)=.*$/\1=15s/' /etc/systemd/system.conf" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + echo "sed -i -r 's/^.*(DefaultTimeoutStopSec)=.*$/\1=15s/' /etc/systemd/system.conf" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" # SSH mkdir -p "$SKEL_DIR/.ssh" ssh-keygen -b 4096 -C "$username@$hostname" -N "" -f "$SKEL_DIR/.ssh/id_rsa" echo 'rm /root/.ssh/id*' >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" echo 'rm /root/.zlogin' >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" - sed -i -r 's/^(.*PermitRootLogin.*)$/#NOPE#\1/' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + sed -r '/.*PermitRootLogin.*/d' "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + cp "$ROOT_DIR/.linux_items/authorized_keys" "$SKEL_DIR/.ssh/authorized_keys" # Root user echo "echo 'root:$ROOT_PASSWORD' | chpasswd" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" @@ -251,6 +256,10 @@ function update_live_env() { # udevil fix echo "mkdir /media" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + # VNC password + echo "mkdir '/home/$username/.vnc'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + echo "echo '$TECH_PASSWORD' | vncpasswd -f > '/home/$username/.vnc/passwd'" >> "$LIVE_DIR/airootfs/root/customize_airootfs.sh" + # Wallpaper mkdir -p "$LIVE_DIR/airootfs/usr/share/wallpaper" cp "$ROOT_DIR/Images/Linux.png" "$LIVE_DIR/airootfs/usr/share/wallpaper/burned.in" @@ -317,9 +326,11 @@ function build_iso() { # Removing cached (and possibly outdated) custom repo packages for package in $(cat "$ROOT_DIR/.linux_items/packages/aur"); do - if [[ -f /var/cache/pacman/pkg/${package}* ]]; then - rm /var/cache/pacman/pkg/${package}* - fi + for p in /var/cache/pacman/pkg/*${package}*; do + if [[ -f "${p}" ]]; then + rm "${p}" + fi + done done # Build ISO