Merge remote-tracking branch 'upstream/dev' into dev

.bin/Scripts/functions/hw_diags.py

@@ -7,6 +7,7 @@ import time
 from collections import OrderedDict
 from functions.osticket import *
 from functions.sensors import *
+from functions.threading import *
 from functions.tmux import *
 
 
@@ -83,6 +84,12 @@ TMUX_LAYOUT = OrderedDict({
   'Top':            {'y': 2,                'Check': True},
   'Started':        {'x': SIDE_PANE_WIDTH,  'Check': True},
   'Progress':       {'x': SIDE_PANE_WIDTH,  'Check': True},
+  # Testing panes
+  'Prime95':        {'y': 11,               'Check': False},
+  'Temps':          {'y': 1000,             'Check': False},
+  'SMART':          {'y': 3,                'Check': True},
+  'badblocks':      {'y': 5,                'Check': True},
+  'I/O Benchmark':  {'y': 1000,             'Check': False},
 })
 
 
@@ -153,6 +160,11 @@ class DiskObj():
     self.smartctl = {}
     self.tests = OrderedDict()
     self.get_details()
+
+    # Try enabling SMART
+    run_program(['sudo', 'smartctl', '--smart=on', self.path], check=False)
+
+    # Get NVMe/SMART data and set description
     self.get_smart_details()
     self.description = '{size} ({tran}) {model} {serial}'.format(
       **self.lsblk)
@@ -421,7 +433,9 @@ class DiskObj():
 
     # Check for attributes
     if KEY_NVME in self.smartctl:
-      self.nvme_attributes.update(self.smartctl[KEY_NVME])
+      self.nvme_attributes = {
+        k: {'name': k, 'raw': int(v), 'raw_str': str(v)}
+        for k, v in self.smartctl[KEY_NVME].items()}
     elif KEY_SMART in self.smartctl:
       for a in self.smartctl[KEY_SMART].get('table', {}):
         try:
@@ -456,7 +470,7 @@ class DiskObj():
       self.check_attributes(silent)
 
       # Check if a self-test is currently running
-      if 'remaining_percent' in self.smart_self_test['status']:
+      if 'remaining_percent' in self.smart_self_test.get('status', ''):
         _msg = 'SMART self-test in progress, all tests disabled'
 
         # Ask to abort
@@ -643,12 +657,25 @@ def build_status_string(label, status, info_label=False):
     **COLORS)
 
 
-def fix_tmux_panes(state, tmux_layout):
+def fix_tmux_panes_loop(state):
+  while True:
+    try:
+      fix_tmux_panes(state)
+      sleep(1)
+    except AttributeError:
+      # tmux_layout attribute has been deleted, exit function
+      return
+    except RuntimeError:
+      # Assuming layout definitions changes mid-run, ignoring
+      pass
+
+
+def fix_tmux_panes(state):
   """Fix pane sizes if the window has been resized."""
   needs_fixed = False
 
   # Check layout
-  for k, v in tmux_layout.items():
+  for k, v in state.tmux_layout.items():
     if not  v.get('Check'):
       # Not concerned with the size of this pane
       continue
@@ -673,7 +700,7 @@ def fix_tmux_panes(state, tmux_layout):
     return
 
   # Update layout
-  for k, v in tmux_layout.items():
+  for k, v in state.tmux_layout.items():
     # Get target
     target = None
     if k != 'Current':
@@ -879,6 +906,18 @@ def run_badblocks_test(state, test):
   if test.disabled:
     return
 
+  def _save_badblocks_output(read_all=False, timeout=0.1):
+    """Get badblocks output and append to both file and var."""
+    _output = ''
+    while _output is not None:
+      _output = test.badblocks_nbsr.read(0.1)
+      if _output is not None:
+        test.badblocks_stderr += _output.decode()
+        with open(test.badblocks_out, 'a') as f:
+          f.write(_output.decode())
+      if not read_all:
+        break
+
   # Prep
   print_log('Starting badblocks test for {}'.format(test.dev.path))
   test.started = True
@@ -890,10 +929,6 @@ def run_badblocks_test(state, test):
     state.panes['Top'],
     text='{}\n{}'.format(
       TOP_PANE_TEXT, test.dev.description))
-  test.tmux_layout = TMUX_LAYOUT.copy()
-  test.tmux_layout.update({
-    'badblocks': {'y': 5, 'Check': True},
-    })
 
   # Create monitor pane
   test.badblocks_out = '{}/badblocks_{}.out'.format(
@@ -908,29 +943,26 @@ def run_badblocks_test(state, test):
 
   # Start badblocks
   print_standard('Running badblocks test...')
-  try:
   test.badblocks_proc = popen_program(
-      ['sudo', 'hw-diags-badblocks', test.dev.path, test.badblocks_out],
+    ['sudo', 'badblocks', '-sv', '-e', '1', test.dev.path],
-      pipe=True)
+    pipe=True, bufsize=1)
-    while True:
+  test.badblocks_nbsr = NonBlockingStreamReader(test.badblocks_proc.stderr)
-      try:
+  test.badblocks_stderr = ''
-        test.badblocks_proc.wait(timeout=1)
-      except subprocess.TimeoutExpired:
-        fix_tmux_panes(state, test.tmux_layout)
-      else:
-        # badblocks finished, exit loop
-        break
 
+  # Update progress loop
+  try:
+    while test.badblocks_proc.poll() is None:
+      _save_badblocks_output()
   except KeyboardInterrupt:
+    run_program(['killall', 'badblocks'], check=False)
     test.aborted = True
 
+  # Save remaining badblocks output
+  _save_badblocks_output(read_all=True)
+
   # Check result and build report
   test.report.append('{BLUE}badblocks{CLEAR}'.format(**COLORS))
-  try:
+  for line in test.badblocks_stderr.splitlines():
-    test.badblocks_out = test.badblocks_proc.stdout.read().decode()
-  except Exception as err:
-    test.badblocks_out = 'Error: {}'.format(err)
-  for line in test.badblocks_out.splitlines():
     line = line.strip()
     if not line or re.search(r'^Checking', line, re.IGNORECASE):
       # Skip empty and progress lines
@@ -964,7 +996,7 @@ def run_badblocks_test(state, test):
   update_progress_pane(state)
 
   # Cleanup
-  tmux_kill_pane(state.panes['badblocks'])
+  tmux_kill_pane(state.panes.pop('badblocks', None))
 
 
 def run_hw_tests(state):
@@ -979,6 +1011,7 @@ def run_hw_tests(state):
   # Build Panes
   update_progress_pane(state)
   build_outer_panes(state)
+  start_tmux_repair_thread(state)
 
   # Show selected tests and create TestObj()s
   print_info('Selected Tests:')
@@ -1031,16 +1064,22 @@ def run_hw_tests(state):
         f = v['Function']
         for test_obj in v['Objects']:
           f(state, test_obj)
+        if not v['Objects']:
+          # No devices available
+          v['Objects'].append(TestObj(dev=None, label=''))
+          v['Objects'][-1].update_status('N/A')
       if k == TESTS_CPU[-1]:
         # Last CPU test run, post CPU results
         state.ost.post_device_results(state.cpu, state.ticket_id)
   except GenericAbort:
     # Cleanup
+    stop_tmux_repair_thread(state)
     tmux_kill_pane(*state.panes.values())
 
     # Rebuild panes
     update_progress_pane(state)
     build_outer_panes(state)
+    start_tmux_repair_thread(state)
 
     # Mark unfinished tests as aborted
     for k, v in state.tests.items():
@@ -1087,6 +1126,7 @@ def run_hw_tests(state):
     print_standard(' ')
 
   # Done
+  sleep(1)
   if state.quick_mode:
     pause('Press Enter to exit... ')
   else:
@@ -1094,6 +1134,7 @@ def run_hw_tests(state):
 
   # Cleanup
   state.ost.disconnect(full=True)
+  stop_tmux_repair_thread(state)
   tmux_kill_pane(*state.panes.values())
 
 
@@ -1114,11 +1155,7 @@ def run_io_benchmark(state, test):
     state.panes['Top'],
     text='{}\n{}'.format(
       TOP_PANE_TEXT, test.dev.description))
-  test.tmux_layout = TMUX_LAYOUT.copy()
+  state.tmux_layout['Current'] = {'y': 15, 'Check': True}
-  test.tmux_layout.update({
-    'io_benchmark': {'y': 1000, 'Check': False},
-    'Current': {'y': 15, 'Check': True},
-    })
 
   # Create monitor pane
   test.io_benchmark_out = '{}/io_benchmark_{}.out'.format(
@@ -1175,9 +1212,6 @@ def run_io_benchmark(state, test):
       # Update offset
       offset += test.dev.dd_chunk_blocks + skip
 
-      # Fix panes
-      fix_tmux_panes(state, test.tmux_layout)
-
   except DeviceTooSmallError:
     # Device too small, skipping test
     test.update_status('N/A')
@@ -1252,7 +1286,8 @@ def run_io_benchmark(state, test):
   update_progress_pane(state)
 
   # Cleanup
-  tmux_kill_pane(state.panes['io_benchmark'])
+  state.tmux_layout.pop('Current', None)
+  tmux_kill_pane(state.panes.pop('io_benchmark', None))
 
 
 def run_keyboard_test():
@@ -1278,12 +1313,6 @@ def run_mprime_test(state, test):
   tmux_update_pane(
     state.panes['Top'],
     text='{}\n{}'.format(TOP_PANE_TEXT, test.dev.name))
-  test.tmux_layout = TMUX_LAYOUT.copy()
-  test.tmux_layout.update({
-    'Temps': {'y': 1000, 'Check': False},
-    'mprime': {'y': 11, 'Check': False},
-    'Current': {'y': 3, 'Check': True},
-    })
 
   # Start live sensor monitor
   test.sensors_out = '{}/sensors.out'.format(global_vars['TmpDir'])
@@ -1296,11 +1325,12 @@ def run_mprime_test(state, test):
     pipe=True)
 
   # Create monitor and worker panes
-  state.panes['mprime'] = tmux_split_window(
+  state.panes['Prime95'] = tmux_split_window(
     lines=10, vertical=True, text=' ')
   state.panes['Temps'] = tmux_split_window(
     behind=True, percent=80, vertical=True, watch=test.sensors_out)
   tmux_resize_pane(global_vars['Env']['TMUX_PANE'], y=3)
+  state.tmux_layout['Current'] = {'y': 3, 'Check': True}
 
   # Get idle temps
   clear_screen()
@@ -1315,7 +1345,7 @@ def run_mprime_test(state, test):
   test.abort_msg = 'If running too hot, press CTRL+c to abort the test'
   run_program(['apple-fans', 'max'])
   tmux_update_pane(
-    state.panes['mprime'],
+    state.panes['Prime95'],
     command=['hw-diags-prime95', global_vars['TmpDir']],
     working_dir=global_vars['TmpDir'])
   time_limit = int(MPRIME_LIMIT) * 60
@@ -1337,9 +1367,6 @@ def run_mprime_test(state, test):
       print('{YELLOW}{msg}{CLEAR}'.format(msg=test.abort_msg, **COLORS))
       update_sensor_data(test.sensor_data)
 
-      # Fix panes
-      fix_tmux_panes(state, test.tmux_layout)
-
       # Wait
       sleep(1)
   except KeyboardInterrupt:
@@ -1357,7 +1384,7 @@ def run_mprime_test(state, test):
   # Stop Prime95 (twice for good measure)
   run_program(['killall', '-s', 'INT', 'mprime'], check=False)
   sleep(1)
-  tmux_kill_pane(state.panes['mprime'])
+  tmux_kill_pane(state.panes.pop('Prime95', None))
 
   # Get cooldown temp
   run_program(['apple-fans', 'auto'])
@@ -1451,7 +1478,11 @@ def run_mprime_test(state, test):
   update_progress_pane(state)
 
   # Cleanup
-  tmux_kill_pane(state.panes['mprime'], state.panes['Temps'])
+  state.tmux_layout.pop('Current', None)
+  tmux_kill_pane(
+    state.panes.pop('Prime95', None),
+    state.panes.pop('Temps', None),
+    )
   test.monitor_proc.kill()
 
 
@@ -1480,10 +1511,6 @@ def run_nvme_smart_tests(state, test):
     state.panes['Top'],
     text='{}\n{}'.format(
       TOP_PANE_TEXT, test.dev.description))
-  test.tmux_layout = TMUX_LAYOUT.copy()
-  test.tmux_layout.update({
-    'smart': {'y': 3, 'Check': True},
-    })
 
   # NVMe
   if test.dev.nvme_attributes:
@@ -1523,7 +1550,7 @@ def run_nvme_smart_tests(state, test):
         global_vars['LogDir'], test.dev.name)
       with open(test.smart_out, 'w') as f:
         f.write('SMART self-test status:\n  Starting...')
-      state.panes['smart'] = tmux_split_window(
+      state.panes['SMART'] = tmux_split_window(
         lines=3, vertical=True, watch=test.smart_out)
 
       # Show attributes
@@ -1538,15 +1565,8 @@ def run_nvme_smart_tests(state, test):
 
       # Monitor progress
       try:
-        for i in range(int(test.timeout*60)):
+        for i in range(int(test.timeout*60/5)):
-          sleep(1)
+          sleep(5)
-
-          # Fix panes
-          fix_tmux_panes(state, test.tmux_layout)
-
-          # Only update SMART progress every 5 seconds
-          if i % 5 != 0:
-            continue
 
           # Update SMART data
           test.dev.get_smart_details()
@@ -1596,7 +1616,7 @@ def run_nvme_smart_tests(state, test):
           test.dev.disable_test(t, 'Denied')
 
       # Cleanup
-      tmux_kill_pane(state.panes['smart'])
+      tmux_kill_pane(state.panes.pop('SMART', None))
 
   # Save report
   test.report = test.dev.generate_attribute_report(
@@ -1651,11 +1671,27 @@ def show_results(state):
     for disk in state.disks:
       show_report(disk.generate_disk_report(), log_report=True)
       print_standard(' ')
+    if not state.disks:
+      print_warning('No devices')
+      print_standard(' ')
 
   # Update progress
   update_progress_pane(state)
 
 
+def start_tmux_repair_thread(state):
+  """Fix tmux panes as long as state.tmux_layout attribute exists."""
+  state.tmux_layout = TMUX_LAYOUT.copy()
+  start_thread(fix_tmux_panes_loop, args=[state])
+
+
+def stop_tmux_repair_thread(state):
+  """Stop previous thread by causing an AttributeError in the thread."""
+  if hasattr(state, 'tmux_layout'):
+    del state.tmux_layout
+    sleep(1)
+
+
 def update_main_options(state, selection, main_options):
   """Update menu and state based on selection."""
   index = int(selection) - 1

.bin/Scripts/functions/sensors.py

@@ -108,14 +108,27 @@ def get_raw_sensor_data():
   """Read sensor data and return dict."""
   data = {}
   cmd = ['sensors', '-j']
+
+  # Get raw data
   try:
     result = run_program(cmd)
-    data = json.loads(result.stdout.decode())
   except subprocess.CalledProcessError:
     # Assuming no sensors available, return empty dict below
     pass
 
-  return data
+  # Workaround for bad sensors
+  raw_data = []
+  for line in result.stdout.decode().splitlines():
+    if line.strip() == ',':
+      # Assuming malformatted line caused by missing data
+      continue
+    raw_data.append(line)
+
+  # Parse JSON data
+  json_data = json.loads('\n'.join(raw_data))
+
+  # Done
+  return json_data
 
 
 def get_sensor_data():
@@ -134,6 +147,9 @@ def get_sensor_data():
     ## current temp is labeled xxxx_input
     for _source, _labels in _sources.items():
       for _label, _temp in _labels.items():
+        if _label.startswith('fan'):
+          # Skip fan RPMs
+          continue
         if 'input' in _label:
           sensor_data[_section][_adapter][_source] = {
             'Current': _temp,

.bin/Scripts/functions/threading.py

@@ -0,0 +1,47 @@
+# Wizard Kit: Functions - Threading
+
+from threading import Thread
+from queue import Queue, Empty
+
+# Classes
+class NonBlockingStreamReader():
+  """Class to allow non-blocking reads from a stream."""
+  # Credits:
+  ## https://gist.github.com/EyalAr/7915597
+  ## https://stackoverflow.com/a/4896288
+
+  def __init__(self, stream):
+    self.stream = stream
+    self.queue = Queue()
+
+    def populate_queue(stream, queue):
+      """Collect lines from stream and put them in queue."""
+      while True:
+        line = stream.read(1)
+        if line:
+          queue.put(line)
+
+    self.thread = start_thread(
+      populate_queue,
+      args=(self.stream, self.queue))
+
+  def read(self, timeout=None):
+    try:
+      return self.queue.get(block = timeout is not None,
+          timeout = timeout)
+    except Empty:
+      return None
+
+
+# Functions
+def start_thread(function, args=[], daemon=True):
+  """Run function as thread in background, returns Thread object."""
+  thread = Thread(target=function, args=args, daemon=daemon)
+  thread.start()
+  return thread
+
+
+if __name__ == '__main__':
+  print("This file is not meant to be called directly.")
+
+# vim: sts=2 sw=2 ts=2

.bin/Scripts/functions/tmux.py

@@ -44,6 +44,9 @@ def tmux_kill_pane(*panes):
   """Kill tmux pane by id."""
   cmd = ['tmux', 'kill-pane', '-t']
   for pane_id in panes:
+    if not pane_id:
+      # Skip empty strings, None values, etc
+      continue
     run_program(cmd+[pane_id], check=False)
 
 

.bin/Scripts/hw-diags-badblocks

@@ -1,18 +0,0 @@
-#!/bin/bash
-#
-## Wizard Kit: HW Diagnostics - badblocks
-
-function usage {
-    echo "Usage: $0 device log-file"
-    echo "  e.g. $0 /dev/sda /tmp/tmp.XXXXXXX/badblocks.log"
-}
-
-# Bail early
-if [ ! -b "$1" ]; then
-    usage
-    exit 1
-fi
-
-# Run Badblocks
-sudo badblocks -sv -e 1 "$1" 2>&1 | tee -a "$2"
-

.bin/Scripts/hw-info

@@ -105,8 +105,3 @@ echo -e "${BLUE}Drives${CLEAR}"
 hw-drive-info | sed 's/^/    /'
 echo ""
 
-# Sensors
-echo -e "${BLUE}Sensors${CLEAR}"
-hw-sensors | sed 's/^/    /'
-echo ""
-

.bin/Scripts/hw-sensors-monitor

@@ -10,7 +10,7 @@ os.chdir(os.path.dirname(os.path.realpath(__file__)))
 sys.path.append(os.getcwd())
 from functions.sensors import *
 from functions.tmux import *
-init_global_vars()
+init_global_vars(silent=True)
 
 if __name__ == '__main__':
   background = False

.linux_items/packages/aur

@@ -2,6 +2,7 @@ aic94xx-firmware
 bash-pipes
 hfsprogs
 i3lock-fancy-git
+macbook12-spi-driver-dkms
 mprime
 openbox-patched
 smartmontools-svn

.linux_items/packages/live_add_min

@@ -0,0 +1 @@
+macbook12-spi-driver-dkms

Build Linux

@@ -198,7 +198,9 @@ function update_live_env() {
         sed -i "/$p/d" "$LIVE_DIR/packages.x86_64"
     done < "$ROOT_DIR/.linux_items/packages/live_remove"
     cat "$ROOT_DIR/.linux_items/packages/live_add" >> "$LIVE_DIR/packages.x86_64"
-    if [[ "${1:-}" != "--minimal" ]]; then
+    if [[ "${1:-}" == "--minimal" ]]; then
+        cat "$ROOT_DIR/.linux_items/packages/live_add_min" >> "$LIVE_DIR/packages.x86_64"
+    else
         cat "$ROOT_DIR/.linux_items/packages/live_add_x" >> "$LIVE_DIR/packages.x86_64"
     fi
     echo "[custom]" >> "$LIVE_DIR/pacman.conf"