Merge remote-tracking branch 'upstream/dev' into dev

scripts/wk/exe.py

@@ -32,8 +32,12 @@ class NonBlockingStreamReader():
 
     def populate_queue(stream, queue):
       """Collect lines from stream and put them in queue."""
-      while True:
+      while not stream.closed:
+        try:
           line = stream.read(1)
+        except ValueError:
+          # Assuming the stream was closed
+          line = None
         if line:
           queue.put(line)
 
@@ -42,6 +46,10 @@ class NonBlockingStreamReader():
       args=(self.stream, self.queue),
       )
 
+  def stop(self):
+    """Stop reading from input stream."""
+    self.stream.close()
+
   def read(self, timeout=None):
     """Read from queue if possible, returns item from queue."""
     try:
@@ -51,6 +59,7 @@ class NonBlockingStreamReader():
 
   def save_to_file(self, proc, out_path):
     """Continuously save output to file while proc is running."""
+    LOG.debug('Saving process %s output to %s', proc, out_path)
     while proc.poll() is None:
       out = b''
       out_bytes = b''
@@ -61,6 +70,9 @@ class NonBlockingStreamReader():
       with open(out_path, 'a') as _f:
         _f.write(out_bytes.decode('utf-8', errors='ignore'))
 
+    # Close stream to prevent 100% CPU usage
+    self.stream.close()
+
 
 # Functions
 def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs):
@@ -70,10 +82,9 @@ def build_cmd_kwargs(cmd, minimized=False, pipe=True, shell=False, **kwargs):
   NOTE: If no encoding specified then UTF-8 will be used.
   """
   LOG.debug(
-    'cmd: %s, minimized: %s, pipe: %s, shell: %s',
+    'cmd: %s, minimized: %s, pipe: %s, shell: %s, kwargs: %s',
-    cmd, minimized, pipe, shell,
+    cmd, minimized, pipe, shell, kwargs,
     )
-  LOG.debug('kwargs: %s', kwargs)
   cmd_kwargs = {
     'args': cmd,
     'shell': shell,
@@ -118,6 +129,7 @@ def get_json_from_command(cmd, check=True, encoding='utf-8', errors='ignore'):
   If the data can't be decoded then either an exception is raised
   or an empty dict is returned depending on errors.
   """
+  LOG.debug('Loading JSON data from cmd: %s', cmd)
   json_data = {}
 
   try:
@@ -187,9 +199,15 @@ def popen_program(cmd, minimized=False, pipe=False, shell=False, **kwargs):
     pipe=pipe,
     shell=shell,
     **kwargs)
+  try:
+    proc = subprocess.Popen(**cmd_kwargs)
+  except FileNotFoundError:
+    LOG.error('Command not found: %s', cmd)
+    raise
+  LOG.debug('proc: %s', proc)
 
-  # Ready to run program
+  # Done
-  return subprocess.Popen(**cmd_kwargs)
+  return proc
 
 
 def run_program(cmd, check=True, pipe=True, shell=False, **kwargs):
@@ -206,7 +224,11 @@ def run_program(cmd, check=True, pipe=True, shell=False, **kwargs):
     pipe=pipe,
     shell=shell,
     **kwargs)
+  try:
     proc = subprocess.run(**cmd_kwargs)
+  except FileNotFoundError:
+    LOG.error('Command not found: %s', cmd)
+    raise
   LOG.debug('proc: %s', proc)
 
   # Done
@@ -215,6 +237,10 @@ def run_program(cmd, check=True, pipe=True, shell=False, **kwargs):
 
 def start_thread(function, args=None, daemon=True):
   """Run function as thread in background, returns Thread object."""
+  LOG.debug(
+    'Starting background thread for function: %s, args: %s, daemon: %s',
+    function, args, daemon,
+    )
   args = args if args else []
   thread = Thread(target=function, args=args, daemon=daemon)
   thread.start()

scripts/wk/hw/diags.py

@@ -132,7 +132,7 @@ class State():
     self.tests = OrderedDict({
       'CPU & Cooling': {
         'Enabled': False,
-        'Function': cpu_mprime_test,
+        'Function': cpu_stress_tests,
         'Objects': [],
         },
       'Disk Attributes': {
@@ -331,7 +331,7 @@ class State():
       if not details['Selected']:
         continue
       if 'CPU' in name:
-        # Create two Test objects which will both be used by cpu_mprime_test
+        # Create two Test objects which will both be used by cpu_stress_tests
         # NOTE: Prime95 should be added first
         test_mprime_obj = hw_obj.Test(dev=self.cpu, label='Prime95')
         test_cooling_obj = hw_obj.Test(dev=self.cpu, label='Cooling')
@@ -597,9 +597,12 @@ def calc_io_dd_values(dev_size):
     }
 
 
-def check_cooling_results(test_obj, sensors):
+def check_cooling_results(test_obj, sensors, run_sysbench=False):
   """Check cooling results and update test_obj."""
   max_temp = sensors.cpu_max_temp()
+  temp_labels = ['Idle', 'Max', 'Cooldown']
+  if run_sysbench:
+    temp_labels.append('Sysbench')
 
   # Check temps
   if not max_temp:
@@ -612,8 +615,7 @@ def check_cooling_results(test_obj, sensors):
     test_obj.set_status('Passed')
 
   # Add temps to report
-  for line in sensors.generate_report(
+  for line in sensors.generate_report(*temp_labels, only_cpu=True):
-      'Idle', 'Max', 'Cooldown', only_cpu=True):
     test_obj.report.append(f'  {line}')
 
 
@@ -767,12 +769,13 @@ def check_self_test_results(test_obj, aborted=False):
         test_obj.set_status('Failed')
 
 
-def cpu_mprime_test(state, test_objects):
+def cpu_stress_tests(state, test_objects):
   # pylint: disable=too-many-statements
-  """CPU & cooling check using Prime95."""
+  """CPU & cooling check using Prime95 and Sysbench."""
   LOG.info('CPU Test (Prime95)')
   aborted = False
   prime_log = pathlib.Path(f'{state.log_dir}/prime.log')
+  run_sysbench = False
   sensors_out = pathlib.Path(f'{state.log_dir}/sensors.out')
   test_mprime_obj, test_cooling_obj = test_objects
 
@@ -795,7 +798,7 @@ def cpu_mprime_test(state, test_objects):
   # Create monitor and worker panes
   state.update_progress_pane()
   state.panes['Prime95'] = tmux.split_window(
-    lines=10, vertical=True, watch_file=prime_log)
+    lines=10, vertical=True, watch_file=prime_log, watch_cmd='tail')
   if PLATFORM == 'Darwin':
     state.panes['Temps'] = tmux.split_window(
       behind=True, percent=80, vertical=True, cmd='./hw-sensors')
@@ -811,7 +814,7 @@ def cpu_mprime_test(state, test_objects):
   sensors.save_average_temps(temp_label='Idle', seconds=5)
 
   # Stress CPU
-  std.print_info('Starting stress test')
+  std.print_info('Running stress test')
   set_apple_fan_speed('max')
   proc_mprime = start_mprime(state.log_dir, prime_log)
 
@@ -842,9 +845,43 @@ def cpu_mprime_test(state, test_objects):
   test_mprime_obj.report.append(std.color_string('Prime95', 'BLUE'))
   check_mprime_results(test_obj=test_mprime_obj, working_dir=state.log_dir)
 
+  # Run Sysbench test if necessary
+  run_sysbench = (
+    not aborted and sensors.cpu_max_temp() >= cfg.hw.CPU_FAILURE_TEMP
+    )
+  if run_sysbench:
+    LOG.info('CPU Test (Sysbench)')
+    std.print_standard('Letting CPU cooldown more...')
+    std.sleep(30)
+    std.clear_screen()
+    std.print_info('Running alternate stress test')
+    print('')
+    proc_sysbench, filehandle_sysbench = start_sysbench(
+      sensors,
+      sensors_out,
+      log_path=prime_log.with_name('sysbench.log'),
+      pane=state.panes['Prime95'],
+      )
+    try:
+      print_countdown(proc=proc_sysbench, seconds=cfg.hw.CPU_TEST_MINUTES*60)
+    except AttributeError:
+      # Assuming the sysbench process wasn't found and proc was set to None
+      LOG.error('Failed to find sysbench process', exc_info=True)
+    except KeyboardInterrupt:
+      aborted = True
+    stop_sysbench(proc_sysbench, filehandle_sysbench)
+
+    # Update progress
+    # NOTE: CPU critical temp check isn't really necessary
+    #       Hard to imagine it wasn't hit during Prime95 but was in sysbench
+    if sensors.cpu_reached_critical_temp() or aborted:
+      test_cooling_obj.set_status('Aborted')
+      test_mprime_obj.set_status('Aborted')
+      state.update_progress_pane()
+
   # Check Cooling results
   test_cooling_obj.report.append(std.color_string('Temps', 'BLUE'))
-  check_cooling_results(test_obj=test_cooling_obj, sensors=sensors)
+  check_cooling_results(test_cooling_obj, sensors, run_sysbench)
 
   # Post results to osTicket
   if not state.ost.disabled:
@@ -1608,7 +1645,8 @@ def print_countdown(proc, seconds):
     except subprocess.TimeoutExpired:
       # proc still going, continue
       pass
-    if proc.poll() is not None:
+    if ((hasattr(proc, 'poll') and proc.poll() is not None)
+        or (hasattr(proc, 'is_running') and not proc.is_running())):
       # proc exited, stop countdown
       break
 
@@ -1800,6 +1838,35 @@ def start_mprime(working_dir, log_path):
   return proc_mprime
 
 
+def start_sysbench(sensors, sensors_out, log_path, pane):
+  """Start sysbench, returns tuple with Popen object and file handle."""
+  set_apple_fan_speed('max')
+  sysbench_cmd = [
+    'sysbench',
+    f'--threads={exe.psutil.cpu_count()}',
+    '--cpu-max-prime=1000000000',
+    'cpu',
+    'run',
+    ]
+
+  # Restart background monitor for Sysbench
+  sensors.stop_background_monitor()
+  sensors.start_background_monitor(
+    sensors_out,
+    alt_max='Sysbench',
+    thermal_action=('killall', 'sysbench', '-INT'),
+    )
+
+  # Update bottom pane
+  tmux.respawn_pane(pane, watch_file=log_path, watch_cmd='tail')
+
+  # Start sysbench
+  filehandle_sysbench = open(log_path, 'a')
+  proc_sysbench = exe.popen_program(sysbench_cmd, stdout=filehandle_sysbench)
+
+  # Done
+  return (proc_sysbench, filehandle_sysbench)
+
 def stop_mprime(proc_mprime):
   """Stop mprime gracefully, then forcefully as needed."""
   proc_mprime.terminate()
@@ -1810,6 +1877,18 @@ def stop_mprime(proc_mprime):
   set_apple_fan_speed('auto')
 
 
+def stop_sysbench(proc_sysbench, filehandle_sysbench):
+  """Stop sysbench."""
+  proc_sysbench.terminate()
+  try:
+    proc_sysbench.wait(timeout=5)
+  except subprocess.TimeoutExpired:
+    proc_sysbench.kill()
+  filehandle_sysbench.flush()
+  filehandle_sysbench.close()
+  set_apple_fan_speed('auto')
+
+
 def sync_clock():
   """Sync clock under macOS using sntp."""
   cmd = ['sudo', 'sntp', '-Ss', 'us.pool.ntp.org']

scripts/wk/hw/sensors.py

@@ -10,6 +10,7 @@ from subprocess import CalledProcessError
 
 from wk.cfg.hw import CPU_CRITICAL_TEMP, SMC_IDS, TEMP_COLORS
 from wk.exe import run_program, start_thread
+from wk.io import non_clobber_path
 from wk.std import PLATFORM, color_string, sleep
 
 
@@ -115,20 +116,27 @@ class Sensors():
     return report
 
   def monitor_to_file(
-      self, out_path,
+      self, out_path, alt_max=None,
       exit_on_thermal_limit=True, temp_labels=None, thermal_action=None):
+    # pylint: disable=too-many-arguments
     """Write report to path every second until stopped.
 
     thermal_action is a cmd to run if ThermalLimitReachedError is caught.
     """
     stop_path = pathlib.Path(out_path).resolve().with_suffix('.stop')
+    if stop_path.exists():
+      # Rename existing file to allow thread to start as expected
+      # Yes this is excessive but safe
+      stop_path.rename(non_clobber_path(stop_path))
     if not temp_labels:
-      temp_labels = ('Current', 'Max')
+      temp_labels = ['Current', 'Max']
+    if alt_max:
+      temp_labels.append(alt_max)
 
     # Start loop
     while True:
       try:
-        self.update_sensor_data(exit_on_thermal_limit)
+        self.update_sensor_data(alt_max, exit_on_thermal_limit)
       except ThermalLimitReachedError:
         if thermal_action:
           run_program(thermal_action, check=False)
@@ -169,8 +177,9 @@ class Sensors():
             source_data[temp_label] = 0
 
   def start_background_monitor(
-      self, out_path,
+      self, out_path, alt_max=None,
       exit_on_thermal_limit=True, temp_labels=None, thermal_action=None):
+    # pylint: disable=too-many-arguments
     """Start background thread to save report to file.
 
     thermal_action is a cmd to run if ThermalLimitReachedError is caught.
@@ -181,7 +190,9 @@ class Sensors():
     self.out_path = pathlib.Path(out_path)
     self.background_thread = start_thread(
       self.monitor_to_file,
-      args=(out_path, exit_on_thermal_limit, temp_labels, thermal_action),
+      args=(
+        out_path, alt_max, exit_on_thermal_limit, temp_labels, thermal_action,
+        ),
       )
 
   def stop_background_monitor(self):
@@ -193,14 +204,14 @@ class Sensors():
     self.background_thread = None
     self.out_path = None
 
-  def update_sensor_data(self, exit_on_thermal_limit=True):
+  def update_sensor_data(self, alt_max=None, exit_on_thermal_limit=True):
     """Update sensor data via OS-specific means."""
     if PLATFORM == 'Darwin':
-      self.update_sensor_data_macos(exit_on_thermal_limit)
+      self.update_sensor_data_macos(alt_max, exit_on_thermal_limit)
     elif PLATFORM == 'Linux':
-      self.update_sensor_data_linux(exit_on_thermal_limit)
+      self.update_sensor_data_linux(alt_max, exit_on_thermal_limit)
 
-  def update_sensor_data_linux(self, exit_on_thermal_limit=True):
+  def update_sensor_data_linux(self, alt_max, exit_on_thermal_limit=True):
     """Update sensor data via lm_sensors."""
     lm_sensor_data = get_sensor_data_lm()
     for section, adapters in self.data.items():
@@ -212,6 +223,8 @@ class Sensors():
             source_data['Current'] = temp
             source_data['Max'] = max(temp, source_data['Max'])
             source_data['Temps'].append(temp)
+            if alt_max:
+              source_data[alt_max] = max(temp, source_data.get(alt_max, 0))
           except KeyError:
             # Dumb workaround for Dell sensors with changing source names
             pass
@@ -221,7 +234,7 @@ class Sensors():
             if source_data['Current'] >= CPU_CRITICAL_TEMP:
               raise ThermalLimitReachedError('CPU temps reached limit')
 
-  def update_sensor_data_macos(self, exit_on_thermal_limit=True):
+  def update_sensor_data_macos(self, alt_max, exit_on_thermal_limit=True):
     """Update sensor data via SMC."""
     for section, adapters in self.data.items():
       for sources in adapters.values():
@@ -239,6 +252,8 @@ class Sensors():
           source_data['Current'] = temp
           source_data['Max'] = max(temp, source_data['Max'])
           source_data['Temps'].append(temp)
+          if alt_max:
+            source_data[alt_max] = max(temp, source_data.get(alt_max, 0))
 
           # Raise exception if thermal limit reached
           if exit_on_thermal_limit and section == 'CPUTemps':

setup/linux/packages/base

@@ -68,6 +68,7 @@ smartmontools-svn
 smbclient
 speedtest-cli
 sudo
+sysbench
 sysfsutils
 syslinux
 systemd-sysvcompat

setup/macos/install-deps

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 
 set -o errexit
@@ -103,3 +103,11 @@ git clone https://github.com/yuyichao/gnuplot-py gnuplot-py
 cd gnuplot-py
 git checkout 2c2218dc67
 python3 setup.py install
+
+# Sysbench
+git clone https://github.com/akopytov/sysbench sysbench
+cd sysbench
+./autogen.sh LDFLAGS=-L/usr/local/opt/openssl/lib --without-mysql
+./configure LDFLAGS=-L/usr/local/opt/openssl/lib --without-mysql
+make MACOSX_DEPLOYMENT_TARGET="${OS_VERSION:0:5}" -j
+sudo mv -nv sysbench/src/sysbench /usr/local/bin/

setup/macos/live-macos-startup

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 ## Init macOS env
 

setup/macos/update-base-image

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 ## Update BaseImage for use as WK