New osTicket report layout

* Block character graph has been reduced to one line
* A PNG graph is exported using gnuplot
* The graph is uploaded to imgur and a link is included in the report
* The graph is also uploaded to the BENCHMARK_SERVER for redundancy

.bin/Scripts/functions/hw_diags.py

@@ -1,10 +1,14 @@
 # Wizard Kit: Functions - HW Diagnostics
 
+import base64
+import Gnuplot
 import json
+import math
 import time
 import mysql.connector as mariadb
 
 from functions.common import *
+from numpy import *
 
 # Database connection
 ost_db = {
@@ -43,8 +47,9 @@ IO_VARS = {
     'Minimum Test Size': 10*1024**3,
     'Alt Test Size Factor': 0.01,
     'Progress Refresh Rate': 5,
-    'Scale 16': [2**(0.56*x)+(16*x) for x in range(1,17)],
+    'Scale 8': [2**(0.56*(x+1))+(16*(x+1)) for x in range(8)],
-    'Scale 32': [2**(0.56*x/2)+(16*x/2) for x in range(1,33)],
+    'Scale 16': [2**(0.56*(x+1))+(16*(x+1)) for x in range(16)],
+    'Scale 32': [2**(0.56*(x+1)/2)+(16*(x+1)/2) for x in range(32)],
     'Threshold Fail': 65*1024**2,
     'Threshold Warn': 135*1024**2,
     'Threshold Great': 750*1024**2,
@@ -87,6 +92,7 @@ TESTS = {
         },
     'iobenchmark': {
         'Enabled': False,
+        'Rates': {},
         'Results': {},
         'Status': {},
         },
@@ -126,7 +132,37 @@ def disconnect_from_db():
     if ost_db['Tunnel']:
         ost_db['Tunnel'].kill()
 
-def generate_horizontal_graph(rates):
+def export_png_graph(name, dev):
+    """Exports PNG graph using gnuplot, returns file path as str."""
+    max_rate = max(800, max(rates))
+    out_path = '{}/iobenchmark.png'.format(global_vars['TmpDir'])
+    plot_data = '{}/iobenchmark-{}-raw.log'.format(global_vars['LogDir'], name)
+
+    # Adjust Y-axis range to either 1000 or roughly max rate + 200
+    y_range = math.ceil(max_rate/100)*100 + 200
+
+    # Run gnuplot commands
+    g = Gnuplot.Gnuplot()
+    g('set output "{}"'.format(out_path))
+    g('set terminal png large size 660,300 truecolor font "Noto Sans,11"')
+    g('set title "I/O Benchmark"')
+    g('set yrange [0:{}]'.format(y_range))
+    g('set style data lines')
+    g('plot "{data}" title "{size} ({tran}) {model} {serial}"'.format(
+        data=plot_data,
+        size=dev['lsblk'].get('size', '???b'),
+        tran=dev['lsblk'].get('tran', '???'),
+        model=dev['lsblk'].get('model', 'Unknown Model'),
+        serial=dev['lsblk'].get('serial', 'Unknown Serial'),
+        ))
+
+    # Cleanup
+    g.close()
+    del(g)
+
+    return out_path
+
+def generate_horizontal_graph(rates, oneline=False):
     """Generate two-line horizontal graph from rates, returns str."""
     line_1 = ''
     line_2 = ''
@@ -134,6 +170,8 @@ def generate_horizontal_graph(rates):
     line_4 = ''
     for r in rates:
         step = get_graph_step(r, scale=32)
+        if oneline:
+            step = get_graph_step(r, scale=8)
 
         # Set color
         r_color = COLORS['CLEAR']
@@ -170,6 +208,9 @@ def generate_horizontal_graph(rates):
     line_2 += COLORS['CLEAR']
     line_3 += COLORS['CLEAR']
     line_4 += COLORS['CLEAR']
+    if oneline:
+        return line_4
+    else:
         return '\n'.join([line_1, line_2, line_3, line_4])
 
 def get_graph_step(rate, scale=16):
@@ -550,19 +591,39 @@ def post_drive_results(ticket_number):
         # I/O Benchmark
         io_status = TESTS['iobenchmark']['Status'].get(name, None)
         if TESTS['iobenchmark']['Enabled'] and io_status not in ['Denied', 'Skipped']:
-            report.append('I/O Benchmark:')
+            one_line_graph = generate_horizontal_graph(
-            io_result = TESTS['iobenchmark']['Results'].get(
+                rates=TESTS['iobenchmark']['Data'][name]['Read Rates'],
-                name,
+                oneline=True)
-                'ERROR: Failed to read log.')
-            for line in io_result.splitlines():
-                line = line.strip()
-                if not line:
-                    continue
-                # Strip colors from line
             for c in COLORS.values():
-                    line = line.replace(c, '')
+                one_line_graph = one_line_graph.replace(c, '')
-                report.append(line)
+            report.append('I/O Benchmark:')
-            report.append('')
+            report.append(one_line_graph)
+            report.append('{}'.format(
+                TESTS['iobenchmark']['Data'][name]['Avg/Min/Max']))
+
+            # Export PNG
+            try:
+                png_path = export_png_graph(name, dev)
+            except:
+                png_path = None
+
+            # imgur
+            try:
+                url = upload_to_imgur(image_path)
+            except:
+                # Oh well
+                pass
+            else:
+                report.append('Imgur: {}'.format(url))
+
+            # Nextcloud
+            try:
+                url = upload_to_nextcloud(image_path, ticket_number, name)
+            except:
+                # Oh well
+                pass
+            else:
+                report.append('Nextcloud: {}'.format(url))
 
         # TODO-REMOVE TESTING
         with open('/home/twoshirt/__ost_report_{}.txt'.format(name), 'w') as f:
@@ -744,7 +805,9 @@ def run_iobenchmark(ticket_number):
 
             # Run dd read tests
             offset = 0
-            read_rates = []
+            TESTS['iobenchmark']['Data'][name] = {
+                'Graph': [],
+                'Read Rates': []}
             for i in range(test_chunks):
                 i += 1
                 s = skip_count
@@ -759,12 +822,18 @@ def run_iobenchmark(ticket_number):
                     o='/dev/null')
                 result = run_program(cmd.split())
                 result_str = result.stderr.decode().replace('\n', '')
-                read_rates.append(get_read_rate(result_str))
+                cur_rate = get_read_rate(result_str)
+                TESTS['iobenchmark']['Data'][name]['Read Rates'].append(
+                    cur_rate)
+                TESTS['iobenchmark']['Data'][name]['Graph'].append(
+                    '{percent} {rate}'.format(
+                        percent=i/test_chunks*100,
+                        rate=cur_rate/(1024**2)))
                 if i % IO_VARS['Progress Refresh Rate'] == 0:
                     # Update vertical graph
                     update_io_progress(
                         percent=i/test_chunks*100,
-                        rate=read_rates[-1],
+                        rate=cur_rate,
                         progress_file=progress_file)
                 # Update offset
                 offset += s + c
@@ -774,24 +843,28 @@ def run_iobenchmark(ticket_number):
             run_program(['tmux', 'kill-pane', '-t', bottom_pane])
 
             # Build report
+            avg_min_max = 'Average read speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format(
+                sum(TESTS['iobenchmark']['Data'][name]['Read Rates'])/len(
+                    TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2),
+                min(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2),
+                max(TESTS['iobenchmark']['Data'][name]['Read Rates'])/(1024**2))
+            TESTS['iobenchmark']['Data'][name]['Avg/Min/Max'] = avg_min_max
             h_graph_rates = []
             pos = 0
             width = int(test_chunks / IO_VARS['Graph Horizontal Width'])
             for i in range(IO_VARS['Graph Horizontal Width']):
                 # Append average rate for WIDTH number of rates to new array
-                h_graph_rates.append(sum(read_rates[pos:pos+width])/width)
+                h_graph_rates.append(sum(
+                    TESTS['iobenchmark']['Data'][name]['Read Rates'][pos:pos+width])/width)
                 pos += width
             report = generate_horizontal_graph(h_graph_rates)
-            report += '\nAverage read speed: {:3.1f} MB/s (Min: {:3.1f}, Max: {:3.1f})'.format(
+            report += '\n{}'.format(avg_min_max)
-                sum(read_rates)/len(read_rates)/(1024**2),
-                min(read_rates)/(1024**2),
-                max(read_rates)/(1024**2))
             TESTS['iobenchmark']['Results'][name] = report
 
             # Set CS/NS
-            if min(read_rates) <= IO_VARS['Threshold Fail']:
+            if min(TESTS['iobenchmark']['Data'][name]['Read Rates']) <= IO_VARS['Threshold Fail']:
                 TESTS['iobenchmark']['Status'][name] = 'NS'
-            elif min(read_rates) <= IO_VARS['Threshold Warn']:
+            elif min(TESTS['iobenchmark']['Data'][name]['Read Rates']) <= IO_VARS['Threshold Warn']:
                 TESTS['iobenchmark']['Status'][name] = 'Unknown'
             else:
                 TESTS['iobenchmark']['Status'][name] = 'CS'
@@ -800,8 +873,7 @@ def run_iobenchmark(ticket_number):
             dest_filename = '{}/iobenchmark-{}.log'.format(global_vars['LogDir'], name)
             shutil.move(progress_file, dest_filename)
             with open(dest_filename.replace('.', '-raw.'), 'a') as f:
-                for rate in read_rates:
+                f.write('\n'.join(TESTS['iobenchmark']['Data'][name]['Graph']))
-                    f.write('{} MB/s\n'.format(rate/(1024**2)))
         update_progress()
 
     # Done
@@ -1417,6 +1489,64 @@ def update_progress():
     with open(TESTS['Progress Out'], 'w') as f:
         f.writelines(output)
 
+def upload_to_imgur(image_path):
+    """Upload image to Imgur and return image url as str."""
+    image_data = None
+    image_link = None
+
+    # Bail early
+    if not image_path:
+        raise GenericError
+
+    # Read image file and convert to base64
+    with open(image_path, 'rb') as f:
+        image_data = base64.b64encode(f.read())
+
+    # POST image
+    url = "https://api.imgur.com/3/image"
+    boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW'
+    payload = ('--{boundary}\r\nContent-Disposition: form-data; '
+               'name="image"\r\n\r\n{data}\r\n--{boundary}--')
+    payload = payload.format(boundary=boundary, data=image_data)
+    headers = {
+        'content-type': 'multipart/form-data; boundary={}'.format(boundary),
+        'Authorization': 'Client-ID {}'.format(IMGUR_CLIENT_ID),
+        }
+    response = requests.request("POST", url, data=payload, headers=headers)
+
+    # Return image link
+    if response.ok:
+        json_data = json.loads(response.text)
+        image_link = json_data['data']['link']
+    return image_link
+
+def upload_to_nextcloud(image_path, ticket_number, dev_name):
+    """Upload image to Nextcloud server and return folder url as str."""
+    image_data = None
+
+    # Bail early
+    if not image_path:
+        raise GenericError
+
+    # Read image file and convert to base64
+    with open(image_path, 'rb') as f:
+        image_data = f.read()
+
+    # PUT image
+    url = '{base_url}/{ticket_number}_iobenchmark_{dev_name}_{date}'.format(
+        base_url=BENCHMARK_SERVER['Url'],
+        ticket_number=ticket_number,
+        dev_name=dev_name,
+        date=global_vars.get('Date-Time', 'Unknown Date-Time'))
+    requests.put(
+        url,
+        data=image_data,
+        headers = {'X-Requested-With': 'XMLHttpRequest'},
+        auth = (BENCHMARK_SERVER['User'], BENCHMARK_SERVER['Pass']))
+
+    # Return folder link
+    return BENCHMARK_SERVER['Short Url']
+
 if __name__ == '__main__':
     print("This file is not meant to be called directly.")
 

.bin/Scripts/settings/main.py

@@ -18,6 +18,8 @@ DB_USER='wizardkit'
 DB_PASS='Abracadabra'
 SSH_PORT='22'
 SSH_USER='sql_tunnel'
+# imgur
+IMGUR_CLIENT_ID=''
 # Live Linux
 MPRIME_LIMIT='7'                          # of minutes to run Prime95 during hw-diags
 ROOT_PASSWORD='Abracadabra'
@@ -56,6 +58,13 @@ BACKUP_SERVERS = [
         'RW-Pass':  'Abracadabra',
     },
 ]
+BENCHMARK_SERVER = {
+    'Name':         'BenchmarkServer',
+    'Short Url':    '',
+    'Url':          '',
+    'User':         '',
+    'Pass':         '',
+}
 CRASH_SERVER = {
     'Name':         'CrashServer',
     'Url':          '',