132 lines
4.2 KiB
Python
Executable File
132 lines
4.2 KiB
Python
Executable File
#!/usr/bin/env python3.6
|
|
|
|
import pandas as pd
|
|
import matplotlib.pyplot as plt
|
|
from matplotlib import ticker
|
|
import numpy as np
|
|
import sys
|
|
import re
|
|
import os
|
|
import json
|
|
import libpar as par
|
|
import getopt
|
|
import math
|
|
import concurrent.futures as CF
|
|
|
|
def process_dir(rootdir):
|
|
ret = []
|
|
print("Processing directory " + rootdir + " ...")
|
|
for subdir in os.listdir(rootdir):
|
|
each_dir = os.path.join(rootdir, subdir)
|
|
if os.path.isfile(each_dir) and each_dir.endswith(".txt"):
|
|
output = None
|
|
try:
|
|
with open(each_dir, 'r') as f:
|
|
if len(f.readlines()) <= 1:
|
|
print("Skipping empty file - " + each_dir)
|
|
continue
|
|
|
|
with open(each_dir, 'r') as f:
|
|
output = f.read()
|
|
parser = par.khat_parser()
|
|
parser.parse(output)
|
|
print("Processed raw data - " + each_dir)
|
|
ret.append(parser)
|
|
except:
|
|
print("Unrecognized format - " + subdir)
|
|
|
|
print("")
|
|
return ret
|
|
|
|
|
|
marker_map = ["o", "P", "s", "v", "*", "+", "^", "1", "2", "d", "X", "o", "P", "s", "v", "*", "+", "^", "1", "2", "d", "X"]
|
|
color_map = ["xkcd:black", "xkcd:red", "xkcd:blue", "xkcd:green", "xkcd:cyan", "xkcd:purple", "xkcd:orange", "xkcd:salmon", "xkcd:lightgreen", "xkcd:indigo", "xkcd:brown", "xkcd:bubblegum", "xkcd:lavender", "xkcd:maroon", "xkcd:fern", "xkcd:sky", "xkcd:orchid", "xkcd:sienna"]
|
|
parser_idx_labels = ["srv_hw", "srv_sw", "clt_hw", "clt_sw"]
|
|
|
|
def add_curve(eax, label : str, qps_arr : [], lat_arr : [], marker : str, color : str):
|
|
df_dict = {}
|
|
df_dict['qps'] = qps_arr
|
|
df_dict['lat'] = lat_arr
|
|
|
|
df = pd.DataFrame(df_dict)
|
|
df = df.sort_values('qps')
|
|
eax.plot('qps', 'lat', data = df, label=label, marker=marker, color=color, markersize=8)
|
|
|
|
# adds curves (avg and 99th percentile) for a specific parser idx
|
|
def add_curves(rax, label : str, parsers : [], parser_idx : int, marker : str, color : str):
|
|
qps_arr = []
|
|
avg_arr = []
|
|
p99_arr = []
|
|
|
|
for parser in parsers:
|
|
qps_arr.append(parser.qps)
|
|
each_lat_arr = []
|
|
each_lat_arr.extend(parser.get_stat_arr(parser_idx))
|
|
avg_arr.append(np.mean(each_lat_arr))
|
|
p99_arr.append(np.percentile(each_lat_arr, 99))
|
|
|
|
add_curve(rax[0], label, qps_arr, avg_arr, marker, color)
|
|
add_curve(rax[1], label, qps_arr, p99_arr, marker, color)
|
|
|
|
|
|
# generate the graphs for a parser index
|
|
def generate_graph(aff_to_parser : {}, parser_idx : int, fn : str):
|
|
marker_idx = 0
|
|
color_idx = 0
|
|
|
|
fig, rax = plt.subplots(2, 1)
|
|
rax[0].set_yscale("log")
|
|
rax[0].set_title("Average")
|
|
rax[0].set_xlabel("QPS")
|
|
rax[0].set_ylabel("Latency (ns)")
|
|
rax[0].xaxis.get_major_formatter().set_scientific(False)
|
|
rax[0].yaxis.set_minor_formatter(ticker.ScalarFormatter())
|
|
rax[1].set_yscale("log")
|
|
rax[1].set_title("99th percentile")
|
|
rax[1].set_xlabel("QPS")
|
|
rax[1].set_ylabel("Latency (ns)")
|
|
rax[1].xaxis.get_major_formatter().set_scientific(False)
|
|
rax[1].yaxis.set_minor_formatter(ticker.ScalarFormatter())
|
|
|
|
print("Generating graph => " + fn + "...")
|
|
for aff in aff_to_parser:
|
|
# each affinity gets a different marker type
|
|
marker_type = marker_map[marker_idx]
|
|
color_type = color_map[color_idx]
|
|
marker_idx += 1
|
|
color_idx += 1
|
|
|
|
print(" Processing affinity " + aff + "...")
|
|
|
|
add_curves(rax, aff, aff_to_parser[aff], parser_idx, marker_type, color_type)
|
|
|
|
rax[0].legend()
|
|
rax[1].legend()
|
|
fig.set_size_inches(23.4, 16.5)
|
|
plt.savefig(fn, dpi=150)
|
|
plt.close()
|
|
|
|
def main():
|
|
datdir = None
|
|
options = getopt.getopt(sys.argv[1:], 'd:')[0]
|
|
|
|
for opt, arg in options:
|
|
if opt in ('-d'):
|
|
datdir = arg
|
|
|
|
if datdir == None:
|
|
raise Exception("Must specify -d parameter")
|
|
|
|
dat = {}
|
|
|
|
for subdir in os.listdir(datdir):
|
|
each_dir = os.path.join(datdir, subdir)
|
|
if not os.path.isfile(each_dir):
|
|
dat[subdir] = process_dir(each_dir)
|
|
|
|
for i in range(len(parser_idx_labels)):
|
|
generate_graph(dat, i, datdir + "/" + parser_idx_labels[i])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |