d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

test/vhost_perf: parse fio json results to csv

Browse Source 
Parse fio output to CSV.
Calculate average read and write metrics while
parsing and present final results.

Change-Id: I8f6d7f84205be409f0ccbba7a42eb7888b2cb03d
Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3253
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Maciej Wawryk <maciejx.wawryk@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Michal Berger <michalx.berger@intel.com>
This commit is contained in:
Karol Latecki 2020-07-08 17:43:39 +02:00 committed by Jim Harris
parent 782681e018
commit c3dc7fb5f2
2 changed files with 77 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
test/vhost/common.sh
View File

@ -1155,6 +1155,81 @@ function run_fio() {
fi
}
# Parsing fio results for json output and client-server mode only!
function parse_fio_results() {
local fio_log_dir=$1
local fio_log_filename=$2
local fio_csv_filename
# Variables used in parsing loop
local log_file
local rwmode mixread mixwrite
local lat_key lat_divisor
local client_stats iops bw
local read_avg_lat read_min_lat read_max_lat
local write_avg_lat write_min_lat write_min_lat
declare -A results
results["iops"]=0
results["bw"]=0
results["avg_lat"]=0
results["min_lat"]=0
results["max_lat"]=0
# Loop using the log filename to see if there are any other
# matching files. This is in case we ran fio test multiple times.
log_files=("$fio_log_dir/$fio_log_filename"*)
for log_file in "${log_files[@]}"; do
rwmode=$(jq -r '.["client_stats"][0]["job options"]["rw"]' "$log_file")
mixread=1
mixwrite=1
if [[ $rwmode = *"rw"* ]]; then
mixread=$(jq -r '.["client_stats"][0]["job options"]["rwmixread"]' "$log_file")
mixread=$(bc -l <<< "scale=3; $mixread/100")
mixwrite=$(bc -l <<< "scale=3; 1-$mixread")
fi
client_stats=$(jq -r '.["client_stats"][] | select(.jobname == "All clients")' "$log_file")
# Check latency unit and later normalize to microseconds
lat_key="lat_us"
lat_divisor=1
if jq -er '.read["lat_ns"]' &> /dev/null <<< $client_stats; then
lat_key="lat_ns"
lat_divisor=1000
fi
# Horrific bash float point arithmetic oprations below.
# Viewer discretion is advised.
iops=$(jq -r '[.read["iops"],.write["iops"]] | add' <<< $client_stats)
bw=$(jq -r '[.read["bw"],.write["bw"]] | add' <<< $client_stats)
read_avg_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["mean"]' <<< $client_stats)
read_min_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["min"]' <<< $client_stats)
read_max_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["max"]' <<< $client_stats)
write_avg_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["mean"]' <<< $client_stats)
write_min_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["min"]' <<< $client_stats)
write_max_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["max"]' <<< $client_stats)
results["iops"]=$(bc -l <<< "${results[iops]} + $iops")
results["bw"]=$(bc -l <<< "${results[bw]} + $bw")
results["avg_lat"]=$(bc -l <<< "${results[avg_lat]} + ($mixread*$read_avg_lat + $mixwrite*$write_avg_lat)/$lat_divisor")
results["min_lat"]=$(bc -l <<< "${results[min_lat]} + ($mixread*$read_min_lat + $mixwrite*$write_min_lat)/$lat_divisor")
results["max_lat"]=$(bc -l <<< "${results[max_lat]} + ($mixread*$read_max_lat + $mixwrite*$write_max_lat)/$lat_divisor")
done
results["iops"]=$(bc -l <<< "scale=3; ${results[iops]} / ${#log_files[@]}")
results["bw"]=$(bc -l <<< "scale=3; ${results[bw]} / ${#log_files[@]}")
results["avg_lat"]=$(bc -l <<< "scale=3; ${results[avg_lat]} / ${#log_files[@]}")
results["min_lat"]=$(bc -l <<< "scale=3; ${results[min_lat]} / ${#log_files[@]}")
results["max_lat"]=$(bc -l <<< "scale=3; ${results[max_lat]} / ${#log_files[@]}")
fio_csv_filename="${fio_log_filename%%.*}.csv"
cat <<- EOF > "$fio_log_dir/$fio_csv_filename"
iops,bw,avg_lat,min_lat,max_lat
${results["iops"]},${results["bw"]},${results["avg_lat"]},${results["min_lat"]},${results["max_lat"]}
EOF
}
# Shutdown or kill any running VM and SPDK APP.
#
function at_app_exit() {

2
test/vhost/perf_bench/vhost_perf.sh
View File

@ -446,6 +446,8 @@ for fio_job in ${fio_jobs//,/ }; do
mv $VHOST_DIR/fio_results/$fio_log_fname $VHOST_DIR/fio_results/$fio_log_fname.$i
sleep 1
done
parse_fio_results "$VHOST_DIR/fio_results" "$fio_log_fname"
done
notice "Shutting down virtual machines..."