temp commit
This commit is contained in:
quackerd
parent
2a543d7e4d
commit
f20ae16e31
@ -2,7 +2,9 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#if defined(__x86_64__)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
#include <ctime>
|
||||
#include <cstdio>
|
||||
#include <sys/types.h>
|
||||
@ -20,6 +22,8 @@ constexpr static unsigned long MS2NS = 1000000UL;
|
||||
|
||||
constexpr static int NEXT_CPU_NULL = -1;
|
||||
|
||||
|
||||
#if defined(__x86_64__)
|
||||
static inline int
|
||||
cmask_get_next_cpu(uint64_t *mask)
|
||||
{
|
||||
@ -33,6 +37,7 @@ cmask_get_num_cpus(const uint64_t mask)
|
||||
{
|
||||
return _mm_popcnt_u64(mask);
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline uint64_t
|
||||
get_uptime()
|
||||
|
||||
3
pyenv.sh
3
pyenv.sh
@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
THIS_FOLDER=$(dirname "$0")
|
||||
export PYTHONPATH="$PYTHONPATH:$THIS_FOLDER/scripts/libs"
|
||||
50
scripts/cc_pin.py
Normal file
50
scripts/cc_pin.py
Normal file
@ -0,0 +1,50 @@
|
||||
import os
|
||||
import sys
|
||||
import getopt
|
||||
import subprocess
|
||||
|
||||
options = getopt.getopt(sys.argv[1:], 'b:s:d:p:')[0]
|
||||
|
||||
base=0
|
||||
stride=2
|
||||
num = 0
|
||||
port = 0
|
||||
|
||||
for opt, arg in options:
|
||||
if opt == '-b':
|
||||
base = int(arg)
|
||||
elif opt == '-s':
|
||||
stride = int(arg)
|
||||
elif opt == '-d':
|
||||
num = int(arg)
|
||||
elif opt == '-p':
|
||||
port = int(arg)
|
||||
result = subprocess.run("sysctl -a", shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
lines = result.stdout.decode().split('\n')
|
||||
cclines : list[str] = []
|
||||
for line in lines:
|
||||
if ("irq" in line) and (f"t6nex{num}" in line) and (f"{port}a" in line):
|
||||
cclines.append(line)
|
||||
|
||||
if len(cclines) == 0:
|
||||
print(f"No t6nex {num}a lines from sysctl.\n")
|
||||
exit(1)
|
||||
|
||||
irqs = []
|
||||
for line in cclines:
|
||||
eles = line.split(' ')
|
||||
irq = eles[0]
|
||||
if (irq.startswith("irq") and irq.endswith(":")):
|
||||
irq = irq[3:-1]
|
||||
irqs.append(int(irq))
|
||||
else:
|
||||
print(f"Unknown line format: f{line}")
|
||||
|
||||
print(f"Detected {len(irqs)} irqs:\n{str(irqs)}")
|
||||
|
||||
for irq in irqs:
|
||||
print(f"Setting irq{irq}'s affinity to core {base}...")
|
||||
subprocess.run(f"cpuset -l {base} -x {irq}", check=True, shell=True)
|
||||
base = base + stride
|
||||
|
||||
exit(0)
|
||||
9
scripts/copy-mount.sh
Executable file
9
scripts/copy-mount.sh
Executable file
@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
scp -P77 mount.sh oscar@icelake1-int.rcs.uwaterloo.ca:~/
|
||||
scp -P77 mount_small.sh oscar@icelake1-int.rcs.uwaterloo.ca:~/
|
||||
scp -P77 mount.sh oscar@milan1-int.rcs.uwaterloo.ca:~/
|
||||
scp -P77 mount_small.sh oscar@milan1-int.rcs.uwaterloo.ca:~/
|
||||
scp -P77 mount.sh oscar@icelake2-int.rcs.uwaterloo.ca:~/
|
||||
scp -P77 mount_small.sh oscar@icelake2-int.rcs.uwaterloo.ca:~/
|
||||
scp -P77 mount.sh oscar@milan2-int.rcs.uwaterloo.ca:~/
|
||||
scp -P77 mount_small.sh oscar@milan2-int.rcs.uwaterloo.ca:~/
|
||||
226
scripts/iperf.py
226
scripts/iperf.py
@ -21,7 +21,9 @@ SSL_CERT = "/certs/server.crt"
|
||||
SSL_PKEY = "/certs/server.key"
|
||||
SERVER_PORT_START = 8050
|
||||
MEMLOAD_BLKSZ = 1024*1024*64
|
||||
RUNS=2
|
||||
RUNS=3
|
||||
SANDYBRIDGE_CPULIST : ic.CPUList = ic.CPUList([16, 16])
|
||||
SKYLAKE_CPULIST : ic.CPUList = ic.CPUList([24, 24])
|
||||
MILAN_CPULIST : ic.CPUList = ic.CPUList([64, 64])
|
||||
ICELAKE_CPULIST : ic.CPUList = ic.CPUList([48, 48])
|
||||
|
||||
@ -29,26 +31,39 @@ ICELAKE_CPULIST : ic.CPUList = ic.CPUList([48, 48])
|
||||
NUM_CORES = 12
|
||||
NUM_CLIENTS = 3
|
||||
# KTLS
|
||||
KTLS_ARGS = [#(False, False, False),
|
||||
# (True, False, False),
|
||||
# (False, True, False),
|
||||
(False, True, True),
|
||||
(True, True, True)]
|
||||
KTLS_ARGS = [(False, False, False),
|
||||
(True, False, False),
|
||||
(False, True, False),
|
||||
(False, True, True),
|
||||
(True, True, True)]
|
||||
|
||||
# MEMLOADGEN
|
||||
MEMLOADGEN_ARGS_ICELAKE = []
|
||||
for i in range(0, 100, 15):
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_ICELAKE.append([f"{ICELAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 3, num = NUM_CORES, stride = 4)}.0.{i}",
|
||||
f"{ICELAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 3, num = NUM_CORES, stride = 4)}.1.{i}"])
|
||||
MEMLOADGEN_ARGS_ICELAKE.append(None)
|
||||
|
||||
MEMLOADGEN_ARGS_MILAN = []
|
||||
for i in range(0, 100, 15):
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_MILAN.append([f"{MILAN_CPULIST.get_cpulist_by_offset(domain = 1, offset = 3, num = NUM_CORES, stride = 4)}.0.{i}",
|
||||
f"{MILAN_CPULIST.get_cpulist_by_offset(domain = 0, offset = 3, num = NUM_CORES, stride = 4)}.1.{i}"])
|
||||
MEMLOADGEN_ARGS_MILAN.append(None)
|
||||
|
||||
MEMLOADGEN_ARGS_SKYLAKE = []
|
||||
for i in range(0, 100, 25):
|
||||
MEMLOADGEN_ARGS_SKYLAKE.append([f"{SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 12, num = 6, stride = 2)}.1.{i}",
|
||||
f"{SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 12, num = 6, stride = 2)}.0.{i}"])
|
||||
MEMLOADGEN_ARGS_SKYLAKE.append(None)
|
||||
|
||||
MEMLOADGEN_ARGS_SANDYBRIDGE = []
|
||||
for i in range(0, 100, 25):
|
||||
MEMLOADGEN_ARGS_SANDYBRIDGE.append([f"{SANDYBRIDGE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = 6, stride = 2)}.1.{i}",
|
||||
f"{SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = 6, stride = 2)}.0.{i}"])
|
||||
MEMLOADGEN_ARGS_SANDYBRIDGE.append(None)
|
||||
|
||||
# client thread affinity
|
||||
CLIENT_ASSIGNMENTS = ic.distribute_threads(NUM_CORES, 1, NUM_CLIENTS, 2)
|
||||
CLIENT_ASSIGNMENTS = ic.distribute_threads(NUM_CORES, 1, NUM_CLIENTS, 4)
|
||||
CLIENT_AFFINITY = []
|
||||
for i in range(NUM_CLIENTS):
|
||||
CLIENT_AFFINITY.append(ic.list_to_comma_delimited(CLIENT_ASSIGNMENTS[i]))
|
||||
@ -57,28 +72,32 @@ ICELAKE_SERVER_AFFINITY_SOCKET0 = ICELAKE_CPULIST.get_cpulist_by_offset(domain =
|
||||
ICELAKE_SERVER_AFFINITY_SOCKET1 = ICELAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = NUM_CORES, stride = 4)
|
||||
MILAN_SERVER_AFFINITY_SOCKET0 = MILAN_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = NUM_CORES, stride = 4)
|
||||
MILAN_SERVER_AFFINITY_SOCKET1= MILAN_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = NUM_CORES, stride = 4)
|
||||
# file system
|
||||
FILE_PATHS = ["/tmpfs/large_file_#p"]
|
||||
#FILE_PATHS = ["/tmpfs0/small_file_0_#p", "/tmpfs1/small_file_1_#p"]
|
||||
# mmap
|
||||
USE_MMAP = [False]
|
||||
SKYLAKE_SERVER_AFFINITY_SOCKET0 = SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = 8, stride = 2)
|
||||
SKYLAKE_SERVER_AFFINITY_SOCKET1= SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = 8, stride = 2)
|
||||
SANDYBRIDGE_SERVER_AFFINITY_SOCKET0= SANDYBRIDGE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = 8, stride = 2)
|
||||
SANDYBRIDGE_SERVER_AFFINITY_SOCKET1= SANDYBRIDGE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = 8, stride = 4)
|
||||
|
||||
# file system
|
||||
#FILE_PATHS = ["/tank/large_file_#p"]
|
||||
FILE_PATHS = ["/tmpfs1/small_file_1_#p"]
|
||||
# mmap
|
||||
USE_MMAP = [False, True]
|
||||
all_args : list[ic.ArgTypes] = []
|
||||
|
||||
# args start
|
||||
all_args : list[ic.ArgTypes] = []
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca"])
|
||||
arg_types.add_arg(["icelake1-int.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake1-int.rcs.uwaterloo.ca", "milan2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca"])
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.100.102"])
|
||||
arg_types.add_arg(["192.168.100.101"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.100.101", "192.168.100.104", "192.168.100.103"])
|
||||
arg_types.add_arg(["192.168.100.102", "192.168.100.103"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(CLIENT_AFFINITY)
|
||||
# affinity
|
||||
arg_types.add_arg(ICELAKE_SERVER_AFFINITY_SOCKET0, ICELAKE_SERVER_AFFINITY_SOCKET1)
|
||||
arg_types.add_arg(ICELAKE_SERVER_AFFINITY_SOCKET1, ICELAKE_SERVER_AFFINITY_SOCKET0)
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
@ -89,22 +108,23 @@ arg_types.add_arg(*FILE_PATHS)
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
all_args.append(arg_types)
|
||||
#all_args.append(arg_types)
|
||||
|
||||
|
||||
# args start
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["icelake1-int.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca"])
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.100.101"])
|
||||
arg_types.add_arg(["192.168.101.101"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.100.102", "192.168.100.104", "192.168.100.103"])
|
||||
arg_types.add_arg(["192.168.101.102", "192.168.101.103"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(CLIENT_AFFINITY)
|
||||
# affinity
|
||||
arg_types.add_arg(ICELAKE_SERVER_AFFINITY_SOCKET0)
|
||||
arg_types.add_arg(ICELAKE_SERVER_AFFINITY_SOCKET1)
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
@ -115,6 +135,31 @@ arg_types.add_arg(*FILE_PATHS)
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
#all_args.append(arg_types)
|
||||
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["milan1-int.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan2-int.rcs.uwaterloo.ca", "icelake1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.100.103"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.100.102", "192.168.100.104", "192.168.100.101"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(CLIENT_AFFINITY)
|
||||
# affinity
|
||||
arg_types.add_arg(MILAN_SERVER_AFFINITY_SOCKET0, MILAN_SERVER_AFFINITY_SOCKET1)
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
arg_types.add_arg(*MEMLOADGEN_ARGS_MILAN)
|
||||
# filepath
|
||||
arg_types.add_arg(*FILE_PATHS)
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
all_args.append(arg_types)
|
||||
|
||||
|
||||
@ -124,13 +169,13 @@ arg_types.add_arg(["milan1-int.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan2-int.rcs.uwaterloo.ca", "icelake1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.100.103"])
|
||||
arg_types.add_arg(["192.168.101.103"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.100.102", "192.168.100.104", "192.168.100.101"])
|
||||
arg_types.add_arg(["192.168.101.102", "192.168.101.104", "192.168.101.101"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(CLIENT_AFFINITY)
|
||||
# affinity
|
||||
arg_types.add_arg(MILAN_SERVER_AFFINITY_SOCKET0)
|
||||
arg_types.add_arg(MILAN_SERVER_AFFINITY_SOCKET1)
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
@ -143,7 +188,6 @@ arg_types.add_arg(False)
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
all_args.append(arg_types)
|
||||
|
||||
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["milan2-int.rcs.uwaterloo.ca"])
|
||||
@ -167,7 +211,84 @@ arg_types.add_arg(*FILE_PATHS)
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
all_args.append(arg_types)
|
||||
#all_args.append(arg_types)
|
||||
|
||||
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["milan2-int.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca", "icelake1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.101.104"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.101.102", "192.168.101.103", "192.168.101.101"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(CLIENT_AFFINITY)
|
||||
# affinity
|
||||
arg_types.add_arg(MILAN_SERVER_AFFINITY_SOCKET1)
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
#arg_types.add_arg(*MEMLOADGEN_ARGS_MILAN_MLX)
|
||||
# filepath
|
||||
arg_types.add_arg(*FILE_PATHS)
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
#all_args.append(arg_types)
|
||||
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["skylake3.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca", "icelake1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.101.92"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.101.102", "192.168.101.103", "192.168.101.101"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(["1,3", "1,3", "1,3"])
|
||||
# affinitybb
|
||||
arg_types.add_arg("25,27,29,31,33,35", "1,3,5,7,9,11")
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
arg_types.add_arg(*MEMLOADGEN_ARGS_SKYLAKE)
|
||||
# filepath
|
||||
arg_types.add_arg("/tmpfs/small_file_#p")
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
#all_args.append(arg_types)
|
||||
|
||||
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["skylake3.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca", "icelake1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.100.92"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.100.102", "192.168.100.103", "192.168.100.101"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(["1,3", "1,3", "1,3"])
|
||||
# affinitybb
|
||||
arg_types.add_arg("1,3,5,7,9,11")
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
arg_types.add_arg(*MEMLOADGEN_ARGS_SKYLAKE)
|
||||
# filepath
|
||||
arg_types.add_arg("/tmpfs/small_file_#p")
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
#all_args.append(arg_types)
|
||||
|
||||
|
||||
def parse_comma_list(input : str):
|
||||
@ -193,7 +314,6 @@ def setup_all(conf : ic.Conf):
|
||||
setup_cmd : str = f'''
|
||||
sudo pkg install -y openssl-devel vim curl wget gmake cmake openssl-devel llvm gcc rsync pkgconf isal-kmod ktls_isa-l_crypto-kmod;
|
||||
sudo pkg remove -y iperf iperf3;
|
||||
sudo kldload -n isal; sudo kldload -n ktls_intel-isa-l;
|
||||
sudo rm -rf { BIN_PATH };
|
||||
sudo mkdir -p { BIN_PATH };
|
||||
sudo git clone https://git.quacker.org/d/iperf3-tls { BIN_PATH };
|
||||
@ -285,8 +405,7 @@ def run_exp(conf : ic.Conf):
|
||||
server_cmd = "sudo "
|
||||
server_cmd += f"{EXE_PATH} -s -p " + str(SERVER_PORT_START + cur_srv_proc) + \
|
||||
" -F " + conf.filepath.replace("#p", str(cur_srv_proc)) + \
|
||||
" -A " + eaff + \
|
||||
" -J --logfile " + LOG_FILEPATH + "/" + eaff + ".txt"
|
||||
" -A " + eaff
|
||||
if conf.tls:
|
||||
server_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
if conf.ktls:
|
||||
@ -295,6 +414,7 @@ def run_exp(conf : ic.Conf):
|
||||
server_cmd += " --use-odirect"
|
||||
if conf.mmap:
|
||||
server_cmd += " --use-mmap"
|
||||
server_cmd += " -J --logfile " + LOG_FILEPATH + "/" + eaff + ".txt"
|
||||
# start server
|
||||
tc.log_print("Starting server proc " + str(cur_srv_proc) + "...")
|
||||
tc.log_print(server_cmd)
|
||||
@ -321,14 +441,10 @@ def run_exp(conf : ic.Conf):
|
||||
" -R" + \
|
||||
" -N" + \
|
||||
" -4" + \
|
||||
" -O 10" + \
|
||||
" -J --logfile /dev/null"
|
||||
" -O 10"
|
||||
if conf.tls:
|
||||
client_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
if conf.ktls:
|
||||
client_cmd += " --enable-ssl-ktls"
|
||||
if conf.sendfile:
|
||||
client_cmd += " -Z"
|
||||
client_cmd += " -J --logfile /dev/null"
|
||||
|
||||
tc.log_print(conf.clients[i] + ":\n" + client_cmd)
|
||||
sclts.append(tc.remote_exec([conf.clients[i]], client_cmd, blocking=False)[0])
|
||||
@ -441,7 +557,8 @@ def main():
|
||||
for arg in args:
|
||||
confs.append(ic.Conf(*arg))
|
||||
|
||||
options = getopt.getopt(sys.argv[1:], 'sS')[0]
|
||||
name = None
|
||||
options = getopt.getopt(sys.argv[1:], 'sSn:')[0]
|
||||
for opt, arg in options:
|
||||
if opt in ('-s'):
|
||||
stop_all(confs[0])
|
||||
@ -449,15 +566,40 @@ def main():
|
||||
elif opt in ('-S'):
|
||||
setup_all(confs[0])
|
||||
return
|
||||
elif opt in ('-n'):
|
||||
name = arg
|
||||
|
||||
if name != None:
|
||||
output_dirname = name
|
||||
tc.init("~/results.d/iperf3/" + output_dirname + "_" + datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
|
||||
cpcmd = "cp " + __file__ + " " + tc.get_odir() + "/"
|
||||
tc.log_print(cpcmd)
|
||||
sp.check_call(cpcmd, shell=True)
|
||||
|
||||
server_set = []
|
||||
for conf in confs:
|
||||
print(conf.to_string(verbose=True))
|
||||
print(f"{len(confs)} configs ({RUNS} * {len(confs)} = {RUNS * len(confs)} runs) scheduled:")
|
||||
tc.log_print(conf.to_string(verbose=True))
|
||||
if conf.server[0] not in server_set:
|
||||
server_set.append(conf.server[0])
|
||||
tc.log_print(f"{len(confs)} configs ({RUNS} * {len(confs)} = {RUNS * len(confs)} runs) scheduled:")
|
||||
|
||||
for server in server_set:
|
||||
print(f"Gathering information on {server} (sysctl)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sysctl -a", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.sysctl", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (ifconfig)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "ifconfig", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.ifconfig", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (mount)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "mount", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.mount", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (dmesg)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sudo dmesg", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.dmesg", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
|
||||
for conf in confs:
|
||||
for i in range(0, RUNS):
|
||||
|
||||
397
scripts/iperf_p9.py
Normal file
397
scripts/iperf_p9.py
Normal file
@ -0,0 +1,397 @@
|
||||
import subprocess as sp
|
||||
import time
|
||||
import os
|
||||
import datetime
|
||||
import sys
|
||||
import getopt
|
||||
import numpy as np
|
||||
|
||||
import libpar as par
|
||||
import libtc as tc
|
||||
import iperfconf as ic
|
||||
|
||||
# definitions
|
||||
file_dir : str = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir : str = os.path.join(file_dir,"..")
|
||||
LOG_FILEPATH = "/iperflogs"
|
||||
BIN_PATH = "/iperftls"
|
||||
MLG_PATH = "/numam/build/bin/memloadgen"
|
||||
EXE_PATH = BIN_PATH + "/src/iperf3"
|
||||
SSL_CERT = "/certs/server.crt"
|
||||
SSL_PKEY = "/certs/server.key"
|
||||
SERVER_PORT_START = 8050
|
||||
MEMLOAD_BLKSZ = 1024*1024*64
|
||||
RUNS=3
|
||||
POWER9_CPULIST : ic.CPUList = ic.CPUList([88, 88])
|
||||
|
||||
# Shared Arguments
|
||||
NUM_CORES = 12
|
||||
NUM_CLIENTS = 3
|
||||
# KTLS
|
||||
KTLS_ARGS = [(False, False, False),
|
||||
(True, False, False),
|
||||
(False, True, False),
|
||||
(False, True, True),
|
||||
(True, True, True)]
|
||||
|
||||
MEMLOADGEN_ARGS_POWER9 = []
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_POWER9.append([f"{POWER9_CPULIST.get_cpulist_by_offset(domain = 0, offset = 48, num = 10, stride = 4)}.1.{i}",
|
||||
f"{POWER9_CPULIST.get_cpulist_by_offset(domain = 1, offset = 48, num = 10, stride = 4)}.0.{i}"])
|
||||
MEMLOADGEN_ARGS_POWER9.append(None)
|
||||
MEMLOADGEN_ARGS_POWER9 = [None]
|
||||
|
||||
POWER9_SERVER_AFFINITY_SOCKET0 = POWER9_CPULIST.get_cpulist_by_offset(domain = 0, offset = 2, num = 12, stride = 4)
|
||||
POWER9_SERVER_AFFINITY_SOCKET1= POWER9_CPULIST.get_cpulist_by_offset(domain = 1, offset = 2, num = 12, stride = 4)
|
||||
|
||||
# file system
|
||||
#FILE_PATHS = ["/tank/large_file_#p"]
|
||||
#"/tmpfs1/small_files_#p",
|
||||
FILE_PATHS = ["/tmpfs0/small_files_#p", "/tmpfs1/small_files_#p"]
|
||||
# mmap
|
||||
USE_MMAP = [False, True]
|
||||
all_args : list[ic.ArgTypes] = []
|
||||
|
||||
# args start
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["power9-int.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake1-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.80.80"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.80.101", "192.168.80.103"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(["49,53,57,61,65,69", "65,69,73,77,81,85"])
|
||||
# affinity
|
||||
arg_types.add_arg(POWER9_SERVER_AFFINITY_SOCKET1)
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
arg_types.add_arg(*MEMLOADGEN_ARGS_POWER9)
|
||||
# filepath
|
||||
arg_types.add_arg(*FILE_PATHS)
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
all_args.append(arg_types)
|
||||
|
||||
|
||||
def parse_comma_list(input : str):
|
||||
return input.split(",")
|
||||
|
||||
def run_setup_cmd(conf : ic.Conf, cmd : str):
|
||||
ssrv : list[tuple[str, sp.Popen]] = []
|
||||
tc.log_print(f"Running command on {conf.server[0]}...")
|
||||
ssrv.append((conf.server[0], tc.remote_exec(conf.server, cmd, blocking=False, check=False)[0]))
|
||||
|
||||
for s in conf.clients:
|
||||
tc.log_print(f"Running command on {s}...")
|
||||
ssrv.append((s, tc.remote_exec([s], cmd, blocking=False, check=False)[0]))
|
||||
|
||||
for p in ssrv:
|
||||
_ , stderr = p[1].communicate()
|
||||
if p[1].returncode != 0:
|
||||
print(f"\n{ p[0] } failed. stderr:\n{stderr.decode()}\n")
|
||||
else:
|
||||
print(f"\n{ p[0] } succeeded\n")
|
||||
|
||||
def setup_all(conf : ic.Conf):
|
||||
setup_cmd : str = f'''
|
||||
sudo pkg install -y openssl-devel vim curl wget gmake cmake openssl-devel llvm gcc rsync pkgconf isal-kmod ktls_isa-l_crypto-kmod;
|
||||
sudo pkg remove -y iperf iperf3;
|
||||
sudo rm -rf { BIN_PATH };
|
||||
sudo mkdir -p { BIN_PATH };
|
||||
sudo git clone https://git.quacker.org/d/iperf3-tls { BIN_PATH };
|
||||
cd { BIN_PATH };
|
||||
sudo git checkout dev;
|
||||
sudo ./configure;
|
||||
sudo make -j8;
|
||||
sudo rm -rf /libtopo;
|
||||
sudo mkdir -p /libtopo;
|
||||
sudo git clone https://git.quacker.org/d/libtopo /libtopo;
|
||||
cd /libtopo;
|
||||
sudo mkdir build;
|
||||
cd build;
|
||||
sudo cmake ../;
|
||||
sudo make install;
|
||||
sudo rm -rf /numam;
|
||||
sudo mkdir -p /numam;
|
||||
sudo chmod 777 /numam;
|
||||
'''
|
||||
run_setup_cmd(conf, setup_cmd)
|
||||
|
||||
#rsync
|
||||
all_clts = []
|
||||
all_clts.extend(conf.clients)
|
||||
all_clts.extend(conf.server)
|
||||
dir = f"{os.path.dirname(__file__)}/../"
|
||||
for clt in all_clts:
|
||||
print("Syncing files to " + clt + "...")
|
||||
rsync_cmd = f"rsync -az --no-perms --rsync-path=\"sudo rsync\" --omit-dir-times -e \"ssh -p77\" {dir} {tc.get_ssh_user()}@{clt}:/numam/"
|
||||
sp.check_call(rsync_cmd, shell=True)
|
||||
|
||||
run_setup_cmd(conf, f'''
|
||||
cd /numam;
|
||||
sudo rm -rf build;
|
||||
sudo mkdir -p build;
|
||||
cd build;
|
||||
sudo cmake ../;
|
||||
sudo make memloadgen;
|
||||
''')
|
||||
|
||||
|
||||
def stop_all(conf : ic.Conf, clients_only = False):
|
||||
# stop clients
|
||||
tc.log_print("Stopping clients...")
|
||||
tc.remote_exec(conf.clients, "sudo killall -9 iperf3; sudo killall -9 memloadgen", check=False)
|
||||
|
||||
if not clients_only:
|
||||
tc.log_print("Stopping server...")
|
||||
tc.remote_exec(conf.server, "sudo killall -9 iperf3; sudo killall -9 memloadgen", check=False)
|
||||
|
||||
def prepare_logdir(conf : ic.Conf):
|
||||
tc.log_print("Preparing server log directory...")
|
||||
prep_cmd = "sudo rm -rf " + LOG_FILEPATH
|
||||
tc.log_print(prep_cmd)
|
||||
tc.remote_exec(conf.server, prep_cmd, check=False)
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
prep_cmd = "sudo mkdir -p " + LOG_FILEPATH
|
||||
tc.log_print(prep_cmd)
|
||||
tc.remote_exec(conf.server, prep_cmd, check=True)
|
||||
|
||||
def run_exp(conf : ic.Conf):
|
||||
stop_all(conf)
|
||||
while True:
|
||||
prepare_logdir(conf)
|
||||
|
||||
ssrvs=[]
|
||||
ssrv_names=[]
|
||||
cur_srv_proc = 0
|
||||
|
||||
smlg = []
|
||||
smlg_names = []
|
||||
if conf.memloadgen != None:
|
||||
for emem in conf.memloadgen:
|
||||
mlg_cmd = "sudo "
|
||||
mlg_cpu = emem.split(".")[0]
|
||||
mlg_dom = emem.split(".")[1]
|
||||
mlg_pct = emem.split(".")[2]
|
||||
mlg_cmd += f"{MLG_PATH} -b {MEMLOAD_BLKSZ} -T {MEMLOAD_BLKSZ} -s {mlg_cpu} -d {mlg_dom} -o {LOG_FILEPATH}/memloadgen_{len(smlg)} -p {mlg_pct} -H 4 -w 17"
|
||||
|
||||
tc.log_print("Starting memloadgen...")
|
||||
tc.log_print(mlg_cmd)
|
||||
smlg.append(tc.remote_exec(conf.server, mlg_cmd, blocking=False)[0])
|
||||
smlg_names.append("memloadgen")
|
||||
time.sleep(0.1)
|
||||
time.sleep(5)
|
||||
for eaff in parse_comma_list(conf.affinity):
|
||||
server_cmd = "sudo "
|
||||
server_cmd += f"{EXE_PATH} -s -p " + str(SERVER_PORT_START + cur_srv_proc) + \
|
||||
" -F " + conf.filepath.replace("#p", str(cur_srv_proc)) + \
|
||||
" -A " + eaff
|
||||
if conf.tls:
|
||||
server_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
if conf.ktls:
|
||||
server_cmd += " --enable-ssl-ktls"
|
||||
if conf.odirect:
|
||||
server_cmd += " --use-odirect"
|
||||
if conf.mmap:
|
||||
server_cmd += " --use-mmap"
|
||||
server_cmd += " -J --logfile " + LOG_FILEPATH + "/" + eaff + ".txt"
|
||||
# start server
|
||||
tc.log_print("Starting server proc " + str(cur_srv_proc) + "...")
|
||||
tc.log_print(server_cmd)
|
||||
ssrv = tc.remote_exec(conf.server, server_cmd, blocking=False)[0]
|
||||
ssrvs.append(ssrv)
|
||||
ssrv_names.append("Server " + str(cur_srv_proc))
|
||||
cur_srv_proc = cur_srv_proc + 1
|
||||
time.sleep(0.1)
|
||||
|
||||
# start clients
|
||||
tc.log_print("Starting clients...")
|
||||
sclts = []
|
||||
sclt_names = []
|
||||
clt_number = 0
|
||||
for i in range(len(conf.clients)):
|
||||
client_aff = conf.clients_affinity[i]
|
||||
for eaff in parse_comma_list(client_aff):
|
||||
client_cmd = f"sudo {EXE_PATH} -c " + conf.server_dat[0] + \
|
||||
" -p " + str(SERVER_PORT_START + clt_number) + \
|
||||
" --connect-timeout 1000" + \
|
||||
" -A " + eaff + \
|
||||
" -t 25" + \
|
||||
" -P 16" + \
|
||||
" -R" + \
|
||||
" -N" + \
|
||||
" -4" + \
|
||||
" -O 10"
|
||||
if conf.tls:
|
||||
client_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
client_cmd += " -J --logfile /dev/null"
|
||||
|
||||
tc.log_print(conf.clients[i] + ":\n" + client_cmd)
|
||||
sclts.append(tc.remote_exec([conf.clients[i]], client_cmd, blocking=False)[0])
|
||||
sclt_names.append(conf.clients[i] + "@" + eaff)
|
||||
clt_number = clt_number + 1
|
||||
time.sleep(0.1)
|
||||
# launch stderr monitoring thread
|
||||
exclude = ["Pseudo-terminal"]
|
||||
tc.errthr_create(sclts, sclt_names, exclude)
|
||||
tc.errthr_create(ssrvs, ssrv_names, exclude)
|
||||
if (conf.memloadgen != None):
|
||||
tc.errthr_create(smlg, smlg_names, exclude)
|
||||
tc.errthr_start()
|
||||
cur = 0
|
||||
# selec = select.poll()
|
||||
# selec.register(p.stdout, select.POLLIN)
|
||||
success = False
|
||||
while not success:
|
||||
success = False
|
||||
# either failed or timeout
|
||||
# we use failure detection to save time for long durations
|
||||
if tc.errthr_get_failed():
|
||||
break
|
||||
|
||||
if cur >= 90:
|
||||
tc.log_print("Experiment timed out. Restarting...")
|
||||
break
|
||||
# while selec.poll(1):
|
||||
# print(p.stdout.readline())
|
||||
|
||||
success = True
|
||||
for p in sclts:
|
||||
if p.poll() == None:
|
||||
success = False
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
cur = cur + 1
|
||||
|
||||
tc.errthr_stop()
|
||||
stop_all(conf, clients_only=True)
|
||||
tc.log_print("Cooling down...")
|
||||
time.sleep(3)
|
||||
stop_all(conf)
|
||||
|
||||
if success:
|
||||
if flush_netresult(conf):
|
||||
break
|
||||
|
||||
def flush_netresult(conf : ic.Conf) -> bool:
|
||||
tc.log_print("Keeping results...")
|
||||
# copy log directory back to machine
|
||||
log_output = tc.get_odir()
|
||||
os.makedirs(log_output, exist_ok=True)
|
||||
scp_cmd = "scp -P77 -r " + tc.get_ssh_user() + "@" + conf.server[0] + ":" + LOG_FILEPATH + " " + log_output + "/"
|
||||
tc.log_print(scp_cmd)
|
||||
sp.check_call(scp_cmd, shell=True)
|
||||
|
||||
# parse results
|
||||
log_output = log_output + "/" + os.path.basename(LOG_FILEPATH)
|
||||
logs = os.listdir(log_output)
|
||||
logs_bytes = []
|
||||
memload_bytes = []
|
||||
for log in logs:
|
||||
tc.log_print("Processing " + log + "...")
|
||||
if os.path.isfile(log_output + "/" + log) and log.endswith(".txt"):
|
||||
with open(log_output + "/" + log, "r") as f:
|
||||
buf = f.read()
|
||||
if len(buf) > 0:
|
||||
logs_bytes.append(buf)
|
||||
else:
|
||||
tc.log_print("Warning: log file empty for " + log + ". Retrying...")
|
||||
return False
|
||||
if os.path.isfile(log_output + "/" + log) and ("memloadgen" in log):
|
||||
with open(log_output + "/" + log, "r") as f:
|
||||
memloadbuf = f.read()
|
||||
if len(memloadbuf) == 0:
|
||||
tc.log_print("Warning: log file empty for " + log + ". Retrying...")
|
||||
return False
|
||||
else:
|
||||
memload_bytes.append(memloadbuf)
|
||||
|
||||
try:
|
||||
parser = par.iperf_json_parser(logs_bytes)
|
||||
bps = []
|
||||
for ml in memload_bytes:
|
||||
memparser = par.memloadgen_parser(ml, 22, 32)
|
||||
bps.append(memparser.bps)
|
||||
|
||||
tc.log_print("Aggregated throughput: " + "{:.2f}".format(parser.aggregate_egress_bps / 8.0) + " B/s " + \
|
||||
"{:.2f}".format(parser.aggregate_egress_bps / 8.0 / 1024.0 / 1024.0) + " MB/s " + \
|
||||
"{:.2f}".format(parser.aggregate_egress_bps / 8.0 / 1024.0 / 1024.0 / 1024.0) + " GB/s" + \
|
||||
" | Memload generator: " + "{:.2f}".format(np.sum(bps) / 1024.0 / 1024.0 / 1024.0) + " GB/s" )
|
||||
except Exception as e:
|
||||
tc.log_print("Warning: exception for parsing logs: " + str(e) + " restarting...")
|
||||
scp_cmd = "sudo rm -rf " + log_output + "/*"
|
||||
tc.log_print(scp_cmd)
|
||||
sp.check_call(scp_cmd, shell=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
def main():
|
||||
tc.set_ssh_param("-o StrictHostKeyChecking=no -p77")
|
||||
tc.set_ssh_user("oscar")
|
||||
output_dirname = "run"
|
||||
|
||||
confs : list[ic.Conf] = []
|
||||
for argtype in all_args:
|
||||
args = argtype.get_fields()
|
||||
for arg in args:
|
||||
confs.append(ic.Conf(*arg))
|
||||
|
||||
name = None
|
||||
options = getopt.getopt(sys.argv[1:], 'sSn:')[0]
|
||||
for opt, arg in options:
|
||||
if opt in ('-s'):
|
||||
stop_all(confs[0])
|
||||
return
|
||||
elif opt in ('-S'):
|
||||
setup_all(confs[0])
|
||||
return
|
||||
elif opt in ('-n'):
|
||||
name = arg
|
||||
|
||||
if name != None:
|
||||
output_dirname = name
|
||||
tc.init("~/results.d/iperf3/" + output_dirname + "_" + datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
|
||||
cpcmd = "cp " + __file__ + " " + tc.get_odir() + "/"
|
||||
tc.log_print(cpcmd)
|
||||
sp.check_call(cpcmd, shell=True)
|
||||
|
||||
server_set = []
|
||||
for conf in confs:
|
||||
tc.log_print(conf.to_string(verbose=True))
|
||||
if conf.server[0] not in server_set:
|
||||
server_set.append(conf.server[0])
|
||||
tc.log_print(f"{len(confs)} configs ({RUNS} * {len(confs)} = {RUNS * len(confs)} runs) scheduled:")
|
||||
|
||||
for server in server_set:
|
||||
print(f"Gathering information on {server} (sysctl)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sysctl -a", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.sysctl", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (ifconfig)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "ifconfig", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.ifconfig", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (mount)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "mount", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.mount", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (dmesg)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sudo dmesg", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.dmesg", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
|
||||
for conf in confs:
|
||||
for i in range(0, RUNS):
|
||||
tc.begin(conf.to_string()+f"_run.{i}")
|
||||
run_exp(conf)
|
||||
tc.end()
|
||||
main()
|
||||
454
scripts/iperf_sand.py
Normal file
454
scripts/iperf_sand.py
Normal file
@ -0,0 +1,454 @@
|
||||
import subprocess as sp
|
||||
import time
|
||||
import os
|
||||
import datetime
|
||||
import sys
|
||||
import getopt
|
||||
import numpy as np
|
||||
|
||||
import libpar as par
|
||||
import libtc as tc
|
||||
import iperfconf as ic
|
||||
|
||||
# definitions
|
||||
file_dir : str = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir : str = os.path.join(file_dir,"..")
|
||||
LOG_FILEPATH = "/iperflogs"
|
||||
BIN_PATH = "/iperftls"
|
||||
MLG_PATH = "/numam/build/bin/memloadgen"
|
||||
EXE_PATH = BIN_PATH + "/src/iperf3"
|
||||
SSL_CERT = "/certs/server.crt"
|
||||
SSL_PKEY = "/certs/server.key"
|
||||
SERVER_PORT_START = 8050
|
||||
MEMLOAD_BLKSZ = 1024*1024*64
|
||||
RUNS=3
|
||||
SANDYBRIDGE_CPULIST : ic.CPUList = ic.CPUList([16, 16])
|
||||
SKYLAKE_CPULIST : ic.CPUList = ic.CPUList([24, 24])
|
||||
MILAN_CPULIST : ic.CPUList = ic.CPUList([64, 64])
|
||||
ICELAKE_CPULIST : ic.CPUList = ic.CPUList([48, 48])
|
||||
|
||||
# Shared Arguments
|
||||
NUM_CORES = 12
|
||||
NUM_CLIENTS = 3
|
||||
# KTLS
|
||||
KTLS_ARGS = [#(False, False, False),
|
||||
#(True, False, False),
|
||||
#(False, True, False),
|
||||
(False, True, True),
|
||||
(True, True, True)]
|
||||
|
||||
# MEMLOADGEN
|
||||
MEMLOADGEN_ARGS_ICELAKE = []
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_ICELAKE.append([f"{ICELAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 3, num = NUM_CORES, stride = 4)}.0.{i}",
|
||||
f"{ICELAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 3, num = NUM_CORES, stride = 4)}.1.{i}"])
|
||||
MEMLOADGEN_ARGS_ICELAKE.append(None)
|
||||
|
||||
MEMLOADGEN_ARGS_MILAN = []
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_MILAN.append([f"{MILAN_CPULIST.get_cpulist_by_offset(domain = 1, offset = 3, num = NUM_CORES, stride = 4)}.0.{i}",
|
||||
f"{MILAN_CPULIST.get_cpulist_by_offset(domain = 0, offset = 3, num = NUM_CORES, stride = 4)}.1.{i}"])
|
||||
MEMLOADGEN_ARGS_MILAN.append(None)
|
||||
|
||||
MEMLOADGEN_ARGS_SKYLAKE = []
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_SKYLAKE.append([f"{SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 12, num = 6, stride = 2)}.1.{i}",
|
||||
f"{SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 12, num = 6, stride = 2)}.0.{i}"])
|
||||
MEMLOADGEN_ARGS_SKYLAKE.append(None)
|
||||
|
||||
MEMLOADGEN_ARGS_SANDYBRIDGE = []
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_SANDYBRIDGE.append([f"{SANDYBRIDGE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 12, num = 4, stride = 1)}.1.{i}",
|
||||
f"{SANDYBRIDGE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 12, num = 4, stride = 1)}.0.{i}"])
|
||||
MEMLOADGEN_ARGS_SANDYBRIDGE.append(None)
|
||||
|
||||
# client thread affinity
|
||||
CLIENT_ASSIGNMENTS = ic.distribute_threads(NUM_CORES, 1, NUM_CLIENTS, 4)
|
||||
CLIENT_AFFINITY = []
|
||||
for i in range(NUM_CLIENTS):
|
||||
CLIENT_AFFINITY.append(ic.list_to_comma_delimited(CLIENT_ASSIGNMENTS[i]))
|
||||
# server thread affinity
|
||||
ICELAKE_SERVER_AFFINITY_SOCKET0 = ICELAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = NUM_CORES, stride = 4)
|
||||
ICELAKE_SERVER_AFFINITY_SOCKET1 = ICELAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = NUM_CORES, stride = 4)
|
||||
MILAN_SERVER_AFFINITY_SOCKET0 = MILAN_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = NUM_CORES, stride = 4)
|
||||
MILAN_SERVER_AFFINITY_SOCKET1= MILAN_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = NUM_CORES, stride = 4)
|
||||
SKYLAKE_SERVER_AFFINITY_SOCKET0 = SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = 8, stride = 2)
|
||||
SKYLAKE_SERVER_AFFINITY_SOCKET1= SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = 8, stride = 2)
|
||||
SANDYBRIDGE_SERVER_AFFINITY_SOCKET0= SANDYBRIDGE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 0, num = 8, stride = 2)
|
||||
SANDYBRIDGE_SERVER_AFFINITY_SOCKET1= SANDYBRIDGE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 0, num = 8, stride = 2)
|
||||
|
||||
# file system
|
||||
#FILE_PATHS = ["/tank/large_file_#p"]
|
||||
#"/tmpfs0/small_files_#p",
|
||||
FILE_PATHS = ["/tmpfs0/small_files_#p", "/tmpfs1/small_files_#p"]
|
||||
# mmap
|
||||
USE_MMAP = [False, True]
|
||||
all_args : list[ic.ArgTypes] = []
|
||||
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["sandybridge4.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["milan2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.70.74"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.70.103", "192.168.70.104"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(["1,3,5", "1,3,5"])
|
||||
# affinity
|
||||
#arg_types.add_arg("17,19,21,23,25,27")
|
||||
arg_types.add_arg("1,3,5,7,9,11")
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
arg_types.add_arg(*MEMLOADGEN_ARGS_SANDYBRIDGE)
|
||||
# filepath
|
||||
arg_types.add_arg(*FILE_PATHS)
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
all_args.append(arg_types)
|
||||
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["sandybridge4.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["milan2-int.rcs.uwaterloo.ca", "milan1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.70.74"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.70.103", "192.168.70.104"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(["1,3,5", "1,3,5"])
|
||||
# affinity
|
||||
arg_types.add_arg("17,19,21,23,25,27")
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
arg_types.add_arg(*MEMLOADGEN_ARGS_SANDYBRIDGE)
|
||||
# filepath
|
||||
arg_types.add_arg(*FILE_PATHS)
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
#all_args.append(arg_types)
|
||||
|
||||
|
||||
def parse_comma_list(input : str):
|
||||
return input.split(",")
|
||||
|
||||
def run_setup_cmd(conf : ic.Conf, cmd : str):
|
||||
ssrv : list[tuple[str, sp.Popen]] = []
|
||||
tc.log_print(f"Running command on {conf.server[0]}...")
|
||||
ssrv.append((conf.server[0], tc.remote_exec(conf.server, cmd, blocking=False, check=False)[0]))
|
||||
|
||||
for s in conf.clients:
|
||||
tc.log_print(f"Running command on {s}...")
|
||||
ssrv.append((s, tc.remote_exec([s], cmd, blocking=False, check=False)[0]))
|
||||
|
||||
for p in ssrv:
|
||||
_ , stderr = p[1].communicate()
|
||||
if p[1].returncode != 0:
|
||||
print(f"\n{ p[0] } failed. stderr:\n{stderr.decode()}\n")
|
||||
else:
|
||||
print(f"\n{ p[0] } succeeded\n")
|
||||
|
||||
def setup_all(conf : ic.Conf):
|
||||
setup_cmd : str = f'''
|
||||
sudo pkg install -y openssl-devel vim curl wget gmake cmake openssl-devel llvm gcc rsync pkgconf isal-kmod ktls_isa-l_crypto-kmod;
|
||||
sudo pkg remove -y iperf iperf3;
|
||||
sudo rm -rf { BIN_PATH };
|
||||
sudo mkdir -p { BIN_PATH };
|
||||
sudo git clone https://git.quacker.org/d/iperf3-tls { BIN_PATH };
|
||||
cd { BIN_PATH };
|
||||
sudo git checkout dev;
|
||||
sudo ./configure;
|
||||
sudo make -j8;
|
||||
sudo rm -rf /libtopo;
|
||||
sudo mkdir -p /libtopo;
|
||||
sudo git clone https://git.quacker.org/d/libtopo /libtopo;
|
||||
cd /libtopo;
|
||||
sudo mkdir build;
|
||||
cd build;
|
||||
sudo cmake ../;
|
||||
sudo make install;
|
||||
sudo rm -rf /numam;
|
||||
sudo mkdir -p /numam;
|
||||
sudo chmod 777 /numam;
|
||||
'''
|
||||
run_setup_cmd(conf, setup_cmd)
|
||||
|
||||
#rsync
|
||||
all_clts = []
|
||||
all_clts.extend(conf.clients)
|
||||
all_clts.extend(conf.server)
|
||||
dir = f"{os.path.dirname(__file__)}/../"
|
||||
for clt in all_clts:
|
||||
print("Syncing files to " + clt + "...")
|
||||
rsync_cmd = f"rsync -az --no-perms --rsync-path=\"sudo rsync\" --omit-dir-times -e \"ssh -p77\" {dir} {tc.get_ssh_user()}@{clt}:/numam/"
|
||||
sp.check_call(rsync_cmd, shell=True)
|
||||
|
||||
run_setup_cmd(conf, f'''
|
||||
cd /numam;
|
||||
sudo rm -rf build;
|
||||
sudo mkdir -p build;
|
||||
cd build;
|
||||
sudo cmake ../;
|
||||
sudo make memloadgen;
|
||||
''')
|
||||
|
||||
|
||||
def stop_all(conf : ic.Conf, clients_only = False):
|
||||
# stop clients
|
||||
tc.log_print("Stopping clients...")
|
||||
tc.remote_exec(conf.clients, "sudo killall -9 iperf3; sudo killall -9 memloadgen", check=False)
|
||||
|
||||
if not clients_only:
|
||||
tc.log_print("Stopping server...")
|
||||
tc.remote_exec(conf.server, "sudo killall -9 iperf3; sudo killall -9 memloadgen", check=False)
|
||||
|
||||
def prepare_logdir(conf : ic.Conf):
|
||||
tc.log_print("Preparing server log directory...")
|
||||
prep_cmd = "sudo rm -rf " + LOG_FILEPATH
|
||||
tc.log_print(prep_cmd)
|
||||
tc.remote_exec(conf.server, prep_cmd, check=False)
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
prep_cmd = "sudo mkdir -p " + LOG_FILEPATH
|
||||
tc.log_print(prep_cmd)
|
||||
tc.remote_exec(conf.server, prep_cmd, check=True)
|
||||
|
||||
def run_exp(conf : ic.Conf):
|
||||
stop_all(conf)
|
||||
while True:
|
||||
prepare_logdir(conf)
|
||||
|
||||
ssrvs=[]
|
||||
ssrv_names=[]
|
||||
cur_srv_proc = 0
|
||||
|
||||
smlg = []
|
||||
smlg_names = []
|
||||
if conf.memloadgen != None:
|
||||
for emem in conf.memloadgen:
|
||||
mlg_cmd = "sudo "
|
||||
mlg_cpu = emem.split(".")[0]
|
||||
mlg_dom = emem.split(".")[1]
|
||||
mlg_pct = emem.split(".")[2]
|
||||
mlg_cmd += f"{MLG_PATH} -b {MEMLOAD_BLKSZ} -T {MEMLOAD_BLKSZ} -s {mlg_cpu} -d {mlg_dom} -o {LOG_FILEPATH}/memloadgen_{len(smlg)} -p {mlg_pct} -H 4 -w 17"
|
||||
|
||||
tc.log_print("Starting memloadgen...")
|
||||
tc.log_print(mlg_cmd)
|
||||
smlg.append(tc.remote_exec(conf.server, mlg_cmd, blocking=False)[0])
|
||||
smlg_names.append("memloadgen")
|
||||
time.sleep(0.1)
|
||||
time.sleep(5)
|
||||
for eaff in parse_comma_list(conf.affinity):
|
||||
server_cmd = "sudo "
|
||||
server_cmd += f"{EXE_PATH} -s -p " + str(SERVER_PORT_START + cur_srv_proc) + \
|
||||
" -F " + conf.filepath.replace("#p", str(cur_srv_proc)) + \
|
||||
" -A " + eaff
|
||||
if conf.tls:
|
||||
server_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
if conf.ktls:
|
||||
server_cmd += " --enable-ssl-ktls"
|
||||
if conf.odirect:
|
||||
server_cmd += " --use-odirect"
|
||||
if conf.mmap:
|
||||
server_cmd += " --use-mmap"
|
||||
server_cmd += " -J --logfile " + LOG_FILEPATH + "/" + eaff + ".txt"
|
||||
# start server
|
||||
tc.log_print("Starting server proc " + str(cur_srv_proc) + "...")
|
||||
tc.log_print(server_cmd)
|
||||
ssrv = tc.remote_exec(conf.server, server_cmd, blocking=False)[0]
|
||||
ssrvs.append(ssrv)
|
||||
ssrv_names.append("Server " + str(cur_srv_proc))
|
||||
cur_srv_proc = cur_srv_proc + 1
|
||||
time.sleep(0.1)
|
||||
|
||||
# start clients
|
||||
tc.log_print("Starting clients...")
|
||||
sclts = []
|
||||
sclt_names = []
|
||||
clt_number = 0
|
||||
for i in range(len(conf.clients)):
|
||||
client_aff = conf.clients_affinity[i]
|
||||
for eaff in parse_comma_list(client_aff):
|
||||
client_cmd = f"sudo {EXE_PATH} -c " + conf.server_dat[0] + \
|
||||
" -p " + str(SERVER_PORT_START + clt_number) + \
|
||||
" --connect-timeout 1000" + \
|
||||
" -A " + eaff + \
|
||||
" -t 25" + \
|
||||
" -P 16" + \
|
||||
" -R" + \
|
||||
" -N" + \
|
||||
" -4" + \
|
||||
" -O 10"
|
||||
if conf.tls:
|
||||
client_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
client_cmd += " -J --logfile /dev/null"
|
||||
|
||||
tc.log_print(conf.clients[i] + ":\n" + client_cmd)
|
||||
sclts.append(tc.remote_exec([conf.clients[i]], client_cmd, blocking=False)[0])
|
||||
sclt_names.append(conf.clients[i] + "@" + eaff)
|
||||
clt_number = clt_number + 1
|
||||
time.sleep(0.1)
|
||||
# launch stderr monitoring thread
|
||||
exclude = ["Pseudo-terminal"]
|
||||
tc.errthr_create(sclts, sclt_names, exclude)
|
||||
tc.errthr_create(ssrvs, ssrv_names, exclude)
|
||||
if (conf.memloadgen != None):
|
||||
tc.errthr_create(smlg, smlg_names, exclude)
|
||||
tc.errthr_start()
|
||||
cur = 0
|
||||
# selec = select.poll()
|
||||
# selec.register(p.stdout, select.POLLIN)
|
||||
success = False
|
||||
while not success:
|
||||
success = False
|
||||
# either failed or timeout
|
||||
# we use failure detection to save time for long durations
|
||||
if tc.errthr_get_failed():
|
||||
break
|
||||
|
||||
if cur >= 90:
|
||||
tc.log_print("Experiment timed out. Restarting...")
|
||||
break
|
||||
# while selec.poll(1):
|
||||
# print(p.stdout.readline())
|
||||
|
||||
success = True
|
||||
for p in sclts:
|
||||
if p.poll() == None:
|
||||
success = False
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
cur = cur + 1
|
||||
|
||||
tc.errthr_stop()
|
||||
stop_all(conf, clients_only=True)
|
||||
tc.log_print("Cooling down...")
|
||||
time.sleep(3)
|
||||
stop_all(conf)
|
||||
|
||||
if success:
|
||||
if flush_netresult(conf):
|
||||
break
|
||||
|
||||
def flush_netresult(conf : ic.Conf) -> bool:
|
||||
tc.log_print("Keeping results...")
|
||||
# copy log directory back to machine
|
||||
log_output = tc.get_odir()
|
||||
os.makedirs(log_output, exist_ok=True)
|
||||
scp_cmd = "scp -P77 -r " + tc.get_ssh_user() + "@" + conf.server[0] + ":" + LOG_FILEPATH + " " + log_output + "/"
|
||||
tc.log_print(scp_cmd)
|
||||
sp.check_call(scp_cmd, shell=True)
|
||||
|
||||
# parse results
|
||||
log_output = log_output + "/" + os.path.basename(LOG_FILEPATH)
|
||||
logs = os.listdir(log_output)
|
||||
logs_bytes = []
|
||||
memload_bytes = []
|
||||
for log in logs:
|
||||
tc.log_print("Processing " + log + "...")
|
||||
if os.path.isfile(log_output + "/" + log) and log.endswith(".txt"):
|
||||
with open(log_output + "/" + log, "r") as f:
|
||||
buf = f.read()
|
||||
if len(buf) > 0:
|
||||
logs_bytes.append(buf)
|
||||
else:
|
||||
tc.log_print("Warning: log file empty for " + log + ". Retrying...")
|
||||
return False
|
||||
if os.path.isfile(log_output + "/" + log) and ("memloadgen" in log):
|
||||
with open(log_output + "/" + log, "r") as f:
|
||||
memloadbuf = f.read()
|
||||
if len(memloadbuf) == 0:
|
||||
tc.log_print("Warning: log file empty for " + log + ". Retrying...")
|
||||
return False
|
||||
else:
|
||||
memload_bytes.append(memloadbuf)
|
||||
|
||||
try:
|
||||
parser = par.iperf_json_parser(logs_bytes)
|
||||
bps = []
|
||||
for ml in memload_bytes:
|
||||
memparser = par.memloadgen_parser(ml, 22, 32)
|
||||
bps.append(memparser.bps)
|
||||
|
||||
tc.log_print("Aggregated throughput: " + "{:.2f}".format(parser.aggregate_egress_bps / 8.0) + " B/s " + \
|
||||
"{:.2f}".format(parser.aggregate_egress_bps / 8.0 / 1024.0 / 1024.0) + " MB/s " + \
|
||||
"{:.2f}".format(parser.aggregate_egress_bps / 8.0 / 1024.0 / 1024.0 / 1024.0) + " GB/s" + \
|
||||
" | Memload generator: " + "{:.2f}".format(np.sum(bps) / 1024.0 / 1024.0 / 1024.0) + " GB/s" )
|
||||
except Exception as e:
|
||||
tc.log_print("Warning: exception for parsing logs: " + str(e) + " restarting...")
|
||||
scp_cmd = "sudo rm -rf " + log_output + "/*"
|
||||
tc.log_print(scp_cmd)
|
||||
sp.check_call(scp_cmd, shell=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
def main():
|
||||
tc.set_ssh_param("-o StrictHostKeyChecking=no -p77")
|
||||
tc.set_ssh_user("oscar")
|
||||
output_dirname = "run"
|
||||
|
||||
confs : list[ic.Conf] = []
|
||||
for argtype in all_args:
|
||||
args = argtype.get_fields()
|
||||
for arg in args:
|
||||
confs.append(ic.Conf(*arg))
|
||||
|
||||
name = None
|
||||
options = getopt.getopt(sys.argv[1:], 'sSn:')[0]
|
||||
for opt, arg in options:
|
||||
if opt in ('-s'):
|
||||
stop_all(confs[0])
|
||||
return
|
||||
elif opt in ('-S'):
|
||||
setup_all(confs[0])
|
||||
return
|
||||
elif opt in ('-n'):
|
||||
name = arg
|
||||
|
||||
if name != None:
|
||||
output_dirname = name
|
||||
tc.init("~/results.d/iperf3/" + output_dirname + "_" + datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
|
||||
cpcmd = "cp " + __file__ + " " + tc.get_odir() + "/"
|
||||
tc.log_print(cpcmd)
|
||||
sp.check_call(cpcmd, shell=True)
|
||||
|
||||
server_set = []
|
||||
for conf in confs:
|
||||
tc.log_print(conf.to_string(verbose=True))
|
||||
if conf.server[0] not in server_set:
|
||||
server_set.append(conf.server[0])
|
||||
tc.log_print(f"{len(confs)} configs ({RUNS} * {len(confs)} = {RUNS * len(confs)} runs) scheduled:")
|
||||
|
||||
for server in server_set:
|
||||
print(f"Gathering information on {server} (sysctl)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sysctl -a", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.sysctl", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (ifconfig)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "ifconfig", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.ifconfig", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (mount)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "mount", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.mount", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (dmesg)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sudo dmesg", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.dmesg", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
for conf in confs:
|
||||
for i in range(0, RUNS):
|
||||
tc.begin(conf.to_string()+f"_run.{i}")
|
||||
run_exp(conf)
|
||||
tc.end()
|
||||
main()
|
||||
399
scripts/iperf_sky.py
Normal file
399
scripts/iperf_sky.py
Normal file
@ -0,0 +1,399 @@
|
||||
import subprocess as sp
|
||||
import time
|
||||
import os
|
||||
import datetime
|
||||
import sys
|
||||
import getopt
|
||||
import numpy as np
|
||||
|
||||
import libpar as par
|
||||
import libtc as tc
|
||||
import iperfconf as ic
|
||||
|
||||
# definitions
|
||||
file_dir : str = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir : str = os.path.join(file_dir,"..")
|
||||
LOG_FILEPATH = "/iperflogs"
|
||||
BIN_PATH = "/iperftls"
|
||||
MLG_PATH = "/numam/build/bin/memloadgen"
|
||||
EXE_PATH = BIN_PATH + "/src/iperf3"
|
||||
SSL_CERT = "/certs/server.crt"
|
||||
SSL_PKEY = "/certs/server.key"
|
||||
SERVER_PORT_START = 8050
|
||||
MEMLOAD_BLKSZ = 1024*1024*64
|
||||
RUNS=3
|
||||
SANDYBRIDGE_CPULIST : ic.CPUList = ic.CPUList([16, 16])
|
||||
SKYLAKE_CPULIST : ic.CPUList = ic.CPUList([24, 24])
|
||||
MILAN_CPULIST : ic.CPUList = ic.CPUList([64, 64])
|
||||
ICELAKE_CPULIST : ic.CPUList = ic.CPUList([48, 48])
|
||||
|
||||
# Shared Arguments
|
||||
NUM_CORES = 12
|
||||
NUM_CLIENTS = 3
|
||||
# KTLS
|
||||
KTLS_ARGS = [#(False, False, False),
|
||||
#(True, False, False),
|
||||
#(False, True, False),
|
||||
(False, True, True),
|
||||
(True, True, True)]
|
||||
|
||||
MEMLOADGEN_ARGS_SKYLAKE = []
|
||||
for i in range(0, 100, 10):
|
||||
MEMLOADGEN_ARGS_SKYLAKE.append([f"{SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 12, num = 6, stride = 2)}.1.{i}",
|
||||
f"{SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 12, num = 6, stride = 2)}.0.{i}"])
|
||||
MEMLOADGEN_ARGS_SKYLAKE.append(None)
|
||||
|
||||
SKYLAKE_SERVER_AFFINITY_SOCKET0 = SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 0, offset = 1, num = 6, stride = 2)
|
||||
SKYLAKE_SERVER_AFFINITY_SOCKET1= SKYLAKE_CPULIST.get_cpulist_by_offset(domain = 1, offset = 1, num = 6, stride = 2)
|
||||
|
||||
# file system
|
||||
#FILE_PATHS = ["/tank/large_file_#p"]
|
||||
#"/tmpfs1/small_files_#p",
|
||||
FILE_PATHS = ["/tmpfs1/small_files_#p", "/tmpfs0/small_files_#p"]
|
||||
# mmap
|
||||
USE_MMAP = [False, True]
|
||||
all_args : list[ic.ArgTypes] = []
|
||||
|
||||
# args start
|
||||
arg_types : ic.ArgTypes = ic.ArgTypes()
|
||||
#server
|
||||
arg_types.add_arg(["skylake3.rcs.uwaterloo.ca"])
|
||||
#clients
|
||||
arg_types.add_arg(["icelake2-int.rcs.uwaterloo.ca", "icelake1-int.rcs.uwaterloo.ca"])
|
||||
#server_dat
|
||||
arg_types.add_arg(["192.168.90.93"])
|
||||
#clients_dat
|
||||
arg_types.add_arg(["192.168.90.102", "192.168.90.101"])
|
||||
#clients_affinity
|
||||
arg_types.add_arg(["1,3,5", "1,3,5"])
|
||||
# affinity
|
||||
arg_types.add_arg(SKYLAKE_SERVER_AFFINITY_SOCKET1)
|
||||
# sendfile/tls/ktls
|
||||
arg_types.add_args(*KTLS_ARGS)
|
||||
# memloadgen
|
||||
arg_types.add_arg(*MEMLOADGEN_ARGS_SKYLAKE)
|
||||
# filepath
|
||||
arg_types.add_arg(*FILE_PATHS)
|
||||
# ODIRECT
|
||||
arg_types.add_arg(False)
|
||||
# mmap
|
||||
arg_types.add_arg(*USE_MMAP)
|
||||
all_args.append(arg_types)
|
||||
|
||||
|
||||
def parse_comma_list(input : str):
|
||||
return input.split(",")
|
||||
|
||||
def run_setup_cmd(conf : ic.Conf, cmd : str):
|
||||
ssrv : list[tuple[str, sp.Popen]] = []
|
||||
tc.log_print(f"Running command on {conf.server[0]}...")
|
||||
ssrv.append((conf.server[0], tc.remote_exec(conf.server, cmd, blocking=False, check=False)[0]))
|
||||
|
||||
for s in conf.clients:
|
||||
tc.log_print(f"Running command on {s}...")
|
||||
ssrv.append((s, tc.remote_exec([s], cmd, blocking=False, check=False)[0]))
|
||||
|
||||
for p in ssrv:
|
||||
_ , stderr = p[1].communicate()
|
||||
if p[1].returncode != 0:
|
||||
print(f"\n{ p[0] } failed. stderr:\n{stderr.decode()}\n")
|
||||
else:
|
||||
print(f"\n{ p[0] } succeeded\n")
|
||||
|
||||
def setup_all(conf : ic.Conf):
|
||||
setup_cmd : str = f'''
|
||||
sudo pkg install -y openssl-devel vim curl wget gmake cmake openssl-devel llvm gcc rsync pkgconf isal-kmod ktls_isa-l_crypto-kmod;
|
||||
sudo pkg remove -y iperf iperf3;
|
||||
sudo rm -rf { BIN_PATH };
|
||||
sudo mkdir -p { BIN_PATH };
|
||||
sudo git clone https://git.quacker.org/d/iperf3-tls { BIN_PATH };
|
||||
cd { BIN_PATH };
|
||||
sudo git checkout dev;
|
||||
sudo ./configure;
|
||||
sudo make -j8;
|
||||
sudo rm -rf /libtopo;
|
||||
sudo mkdir -p /libtopo;
|
||||
sudo git clone https://git.quacker.org/d/libtopo /libtopo;
|
||||
cd /libtopo;
|
||||
sudo mkdir build;
|
||||
cd build;
|
||||
sudo cmake ../;
|
||||
sudo make install;
|
||||
sudo rm -rf /numam;
|
||||
sudo mkdir -p /numam;
|
||||
sudo chmod 777 /numam;
|
||||
'''
|
||||
run_setup_cmd(conf, setup_cmd)
|
||||
|
||||
#rsync
|
||||
all_clts = []
|
||||
all_clts.extend(conf.clients)
|
||||
all_clts.extend(conf.server)
|
||||
dir = f"{os.path.dirname(__file__)}/../"
|
||||
for clt in all_clts:
|
||||
print("Syncing files to " + clt + "...")
|
||||
rsync_cmd = f"rsync -az --no-perms --rsync-path=\"sudo rsync\" --omit-dir-times -e \"ssh -p77\" {dir} {tc.get_ssh_user()}@{clt}:/numam/"
|
||||
sp.check_call(rsync_cmd, shell=True)
|
||||
|
||||
run_setup_cmd(conf, f'''
|
||||
cd /numam;
|
||||
sudo rm -rf build;
|
||||
sudo mkdir -p build;
|
||||
cd build;
|
||||
sudo cmake ../;
|
||||
sudo make memloadgen;
|
||||
''')
|
||||
|
||||
|
||||
def stop_all(conf : ic.Conf, clients_only = False):
|
||||
# stop clients
|
||||
tc.log_print("Stopping clients...")
|
||||
tc.remote_exec(conf.clients, "sudo killall -9 iperf3; sudo killall -9 memloadgen", check=False)
|
||||
|
||||
if not clients_only:
|
||||
tc.log_print("Stopping server...")
|
||||
tc.remote_exec(conf.server, "sudo killall -9 iperf3; sudo killall -9 memloadgen", check=False)
|
||||
|
||||
def prepare_logdir(conf : ic.Conf):
|
||||
tc.log_print("Preparing server log directory...")
|
||||
prep_cmd = "sudo rm -rf " + LOG_FILEPATH
|
||||
tc.log_print(prep_cmd)
|
||||
tc.remote_exec(conf.server, prep_cmd, check=False)
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
prep_cmd = "sudo mkdir -p " + LOG_FILEPATH
|
||||
tc.log_print(prep_cmd)
|
||||
tc.remote_exec(conf.server, prep_cmd, check=True)
|
||||
|
||||
def run_exp(conf : ic.Conf):
|
||||
stop_all(conf)
|
||||
while True:
|
||||
prepare_logdir(conf)
|
||||
|
||||
ssrvs=[]
|
||||
ssrv_names=[]
|
||||
cur_srv_proc = 0
|
||||
|
||||
smlg = []
|
||||
smlg_names = []
|
||||
if conf.memloadgen != None:
|
||||
for emem in conf.memloadgen:
|
||||
mlg_cmd = "sudo "
|
||||
mlg_cpu = emem.split(".")[0]
|
||||
mlg_dom = emem.split(".")[1]
|
||||
mlg_pct = emem.split(".")[2]
|
||||
mlg_cmd += f"{MLG_PATH} -b {MEMLOAD_BLKSZ} -T {MEMLOAD_BLKSZ} -s {mlg_cpu} -d {mlg_dom} -o {LOG_FILEPATH}/memloadgen_{len(smlg)} -p {mlg_pct} -H 4 -w 17"
|
||||
|
||||
tc.log_print("Starting memloadgen...")
|
||||
tc.log_print(mlg_cmd)
|
||||
smlg.append(tc.remote_exec(conf.server, mlg_cmd, blocking=False)[0])
|
||||
smlg_names.append("memloadgen")
|
||||
time.sleep(0.1)
|
||||
time.sleep(5)
|
||||
for eaff in parse_comma_list(conf.affinity):
|
||||
server_cmd = "sudo "
|
||||
server_cmd += f"{EXE_PATH} -s -p " + str(SERVER_PORT_START + cur_srv_proc) + \
|
||||
" -F " + conf.filepath.replace("#p", str(cur_srv_proc)) + \
|
||||
" -A " + eaff
|
||||
if conf.tls:
|
||||
server_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
if conf.ktls:
|
||||
server_cmd += " --enable-ssl-ktls"
|
||||
if conf.odirect:
|
||||
server_cmd += " --use-odirect"
|
||||
if conf.mmap:
|
||||
server_cmd += " --use-mmap"
|
||||
server_cmd += " -J --logfile " + LOG_FILEPATH + "/" + eaff + ".txt"
|
||||
# start server
|
||||
tc.log_print("Starting server proc " + str(cur_srv_proc) + "...")
|
||||
tc.log_print(server_cmd)
|
||||
ssrv = tc.remote_exec(conf.server, server_cmd, blocking=False)[0]
|
||||
ssrvs.append(ssrv)
|
||||
ssrv_names.append("Server " + str(cur_srv_proc))
|
||||
cur_srv_proc = cur_srv_proc + 1
|
||||
time.sleep(0.1)
|
||||
|
||||
# start clients
|
||||
tc.log_print("Starting clients...")
|
||||
sclts = []
|
||||
sclt_names = []
|
||||
clt_number = 0
|
||||
for i in range(len(conf.clients)):
|
||||
client_aff = conf.clients_affinity[i]
|
||||
for eaff in parse_comma_list(client_aff):
|
||||
client_cmd = f"sudo {EXE_PATH} -c " + conf.server_dat[0] + \
|
||||
" -p " + str(SERVER_PORT_START + clt_number) + \
|
||||
" --connect-timeout 1000" + \
|
||||
" -A " + eaff + \
|
||||
" -t 25" + \
|
||||
" -P 16" + \
|
||||
" -R" + \
|
||||
" -N" + \
|
||||
" -4" + \
|
||||
" -O 10"
|
||||
if conf.tls:
|
||||
client_cmd += f" --enable-ssl-over-tcp --ssl-certificate {SSL_CERT} --ssl-private-key {SSL_PKEY}"
|
||||
client_cmd += " -J --logfile /dev/null"
|
||||
|
||||
tc.log_print(conf.clients[i] + ":\n" + client_cmd)
|
||||
sclts.append(tc.remote_exec([conf.clients[i]], client_cmd, blocking=False)[0])
|
||||
sclt_names.append(conf.clients[i] + "@" + eaff)
|
||||
clt_number = clt_number + 1
|
||||
time.sleep(0.1)
|
||||
# launch stderr monitoring thread
|
||||
exclude = ["Pseudo-terminal"]
|
||||
tc.errthr_create(sclts, sclt_names, exclude)
|
||||
tc.errthr_create(ssrvs, ssrv_names, exclude)
|
||||
if (conf.memloadgen != None):
|
||||
tc.errthr_create(smlg, smlg_names, exclude)
|
||||
tc.errthr_start()
|
||||
cur = 0
|
||||
# selec = select.poll()
|
||||
# selec.register(p.stdout, select.POLLIN)
|
||||
success = False
|
||||
while not success:
|
||||
success = False
|
||||
# either failed or timeout
|
||||
# we use failure detection to save time for long durations
|
||||
if tc.errthr_get_failed():
|
||||
break
|
||||
|
||||
if cur >= 90:
|
||||
tc.log_print("Experiment timed out. Restarting...")
|
||||
break
|
||||
# while selec.poll(1):
|
||||
# print(p.stdout.readline())
|
||||
|
||||
success = True
|
||||
for p in sclts:
|
||||
if p.poll() == None:
|
||||
success = False
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
cur = cur + 1
|
||||
|
||||
tc.errthr_stop()
|
||||
stop_all(conf, clients_only=True)
|
||||
tc.log_print("Cooling down...")
|
||||
time.sleep(3)
|
||||
stop_all(conf)
|
||||
|
||||
if success:
|
||||
if flush_netresult(conf):
|
||||
break
|
||||
|
||||
def flush_netresult(conf : ic.Conf) -> bool:
|
||||
tc.log_print("Keeping results...")
|
||||
# copy log directory back to machine
|
||||
log_output = tc.get_odir()
|
||||
os.makedirs(log_output, exist_ok=True)
|
||||
scp_cmd = "scp -P77 -r " + tc.get_ssh_user() + "@" + conf.server[0] + ":" + LOG_FILEPATH + " " + log_output + "/"
|
||||
tc.log_print(scp_cmd)
|
||||
sp.check_call(scp_cmd, shell=True)
|
||||
|
||||
# parse results
|
||||
log_output = log_output + "/" + os.path.basename(LOG_FILEPATH)
|
||||
logs = os.listdir(log_output)
|
||||
logs_bytes = []
|
||||
memload_bytes = []
|
||||
for log in logs:
|
||||
tc.log_print("Processing " + log + "...")
|
||||
if os.path.isfile(log_output + "/" + log) and log.endswith(".txt"):
|
||||
with open(log_output + "/" + log, "r") as f:
|
||||
buf = f.read()
|
||||
if len(buf) > 0:
|
||||
logs_bytes.append(buf)
|
||||
else:
|
||||
tc.log_print("Warning: log file empty for " + log + ". Retrying...")
|
||||
return False
|
||||
if os.path.isfile(log_output + "/" + log) and ("memloadgen" in log):
|
||||
with open(log_output + "/" + log, "r") as f:
|
||||
memloadbuf = f.read()
|
||||
if len(memloadbuf) == 0:
|
||||
tc.log_print("Warning: log file empty for " + log + ". Retrying...")
|
||||
return False
|
||||
else:
|
||||
memload_bytes.append(memloadbuf)
|
||||
|
||||
try:
|
||||
parser = par.iperf_json_parser(logs_bytes)
|
||||
bps = []
|
||||
for ml in memload_bytes:
|
||||
memparser = par.memloadgen_parser(ml, 22, 32)
|
||||
bps.append(memparser.bps)
|
||||
|
||||
tc.log_print("Aggregated throughput: " + "{:.2f}".format(parser.aggregate_egress_bps / 8.0) + " B/s " + \
|
||||
"{:.2f}".format(parser.aggregate_egress_bps / 8.0 / 1024.0 / 1024.0) + " MB/s " + \
|
||||
"{:.2f}".format(parser.aggregate_egress_bps / 8.0 / 1024.0 / 1024.0 / 1024.0) + " GB/s" + \
|
||||
" | Memload generator: " + "{:.2f}".format(np.sum(bps) / 1024.0 / 1024.0 / 1024.0) + " GB/s" )
|
||||
except Exception as e:
|
||||
tc.log_print("Warning: exception for parsing logs: " + str(e) + " restarting...")
|
||||
scp_cmd = "sudo rm -rf " + log_output + "/*"
|
||||
tc.log_print(scp_cmd)
|
||||
sp.check_call(scp_cmd, shell=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
def main():
|
||||
tc.set_ssh_param("-o StrictHostKeyChecking=no -p77")
|
||||
tc.set_ssh_user("oscar")
|
||||
output_dirname = "run"
|
||||
|
||||
confs : list[ic.Conf] = []
|
||||
for argtype in all_args:
|
||||
args = argtype.get_fields()
|
||||
for arg in args:
|
||||
confs.append(ic.Conf(*arg))
|
||||
|
||||
name = None
|
||||
options = getopt.getopt(sys.argv[1:], 'sSn:')[0]
|
||||
for opt, arg in options:
|
||||
if opt in ('-s'):
|
||||
stop_all(confs[0])
|
||||
return
|
||||
elif opt in ('-S'):
|
||||
setup_all(confs[0])
|
||||
return
|
||||
elif opt in ('-n'):
|
||||
name = arg
|
||||
|
||||
if name != None:
|
||||
output_dirname = name
|
||||
tc.init("~/results.d/iperf3/" + output_dirname + "_" + datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
|
||||
cpcmd = "cp " + __file__ + " " + tc.get_odir() + "/"
|
||||
tc.log_print(cpcmd)
|
||||
sp.check_call(cpcmd, shell=True)
|
||||
|
||||
server_set = []
|
||||
for conf in confs:
|
||||
tc.log_print(conf.to_string(verbose=True))
|
||||
if conf.server[0] not in server_set:
|
||||
server_set.append(conf.server[0])
|
||||
tc.log_print(f"{len(confs)} configs ({RUNS} * {len(confs)} = {RUNS * len(confs)} runs) scheduled:")
|
||||
|
||||
for server in server_set:
|
||||
print(f"Gathering information on {server} (sysctl)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sysctl -a", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.sysctl", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (ifconfig)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "ifconfig", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.ifconfig", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (mount)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "mount", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.mount", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
print(f"Gathering information on {server} (dmesg)...")
|
||||
p : sp.Popen = tc.remote_exec([server], "sudo dmesg", blocking=False, check=True)[0]
|
||||
with open(f"{tc.get_odir()}/{server}.dmesg", "w") as f:
|
||||
f.write(p.communicate()[0].decode())
|
||||
|
||||
for conf in confs:
|
||||
for i in range(0, RUNS):
|
||||
tc.begin(conf.to_string()+f"_run.{i}")
|
||||
run_exp(conf)
|
||||
tc.end()
|
||||
main()
|
||||
@ -2,6 +2,51 @@ import numpy as np
|
||||
import os
|
||||
import libpar as par
|
||||
|
||||
class IperfConfObj:
|
||||
KTLS_MODE_SW = "sw"
|
||||
KTLS_MODE_IFNET = "IFNET"
|
||||
KTLS_MODE_NONE = "none"
|
||||
KTLS_MODE_TOE = "toe"
|
||||
STORAGE_NVDIMM = "nvdimm"
|
||||
STORAGE_TMPFS = "tmpfs"
|
||||
STORAGE_NVME = "nvme"
|
||||
STORAGE_RAID_0 = "0"
|
||||
STORAGE_RAID_NONE = "none"
|
||||
SERVER_ICELAKE = "icelake"
|
||||
SERVER_MILAN = "milan"
|
||||
SERVER_SKYLAKE = "skylake"
|
||||
NIC_INTEL_100 = "ice100"
|
||||
NIC_CHELSIO_100 = "cc100"
|
||||
NIC_MELLANOX_100 = "mlx100"
|
||||
|
||||
def __init__(self):
|
||||
self.conf_odirect = False
|
||||
self.conf_memloadgen_pct = 0
|
||||
self.conf_memloadgen_num = 1
|
||||
self.conf_memloadgen_threads = []
|
||||
self.conf_ktls = False
|
||||
self.conf_tls = False
|
||||
self.conf_sendfile = False
|
||||
self.conf_storage = self.STORAGE_TMPFS
|
||||
self.conf_storage_domain = 0
|
||||
self.conf_nic_domain = 0
|
||||
self.conf_nic_name = self.NIC_INTEL_100
|
||||
self.conf_ktls_mode = self.KTLS_MODE_NONE
|
||||
self.conf_storage_raid = self.STORAGE_RAID_NONE
|
||||
self.conf_storage_num = 1
|
||||
self.conf_mmap = False
|
||||
self.conf_server = self.SERVER_ICELAKE
|
||||
self.num_clients = 1
|
||||
self.server_threads = 1
|
||||
self.server_threads_base = 0
|
||||
self.server_threads_stride = 1
|
||||
self.stat_memloadgen = 0
|
||||
self.stat_throughput = 0
|
||||
self.log_driver = ""
|
||||
self.log_memloadgen = {}
|
||||
self.log_iperf = {}
|
||||
|
||||
|
||||
class Conf:
|
||||
def __init__(self, server, clients, server_dat, clients_dat, clients_affinity, affinity, sendfile, tls, ktls, memloadgen, filepath, odirect, mmap):
|
||||
self.affinity = affinity
|
||||
@ -19,7 +64,7 @@ class Conf:
|
||||
self.mmap = mmap
|
||||
|
||||
def to_string(self, verbose = False):
|
||||
ret = f"server.{self.server[0]}_affinity.{self.affinity}_sendfile.{self.sendfile}_tls.{self.tls}_ktls.{self.ktls}_"
|
||||
ret = f"server.{self.server[0]}@{self.server_dat[0]}_affinity.{self.affinity}_sendfile.{self.sendfile}_tls.{self.tls}_ktls.{self.ktls}_"
|
||||
if verbose:
|
||||
ret += f"memloadgen.{False if self.memloadgen == None else (self.memloadgen[0] + '+' + self.memloadgen[1])}"
|
||||
else:
|
||||
|
||||
@ -41,7 +41,7 @@ def begin(name):
|
||||
def end():
|
||||
global tc_cur_test
|
||||
log_print("\n===== Test #" + str(tc_test_id) + " - " + tc_cur_test + " completed =====")
|
||||
tc_cur_test = None
|
||||
tc_cur_test = ""
|
||||
|
||||
def get_odir():
|
||||
return tc_output_dir + "/" + tc_cur_test
|
||||
@ -79,7 +79,7 @@ def get_ssh_user():
|
||||
global ssh_user
|
||||
return ssh_user
|
||||
|
||||
def remote_exec(srv, cmd, blocking=True, check=True):
|
||||
def remote_exec(srv : list[str], cmd : str, blocking=True, check=True) -> sp.Popen:
|
||||
sub = []
|
||||
for s in srv:
|
||||
p = sp.Popen(["ssh " + ssh_param + " " + ((ssh_user + "@") if ssh_user != None else "") + s + " \"" + cmd +"\""], shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
|
||||
|
||||
6
scripts/mount.sh
Normal file
6
scripts/mount.sh
Normal file
@ -0,0 +1,6 @@
|
||||
sudo mkdir -p /mnt/zroot
|
||||
sudo fsck /dev/$1
|
||||
mount -t ufs /dev/$1 /mnt/zroot
|
||||
sudo mkdir -p /tmpfs
|
||||
mount -t tmpfs tmpfs /tmpfs
|
||||
cp /mnt/zroot/large* /tmpfs/
|
||||
20
scripts/mount_small.sh
Normal file
20
scripts/mount_small.sh
Normal file
@ -0,0 +1,20 @@
|
||||
sudo mkdir -p /mnt/zroot
|
||||
sudo fsck /dev/$1
|
||||
mount -t ufs /dev/$1 /mnt/zroot
|
||||
sudo mkdir -p /tmpfs0
|
||||
sudo mkdir -p /tmpfs1
|
||||
|
||||
sysctl vfs.tmpfs.domain_target=0
|
||||
sysctl vfs.tmpfs.domain_policy=3
|
||||
|
||||
mount -t tmpfs tmpfs /tmpfs0
|
||||
cp /mnt/zroot/small_file_0_* /tmpfs0/
|
||||
|
||||
sysctl vfs.tmpfs.domain_target=1
|
||||
sysctl vfs.tmpfs.domain_policy=3
|
||||
|
||||
mount -t tmpfs tmpfs /tmpfs1
|
||||
cp /mnt/zroot/small_file_1_* /tmpfs1/
|
||||
|
||||
sysctl vfs.tmpfs.domain_target=0
|
||||
sysctl vfs.tmpfs.domain_policy=1
|
||||
Loading…
Reference in New Issue
Block a user