scripts: add a parser for the dpdk memory stats.
This script takes an optional dpdk stats file argument and tries to parse the information from that file into a comprehensible structure. It then provides a basic UI for printing that memory information at various levels of granularity. Signed-off-by: Seth Howell <seth.howell@intel.com> Change-Id: Ibbb09f41122be18b7e640781f3e140618e52c0ab Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/477635 Community-CI: SPDK CI Jenkins <sys_sgci@intel.com> Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Seth Howell
Jim Harris
parent
076821454d
commit
0e9727db73
406
scripts/dpdk_mem_info.py
Executable file
406
scripts/dpdk_mem_info.py
Executable file
@ -0,0 +1,406 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class memory:
|
||||
def __init__(self, size):
|
||||
self.size = size
|
||||
self.heaps = []
|
||||
self.mempools = []
|
||||
self.memzones = []
|
||||
|
||||
def get_size(self):
|
||||
return self.size
|
||||
|
||||
def add_mempool(self, pool):
|
||||
self.mempools.append(pool)
|
||||
|
||||
def add_memzone(self, zone):
|
||||
self.memzones.append(zone)
|
||||
|
||||
def add_heap(self, heap):
|
||||
self.heaps.append(heap)
|
||||
|
||||
def get_total_heap_size(self):
|
||||
size = 0
|
||||
for heap in self.heaps:
|
||||
size = size + heap.size
|
||||
return size
|
||||
|
||||
def get_total_mempool_size(self):
|
||||
size = 0
|
||||
for pool in self.mempools:
|
||||
size = size + pool.get_memzone_size_sum()
|
||||
return size
|
||||
|
||||
def get_total_memzone_size(self):
|
||||
size = 0
|
||||
for zone in self.memzones:
|
||||
size = size + zone.size
|
||||
return size
|
||||
|
||||
def print_summary(self):
|
||||
print("DPDK memory size {} in {} heap(s)"
|
||||
.format(B_to_MiB(self.size), len(self.heaps)))
|
||||
print("{} heaps totaling size {}".format(len(self.heaps), B_to_MiB(self.get_total_heap_size())))
|
||||
for x in sorted(self.heaps, key=lambda x: x.size, reverse=True):
|
||||
x.print_summary(' ')
|
||||
print("end heaps----------")
|
||||
print("{} mempools totaling size {}".format(len(self.mempools), B_to_MiB(self.get_total_mempool_size())))
|
||||
for x in sorted(self.mempools, key=lambda x: x.get_memzone_size_sum(), reverse=True):
|
||||
x.print_summary(' ')
|
||||
print("end mempools-------")
|
||||
print("{} memzones totaling size {}".format(len(self.memzones), B_to_MiB(self.get_total_memzone_size())))
|
||||
for x in sorted(self.memzones, key=lambda x: x.size, reverse=True):
|
||||
x.print_summary(' ')
|
||||
print("end memzones-------")
|
||||
|
||||
def print_heap_summary(self, heap_id):
|
||||
for heap in self.heaps:
|
||||
if heap_id == heap.id:
|
||||
heap.print_detailed_stats()
|
||||
break
|
||||
else:
|
||||
print("heap id {} is invalid. please see the summary for valid heaps.\n".format(heap_id))
|
||||
|
||||
def print_mempool_summary(self, name):
|
||||
for pool in self.mempools:
|
||||
if name == pool.name:
|
||||
pool.print_detailed_stats()
|
||||
break
|
||||
else:
|
||||
print("mempool name {} is invalid. please see the summary for valid mempools.\n".format(name))
|
||||
|
||||
def print_memzone_summary(self, name):
|
||||
for zone in self.memzones:
|
||||
if name == zone.name:
|
||||
zone.print_detailed_stats("")
|
||||
break
|
||||
else:
|
||||
print("memzone name {} is invalid. please see the summary for valid memzone.\n".format(name))
|
||||
|
||||
def associate_heap_elements_and_memzones(self):
|
||||
for zone in self.memzones:
|
||||
for heap_obj in self.heaps:
|
||||
for element in heap_obj.busy_malloc_elements:
|
||||
if element.check_memzone_compatibility(zone):
|
||||
heap_obj.busy_memzone_elements.append(element)
|
||||
heap_obj.busy_malloc_elements.remove(element)
|
||||
|
||||
def associate_memzones_and_mempools(self):
|
||||
for pool in self.mempools:
|
||||
for zone in self.memzones:
|
||||
if pool.name in zone.name:
|
||||
pool.add_memzone(zone)
|
||||
|
||||
for pool in self.mempools:
|
||||
for zone in pool.memzones:
|
||||
if zone in self.memzones:
|
||||
self.memzones.remove(zone)
|
||||
|
||||
|
||||
class heap_elem_status(Enum):
|
||||
FREE = 0
|
||||
BUSY = 1
|
||||
|
||||
|
||||
class heap_element:
|
||||
def __init__(self, size, status, addr):
|
||||
self.status = status
|
||||
self.size = size
|
||||
self.addr = addr
|
||||
self.memzone = None
|
||||
|
||||
def print_summary(self, header):
|
||||
print("{}element at address: {} with size: {:>15}".format(header, hex(self.addr), B_to_MiB(self.size)))
|
||||
|
||||
def check_memzone_compatibility(self, memzone):
|
||||
ele_fini_addr = self.addr + self.size
|
||||
memzone_fini_addr = memzone.address + memzone.size
|
||||
if (self.addr <= memzone.address and ele_fini_addr >= memzone_fini_addr):
|
||||
self.memzone = memzone
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class heap:
|
||||
def __init__(self, id, size, num_allocations):
|
||||
self.id = id
|
||||
self.size = size
|
||||
self.num_allocations = num_allocations
|
||||
self.free_elements = []
|
||||
self.busy_malloc_elements = []
|
||||
self.busy_memzone_elements = []
|
||||
|
||||
def add_element(self, element):
|
||||
if element.status == heap_elem_status.FREE:
|
||||
self.free_elements.append(element)
|
||||
else:
|
||||
self.busy_malloc_elements.append(element)
|
||||
|
||||
def print_element_stats(self, list_to_print, list_type, header):
|
||||
print("{}list of {} elements. size: {}".format(header, list_type, B_to_MiB(self.get_element_size(list_to_print))))
|
||||
for x in sorted(list_to_print, key=lambda x: x.size, reverse=True):
|
||||
x.print_summary("{} ".format(header))
|
||||
if x.memzone is not None:
|
||||
x.memzone.print_summary(" {}associated memzone info: ".format(header))
|
||||
|
||||
def get_element_size(self, list_to_check):
|
||||
size = 0
|
||||
for element in list_to_check:
|
||||
size = size + element.size
|
||||
return size
|
||||
|
||||
def print_summary(self, header):
|
||||
print("{}size: {:>15} heap id: {}".format(header, B_to_MiB(self.size), self.id))
|
||||
|
||||
def print_detailed_stats(self):
|
||||
print("heap id: {} total size: {} number of busy elements: {} number of free elements: {}"
|
||||
.format(self.id, B_to_MiB(self.size), len(self.busy_malloc_elements), len(self.free_elements)))
|
||||
self.print_element_stats(self.free_elements, "free", " ")
|
||||
self.print_element_stats(self.busy_malloc_elements, "standard malloc", " ")
|
||||
self.print_element_stats(self.busy_memzone_elements, "memzone associated", " ")
|
||||
|
||||
|
||||
class mempool:
|
||||
def __init__(self, name, num_objs, num_populated_objs, obj_size):
|
||||
self.name = name
|
||||
self.num_objs = num_objs
|
||||
self.num_populated_objs = num_populated_objs
|
||||
self.obj_size = obj_size
|
||||
self.memzones = []
|
||||
|
||||
def add_memzone(self, memzone):
|
||||
self.memzones.append(memzone)
|
||||
|
||||
def get_memzone_size_sum(self):
|
||||
size = 0
|
||||
for zone in self.memzones:
|
||||
size = size + zone.size
|
||||
return size
|
||||
|
||||
def print_summary(self, header):
|
||||
print("{}size: {:>15} name: {}"
|
||||
.format(header, B_to_MiB(self.get_memzone_size_sum()), self.name))
|
||||
|
||||
def print_detailed_stats(self):
|
||||
print("size: {:>15} name: {} comprised of {} memzone(s):"
|
||||
.format(B_to_MiB(self.get_memzone_size_sum()), self.name, len(self.memzones)))
|
||||
for x in sorted(self.memzones, key=lambda x: x.size, reverse=True):
|
||||
x.print_detailed_stats(" ")
|
||||
|
||||
|
||||
class memzone:
|
||||
def __init__(self, name, size, address):
|
||||
self.name = name
|
||||
self.size = size
|
||||
self.address = address
|
||||
self.segments = []
|
||||
|
||||
def add_segment(self, segment):
|
||||
self.segments.append(segment)
|
||||
|
||||
def print_summary(self, header):
|
||||
print("{}size: {:>15} name: {}".format(header, B_to_MiB(self.size), self.name))
|
||||
|
||||
def print_detailed_stats(self, header):
|
||||
self.print_summary(header)
|
||||
print("{}located at address {}".format(header, hex(self.address)))
|
||||
print("{}spanning {} segment(s):".format(header, len(self.segments)))
|
||||
for x in sorted(self.segments, key=lambda x: x.size, reverse=True):
|
||||
x.print_summary(' ')
|
||||
|
||||
|
||||
class segment:
|
||||
def __init__(self, size, address):
|
||||
self.size = size
|
||||
self.address = address
|
||||
|
||||
def print_summary(self, header):
|
||||
print("{}address: {} length: {:>15}".format(header, hex(self.address), B_to_MiB(self.size)))
|
||||
|
||||
|
||||
class parse_state(Enum):
|
||||
PARSE_MEMORY_SIZE = 0
|
||||
PARSE_MEMZONES = 1
|
||||
PARSE_MEMZONE_SEGMENTS = 2
|
||||
PARSE_MEMPOOLS = 3
|
||||
PARSE_MEMPOOL_INFO = 4
|
||||
PARSE_HEAPS = 5
|
||||
PARSE_HEAP_ELEMENTS = 6
|
||||
|
||||
|
||||
def B_to_MiB(raw_value):
|
||||
raw_value = raw_value / (1024.0 * 1024.0)
|
||||
|
||||
return "%6f %s" % (raw_value, "MiB")
|
||||
|
||||
|
||||
def parse_zone(line):
|
||||
zone, info = line.split(':', 1)
|
||||
name, length, addr, trash = info.split(',', 3)
|
||||
|
||||
trash, name = name.split(':', 1)
|
||||
name = name.replace("<", "")
|
||||
name = name.replace(">", "")
|
||||
trash, length = length.split(':', 1)
|
||||
trash, addr = addr.split(':', 1)
|
||||
|
||||
return memzone(name, int(length, 0), int(addr, 0))
|
||||
|
||||
|
||||
def parse_segment(line):
|
||||
trash, addr, iova, length, pagesz = line.split(':')
|
||||
addr, trash = addr.strip().split(' ')
|
||||
length, trash = length.strip().split(' ')
|
||||
|
||||
return segment(int(length, 0), int(addr, 0))
|
||||
|
||||
|
||||
def parse_mempool_name(line):
|
||||
trash, info = line.split()
|
||||
name, addr = line.split('@')
|
||||
name = name.replace("<", "")
|
||||
name = name.replace(">", "")
|
||||
trash, name = name.split()
|
||||
|
||||
return name
|
||||
|
||||
|
||||
def parse_mem_stats(stat_path):
|
||||
state = parse_state.PARSE_MEMORY_SIZE
|
||||
with open(stat_path, "r") as stats:
|
||||
|
||||
line = stats.readline()
|
||||
while line != '':
|
||||
if state == parse_state.PARSE_MEMORY_SIZE:
|
||||
if "DPDK memory size" in line:
|
||||
mem_size = int(line.replace("DPDK memory size ", ""))
|
||||
memory_struct = memory(mem_size)
|
||||
state = parse_state.PARSE_MEMZONES
|
||||
line = stats.readline()
|
||||
|
||||
if state == parse_state.PARSE_MEMZONES:
|
||||
if line.find("Zone") == 0:
|
||||
zone = parse_zone(line)
|
||||
state = parse_state.PARSE_MEMZONE_SEGMENTS
|
||||
line = stats.readline()
|
||||
|
||||
if state == parse_state.PARSE_MEMZONE_SEGMENTS:
|
||||
if line.find("Zone") == 0:
|
||||
memory_struct.add_memzone(zone)
|
||||
state = parse_state.PARSE_MEMZONES
|
||||
continue
|
||||
elif line.lstrip().find("addr:") == 0:
|
||||
segment = parse_segment(line)
|
||||
zone.add_segment(segment)
|
||||
elif "DPDK mempools." in line:
|
||||
state = parse_state.PARSE_MEMPOOLS
|
||||
continue
|
||||
line = stats.readline()
|
||||
|
||||
if state == parse_state.PARSE_MEMPOOLS:
|
||||
mempool_info = {}
|
||||
if line.find("mempool") == 0:
|
||||
mempool_info['name'] = parse_mempool_name(line)
|
||||
state = parse_state.PARSE_MEMPOOL_INFO
|
||||
line = stats.readline()
|
||||
|
||||
if state == parse_state.PARSE_MEMPOOL_INFO:
|
||||
if line.find("mempool") == 0:
|
||||
try:
|
||||
new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0),
|
||||
int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0))
|
||||
memory_struct.add_mempool(new_mempool)
|
||||
except KeyError:
|
||||
print("proper key values not provided for mempool.")
|
||||
state = parse_state.PARSE_MEMPOOLS
|
||||
continue
|
||||
elif "cache" in line:
|
||||
pass
|
||||
elif "DPDK malloc stats." in line:
|
||||
try:
|
||||
new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0),
|
||||
int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0))
|
||||
memory_struct.add_mempool(new_mempool)
|
||||
except KeyError:
|
||||
print("proper key values not provided for mempool.")
|
||||
while "DPDK malloc heaps." not in line:
|
||||
line = stats.readline()
|
||||
state = parse_state.PARSE_HEAPS
|
||||
else:
|
||||
try:
|
||||
field, value = line.strip().split('=')
|
||||
mempool_info[field] = value
|
||||
except Exception as e:
|
||||
pass
|
||||
line = stats.readline()
|
||||
|
||||
if state == parse_state.PARSE_HEAPS:
|
||||
trash, heap_id = line.strip().split(':')
|
||||
line = stats.readline()
|
||||
trash, heap_size = line.split(':')
|
||||
line = stats.readline()
|
||||
trash, num_allocations = line.split(':')
|
||||
if int(heap_size, 0) == 0:
|
||||
pass
|
||||
else:
|
||||
new_heap = heap(heap_id.lstrip(), int(heap_size, 0), int(num_allocations, 0))
|
||||
memory_struct.add_heap(new_heap)
|
||||
state = parse_state.PARSE_HEAP_ELEMENTS
|
||||
|
||||
line = stats.readline()
|
||||
|
||||
if state == parse_state.PARSE_HEAP_ELEMENTS:
|
||||
if line.find("Heap id") == 0:
|
||||
state = parse_state.PARSE_HEAPS
|
||||
continue
|
||||
elif line.find("Malloc element at") == 0:
|
||||
trash, address, status = line.rsplit(maxsplit=2)
|
||||
line = stats.readline()
|
||||
trash, length, trash = line.split(maxsplit=2)
|
||||
line = stats.readline()
|
||||
if "FREE" in status:
|
||||
element = heap_element(int(length, 0), heap_elem_status.FREE, int(address, 0))
|
||||
else:
|
||||
element = heap_element(int(length, 0), heap_elem_status.BUSY, int(address, 0))
|
||||
new_heap.add_element(element)
|
||||
line = stats.readline()
|
||||
|
||||
memory_struct.associate_heap_elements_and_memzones()
|
||||
memory_struct.associate_memzones_and_mempools()
|
||||
return memory_struct
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Dumps memory stats for DPDK. If no arguments are provided, it dumps a general summary.')
|
||||
parser.add_argument('-f', dest="stats_file", help='path to a dpdk memory stats file.', default='/tmp/spdk_mem_dump.txt')
|
||||
parser.add_argument('-m', '--heap', dest="heap", help='Print detailed information about the given heap.', default=None)
|
||||
parser.add_argument('-p', '--mempool', dest="mempool", help='Print detailed information about the given mempool.', default=None)
|
||||
parser.add_argument('-z', '--memzone', dest="memzone", help='Print detailed information about the given memzone.', default=None)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.stats_file):
|
||||
print("Error, specified stats file does not exist. Please make sure you have run the"
|
||||
"env_dpdk_get_mem_stats rpc on the spdk app you want to analyze.")
|
||||
exit(1)
|
||||
|
||||
mem_info = parse_mem_stats(args.stats_file)
|
||||
|
||||
summary = True
|
||||
if args.heap is not None:
|
||||
mem_info.print_heap_summary(args.heap)
|
||||
summary = False
|
||||
if args.mempool is not None:
|
||||
mem_info.print_mempool_summary(args.mempool)
|
||||
summary = False
|
||||
if args.memzone is not None:
|
||||
mem_info.print_memzone_summary(args.memzone)
|
||||
summary = False
|
||||
|
||||
if summary:
|
||||
mem_info.print_summary()
|
||||
Loading…
Reference in New Issue
Block a user