407 lines
15 KiB
Python
407 lines
15 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import os
|
||
|
|
from enum import Enum
|
||
|
|
|
||
|
|
|
||
|
|
class memory:
|
||
|
|
def __init__(self, size):
|
||
|
|
self.size = size
|
||
|
|
self.heaps = []
|
||
|
|
self.mempools = []
|
||
|
|
self.memzones = []
|
||
|
|
|
||
|
|
def get_size(self):
|
||
|
|
return self.size
|
||
|
|
|
||
|
|
def add_mempool(self, pool):
|
||
|
|
self.mempools.append(pool)
|
||
|
|
|
||
|
|
def add_memzone(self, zone):
|
||
|
|
self.memzones.append(zone)
|
||
|
|
|
||
|
|
def add_heap(self, heap):
|
||
|
|
self.heaps.append(heap)
|
||
|
|
|
||
|
|
def get_total_heap_size(self):
|
||
|
|
size = 0
|
||
|
|
for heap in self.heaps:
|
||
|
|
size = size + heap.size
|
||
|
|
return size
|
||
|
|
|
||
|
|
def get_total_mempool_size(self):
|
||
|
|
size = 0
|
||
|
|
for pool in self.mempools:
|
||
|
|
size = size + pool.get_memzone_size_sum()
|
||
|
|
return size
|
||
|
|
|
||
|
|
def get_total_memzone_size(self):
|
||
|
|
size = 0
|
||
|
|
for zone in self.memzones:
|
||
|
|
size = size + zone.size
|
||
|
|
return size
|
||
|
|
|
||
|
|
def print_summary(self):
|
||
|
|
print("DPDK memory size {} in {} heap(s)"
|
||
|
|
.format(B_to_MiB(self.size), len(self.heaps)))
|
||
|
|
print("{} heaps totaling size {}".format(len(self.heaps), B_to_MiB(self.get_total_heap_size())))
|
||
|
|
for x in sorted(self.heaps, key=lambda x: x.size, reverse=True):
|
||
|
|
x.print_summary(' ')
|
||
|
|
print("end heaps----------")
|
||
|
|
print("{} mempools totaling size {}".format(len(self.mempools), B_to_MiB(self.get_total_mempool_size())))
|
||
|
|
for x in sorted(self.mempools, key=lambda x: x.get_memzone_size_sum(), reverse=True):
|
||
|
|
x.print_summary(' ')
|
||
|
|
print("end mempools-------")
|
||
|
|
print("{} memzones totaling size {}".format(len(self.memzones), B_to_MiB(self.get_total_memzone_size())))
|
||
|
|
for x in sorted(self.memzones, key=lambda x: x.size, reverse=True):
|
||
|
|
x.print_summary(' ')
|
||
|
|
print("end memzones-------")
|
||
|
|
|
||
|
|
def print_heap_summary(self, heap_id):
|
||
|
|
for heap in self.heaps:
|
||
|
|
if heap_id == heap.id:
|
||
|
|
heap.print_detailed_stats()
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
print("heap id {} is invalid. please see the summary for valid heaps.\n".format(heap_id))
|
||
|
|
|
||
|
|
def print_mempool_summary(self, name):
|
||
|
|
for pool in self.mempools:
|
||
|
|
if name == pool.name:
|
||
|
|
pool.print_detailed_stats()
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
print("mempool name {} is invalid. please see the summary for valid mempools.\n".format(name))
|
||
|
|
|
||
|
|
def print_memzone_summary(self, name):
|
||
|
|
for zone in self.memzones:
|
||
|
|
if name == zone.name:
|
||
|
|
zone.print_detailed_stats("")
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
print("memzone name {} is invalid. please see the summary for valid memzone.\n".format(name))
|
||
|
|
|
||
|
|
def associate_heap_elements_and_memzones(self):
|
||
|
|
for zone in self.memzones:
|
||
|
|
for heap_obj in self.heaps:
|
||
|
|
for element in heap_obj.busy_malloc_elements:
|
||
|
|
if element.check_memzone_compatibility(zone):
|
||
|
|
heap_obj.busy_memzone_elements.append(element)
|
||
|
|
heap_obj.busy_malloc_elements.remove(element)
|
||
|
|
|
||
|
|
def associate_memzones_and_mempools(self):
|
||
|
|
for pool in self.mempools:
|
||
|
|
for zone in self.memzones:
|
||
|
|
if pool.name in zone.name:
|
||
|
|
pool.add_memzone(zone)
|
||
|
|
|
||
|
|
for pool in self.mempools:
|
||
|
|
for zone in pool.memzones:
|
||
|
|
if zone in self.memzones:
|
||
|
|
self.memzones.remove(zone)
|
||
|
|
|
||
|
|
|
||
|
|
class heap_elem_status(Enum):
|
||
|
|
FREE = 0
|
||
|
|
BUSY = 1
|
||
|
|
|
||
|
|
|
||
|
|
class heap_element:
|
||
|
|
def __init__(self, size, status, addr):
|
||
|
|
self.status = status
|
||
|
|
self.size = size
|
||
|
|
self.addr = addr
|
||
|
|
self.memzone = None
|
||
|
|
|
||
|
|
def print_summary(self, header):
|
||
|
|
print("{}element at address: {} with size: {:>15}".format(header, hex(self.addr), B_to_MiB(self.size)))
|
||
|
|
|
||
|
|
def check_memzone_compatibility(self, memzone):
|
||
|
|
ele_fini_addr = self.addr + self.size
|
||
|
|
memzone_fini_addr = memzone.address + memzone.size
|
||
|
|
if (self.addr <= memzone.address and ele_fini_addr >= memzone_fini_addr):
|
||
|
|
self.memzone = memzone
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
class heap:
|
||
|
|
def __init__(self, id, size, num_allocations):
|
||
|
|
self.id = id
|
||
|
|
self.size = size
|
||
|
|
self.num_allocations = num_allocations
|
||
|
|
self.free_elements = []
|
||
|
|
self.busy_malloc_elements = []
|
||
|
|
self.busy_memzone_elements = []
|
||
|
|
|
||
|
|
def add_element(self, element):
|
||
|
|
if element.status == heap_elem_status.FREE:
|
||
|
|
self.free_elements.append(element)
|
||
|
|
else:
|
||
|
|
self.busy_malloc_elements.append(element)
|
||
|
|
|
||
|
|
def print_element_stats(self, list_to_print, list_type, header):
|
||
|
|
print("{}list of {} elements. size: {}".format(header, list_type, B_to_MiB(self.get_element_size(list_to_print))))
|
||
|
|
for x in sorted(list_to_print, key=lambda x: x.size, reverse=True):
|
||
|
|
x.print_summary("{} ".format(header))
|
||
|
|
if x.memzone is not None:
|
||
|
|
x.memzone.print_summary(" {}associated memzone info: ".format(header))
|
||
|
|
|
||
|
|
def get_element_size(self, list_to_check):
|
||
|
|
size = 0
|
||
|
|
for element in list_to_check:
|
||
|
|
size = size + element.size
|
||
|
|
return size
|
||
|
|
|
||
|
|
def print_summary(self, header):
|
||
|
|
print("{}size: {:>15} heap id: {}".format(header, B_to_MiB(self.size), self.id))
|
||
|
|
|
||
|
|
def print_detailed_stats(self):
|
||
|
|
print("heap id: {} total size: {} number of busy elements: {} number of free elements: {}"
|
||
|
|
.format(self.id, B_to_MiB(self.size), len(self.busy_malloc_elements), len(self.free_elements)))
|
||
|
|
self.print_element_stats(self.free_elements, "free", " ")
|
||
|
|
self.print_element_stats(self.busy_malloc_elements, "standard malloc", " ")
|
||
|
|
self.print_element_stats(self.busy_memzone_elements, "memzone associated", " ")
|
||
|
|
|
||
|
|
|
||
|
|
class mempool:
|
||
|
|
def __init__(self, name, num_objs, num_populated_objs, obj_size):
|
||
|
|
self.name = name
|
||
|
|
self.num_objs = num_objs
|
||
|
|
self.num_populated_objs = num_populated_objs
|
||
|
|
self.obj_size = obj_size
|
||
|
|
self.memzones = []
|
||
|
|
|
||
|
|
def add_memzone(self, memzone):
|
||
|
|
self.memzones.append(memzone)
|
||
|
|
|
||
|
|
def get_memzone_size_sum(self):
|
||
|
|
size = 0
|
||
|
|
for zone in self.memzones:
|
||
|
|
size = size + zone.size
|
||
|
|
return size
|
||
|
|
|
||
|
|
def print_summary(self, header):
|
||
|
|
print("{}size: {:>15} name: {}"
|
||
|
|
.format(header, B_to_MiB(self.get_memzone_size_sum()), self.name))
|
||
|
|
|
||
|
|
def print_detailed_stats(self):
|
||
|
|
print("size: {:>15} name: {} comprised of {} memzone(s):"
|
||
|
|
.format(B_to_MiB(self.get_memzone_size_sum()), self.name, len(self.memzones)))
|
||
|
|
for x in sorted(self.memzones, key=lambda x: x.size, reverse=True):
|
||
|
|
x.print_detailed_stats(" ")
|
||
|
|
|
||
|
|
|
||
|
|
class memzone:
|
||
|
|
def __init__(self, name, size, address):
|
||
|
|
self.name = name
|
||
|
|
self.size = size
|
||
|
|
self.address = address
|
||
|
|
self.segments = []
|
||
|
|
|
||
|
|
def add_segment(self, segment):
|
||
|
|
self.segments.append(segment)
|
||
|
|
|
||
|
|
def print_summary(self, header):
|
||
|
|
print("{}size: {:>15} name: {}".format(header, B_to_MiB(self.size), self.name))
|
||
|
|
|
||
|
|
def print_detailed_stats(self, header):
|
||
|
|
self.print_summary(header)
|
||
|
|
print("{}located at address {}".format(header, hex(self.address)))
|
||
|
|
print("{}spanning {} segment(s):".format(header, len(self.segments)))
|
||
|
|
for x in sorted(self.segments, key=lambda x: x.size, reverse=True):
|
||
|
|
x.print_summary(' ')
|
||
|
|
|
||
|
|
|
||
|
|
class segment:
|
||
|
|
def __init__(self, size, address):
|
||
|
|
self.size = size
|
||
|
|
self.address = address
|
||
|
|
|
||
|
|
def print_summary(self, header):
|
||
|
|
print("{}address: {} length: {:>15}".format(header, hex(self.address), B_to_MiB(self.size)))
|
||
|
|
|
||
|
|
|
||
|
|
class parse_state(Enum):
|
||
|
|
PARSE_MEMORY_SIZE = 0
|
||
|
|
PARSE_MEMZONES = 1
|
||
|
|
PARSE_MEMZONE_SEGMENTS = 2
|
||
|
|
PARSE_MEMPOOLS = 3
|
||
|
|
PARSE_MEMPOOL_INFO = 4
|
||
|
|
PARSE_HEAPS = 5
|
||
|
|
PARSE_HEAP_ELEMENTS = 6
|
||
|
|
|
||
|
|
|
||
|
|
def B_to_MiB(raw_value):
|
||
|
|
raw_value = raw_value / (1024.0 * 1024.0)
|
||
|
|
|
||
|
|
return "%6f %s" % (raw_value, "MiB")
|
||
|
|
|
||
|
|
|
||
|
|
def parse_zone(line):
|
||
|
|
zone, info = line.split(':', 1)
|
||
|
|
name, length, addr, trash = info.split(',', 3)
|
||
|
|
|
||
|
|
trash, name = name.split(':', 1)
|
||
|
|
name = name.replace("<", "")
|
||
|
|
name = name.replace(">", "")
|
||
|
|
trash, length = length.split(':', 1)
|
||
|
|
trash, addr = addr.split(':', 1)
|
||
|
|
|
||
|
|
return memzone(name, int(length, 0), int(addr, 0))
|
||
|
|
|
||
|
|
|
||
|
|
def parse_segment(line):
|
||
|
|
trash, addr, iova, length, pagesz = line.split(':')
|
||
|
|
addr, trash = addr.strip().split(' ')
|
||
|
|
length, trash = length.strip().split(' ')
|
||
|
|
|
||
|
|
return segment(int(length, 0), int(addr, 0))
|
||
|
|
|
||
|
|
|
||
|
|
def parse_mempool_name(line):
|
||
|
|
trash, info = line.split()
|
||
|
|
name, addr = line.split('@')
|
||
|
|
name = name.replace("<", "")
|
||
|
|
name = name.replace(">", "")
|
||
|
|
trash, name = name.split()
|
||
|
|
|
||
|
|
return name
|
||
|
|
|
||
|
|
|
||
|
|
def parse_mem_stats(stat_path):
|
||
|
|
state = parse_state.PARSE_MEMORY_SIZE
|
||
|
|
with open(stat_path, "r") as stats:
|
||
|
|
|
||
|
|
line = stats.readline()
|
||
|
|
while line != '':
|
||
|
|
if state == parse_state.PARSE_MEMORY_SIZE:
|
||
|
|
if "DPDK memory size" in line:
|
||
|
|
mem_size = int(line.replace("DPDK memory size ", ""))
|
||
|
|
memory_struct = memory(mem_size)
|
||
|
|
state = parse_state.PARSE_MEMZONES
|
||
|
|
line = stats.readline()
|
||
|
|
|
||
|
|
if state == parse_state.PARSE_MEMZONES:
|
||
|
|
if line.find("Zone") == 0:
|
||
|
|
zone = parse_zone(line)
|
||
|
|
state = parse_state.PARSE_MEMZONE_SEGMENTS
|
||
|
|
line = stats.readline()
|
||
|
|
|
||
|
|
if state == parse_state.PARSE_MEMZONE_SEGMENTS:
|
||
|
|
if line.find("Zone") == 0:
|
||
|
|
memory_struct.add_memzone(zone)
|
||
|
|
state = parse_state.PARSE_MEMZONES
|
||
|
|
continue
|
||
|
|
elif line.lstrip().find("addr:") == 0:
|
||
|
|
segment = parse_segment(line)
|
||
|
|
zone.add_segment(segment)
|
||
|
|
elif "DPDK mempools." in line:
|
||
|
|
state = parse_state.PARSE_MEMPOOLS
|
||
|
|
continue
|
||
|
|
line = stats.readline()
|
||
|
|
|
||
|
|
if state == parse_state.PARSE_MEMPOOLS:
|
||
|
|
mempool_info = {}
|
||
|
|
if line.find("mempool") == 0:
|
||
|
|
mempool_info['name'] = parse_mempool_name(line)
|
||
|
|
state = parse_state.PARSE_MEMPOOL_INFO
|
||
|
|
line = stats.readline()
|
||
|
|
|
||
|
|
if state == parse_state.PARSE_MEMPOOL_INFO:
|
||
|
|
if line.find("mempool") == 0:
|
||
|
|
try:
|
||
|
|
new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0),
|
||
|
|
int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0))
|
||
|
|
memory_struct.add_mempool(new_mempool)
|
||
|
|
except KeyError:
|
||
|
|
print("proper key values not provided for mempool.")
|
||
|
|
state = parse_state.PARSE_MEMPOOLS
|
||
|
|
continue
|
||
|
|
elif "cache" in line:
|
||
|
|
pass
|
||
|
|
elif "DPDK malloc stats." in line:
|
||
|
|
try:
|
||
|
|
new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0),
|
||
|
|
int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0))
|
||
|
|
memory_struct.add_mempool(new_mempool)
|
||
|
|
except KeyError:
|
||
|
|
print("proper key values not provided for mempool.")
|
||
|
|
while "DPDK malloc heaps." not in line:
|
||
|
|
line = stats.readline()
|
||
|
|
state = parse_state.PARSE_HEAPS
|
||
|
|
else:
|
||
|
|
try:
|
||
|
|
field, value = line.strip().split('=')
|
||
|
|
mempool_info[field] = value
|
||
|
|
except Exception as e:
|
||
|
|
pass
|
||
|
|
line = stats.readline()
|
||
|
|
|
||
|
|
if state == parse_state.PARSE_HEAPS:
|
||
|
|
trash, heap_id = line.strip().split(':')
|
||
|
|
line = stats.readline()
|
||
|
|
trash, heap_size = line.split(':')
|
||
|
|
line = stats.readline()
|
||
|
|
trash, num_allocations = line.split(':')
|
||
|
|
if int(heap_size, 0) == 0:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
new_heap = heap(heap_id.lstrip(), int(heap_size, 0), int(num_allocations, 0))
|
||
|
|
memory_struct.add_heap(new_heap)
|
||
|
|
state = parse_state.PARSE_HEAP_ELEMENTS
|
||
|
|
|
||
|
|
line = stats.readline()
|
||
|
|
|
||
|
|
if state == parse_state.PARSE_HEAP_ELEMENTS:
|
||
|
|
if line.find("Heap id") == 0:
|
||
|
|
state = parse_state.PARSE_HEAPS
|
||
|
|
continue
|
||
|
|
elif line.find("Malloc element at") == 0:
|
||
|
|
trash, address, status = line.rsplit(maxsplit=2)
|
||
|
|
line = stats.readline()
|
||
|
|
trash, length, trash = line.split(maxsplit=2)
|
||
|
|
line = stats.readline()
|
||
|
|
if "FREE" in status:
|
||
|
|
element = heap_element(int(length, 0), heap_elem_status.FREE, int(address, 0))
|
||
|
|
else:
|
||
|
|
element = heap_element(int(length, 0), heap_elem_status.BUSY, int(address, 0))
|
||
|
|
new_heap.add_element(element)
|
||
|
|
line = stats.readline()
|
||
|
|
|
||
|
|
memory_struct.associate_heap_elements_and_memzones()
|
||
|
|
memory_struct.associate_memzones_and_mempools()
|
||
|
|
return memory_struct
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
parser = argparse.ArgumentParser(description='Dumps memory stats for DPDK. If no arguments are provided, it dumps a general summary.')
|
||
|
|
parser.add_argument('-f', dest="stats_file", help='path to a dpdk memory stats file.', default='/tmp/spdk_mem_dump.txt')
|
||
|
|
parser.add_argument('-m', '--heap', dest="heap", help='Print detailed information about the given heap.', default=None)
|
||
|
|
parser.add_argument('-p', '--mempool', dest="mempool", help='Print detailed information about the given mempool.', default=None)
|
||
|
|
parser.add_argument('-z', '--memzone', dest="memzone", help='Print detailed information about the given memzone.', default=None)
|
||
|
|
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
if not os.path.exists(args.stats_file):
|
||
|
|
print("Error, specified stats file does not exist. Please make sure you have run the"
|
||
|
|
"env_dpdk_get_mem_stats rpc on the spdk app you want to analyze.")
|
||
|
|
exit(1)
|
||
|
|
|
||
|
|
mem_info = parse_mem_stats(args.stats_file)
|
||
|
|
|
||
|
|
summary = True
|
||
|
|
if args.heap is not None:
|
||
|
|
mem_info.print_heap_summary(args.heap)
|
||
|
|
summary = False
|
||
|
|
if args.mempool is not None:
|
||
|
|
mem_info.print_mempool_summary(args.mempool)
|
||
|
|
summary = False
|
||
|
|
if args.memzone is not None:
|
||
|
|
mem_info.print_memzone_summary(args.memzone)
|
||
|
|
summary = False
|
||
|
|
|
||
|
|
if summary:
|
||
|
|
mem_info.print_summary()
|