[Python] generic gdb python helpers to traverse c++ STL container and invoke callback for the...

I want to debug c++ application core and gcore files.
This includes creating reports for relevant data in data repositories of the binary.
Those repositories are based on STL containers (std::map<>, std::set<>, std::vector<>, ...)
For this I use my own "generic" python helpers to automatically traverse STL containers, invoke callback function to handle the data, and create data reports.

Here is an example:

Test c++ program:

#include <string>
#include <map>
#include <iostream>

struct DemoStruct
{
DemoStruct(int arg1, double arg2, std::string arg3, std::string arg4, char arg5, double arg6)
: wantSeeThis_1(arg1),
wantIgnoreThis_1(arg2),
wantSeeThis_2(arg3),
wantIgnoreThis_2(arg4),
wantIgnoreThis_3(arg5),
wantSeeThis_3(arg6)
{ };

~DemoStruct() = default;

int wantSeeThis_1;
double wantIgnoreThis_1;
std::string wantSeeThis_2;
std::string wantIgnoreThis_2;
char wantIgnoreThis_3;
double wantSeeThis_3;
};

std::map<int, DemoStruct> staticData = {
{ 1, {1, 2.0, "hello", "world", 'a', 3.0 } },
{ 2, {10, 3.2, "hi", "world", 'b', 0.3 } },
{ 3, {50, 4.7, "salute", "world", 'c', 11.5 } },
};

int main(int, char**)
{
// set breakpoint here and inspect data
return 0;
}


compile it with:

> g++ -std=c++11 -o main -g3 -O0 main.cpp


use #2 python scripts for debugging:

File #1:

> cat stlhelper.py
# -*- coding: utf-8 -*-

import gdb
import sys
import re
import itertools
import six


#
# base class for PrintGenricXY
#
class PrintGenericCommon (gdb.Function):
"""print generic common"""

def __init__ (self):
super (PrintGenericCommon, self).__init__ ("PrintGenericList")

#
# data is hosted by the subject and we don't know their representation
# subject.ElemItemString() is responsible to format properly (might be complex data)
#
def PrintResult(self, subject):
# calculate the max sizes per element of all lines for alignment
sizes = subject.sizesDef
for x in subject.result:
for key, value in six.iteritems(sizes):
valStr = subject.ElemItemString(x[key])
if len(valStr) > value:
sizes[key] = len(valStr)

totalSize = 0
for key, value in six.iteritems(sizes):
sizes[key] += 2
totalSize += sizes[key]

print("-" * totalSize)
for i in range(len(subject.result)):
myStr = ''
for j in range(len(subject.elements)):
setSize=sizes[subject.elements[j]]
if (j == len(subject.elements) - 1):
setSize=0 # don't auto-align the last element, which often can have some large elements in the list
myStr += "%-*s" % (setSize, subject.ElemItemString(subject.result[subject.elements[j]]))
print (myStr)
# print optional __VERBOSE data in a next line
if '__VERBOSE' in subject.result:
str = subject.ElemItemString(subject.result['__VERBOSE'])
lines = str.splitlines()
for line in lines:
print ('\t' + line)
if len(str) > 0 and str[-1] == '\n':
print('')
if i == 0:
print ("-" * totalSize)


#
# generic printer for std::map<> and std::set<>
#
class PrintGenericMapOrSet (PrintGenericCommon):
"""print generic map/set"""

regex = re.compile('\$count')

maxNoResults = -1

def __init__ (self):
super (PrintGenericCommon, self).__init__ ("PrintGenericMapOrSet")

def setMaxNoResults(self, maxNoResults):
self.maxNoResults = maxNoResults


def invokeCore(self, headlines, map, valueType, isConst, isPointer, valueHandler):
collectedLines = []
map = map['_M_t']['_M_impl']
count = map['_M_node_count']
for x in headlines:
collectedLines.append(self.regex.sub(str(count), x))
node = map['_M_header']['_M_left']
valuetype = gdb.lookup_type (valueType)
if isConst:
valuetype = valuetype.const()
if isPointer:
valuetype = valuetype.pointer()
nodetype = gdb.lookup_type('std::_Rb_tree_node < %s >' % valuetype)
nodetype = nodetype.pointer()
i = 0
while i < count:
collectedLines.append(valueHandler(i, node.cast (nodetype).dereference()['_M_value_field'], node))
if node.dereference()['_M_right']:
node = node.dereference()['_M_right']
while node.dereference()['_M_left']:
node = node.dereference()['_M_left']
else:
parent = node.dereference()['_M_parent']
while node == parent.dereference()['_M_right']:
node = parent
parent = parent.dereference()['_M_parent']
if node.dereference()['_M_right'] != parent:
node = parent
i += 1
if self.maxNoResults > -1 and i >= self.maxNoResults:
break;
return "\n".join(collectedLines)

def invoke (self, headlines, mapLoc, valueType, valueHandler, typeWrapper):
map = gdb.parse_and_eval (mapLoc)
return self.invokeCore(headlines, map, valueType, valueHandler, typeWrapper);

#
# generic printer for std::map<>
#
class PrintGenericMap (gdb.Function):
"""print generic map"""

handler = PrintGenericMapOrSet()

def __init__ (self):
super (PrintGenericMap, self).__init__ ("PrintGenericMap")

def setMaxNoResults(self, maxNoResults):
return self.handler.setMaxNoResults(maxNoResults)

def invokeCore(self, headlines, map, valueType, valueHandler):
return self.handler.invokeCore(headlines, map, 'std:air' + valueType, False, False, valueHandler)

def invoke (self, headlines, mapLoc, valueType, valueHandler, maxNoResults):
return self.handler.invoke(headlines, mapLoc, 'std:air' + valueType, False, False, valueHandler);

def PrintResult(self, subject):
return self.handler.PrintResult(subject)


#
# generic printer for std::set<>
#
class PrintGenericSet (gdb.Function):
"""print generic set"""

handler = PrintGenericMapOrSet()

def __init__ (self):
super (PrintGenericSet, self).__init__ ("PrintGenericSet")

def setMaxNoResults(self, maxNoResults):
return self.handler.setMaxNoResults(maxNoResults)

def invokeCore(self, headlines, map, valueType, isConst, isPointer, valueHandler):
return self.handler.invokeCore(headlines, map, valueType, isConst, isPointer, valueHandler)

def invoke (self, headlines, mapLoc, valueType, isConst, isPointer, valueHandler, maxNoResults):
return self.handler.invoke(headlines, mapLoc, valueType, isConst, isPointer, valueHandler);

def PrintResult(self, subject):
return self.handler.PrintResult(subject)


File #2:

> cat main.py
# -*- coding: utf-8 -*-

import gdb, platform, six, copy

from stlhelper import PrintGenericMap, PrintGenericSet

class PrintStaticData (gdb.Command):
"""Print STL data demo.\n"""

printer = None
result = []
headline = {}
sizesDef = {}

elements = [
'key',
'see_1',
'see_2',
'see_3',
]

errorEntry = {
'key' : '?',
'see_1' : '?',
'see_2' : '?',
'see_3' : '?',
}

def __init__ (self):
super (PrintStaticData, self).__init__ ("PrintStaticData", gdb.COMMAND_USER)
self.headline = {
'key' : 'Key',
'see_1' : 'Value #1',
'see_2' : 'Value #2',
'see_3' : 'Value #3',
}

def ReInit(self):
if(platform.python_version().startswith("2")):
for key, value in six.iteritems(self.headline):
self.sizesDef[key] = 0
else:
for key in self.headline.keys():
self.sizesDef[key] = 0
self.result = []
self.result.append(self.headline)

def ElemItemString(self, elem):
return elem

def Traverse(self, index, pair, node = None):
first = pair['first']
second = pair['second']
newEntry = copy.deepcopy(self.errorEntry)
newEntry['key'] = str(first)
newEntry['see_1'] = str(second['wantSeeThis_1'])
newEntry['see_2'] = str(second['wantSeeThis_2'])
newEntry['see_3'] = str(second['wantSeeThis_3'])
self.result.append(newEntry)
return ''

def invoke (self, arg, from_tty):
self.ReInit()

self.printer = PrintGenericMap()

data = gdb.parse_and_eval("staticData")
result = self.printer.invokeCore(
[],
data,
'<int const, DemoStruct>',
self.Traverse)

self.printer.PrintResult(self)

PrintStaticData()


.gdbinit file (adjust for your own paths):

cat $HOME/.gdbinit
set print object
set print pretty
set print static off
set pagination off
set auto-load safe-path /

python
sys.path.insert(0, "/home/me/TEST")
import main
end

set history filename ~/.gdb_history
set history save


Execute the demo in 'gdb' with e.g. gcc 4.8.5

> gdb main
(gdb) break 35
Breakpoint 1 at 0x400c18: file main.cpp, line 35.

(gdb) r
Starting program: /home/me/TEST/main

Breakpoint 1, main () at main.cpp:35
35 return 0;

(gdb) PrintStaticData
----------------------------------------------
Key Value #1 Value #2 Value #3
----------------------------------------------
1 1 "hello" 3
2 10 "hi" 0.29999999999999999
3 50 "salute" 11.5


So far so good.

The problem now is, that it does not work for gcc 9.4.0, anymore. :-(
The reason is, that the internal data structure for STL type std::map<> changed.
My python helpers are not really "generic" - they depend on the concrete STL implementation.
Using gcc 9.4.0 I hit

(gdb) PrintStaticData
Python Exception <class 'gdb.error'> There is no member or method named _M_value_field.:
Error occurred in Python: There is no member or method named _M_value_field.


There is no _M_value_field anymore - which I use here in stdlhelper.py:

collectedLines.append(valueHandler(i, node.cast (nodetype).dereference()['_M_value_field'], node))


So I could try to update the python code to also handle the new STL implementation.
But I think (hope), that there is already some generic tool in the gcc python STL toolbox available to do the same:
Traverse any STL container - without need to know about it's implementation - and invoke a callback function to handle each element that is stored in the container.

Is such available from gcc python helpers?
Maybe some link to documentation / howto?

Continue reading...