[Python] Can tree-sitter be used on a QsciLexerCustom in PyQt?

I want to implement a custom lexer (syntax highlighting) for my QScintilla based IDE in PyQt. I tried to use tree sitter's python bindings to parse the editor's text, then use a bunch of if/else statements to figure out the lengths of each node in the language (more on setStyling here)

# ...
import tree_sitter_python as PYTHON
from tree_sitter import Parser, Node, Language

class PythonLexer(QsciLexerCustom):
DEFAULT = 0
KEYWORD = 1
TYPES = 2
STRING = 3
KEYARGS = 4
BRACKETS = 5
COMMENTS = 6
CONSTANTS = 7
FUNCTIONS = 8
CLASS_DEF = 9
FUNCTION_DEF = 10

def __init__(self, editor: QsciScintilla):
super().__init__(editor, 'Python')

self.editor = editor
self.language_name = language_name

defaults = {}
defaults['color'] = '#ffffff'
defaults['paper'] = '#1e1e1e'
defaults['font'] = ('JetBrains Mono', 14)

self.setDefaultColor(QColor(defaults['color']))
self.setDefaultPaper(QColor(defaults['paper']))
self.setDefaultFont(QFont(defaults['font'][0], defaults['font'][1]))
self.createStyle()

self.parser = Parser(Language(PYTHON.language()))

def language(self):
return self.language_name

def description(self, style):
if style == self.DEFAULT:
return 'DEFAULT'

elif style == self.KEYWORD:
return 'KEYWORD'

elif style == self.TYPES:
return 'TYPES'

elif style == self.STRING:
return 'STRING'

elif style == self.KEYARGS:
return 'kWARGS'

elif style == self.BRACKETS:
return 'BRACKETS'

elif style == self.COMMENTS:
return 'COMMENTS'

elif style == self.CONSTANTS:
return 'CONSTANTS'

elif style == self.FUNCTIONS:
return 'FUNCTIONS'

elif style == self.CLASS_DEF:
return 'CLASS_DEF'

elif style == self.FUNCTION_DEF:
return 'FUNCTION_DEF'

return ''

def createStyle(self):
normal = QColor('#abb2bf')
font = QFont('JetBrains Mono', 14)
italic = font
italic.setItalic(True)

self.setFont(italic, PythonLexer.COMMENTS)
self.setColor(normal, PythonLexer.DEFAULT)
self.setColor(normal, PythonLexer.BRACKETS)
self.setColor(QColor('#7f848e'), PythonLexer.COMMENTS)
self.setColor(QColor('#c678dd'), PythonLexer.KEYWORD)
self.setColor(QColor('#e5C07b'), PythonLexer.CLASS_DEF)
self.setColor(QColor('#61afef'), PythonLexer.FUNCTIONS)
self.setColor(QColor('#61afef'), PythonLexer.FUNCTION_DEF)
self.setColor(QColor('#56b6c2'), PythonLexer.TYPES)
self.setColor(QColor('#d19a66'), PythonLexer.CONSTANTS)
self.setColor(QColor('#98c379'), PythonLexer.STRING)

def styleText(self, start, end):
self.startStyling(start)

raw_bytes = self.editor.bytes(start, end)
text = raw_bytes.data().decode('utf-8').replace('\0', '')
tree = self.parser.parse(bytes(text, 'utf-8'))
print(tree.root_node)

highlights = []
self.buildHighlights(tree.root_node, highlights)
highlights.sort(key=lambda h: h[0]) # sort by start byte

for start_byte, end_byte, style in highlights:
print(f'Start byte: {start_byte}, Line length: {end_byte}, Style: {style}')
self.setStyling(end_byte, style)

def buildHighlights(self, node: Node, highlights: list):
for child in node.children:
style = None

if child.type == 'comment':
style = PythonLexer.COMMENTS

elif child.type == 'string':
style = PythonLexer.STRING

elif child.type == 'call':
# builtins
style = PythonLexer.TYPES

elif child.type == 'function_definition':
style = PythonLexer.FUNCTION_DEF

elif child.type == 'class_definition':
style = PythonLexer.CLASS_DEF

elif child.type == 'keyword':
style = PythonLexer.KEYWORD

elif child.type == 'integer':
style = PythonLexer.CONSTANTS

elif child.text in ('(', ')', '{', '}', '[', ']'):
style = PythonLexer.BRACKETS

else:
style = PythonLexer.DEFAULT

if style:
highlights.append((child.start_byte, child.end_byte, style))

self.buildHighlights(child, highlights)


The problem I encounter is this doesn't seem to work beyond strings and comments. Colors are mixed and matched randomly:

syntax highlighting attempt

It gets even worse as I edit the text.

Is there a better way to do this? I am aware of the tokenizing method via re but I would strongly prefer tree sitter due to it's language support.

Any help is appreciated

Continue reading...