[Python] How to get all the subclasses of an abstract class in different files

I'm trying to apply design pattern to make it extendable and easy to maintain.

Here is my file structure:

/evaluation
│── metrics/
│──│── metric.py # abstract class
│──│── QWK.py
│──│── PCC.py
│── evaluators/
│──│── __init__.py
│──│── evaluator.py # abstract class
│──│── score_evaluator.py
│──│── feedback_evaluator.py
│── evaluator_factory.py


evaluator.py

class Evaluator(ABC):
def __init__(self, evaluation_config, framework_config, **kwargs):

self.evaluation_config = evaluation_config
self.framework_config = framework_config

self.logger = WandbLogger()

@abstractmethod
def load_evaluation_data(self, data_path):
pass

@abstractmethod
def __call__(self, *args, **kwargs):
pass


score_evaluator.py

class ScoreEvaluator(Evaluator):
def __init__(self, evaluation_config, framework_config, **kwargs):
super().__init__(evaluation_config, framework_config, **kwargs)

self.score_generator = ScoreGenerator(llm=LLMClient(),
vlm=VLMClient(),
config=self.framework_config)


def load_evaluation_data(self, data_path):
...
return evaluation_data


async def __call__(self, *args, **kwargs):
...

return evaluation_results



evaluator_factory.py

from evaluation.evaluators.evaluator import Evaluator

class EvaluatorFactory:
def __init__(self, framework_config, evaluation_config, **kwargs):
self.framework_config = framework_config
self.evaluation_config = evaluation_config
self.kwargs = kwargs

self.evaluators = self._create_evaluators()

def _create_evaluators(self):
evaluators = []
evaluator_classes = Evaluator.__subclasses__()
print(evaluator_classes) # bug here, get empty list
evaluator_module = importlib.import_module("evaluation.evaluators.evaluator")
print(evaluator_module) # second attempt using importlib, and inspect still get empty dict.
evaluators = {
name: cls()
for name, cls in inspect.getmembers(evaluator_module, inspect.isclass)
if issubclass(cls, evaluator_module.Evaluator) and cls is not evaluator_module.Evaluator
}

print(evaluators)
for cls in evaluator_classes:
evaluators.append(cls(self.framework_config, self.evaluation_config, **self.kwargs))

return evaluators

async def evaluate(self):
evaluation_results = {}
print(len(self.evaluators))
for evaluator in self.evaluators:
results = await evaluator()
evaluation_results[evaluator.__name__] = results

return evaluation_results



Dear design pattern specialists, help me.

I also want to design something with the metrics:

• As some metrics are only designed for specific evaluator, so I don't know how is the best practice to design this, for example when someone in the team try to create a new metric for an evaluator, it's easy to scale without modify the code.

Thank youuuuuu so much~


Edit 1

I figured out that __subclasses__ only works when the subclasses already loaded in memory, to do so I need to register it in the __init__ file:

evaluators/__init__.py

from evaluation.evaluators.score_evaluator import ScoreEvaluator
from evaluation.evaluators.feedback_evaluator import FeedbackEvaluator

__all__ = ["ScoreEvaluator", "FeedbackEvaluator"]


Still waiting for any other approaches and my second concern

Continue reading...