[Python] How to match and populate a generic Pydantic model with deeply nested fields from...

I'm building a generalized Pydantic model (let's call it model X) that I want to use to match and extract values from multiple different JSON schemas (model A, B, C, etc.). These schemas are fetched via API and cannot be modified at the source.

My process:

1. GET JSON Schema A via API
2. GET populated JSON data for A
3. Generate a Pydantic model from schema A (datamodel_code_generator)
4. GET my Generalized schema --> generate pydantic model X
5. Create an empty instance of generalized model X (problematic due to required fields)
6. Match model A to model X
7. Populate X with values that have the same path and compatible type (e.g., if bith have system.model type: str)

Challenges:

• I don’t control the incoming schema structures or $ref names — only the content.
• Although the overall structure is similar, the naming and typing sometimes differ (e.g., system.type might be a string in one schema and a list in another) or ($ref/subclass names vary).
• Required fields in model X make instantiating an "empty but fully structured" version difficult.

Aim:

find a robust way to:

• Create a fully structured (but unpopulated) instance of X
• Match paths/types between A and X even when $ref names differ
• Create for flexible population and validation of shared structures

?? Any advice on:

• What would be a good strategy for this kind of schema abstraction and instance matching?
• libraries, techniques, tools, patterns or other to help matching/populating fields?

Simplified Generic Schema


{
"required": ["systems", "results"],
"properties": {
"systems": {
"$ref": "#/definitions/System"
},
"results": {
"$ref": "#/definitions/ResultList"
}
},
"definitions": {
"System": {
"type": "object",
"required": ["vendor", "model", "type", "operator"],
"properties": {
"vendor": { "type": "string" },
"model": { "type": "string" },
"type": { "type": "string" },
"operator": { "type": "string" }
}
},
"Result": {
"type": "object",
"properties": {
"x": { "type": "integer" },
"y": { "type": "integer" }
},
"required": ["x", "y"]
},
"ResultList": {
"type": "array",
"items": { "$ref": "#/definitions/Result" }
}
}
}

simplified example of schema A


{
"$schema": "http://json-schema.org/draft-07/schema",
"type": "object",
"required": ["systems", "users", "results"],
"properties": {
"systems": {
"type": "array",
"items": { "$ref": "#/definitions/turbineSystem" }
},
"users": {
"type": "array",
"items": { "$ref": "#/definitions/User" }
},
"results": {
"$ref": "#/definitions/ResultList"
}
},
"definitions": {
"turbineSystem": {
"type": "object",
"required": ["vendor", "model", "type", "serial_number"],
"properties": {
"vendor": { "type": "string" },
"model": { "type": "string" },
"type": {
"type": "array",
"items": { "type": "string" }
},
"serial_number": { "type": "string" }
}
},
"User": {
"type": "object",
"properties": {
"name": { "type": "string" }
}
},
"turbineResult": {
"type": "object",
"required": ["x", "y"],
"properties": {
"x": { "type": "string" }, // ← conflict: was integer in Generic
"y": { "type": "integer" }
}
},
"ResultList": {
"type": "array",
"items": { "$ref": "#/definitions/turbineResult" } // ← conflict: reference is different in Generic
}
}
}


I tried:

• Match the structure of a JSON schemas (Model A) with a simpler Generic/generalized schema via pydantic models to assure types.
• Auto-populated model X with values from model A when paths and types align.

I expected:

• To create a "template" instance of Model X with placeholder/default values so it can be filled with matching values from a populated model A object.
• To automatically populate Model X with only the fields from Model A that: Exist in both models, have compatible types, and match by field path (e.g. systems[].model, results[].x, etc.)
• to make a flexible, reusable way to abstract over many different schemas using a shared generalized structure

import json
import os
from datamodel_code_generator.parser.jsonschema import JsonSchemaParser

import tempfile

import importlib.util
import sys

from pydantic import BaseModel


## Works as it should
def get_pydantic_model_from_jsonschema(json_schema: dict) -> type[BaseModel]:
parser = JsonSchemaParser(str(json_schema))

model_code = parser.parse()

# Save to a temporary .py file - i only need to model during runtime
with tempfile.NamedTemporaryFile(delete=False, suffix=".py") as temp_file:
temp_file.write(model_code.encode('utf-8'))
temp_file_path = temp_file.name

# Import the generated module
spec = importlib.util.spec_from_file_location("generated_model", temp_file_path)
generated_model = importlib.util.module_from_spec(spec)
sys.modules["generated_model"] = generated_model
spec.loader.exec_module(generated_model)

os.remove(temp_file_path)

model_class = getattr(generated_model, "Model")

return model_class

# Doesn't work
def create_empty_instance(model_class: type[BaseModel]) -> BaseModel:
# Create an empty instance of the Pydantic model, with default values
def fill(model_cls: type[BaseModel]) -> dict:
result = {}
for field_name, field_info in model_cls.model_fields.items():
field_type = field_info.annotation
if field_type == str:
result[field_name] = ""
elif field_type == int:
result[field_name] = 0
elif field_type == bool:
result[field_name] = False
else:
result[field_name] = None
return result
return model_class(**fill(model_class))

# doesn't work
def populate_model_from_another(source_model: BaseModel, target_model: BaseModel) -> BaseModel:
# Copy matching fields from source model to target model
def add_matching_field_values(source: any, target: any):
for field in source.model_fields:
if field in target.model_fields:
src_val = getattr(source, field)
tgt_val = getattr(target, field)

# Recursive step for nested models
if isinstance(src_val, BaseModel) and isinstance(tgt_val, BaseModel):
add_matching_field_values(src_val, tgt_val)
elif isinstance(src_val, list) and isinstance(tgt_val, list):
for src_item, tgt_item in zip(src_val, tgt_val):
add_matching_field_values(src_item, tgt_item)
else:
setattr(target, field, src_val)

add_matching_field_values(source_model, target_model)

return target_model


# ___________________________________________________________________________________
# Load Model A (source) schema and instance
with open('model_a_schema.json', 'r') as f:
model_a_schema = json.load(f)

with open('model_a_data.json', 'r') as f:
model_a_data = json.load(f)

model_a_class = get_pydantic_model_from_jsonschema(model_a_schema)
model_a_instance = model_a_class(**model_a_data)

# Load Generic Model X schema
with open('generic_model_x_schema.json', 'r') as f:
generic_model_x_schema = json.load(f)

generic_model_x_class = get_pydantic_model_from_jsonschema(generic_model_x_schema)

# Does't work: Create empty instance/template
generic_model_x_instance = create_empty_instance(generic_model_x_class)

# Does't work: Populate the Generic Model X instance with matching data from Model A
populated_generic_model_x = populate_model_from_another(model_a_instance, generic_model_x_instance)

print(populated_generic_model_x)

Continue reading...