tortoise-orm
tortoise-orm copied to clipboard
tortoise
pydantic_model_creator does not support Forward references using ReverseRelation in different files
Describe the bug
If I use Forward references in the same file as I use ReverseRelation this does not generate problems.
import json
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from tortoise import Model
from tortoise import fields
class Foo(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
bar = fields.ReverseRelation["Bar"]
class Bar(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
foo: fields.ForeignKeyRelation[Foo] = fields.ForeignKeyField(
"models.Foo", on_delete=fields.CASCADE
)
Tortoise.init_models(models_paths=["__main__"], app_label="models")
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))
Output:
{
"$defs": {
"Foo_tpibnp_leaf": {
"additionalProperties": false,
"properties": {
"id": {
"maximum": 2147483647,
"minimum": -2147483648,
"title": "Id",
"type": "integer"
},
"name": {
"maxLength": 255,
"title": "Name",
"type": "string"
}
},
"required": [
"id",
"name"
],
"title": "Foo",
"type": "object"
}
},
"additionalProperties": false,
"properties": {
"id": {
"maximum": 2147483647,
"minimum": -2147483648,
"title": "Id",
"type": "integer"
},
"name": {
"maxLength": 255,
"title": "Name",
"type": "string"
},
"foo": {
"$ref": "#/$defs/Foo_tpibnp_leaf"
}
},
"required": [
"id",
"name",
"foo"
],
"title": "BarIn",
"type": "object"
}
But if I separate them into different files:
├── models
│ ├───__init__.py
│ ├───bar.py
│ └───foo.py
└── main.py
foo.py
from tortoise import Model
from tortoise import fields
from models.bar import Bar
class Foo(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
bar = fields.ReverseRelation[Bar]
bar.py
from __future__ import annotations
from typing import TYPE_CHECKING
from tortoise import Model
from tortoise import fields
if TYPE_CHECKING:
from models.foo import Foo
class Bar(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
foo: fields.ForeignKeyRelation[Foo] = fields.ForeignKeyField(
"models.Foo", on_delete=fields.CASCADE
)
main.py
from __future__ import annotations
import json
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from models import foo, bar
from models.bar import Bar
Tortoise.init_models(models_paths=[foo, bar], app_label="models")
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))
Getting the following:
Traceback (most recent call last):
File "c:\Users\HP\Documents\apps\test_tortoirse_orm\main.py", line 10, in <module>
BarIn = pydantic_model_creator(Bar, name="BarIn")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\HP\Documents\apps\test_tortoirse_orm\.venv\Lib\site-packages\tortoise\contrib\pydantic\creator.py", line 625, in pydantic_model_creator
pmc = PydanticModelCreator(
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\HP\Documents\apps\test_tortoirse_orm\.venv\Lib\site-packages\tortoise\contrib\pydantic\creator.py", line 300, in __init__
self._annotations = get_annotations(cls)
^^^^^^^^^^^^^^^^^^^^
File "C:\Users\HP\Documents\apps\test_tortoirse_orm\.venv\Lib\site-packages\tortoise\contrib\pydantic\utils.py", line 15, in get_annotations
return typing.get_type_hints(method or cls)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\HP\AppData\Local\Programs\Python\Python312\Lib\typing.py", line 2273, in get_type_hints
value = _eval_type(value, base_globals, base_locals, base.__type_params__)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\HP\AppData\Local\Programs\Python\Python312\Lib\typing.py", line 415, in _eval_type
return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\HP\AppData\Local\Programs\Python\Python312\Lib\typing.py", line 947, in _evaluate
eval(self.__forward_code__, globalns, localns),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<string>", line 1, in <module>
NameError: name 'Foo' is not defined
Checking I found that the problem has its root in the following code:
def get_annotations(cls: "Type[Model]", method: Optional[Callable] = None) -> Dict[str, Any]:
"""
Get all annotations including base classes
:param cls: The model class we need annotations from
:param method: If specified, we try to get the annotations for the callable
:return: The list of annotations
"""
return typing.get_type_hints(method or cls)
And looking at the change history I implemented the following modification:
def get_annotations(cls: "Type[Model]", method: Optional[Callable] = None) -> Dict[str, Any]:
"""
Get all annotations including base classes
:param cls: The model class we need annotations from
:param method: If specified, we try to get the annotations for the callable
:return: The list of annotations
"""
localns = (
tortoise.Tortoise.apps.get(cls._meta.app, None)
if cls._meta.app
else None
)
return typing.get_type_hints(method or cls, localns=localns)
Solving my problem and getting:
{
"$defs": {
"Foo_ijnnvp_leaf": {
"additionalProperties": false,
"properties": {
"id": {
"maximum": 2147483647,
"minimum": -2147483648,
"title": "Id",
"type": "integer"
},
"name": {
"maxLength": 255,
"title": "Name",
"type": "string"
}
},
"required": [
"id",
"name"
],
"title": "Foo",
"type": "object"
}
},
"additionalProperties": false,
"properties": {
"id": {
"maximum": 2147483647,
"minimum": -2147483648,
"title": "Id",
"type": "integer"
},
"name": {
"maxLength": 255,
"title": "Name",
"type": "string"
},
"foo": {
"$ref": "#/$defs/Foo_ijnnvp_leaf"
}
},
"required": [
"id",
"name",
"foo"
],
"title": "BarIn",
"type": "object"
}
My solution is different from what was generated by issue #1552, It indicates there that the error is due to setting the globalns.
@eyllanesc-JE is it the complete traceback?
Traceback (most recent call last):
....
File "<string>", line 1, in <module>
NameError: name 'Foo' is not defined
just change
if TYPE_CHECKING:
from foo import Foo
to
from foo import Foo
during runtime, TYPE_CHECKING is always False. So it won't import Foo.
Full example, working with python3.12:
first option with ForeignKeyRelation
main.py
import json
from typing import Callable, Coroutine
from tortoise import run_async, Tortoise
from tortoise.contrib.pydantic import pydantic_model_creator
from bar import Bar
def run(func: Callable[..., Coroutine]) -> None:
run_async(func())
async def do_stuff():
await Tortoise.init(
db_url='sqlite:///:memory:',
modules={'models': ['foo', 'bar']}
)
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=2))
if __name__ == '__main__':
run(do_stuff)
foo.py
from tortoise import fields
from tortoise import Model
class Foo(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
bar.py
from tortoise import fields
from tortoise import Model
from foo import Foo
class Bar(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
foo: fields.ForeignKeyRelation[Foo] = fields.ForeignKeyField(
"models.Foo", on_delete=fields.CASCADE
)
second option without ForeignKeyRelation
bar.py
from tortoise import fields
from tortoise import Model
class Bar(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
foo = fields.ForeignKeyField(
"models.Foo", on_delete=fields.CASCADE
)
@markus-96 Thanks for the comments. I realized that I have not focused my problem correctly. It is not so much the ForeignKeyField but the ForwardRef that is not handled correctly by pydantic_model_creator.
- I use
TYPE_CHECKINGfor my projects to avoid circular imports. - I am using mypy so I need the typehints so that is why I cannot use the second option.
I will improve my post to focus only on ForwardRef.
@eyllanesc-JE if you move from models.foo import Foo WITHOUT the if TYPE_CHECKING: after the Bar definition:
class Bar(Model):
foo: fields.ForeignKeyRelation["Foo"] = fields.ForeignKeyField(
"models.Foo", on_delete=fields.CASCADE
)
from models.foo import Foo
Does it fix the issue?
the following is working, but a little bit ugly, but it should be more compliant to PEP8 (imports to the top of the file):
from typing import TYPE_CHECKING
import tortoise
from tortoise import fields
from tortoise import Model
from typing_extensions import TypeVar
if TYPE_CHECKING:
from models.foo import Foo
else:
Foo = TypeVar('Foo', bound=Model)
class Bar(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
foo: fields.ForeignKeyRelation["Foo"] = fields.ForeignKeyField(
"models.Foo", on_delete=fields.CASCADE
)
The problem I have with the proposed solution of @eyllanesc-JE is that imports like import bar and then using them like ForeignKeyRelation["bar.Bar"] will not work. So it would need to be clearly documented how to import reliant models.
Also, if you have a structure with multiple apps, you will not be able to reference any models from the other app. ie:
models
|- bar.py
|- foo.py
models2
|- foo2.py
main.py
(bar.py and foo.py are the same as in the comments above...)
foo2.py
[...]
class Foo2(Model):
id = fields.IntField(primary_key=True)
name_s = fields.CharField(max_length=255)
bar: fields.ForeignKeyRelation["Bar"] = fields.ForeignKeyField(
"models.Bar", on_delete=fields.CASCADE, related_name='foos2'
)
main.py
import json
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from models.bar import Bar
app_modules = {'models': ['models.foo', 'models.bar'], 'models2': ['models2.foo2']}
for name, modules in app_modules.items():
Tortoise.init_models(modules, name, _init_relations=False)
for name, modules in app_modules.items():
Tortoise.init_models(modules, name)
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))
So, I think the ugly way of "importing" stuff is the most reliable one.
Full example
models/bar.py
from typing import TYPE_CHECKING
import tortoise
from tortoise import fields
from tortoise import Model
if TYPE_CHECKING:
from models.foo import Foo
from models2.foo2 import Foo2
else:
Foo = TypeVar('Foo', bound=Model)
Foo2 = TypeVar('Foo2', bound=Model)
class Bar(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
foo: fields.ForeignKeyRelation["Foo"] = fields.ForeignKeyField(
"models.Foo", on_delete=fields.CASCADE
)
foo2: fields.ForeignKeyRelation["Foo2"] = fields.ForeignKeyField(
"models2.Foo2", on_delete=fields.CASCADE
)
models/foo.py
from typing import TYPE_CHECKING
import tortoise
from tortoise import fields
from tortoise import Model
if TYPE_CHECKING:
from models.bar import Bar
else:
Bar = TypeVar('Bar', bound=Model)
class Foo(Model):
id = fields.IntField(primary_key=True)
name = fields.CharField(max_length=255)
bar: fields.ForeignKeyRelation["Bar"] = fields.ForeignKeyField(
"models.Bar", on_delete=fields.CASCADE
)
models2/foo2.py
from typing import TYPE_CHECKING
import tortoise
from tortoise import fields
from tortoise import Model
if TYPE_CHECKING:
from models.bar import Bar
else:
Bar = TypeVar('Bar', bound=Model)
class Foo2(Model):
id = fields.IntField(primary_key=True)
name_s = fields.CharField(max_length=255)
bar: fields.ForeignKeyRelation["Bar"] = fields.ForeignKeyField(
"models.Bar", on_delete=fields.CASCADE, related_name='foos2'
)
main.py
import json
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import Tortoise
from models.bar import Bar
app_modules = {'models': ['models.foo', 'models.bar'], 'models2': ['models2.foo2']}
for name, modules in app_modules.items():
Tortoise.init_models(modules, name, _init_relations=False)
for name, modules in app_modules.items():
Tortoise.init_models(modules, name)
BarIn = pydantic_model_creator(Bar, name="BarIn")
print(json.dumps(BarIn.model_json_schema(), indent=4))
(Edit: simpler main.py) (Edit2: much simpler import-section)