tavern
tavern copied to clipboard
taverntesting
Clean validation of Tavern files is required to allow using it in a productive way
I'm stuck for hours with a simple test with one config file and one external stage file.
Each time there is a single mistake, I'm left with a stack trace of 500 line without clear indication about what is wrong.
to_verify = {'test_name': 'Full path, from client sync to period renewal.', 'includes': [{'test_name': 'Database delete stage', 'i...prod/command/SynchronizeClients'}, 'response': {'status_code': 502, 'headers': {'content-type': 'application/json'}}}]}
schema = {'name': 'Test schema', 'desc': 'Matches test blocks', 'schema;any_request_json': {'func': 'validate_request_json', 't... 'map', 'mapping': {'username': {'type': 'str', 'required': True}, 'password': {'type': 'str', 'required': False}}}}}}}
def verify_generic(to_verify, schema):
"""Verify a generic file against a given schema
Args:
to_verify (dict): Filename of source tests to check
schema (dict): Schema to verify against
Raises:
BadSchemaError: Schema did not match
"""
logger.debug("Verifying %s against %s", to_verify, schema)
here = os.path.dirname(os.path.abspath(__file__))
extension_module_filename = os.path.join(here, "extensions.py")
verifier = core.Core(
source_data=to_verify,
schema_data=schema,
extensions=[extension_module_filename],
)
try:
> verifier.validate()
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/tavern/schemas/files.py:106:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <pykwalify.core.Core object at 0x7f0663c92490>, raise_exception = True
def validate(self, raise_exception=True):
"""
"""
log.debug(u"starting core")
self._start_validate(self.source)
self.validation_errors = [unicode(error) for error in self.errors]
self.validation_errors_exceptions = self.errors
if self.errors is None or len(self.errors) == 0:
log.info(u"validation.valid")
else:
log.error(u"validation.invalid")
log.error(u" --- All found errors ---")
log.error(self.validation_errors)
if raise_exception:
> raise SchemaError(u"Schema validation failed:\n - {error_msg}.".format(
error_msg=u'.\n - '.join(self.validation_errors)))
E pykwalify.errors.SchemaError: <SchemaError: error code 2: Schema validation failed:
E - Cannot find required key 'name'. Path: '/includes/0'.
E - Cannot find required key 'description'. Path: '/includes/0'.
E - Key 'test_name' was not defined. Path: '/includes/0'.
E - Key 'includes' was not defined. Path: '/includes/0'.: Path: '/'>
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/pykwalify/core.py:194: SchemaError
The above exception was the direct cause of the following exception:
cls = <class '_pytest.runner.CallInfo'>
func = <function call_runtest_hook.<locals>.<lambda> at 0x7f0662a263a0>
when = 'call'
reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)
@classmethod
def from_call(
cls,
func: "Callable[[], TResult]",
when: "Literal['collect', 'setup', 'call', 'teardown']",
reraise: Optional[
Union[Type[BaseException], Tuple[Type[BaseException], ...]]
] = None,
) -> "CallInfo[TResult]":
excinfo = None
start = timing.time()
precise_start = timing.perf_counter()
try:
> result: Optional[TResult] = func()
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/_pytest/runner.py:311:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> lambda: ihook(item=item, **kwds), when=when, reraise=reraise
)
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/_pytest/runner.py:255:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <_HookCaller 'pytest_runtest_call'>, args = ()
kwargs = {'item': <YamlItem Full path, from client sync to period renewal.>}
notincall = set()
def __call__(self, *args, **kwargs):
if args:
raise TypeError("hook calling supports only keyword arguments")
assert not self.is_historic()
if self.spec and self.spec.argnames:
notincall = (
set(self.spec.argnames) - set(["__multicall__"]) - set(kwargs.keys())
)
if notincall:
warnings.warn(
"Argument(s) {} which are declared in the hookspec "
"can not be found in this hook call".format(tuple(notincall)),
stacklevel=2,
)
> return self._hookexec(self, self.get_hookimpls(), kwargs)
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/pluggy/hooks.py:286:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <_pytest.config.PytestPluginManager object at 0x7f066319fac0>
hook = <_HookCaller 'pytest_runtest_call'>
methods = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/home/detailoc/dev/dependencies/python/aws38/li...xception' from '/home/detailoc/dev/dependencies/python/aws38/lib/python3.8/site-packages/_pytest/threadexception.py'>>]
kwargs = {'item': <YamlItem Full path, from client sync to period renewal.>}
def _hookexec(self, hook, methods, kwargs):
# called from all hookcaller instances.
# enable_tracing will set its own wrapping function at self._inner_hookexec
> return self._inner_hookexec(hook, methods, kwargs)
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/pluggy/manager.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
hook = <_HookCaller 'pytest_runtest_call'>
methods = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/home/detailoc/dev/dependencies/python/aws38/li...xception' from '/home/detailoc/dev/dependencies/python/aws38/lib/python3.8/site-packages/_pytest/threadexception.py'>>]
kwargs = {'item': <YamlItem Full path, from client sync to period renewal.>}
> self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(
methods,
kwargs,
firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
)
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/pluggy/manager.py:84:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/home/detailoc/dev/dependencies/python/aws38/li...xception' from '/home/detailoc/dev/dependencies/python/aws38/lib/python3.8/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Full path, from client sync to period renewal.>}
firstresult = False
def _multicall(hook_impls, caller_kwargs, firstresult=False):
"""Execute a call into multiple python functions/methods and return the
result(s).
``caller_kwargs`` comes from _HookCaller.__call__().
"""
__tracebackhide__ = True
results = []
excinfo = None
try: # run impl and wrapper setup functions in a loop
teardowns = []
try:
for hook_impl in reversed(hook_impls):
try:
args = [caller_kwargs[argname] for argname in hook_impl.argnames]
except KeyError:
for argname in hook_impl.argnames:
if argname not in caller_kwargs:
raise HookCallError(
"hook call must provide argument %r" % (argname,)
)
if hook_impl.hookwrapper:
try:
gen = hook_impl.function(*args)
next(gen) # first yield
teardowns.append(gen)
except StopIteration:
_raise_wrapfail(gen, "did not yield")
else:
res = hook_impl.function(*args)
if res is not None:
results.append(res)
if firstresult: # halt further impl calls
break
except BaseException:
excinfo = sys.exc_info()
finally:
if firstresult: # first result hooks return a single value
outcome = _Result(results[0] if results else None, excinfo)
else:
outcome = _Result(results, excinfo)
# run all wrapper post-yield blocks
for gen in reversed(teardowns):
try:
gen.send(outcome)
_raise_wrapfail(gen, "has second yield")
except StopIteration:
pass
> return outcome.get_result()
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/pluggy/callers.py:208:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <pluggy.callers._Result object at 0x7f0663c92640>
def get_result(self):
"""Get the result(s) for this hook call.
If the hook was marked as a ``firstresult`` only a single value
will be returned otherwise a list of results.
"""
__tracebackhide__ = True
if self._excinfo is None:
return self._result
else:
ex = self._excinfo
if _py3:
> raise ex[1].with_traceback(ex[2])
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/pluggy/callers.py:80:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/home/detailoc/dev/dependencies/python/aws38/li...xception' from '/home/detailoc/dev/dependencies/python/aws38/lib/python3.8/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Full path, from client sync to period renewal.>}
firstresult = False
def _multicall(hook_impls, caller_kwargs, firstresult=False):
"""Execute a call into multiple python functions/methods and return the
result(s).
``caller_kwargs`` comes from _HookCaller.__call__().
"""
__tracebackhide__ = True
results = []
excinfo = None
try: # run impl and wrapper setup functions in a loop
teardowns = []
try:
for hook_impl in reversed(hook_impls):
try:
args = [caller_kwargs[argname] for argname in hook_impl.argnames]
except KeyError:
for argname in hook_impl.argnames:
if argname not in caller_kwargs:
raise HookCallError(
"hook call must provide argument %r" % (argname,)
)
if hook_impl.hookwrapper:
try:
gen = hook_impl.function(*args)
next(gen) # first yield
teardowns.append(gen)
except StopIteration:
_raise_wrapfail(gen, "did not yield")
else:
> res = hook_impl.function(*args)
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/pluggy/callers.py:187:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
item = <YamlItem Full path, from client sync to period renewal.>
def pytest_runtest_call(item: Item) -> None:
_update_current_test_var(item, "call")
try:
del sys.last_type
del sys.last_value
del sys.last_traceback
except AttributeError:
pass
try:
item.runtest()
except Exception as e:
# Store trace info to allow postmortem debugging
sys.last_type = type(e)
sys.last_value = e
assert e.__traceback__ is not None
# Skip *this* frame
sys.last_traceback = e.__traceback__.tb_next
> raise e
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/_pytest/runner.py:170:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
item = <YamlItem Full path, from client sync to period renewal.>
def pytest_runtest_call(item: Item) -> None:
_update_current_test_var(item, "call")
try:
del sys.last_type
del sys.last_value
del sys.last_traceback
except AttributeError:
pass
try:
> item.runtest()
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/_pytest/runner.py:162:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <YamlItem Full path, from client sync to period renewal.>
def runtest(self):
# Do a deep copy because this sometimes still retains things from previous tests(?)
self.global_cfg = copy.deepcopy(load_global_cfg(self.config))
self.global_cfg.setdefault("variables", {})
load_plugins(self.global_cfg)
self.global_cfg["tavern_internal"] = {"pytest_hook_caller": self.config.hook}
# INTERNAL
# NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail
# instead. This doesn't differentiate between an error in verification
# and an error when running the test though.
xfail = self.spec.get("_xfail", False)
try:
fixture_values = self._load_fixture_values()
self.global_cfg["variables"].update(fixture_values)
call_hook(
self.global_cfg,
"pytest_tavern_beta_before_every_test_run",
test_dict=self.spec,
variables=self.global_cfg["variables"],
)
> verify_tests(self.spec)
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/tavern/testutils/pytesthook/item.py:184:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
test_spec = {'test_name': 'Full path, from client sync to period renewal.', 'includes': [{'test_name': 'Database delete stage', 'i...prod/command/SynchronizeClients'}, 'response': {'status_code': 502, 'headers': {'content-type': 'application/json'}}}]}
with_plugins = True
def verify_tests(test_spec, with_plugins=True):
"""Verify that a specific test block is correct
Todo:
Load schema file once. Requires some caching of the file
Args:
test_spec (dict): Test in dictionary form
Raises:
BadSchemaError: Schema did not match
"""
here = os.path.dirname(os.path.abspath(__file__))
schema_filename = os.path.join(here, "tests.schema.yaml")
schema = load_schema_file(schema_filename, with_plugins)
> verify_generic(test_spec, schema)
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/tavern/schemas/files.py:152:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
to_verify = {'test_name': 'Full path, from client sync to period renewal.', 'includes': [{'test_name': 'Database delete stage', 'i...prod/command/SynchronizeClients'}, 'response': {'status_code': 502, 'headers': {'content-type': 'application/json'}}}]}
schema = {'name': 'Test schema', 'desc': 'Matches test blocks', 'schema;any_request_json': {'func': 'validate_request_json', 't... 'map', 'mapping': {'username': {'type': 'str', 'required': True}, 'password': {'type': 'str', 'required': False}}}}}}}
def verify_generic(to_verify, schema):
"""Verify a generic file against a given schema
Args:
to_verify (dict): Filename of source tests to check
schema (dict): Schema to verify against
Raises:
BadSchemaError: Schema did not match
"""
logger.debug("Verifying %s against %s", to_verify, schema)
here = os.path.dirname(os.path.abspath(__file__))
extension_module_filename = os.path.join(here, "extensions.py")
verifier = core.Core(
source_data=to_verify,
schema_data=schema,
extensions=[extension_module_filename],
)
try:
verifier.validate()
except pykwalify.errors.PyKwalifyException as e:
logger.exception("Error validating %s", to_verify)
> raise BadSchemaError() from e
E tavern.util.exceptions.BadSchemaError
../../../../../dependencies/python/aws38/lib/python3.8/site-packages/tavern/schemas/files.py:109: BadSchemaError
========================= 1 failed, 1 passed in 0.76s ==========================
Process finished with exit code 1
This is just an example, but Tavern crashes here and there depending on the issue. That is not possible to use this in a productive way.
This is partially the reason I'm switching to using jsonschema in the next release, it should be able to provide more useful errors like
../../.tox/py38-generic/lib/python3.8/site-packages/tavern/_core/schema/jsonschema.py:134: in verify_jsonschema
raise BadSchemaError(msg) from e
E tavern._core.exceptions.BadSchemaError:
E ---
E
E Additional properties are not allowed ('blllblbb' was unexpected)
E
E - name: Send with basic auth
E blllblbb: gg
E request:
E url: "{global_host}/authtest/basic"
E method: GET
E auth:
E - "fakeuser"
E - "fakepass"
E response:
E status_code: 200
E json:
E auth_type: basic
E auth_user: fakeuser
E auth_pass: fakepass
For the time being you can also use the pytest -q flag which won't print so many lines of code in each traceback
