pycuda
pycuda copied to clipboard
inducer
Segfault on Windows with CUDA 9
Testing pycuda-2017.1.1 from git master branch (a7de45b) on Python 3.6.3, 64-bit, Windows 10, with CUDA 9.0.176 and Visual Studio 2015, test_dot_allocator crashes:
platform win32 -- Python 3.6.3, pytest-3.3.0, py-1.5.2, pluggy-0.6.0 -- x:\python36\python.exe
cachedir: ..\.cache
rootdir: D:\Build\PyCUDA\pycuda-git, inifile:
plugins: pep8-1.0.6, cov-2.5.1, palladium-1.1.0.1, celery-4.1.0
collected 118 items
test_cumath.py::TestMath::test_fmod <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 0%]
test_cumath.py::TestMath::test_ldexp <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 1%]
test_cumath.py::TestMath::test_modf <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 2%]
test_cumath.py::TestMath::test_frexp <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 3%]
test_cumath.py::TestMath::test_unary_func_kwargs <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 4%]
test_cumath.py::test_ceil <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 5%]
test_cumath.py::test_floor <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 5%]
test_cumath.py::test_fabs <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 6%]
test_cumath.py::test_exp <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 7%]
test_cumath.py::test_exp_c <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 8%]
test_cumath.py::test_log <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 9%]
test_cumath.py::test_log10 <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 10%]
test_cumath.py::test_sqrt <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 11%]
test_cumath.py::test_sin <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 11%]
test_cumath.py::test_sin_c <- x:\python36\lib\site-packages\pycuda\tools.py FAILED [ 12%]
test_cumath.py::test_cos <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 13%]
test_cumath.py::test_cos_c <- x:\python36\lib\site-packages\pycuda\tools.py FAILED [ 14%]
test_cumath.py::test_asin <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 15%]
test_cumath.py::test_acos <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 16%]
test_cumath.py::test_tan <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 16%]
test_cumath.py::test_tan_c <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 17%]
test_cumath.py::test_atan <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 18%]
test_cumath.py::test_sinh <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 19%]
test_cumath.py::test_sinh_c <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 20%]
test_cumath.py::test_cosh <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 21%]
test_cumath.py::test_cosh_c <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 22%]
test_cumath.py::test_tanh <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 22%]
test_cumath.py::test_tanh_c <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 23%]
test_driver.py::TestDriver::test_memory <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 24%]
test_driver.py::TestDriver::test_simple_kernel <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 25%]
test_driver.py::TestDriver::test_simple_kernel_2 <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 26%]
test_driver.py::TestDriver::test_vector_types <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 27%]
test_driver.py::TestDriver::test_streamed_kernel <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 27%]
test_driver.py::TestDriver::test_gpuarray <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 28%]
test_driver.py::TestDriver::test_2d_texture <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 29%]
test_driver.py::TestDriver::test_multiple_2d_textures <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 30%]
test_driver.py::TestDriver::test_multichannel_2d_texture <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 31%]
test_driver.py::TestDriver::test_multichannel_linear_texture <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 32%]
test_driver.py::TestDriver::test_2d_fp_textures <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 33%]
test_driver.py::TestDriver::test_2d_fp_texturesLayered <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 33%]
test_driver.py::TestDriver::test_3d_fp_textures <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 34%]
test_driver.py::TestDriver::test_3d_fp_surfaces <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 35%]
test_driver.py::TestDriver::test_2d_fp_surfaces <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 36%]
test_driver.py::TestDriver::test_large_smem <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 37%]
test_driver.py::TestDriver::test_bitlog <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 38%]
test_driver.py::TestDriver::test_mempool_2 <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 38%]
test_driver.py::TestDriver::test_mempool <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 39%]
test_driver.py::TestDriver::test_multi_context <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 40%]
test_driver.py::TestDriver::test_3d_texture <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 41%]
test_driver.py::TestDriver::test_prepared_invocation <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 42%]
test_driver.py::TestDriver::test_prepared_with_vector <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 43%]
test_driver.py::TestDriver::test_fp_textures <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 44%]
test_driver.py::TestDriver::test_constant_memory <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 44%]
test_driver.py::TestDriver::test_register_host_memory <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 45%]
test_driver.py::TestDriver::test_recursive_launch <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 46%]
test_driver.py::TestDriver::test_jit_link_module <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 47%]
test_driver.py::test_import_pyopencl_before_pycuda PASSED [ 48%]
test_gpuarray.py::TestGPUArray::test_pow_array <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 49%]
test_gpuarray.py::TestGPUArray::test_pow_number <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 50%]
test_gpuarray.py::TestGPUArray::test_numpy_integer_shape <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 50%]
test_gpuarray.py::TestGPUArray::test_ndarray_shape <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 51%]
test_gpuarray.py::TestGPUArray::test_abs <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 52%]
test_gpuarray.py::TestGPUArray::test_len <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 53%]
test_gpuarray.py::TestGPUArray::test_multiply <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 54%]
test_gpuarray.py::TestGPUArray::test_rmul_yields_right_type <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 55%]
test_gpuarray.py::TestGPUArray::test_multiply_array <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 55%]
test_gpuarray.py::TestGPUArray::test_addition_array <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 56%]
test_gpuarray.py::TestGPUArray::test_iaddition_array <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 57%]
test_gpuarray.py::TestGPUArray::test_addition_scalar <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 58%]
test_gpuarray.py::TestGPUArray::test_iaddition_scalar <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 59%]
test_gpuarray.py::TestGPUArray::test_substract_array <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 60%]
test_gpuarray.py::TestGPUArray::test_substract_scalar <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 61%]
test_gpuarray.py::TestGPUArray::test_divide_scalar <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 61%]
test_gpuarray.py::TestGPUArray::test_divide_array <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 62%]
test_gpuarray.py::TestGPUArray::test_random <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 63%]
test_gpuarray.py::TestGPUArray::test_curand_wrappers <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 64%]
test_gpuarray.py::TestGPUArray::test_array_gt <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 65%]
test_gpuarray.py::TestGPUArray::test_array_lt <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 66%]
test_gpuarray.py::TestGPUArray::test_array_le <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 66%]
test_gpuarray.py::TestGPUArray::test_array_ge <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 67%]
test_gpuarray.py::TestGPUArray::test_array_eq <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 68%]
test_gpuarray.py::TestGPUArray::test_array_ne <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 69%]
test_gpuarray.py::TestGPUArray::test_nan_arithmetic <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 70%]
test_gpuarray.py::TestGPUArray::test_elwise_kernel <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 71%]
test_gpuarray.py::TestGPUArray::test_ranged_elwise_kernel <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 72%]
test_gpuarray.py::TestGPUArray::test_take <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 72%]
test_gpuarray.py::TestGPUArray::test_arange <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 73%]
test_gpuarray.py::TestGPUArray::test_reverse <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 74%]
test_gpuarray.py::TestGPUArray::test_sum <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 75%]
test_gpuarray.py::TestGPUArray::test_minmax <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 76%]
test_gpuarray.py::TestGPUArray::test_subset_minmax <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 77%]
test_gpuarray.py::TestGPUArray::test_dot <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 77%]
test_gpuarray.py::TestGPUArray::test_slice <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 78%]
test_gpuarray.py::TestGPUArray::test_2d_slice_c <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 79%]
test_gpuarray.py::TestGPUArray::test_2d_slice_f <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 80%]
test_gpuarray.py::TestGPUArray::test_if_positive <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 81%]
test_gpuarray.py::TestGPUArray::test_take_put <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 82%]
test_gpuarray.py::TestGPUArray::test_astype <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 83%]
test_gpuarray.py::TestGPUArray::test_complex_bits <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 83%]
test_gpuarray.py::TestGPUArray::test_pass_slice_to_kernel <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 84%]
test_gpuarray.py::TestGPUArray::test_scan <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 85%]
test_gpuarray.py::TestGPUArray::test_stride_preservation <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 86%]
test_gpuarray.py::TestGPUArray::test_vector_fill <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 87%]
test_gpuarray.py::TestGPUArray::test_create_complex_zeros <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 88%]
test_gpuarray.py::TestGPUArray::test_reshape <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 88%]
test_gpuarray.py::TestGPUArray::test_view <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 89%]
test_gpuarray.py::TestGPUArray::test_squeeze <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 90%]
test_gpuarray.py::TestGPUArray::test_struct_reduce <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 91%]
test_gpuarray.py::TestGPUArray::test_sum_allocator <- x:\python36\lib\site-packages\pycuda\tools.py PASSED [ 92%]
test_gpuarray.py::TestGPUArray::test_dot_allocator <- x:\python36\lib\site-packages\pycuda\tools.py <Windows fatal exception: code 0xe06d7363>
The crash is in cuda.hpp, line 887 during garbage collection (it seems):
ucrtbase.dll!abort() Unknown Non-user code. Symbols loaded.
ucrtbase.dll!terminate() Unknown Non-user code. Symbols loaded.
vcruntime140.dll!FindHandler(EHExceptionRecord * pExcept, unsigned __int64 * pRN, _CONTEXT * pContext, _xDISPATCHER_CONTEXT * pDC, const _s_FuncInfo * pFuncInfo, unsigned char recursive, int) Line 762 C++ Non-user code. Symbols loaded.
vcruntime140.dll!__InternalCxxFrameHandler(EHExceptionRecord * pExcept, unsigned __int64 * pRN, _CONTEXT * pContext, _xDISPATCHER_CONTEXT * pDC, const _s_FuncInfo * pFuncInfo, int recursive, unsigned __int64 *) Line 449 C++ Non-user code. Symbols loaded.
vcruntime140.dll!__CxxFrameHandler3(EHExceptionRecord * pExcept, unsigned __int64 RN, _CONTEXT * pContext, _xDISPATCHER_CONTEXT * pDC) Line 215 C++ Non-user code. Symbols loaded.
ntdll.dll!00007ffaefa74c3d() Unknown Non-user code. Cannot find or open the PDB file.
ntdll.dll!00007ffaef9ed1b8() Unknown Non-user code. Cannot find or open the PDB file.
ntdll.dll!00007ffaef9eb623() Unknown Non-user code. Cannot find or open the PDB file.
KernelBase.dll!00007ffaebe93fb8() Unknown Non-user code. Cannot find or open the PDB file.
[External Code] Annotated Frame
> _driver.cp36-win_amd64.pyd!pycuda::explicit_context_dependent::acquire_context() Line 887 C++ Symbols loaded.
_driver.cp36-win_amd64.pyd!`anonymous namespace'::context_dependent_memory_pool<`anonymous namespace'::device_allocator>::start_holding_blocks() Line 108 C++ Symbols loaded.
_driver.cp36-win_amd64.pyd!pycuda::memory_pool<`anonymous namespace'::device_allocator>::inc_held_blocks() Line 148 C++ Symbols loaded.
_driver.cp36-win_amd64.pyd!pycuda::memory_pool<`anonymous namespace'::device_allocator>::free(unsigned __int64 p, unsigned __int64 size) Line 232 C++ Symbols loaded.
_driver.cp36-win_amd64.pyd!pycuda::pooled_allocation<`anonymous namespace'::context_dependent_memory_pool<`anonymous namespace'::device_allocator> >::free() Line 351 C++ Symbols loaded.
_driver.cp36-win_amd64.pyd!pycuda::pooled_allocation<`anonymous namespace'::context_dependent_memory_pool<`anonymous namespace'::device_allocator> >::~pooled_allocation<`anonymous namespace'::context_dependent_memory_pool<`anonymous namespace'::device_allocator> >() Line 343 C++ Symbols loaded.
[External Code] Annotated Frame
python36.dll!subtype_dealloc(_object * self) Line 1123 C Symbols loaded.
python36.dll!dict_dealloc(PyDictObject * mp) Line 2016 C Symbols loaded.
python36.dll!subtype_dealloc(_object * self) Line 1208 C Symbols loaded.
python36.dll!frame_tp_clear(_frame * f) Line 508 C Symbols loaded.
python36.dll!delete_garbage(_gc_head * collectable, _gc_head * old) Line 868 C Symbols loaded.
python36.dll!collect(int generation, __int64 * n_collected, __int64 * n_uncollectable, int nofail) Line 1024 C Symbols loaded.
python36.dll!collect_with_callback(int generation) Line 1129 C Symbols loaded.
python36.dll!gc_collect(_object * self, _object * args, _object * kws) Line 1223 C Symbols loaded.
python36.dll!_PyCFunction_FastCallDict(_object * func_obj, _object * * args, __int64 nargs, _object * kwargs) Line 232 C Symbols loaded.
[Inline Frame] python36.dll!_PyCFunction_FastCallKeywords(_object *) Line 294 C Symbols loaded.
python36.dll!call_function(_object * * * pp_stack, __int64 oparg, _object * kwnames) Line 4824 C Symbols loaded.
<snip>
We're seeing something pretty similar on Linux: https://gitlab.tiker.net/inducer/pycuda/-/jobs/38205
I don't have a lot of time to debug this right now (tenure track...), but I'd appreciate patches. I'll disable that test for now with a link to this issue.
Let me know if the skipping helped for you. It didn't for me: https://gitlab.tiker.net/inducer/pycuda/pipelines/9559
Let me know if the skipping helped for you.
Yes, the tests pass:
platform win32 -- Python 3.6.3, pytest-3.3.0, py-1.5.2, pluggy-0.6.0
rootdir: D:\Build\PyCUDA\pycuda-git, inifile:
plugins: pep8-1.0.6, cov-2.5.1, palladium-1.1.0.1, celery-4.1.0
collected 61 items
test_gpuarray.py ....................................................s.. [ 90%]
...... [100%]
I see similar problem on Linux in test
Let me know if the skipping helped for you. It didn't for me: https://gitlab.tiker.net/inducer/pycuda/pipelines/9559
I'm not sure which test fails in that pipline, but I see similar crash in test_sum_allocator:
(gdb) bt
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007ffff734afb1 in __GI_abort () at abort.c:79
#2 0x00007fffe94c9095 in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/gcc/x86_64-pc-linux-gnu/7.3.0/libstdc++.so.6
#3 0x00007fffe94c6c66 in ?? () from /usr/lib/gcc/x86_64-pc-linux-gnu/7.3.0/libstdc++.so.6
#4 0x00007fffe94c5b19 in ?? () from /usr/lib/gcc/x86_64-pc-linux-gnu/7.3.0/libstdc++.so.6
#5 0x00007fffe94c6538 in __gxx_personality_v0 () from /usr/lib/gcc/x86_64-pc-linux-gnu/7.3.0/libstdc++.so.6
#6 0x00007ffff030cbf3 in ?? () from /usr/lib/gcc/x86_64-pc-linux-gnu/7.3.0/libgcc_s.so.1
#7 0x00007ffff030d112 in _Unwind_RaiseException () from /usr/lib/gcc/x86_64-pc-linux-gnu/7.3.0/libgcc_s.so.1
#8 0x00007fffe94c6ee6 in __cxa_throw () from /usr/lib/gcc/x86_64-pc-linux-gnu/7.3.0/libstdc++.so.6
#9 0x00007fffee9c9155 in pycuda::explicit_context_dependent::acquire_context() () from /usr/lib64/python3.6/site-packages/pycuda/_driver.cpython-36m-x86_64-linux-gnu.so
#10 0x00007fffee9c4d30 in ?? () from /usr/lib64/python3.6/site-packages/pycuda/_driver.cpython-36m-x86_64-linux-gnu.so
#11 0x00007fffee9c5239 in ?? () from /usr/lib64/python3.6/site-packages/pycuda/_driver.cpython-36m-x86_64-linux-gnu.so
#12 0x00007fffee9d9c4c in ?? () from /usr/lib64/python3.6/site-packages/pycuda/_driver.cpython-36m-x86_64-linux-gnu.so
I can believe that it would behave the same way. I've disabled it, too, pending a fix.