JuliaGPU
Revising recursive function results in `InvalidIRError`
When running this CUDA.jl code with Revise.jl
using CUDA
function recursive_sub(b::Int64)
if b > 0
recursive_sub(b-1)
end
return
end
function reproduce_main()
function recursive_kernel()
recursive_sub(5)
return
end
@cuda threads=5 recursive_kernel()
end
everything works as expected, but when you change recursive_sub with something like a print statement the kernel cannot execute anymore due to InvalidIRError after something that looks like a stackoverflow in GPUCompiler.
Stacktrace
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:72
invalidate at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:53 [inlined]
callback at /home/ajuvercr/.julia/packages/GPUCompiler/XwWPj/src/jlgen.jl:38
unknown function (ip: 0x7f7d47e39bd8)
_jl_invoke at /home/ajuvercr/julia-1.6/src/gf.c:2237 [inlined]
jl_apply_generic at /home/ajuvercr/julia-1.6/src/gf.c:2419
jl_apply at /home/ajuvercr/julia-1.6/src/julia.h:1703 [inlined]
invalidate_external at /home/ajuvercr/julia-1.6/src/gf.c:1339
jl_method_table_invalidate at /home/ajuvercr/julia-1.6/src/gf.c:1561
jl_method_table_disable at /home/ajuvercr/julia-1.6/src/gf.c:1582
delete_method at ./reflection.jl:1535 [inlined]
#79 at /home/ajuvercr/.julia/packages/Revise/VxkZO/src/packagedef.jl:276
unknown function (ip: 0x7f7d47e16118)
_jl_invoke at /home/ajuvercr/julia-1.6/src/gf.c:2237 [inlined]
jl_apply_generic at /home/ajuvercr/julia-1.6/src/gf.c:2419
with_logstate at ./logging.jl:491
with_logger at ./logging.jl:603 [inlined]
delete_missing! at /home/ajuvercr/.julia/packages/Revise/VxkZO/src/packagedef.jl:242 [inlined]
delete_missing! at /home/ajuvercr/.julia/packages/Revise/VxkZO/src/packagedef.jl:304
handle_deletions at /home/ajuvercr/.julia/packages/Revise/VxkZO/src/packagedef.jl:647
#revise#98 at /home/ajuvercr/.julia/packages/Revise/VxkZO/src/packagedef.jl:746
revise at /home/ajuvercr/.julia/packages/Revise/VxkZO/src/packagedef.jl:734
unknown function (ip: 0x7f7d47df008d)
_jl_invoke at /home/ajuvercr/julia-1.6/src/gf.c:2237 [inlined]
jl_apply_generic at /home/ajuvercr/julia-1.6/src/gf.c:2419
jl_apply at /home/ajuvercr/julia-1.6/src/julia.h:1703 [inlined]
jl_f__call_latest at /home/ajuvercr/julia-1.6/src/builtins.c:714
#invokelatest#2 at ./essentials.jl:708 [inlined]
invokelatest at ./essentials.jl:706
_jl_invoke at /home/ajuvercr/julia-1.6/src/gf.c:2237 [inlined]
jl_apply_generic at /home/ajuvercr/julia-1.6/src/gf.c:2419
jl_apply at /home/ajuvercr/julia-1.6/src/julia.h:1703 [inlined]
do_call at /home/ajuvercr/julia-1.6/src/interpreter.c:115
eval_value at /home/ajuvercr/julia-1.6/src/interpreter.c:204
eval_stmt_value at /home/ajuvercr/julia-1.6/src/interpreter.c:155 [inlined]
eval_body at /home/ajuvercr/julia-1.6/src/interpreter.c:557
jl_interpret_toplevel_thunk at /home/ajuvercr/julia-1.6/src/interpreter.c:669
top-level scope at none:1
jl_toplevel_eval_flex at /home/ajuvercr/julia-1.6/src/toplevel.c:877
jl_toplevel_eval_flex at /home/ajuvercr/julia-1.6/src/toplevel.c:825
jl_toplevel_eval_in at /home/ajuvercr/julia-1.6/src/toplevel.c:929
eval at ./boot.jl:360 [inlined]
eval_user_input at /home/ajuvercr/julia-1.6/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:139
repl_backend_loop at /home/ajuvercr/julia-1.6/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:200
start_repl_backend at /home/ajuvercr/julia-1.6/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:185
#run_repl#42 at /home/ajuvercr/julia-1.6/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:317
run_repl at /home/ajuvercr/julia-1.6/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:305
_jl_invoke at /home/ajuvercr/julia-1.6/src/gf.c:2237 [inlined]
jl_apply_generic at /home/ajuvercr/julia-1.6/src/gf.c:2419
#876 at ./client.jl:387
jfptr_YY.876_44055 at /home/ajuvercr/julia-1.6/usr/lib/julia/sys.so (unknown line)
_jl_invoke at /home/ajuvercr/julia-1.6/src/gf.c:2237 [inlined]
jl_apply_generic at /home/ajuvercr/julia-1.6/src/gf.c:2419
jl_apply at /home/ajuvercr/julia-1.6/src/julia.h:1703 [inlined]
jl_f__call_latest at /home/ajuvercr/julia-1.6/src/builtins.c:714
#invokelatest#2 at ./essentials.jl:708 [inlined]
invokelatest at ./essentials.jl:706 [inlined]
run_main_repl at ./client.jl:372
exec_options at ./client.jl:302
_start at ./client.jl:485
jfptr__start_49096 at /home/ajuvercr/julia-1.6/usr/lib/julia/sys.so (unknown line)
_jl_invoke at /home/ajuvercr/julia-1.6/src/gf.c:2237 [inlined]
jl_apply_generic at /home/ajuvercr/julia-1.6/src/gf.c:2419
jl_apply at /home/ajuvercr/julia-1.6/src/julia.h:1703 [inlined]
true_main at /home/ajuvercr/julia-1.6/src/jlapi.c:560
repl_entrypoint at /home/ajuvercr/julia-1.6/src/jlapi.c:702
main at julia-1.6 (unknown line)
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
_start at julia-1.6 (unknown line)
ERROR: InvalidIRError: compiling kernel recursive_kernel() resulted in invalid LLVM IR
Reason: unsupported dynamic function invocation (call to recursive_sub(b::Int64) in Main at /home/ajuvercr/test.jl:12)
Stacktrace:
[1] recursive_sub
@ ~/test.jl:15
[2] recursive_kernel
@ ~/test.jl:23
Stacktrace:
[1] check_ir(job::GPUCompiler.CompilerJob{GPUCompiler.PTXCompilerTarget, CUDA.CUDACompilerParams, GPUCompiler.FunctionSpec{var"#recursive_kernel#1", Tuple{}}}, args::LLVM.Module)
@ GPUCompiler ~/.julia/packages/GPUCompiler/XwWPj/src/validation.jl:123
[2] macro expansion
@ ~/.julia/packages/GPUCompiler/XwWPj/src/driver.jl:288 [inlined]
[3] macro expansion
@ ~/.julia/packages/TimerOutputs/4QAIk/src/TimerOutput.jl:206 [inlined]
[4] macro expansion
@ ~/.julia/packages/GPUCompiler/XwWPj/src/driver.jl:286 [inlined]
[5] emit_asm(job::GPUCompiler.CompilerJob, ir::LLVM.Module, kernel::LLVM.Function; strip::Bool, validate::Bool, format::LLVM.API.LLVMCodeGenFileType)
@ GPUCompiler ~/.julia/packages/GPUCompiler/XwWPj/src/utils.jl:62
[6] cufunction_compile(job::GPUCompiler.CompilerJob)
@ CUDA ~/.julia/packages/CUDA/qEV3Y/src/compiler/execution.jl:306
[7] check_cache
@ ~/.julia/packages/GPUCompiler/XwWPj/src/cache.jl:44 [inlined]
[8] cached_compilation
@ ~/test.jl:23 [inlined]
[9] cached_compilation(cache::Dict{UInt64, Any}, job::GPUCompiler.CompilerJob{GPUCompiler.PTXCompilerTarget, CUDA.CUDACompilerParams, GPUCompiler.FunctionSpec{var"#recursive_kernel#1", Tuple{}}}, compiler::typeof(CUDA.cufunction_compile), linker::typeof(CUDA.cufunction_link))
@ GPUCompiler ~/.julia/packages/GPUCompiler/XwWPj/src/cache.jl:0
[10] cufunction(f::var"#recursive_kernel#1", tt::Type{Tuple{}}; name::Nothing, kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ CUDA ~/.julia/packages/CUDA/qEV3Y/src/compiler/execution.jl:294
[11] cufunction
@ ~/.julia/packages/CUDA/qEV3Y/src/compiler/execution.jl:288 [inlined]
[12] macro expansion
@ ~/.julia/packages/CUDA/qEV3Y/src/compiler/execution.jl:102 [inlined]
[13] reproduce_main()
@ Main ~/test.jl:26
[14] top-level scope
@ REPL[2]:1
when you change
recursive_subwith something like a print statement
To be clear, that's with Revise in a running session, right?
MWE
$ julia -L test/definitions/native.jl_apply
julia> function inner(b::Int64)
if b > 0
inner(b-1)
end
return
end
inner (generic function with 1 method)
julia> outer() = inner(5)
outer (generic function with 1 method)
julia> native_code_llvm(outer, Tuple{})
; @ REPL[2]:1 within `outer'
define void @julia_outer_605() local_unnamed_addr {
top:
; ┌ @ REPL[1]:3 within `inner'
call fastcc void @julia_inner_608(i64 signext 4)
; └
ret void
}
julia> function inner(b::Int64)
if b > 0
inner(b-2)
end
return
end
error in invalidation callback: StackOverflowError()
jl_inst_arg_tuple_type at /buildworker/worker/package_linux64/build/src/jltypes.c:1407
jl_f_tuple at /buildworker/worker/package_linux64/build/src/builtins.c:743
iterate at ./array.jl:777 [inlined]
iterate at ./array.jl:777
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
invalidate at /home/tim/Julia/pkg/GPUCompiler/src/jlgen.jl:82
invalidate at /home/tim/Julia/pkg/GPUCompiler/src/jlgen.jl:83
...
invalidate at /home/tim/Julia/pkg/GPUCompiler/src/jlgen.jl:83
invalidate at /home/tim/Julia/pkg/GPUCompiler/src/jlgen.jl:64 [inlined]
callback at /home/tim/Julia/pkg/GPUCompiler/src/jlgen.jl:49
unknown function (ip: 0x7f87b9568428)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
invalidate_external at /buildworker/worker/package_linux64/build/src/gf.c:1339
jl_method_table_invalidate at /buildworker/worker/package_linux64/build/src/gf.c:1561
jl_method_table_insert at /buildworker/worker/package_linux64/build/src/gf.c:1640
jl_method_def at /buildworker/worker/package_linux64/build/src/method.c:818
eval_methoddef at /buildworker/worker/package_linux64/build/src/interpreter.c:100
eval_body at /buildworker/worker/package_linux64/build/src/interpreter.c:521
jl_interpret_toplevel_thunk at /buildworker/worker/package_linux64/build/src/interpreter.c:669
top-level scope at REPL[4]:1
jl_toplevel_eval_flex at /buildworker/worker/package_linux64/build/src/toplevel.c:877
jl_toplevel_eval_flex at /buildworker/worker/package_linux64/build/src/toplevel.c:825
jl_toplevel_eval_in at /buildworker/worker/package_linux64/build/src/toplevel.c:929
eval at ./boot.jl:360 [inlined]
eval_user_input at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:139
repl_backend_loop at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:200
start_repl_backend at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:185
#run_repl#42 at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:317
run_repl at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:305
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
#874 at ./client.jl:387
jfptr_YY.874_46600.clone_1 at /home/tim/.cache/julia/binaries/1.6/x64/lib/julia/sys.so (unknown line)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
jl_f__call_latest at /buildworker/worker/package_linux64/build/src/builtins.c:714
#invokelatest#2 at ./essentials.jl:708 [inlined]
invokelatest at ./essentials.jl:706 [inlined]
run_main_repl at ./client.jl:372
exec_options at ./client.jl:302
_start at ./client.jl:485
jfptr__start_41020.clone_1 at /home/tim/.cache/julia/binaries/1.6/x64/lib/julia/sys.so (unknown line)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
true_main at /buildworker/worker/package_linux64/build/src/jlapi.c:560
repl_entrypoint at /buildworker/worker/package_linux64/build/src/jlapi.c:702
main at julia (unknown line)
__libc_start_main at /usr/lib/libc.so.6 (unknown line)
unknown function (ip: 0x4007d8)
inner (generic function with 1 method)
julia> native_code_llvm(outer, Tuple{})
; @ REPL[2]:1 within `outer'
define void @julia_outer_2061() local_unnamed_addr {
top:
%0 = alloca {}*, align 8
; ┌ @ REPL[1]:3 within `inner'
%1 = getelementptr inbounds {}*, {}** %0, i32 0
store {}* inttoptr (i64 140221913301280 to {}*), {}** %1, align 8
%2 = call nonnull {}* @jl_invoke({}* inttoptr (i64 140221922362784 to {}*), {}** %0, i32 1, {}* inttoptr (i64 140221966288784 to {}*))
; └
ret void
}