arrow-julia icon indicating copy to clipboard operation
arrow-julia copied to clipboard

error serializing `Union{Missing,Nothing,Nanosecond}`

Open kleinschmidt opened this issue 4 years ago • 1 comments

using Dates, Arrow
io = Arrow.tobuffer((x=[missing; nothing; Nanosecond(1)], ))
ERROR: BoundsError: attempt to access DataType at index [3]
Stacktrace:
  [1] isatypeid(x::Arrow.Duration{Arrow.Flatbuf.TimeUnitModule.NANOSECOND}, #unused#::Type{Nanosecond}, #unused#::Type{Tuple{Union{Missing, Nothing}, Nanosecond}}, i::Int64) (repeats 2 times)
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/unions.jl:50
  [2] isatypeid(x::Arrow.Duration{Arrow.Flatbuf.TimeUnitModule.NANOSECOND}, #unused#::Type{Tuple{Union{Missing, Nothing}, Nanosecond}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/unions.jl:49
  [3] todense(#unused#::Type{Arrow.UnionT{Arrow.Flatbuf.UnionModeModule.Dense, nothing, Tuple{Union{Missing, Nothing}, Nanosecond}}}, x::ArrowTypes.ToArrow{Union{Missing, Nothing, Nanosecond}, Vector{Union{Missing, Nothing, Nanosecond}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/unions.jl:113
  [4] arrowvector(::ArrowTypes.UnionKind, x::Arrow.DenseUnionVector{ArrowTypes.ToArrow{Union{Missing, Nothing, Nanosecond}, Vector{Union{Missing, Nothing, Nanosecond}}}, Tuple{Union{Missing, Nothing}, Nanosecond}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :compression, :largelists, :dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/unions.jl:219
  [5] arrowvector(::Type{Arrow.UnionT{Arrow.Flatbuf.UnionModeModule.Dense, nothing, Tuple{Union{Missing, Nothing}, Nanosecond}}}, x::Arrow.DenseUnionVector{ArrowTypes.ToArrow{Union{Missing, Nothing, Nanosecond}, Vector{Union{Missing, Nothing, Nanosecond}}}, Tuple{Union{Missing, Nothing}, Nanosecond}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :compression, :largelists, :dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/arraytypes.jl:85
  [6] arrowvector(x::Arrow.DenseUnionVector{ArrowTypes.ToArrow{Union{Missing, Nothing, Nanosecond}, Vector{Union{Missing, Nothing, Nanosecond}}}, Tuple{Union{Missing, Nothing}, Nanosecond}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:denseunions, :compression, :largelists, :dictencodenested), Tuple{Bool, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/arraytypes.jl:69
  [7] arrowvector(U::Union, x::ArrowTypes.ToArrow{Union{Missing, Nothing, Nanosecond}, Vector{Union{Missing, Nothing, Nanosecond}}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; denseunions::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/unions.jl:210
  [8] arrowvector(x::Vector{Union{Missing, Nothing, Nanosecond}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists, :denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/arraytypes.jl:69
  [9] toarrowvector(x::Vector{Union{Missing, Nothing, Nanosecond}}, i::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing, kw::Base.Iterators.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists, :denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool, Bool, Bool, Int64}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/arraytypes/arraytypes.jl:36
 [10] (::Arrow.var"#111#112"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool, Bool, Int64, Vector{Arrow.DictEncoding}, Vector{Type}, Vector{Any}})(col::Vector{Union{Missing, Nothing, Nanosecond}}, i::Int64, nm::Symbol)
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/write.jl:246
 [11] eachcolumn
    @ ~/.julia/packages/Tables/E99hk/src/utils.jl:70 [inlined]
 [12] toarrowtable(cols::NamedTuple{(:x,), Tuple{Vector{Union{Missing, Nothing, Nanosecond}}}}, dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64)
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/write.jl:245
 [13] macro expansion
    @ ~/.julia/packages/Arrow/PQ5Mm/src/write.jl:140 [inlined]
 [14] macro expansion
    @ ./task.jl:382 [inlined]
 [15] write(io::IOBuffer, source::NamedTuple{(:x,), Tuple{Vector{Union{Missing, Nothing, Nanosecond}}}}, writetofile::Bool, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, alignment::Int64, maxdepth::Int64, ntasks::Float64)
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/write.jl:132
 [16] #write#100
    @ ~/.julia/packages/Arrow/PQ5Mm/src/write.jl:95 [inlined]
 [17] write
    @ ~/.julia/packages/Arrow/PQ5Mm/src/write.jl:95 [inlined]
 [18] #tobuffer#3
    @ ~/.julia/packages/Arrow/PQ5Mm/src/utils.jl:206 [inlined]
 [19] tobuffer(data::NamedTuple{(:x,), Tuple{Vector{Union{Missing, Nothing, Nanosecond}}}})
    @ Arrow ~/.julia/packages/Arrow/PQ5Mm/src/utils.jl:205
 [20] top-level scope
    @ REPL[15]:1

Seems to work fine using [missing, Nanosecond(1)], [nothing, Nanosecond(1)], [missing, nothing, 1.0], and even

struct MyStruct
    x
end

io = Arrow.tobuffer((x=[missing, nothing, MyStruct(1)], ))

kleinschmidt avatar Aug 17 '21 16:08 kleinschmidt

I ran into the same issue recently. Any update here? Thanks.

baumgold avatar Feb 15 '22 02:02 baumgold