Stop lying about eltype

Author: andreasnoack

src/abstractdataarray.jl

@@ -4,7 +4,7 @@
 An `N`-dimensional `AbstractArray` whose entries can take on values of type
 `T` or the value `NA`.
 """
-abstract type AbstractDataArray{T, N} <: AbstractArray{T, N} end
+abstract type AbstractDataArray{T, N} <: AbstractArray{Union{T, NAtype}, N} end
 
 """
     AbstractDataVector{T}
@@ -20,8 +20,6 @@ A 2-dimensional [`AbstractDataArray`](@ref) with element type `T`.
 """
 const AbstractDataMatrix{T} = AbstractDataArray{T, 2}
 
-Base.eltype{T, N}(d::AbstractDataArray{T, N}) = T
-
 # Generic iteration over AbstractDataArray's
 
 Base.start(x::AbstractDataArray) = 1

src/broadcast.jl

@@ -189,9 +189,30 @@ Base.Broadcast._containertype(::Type{T}) where T<:DataArray           = DataArra
 Base.Broadcast._containertype(::Type{T}) where T<:PooledDataArray     = PooledDataArray
 Base.Broadcast.broadcast_indices(::Type{T}, A) where T<:AbstractDataArray = indices(A)
 
+@inline function broadcast_t(f, ::Type{T}, shape, A, Bs...) where {T}
+    dest = Base.Broadcast.containertype(A, Bs...)(extractT(T), Base.index_lengths(shape...))
+    return broadcast!(f, dest, A, Bs...)
+end
+
+# This is mainly to handle isna.(x) since isna is probably the only
+# function that can guarantee that NAs will never propagate
+@inline function broadcast_t(f, ::Type{Bool}, shape, A, Bs...)
+    dest = similar(BitArray, shape)
+    return broadcast!(f, dest, A, Bs...)
+end
+
+# This one is almost identical to the version in Base and can hopefully be
+# removed at some point. The main issue in Base is that it tests for
+# isleaftype(T) which is false for Union{T,NAtype}. If the test in Base
+# can be modified to cover simple unions of leaftypes then this method
+# can probably be deleted and the two _t methods adjusted to match the Base
+# invokation from Base.Broadcast.broadcast_c
 @inline function Base.Broadcast.broadcast_c{S<:AbstractDataArray}(f, ::Type{S}, A, Bs...)
     T     = Base.Broadcast._broadcast_eltype(f, A, Bs...)
     shape = Base.Broadcast.broadcast_indices(A, Bs...)
-    dest = S(T, Base.index_lengths(shape...))
-    return broadcast!(f, dest, A, Bs...)
+    return broadcast_t(f, T, shape, A, Bs...)
 end
+
+# This one is much faster than normal broadcasting but the method won't get called
+# in fusing operations like (!).(isna.(x))
+Base.broadcast(::typeof(isna), da::DataArray) = copy(da.na)

src/dataarray.jl

@@ -162,7 +162,7 @@ function Base.resize!{T}(da::DataArray{T,1}, n::Int)
 end
 
 function Base.similar(da::DataArray, T::Type, dims::Dims) #-> DataArray{T}
-    return DataArray(Array{T}(dims), trues(dims))
+    return DataArray(Array{extractT(T)}(dims), trues(dims))
 end
 
 Base.size(d::DataArray) = size(d.data) # -> (Int...)
@@ -244,8 +244,6 @@ end
 
 dropna(dv::DataVector) = dv.data[.!dv.na] # -> Vector
 
-Base.broadcast(::typeof(isna), da::DataArray) = copy(da.na)
-
 Base.any(::typeof(isna), da::DataArray) = any(da.na) # -> Bool
 Base.all(::typeof(isna), da::DataArray) = all(da.na) # -> Bool
 

src/natype.jl

@@ -36,12 +36,33 @@ struct NAException <: Exception
 end
 NAException() = NAException("NA found")
 
+# Restrict to Number to avoid infinite recursion
+## Numbers
+Base.promote_rule(::Type{Union{T,NAtype}}, ::Type{Union{S,NAtype}}) where {T<:Number,S<:Number} =
+    Union{promote_type(T, S),NAtype}
+Base.promote_rule(::Type{Union{T,NAtype}}, ::Type{S}) where {T<:Number,S<:Number} =
+    Union{promote_type(T, S),NAtype}
+## Dates
+Base.promote_rule(::Type{Union{T,NAtype}}, ::Type{Union{S,NAtype}}) where {T<:Dates.AbstractTime,S<:Dates.AbstractTime} =
+    Union{promote_type(T, S),NAtype}
+Base.promote_rule(::Type{Union{T,NAtype}}, ::Type{S}) where {T<:Dates.AbstractTime,S<:Dates.AbstractTime} =
+    Union{promote_type(T, S),NAtype}
+
+# Restrict to Number to avoid maching everything
+Base.convert(::Type{Union{T,NAtype}}, x::Number)             where {T<:Number}             = convert(T, x)
+Base.convert(::Type{Union{T,NAtype}}, x::Dates.AbstractTime) where {T<:Dates.AbstractTime} = convert(T, x)
+
 Base.length(x::NAtype) = 1
 Base.size(x::NAtype) = ()
 Base.size(x::NAtype, i::Integer) = i < 1 ? throw(BoundsError()) : 1
 Base.ndims(x::NAtype) = 0
 Base.getindex(x::NAtype, i) = i == 1 ? NA : throw(BoundsError())
 
+extractT(::Type{T}) where {T} = T
+extractT(::Type{Union{T,NAtype}}) where {T} = T
+
+Base.zero(::Type{Union{T,NAtype}}) where {T} = zero(T)
+
 """
     isna(x) -> Bool
 

src/operators.jl

@@ -205,6 +205,7 @@ for f in [:+,:-,:*,:/]
 end
 
 # Unary operators, DataArrays.
+@dataarray_unary(+, Any, T)
 @dataarray_unary(-, Bool, Int)
 @dataarray_unary(-, Any, T)
 @dataarray_unary(!, Bool, T)
@@ -531,31 +532,31 @@ function (-){TA,TJ<:Number}(J::UniformScaling{TJ},A::DataArray{TA,2})
 end
 
 (+)(A::DataArray{Bool,2},J::UniformScaling{Bool}) =
-    invoke(+, Tuple{AbstractArray{Bool,2},UniformScaling{Bool}}, A, J)
+    invoke(+, Tuple{AbstractArray{Union{Bool,NAtype},2},UniformScaling{Bool}}, A, J)
 (+)(J::UniformScaling{Bool},A::DataArray{Bool,2}) =
-    invoke(+, Tuple{UniformScaling{Bool},AbstractArray{Bool,2}}, J, A)
+    invoke(+, Tuple{UniformScaling{Bool},AbstractArray{Union{Bool,NAtype},2}}, J, A)
 (-)(A::DataArray{Bool,2},J::UniformScaling{Bool}) =
-    invoke(-, Tuple{AbstractArray{Bool,2},UniformScaling{Bool}}, A, J)
+    invoke(-, Tuple{AbstractArray{Union{Bool,NAtype},2},UniformScaling{Bool}}, A, J)
 (-)(J::UniformScaling{Bool},A::DataArray{Bool,2}) =
-    invoke(-, Tuple{UniformScaling{Bool},AbstractArray{Bool,2}}, J, A)
+    invoke(-, Tuple{UniformScaling{Bool},AbstractArray{Union{Bool,NAtype},2}}, J, A)
 
 (+){TA,TJ}(A::AbstractDataArray{TA,2},J::UniformScaling{TJ}) =
-    invoke(+, Tuple{AbstractArray{TA,2},UniformScaling{TJ}}, A, J)
+    invoke(+, Tuple{AbstractArray{Union{TA,NAtype},2},UniformScaling{TJ}}, A, J)
 (+){TA}(J::UniformScaling,A::AbstractDataArray{TA,2}) =
-    invoke(+, Tuple{UniformScaling,AbstractArray{TA,2}}, J, A)
+    invoke(+, Tuple{UniformScaling,AbstractArray{Union{TA,NAtype},2}}, J, A)
 (-){TA,TJ<:Number}(A::AbstractDataArray{TA,2},J::UniformScaling{TJ}) =
-    invoke(-, Tuple{AbstractArray{TA,2},UniformScaling{TJ}}, A, J)
+    invoke(-, Tuple{AbstractArray{Union{TA,NAtype},2},UniformScaling{TJ}}, A, J)
 (-){TA,TJ<:Number}(J::UniformScaling{TJ},A::AbstractDataArray{TA,2}) =
-    invoke(-, Tuple{UniformScaling{TJ},AbstractArray{TA,2}}, J, A)
+    invoke(-, Tuple{UniformScaling{TJ},AbstractArray{Union{TA,NAtype},2}}, J, A)
 
 (+)(A::AbstractDataArray{Bool,2},J::UniformScaling{Bool}) =
-    invoke(+, Tuple{AbstractArray{Bool,2},UniformScaling{Bool}}, A, J)
+    invoke(+, Tuple{AbstractArray{Union{Bool,NAtype},2},UniformScaling{Bool}}, A, J)
 (+)(J::UniformScaling{Bool},A::AbstractDataArray{Bool,2}) =
-    invoke(+, Tuple{UniformScaling{Bool},AbstractArray{Bool,2}}, J, A)
+    invoke(+, Tuple{UniformScaling{Bool},AbstractArray{Union{Bool,NAtype},2}}, J, A)
 (-)(A::AbstractDataArray{Bool,2},J::UniformScaling{Bool}) =
-    invoke(-, Tuple{AbstractArray{Bool,2},UniformScaling{Bool}}, A, J)
+    invoke(-, Tuple{AbstractArray{Union{Bool,NAtype},2},UniformScaling{Bool}}, A, J)
 (-)(J::UniformScaling{Bool},A::AbstractDataArray{Bool,2}) =
-    invoke(-, Tuple{UniformScaling{Bool},AbstractArray{Bool,2}}, J, A)
+    invoke(-, Tuple{UniformScaling{Bool},AbstractArray{Union{BoolNAtype},2}}, J, A)
 
 end # if isdefined(Base, :UniformScaling)
 

src/pooleddataarray.jl

@@ -106,9 +106,9 @@ end
 PooledDataArray(d::PooledDataArray) = d
 
 # Constructor from array, w/ pool, missingness, and ref type
-function PooledDataArray{T,R<:Integer,N}(d::AbstractArray{T, N},
+function PooledDataArray{T,R<:Integer,N}(d::AbstractArray{<:Union{T,NAtype}, N},
                                          pool::Vector{T},
-                                         m::AbstractArray{Bool, N},
+                                         m::AbstractArray{<:Union{Bool,NAtype}, N},
                                          r::Type{R} = DEFAULT_POOLED_REF_TYPE)
     if length(pool) > typemax(R)
         throw(ArgumentError("Cannot construct a PooledDataVector with type $R with a pool of size $(length(pool))"))
@@ -466,7 +466,7 @@ julia> p # has been modified
  "B"
 ```
 """
-function setlevels!{T,R}(x::PooledDataArray{T,R}, newpool::AbstractVector{T})
+function setlevels!{T,R}(x::PooledDataArray{T,R}, newpool::AbstractVector)
     if newpool == myunique(newpool) # no NAs or duplicates
         x.pool = newpool
         return x
@@ -483,9 +483,6 @@ function setlevels!{T,R}(x::PooledDataArray{T,R}, newpool::AbstractVector{T})
     end
 end
 
-setlevels!{T, R}(x::PooledDataArray{T, R},
-                 newpool::AbstractVector) = setlevels!(x, convert(Array{T}, newpool))
-
 function setlevels(x::PooledDataArray, d::Dict)
     newpool = copy(DataArray(x.pool))
     # An NA in `v` is put in the pool; that will cause it to become NA
@@ -553,7 +550,7 @@ end
 ##############################################################################
 
 function Base.similar{T,R}(pda::PooledDataArray{T,R}, S::Type, dims::Dims)
-    PooledDataArray(RefArray(zeros(R, dims)), S[])
+    PooledDataArray(RefArray(zeros(R, dims)), extractT(S)[])
 end
 
 ##############################################################################

src/reducedim.jl

@@ -303,7 +303,7 @@ end
 
 function Base.mean!{T}(R::AbstractArray{T}, A::DataArray; skipna::Bool=false,
                        init::Bool=true)
-    init && fill!(R, zero(eltype(R)))
+    init && fill!(R, 0)
     if skipna
         C = Array{Int}(size(R))
         _mapreducedim_skipna_impl!(identity, +, R, C, A)

src/statistics.jl

@@ -39,25 +39,25 @@ gl(n::Integer, k::Integer) = gl(n, k, n*k)
 StatsBase.describe(X::DataVector) = StatsBase.describe(STDOUT, X)
 
 function StatsBase.describe{T<:Real}(io::IO, X::AbstractDataVector{T})
-    nacount = sum(isna.(X))
+    nacount = sum(isna, X)
     pna = 100nacount/length(X)
     if pna != 100 # describe will fail if dropna returns an empty vector
         describe(io, dropna(X))
     else
         println(io, "Summary Stats:")
-        println(io, "Type:           $(eltype(X))")
+        println(io, "Type:           $(T)")
     end
     println(io, "Number Missing: $(nacount)")
     @printf(io, "%% Missing:      %.6f\n", pna)
     return
 end
 
 function StatsBase.describe(io::IO, X::AbstractDataVector)
-    nacount = sum(isna.(X))
+    nacount = sum(isna, X)
     pna = 100nacount/length(X)
     println(io, "Summary Stats:")
     println(io, "Length:         $(length(X))")
-    println(io, "Type:           $(eltype(X))")
+    println(io, "Type:           $(extractT(eltype(X)))")
     println(io, "Number Unique:  $(length(unique(X)))")
     println(io, "Number Missing: $(nacount)")
     @printf(io, "%% Missing:      %.6f\n", pna)

test/broadcast.jl

@@ -128,4 +128,12 @@
     @test map!(abs, x, x) == @data([1, 2])
     @test isequal(map!(+, DataArray(Float64, 3), @data([1, NA, 3]), @data([NA, 2, 3])), @data([NA, NA, 6]))
     @test map!(isequal, DataArray(Float64, 3), @data([1, NA, NA]), @data([1, NA, 3])) == @data([true, true, false])
+
+    # isna doesn't propagate NAs so it should return BitArrays
+    x = isna.(@data [NA, 1, 2])
+    @test x isa BitArray
+    @test x == [true, false, false]
+    x = (!).(isna.(@data [NA, 1, 2]))
+    @test x isa BitArray
+    @test x == [false, true, true]
 end

test/constructors.jl

@@ -25,7 +25,7 @@
     @test isequal(dv, convert(DataArray, 1:3))
 
     dv = DataArray(Int, 3)
-    @test isequal(eltype(dv), Int)
+    @test isequal(eltype(dv), Union{Int,NAtype})
     @test isequal(dv.na, trues(3))
 
     dv = convert(DataArray, zeros(3))
@@ -67,7 +67,7 @@
     @test isequal(pdv, convert(PooledDataArray, PooledDataArray([1, 2, 3])))
 
     pdv = PooledDataArray(Int, 3)
-    @test isequal(eltype(pdv), Int)
+    @test isequal(eltype(pdv), Union{Int,NAtype})
     @test all(isna.(pdv) .== trues(3))
 
     pdv = convert(PooledDataArray, zeros(3))
@@ -106,7 +106,7 @@
     @test isequal(dm, convert(DataArray, trues(2, 2)))
 
     dm = DataArray(Int, 2, 2)
-    @test isequal(eltype(dm), Int)
+    @test isequal(eltype(dm), Union{Int,NAtype})
     @test isequal(dm.na, trues(2, 2))
 
     @test_nowarn convert(DataArray, zeros(2, 2))

test/data.jl

@@ -71,13 +71,13 @@
     @test size(dvint) == (4,)
     @test length(dvint) == 4
     @test sum(isna.(dvint)) == 1
-    @test eltype(dvint) == Int
+    @test eltype(dvint) == Union{Int,NAtype}
 
     #test_group("PooledDataVector methods")
     @test size(pdvstr) == (7,)
     @test length(pdvstr) == 7
     @test sum(isna.(pdvstr)) == 1
-    @test eltype(pdvstr) == String
+    @test eltype(pdvstr) == Union{String,NAtype}
 
     #test_group("DataVector operations")
     @test isequal(dvint .+ 1, DataArray([2, 3, 4, 5], [false, false, true, false]))
@@ -99,7 +99,7 @@
     @test all(convert(Vector{Int}, dvint2) .== [5:8;])
     @test all([i + 1 for i in dvint2] .== [6:9;])
     @test all([length(x)::Int for x in dvstr] == [3, 3, 1, 4])
-    @test repr(dvint) == "[1, 2, NA, 4]"
+    @test repr(dvint) == "Union{DataArrays.NAtype, $Int}[1, 2, NA, 4]"
 
     #test_group("PooledDataVector to something else")
     @test all(dropna(pdvstr) .== ["one", "one", "two", "two", "one", "one"])

test/dataarray.jl

@@ -99,7 +99,7 @@
     end
 
     # Inferrability of map (#276)
-    @test eltype(map(x -> x > 1, @data [1, 2])) == Bool
+    @test eltype(map(x -> x > 1, @data [1, 2])) == Union{Bool,NAtype}
 
     @testset "Issue #278" begin
         x = @data ones(4)

test/nas.jl

@@ -62,4 +62,12 @@
     @test_throws NAException for v in each_failna(dv); end
     @test collect(each_dropna(dv)) == a
     @test collect(each_replacena(dv, 4)) == [4, 4, a..., 4]
+
+    @testset "promotion" for (T1, T2) in ((Int, Float64),
+                                          (Dates.Minute, Dates.Second))
+        @test promote_type(T1, Union{T2,NAtype})               == Union{T2,NAtype}
+        @test promote_type(Union{T1,NAtype}, T2)               == Union{T2,NAtype}
+        @test promote_type(Union{T1,NAtype}, Union{T2,NAtype}) == Union{T2,NAtype}
+    end
+
 end