<|start_filename|>test/convention/scitype.jl<|end_filename|>
@testset "Explicit Scitype tests" begin
    STB = ScientificTypesBase
    ST = ScientificTypes
    @test STB.Scitype(Int, ST.CONV) == Count
    @test STB.Scitype(Float64, ST.CONV) == Continuous
    @test STB.Scitype(Float32, ST.CONV) == Continuous
    @test STB.Scitype(Float16, ST.CONV) == Continuous
    @test STB.Scitype(SubString, ST.CONV) == Textual
end

@testset "scitype of (In)Finite elements" begin
    cv = categorical(['x', 'y'])
    c  = cv[1]
    uv = categorical(['x', 'y'], ordered=true)
    u  = uv[1]
    @test scitype((4, 4.5, c, u, "X")) == Tuple{
                       Count,Continuous,Multiclass{2},OrderedFactor{2},Textual}
end

@testset "elscitype" begin
    X = randn(5, 5)
    @test scitype(X) == Arr{Continuous, 2}
    @test elscitype(X) == Continuous
    X = [1, 2, missing, 5]
    @test elscitype(X) == Union{Missing, Count}
    @test elscitype(X[1:2]) == Union{Missing, Count}
    X = categorical([1, 2, 3, missing, 2, 1, missing])
    @test elscitype(X) == Union{Missing,Multiclass{3}}
    X = categorical("lksfjalksjdflkjsdlkfjasldkfj" |> collect)
    @test elscitype(X) <: Multiclass
    X = coerce(categorical([1,2,3,1,2,3]), OrderedFactor)
    @test elscitype(X) <: Union{Missing, OrderedFactor}
end

A = Any[2 4.5;
        4 4.5;
        6 4.5]

@testset "scitype_union" begin
    @test scitype_union(A) == Union{Count,Continuous}
    @test scitype_union(randn(1000000)) == Continuous
    @test scitype_union(1) == Count
    @test scitype_union([1]) == Count
    @test scitype_union(Any[1]) == Count
    @test scitype_union([1, 2.0, "3"]) == Union{Continuous, Count, Textual}
end

@testset "scitype of Arrays" begin
    # COUNT
    @test scitype(A)               == Arr{Union{Count,Continuous}, 2}
    @test scitype([1, 2, 3])       == Vec{Count}
    @test scitype([1, missing, 3]) == Vec{Union{Missing,Count}}
    @test scitype(Any[1, 2, 3])    == Vec{Count}
    @test scitype(Any[1, missing]) == Vec{Union{Missing,Count}}

    # CONTINUOUS
    @test scitype([1.0, 2.0, 3.0])    == Vec{Continuous}
    @test scitype(Any[1.0, missing])  == Vec{Union{Missing,Continuous}}
    @test scitype(Any[1.0, 2.0, 3.0]) == Vec{Continuous}
    @test scitype(Any[1.0, missing])  == Vec{Union{Missing,Continuous}}

    # MULTICLASS
    @test scitype(categorical(1:4))         == Vec{Multiclass{4}}
    @test scitype(view(categorical(1:4), 1:3)) == Vec{Multiclass{4}}
    @test scitype(Any[categorical(1:4)...]) == Vec{Multiclass{4}}
    @test scitype(categorical([1, missing, 3])) ==
        Vec{Union{Multiclass{2},Missing}}
    
        a = ["aa", "bb", "aa", "bb"] |> categorical
    @test scitype(a[1]) == Multiclass{2}
 
    # NOTE: the slice here does not contain missings but the machine type
    # still contains a missing so the scitype remains with a missing
    @test scitype(categorical([1, missing, 3])[1:1]) ==
        Vec{Union{Multiclass{2},Missing}}

    # ORDEREDFACTOR
    @test scitype(categorical(1:4, ordered=true)) ==
        Vec{OrderedFactor{4}}
    @test scitype(Any[categorical(1:4, ordered=true)...]) ==
        Vec{OrderedFactor{4}}
    @test scitype(categorical([1, missing, 3], ordered=true)) ==
        Vec{Union{OrderedFactor{2},Missing}}
    # NOTE: see note for multiclass
    @test scitype(categorical([1, missing, 3], ordered=true)[1:1]) ==
        Vec{Union{OrderedFactor{2},Missing}}
end

@testset "scitype of Images" begin
    black = RGB(0, 0, 0)
    color_image = fill(black, (10, 20))
    @test scitype(color_image) == ColorImage{10,20}

    color_image2 = fill(black, (5, 3))
    v = [color_image, color_image2, color_image2]
    @test scitype(v) ==
        Vec{Union{ColorImage{10,20},ColorImage{5,3}}}

    white = Gray(1.0)
    gray_image = fill(white, (10, 20))
    @test scitype(gray_image) == GrayImage{10,20}
end

# @testset "PersistenceDiagrams" begin
#     diagram = PersistenceDiagram([(1, Inf), (2, 3)], dim=0)
#     @test scitype(diagram) == PersistenceDiagram

#     diagrams = [diagram, diagram, diagram]
#     @test scitype(diagrams) == Vec{PersistenceDiagram}
# end

@testset "scitype for objects with temporal types" begin
    d = Date(2020, 4, 21)
    t = Time(8, 15, 42)
    dt = now()
    @test scitype(d) == ScientificDate
    @test scitype(t) == ScientificTime
    @test scitype(dt) == ScientificDateTime
    @test scitype(vcat([missing, ], fill(d, 3))) ==
        AbstractVector{Union{Missing,ScientificDate}}
    @test scitype(vcat([missing, ], fill(t, 3))) ==
        AbstractVector{Union{Missing,ScientificTime}}
    @test scitype(vcat([missing, ], fill(dt, 3))) ==
        AbstractVector{Union{Missing,ScientificDateTime}}
    @test scitype([missing, d, t]) ==
        AbstractVector{Union{Missing,ScientificTimeType}}
end

struct FooSampleable <: Dist.Sampleable{Dist.ArrayLikeVariate{0},
    Dist.Discrete}
end

@testset "scitypes of distributions" begin
    @test scitype(Dist.Normal()) == Density{Continuous}
    @test scitype(Dist.Poisson()) == Density{Count}
    @test scitype(Dist.Categorical(3)) == Density{Count}
    X = [1 2; 3 4]
    @test scitype(Dist.MultivariateNormal(X'X)) ==
    Density{AbstractVector{Continuous}}
    @test scitype(FooSampleable()) == Sampleable{Count}
end

@testset "Tables scitype" begin
    ST = ScientificTypes
    ## Test scitype for Column oriented tables
    X = (
        x = rand(5),
        y = rand(Int, 5),
        z = categorical(collect("asdfa")),
        w = rand(5)
    )
    t = scitype(X)
    @test t <: Table(Continuous, Finite, Count)
    @test t <: Table(Infinite, Multiclass)
    @test !(t <: Table(Continuous, Union{Missing, Count}))
    @test ST._cols_scitype(
        Tables.columns(X), Tables.Schema(Tables.columnnames(X), nothing)
    ) == t

    ## scitype tests for row oriented tables.
    Y = Tables.rowtable(X)
    r = scitype(Y)
    @test r <: Table(Continuous, Finite, Count)
    @test r <: Table(Infinite, Multiclass)
    @test !(r <: Table(Continuous, Union{Missing, Count}))
    rows = Tables.rows(Y)
    @test ST._rows_scitype(
        rows, Tables.Schema(Tables.columnnames(iterate(rows, 1)[1]), nothing)
    ) == r
    # ExtremelyWide row oriented table
    @test ST._rows_scitype(
        rows, 
        Tables.Schema(
            Tables.columnnames(iterate(rows, 1)[1]),
            (Int, Int, CategoricalValue{Char, UInt32}, Float64);
            stored = true
        )
    ) == r

    # test schema for column oreinted tables with number of columns 
    # exceeding COLS_SPECIALIZATION_THRESHOLD.
    nt = NamedTuple{
            Tuple(Symbol("x$i") for i in Base.OneTo(ST.COLS_SPECIALIZATION_THRESHOLD + 1))
        }(
            Tuple(rand(2) for i in Base.OneTo(ST.COLS_SPECIALIZATION_THRESHOLD + 1))
    )
    @test scitype(nt) <: Table(Continuous)

    #issue 146
    X = Tables.table(coerce(rand("abc", 5, 3), Multiclass))
    @test scitype(X) === Table{AbstractVector{Multiclass{3}}}
end

# TODO: re-instate when julia 1.0 is no longer LTS release:

# @testset "csvfile" begin
#     X = (x = rand(4), )
#     CSV.write("test.csv", X)
#     file = CSV.File("test.csv")
#     @test scitype(file) == scitype(X)
#     rm("test.csv")
# end

<|start_filename|>test/convention/coerce.jl<|end_filename|>
# -----------------------------------------------------------------------------------------
##############
## Basic (Behavioural tests)
##############

# ----------------------------------------------------------------------------------------------------------------------
###############
## Detailed Tests
###############

@testset "coerce arrays" begin
    A = rand(Int, 2, 3)
    z = rand(Char, 2, 3)
    y = Any[1.0 2; 3 4]
    @test scitype_union(coerce(A, Continuous)) == Continuous
    @test scitype_union(coerce(A, OrderedFactor)) <: OrderedFactor
    @test scitype_union(coerce(z, Multiclass)) <: Multiclass
    @test scitype_union(coerce(y, Count)) === Count
    
    # test fix for issue 39
    y = collect(Int64, 1:5)
    @test_throws ScientificTypes.CoercionError coerce(y, Float64)
    @test_throws ScientificTypes.CoercionError coerce(y, Textual)

    @testset "coersion of Real->OrderedFactor" begin
        v = [0.1, 0.2, 0.2, 0.3, missing, 0.1]
        w = [0.1, 0.2, 0.2, 0.3, 0.1]
        @test_logs((:info, r"Trying to coerce from `Union{Missing,"),
                    global cv = coerce(v, OrderedFactor))
        cw = coerce(w, OrderedFactor)
        @test all(skipmissing(unique(cv)) .== [0.1, 0.2, 0.3])
        @test all(unique(cw) .== [0.1, 0.2, 0.3])
    end

    @testset "coersion of Any->MultiClass" begin
        v2 = Symbol.(collect("asdfghj"))
        global v2c = @test_logs((:warn, r"Converting array"),
                                coerce(v2, Multiclass))
        @test scitype_union(v2c) == Multiclass{7}
        @test eltype(v2c) <: Union{Missing,CategoricalValue{String}}

        v3 = Any[1, 2, 3]
        coerce(v3, Multiclass) # should be no warning here

        # normal behaviour
        v1 = categorical([1,2,1,2,1,2,missing])
        v2 = collect("aksldjfalsdjkfslkjdfalksjdf")
        @test_logs((:info, r"Trying to coerce from `Union"),
                global v1c = coerce(v1, Multiclass))
        global v2c # otherwise julia complains...
        v2c = coerce(v2, Multiclass)
        @test scitype_union(v1c) == Union{Missing,Multiclass{2}}
        @test scitype_union(v2c) == Multiclass{7}
        @test eltype(v1c) <: Union{Missing,CategoricalValue{Int64}}
        @test eltype(v2c) <: CategoricalValue{Char}
    end

    @testset "coerce Categorical->Continuous" begin
        a = categorical(["a","b","a","b",missing])
        a1 = coerce(a, Union{Count,Missing})
        @test scitype_union(a1) == Union{Missing,Count}
        @test all(skipmissing(a1 .== [1, 2, 1, 2, missing]))
        a1 = coerce(a, Union{Continuous,Missing})
        @test scitype_union(a1) == Union{Missing,Continuous}
        @test all(skipmissing(a1 .== [1., 2., 1., 2., missing]))

        y = categorical(1:10, ordered=true)
        new_order = [4, 10, 9, 7, 6, 2, 8, 3, 1, 5]
        levels!(y, new_order)
        @test all(coerce(y, Count) .== sortperm(new_order))
        @test all(coerce(y, Count) .== [9, 6, 8, 1, 10, 5, 4, 7, 3, 2])

        y = categorical([1:10..., missing, 11], ordered=true)
        @test all(skipmissing(coerce(y, Union{Continuous, Missing}) .==
                            float([1:10...,missing,11])))
    end

    # see issue 9
    @testset "coerce Text=>Num" begin
        x = ["52", "15", "125"]
        cx = coerce(x, Count)
        @test cx == [52, 15, 125]
        cx = coerce(x, Continuous)
        @test cx == [52.0, 15.0, 125.0]
        x = ["52", "15", "125", missing]
        cx = coerce(x, Union{Count,Missing})
        @test all(cx .=== [52, 15, 125, missing])
        cx = coerce(x, Union{Missing,Continuous})
        @test all(cx .=== [52.0, 15.0, 125.0, missing])
        @test_logs (:info, "Trying to coerce from `Union{Missing, String}` to `Count`.\nCoerced to `Union{Missing,Count}` instead.") coerce(x, Count)
    end
end

@testset "coersion of Images" begin
    #1. Single GrayImage:
    img = rand(10, 10)
    @test coerce(img, GrayImage) |> size == size(img)
    @test scitype(coerce(img, GrayImage)) == GrayImage{10, 10}

    #2. Single colorimage
    img = rand(10, 10, 3)
    @test coerce(img, ColorImage) |> size == (10, 10)
    @test scitype(coerce(img, ColorImage)) == ColorImage{10, 10}

    #3. Collection: 3dcollection -> GrayImage
    imgs = rand(10, 10, 3)
    @test coerce(imgs, GrayImage) |> size == (3,)
    @test scitype(coerce(imgs, GrayImage)) == AbstractArray{GrayImage{10, 10},1}

    #4. Collection: 4dcollection -> GrayImage
    imgs = rand(10, 10, 1, 3)
    @test coerce(imgs, GrayImage) |> size == (3,)
    @test scitype(coerce(imgs, GrayImage)) == AbstractArray{GrayImage{10, 10},1}

    #5. Collection : 4dcollection -> ColorImage
    imgs = rand(10, 10, 3, 5)
    @test coerce(imgs, ColorImage) |> size == (5,)
    @test scitype(coerce(imgs, ColorImage)) == AbstractArray{ColorImage{10,10},1}
end

# TODO: CSV as testing dep is causing issues for julia 1.0. Perhaps
# the followinng can be re-instated when we have a new LTS release:

# if VERSION ≥ v"1.3.0-"
#    @testset "Coerce Col2" begin
#       X = Tables.table(ones(1_000, 2))
#       tmp = tempname()
#       CSV.write(tmp, X)
#       data = CSV.read(tmp, DataFrame)
#       # data.Column1 and data.Column2 are Column2 (as of CSV 5.19)
#       @test data.Column1 isa AbstractArray{<:AbstractFloat}
#       dc = coerce(data, autotype(data, :discrete_to_continuous))
#       @test scitype(dc) == Table{AbstractArray{Continuous,1}}
#       rm(tmp)
#    end
# end


@testset "misc" begin
    x = Any['a', 5]
    @test (@test_logs (:info, "Char value encountered, such value will be coerced according to the corresponding numeric value (e.g. 'A' to 65).") coerce(x, Count)) == [97, 5]
    x = categorical(['a','b','a','b'])
    @test coerce(x, Continuous) == [1.0,2.0,1.0,2.0]
    y = [missing, 1, 2]
    x = y[2:end]
    c = ScientificTypes._tighten_if_needed(x, eltype(x), true)
    @test c == [1, 2]
    @test eltype(c) == Int
    y = categorical([missing,1,2])
    x = y[2:end]
    @test eltype(x) >: Missing
    c = ScientificTypes._tighten_if_needed(x, eltype(x), true)
    @test c == [1,2]
    @test eltype(c) == Int
    c = coerce(x, OrderedFactor, tight=true)
    @test c == categorical([1,2])
    @test !(eltype(c) >: Missing)
end

<|start_filename|>src/convention/scitype.jl<|end_filename|>
# -----------------------------------------------------------------------------------------
# This file includes the single argument definition of `scitype` method and corresponding 
# convenience functions. It also includes definitions for `scitype/Scitype` of different 
# objects with respect to the `DefaultConvention``.
# -----------------------------------------------------------------------------------------
"""
The scientific type (interpretation) of `X`, distinct from its
machine type.

### Examples
```
julia> scitype(3.14)
Continuous

julia> scitype([1, 2, missing])
AbstractVector{Union{Missing, Count}} 

julia> scitype((5, "beige"))
Tuple{Count, Textual}

julia> using CategoricalArrays

julia> X = (gender = categorical(['M', 'M', 'F', 'M', 'F']),
     ndevices = [1, 3, 2, 3, 2])

julia> scitype(X)
Table{Union{AbstractVector{Count}, AbstractVector{Multiclass{2}}}}
```
"""
scitype(X) = ST.scitype(X, CONV)

function ST.scitype(@nospecialize(X), C::DefaultConvention)
    return _scitype(X, C, vtrait(X)) 
end

function _scitype(X, C, ::Val{:other})
    return ST.fallback_scitype(X, C)
end

# -----------------------------------------------------------------------------------------
# Basic (scalar) scitypes
ST.scitype(::Integer, ::DefaultConvention) = Count
ST.scitype(::AbstractFloat, ::DefaultConvention) = Continuous
ST.scitype(::AbstractString, ::DefaultConvention) = Textual
ST.scitype(::TimeType, ::DefaultConvention) = ScientificTimeType
ST.scitype(::Time, ::DefaultConvention) = ScientificTime
ST.scitype(::Date, ::DefaultConvention) = ScientificDate
ST.scitype(::DateTime, ::DefaultConvention) = ScientificDateTime

# -----------------------------------------------------------------------------------------
# Image scitypes
ST.scitype(img::Arr{<:Gray,2}, ::DefaultConvention) = GrayImage{size(img)...}
ST.scitype(img::Arr{<:AbstractRGB,2}, ::DefaultConvention) =
ColorImage{size(img)...}

# -----------------------------------------------------------------------------------------
# CategoricalArray scitype
function ST.scitype(c::Cat, ::DefaultConvention)
    nc = length(levels(c.pool))
    return ifelse(c.pool.ordered, OrderedFactor{nc}, Multiclass{nc})
end

const CatArrOrSub{T, N} =
    Union{CategoricalArray{T, N}, SubArray{T, N, <:CategoricalArray}}

function ST.scitype(A::CatArrOrSub{T, N}, ::DefaultConvention) where {T, N}
    nlevels = length(levels(A))
    S = ifelse(isordered(A), OrderedFactor{nlevels}, Multiclass{nlevels})
    T >: Missing && (S = Union{S, Missing})
    return AbstractArray{S, N}
end

# -----------------------------------------------------------------------------------------
## Table scitypes

function _scitype(X, C, ::Val{:table})
    if Tables.columnaccess(X)
        return cols_scitype(X)
    else
        return rows_scitype(X)
    end
end

function cols_scitype(X)
    cols = Tables.columns(X)
    sch = Tables.schema(cols)
    return _cols_scitype(cols, sch)
end

function _cols_scitype(cols, ::Union{Nothing, Tables.Schema{nothing, nothing}})
    col_names = Tables.columnnames(cols)
    scitypes = map(col_names) do name
        scitype(Tables.getcolumn(cols, name))
    end
    return Table{Union{scitypes...}}
end

function _cols_scitype(cols, sch::Tables.Schema{names, types}) where {names, types}
    N = length(names)
    if N <= COLS_SPECIALIZATION_THRESHOLD
        return __cols_scitype(cols, sch) 
    else
        scitypes = if types === nothing
            Type[scitype(Tables.getcolumn(cols, names[i])) for i in Base.OneTo(N)]
        else
            Type[
                scitype(
                    Tables.getcolumn(
                    cols, fieldtype(types, i), i, names[i])
                ) for i in Base.OneTo(N)
            ]
            
        end
        return Table{Union{scitypes...}}
    end
end

@inline function __cols_scitype(
    cols, 
    sch::Tables.Schema{names, types}
) where {names, types}
    N = length(names)
    if @generated
        stypes = if types === nothing
            (
                :(scitype(Tables.getcolumn(cols, $(Meta.QuoteNode(names[i]))))) 
                for i in Base.OneTo(N)
            )
        else
            (
                quote
                    scitype(
                        Tables.getcolumn(
                            cols, $(fieldtype(types, i)), $i,  $(Meta.QuoteNode(names[i]))
                        );
                    )
                end 
                for i in Base.OneTo(N)
            )
        end
        return :(Table{Union{$(stypes...)}})
    else
        stypes = if types === nothing 
            (scitype(Tables.getcolumn(cols, names[i])) for i in Base.OneTo(N))
        else
            (
                scitype(Tables.getcolumn(cols, fieldtype(types, i), i, names[i])) 
                for i in Base.OneTo(N)
            )
        end
        return Table{Union{stypes...}}
    end
end

function rows_scitype(X)
    rows = Tables.rows(X)
    sch = Tables.schema(rows)
    return _rows_scitype(rows, sch)
end

function _rows_scitype(rows, sch::Tables.Schema{names, types}) where {names, types}
    rows_ = RowsWrapper(rows)
    cols = if length(names) <= ROWS_SPECIALIZATION_THRESHOLD
        Tables.columns(rows_)
    else
        Tables.dictcolumntable(rows_)
    end
    return _cols_scitype(cols, sch)
end

function _rows_scitype(rows, sch::Union{Nothing, Tables.Schema{nothing, nothing}})
    cols = Tables.dictcolumntable(RowsWrapper(rows))
    return _cols_scitype(cols, sch)
end

# -----------------------------------------------------------------------------------------
# Distributions scitype

const Dist = Distributions

scalar_scitype(::Type) = Unknown
scalar_scitype(::Type{<:Dist.Discrete}) = Count
scalar_scitype(::Type{<:Dist.Continuous}) = Continuous

function space_scitype(
    variate_form::Type{<:Dist.ArrayLikeVariate{0}},
    value_support
)
    return scalar_scitype(value_support)
end

function space_scitype(
    variate_form::Type{<:Dist.ArrayLikeVariate{N}},
    value_support
) where N
    return AbstractArray{scalar_scitype(value_support), N}
end

function ST.scitype(::Distributions.Sampleable{F, S}, ::DefaultConvention) where {F, S}
    return Sampleable{space_scitype(F, S)}
end

function ST.scitype(::Distributions.Distribution{F,S}, ::DefaultConvention) where {F, S}
    return Density{space_scitype(F, S)}
end

# -----------------------------------------------------------------------------------------
# `Scitype` definitions for fast array broadcasting

ST.Scitype(::Type{<:Integer}, ::DefaultConvention) = Count
ST.Scitype(::Type{<:AbstractFloat}, ::DefaultConvention) = Continuous
ST.Scitype(::Type{<:AbstractString}, ::DefaultConvention) = Textual
ST.Scitype(::Type{<:TimeType}, ::DefaultConvention) = ScientificTimeType
ST.Scitype(::Type{<:Date}, ::DefaultConvention) = ScientificDate
ST.Scitype(::Type{<:Time}, ::DefaultConvention) = ScientificTime
ST.Scitype(::Type{<:DateTime}, ::DefaultConvention) = ScientificDateTime
# ST.Scitype(::Type{<:PersistenceDiagram}, ::DefaultConvention) = PersistenceDiagram

# -----------------------------------------------------------------------------------------
## convenience methods
"""
    elscitype(A)
Return the element scientific type of an abstract array `A`. By definition, if
`scitype(A) = AbstractArray{S,N}`, then `elscitype(A) = S`.
"""
elscitype(X) = elscitype(collect(X))
elscitype(X::Arr) = eltype(scitype(X))
 
"""
    scitype_union(A)
Return the type union, over all elements `x` generated by the iterable `A`,
of `scitype(x)`. See also [`scitype`](@ref).
"""
scitype_union(X) = ST.scitype_union(X, DefaultConvention())

<|start_filename|>test/coerce.jl<|end_filename|>
@testset "Basic Type coercion tests" begin
    X = (x=10:10:44, y=1:4, z=collect("abcd"))

    @test_throws ScientificTypes.CoercionError coerce(X, :x=>Float64)
    @test_throws ScientificTypes.CoercionError coerce(X, :x=>Textual)

    types = Dict(:x => Continuous, :z => Multiclass)
    X_coerced = coerce(X, types)
    @test X_coerced ==  coerce(X, :x => Continuous, :z => Multiclass)
    @test scitype_union(X_coerced.x) === Continuous
    @test scitype_union(X_coerced.z) <: Multiclass
    @test !X_coerced.z.pool.ordered
    @test_throws MethodError coerce(["a", "b", "c"], Count)

    y = collect(Float64, 1:5)
    y_coerced = coerce(y, Count)
    @test scitype_union(y_coerced) === Count
    @test y_coerced == y
    y = [1//2, 3//4, 6//5]
    y_coerced = coerce(y, Continuous)
    @test scitype_union(y_coerced) === Continuous
    @test y_coerced ≈ y
    X_coerced = coerce(X, Dict(:z => OrderedFactor))
    @test X_coerced == coerce(X, :z => OrderedFactor)
    @test X_coerced.x === X.x
    @test scitype_union(X_coerced.z) <: OrderedFactor
    @test X_coerced.z.pool.ordered

    # Check no-op coercion
    y = rand(Float64, 5)
    @test coerce(y, Continuous) === y
    y = rand(Float32, 5)
    @test coerce(y, Continuous) === y
    y = rand(BigFloat, 5)
    @test coerce(y, Continuous) === y
    y = rand(Int, 5)
    @test coerce(y, Count) === y
    y = big.(y)
    @test coerce(y, Count) === y
    y = rand(UInt32, 5)
    @test coerce(y, Count) === y
    X_coerced = coerce(X, Dict{Symbol, Type}())
    @test X_coerced.x === X.x
    @test X_coerced.z === X.z
    z = categorical(X.z)
    @test coerce(z, Multiclass) === z
    z = categorical(X.z, compress=true, ordered=false)
    @test coerce(z, Multiclass) === z
    z = categorical(X.z, compress=true, ordered=true)
    @test coerce(z, OrderedFactor) === z

    # missing values
    y_coerced = @test_logs(
        (:info, r"Trying to coerce from `Union{Missing,"),
        coerce([4, 7, missing], Continuous))
    @test ismissing(y_coerced == [4.0, 7.0, missing])
    @test scitype_union(y_coerced) === Union{Missing,Continuous}
    y_coerced = @test_logs(
        (:info, r"Trying to coerce from `Any"),
        coerce(Any[4, 7.0, missing], Continuous))
    @test ismissing(y_coerced == [4.0, 7.0, missing])
    @test scitype_union(y_coerced) === Union{Missing,Continuous}
    y_coerced = @test_logs(
        (:info, r"Trying to coerce from `Union{Missing,"),
        coerce([4.0, 7.0, missing], Count))
    @test ismissing(y_coerced == [4, 7, missing])
    @test scitype_union(y_coerced) === Union{Missing,Count}
    y_coerced = @test_logs(
        (:info, r"Trying to coerce from `Any"),
        coerce(Any[4, 7.0, missing], Count))
    @test ismissing(y_coerced == [4, 7, missing])
    @test scitype_union(y_coerced) === Union{Missing,Count}
    @test scitype_union(@test_logs(
        (:info, r"Trying to coerce from `Union{Missing,"),
        coerce(['x', 'y', missing], Multiclass))) ===
            Union{Missing, Multiclass{2}}
    @test scitype_union(@test_logs(
        (:info, r"Trying to coerce from `Union{Missing,"),
        coerce(['x', 'y', missing], OrderedFactor))) ===
            Union{Missing, OrderedFactor{2}}
    # non-missing Any vectors
    @test coerce(Any[4, 7],     Continuous) == [4.0, 7.0]
    @test coerce(Any[4.0, 7.0], Continuous) == [4, 7]

    # Finite conversions:
    @test scitype_union(coerce(['x', 'y'], Finite)) === Multiclass{2}
    @test scitype_union(@test_logs(
        (:info, r"Trying to coerce from `Union{Missing,"),
        coerce(['x', 'y', missing], Finite))) === Union{Missing, Multiclass{2}}

    # More finite conversions (to check resolution of #48):
    y = categorical([1, 2, 3, missing]) # unordered
    yc = coerce(y, Union{Missing,OrderedFactor})
    @test isordered(yc)
    @test yc[1].pool.ordered
    @test scitype(yc) == Vec{Union{Missing, OrderedFactor{3}}}
    @test scitype_union(yc) == Union{Missing, OrderedFactor{3}}
    @test scitype_union(y) == Union{Missing, Multiclass{3}}

    # tests fix for issue https://github.com/JuliaAI/ScientificTypes.jl/issues/161
    X = (x=10:10:44, y=1:4, z=collect("abcd"))
    Xc = coerce(X, :x => Continuous, "y" => Continuous)
    @test scitype_union(Xc.x) === Continuous
    @test scitype_union(Xc.y) === Continuous
end

# issue #62 (ScientficTypes)
@testset "coersion of Type=>Type" begin
    X = (x=[1,2,1,2,5,1,0,7],
         y=[0,1,0,1,0,1,0,1],
         z=['a','b','a','b','a','a',missing,missing])
    Xc = coerce(X, :y=>OrderedFactor)
    Xc = coerce(Xc, Count=>Continuous)
    @test elscitype(Xc.x) == Continuous
    @test elscitype(Xc.y) == OrderedFactor{2}
    Xc = coerce(Xc, OrderedFactor=>Count)
    @test elscitype(Xc.y) == Count
    Xc = coerce(Xc, :z=>Multiclass, verbosity=0)
    Xc = coerce(Xc, Multiclass=>OrderedFactor, verbosity=0)
    @test elscitype(Xc.z) == Union{Missing,OrderedFactor{2}}
    Xc = coerce(X, Count=>Continuous, Unknown=>Multiclass, verbosity=0)
    @test elscitype(Xc.x) == Continuous
    @test elscitype(Xc.y) == Continuous
    @test elscitype(Xc.z) == Union{Missing, Multiclass{2}}
end

# issue #13
@testset "coersion of mixture of Type=>Type and Symbol=>Type" begin
    X = (x=10:10:44, y=1:4, z=collect("abcd"), w=["a", "b", "c", missing])
    @test ScientificTypes.feature_scitype_pairs(:x => Continuous, X) ==
        [:x => Continuous, ]
    @test ScientificTypes.feature_scitype_pairs(Count => Continuous, X) ==
        [:x => Union{Continuous}, :y=> Union{Continuous}]
    X1 = coerce(X, :z => Multiclass, Count=>Continuous)
    @test schema(X1).scitypes ==
        (Continuous, Continuous, Multiclass{4}, Union{Missing,Textual})
    X2 = coerce(X, :z => Multiclass, :w=>Multiclass, Count=>Continuous,
                verbosity=0)
    @test schema(X2).scitypes ==
        (Continuous, Continuous, Multiclass{4}, Union{Missing,Multiclass{3}})
    @test_throws ArgumentError coerce(X, Count => Continuous, :x=>Multiclass)
end

@testset "coerce!" begin
    df = DataFrame((x=ones(Int,5), y=ones(5)))
     @test scitype(df) == Table{Union{AbstractArray{Continuous,1},
                                      AbstractArray{Count,1}}}
    coerce!(df, :x=>Continuous)
    @test scitype(df) == Table{AbstractArray{Continuous,1}}
 
    df = DataFrame((
          x=ones(Int, 50),
          y=ones(50),
          z=collect("abbabaacbcabbabaacbbcccbccbbabaaacacbcabcbccaabaaa")
          ))
     @test scitype(df) == Table{Union{AbstractArray{Continuous,1},
                                      AbstractArray{Count,1},
                                      AbstractArray{Unknown,1}}}
 
    coerce!(df, autotype(df, :few_to_finite))
     @test scitype(df) == Table{Union{AbstractArray{Multiclass{3},1},
                                      AbstractArray{OrderedFactor{1},1}}}
 
    @test_throws ScientificTypes.CoercionError coerce!(randn(5, 5))

    df2 = DataFrame(x=[1,2,3,4],
    y=["a","b","c","a"],
    z = Union{Missing,Int}[10, 20, 30, 40])
    coerce!(df2, Textual=>Finite)
    coerce!(df2, Union{Missing, Count}=>Count, tight=true) # issue #166
    @test scitype(df2) == Table{Union{AbstractArray{Count,1},
                      AbstractArray{Multiclass{3},1} }}
    
    # issue #9 (coerce Text => Num)
    df3 = DataFrame(x=["1","2","3"], y=[2,3,4])
    coerce!(df3, Textual=>Count)
    @test scitype(df3) == Table{AbstractArray{Count,1}}
    
    x = [1,2,3,4]
    @test_throws ScientificTypes.CoercionError coerce!(x, Continuous)

end


<|start_filename|>src/schema.jl<|end_filename|>
#=
Functionalities supporting the schema of `X` when `X` is a `Tables.jl`
compatible table.
=#
struct Schema{names, scitypes, types}
    storednames::Union{Nothing, Vector{Symbol}}
    storedscitypes::Union{Nothing, Vector{Type}}
    storedtypes::Union{Nothing, Vector{Type}}
end

"""
    Schema(names, scitypes, types)

Constructor for a `Schema` object.
"""
function Schema(
    names::Tuple{Vararg{Symbol}},
    scitypes::Type{S},
    types::Type{T},
) where {T<:Tuple, S<:Tuple}
    return Schema{names, S, T}()
end

function Schema{names, scitypes, types}() where {names, scitypes, types}
    return Schema{names, scitypes, types}(nothing, nothing, nothing)
end

function Schema(
    names::Tuple{Vararg{Symbol}},
    scitypes::Type{S},
    ::Nothing
) where {S<:Tuple}
    return Schema{names, S, nothing}(nothing, nothing, nothing)
end

# whether names/types are stored or not
function stored(::Schema{names, scitypes, types}) where {names, scitypes, types}
    return (names === nothing && scitypes === nothing && types === nothing)
end

stored(::Nothing) = false
tuple_of_symbols(x::NTuple{N, Symbol}) where {N} = x
tuple_of_symbols(x) = Tuple(map(Symbol, x))
 
@inline function Schema(names, scitypes, types; stored::Bool=false)
    if stored || length(names) > SCHEMA_SPECIALIZATION_THRESHOLD
        return Schema{nothing, nothing, nothing}(
            [Symbol(x) for x in names],
            Type[T for T in scitypes],
            Type[T for T in types]
        )
    else 
        return Schema{tuple_of_symbols(names), Tuple{scitypes...}, Tuple{types...}}()
    end
end

if VERSION < v"1.1"
    fieldtypes(t) = Tuple(fieldtype(t, i) for i = 1:fieldcount(t))
end

# Note that `getproperty(::Schema, :name)`, `getproperty(::Schema, :scitypes)`
# and `getproperty(::Schema, :types)` cannot return `nothing` even though the 
# definition below allows for this case. This is because the nature of the `schema`
# function defined below.   
function Base.getproperty(sch::Schema{names, scitypes, types},
                          field::Symbol) where {names, scitypes, types}
    if field === :names
        return names === nothing ? getfield(sch, :storednames) : names
    elseif field === :scitypes
        return scitypes === nothing ? 
            (S = getfield(sch, :storedscitypes); S !== nothing ? S : nothing) : fieldtypes(scitypes)
    elseif field === :types
        return types === nothing ? 
            (T = getfield(sch, :storedtypes); T !== nothing ? T : nothing) : fieldtypes(types)
    else
        throw(ArgumentError("unsupported property for ScientificTypes.Schema"))
    end
end

Base.propertynames(sch::Schema) = (:names, :scitypes, :types)

# make `Schema` a Tables.jl compatible object.
Tables.istable(::Type{<:Schema}) = true
Tables.columnaccess(::Schema) = true
Tables.columns(s::Schema) = s
Tables.columnnames(sch::Schema) = propertynames(sch)
Tables.schema(sch::Schema) = Tables.Schema(Tables.columnnames(sch), Tuple{Symbol, Type, Type})
Tables.getcolumn(sch::Schema, i::Int) = Tables.getcolumn(sch, Tables.columnnames(sch)[i])

function Tables.getcolumn(sch::Schema, nm::Symbol)
    col = getproperty(sch, nm)
    if col === nothing
        N = length(getproperty(sch, :names))
        return fill(nothing, N)
    end
    return col
end

"""
    schema(X)

Inspect the column types and scitypes of a tabular object.
returns `nothing` if the column types and/or scitypes can't be inspected.

## Example

```
X = (ncalls=[1, 2, 4], mean_delay=[2.0, 5.7, 6.0])
schema(X)
```
"""
schema(X) = schema(X, vtrait(X))

# Fallback
function schema(X, ::Val{:other})
    throw(
        ArgumentError("Cannot inspect the internal scitypes of a non-tabular object. ")
    )
end

function schema(X, ::Val{:table})
    if Tables.columnaccess(X)
        return cols_schema(X)
    else
        return rows_schema(X)
    end
end

function cols_schema(X)
    cols = Tables.columns(X)
    sch = Tables.schema(cols)
    sch === nothing && return nothing
    return _cols_schema(cols, sch)
end

function _cols_schema(cols, sch::Tables.Schema{names, types}) where {names, types}
    N = length(names)
    if N <= COLS_SPECIALIZATION_THRESHOLD
        return __cols_schema(cols, sch)
    else
        stypes = if types === nothing
            Type[
                elscitype(Tables.getcolumn(cols, names[i])) 
                for i in Base.OneTo(N)
            ]
        else
            Type[
                elscitype(Tables.getcolumn(cols, fieldtype(types, i), i, names[i])) 
                for i in Base.OneTo(N)
            ]
        end
        return Schema(names, Tuple{stypes...}, types)
    end
end

@inline function __cols_schema(cols, sch::Tables.Schema{names, types}) where {names, types}
    N = length(names)
    if @generated
        stypes = if types === nothing
            (
                :(elscitype(Tables.getcolumn(cols, $(Meta.QuoteNode(names[i]))))) 
                for i in Base.OneTo(N)
            )
        else
            (
                quote
                    elscitype(
                        Tables.getcolumn(
                            cols, $(fieldtype(types, i)), $i,  $(Meta.QuoteNode(names[i]))
                        )
                    )
                end 
                for i in Base.OneTo(N)
            )
        end
        
        return :(Schema(names, Tuple{$(stypes...)}, types))
        
    else
        
        stypes = if types === nothing 
            (
                elscitype(Tables.getcolumn(cols, names[i])) 
                for i in Base.OneTo(N)
            )
        else
            (
                elscitype(Tables.getcolumn(cols, fieldtype(types, i), i, names[i])) 
                for i in Base.OneTo(N)
            )
        end

        return Schema(names, Tuple{stypes...}, types)
        
    end

end

# For extremely wide column oriented tables.
@inline function _cols_schema(cols, sch::Tables.Schema{nothing, nothing})
    names = sch.names
    types = sch.types
    len = length(names)
    stypes = Vector{Type}(undef, len)
    @simd for ind in eachindex(names)
        name = @inbounds(names[ind])
        @inbounds(
            stypes[ind] = elscitype(Tables.getcolumn(cols, name))
        )
    end
    return Schema(names, stypes, types)
end

struct RowsWrapper{T}
    rows::T
end

Tables.istable(::Type{<:RowsWrapper}) = true
Tables.rowaccess(::RowsWrapper) = true
Tables.rows(x::RowsWrapper) = x.rows
Tables.columns(x::RowsWrapper) = Tables.columns(x.rows)

function rows_schema(X)
    rows = Tables.rows(X)
    sch = Tables.schema(rows)
    sch === nothing && return nothing
    return _rows_schema(rows, sch)
end

function _rows_schema(rows, sch::Tables.Schema{names, types}) where {names, types}
    rows_ = RowsWrapper(rows)
    cols = if length(names) <= ROWS_SPECIALIZATION_THRESHOLD
            Tables.columns(rows_)
    else
        Tables.dictcolumntable(rows_)
    end
    return _cols_schema(cols, sch)
end

# for extremely wide row oriented tables
function _rows_schema(rows, sch::Tables.Schema{nothing, nothing})
    cols = Tables.dictcolumntable(RowsWrapper(rows))
    return _cols_schema(cols, sch)
end

function Base.show(io::IO, ::MIME"text/plain", s::Schema)
    data = Tables.matrix(s)
    header = (["names", "scitypes", "types"],)
    pretty_table(io, data, header=header;
                 header_crayon=Crayon(bold=false),
                 alignment=:l)
end



<|start_filename|>test/_benchmark.jl<|end_filename|>
# Last run: 7/1/2020

using ScientificTypes, DataFrames, Random

function foo()
    r = rand()
    r < 0.2 && return "AAA"
    r < 0.4 && return "BBB"
    r < 0.6 && return "CCC"
    r < 0.8 && return "DDD"
    return "EEE"
end

Random.seed!(5161)
v1 = [foo() for i in 1:1_000_000];
v2 = [foo() for i in 1:1_000_000];
df = DataFrame((x1=v1,x2=v2));

using BenchmarkTools
using CategoricalArrays

@btime categorical($v1); # 28.495 ms (1000046 allocations: 34.34 MiB)
@btime coerce($v1, Multiclass);
#this branch ==> 28.755 ms (1000046 allocations: 34.34 MiB)
#master      ==> 46.380 ms (1000080 allocations: 31.48 MiB)

function foo()
    r = rand()
    r < 0.2 && return "AAA"
    r < 0.4 && return "BBB"
    r < 0.6 && return "CCC"
    r < 0.8 && return "DDD"
    return missing
end

Random.seed!(51261)
v2 = [foo() for i in 1:1_000_000];

@btime categorical($v2); # 25.051 ms (799786 allocations: 28.22 MiB)
@btime coerce($v2, Union{Missing,Multiclass});
#this branch ==> 25.712 ms (799787 allocations: 28.22 MiB)
#master      ==> 139.152 ms (1599678 allocations: 51.69 MiB)

using Random, Distributions

n = 1_115_000
Random.seed!(51611)
df  = DataFrame(
        rs  = rand(["A", "B", "C", "D", "E", "F", "G"], n),
        es  = rand(["1","3","5","7","8","9"], n),
        inc = rand([collect(5000:15000); [missing for _ ∈ 1:50]], n),
        gr  = rand([randstring(6) for _ in 1:400], n),
        year = rand(2012:2017, n),
        lt = rand([12, 24, 36, 48, 60, 72], n),
        pb = Int.(round.(rand(LogNormal(8.07, 1.116), n))),
        cb = rand(LogNormal(7.88, 1.288), n),
        ci = rand(0.03:0.01:0.17, n),
        db = rand([repeat(collect(1945:2000),100); missing], n),
        ms = rand(["1", "2", "NA"], n),
        ho = BitArray(rand([true, false], n)),
        el = Int.(round.(rand(LogNormal(1.0, 0.72), n))),
        fl = rand(n),
        ft = rand(n),
        ea = BitArray(rand([true, false], n))
        );

@btime schema($df);
#this branch ==> 296.510 μs (140 allocations: 10.78 KiB)
#master      ==> 270.863 μs (140 allocations: 10.78 KiB)

@btime coerce($df, autotype($df, :few_to_finite));
#this branch ==> 979.098 ms (11152394 allocations: 636.62 MiB)
#master      ==> 1.415 s (12273798 allocations: 609.21 MiB)


<|start_filename|>src/ScientificTypes.jl<|end_filename|>
module ScientificTypes

# Dependencies
using Reexport
@reexport using ScientificTypesBase
export scitype, elscitype, scitype_union
using Tables
using CategoricalArrays
using ColorTypes
using PrettyTables
using Dates
import Distributions

# exports
export coerce, coerce!, autotype, schema, levels, levels!

# --------------------------------------------------------------------------------------
# Abbreviations

const ST  = ScientificTypesBase
const Arr = AbstractArray
const CArr = CategoricalArray
const Cat = CategoricalValue
const COLS_SPECIALIZATION_THRESHOLD = 30
const ROWS_SPECIALIZATION_THRESHOLD = 10000
const SCHEMA_SPECIALIZATION_THRESHOLD = Tables.SCHEMA_SPECIALIZATION_THRESHOLD

#---------------------------------------------------------------------------------------
# Define convention
struct DefaultConvention <: Convention end
const CONV = DefaultConvention()
# -------------------------------------------------------------
# vtrait function, returns either `Val{:table}()` or `Val{:other}()`
vtrait(X) = Val{ifelse(Tables.istable(X), :table, :other)}()

# -------------------------------------------------------------
# Includes

include("coerce.jl")
include("schema.jl")
include("autotype.jl")

include("convention/scitype.jl")
include("convention/coerce.jl")

end # module


<|start_filename|>test/schema.jl<|end_filename|>
@testset "Schema Object" begin
    ST = ScientificTypes
    sch = ST.Schema((:a, :b), (Count, Count), (Int, Int))
    @test sch isa ST.Schema{(:a, :b), Tuple{Count,Count}, Tuple{Int64,Int64}}
    @test sch.names == (:a, :b)
    @test sch.types == (Int, Int)
    @test sch.scitypes == (Count, Count)

    @test_throws ArgumentError sch.something
    @test propertynames(sch) == (:names, :scitypes, :types)

    # Tables.jl interface
    @test Tables.istable(sch) == true
    @test Tables.columnaccess(sch) == true
    @test Tables.columns(sch) == sch
    @test Tables.columnnames(sch) == (:names, :scitypes, :types)
    @test Tables.schema(sch) == Tables.Schema(Tables.columnnames(sch), Tuple{Symbol, Type, Type})
    @test Tables.getcolumn(sch, 1) == (:a, :b)
    @test Tables.getcolumn(sch, 2) == (Count, Count)
    @test Tables.getcolumn(sch, 3) == (Int, Int)
    @test Tables.getcolumn(sch, :types) == (Int, Int)
    @test Tables.getcolumn(sch, :names) == (:a, :b)

    sch2 = ST.Schema((:a, :b), Tuple{Count, Count}, nothing)
    @test sch2 isa ST.Schema{(:a, :b), Tuple{Count,Count}, nothing}
    @test ST.stored(sch2) == false
    @test Tables.getcolumn(sch2, :types) == [nothing, nothing]
    @test Tables.getcolumn(sch2, 3) == [nothing, nothing]

    sch3 = ST.Schema((:a, :b), (Count, Count), (Int, Int); stored=true)
    @test sch3 isa ST.Schema{nothing, nothing, nothing}
    @test ST.stored(sch3) == true

    # Schema show
    df = DataFrame(x=[1.0,2.0,3.0], y=["a","b","c"])
    s = schema(df)
    io = IOBuffer()
    show(io, MIME("text/plain"), ScientificTypes.schema(df))
    @test String(take!(io)) == "┌───────┬────────────┬─────────┐\n│ names │ scitypes   │ types   │\n├───────┼────────────┼─────────┤\n│ x     │ Continuous │ Float64 │\n│ y     │ Textual    │ String  │\n└───────┴────────────┴─────────┘\n"
end

struct MySchemalessTable{U, V}
    x::Vector{U}
    y::Vector{V}
end
 
Tables.istable(::MySchemalessTable) = true
Tables.columnaccess(::Type{<:MySchemalessTable}) = true
Tables.columns(t::MySchemalessTable) = t
Tables.columnnames(::MySchemalessTable) = (:x, :y)

function Tables.getcolumn(t::MySchemalessTable, i::Int)
    return Tables.getcolumn(t, Tables.columnnames(t)[i])
end

function Tables.getcolumn(t::MySchemalessTable, nm::Symbol)
    return ifelse(nm === :x, t.x, t.y)
end

Tables.schema(t::MySchemalessTable) = nothing

struct ExtremelyWideTable{U, V}
    a::Vector{U}
    b::Vector{V}
end
 
Tables.istable(::ExtremelyWideTable) = true
Tables.columnaccess(::Type{<:ExtremelyWideTable}) = true
Tables.columns(t::ExtremelyWideTable) = t
Tables.columnnames(::ExtremelyWideTable) = (:a, :b)

function Tables.getcolumn(t::ExtremelyWideTable, i::Int)
    return Tables.getcolumn(t, Tables.columnnames(t)[i])
end

function Tables.getcolumn(t::ExtremelyWideTable, nm::Symbol)
    return ifelse(nm === :a, t.a, t.b)
end
function Tables.schema(t::ExtremelyWideTable{U, V}) where {U, V}
    return Tables.Schema(Tables.columnnames(t), (U, V); stored=true)
end

@testset "schema" begin
    ST = ScientificTypes
    # NamedTuple schema
    X = (
        x = rand(5),
        y = rand(Int, 5),
        z = categorical(collect("asdfa")),
        w = rand(5)
     )
     s = schema(X)
    @test s.scitypes == (Continuous, Count, Multiclass{4}, Continuous)
    @test s.types == (Float64, Int64, CategoricalValue{Char,UInt32}, Float64)

    # Dataframe schema
    df = DataFrame(X = [1,2,3,missing], Y = [1.0,2.0,3.0,missing])
    sch = schema(df)
    @test sch.scitypes[1] == Union{Missing, Count}
    @test sch.scitypes[2] == Union{Missing, Continuous}
    df2 = df[1:3,:]
    sch = schema(df2)
    @test sch.scitypes[1] == Union{Missing, Count}
    @test sch.scitypes[2] == Union{Missing, Continuous}

    # schema of non-tabular objects
    @test_throws ArgumentError schema([:x, :y])
 
    # PR #61 "schema check for `Tables.DictColumn`"
    X1 = Dict(:a=>rand(5), :b=>rand(Int, 5))
    s1 = schema(X1)
    @test s1.scitypes == (Continuous, Count)
    @test s1.types == (Float64, Int64)
 
    # issue 47 (schema for objects, `X` with, `Tables.schema(X) == nothing`)
    X2 = MySchemalessTable(rand(3), rand(Int, 3))
    s2 = schema(X2)
    @test s2 === nothing
    @test ST.__cols_schema(
        Tables.columns(X2), Tables.Schema(Tables.columnnames(X2), nothing)
    ) == ST.Schema(Tables.columnnames(X2), Tuple{Continuous, Count}, nothing)

    # For ExtremelyWideTable's
    X3 = ExtremelyWideTable(rand(3), rand(Int, 3))
    s3 = schema(X3)
    @test s3.names == Tables.columnnames(X3)
    @test s3.scitypes == (Continuous, Count)
    @test s3.types == (Float64, Int)
    @test ST._cols_schema(
        Tables.columns(X3), Tables.schema(X3)
    ) == ST.Schema(Tables.columnnames(X3), (Continuous, Count), (Float64, Int))
    @test ST._rows_schema(
        Tables.rows(X3), Tables.schema(X3)
    ) == ST.Schema(Tables.columnnames(X3), (Continuous, Count), (Float64, Int))
    
    # test schema for column oreinted tables with number of columns 
    # exceeding COLS_SPECIALIZATION_THRESHOLD.
    nt = NamedTuple{
            Tuple(Symbol("x$i") for i in Base.OneTo(ST.COLS_SPECIALIZATION_THRESHOLD + 1))
        }(
            Tuple(rand(2) for i in Base.OneTo(ST.COLS_SPECIALIZATION_THRESHOLD + 1))
    )
    @test ST.__cols_schema(
        Tables.columns(nt), Tables.Schema(Tables.columnnames(nt), nothing)
    ) == ST.Schema(
        Tables.columnnames(nt),
        NTuple{ST.COLS_SPECIALIZATION_THRESHOLD + 1, Continuous},
        nothing
    )

    @test ST.__cols_schema(
        Tables.columns(nt), Tables.schema(nt)
    ) == ST.Schema(
        Tables.columnnames(nt),
        NTuple{ST.COLS_SPECIALIZATION_THRESHOLD + 1, Continuous},
        NTuple{ST.COLS_SPECIALIZATION_THRESHOLD + 1, Float64}
    )  

 end
 

<|start_filename|>test/runtests.jl<|end_filename|>
using Test, ScientificTypes, ScientificTypesBase, Random
using Tables, CategoricalArrays, DataFrames
using ColorTypes
using Dates

# using CSV # dropped until julia release new LTS as issue for 1.0
import Distributions

const Dist = Distributions
const Arr  = AbstractArray
const CArr = CategoricalArray
const Cat  = CategoricalValue
const Vec  = AbstractVector

include("convention/coerce.jl")
include("convention/scitype.jl")

include("autotype.jl")
include("coerce.jl")
include("schema.jl")
