# -*- coding: utf-8 -*-
'''
Copyright (C) 2016 Turi
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
'''
from ..data_structures.sarray import SArray
from ..util.timezone import GMT
from . import util

import binascii
import pandas as pd
import numpy as np
import unittest
import random
import datetime as dt
import copy
import os
import math
import shutil
import array
import time
import itertools
import warnings
import functools
import tempfile
import sys
import six
from nose.tools import nottest

#######################################################
# Metrics tracking tests are in test_usage_metrics.py #
#######################################################

class SArrayTest(unittest.TestCase):
    def setUp(self):
        self.int_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        self.bool_data = [x % 2 == 0 for x in range(10)]
        self.datetime_data = [dt.datetime(2013, 5, 7, 10, 4, 10),
                dt.datetime(1902, 10, 21, 10, 34, 10).replace(tzinfo=GMT(0.0)),None]
        self.datetime_data2 = [dt.datetime(2013, 5, 7, 10, 4, 10, 109321),
                dt.datetime(1902, 10, 21, 10, 34, 10, 991111).replace(tzinfo=GMT(0.0)),None]
        self.float_data = [1., 2., 3., 4., 5., 6., 7., 8., 9., 10.]
        self.string_data = ["abc", "def", "hello", "world", "pika", "chu", "hello", "world"]
        self.vec_data = [array.array('d', [i, i+1]) for i in self.int_data]
        self.list_data = [[i, str(i), i * 1.0] for i in self.int_data]
        self.dict_data =  [{str(i): i, i : float(i)} for i in self.int_data]
        self.url = "http://s3-us-west-2.amazonaws.com/testdatasets/a_to_z.txt.gz"

    def __test_equal(self, _sarray, _data, _type):
        self.assertEqual(_sarray.dtype(), _type)
        self.assertEqual(len(_sarray), len(_data))
        self.assertSequenceEqual(list(_sarray.head(_sarray.size())), _data)

    def __test_almost_equal(self, _sarray, _data, _type):
        self.assertEqual(_sarray.dtype(), _type)
        self.assertEqual(len(_sarray), len(_data))
        l = list(_sarray)
        for i in range(len(l)):
            if type(l[i]) in (list, array.array): 
                for j in range(len(l[i])):
                    self.assertAlmostEqual(l[i][j], _data[i][j])
            else:
                self.assertAlmostEqual(l[i], _data[i])

    def __test_creation(self, data, dtype, expected):
        """
        Create sarray from data with dtype, and test it equals to
        expected.
        """
        s = SArray(data, dtype)
        self.__test_equal(s, expected, dtype)
        s = SArray(pd.Series(data), dtype)
        self.__test_equal(s, expected, dtype)

    def __test_creation_type_inference(self, data, expected_dtype, expected):
        """
        Create sarray from data with dtype, and test it equals to
        expected.
        """
        s = SArray(data)
        self.__test_equal(s, expected, expected_dtype)
        s = SArray(pd.Series(data))
        self.__test_equal(s, expected, expected_dtype)

    def test_creation(self):
        self.__test_creation(self.int_data, int, self.int_data)
        self.__test_creation(self.int_data, float, [float(x) for x in self.int_data])
        self.__test_creation(self.int_data, str, [str(x) for x in self.int_data])

        self.__test_creation(self.float_data, float, self.float_data)
        self.assertRaises(TypeError, self.__test_creation, [self.float_data, int])

        self.__test_creation(self.string_data, str, self.string_data)
        self.assertRaises(TypeError, self.__test_creation, [self.string_data, int])
        self.assertRaises(TypeError, self.__test_creation, [self.string_data, float])

        expected_output = [chr(x) for x in range(ord('a'), ord('a') + 26)]
        self.__test_equal(SArray(self.url, str), expected_output, str)

        self.__test_creation(self.vec_data, array.array, self.vec_data)
        self.__test_creation(self.list_data, list, self.list_data)

        self.__test_creation(self.dict_data, dict, self.dict_data)

        # test with type inference
        self.__test_creation_type_inference(self.int_data, int, self.int_data)
        self.__test_creation_type_inference(self.float_data, float, self.float_data)
        self.__test_creation_type_inference(self.bool_data, int, [int(x) for x in self.bool_data])
        self.__test_creation_type_inference(self.string_data, str, self.string_data)
        self.__test_creation_type_inference(self.vec_data, array.array, self.vec_data)
        self.__test_creation_type_inference([np.bool_(True),np.bool_(False)],int,[1,0])
        self.__test_creation((1,2,3,4), int, [1,2,3,4])

    def test_list_with_none_creation(self):
        tlist=[[2,3,4],[5,6],[4,5,10,None]]
        g=SArray(tlist)
        self.assertEqual(len(g), len(tlist))
        for i in range(len(tlist)):
          self.assertEqual(g[i], tlist[i])

    def test_list_with_array_creation(self):
        import array
        t = array.array('d',[1.1,2,3,4,5.5])
        g=SArray(t)
        self.assertEqual(len(g), len(t))
        self.assertEqual(g.dtype(), float)
        glist = list(g)
        for i in range(len(glist)):
          self.assertAlmostEqual(glist[i], t[i])

        t = array.array('i',[1,2,3,4,5])
        g=SArray(t)
        self.assertEqual(len(g), len(t))
        self.assertEqual(g.dtype(), int)
        glist = list(g)
        for i in range(len(glist)):
          self.assertEqual(glist[i], t[i])


    def test_in(self):
        sint = SArray(self.int_data, int)
        self.assertTrue(5 in sint)
        self.assertFalse(20 in sint)
        sstr = SArray(self.string_data, str)
        self.assertTrue("abc" in sstr)
        self.assertFalse("zzzzzz" in sstr)
        self.assertFalse("" in sstr)
        self.__test_equal(sstr.contains("ll"), ["ll" in i for i in self.string_data], int)
        self.__test_equal(sstr.contains("a"), ["a" in i for i in self.string_data], int)

        svec = SArray([[1.0,2.0],[2.0,3.0],[3.0,4.0],[4.0,5.0]], array.array)
        self.__test_equal(svec.contains(1.0), [1,0,0,0], int)
        self.__test_equal(svec.contains(0.0), [0,0,0,0], int)
        self.__test_equal(svec.contains(2), [1,1,0,0], int)

        slist = SArray([[1,"22"],[2,"33"],[3,"44"],[4,None]], list)
        self.__test_equal(slist.contains(1.0), [1,0,0,0], int)
        self.__test_equal(slist.contains(3), [0,0,1,0], int)
        self.__test_equal(slist.contains("33"), [0,1,0,0], int)
        self.__test_equal(slist.contains("3"), [0,0,0,0], int)
        self.__test_equal(slist.contains(None), [0,0,0,1], int)

        sdict = SArray([{1:"2"},{2:"3"},{3:"4"},{"4":"5"}], dict)
        self.__test_equal(sdict.contains(1.0), [1,0,0,0], int)
        self.__test_equal(sdict.contains(3), [0,0,1,0], int)
        self.__test_equal(sdict.contains("4"), [0,0,0,1], int)
        self.__test_equal(sdict.contains("3"), [0,0,0,0], int)


        self.__test_equal(SArray(['ab','bc','cd']).is_in('abc'), [1,1,0], int)
        self.__test_equal(SArray(['a','b','c']).is_in(['a','b']), [1,1,0], int)
        self.__test_equal(SArray([1,2,3]).is_in(array.array('d',[1.0,2.0])), [1,1,0], int)
        self.__test_equal(SArray([1,2,None]).is_in([1, None]), [1,0,1], int)
        self.__test_equal(SArray([1,2,None]).is_in([1]), [1,0,0], int)

    def test_save_load(self):
        # Make sure these files don't exist before testing
        self._remove_sarray_files("intarr")
        self._remove_sarray_files("fltarr")
        self._remove_sarray_files("strarr")
        self._remove_sarray_files("vecarr")
        self._remove_sarray_files("listarr")
        self._remove_sarray_files("dictarr")

        sint = SArray(self.int_data, int)
        sflt = SArray([float(x) for x in self.int_data], float)
        sstr = SArray([str(x) for x in self.int_data], str)
        svec = SArray(self.vec_data, array.array)
        slist = SArray(self.list_data, list)
        sdict = SArray(self.dict_data, dict)

        sint.save('intarr.sidx')
        sflt.save('fltarr.sidx')
        sstr.save('strarr.sidx')
        svec.save('vecarr.sidx')
        slist.save('listarr.sidx')
        sdict.save('dictarr.sidx')

        sint2 = SArray('intarr.sidx')
        sflt2 = SArray('fltarr.sidx')
        sstr2 = SArray('strarr.sidx')
        svec2 = SArray('vecarr.sidx')
        slist2 = SArray('listarr.sidx')
        sdict2 = SArray('dictarr.sidx')
        self.assertRaises(IOError, lambda: SArray('__no_such_file__.sidx'))

        self.__test_equal(sint2, self.int_data, int)
        self.__test_equal(sflt2, [float(x) for x in self.int_data], float)
        self.__test_equal(sstr2, [str(x) for x in self.int_data], str)
        self.__test_equal(svec2, self.vec_data, array.array)
        self.__test_equal(slist2, self.list_data, list)
        self.__test_equal(sdict2, self.dict_data, dict)

        # Bad permission
        # Windows has a way more ridiculous way of setting permissions. I'm
        # sure windows will stop us from writing if we don't have
        # permission...probably no reason to test
        if sys.platform != 'win32':
            test_dir = 'test_dir'
            if os.path.exists(test_dir):
                os.removedirs(test_dir)
            os.makedirs(test_dir, mode=0000)
            with self.assertRaises(IOError):
                sint.save(os.path.join(test_dir, 'bad.sidx'))

            # Permissions will affect this test first, so no need
            # to write something here
            with self.assertRaises(IOError):
              sint3 = SArray(os.path.join(test_dir, 'bad.sidx'))

            os.removedirs(test_dir)

        #cleanup
        del sint2
        del sflt2
        del sstr2
        del svec2
        del slist2
        del sdict2
        self._remove_sarray_files("intarr")
        self._remove_sarray_files("fltarr")
        self._remove_sarray_files("strarr")
        self._remove_sarray_files("vecarr")
        self._remove_sarray_files("listarr")
        self._remove_sarray_files("dictarr")

    def test_save_load_text(self):
        self._remove_single_file('txt_int_arr.txt')
        sint = SArray(self.int_data, int)
        sint.save('txt_int_arr.txt')
        self.assertTrue(os.path.exists('txt_int_arr.txt'))
        f = open('txt_int_arr.txt')
        lines = f.readlines()
        for i in range(len(sint)):
            self.assertEquals(int(lines[i]), sint[i])
        self._remove_single_file('txt_int_arr.txt')

        self._remove_single_file('txt_int_arr')
        sint.save('txt_int_arr', format='text')
        self.assertTrue(os.path.exists('txt_int_arr'))
        f = open('txt_int_arr')
        lines = f.readlines()
        for i in range(len(sint)):
            self.assertEquals(int(lines[i]), sint[i])
        self._remove_single_file('txt_int_arr')

    def _remove_single_file(self, filename):
        try:
            os.remove(filename)
        except:
            pass

    def _remove_sarray_files(self, prefix):
        filelist = [ f for f in os.listdir(".") if f.startswith(prefix) ]
        for f in filelist:
            shutil.rmtree(f)

    def test_transform(self):
        sa_char = SArray(self.url, str)
        sa_int = sa_char.apply(lambda char: ord(char), int)

        expected_output = [x for x in range(ord('a'), ord('a') + 26)]
        self.__test_equal(sa_int, expected_output, int)

        # Test randomness across segments, randomized sarray should have different elemetns.
        sa_random = SArray(range(0, 16), int).apply(lambda x: random.randint(0, 1000), int)
        vec = list(sa_random.head(sa_random.size()))
        self.assertFalse(all([x == vec[0] for x in vec]))

        # test transform with missing values
        sa = SArray([1,2,3,None,4,5])
        sa1 = sa.apply(lambda x : x + 1)
        self.__test_equal(sa1, [2,3,4,None,5,6], int)

    def test_transform_with_multiple_lambda(self):
        sa_char = SArray(self.url, str)
        sa_int = sa_char.apply(lambda char: ord(char), int)
        sa2_int = sa_int.apply(lambda val: val + 1, int)

        expected_output = [x for x in range(ord('a') + 1, ord('a') + 26 + 1)]
        self.__test_equal(sa2_int, expected_output, int)

    def test_transform_with_exception(self):
        sa_char = SArray(['a' for i in range(10000)], str)
        # # type mismatch exception
        self.assertRaises(TypeError, lambda: sa_char.apply(lambda char: char, int).head(1))
        # # divide by 0 exception
        self.assertRaises(ZeroDivisionError, lambda: sa_char.apply(lambda char: ord(char) / 0, float))

    def test_transform_with_type_inference(self):
        sa_char = SArray(self.url, str)
        sa_int = sa_char.apply(lambda char: ord(char))

        expected_output = [x for x in range(ord('a'), ord('a') + 26)]
        self.__test_equal(sa_int, expected_output, int)

        sa_bool = sa_char.apply(lambda char: ord(char) > ord('c'))
        expected_output = [int(x > ord('c')) for x in range(ord('a'), ord('a') + 26)]
        self.__test_equal(sa_bool, expected_output, int)

        # # divide by 0 exception
        self.assertRaises(ZeroDivisionError, lambda: sa_char.apply(lambda char: ord(char) / 0))

        # Test randomness across segments, randomized sarray should have different elemetns.
        sa_random = SArray(range(0, 16), int).apply(lambda x: random.randint(0, 1000))
        vec = list(sa_random.head(sa_random.size()))
        self.assertFalse(all([x == vec[0] for x in vec]))

    def test_transform_on_lists(self):
        sa_int =  SArray(self.int_data, int)
        sa_vec2 = sa_int.apply(lambda x: [x, x+1, str(x)])
        expected = [[i, i + 1, str(i)] for i in self.int_data]
        self.__test_equal(sa_vec2, expected, list)
        sa_int_again = sa_vec2.apply(lambda x: int(x[0]))
        self.__test_equal(sa_int_again, self.int_data, int)

        # transform from vector to vector
        sa_vec = SArray(self.vec_data, array.array)
        sa_vec2 = sa_vec.apply(lambda x: x)
        self.__test_equal(sa_vec2, self.vec_data, array.array)

        # transform on list
        sa_list = SArray(self.list_data, list)
        sa_list2 = sa_list.apply(lambda x: x)
        self.__test_equal(sa_list2, self.list_data, list)

        # transform dict to list
        sa_dict = SArray(self.dict_data, dict)
        # Python 3 doesn't return keys in same order from identical dictionaries.
        sort_by_type = lambda x : str(type(x))
        sa_list = sa_dict.apply(lambda x: sorted(list(x), key = sort_by_type))
        self.__test_equal(sa_list, [sorted(list(x), key = sort_by_type) for x in self.dict_data], list)

    def test_transform_dict(self):
        # lambda accesses dict
        sa_dict = SArray([{'a':1}, {1:2}, {'c': 'a'}, None], dict)
        sa_bool_r = sa_dict.apply(lambda x: 'a' in x if x != None else None, skip_undefined=False)
        expected_output = [1, 0, 0, None]
        self.__test_equal(sa_bool_r, expected_output, int)

        # lambda returns dict
        expected_output = [{'a':1}, {1:2}, None, {'c': 'a'}]
        sa_dict = SArray(expected_output, dict)
        lambda_out = sa_dict.apply(lambda x: x)
        self.__test_equal(lambda_out, expected_output, dict)

    def test_filter_dict(self):
        expected_output = [{'a':1}]
        sa_dict = SArray(expected_output, dict)
        ret = sa_dict.filter(lambda x: 'a' in x)
        self.__test_equal(ret, expected_output, dict)

        # try second time to make sure the lambda system still works
        expected_output = [{1:2}]
        sa_dict = SArray(expected_output, dict)
        lambda_out = sa_dict.filter(lambda x: 1 in x)
        self.__test_equal(lambda_out, expected_output, dict)

    def test_filter(self):
        # test empty
        s = SArray([], float)
        no_change = s.filter(lambda x : x == 0)
        self.assertEqual(no_change.size(), 0)

        # test normal case
        s = SArray(self.int_data, int)
        middle_of_array = s.filter(lambda x: x > 3 and x < 8)
        self.assertEqual(list(middle_of_array.head(10)), [x for x in range(4,8)])

        # test normal string case
        s = SArray(self.string_data, str)
        exp_val_list = [x for x in self.string_data if x != 'world']
        # Remove all words whose second letter is not in the first half of the alphabet
        second_letter = s.filter(lambda x: len(x) > 1 and (ord(x[1]) > ord('a')) and (ord(x[1]) < ord('n')))
        self.assertEqual(list(second_letter.head(10)), exp_val_list)

        # test not-a-lambda
        def a_filter_func(x):
            return ((x > 4.4) and (x < 6.8))

        s = SArray(self.int_data, float)
        another = s.filter(a_filter_func)
        self.assertEqual(list(another.head(10)), [5.,6.])

        sa = SArray(self.float_data)

        # filter by self
        sa2 = sa[sa]
        self.assertEquals(list(sa.head(10)), list(sa2.head(10)))

        # filter by zeros
        sa_filter = SArray([0,0,0,0,0,0,0,0,0,0])
        sa2 = sa[sa_filter]
        self.assertEquals(len(sa2), 0)

        # filter by wrong size
        sa_filter = SArray([0,2,5])
        with self.assertRaises(IndexError):
            sa2 = sa[sa_filter]


    def test_any_all(self):
        s = SArray([0,1,2,3,4,5,6,7,8,9], int)
        self.assertEqual(s.any(), True)
        self.assertEqual(s.all(), False)
        s = SArray([0,0,0,0,0], int)
        self.assertEqual(s.any(), False)
        self.assertEqual(s.all(), False)

        s = SArray(self.string_data, str)
        self.assertEqual(s.any(), True)
        self.assertEqual(s.all(), True)

        s = SArray(self.int_data, int)
        self.assertEqual(s.any(), True)
        self.assertEqual(s.all(), True)

        # test empty
        s = SArray([], int)
        self.assertEqual(s.any(), False)
        self.assertEqual(s.all(), True)

        s = SArray([[], []], array.array)
        self.assertEqual(s.any(), False)
        self.assertEqual(s.all(), False)

        s = SArray([[],[1.0]], array.array)
        self.assertEqual(s.any(), True)
        self.assertEqual(s.all(), False)

    def test_astype(self):
        # test empty
        s = SArray([], int)
        as_out = s.astype(float)
        self.assertEqual(as_out.dtype(), float)

        # test float -> int
        s = SArray(list(map(lambda x: x+0.2, self.float_data)), float)
        as_out = s.astype(int)
        self.assertEqual(list(as_out.head(10)), self.int_data)

        # test int->string
        s = SArray(self.int_data, int)
        as_out = s.astype(str)
        self.assertEqual(list(as_out.head(10)), list(map(lambda x: str(x), self.int_data)))

        i_out = as_out.astype(int)
        self.assertEqual(list(i_out.head(10)), list(s.head(10)))

        s = SArray(self.vec_data, array.array)

        with self.assertRaises(RuntimeError):
            s.astype(int)
        with self.assertRaises(RuntimeError):
            s.astype(float)

        s = SArray(["a","1","2","3"])
        with self.assertRaises(RuntimeError):
            s.astype(int)

        self.assertEqual(list(s.astype(int,True).head(4)), [None,1,2,3])

        s = SArray(["[1 2 3]","[4;5]"])
        ret = list(s.astype(array.array).head(2))
        self.assertEqual(ret, [array.array('d',[1,2,3]),array.array('d',[4,5])])

        s = SArray(["[1,\"b\",3]","[4,5]"])
        ret = list(s.astype(list).head(2))
        self.assertEqual(ret, [[1,"b",3],[4,5]])

        s = SArray(["{\"a\":2,\"b\":3}","{}"])
        ret = list(s.astype(dict).head(2))
        self.assertEqual(ret, [{"a":2,"b":3},{}])

        s = SArray(["[1abc]"])
        ret = list(s.astype(list).head(1))
        self.assertEqual(ret, [["1abc"]])

        s = SArray(["{1xyz:1a,2b:2}"])
        ret = list(s.astype(dict).head(1))
        self.assertEqual(ret, [{"1xyz":"1a","2b":2}])

        # astype between list and array
        s = SArray([array.array('d',[1.0,2.0]), array.array('d',[2.0,3.0])])
        ret = list(s.astype(list))
        self.assertEqual(ret, [[1.0, 2.0], [2.0,3.0]])
        ret = list(s.astype(list).astype(array.array))
        self.assertEqual(list(s), list(ret))
        with self.assertRaises(RuntimeError):
            ret = list(SArray([["a",1.0],["b",2.0]]).astype(array.array))

        badcast = list(SArray([["a",1.0],["b",2.0]]).astype(array.array, undefined_on_failure=True))
        self.assertEqual(badcast, [None, None])

    def test_clip(self):
        # invalid types
        s = SArray(self.string_data, str)
        with self.assertRaises(RuntimeError):
            s.clip(25,26)
        with self.assertRaises(RuntimeError):
            s.clip_lower(25)
        with self.assertRaises(RuntimeError):
            s.clip_upper(26)

        # int w/ int, test lower and upper functions too
        # int w/float, no change
        s = SArray(self.int_data, int)
        clip_out = s.clip(3,7).head(10)
        # test that our list isn't cast to float if nothing happened
        clip_out_nc = s.clip(0.2, 10.2).head(10)
        lclip_out = s.clip_lower(3).head(10)
        rclip_out = s.clip_upper(7).head(10)
        self.assertEqual(len(clip_out), len(self.int_data))
        self.assertEqual(len(lclip_out), len(self.int_data))
        self.assertEqual(len(rclip_out), len(self.int_data))
        for i in range(0,len(clip_out)):
            if i < 2:
                self.assertEqual(clip_out[i], 3)
                self.assertEqual(lclip_out[i], 3)
                self.assertEqual(rclip_out[i], self.int_data[i])
                self.assertEqual(clip_out_nc[i], self.int_data[i])
            elif i > 6:
                self.assertEqual(clip_out[i], 7)
                self.assertEqual(lclip_out[i], self.int_data[i])
                self.assertEqual(rclip_out[i], 7)
                self.assertEqual(clip_out_nc[i], self.int_data[i])
            else:
                self.assertEqual(clip_out[i], self.int_data[i])
                self.assertEqual(clip_out_nc[i], self.int_data[i])

        # int w/float, change
        # float w/int
        # float w/float
        clip_out = s.clip(2.8, 7.2).head(10)
        fs = SArray(self.float_data, float)
        ficlip_out = fs.clip(3, 7).head(10)
        ffclip_out = fs.clip(2.8, 7.2).head(10)
        for i in range(0,len(clip_out)):
            if i < 2:
                self.assertAlmostEqual(clip_out[i], 2.8)
                self.assertAlmostEqual(ffclip_out[i], 2.8)
                self.assertAlmostEqual(ficlip_out[i], 3.)
            elif i > 6:
                self.assertAlmostEqual(clip_out[i], 7.2)
                self.assertAlmostEqual(ffclip_out[i], 7.2)
                self.assertAlmostEqual(ficlip_out[i], 7.)
            else:
                self.assertAlmostEqual(clip_out[i], self.float_data[i])
                self.assertAlmostEqual(ffclip_out[i], self.float_data[i])
                self.assertAlmostEqual(ficlip_out[i], self.float_data[i])

        vs = SArray(self.vec_data, array.array);
        clipvs = vs.clip(3, 7).head(100)
        self.assertEqual(len(clipvs), len(self.vec_data));
        for i in range(0, len(clipvs)):
            a = clipvs[i]
            b = self.vec_data[i]
            self.assertEqual(len(a), len(b))
            for j in range(0, len(b)):
                if b[j] < 3:
                    b[j] = 3
                elif b[j] > 7:
                    b[j] = 7
            self.assertEqual(a, b)

    def test_missing(self):
        s=SArray(self.int_data, int)
        self.assertEqual(s.num_missing(), 0)
        s=SArray(self.int_data + [None], int)
        self.assertEqual(s.num_missing(), 1)

        s=SArray(self.float_data, float)
        self.assertEqual(s.num_missing(), 0)
        s=SArray(self.float_data + [None], float)
        self.assertEqual(s.num_missing(), 1)

        s=SArray(self.string_data, str)
        self.assertEqual(s.num_missing(), 0)
        s=SArray(self.string_data + [None], str)
        self.assertEqual(s.num_missing(), 1)

        s=SArray(self.vec_data, array.array)
        self.assertEqual(s.num_missing(), 0)
        s=SArray(self.vec_data + [None], array.array)
        self.assertEqual(s.num_missing(), 1)


    def test_nonzero(self):
        # test empty
        s = SArray([],int)
        nz_out = s.nnz()
        self.assertEqual(nz_out, 0)

        # test all nonzero
        s = SArray(self.float_data, float)
        nz_out = s.nnz()
        self.assertEqual(nz_out, len(self.float_data))

        # test all zero
        s = SArray([0 for x in range(0,10)], int)
        nz_out = s.nnz()
        self.assertEqual(nz_out, 0)

        # test strings
        str_list = copy.deepcopy(self.string_data)
        str_list.append("")
        s = SArray(str_list, str)
        nz_out = s.nnz()
        self.assertEqual(nz_out, len(self.string_data))

    def test_std_var(self):
        # test empty
        s = SArray([], int)
        self.assertTrue(s.std() is None)
        self.assertTrue(s.var() is None)

        # increasing ints
        s = SArray(self.int_data, int)
        self.assertAlmostEqual(s.var(), 8.25)
        self.assertAlmostEqual(s.std(), 2.8722813)

        # increasing floats
        s = SArray(self.float_data, float)
        self.assertAlmostEqual(s.var(), 8.25)
        self.assertAlmostEqual(s.std(), 2.8722813)

        # vary ddof
        self.assertAlmostEqual(s.var(ddof=3), 11.7857143)
        self.assertAlmostEqual(s.var(ddof=6), 20.625)
        self.assertAlmostEqual(s.var(ddof=9), 82.5)
        self.assertAlmostEqual(s.std(ddof=3), 3.4330328)
        self.assertAlmostEqual(s.std(ddof=6), 4.5414755)
        self.assertAlmostEqual(s.std(ddof=9), 9.08295106)

        # bad ddof
        with self.assertRaises(RuntimeError):
            s.var(ddof=11)
        with self.assertRaises(RuntimeError):
            s.std(ddof=11)
        # bad type
        s = SArray(self.string_data, str)
        with self.assertRaises(RuntimeError):
            s.std()
        with self.assertRaises(RuntimeError):
            s.var()

        # overflow test
        huge_int = 9223372036854775807
        s = SArray([1, huge_int], int)
        self.assertAlmostEqual(s.var(), 21267647932558653957237540927630737409.0)
        self.assertAlmostEqual(s.std(), 4611686018427387900.0)

    def test_tail(self):
        # test empty
        s = SArray([], int)
        self.assertEqual(len(s.tail()), 0)

        # test standard tail
        s = SArray([x for x in range(0,40)], int)
        self.assertEqual(list(s.tail()), [x for x in range(30,40)])

        # smaller amount
        self.assertEqual(list(s.tail(3)), [x for x in range(37,40)])

        # larger amount
        self.assertEqual(list(s.tail(40)), [x for x in range(0,40)])

        # too large
        self.assertEqual(list(s.tail(81)), [x for x in range(0,40)])

    def test_max_min_sum_mean(self):
        # negative and positive
        s = SArray([-2,-1,0,1,2], int)
        self.assertEqual(s.max(), 2)
        self.assertEqual(s.min(), -2)
        self.assertEqual(s.sum(), 0)
        self.assertAlmostEqual(s.mean(), 0.)

        # test valid and invalid types
        s = SArray(self.string_data, str)
        with self.assertRaises(RuntimeError):
            s.max()
        with self.assertRaises(RuntimeError):
            s.min()
        with self.assertRaises(RuntimeError):
            s.sum()
        with self.assertRaises(RuntimeError):
            s.mean()

        s = SArray(self.int_data, int)
        self.assertEqual(s.max(), 10)
        self.assertEqual(s.min(), 1)
        self.assertEqual(s.sum(), 55)
        self.assertAlmostEqual(s.mean(), 5.5)

        s = SArray(self.float_data, float)
        self.assertEqual(s.max(), 10.)
        self.assertEqual(s.min(), 1.)
        self.assertEqual(s.sum(), 55.)
        self.assertAlmostEqual(s.mean(), 5.5)

        # test all negative
        s = SArray(list(map(lambda x: x*-1, self.int_data)), int)
        self.assertEqual(s.max(), -1)
        self.assertEqual(s.min(), -10)
        self.assertEqual(s.sum(), -55)
        self.assertAlmostEqual(s.mean(), -5.5)

        # test empty
        s = SArray([], float)
        self.assertTrue(s.max() is None)
        self.assertTrue(s.min() is None)
        self.assertTrue(s.mean() is None)

        # test sum
        t = SArray([], float).sum()
        self.assertTrue(type(t) == float)
        self.assertTrue(t == 0.0)
        t = SArray([], int).sum()
        self.assertTrue(type(t) == int or type(t) == long)
        self.assertTrue(t == 0)
        self.assertTrue(SArray([], array.array).sum() == array.array('d',[]))

        # test big ints
        huge_int = 9223372036854775807
        s = SArray([1, huge_int], int)
        self.assertEqual(s.max(), huge_int)
        self.assertEqual(s.min(), 1)
        # yes, we overflow
        self.assertEqual(s.sum(), (huge_int+1)*-1)
        # ...but not here
        self.assertAlmostEqual(s.mean(), 4611686018427387904.)

        a = SArray([[1,2],[1,2],[1,2]], array.array)
        self.assertEqual(a.sum(), array.array('d', [3,6]))
        self.assertEqual(a.mean(), array.array('d', [1,2]))
        with self.assertRaises(RuntimeError):
            a.max()
        with self.assertRaises(RuntimeError):
            a.min()

        a = SArray([[1,2],[1,2],[1,2,3]], array.array)
        with self.assertRaises(RuntimeError):
            a.sum()
        with self.assertRaises(RuntimeError):
            a.mean()

    def test_max_min_sum_mean_missing(self):
        # negative and positive
        s = SArray([-2,0,None,None,None], int)
        self.assertEqual(s.max(), 0)
        self.assertEqual(s.min(), -2)
        self.assertEqual(s.sum(), -2)
        self.assertAlmostEqual(s.mean(), -1)

        s = SArray([None,None,None], int)
        self.assertEqual(s.max(), None)
        self.assertEqual(s.min(), None)
        self.assertEqual(s.sum(), 0)
        self.assertEqual(s.mean(), None)

    def test_python_special_functions(self):
        s = SArray([], int)
        self.assertEqual(len(s), 0)
        self.assertEqual(str(s), '[]')
        self.assertRaises(ValueError, lambda: bool(s))

        # increasing ints
        s = SArray(self.int_data, int)
        self.assertEqual(len(s), len(self.int_data))
        self.assertEqual(list(s), self.int_data)
        self.assertRaises(ValueError, lambda: bool(s))

        realsum = sum(self.int_data)
        sum1 = sum([x for x in s])
        sum2 = s.sum()
        sum3 = s.apply(lambda x:x, int).sum()

        self.assertEquals(sum1, realsum)
        self.assertEquals(sum2, realsum)
        self.assertEquals(sum3, realsum)

        # abs
        s=np.array(range(-10, 10))
        t = SArray(s, int)
        self.__test_equal(abs(t), list(abs(s)), int)
        t = SArray(s, float)
        self.__test_equal(abs(t), list(abs(s)), float)
        t = SArray([s], array.array)
        self.__test_equal(SArray(abs(t)[0]), list(abs(s)), float)

    def test_scalar_operators(self):
        s=np.array([1,2,3,4,5,6,7,8,9,10]);
        t = SArray(s, int)
        self.__test_equal(t + 1, list(s + 1), int)
        self.__test_equal(t - 1, list(s - 1), int)
        # we handle division differently. All divisions cast to float
        self.__test_equal(t / 2, list(s / 2.0), float)
        self.__test_equal(t * 2, list(s * 2), int)
        self.__test_equal(t ** 2, list(s ** 2), float)
        self.__test_almost_equal(t ** 0.5, list(s ** 0.5), float)
        self.__test_equal(((t ** 2) ** 0.5 + 1e-8).astype(int), list(s), int)
        self.__test_equal(t < 5, list(s < 5), int)
        self.__test_equal(t > 5, list(s > 5), int)
        self.__test_equal(t <= 5, list(s <= 5), int)
        self.__test_equal(t >= 5, list(s >= 5), int)
        self.__test_equal(t == 5, list(s == 5), int)
        self.__test_equal(t != 5, list(s != 5), int)
        self.__test_equal(t % 5, list(s % 5), int)
        self.__test_equal(t // 5, list(s // 5), int)
        self.__test_equal(t + 1, list(s + 1), int)
        self.__test_equal(+t, list(+s), int)
        self.__test_equal(-t, list(-s), int)
        self.__test_equal(1.5 - t, list(1.5 - s), float)
        self.__test_equal(2.0 / t, list(2.0 / s), float)
        self.__test_equal(2 / t, list(2.0 / s), float)
        self.__test_equal(2.5 * t, list(2.5 * s), float)
        self.__test_equal(2**t, list(2**s), float)

        s_neg = np.array([-1,-2,-3,5,6,7,8,9,10]);
        t_neg = SArray(s_neg, int)
        self.__test_equal(t_neg // 5, list(s_neg // 5), int)
        self.__test_equal(t_neg % 5, list(s_neg % 5), int)
        
        s=["a","b","c"]
        t = SArray(s, str)
        self.__test_equal(t + "x", [i + "x" for i in s], str)
        with self.assertRaises(RuntimeError):
            t - 'x'
        with self.assertRaises(RuntimeError):
            t * 'x'
        with self.assertRaises(RuntimeError):
            t / 'x'

        s = SArray(self.vec_data, array.array)
        self.__test_equal(s + 1, [array.array('d', [float(j) + 1 for j in i]) for i in self.vec_data], array.array)
        self.__test_equal(s - 1, [array.array('d', [float(j) - 1 for j in i]) for i in self.vec_data], array.array)
        self.__test_equal(s * 2, [array.array('d', [float(j) * 2 for j in i]) for i in self.vec_data], array.array)
        self.__test_equal(s / 2, [array.array('d', [float(j) / 2 for j in i]) for i in self.vec_data], array.array)
        s = SArray([1,2,3,4,None])
        self.__test_equal(s == None, [0,0,0,0,1], int)
        self.__test_equal(s != None, [1,1,1,1,0], int)

    def test_modulus_operator(self):
        l = [-5,-4,-3,-2,-1,0,1,2,3,4,5]
        t = SArray(l, int)
        self.__test_equal(t % 2, [i % 2 for i in l], int)
        self.__test_equal(t % -2, [i % -2 for i in l], int)

    def test_vector_operators(self):
        s=np.array([1,2,3,4,5,6,7,8,9,10]);
        s2=np.array([5,4,3,2,1,10,9,8,7,6]);
        t = SArray(s, int)
        t2 = SArray(s2, int)
        self.__test_equal(t + t2, list(s + s2), int)
        self.__test_equal(t - t2, list(s - s2), int)
        # we handle division differently. All divisions cast to float
        self.__test_equal(t / t2, list(s.astype(float) / s2), float)
        self.__test_equal(t * t2, list(s * s2), int)
        self.__test_equal(t ** t2, list(s ** s2), float)
        self.__test_almost_equal(t ** (1.0 / t2), list(s ** (1.0 / s2)), float)
        self.__test_equal(t > t2, list(s > s2), int)
        self.__test_equal(t <= t2, list(s <= s2), int)
        self.__test_equal(t >= t2, list(s >= s2), int)
        self.__test_equal(t == t2, list(s == s2), int)
        self.__test_equal(t != t2, list(s != s2), int)

        s = SArray(self.vec_data, array.array)
        self.__test_almost_equal(s + s, [array.array('d', [float(j) + float(j) for j in i]) for i in self.vec_data], array.array)
        self.__test_almost_equal(s - s, [array.array('d', [float(j) - float(j) for j in i]) for i in self.vec_data], array.array)
        self.__test_almost_equal(s * s, [array.array('d', [float(j) * float(j) for j in i]) for i in self.vec_data], array.array)
        self.__test_almost_equal(s / s, [array.array('d', [float(j) / float(j) for j in i]) for i in self.vec_data], array.array)
        self.__test_almost_equal(s ** s, [array.array('d', [float(j) ** float(j) for j in i]) for i in self.vec_data], array.array)
        self.__test_almost_equal(s // s, [array.array('d', [float(j) // float(j) for j in i]) for i in self.vec_data], array.array)
        t = SArray(self.float_data, float)

        self.__test_almost_equal(s + t, [array.array('d', [float(j) + i[1] for j in i[0]]) for i in zip(self.vec_data, self.float_data)], array.array)
        self.__test_almost_equal(s - t, [array.array('d', [float(j) - i[1] for j in i[0]]) for i in zip(self.vec_data, self.float_data)], array.array)
        self.__test_almost_equal(s * t, [array.array('d', [float(j) * i[1] for j in i[0]]) for i in zip(self.vec_data, self.float_data)], array.array)
        self.__test_almost_equal(s / t, [array.array('d', [float(j) / i[1] for j in i[0]]) for i in zip(self.vec_data, self.float_data)], array.array)
        self.__test_almost_equal(s ** t, [array.array('d', [float(j) ** i[1] for j in i[0]]) for i in zip(self.vec_data, self.float_data)], array.array)
        self.__test_almost_equal(s // t, [array.array('d', [float(j) // i[1] for j in i[0]]) for i in zip(self.vec_data, self.float_data)], array.array)
        self.__test_almost_equal(+s, [array.array('d', [float(j) for j in i]) for i in self.vec_data], array.array)
        self.__test_almost_equal(-s, [array.array('d', [-float(j) for j in i]) for i in self.vec_data], array.array)

        neg_float_data = [-v for v in self.float_data]
        t = SArray(neg_float_data, float)
        self.__test_almost_equal(s + t, [array.array('d', [float(j) + i[1] for j in i[0]]) for i in zip(self.vec_data, neg_float_data)], array.array)
        self.__test_almost_equal(s - t, [array.array('d', [float(j) - i[1] for j in i[0]]) for i in zip(self.vec_data, neg_float_data)], array.array)
        self.__test_almost_equal(s * t, [array.array('d', [float(j) * i[1] for j in i[0]]) for i in zip(self.vec_data, neg_float_data)], array.array)
        self.__test_almost_equal(s / t, [array.array('d', [float(j) / i[1] for j in i[0]]) for i in zip(self.vec_data, neg_float_data)], array.array)
        self.__test_almost_equal(s ** t, [array.array('d', [float(j) ** i[1] for j in i[0]]) for i in zip(self.vec_data, neg_float_data)], array.array)
        self.__test_almost_equal(s // t, [array.array('d', [float(j) // i[1] for j in i[0]]) for i in zip(self.vec_data, neg_float_data)], array.array)
        self.__test_almost_equal(t // s, [array.array('d', [i[1] // float(j) for j in i[0]]) for i in zip(self.vec_data, neg_float_data)], array.array)
        
        s = SArray([1,2,3,4,None])
        self.assertTrue((s==s).all())
        s = SArray([1,2,3,4,None])
        self.assertFalse((s!=s).any())

    def test_div_corner(self):

        def try_eq_sa_val(left_val, right_val):
            if type(left_val) is list:
                left_val = array.array('d', left_val)
            if type(right_val) is list:
                right_val = array.array('d', right_val)
            
            left_type = type(left_val)
            v1 = (SArray([left_val], left_type) // right_val)[0]
            
            if type(right_val) is array.array:
                if type(left_val) is array.array:
                    v2 = array.array('d', [lv // rv for lv, rv in zip(left_val, right_val)])
                else:
                    v2 = array.array('d', [left_val // rv for rv in right_val])
            else:
                if type(left_val) is array.array:
                    v2 = array.array('d', [lv // right_val for lv in left_val])
                else:
                    v2 = left_val // right_val
                
            if type(v1) in six.integer_types:
                self.assertTrue(type(v2) in six.integer_types)
            else:
                self.assertEqual(type(v1), type(v2))
            self.assertEqual(v1, v2)

        try_eq_sa_val(1, 2)
        try_eq_sa_val(1.0, 2)
        try_eq_sa_val(1, 2.0)
        try_eq_sa_val(1.0, 2.0)

        try_eq_sa_val(-1, 2)
        try_eq_sa_val(-1.0, 2)
        try_eq_sa_val(-1, 2.0)
        try_eq_sa_val(-1.0, 2.0)

        try_eq_sa_val([1, -1], 2)
        try_eq_sa_val([1, -1], 2.0)

        try_eq_sa_val(2,[3, -3])
        try_eq_sa_val(2.0,[3, -3])
                

    def test_floodiv_corner(self):

        def try_eq_sa_val(left_val, right_val):
            if type(left_val) is list:
                left_val = array.array('d', left_val)
            if type(right_val) is list:
                right_val = array.array('d', right_val)
            
            left_type = type(left_val)
            v1 = (SArray([left_val], left_type) // right_val)[0]
            
            if type(right_val) is array.array:
                if type(left_val) is array.array:
                    v2 = array.array('d', [lv // rv for lv, rv in zip(left_val, right_val)])
                else:
                    v2 = array.array('d', [left_val // rv for rv in right_val])
            else:
                if type(left_val) is array.array:
                    v2 = array.array('d', [lv // right_val for lv in left_val])
                else:
                    v2 = left_val // right_val
                
            if type(v1) in six.integer_types:
                self.assertTrue(type(v2) in six.integer_types)
            else:
                self.assertEqual(type(v1), type(v2))

            self.assertEqual(v1, v2)

        try_eq_sa_val(1, 2)
        try_eq_sa_val(1.0, 2)
        try_eq_sa_val(1, 2.0)
        try_eq_sa_val(1.0, 2.0)

        try_eq_sa_val(-1, 2)
        try_eq_sa_val(-1.0, 2)
        try_eq_sa_val(-1, 2.0)
        try_eq_sa_val(-1.0, 2.0)

        try_eq_sa_val([1, -1], 2)
        try_eq_sa_val([1, -1], 2.0)

        try_eq_sa_val(2,[3, -3])
        try_eq_sa_val(2.0,[3, -3])

        from math import isnan
        
        def try_eq_sa_correct(left_val, right_val, correct):
            if type(left_val) is list:
                left_val = array.array('d', left_val)
            if type(right_val) is list:
                right_val = array.array('d', right_val)
            
            left_type = type(left_val)
            v1 = (SArray([left_val], left_type) // right_val)[0]

            if type(correct) is not list:
                v1 = [v1]
                correct = [correct]

            for v, c in zip(v1, correct):
                if type(v) is float and isnan(v):
                    assert isnan(c)
                else:
                    self.assertEqual(type(v), type(c))
                    self.assertEqual(v, c)
        
        try_eq_sa_correct(1, 0, None)
        try_eq_sa_correct(0, 0, None)
        try_eq_sa_correct(-1, 0, None)
        
        try_eq_sa_correct(1.0, 0, float('inf'))
        try_eq_sa_correct(0.0, 0, float('nan'))
        try_eq_sa_correct(-1.0, 0, float('-inf'))
                
        try_eq_sa_correct([1.0,0,-1], 0, [float('inf'), float('nan'), float('-inf')])
        try_eq_sa_correct(1, [1.0, 0], [1., float('inf')])
        try_eq_sa_correct(-1, [1.0, 0], [-1., float('-inf')])
        try_eq_sa_correct(0, [1.0, 0], [0., float('nan')])
        
    def test_logical_ops(self):
        s=np.array([0,0,0,0,1,1,1,1]);
        s2=np.array([0,1,0,1,0,1,0,1]);
        t = SArray(s, int)
        t2 = SArray(s2, int)
        self.__test_equal(t & t2, list(((s & s2) > 0).astype(int)), int)
        self.__test_equal(t | t2, list(((s | s2) > 0).astype(int)), int)

    def test_string_operators(self):
        s=["a","b","c","d","e","f","g","h","i","j"];
        s2=["e","d","c","b","a","j","i","h","g","f"];

        t = SArray(s, str)
        t2 = SArray(s2, str)
        self.__test_equal(t + t2, ["".join(x) for x in zip(s,s2)], str)
        self.__test_equal(t + "x", [x + "x" for x in s], str)
        self.__test_equal(t < t2, [x < y for (x,y) in zip(s,s2)], int)
        self.__test_equal(t > t2, [x > y for (x,y) in zip(s,s2)], int)
        self.__test_equal(t == t2, [x == y for (x,y) in zip(s,s2)], int)
        self.__test_equal(t != t2, [x != y for (x,y) in zip(s,s2)], int)
        self.__test_equal(t <= t2, [x <= y for (x,y) in zip(s,s2)], int)
        self.__test_equal(t >= t2, [x >= y for (x,y) in zip(s,s2)], int)


    def test_vector_operator_missing_propagation(self):
        t = SArray([1,2,3,4,None,6,7,8,9,None], float) # missing 4th and 9th
        t2 = SArray([None,4,3,2,np.nan,10,9,8,7,6], float) # missing 0th and 4th
        self.assertEquals(len((t + t2).dropna()), 7);
        self.assertEquals(len((t - t2).dropna()), 7);
        self.assertEquals(len((t * t2).dropna()), 7);

    def test_dropna(self):
        no_nas = ['strings', 'yeah', 'nan', 'NaN', 'NA', 'None']
        t = SArray(no_nas)
        self.assertEquals(len(t.dropna()), 6)
        self.assertEquals(list(t.dropna()), no_nas)
        t2 = SArray([None,np.nan])
        self.assertEquals(len(t2.dropna()), 0)
        self.assertEquals(list(SArray(self.int_data).dropna()), self.int_data)
        self.assertEquals(list(SArray(self.float_data).dropna()), self.float_data)

    def test_fillna(self):
        # fillna shouldn't fill anything
        no_nas = ['strings', 'yeah', 'nan', 'NaN', 'NA', 'None']
        t = SArray(no_nas)
        out = t.fillna('hello')
        self.assertEquals(list(out), no_nas)

        # Normal integer case (float auto casted to int)
        t = SArray([53,23,None,np.nan,5])
        self.assertEquals(list(t.fillna(-1.0)), [53,23,-1,-1,5])

        # dict type
        t = SArray(self.dict_data+[None])
        self.assertEquals(list(t.fillna({1:'1'})), self.dict_data+[{1:'1'}])

        # list type
        t = SArray(self.list_data+[None])
        self.assertEquals(list(t.fillna([0,0,0])), self.list_data+[[0,0,0]])

        # vec type
        t = SArray(self.vec_data+[None])
        self.assertEquals(list(t.fillna(array.array('f',[0.0,0.0]))), self.vec_data+[array.array('f',[0.0,0.0])])

        # empty sarray
        t = SArray()
        self.assertEquals(len(t.fillna(0)), 0)

    def test_sample(self):
        sa = SArray(data=self.int_data)
        sa_sample = sa.sample(.5, 9)
        sa_sample2 = sa.sample(.5, 9)

        self.assertEqual(list(sa_sample.head()), list(sa_sample2.head()))

        for i in sa_sample:
            self.assertTrue(i in self.int_data)

        with self.assertRaises(ValueError):
            sa.sample(3)

        sa_sample = SArray().sample(.5, 9)
        self.assertEqual(len(sa_sample), 0)

    def test_hash(self):
        a = SArray([0,1,0,1,0,1,0,1], int)
        b = a.hash()
        zero_hash = b[0]
        one_hash = b[1]
        self.assertTrue((b[a] == one_hash).all())
        self.assertTrue((b[1-a] == zero_hash).all())

        # I can hash other stuff too
        # does not throw
        a.astype(str).hash().__materialize__()

        a.apply(lambda x: [x], list).hash().__materialize__()

        # Nones hash too!
        a = SArray([None, None, None], int).hash()
        self.assertTrue(a[0] is not None)
        self.assertTrue((a == a[0]).all())

        # different seeds give different hash values
        self.assertTrue((a.hash(seed=0) != a.hash(seed=1)).all())


    def test_random_integers(self):
        a = SArray.random_integers(0)
        self.assertEqual(len(a), 0)
        a = SArray.random_integers(1000)
        self.assertEqual(len(a), 1000)

    def test_vector_slice(self):
        d=[[1],[1,2],[1,2,3]]
        g=SArray(d, array.array)
        self.assertEqual(list(g.vector_slice(0).head()), [1,1,1])
        self.assertEqual(list(g.vector_slice(0,2).head()), [None,array.array('d', [1,2]),array.array('d', [1,2])])
        self.assertEqual(list(g.vector_slice(0,3).head()), [None,None,array.array('d', [1,2,3])])

        g=SArray(self.vec_data, array.array);
        self.__test_equal(g.vector_slice(0), self.float_data, float)
        self.__test_equal(g.vector_slice(0, 2), self.vec_data, array.array)

    def _my_subslice(self, arr, start=None, stop=None, step=1):
        return arr.apply(lambda x: x[slice(start, stop, step)], arr.dtype())

    def _slice_equality_test(self, arr, start=None, stop=None, step=1):
        self.assertEqual(
                list(arr.subslice(start, stop, step)),
                list(self._my_subslice(arr,start,stop,step)))

    def test_subslice(self):
        #string slicing
        g=SArray(range(1,1000, 10)).astype(str)
        self._slice_equality_test(g, 0, 2);
        self._slice_equality_test(g, 0, -1, 2);
        self._slice_equality_test(g, -1, -3);
        self._slice_equality_test(g, -1, -2, -1);
        self._slice_equality_test(g, None, None, -1);
        self._slice_equality_test(g, -100, -1);

        #list slicing
        g=SArray(range(1,10)).apply(lambda x: list(range(x)), list)
        self._slice_equality_test(g, 0, 2);
        self._slice_equality_test(g, 0, -1, 2);
        self._slice_equality_test(g, -1, -3);
        self._slice_equality_test(g, -1, -2, -1);
        self._slice_equality_test(g, None, None, -1);
        self._slice_equality_test(g, -100, -1);

        #array slicing
        import array
        g=SArray(range(1,10)).apply(lambda x: array.array('d', range(x)))
        self._slice_equality_test(g, 0, 2);
        self._slice_equality_test(g, 0, -1, 2);
        self._slice_equality_test(g, -1, -3);
        self._slice_equality_test(g, -1, -2, -1);
        self._slice_equality_test(g, None, None, -1);
        self._slice_equality_test(g, -100, -1);

        #this should fail
        with self.assertRaises(TypeError):
            g=SArray(range(1,1000)).subslice(1)

        with self.assertRaises(TypeError):
            g=SArray(range(1,1000)).astype(float).subslice(1)

    def test_lazy_eval(self):
        sa = SArray(range(-10, 10))
        sa = sa + 1
        sa1 = sa >= 0
        sa2 = sa <= 0
        sa3 = sa[sa1 & sa2]
        item_count = sa3.size()
        self.assertEqual(item_count, 1)

    def __test_append(self, data1, data2, dtype):
        sa1 = SArray(data1, dtype)
        sa2 = SArray(data2, dtype)
        sa3 = sa1.append(sa2)
        self.__test_equal(sa3, data1 + data2, dtype)

        sa3 = sa2.append(sa1)
        self.__test_equal(sa3, data2 + data1, dtype)

    def test_append(self):
        n = len(self.int_data)
        m = n // 2

        self.__test_append(self.int_data[0:m], self.int_data[m:n], int)
        self.__test_append(self.bool_data[0:m], self.bool_data[m:n], int)
        self.__test_append(self.string_data[0:m], self.string_data[m:n], str)
        self.__test_append(self.float_data[0:m], self.float_data[m:n], float)
        self.__test_append(self.vec_data[0:m], self.vec_data[m:n], array.array)
        self.__test_append(self.dict_data[0:m], self.dict_data[m:n], dict)

    def test_append_exception(self):
        val1 = [i for i in range(1, 1000)]
        val2 = [str(i) for i in range(-10, 1)]
        sa1 = SArray(val1, int)
        sa2 = SArray(val2, str)
        with self.assertRaises(RuntimeError):
            sa3 = sa1.append(sa2)

    def test_word_count(self):
        sa = SArray(["This is someurl http://someurl!!",
                     "中文 应该也 行",
                     'Сблъсъкът между'])
        expected = [{"this": 1, "http://someurl!!": 1, "someurl": 1, "is": 1},
                    {"中文": 1, "应该也": 1, "行": 1},
                    {"Сблъсъкът": 1, "между": 1}]
        expected2 = [{"This": 1, "http://someurl!!": 1, "someurl": 1, "is": 1},
                     {"中文": 1, "应该也": 1, "行": 1},
                     {"Сблъсъкът": 1, "между": 1}]
        sa1 = sa._count_words()
        self.assertEquals(sa1.dtype(), dict)
        self.__test_equal(sa1, expected, dict)

        sa1 = sa._count_words(to_lower=False)
        self.assertEquals(sa1.dtype(), dict)
        self.__test_equal(sa1, expected2, dict)

        #should fail if the input type is not string
        sa = SArray([1, 2, 3])
        with self.assertRaises(TypeError):
            sa._count_words()

    def test_word_count2(self):
        sa = SArray(["This is some url http://www.someurl.com!!", "Should we? Yes, we should."])
        #TODO: Get some weird unicode whitespace in the Chinese and Russian tests
        expected1 = [{"this": 1, "is": 1, "some": 1, "url": 1, "http://www.someurl.com!!": 1},
                     {"should": 1, "we?": 1, "we": 1, "yes,": 1, "should.": 1}]
        expected2 = [{"this is some url http://www.someurl.com": 1},
                     {"should we": 1, " yes": 1, " we should.": 1}]
        word_counts1 = sa._count_words()
        word_counts2 = sa._count_words(delimiters=["?", "!", ","])

        self.assertEquals(word_counts1.dtype(), dict)
        self.__test_equal(word_counts1, expected1, dict)
        self.assertEquals(word_counts2.dtype(), dict)
        self.__test_equal(word_counts2, expected2, dict)

    def test_ngram_count(self):
        sa_word = SArray(["I like big dogs. They are fun. I LIKE BIG DOGS", "I like.", "I like big"])
        sa_character = SArray(["Fun. is. fun","Fun is fun.","fu", "fun"])

        # Testing word n-gram functionality
        result = sa_word._count_ngrams(3)
        result2 = sa_word._count_ngrams(2)
        result3 = sa_word._count_ngrams(3,"word", to_lower=False)
        result4 = sa_word._count_ngrams(2,"word", to_lower=False)
        expected = [{'fun i like': 1, 'i like big': 2, 'they are fun': 1, 'big dogs they': 1, 'like big dogs': 2, 'are fun i': 1, 'dogs they are': 1}, {}, {'i like big': 1}]
        expected2 = [{'i like': 2, 'dogs they': 1, 'big dogs': 2, 'are fun': 1, 'like big': 2, 'they are': 1, 'fun i': 1}, {'i like': 1}, {'i like': 1, 'like big': 1}]
        expected3 = [{'I like big': 1, 'fun I LIKE': 1, 'I LIKE BIG': 1, 'LIKE BIG DOGS': 1, 'They are fun': 1, 'big dogs They': 1, 'like big dogs': 1, 'are fun I': 1, 'dogs They are': 1}, {}, {'I like big': 1}]
        expected4 = [{'I like': 1, 'like big': 1, 'I LIKE': 1, 'BIG DOGS': 1, 'are fun': 1, 'LIKE BIG': 1, 'big dogs': 1, 'They are': 1, 'dogs They': 1, 'fun I': 1}, {'I like': 1}, {'I like': 1, 'like big': 1}]



        self.assertEquals(result.dtype(), dict)
        self.__test_equal(result, expected, dict)
        self.assertEquals(result2.dtype(), dict)
        self.__test_equal(result2, expected2, dict)
        self.assertEquals(result3.dtype(), dict)
        self.__test_equal(result3, expected3, dict)
        self.assertEquals(result4.dtype(), dict)
        self.__test_equal(result4, expected4, dict)


        #Testing character n-gram functionality
        result5 = sa_character._count_ngrams(3, "character")
        result6 = sa_character._count_ngrams(2, "character")
        result7 = sa_character._count_ngrams(3, "character", to_lower=False)
        result8 = sa_character._count_ngrams(2, "character", to_lower=False)
        result9 = sa_character._count_ngrams(3, "character", to_lower=False, ignore_space=False)
        result10 = sa_character._count_ngrams(2, "character", to_lower=False, ignore_space=False)
        result11 = sa_character._count_ngrams(3, "character", to_lower=True, ignore_space=False)
        result12 = sa_character._count_ngrams(2, "character", to_lower=True, ignore_space=False)
        expected5 = [{'fun': 2, 'nis': 1, 'sfu': 1, 'isf': 1, 'uni': 1}, {'fun': 2, 'nis': 1, 'sfu': 1, 'isf': 1, 'uni': 1}, {}, {'fun': 1}]
        expected6 = [{'ni': 1, 'is': 1, 'un': 2, 'sf': 1, 'fu': 2}, {'ni': 1, 'is': 1, 'un': 2, 'sf': 1, 'fu': 2}, {'fu': 1}, {'un': 1, 'fu': 1}]
        expected7 = [{'sfu': 1, 'Fun': 1, 'uni': 1, 'fun': 1, 'nis': 1, 'isf': 1}, {'sfu': 1, 'Fun': 1, 'uni': 1, 'fun': 1, 'nis': 1, 'isf': 1}, {}, {'fun': 1}]
        expected8 = [{'ni': 1, 'Fu': 1, 'is': 1, 'un': 2, 'sf': 1, 'fu': 1}, {'ni': 1, 'Fu': 1, 'is': 1, 'un': 2, 'sf': 1, 'fu': 1}, {'fu': 1}, {'un': 1, 'fu': 1}]
        expected9 = [{' fu': 1, ' is': 1, 's f': 1, 'un ': 1, 'Fun': 1, 'n i': 1, 'fun': 1, 'is ': 1}, {' fu': 1, ' is': 1, 's f': 1, 'un ': 1, 'Fun': 1, 'n i': 1, 'fun': 1, 'is ': 1}, {}, {'fun': 1}]
        expected10 = [{' f': 1, 'fu': 1, 'n ': 1, 'is': 1, ' i': 1, 'un': 2, 's ': 1, 'Fu': 1}, {' f': 1, 'fu': 1, 'n ': 1, 'is': 1, ' i': 1, 'un': 2, 's ': 1, 'Fu': 1}, {'fu': 1}, {'un': 1, 'fu': 1}]
        expected11 = [{' fu': 1, ' is': 1, 's f': 1, 'un ': 1, 'n i': 1, 'fun': 2, 'is ': 1}, {' fu': 1, ' is': 1, 's f': 1, 'un ': 1, 'n i': 1, 'fun': 2, 'is ': 1}, {}, {'fun': 1}]
        expected12 = [{' f': 1, 'fu': 2, 'n ': 1, 'is': 1, ' i': 1, 'un': 2, 's ': 1}, {' f': 1, 'fu': 2, 'n ': 1, 'is': 1, ' i': 1, 'un': 2, 's ': 1}, {'fu': 1}, {'un': 1, 'fu': 1}]

        self.assertEquals(result5.dtype(), dict)
        self.__test_equal(result5, expected5, dict)
        self.assertEquals(result6.dtype(), dict)
        self.__test_equal(result6, expected6, dict)
        self.assertEquals(result7.dtype(), dict)
        self.__test_equal(result7, expected7, dict)
        self.assertEquals(result8.dtype(), dict)
        self.__test_equal(result8, expected8, dict)
        self.assertEquals(result9.dtype(), dict)
        self.__test_equal(result9, expected9, dict)
        self.assertEquals(result10.dtype(), dict)
        self.__test_equal(result10, expected10, dict)
        self.assertEquals(result11.dtype(), dict)
        self.__test_equal(result11, expected11, dict)
        self.assertEquals(result12.dtype(), dict)
        self.__test_equal(result12, expected12, dict)



        sa = SArray([1, 2, 3])
        with self.assertRaises(TypeError):
            #should fail if the input type is not string
            sa._count_ngrams()

        with self.assertRaises(TypeError):
            #should fail if n is not of type 'int'
            sa_word._count_ngrams(1.01)



        with self.assertRaises(ValueError):
            #should fail with invalid method
            sa_word._count_ngrams(3,"bla")

        with self.assertRaises(ValueError):
            #should fail with n <0
            sa_word._count_ngrams(0)


        with warnings.catch_warnings(record=True) as context:
             sa_word._count_ngrams(10)
             assert len(context) == 1



    def test_dict_keys(self):
        # self.dict_data =  [{str(i): i, i : float(i)} for i in self.int_data]
        sa = SArray(self.dict_data)
        sa_keys = sa.dict_keys()
        self.assertEquals([set(i) for i in sa_keys], [{str(i), i} for i in self.int_data])

        # na value
        d = [{'a': 1}, {None: 2}, {"b": None}, None]
        sa = SArray(d)
        sa_keys = sa.dict_keys()
        self.assertEquals(list(sa_keys), [['a'], [None], ['b'], None])

        #empty SArray
        sa = SArray()
        with self.assertRaises(RuntimeError):
            sa.dict_keys()

        # empty SArray with type
        sa = SArray([], dict)
        self.assertEquals(list(sa.dict_keys().head(10)), [], list)

    def test_dict_values(self):
        # self.dict_data =  [{str(i): i, i : float(i)} for i in self.int_data]
        sa = SArray(self.dict_data)
        sa_values = sa.dict_values()
        self.assertEquals(list(sa_values), [[i, float(i)] for i in self.int_data])

        # na value
        d = [{'a': 1}, {None: 'str'}, {"b": None}, None]
        sa = SArray(d)
        sa_values = sa.dict_values()
        self.assertEquals(list(sa_values), [[1], ['str'], [None], None])

        #empty SArray
        sa = SArray()
        with self.assertRaises(RuntimeError):
            sa.dict_values()

        # empty SArray with type
        sa = SArray([], dict)
        self.assertEquals(list(sa.dict_values().head(10)), [], list)

    def test_dict_trim_by_keys(self):
        # self.dict_data =  [{str(i): i, i : float(i)} for i in self.int_data]
        d = [{'a':1, 'b': [1,2]}, {None: 'str'}, {"b": None, "c": 1}, None]
        sa = SArray(d)
        sa_values = sa.dict_trim_by_keys(['a', 'b'])
        self.assertEquals(list(sa_values), [{}, {None: 'str'}, {"c": 1}, None])

        #empty SArray
        sa = SArray()
        with self.assertRaises(RuntimeError):
            sa.dict_trim_by_keys([])

        sa = SArray([], dict)
        self.assertEquals(list(sa.dict_trim_by_keys([]).head(10)), [], list)

    def test_dict_trim_by_values(self):
        # self.dict_data =  [{str(i): i, i : float(i)} for i in self.int_data]
        d = [{'a':1, 'b': 20, 'c':None}, {"b": 4, None: 5}, None]
        sa = SArray(d)
        sa_values = sa.dict_trim_by_values(5,10)
        self.assertEquals(list(sa_values), [{'c':None}, {None:5}, None])

        # no upper key
        sa_values = sa.dict_trim_by_values(2)
        self.assertEquals(list(sa_values), [{'b': 20, 'c':None}, {"b": 4, None:5}, None])

        # no param
        sa_values = sa.dict_trim_by_values()
        self.assertEquals(list(sa_values), [{'a':1, 'b': 20, 'c':None}, {"b": 4, None: 5}, None])

        # no lower key
        sa_values = sa.dict_trim_by_values(upper=7)
        self.assertEquals(list(sa_values), [{'a':1, 'c':None}, {"b": 4, None: 5}, None])

        #empty SArray
        sa = SArray()
        with self.assertRaises(RuntimeError):
            sa.dict_trim_by_values()

        sa = SArray([], dict)
        self.assertEquals(list(sa.dict_trim_by_values().head(10)), [], list)

    def test_dict_has_any_keys(self):
        d = [{'a':1, 'b': 20, 'c':None}, {"b": 4, None: 5}, None, {'a':0}]
        sa = SArray(d)
        sa_values = sa.dict_has_any_keys([])
        self.assertEquals(list(sa_values), [0,0,None,0])

        sa_values = sa.dict_has_any_keys(['a'])
        self.assertEquals(list(sa_values), [1,0,None,1])

        # one value is auto convert to list
        sa_values = sa.dict_has_any_keys("a")
        self.assertEquals(list(sa_values), [1,0,None,1])

        sa_values = sa.dict_has_any_keys(['a', 'b'])
        self.assertEquals(list(sa_values), [1,1,None,1])

        with self.assertRaises(TypeError):
            sa.dict_has_any_keys()

        #empty SArray
        sa = SArray()
        with self.assertRaises(TypeError):
            sa.dict_has_any_keys()

        sa = SArray([], dict)
        self.assertEquals(list(sa.dict_has_any_keys([]).head(10)), [], list)

    def test_dict_has_all_keys(self):
        d = [{'a':1, 'b': 20, 'c':None}, {"b": 4, None: 5}, None, {'a':0}]
        sa = SArray(d)
        sa_values = sa.dict_has_all_keys([])
        self.assertEquals(list(sa_values), [1,1,None,1])

        sa_values = sa.dict_has_all_keys(['a'])
        self.assertEquals(list(sa_values), [1,0,None,1])

        # one value is auto convert to list
        sa_values = sa.dict_has_all_keys("a")
        self.assertEquals(list(sa_values), [1,0,None,1])

        sa_values = sa.dict_has_all_keys(['a', 'b'])
        self.assertEquals(list(sa_values), [1,0,None,0])

        sa_values = sa.dict_has_all_keys([None, "b"])
        self.assertEquals(list(sa_values), [0,1,None,0])

        with self.assertRaises(TypeError):
            sa.dict_has_all_keys()

        #empty SArray
        sa = SArray()
        with self.assertRaises(TypeError):
            sa.dict_has_all_keys()

        sa = SArray([], dict)
        self.assertEquals(list(sa.dict_has_all_keys([]).head(10)), [], list)

    def test_save_load_cleanup_file(self):
        # simlarly for SArray
        with util.TempDirectory() as f:
            sa = SArray(range(1,1000000))
            sa.save(f)

            # 17 for each sarray, 1 object.bin, 1 ini
            file_count = len(os.listdir(f))
            self.assertTrue(file_count > 2)

            # sf1 now references the on disk file
            sa1 = SArray(f);

            # create another SFrame and save to the same location
            sa2 = SArray([str(i) for i in range(1,100000)])
            sa2.save(f)

            file_count = len(os.listdir(f))
            self.assertTrue(file_count > 2)

            # now sf1 should still be accessible
            self.__test_equal(sa1, list(sa), int)

            # and sf2 is correct too
            sa3 = SArray(f)
            self.__test_equal(sa3, list(sa2), str)

            # when sf1 goes out of scope, the tmp files should be gone
            sa1 = 1
            time.sleep(1)  # give time for the files being deleted
            file_count = len(os.listdir(f))
            self.assertTrue(file_count > 2)

    # list_to_compare must have all unique values for this to work
    def __generic_unique_test(self, list_to_compare):
        test = SArray(list_to_compare + list_to_compare)
        self.assertEquals(sorted(list(test.unique())), sorted(list_to_compare))

    def test_unique(self):
        # Test empty SArray
        test = SArray([])
        self.assertEquals(list(test.unique()), [])

        # Test one value
        test = SArray([1])
        self.assertEquals(list(test.unique()), [1])

        # Test many of one value
        test = SArray([1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])
        self.assertEquals(list(test.unique()), [1])

        # Test all unique values
        test = SArray(self.int_data)
        self.assertEquals(sorted(list(test.unique())), self.int_data)

        # Test an interesting sequence
        interesting_ints = [4654,4352436,5453,7556,45435,4654,5453,4654,5453,1,1,1,5,5,5,8,66,7,7,77,90,-34]
        test = SArray(interesting_ints)
        u = test.unique()
        self.assertEquals(len(u), 13)

        # We do not preserve order
        self.assertEquals(sorted(list(u)), sorted(np.unique(interesting_ints)))

        # Test other types
        self.__generic_unique_test(self.string_data[0:6])

        # only works reliably because these are values that floats can perform
        # reliable equality tests
        self.__generic_unique_test(self.float_data)

        self.__generic_unique_test(self.list_data)
        self.__generic_unique_test(self.vec_data)

        with self.assertRaises(TypeError):
            SArray(self.dict_data).unique()

    def test_item_len(self):
        # empty SArray
        test = SArray([])
        with self.assertRaises(TypeError):
            self.assertEquals(test.item_length())

        # wrong type
        test = SArray([1,2,3])
        with self.assertRaises(TypeError):
            self.assertEquals(test.item_length())

        test = SArray(['1','2','3'])
        with self.assertRaises(TypeError):
            self.assertEquals(test.item_length())

        # vector type
        test = SArray([[], [1], [1,2], [1,2,3], None])
        item_length = test.item_length();
        self.assertEquals(list(item_length), list([0, 1,2,3,None]))

        # dict type
        test = SArray([{}, {'key1': 1}, {'key2':1, 'key1':2}, None])
        self.assertEquals(list(test.item_length()), list([0, 1,2,None]))

        # list type
        test = SArray([[], [1,2], ['str', 'str2'], None])
        self.assertEquals(list(test.item_length()), list([0, 2,2,None]))

    def test_random_access(self):
        t = list(range(0,100000))
        s = SArray(t)
        # simple slices
        self.__test_equal(s[1:10000], t[1:10000], int)
        self.__test_equal(s[0:10000:3], t[0:10000:3], int)
        self.__test_equal(s[1:10000:3], t[1:10000:3], int)
        self.__test_equal(s[2:10000:3], t[2:10000:3], int)
        self.__test_equal(s[3:10000:101], t[3:10000:101], int)
        # negative slices
        self.__test_equal(s[-5:], t[-5:], int)
        self.__test_equal(s[-1:], t[-1:], int)
        self.__test_equal(s[-100:-10], t[-100:-10], int)
        self.__test_equal(s[-100:-10:2], t[-100:-10:2], int)
        # single element reads
        self.assertEquals(s[511], t[511])
        self.assertEquals(s[1912], t[1912])
        self.assertEquals(s[-1], t[-1])
        self.assertEquals(s[-10], t[-10])

        # A cache boundary
        self.assertEquals(s[32*1024-1], t[32*1024-1])
        self.assertEquals(s[32*1024], t[32*1024])

        # totally different
        self.assertEquals(s[19312], t[19312])

        # edge case odities
        self.__test_equal(s[10:100:100], t[10:100:100], int)
        self.__test_equal(s[-100:len(s):10], t[-100:len(t):10], int)
        self.__test_equal(s[-1:-2], t[-1:-2], int)
        self.__test_equal(s[-1:-1000:2], t[-1:-1000:2], int)
        with self.assertRaises(IndexError):
            s[len(s)]

        # with caching abilities; these should be fast, as 32K
        # elements are cached.
        for i in range(0, 100000, 100):
            self.assertEquals(s[i], t[i])
        for i in range(0, 100000, 100):
            self.assertEquals(s[-i], t[-i])

    def test_sort(self):
        test = SArray([1,2,3,5,1,4])
        ascending = SArray([1,1,2,3,4,5])
        descending = SArray([5,4,3,2,1,1])
        result = test.sort()
        self.assertEqual(list(result), list(ascending))
        result = test.sort(ascending = False)
        self.assertEqual(list(result), list(descending))

        with self.assertRaises(TypeError):
            SArray([[1,2], [2,3]]).sort()

    def test_unicode_encode_should_not_fail(self):
        g=SArray([{'a':u'\u2019'}])
        g=SArray([u'123',u'\u2019'])
        g=SArray(['123',u'\u2019'])

    def test_read_from_avro(self):
        encoded_data = 'T2JqAQQWYXZyby5zY2hlbWHsBXsiZmllbGRzIjogW3sidHlwZSI6ICJzdHJpbmciLCAibmFtZSI6ICJidXNpbmVzc19pZCJ9LCB7InR5cGUiOiAic3RyaW5nIiwgIm5hbWUiOiAiZGF0ZSJ9LCB7InR5cGUiOiAic3RyaW5nIiwgIm5hbWUiOiAicmV2aWV3X2lkIn0sIHsidHlwZSI6ICJpbnQiLCAibmFtZSI6ICJzdGFycyJ9LCB7InR5cGUiOiAic3RyaW5nIiwgIm5hbWUiOiAidGV4dCJ9LCB7InR5cGUiOiAic3RyaW5nIiwgIm5hbWUiOiAidHlwZSJ9LCB7InR5cGUiOiAic3RyaW5nIiwgIm5hbWUiOiAidXNlcl9pZCJ9LCB7InR5cGUiOiB7InR5cGUiOiAibWFwIiwgInZhbHVlcyI6ICJpbnQifSwgIm5hbWUiOiAidm90ZXMifV0sICJ0eXBlIjogInJlY29yZCIsICJuYW1lIjogInJldmlldyJ9FGF2cm8uY29kZWMIbnVsbAAON5ELIy6PokglPEeciZP7BOggLHNnQmwzVURFY05ZS3d1VWI5MkNZZEEUMjAwOS0wMS0yNSxaai1SMFpacUlLRng1NkxZMnN1MWlRCIAZVGhlIG93bmVyIG9mIENoaW5hIEtpbmcgaGFkIG5ldmVyIGhlYXJkIG9mIFllbHAuLi51bnRpbCBKaW0gVyByb2xsZWQgdXAgb24gQ2hpbmEgS2luZyEKClRoZSBvd25lciBvZiBDaGluYSBLaW5nLCBNaWNoYWVsLCBpcyB2ZXJ5IGZyaWVuZGx5IGFuZCBjaGF0dHkuICBCZSBQcmVwYXJlZCB0byBjaGF0IGZvciBhIGZldyBtaW51dGVzIGlmIHlvdSBzdHJpa2UgdXAgYSBjb252ZXJzYXRpb24uCgpUaGUgc2VydmljZSBoZXJlIHdhcyB0ZXJyaWZpYy4gIFdlIGhhZCBzZXZlcmFsIHBlb3BsZSBmdXNzaW5nIG92ZXIgdXMgYnV0IHRoZSBwcmltYXJ5IHNlcnZlciwgTWFnZ2llIHdhcyBhIGdlbS4gIAoKTXkgd2lmZSBhbmQgdGhlIGtpZHMgb3B0ZWQgZm9yIHRoZSBBbWVyaWNhbml6ZWQgbWVudSBhbmQgd2VudCB3aXRoIHNwZWNpYWxzIGxpa2Ugc3dlZXQgYW5kIHNvdXIgY2hpY2tlbiwgc2hyaW1wIGluIHdoaXRlIHNhdWNlIGFuZCBnYXJsaWMgYmVlZi4gIEVhY2ggY2FtZSBjYW1lIHdpdGggc291cCwgZWdnIHJvbGwgYW5kIHJpY2UuICBJIHNhbXBsZWQgdGhlIGdhcmxpYyBiZWVmIHdoaWNoIHRoZXkgcHJlcGFyZWQgd2l0aCBhIGt1bmcgcGFvIGJyb3duIHNhdWNlIChhIGRlY2lzaW9uIE1hZ2dpZSBhbmQgbXkgd2lmZSBhcnJpdmVkIGF0IGFmdGVyIHNldmVyYWwgbWludXRlcyBvZiBkaXNjdXNzaW9uKSBpdCBoYWQgYSBuaWNlIHJvYnVzdCBmbGF2b3IgYW5kIHRoZSB2ZWdnaWVzIHdlcmUgZnJlc2ggYW5kIGZsYXZvcmZ1bC4gIEkgIGFsc28gc2FtcGxlZCB0aGUgc2hyaW1wIHdoaWNoIHdlcmUgc3VjY3VsZW50IGFuZCB0aGUgd2hpdGUgc2F1Y2UgaGFkIGEgbGl0dGxlIG1vcmUgZGlzdGluY3RpdmVuZXNzIHRvIGl0IHRoYW4gdGhlIHNhbWUgc2F1Y2UgYXQgbWFueSBDaGluZXNlIHJlc3RhdXJhbnRzLgoKSSBvcmRlcmVkIGZyb20gdGhlIHRyYWRpdGlvbmFsIG1lbnUgYnV0IHdlbnQgbm90IHRvbyBhZHZlbnR1cm91cyB3aXRoIHNpenpsaW5nIHBsYXRlIHdpdGggc2NhbGxvcHMgYW5kIHNocmltcCBpbiBibGFjayBwZXBwZXIgc2F1Y2UuICBWZXJ5IGVuam95YWJsZS4gIEFnYWluLCBzdWNjdWxlbnQgc2hyaW1wLiAgVGhlIHNjYWxsb3BzIHdlcmUgdGFzdHkgYXMgd2VsbC4gIFJlYWxpemluZyB0aGF0IEkgbW92ZWQgaGVyZSBmcm9tIEJvc3RvbiBhbmQgSSBnbyBpbnRvIGFueSBzZWFmb29kIGV4cGVyaWVuY2Ugd2l0aCBkaW1pbmlzaGVkIGV4cGVjdGF0aW9ucyBub3cgdGhhdCBJIGxpdmUgaW4gdGhlIHdlc3QsIEkgaGF2ZSB0byBzYXkgdGhlIHNjYWxsb3BzIGFyZSBhbW9uZyB0aGUgZnJlc2hlciBhbmQganVkaWNpb3VzbHkgcHJlcGFyZWQgdGhhdCBJIGhhdmUgaGFkIGluIFBob2VuaXguCgpPdmVyYWxsIENoaW5hIEtpbmcgZGVsaXZlcmVkIGEgdmVyeSB0YXN0eSBhbmQgdmVyeSBmcmVzaCBtZWFsLiAgVGhleSBoYXZlIGEgZmFpcmx5IGV4dGVuc2l2ZSB0cmFkaXRpb25hbCBtZW51IHdoaWNoIEkgbG9vayBmb3J3YXJkIHRvIGV4cGxvcmluZyBmdXJ0aGVyLgoKVGhhbmtzIHRvIENocmlzdGluZSBPIGZvciBoZXIgcmV2aWV3Li4uYWZ0ZXIgcmVhZGluZyB0aGF0IEkga25ldyBDaGluYSBLaW5nIHdhcyBBLU9LLgxyZXZpZXcsUDJrVms0Y0lXeUs0ZTRoMTRSaEstUQYKZnVubnkIDHVzZWZ1bBIIY29vbA4ALGFyS2NrTWY3bEdOWWpYaktvNkRYY0EUMjAxMi0wNS0wNSxFeVZmaFJEbHlpcDJFcktNT0hFQS1BCKQEV2UndmUgYmVlbiBoZXJlIGEgZmV3IHRpbWVzIGFuZCB3ZSBsb3ZlIGFsbCB0aGUgZnJlc2ggaW5ncmVkaWVudHMuIFRoZSBwaXp6YSBpcyBnb29kIHdoZW4geW91IGVhdCBpdCBmcmVzaCBidXQgaWYgeW91IGxpa2UgdG8gZWF0IHlvdXIgcGl6emEgY29sZCB0aGVuIHlvdSdsbCBiZSBiaXRpbmcgaW50byBoYXJkIGRvdWdoLiBUaGVpciBOdXRlbGxhIHBpenphIGlzIGdvb2QuIFRha2UgYSBtZW51IGFuZCBjaGVjayBvdXQgdGhlaXIgbWVudSBhbmQgaG91cnMgZm9yIHNwZWNpYWxzLgxyZXZpZXcseDFZbDFkcE5jV0NDRWRwTUU5ZGcwZwYKZnVubnkCDHVzZWZ1bAIIY29vbAAADjeRCyMuj6JIJTxHnImT+w=='
 
        test_avro_file = open("test.avro", "wb")
        test_avro_file.write(binascii.a2b_base64(encoded_data))
        test_avro_file.close()
        sa = SArray.from_avro("test.avro")
        self.assertEqual(sa.dtype(), dict)
        self.assertEqual(len(sa), 2)

    def test_from_const(self):
        g = SArray.from_const('a', 100)
        self.assertEqual(len(g), 100)
        self.assertEqual(list(g), ['a']*100)
        g = SArray.from_const(dt.datetime(2013, 5, 7, 10, 4, 10),10)
        self.assertEqual(len(g), 10)
        self.assertEqual(list(g), [dt.datetime(2013, 5, 7, 10, 4, 10)]*10)
        g = SArray.from_const(0, 0)
        self.assertEqual(len(g), 0)

        g = SArray.from_const(None, 100)
        self.assertEquals(list(g), [None] * 100)
        self.assertEqual(g.dtype(), float)

        g = SArray.from_const(None, 100, str)
        self.assertEquals(list(g), [None] * 100)
        self.assertEqual(g.dtype(), str)

        g = SArray.from_const(0, 100, float)
        self.assertEquals(list(g), [0.0] * 100)
        self.assertEqual(g.dtype(), float)

        g = SArray.from_const(0.0, 100, int)
        self.assertEquals(list(g), [0] * 100)
        self.assertEqual(g.dtype(), int)

        g = SArray.from_const(None, 100, float)
        self.assertEquals(list(g), [None] * 100)
        self.assertEqual(g.dtype(), float)

        g = SArray.from_const(None, 100, int)
        self.assertEquals(list(g), [None] * 100)
        self.assertEqual(g.dtype(), int)

        g = SArray.from_const(None, 100, list)
        self.assertEquals(list(g), [None] * 100)
        self.assertEqual(g.dtype(), list)

        g = SArray.from_const([1], 100, list)
        self.assertEquals(list(g), [[1]] * 100)
        self.assertEqual(g.dtype(), list)

    def test_from_sequence(self):
        with self.assertRaises(TypeError):
            g = SArray.from_sequence()
        g = SArray.from_sequence(100)
        self.assertEqual(list(g), list(range(100)))
        g = SArray.from_sequence(10, 100)
        self.assertEqual(list(g), list(range(10, 100)))
        g = SArray.from_sequence(100, 10)
        self.assertEqual(list(g), list(range(100, 10)))

    def test_datetime(self):
        sa = SArray(self.datetime_data)
        self.__test_equal(sa ,self.datetime_data,dt.datetime)
        sa = SArray(self.datetime_data2)
        self.__test_equal(sa ,self.datetime_data2,dt.datetime)

        ret = sa.split_datetime(limit=['year','month','day','hour','minute',
            'second','us','weekday', 'isoweekday','tmweekday'])
        self.assertEqual(ret.num_cols(), 10)
        self.__test_equal(ret['X.year'] , [2013, 1902, None], int)
        self.__test_equal(ret['X.month'] , [5, 10, None], int)
        self.__test_equal(ret['X.day'] , [7, 21, None], int)
        self.__test_equal(ret['X.hour'] , [10, 10, None], int)
        self.__test_equal(ret['X.minute'] , [4, 34, None], int)
        self.__test_equal(ret['X.second'] , [10, 10, None], int)
        self.__test_equal(ret['X.us'] , [109321, 991111, None], int)
        self.__test_equal(ret['X.weekday'] , [1, 1, None], int)
        self.__test_equal(ret['X.isoweekday'] , [2, 2, None], int)
        self.__test_equal(ret['X.tmweekday'] , [2, 2, None], int)

    def test_datetime_difference(self):
        sa = SArray(self.datetime_data)
        sa2 = SArray(self.datetime_data2)
        res = sa2 - sa
        expected = [float(x.microsecond) / 1000000.0 if x is not None else x for x in self.datetime_data2]
        self.assertEqual(len(res), len(expected))
        for i in range(len(res)):
            if res[i] == None:
                self.assertEqual(res[i], expected[i])
            else:
                self.assertAlmostEqual(res[i], expected[i], places=6)

    def test_datetime_lambda(self):
        data = [dt.datetime(2013, 5, 7, 10, 4, 10, 109321),
                dt.datetime(1902, 10, 21, 10, 34, 10, 991111,
                    tzinfo=GMT(1))]
        g=SArray(data)
        gstr=g.apply(lambda x:str(x))
        self.__test_equal(gstr, [str(x) for x in g], str)
        gident=g.apply(lambda x:x)
        self.__test_equal(gident, list(g), dt.datetime)

    def test_datetime_to_str(self):
        sa = SArray(self.datetime_data)
        sa_string_back = sa.datetime_to_str()

        self.__test_equal(sa_string_back,['2013-05-07T10:04:10', '1902-10-21T10:34:10GMT+00', None],str)

        sa = SArray([None,None,None],dtype=dt.datetime)
        sa_string_back = sa.datetime_to_str()

        self.__test_equal(sa_string_back,[None,None,None],str)

        sa = SArray(dtype=dt.datetime)
        sa_string_back = sa.datetime_to_str()

        self.__test_equal(sa_string_back,[],str)

        sa = SArray([None,None,None])
        self.assertRaises(TypeError,sa.datetime_to_str)

        sa = SArray()
        self.assertRaises(TypeError,sa.datetime_to_str)



    def test_str_to_datetime(self):
        sa_string = SArray(['2013-05-07T10:04:10', '1902-10-21T10:34:10GMT+00', None])
        sa_datetime_back = sa_string.str_to_datetime()

        expected = self.datetime_data

        self.__test_equal(sa_datetime_back,expected,dt.datetime)

        sa_string = SArray([None,None,None],str)
        sa_datetime_back = sa_string.str_to_datetime()

        self.__test_equal(sa_datetime_back,[None,None,None],dt.datetime)

        sa_string = SArray(dtype=str)
        sa_datetime_back = sa_string.str_to_datetime()

        self.__test_equal(sa_datetime_back,[],dt.datetime)

        sa = SArray([None,None,None])
        self.assertRaises(TypeError,sa.str_to_datetime)


        sa = SArray()
        self.assertRaises(TypeError,sa.str_to_datetime)

        # hour without leading zero
        sa = SArray(['10/30/2014 9:01'])
        sa = sa.str_to_datetime('%m/%d/%Y %H:%M')
        expected = [dt.datetime(2014, 10, 30, 9, 1)]
        self.__test_equal(sa,expected,dt.datetime)

        # without delimiters
        sa = SArray(['10302014 0901', '10302014 2001'])
        sa = sa.str_to_datetime('%m%d%Y %H%M')
        expected = [dt.datetime(2014, 10, 30, 9, 1),
                    dt.datetime(2014, 10, 30, 20, 1)]
        self.__test_equal(sa,expected,dt.datetime)

        # another without delimiter test
        sa = SArray(['20110623T191001'])
        sa = sa.str_to_datetime("%Y%m%dT%H%M%S%F%q")
        expected = [dt.datetime(2011, 6, 23, 19, 10, 1)]
        self.__test_equal(sa,expected,dt.datetime)

        # am pm
        sa = SArray(['10/30/2014 9:01am', '10/30/2014 9:01pm'])
        sa = sa.str_to_datetime('%m/%d/%Y %H:%M%p')
        expected = [dt.datetime(2014, 10, 30, 9, 1),
                    dt.datetime(2014, 10, 30, 21, 1)]
        self.__test_equal(sa,expected,dt.datetime)

        sa = SArray(['10/30/2014 9:01AM', '10/30/2014 9:01PM'])
        sa = sa.str_to_datetime('%m/%d/%Y %H:%M%P')
        expected = [dt.datetime(2014, 10, 30, 9, 1),
                    dt.datetime(2014, 10, 30, 21, 1)]
        self.__test_equal(sa,expected,dt.datetime)

        # failure 13pm
        sa = SArray(['10/30/2014 13:01pm'])
        with self.assertRaises(RuntimeError):
            sa.str_to_datetime('%m/%d/%Y %H:%M%p')

        # failure hour 13 when %l should only have up to hour 12
        sa = SArray(['10/30/2014 13:01'])
        with self.assertRaises(RuntimeError):
            sa.str_to_datetime('%m/%d/%Y %l:%M')

        with self.assertRaises(RuntimeError):
            sa.str_to_datetime('%m/%d/%Y %L:%M')

        sa = SArray(['2013-05-07T10:04:10',
            '1902-10-21T10:34:10UTC+05:45'])
        expected = [dt.datetime(2013, 5, 7, 10, 4, 10),
                dt.datetime(1902, 10, 21, 10, 34, 10).replace(tzinfo=GMT(5.75))]
        self.__test_equal(sa.str_to_datetime() ,expected,dt.datetime)




    def test_apply_with_partial(self):
        sa = SArray([1, 2, 3, 4, 5])

        def concat_fn(character, number):
            return '%s%d' % (character, number)

        my_partial_fn = functools.partial(concat_fn, 'x')
        sa_transformed = sa.apply(my_partial_fn)
        self.assertEqual(list(sa_transformed), ['x1', 'x2', 'x3', 'x4', 'x5'])

    def test_apply_with_functor(self):
        sa = SArray([1, 2, 3, 4, 5])

        class Concatenator(object):
            def __init__(self, character):
                self.character = character

            def __call__(self, number):
                return '%s%d' % (self.character, number)

        concatenator = Concatenator('x')
        sa_transformed = sa.apply(concatenator)
        self.assertEqual(list(sa_transformed), ['x1', 'x2', 'x3', 'x4', 'x5'])

    def test_argmax_argmin(self):
        sa = SArray([1,4,-1,10,3,5,8])
        index = [sa.argmax(),sa.argmin()]
        expected = [3,2]
        self.assertEqual(index,expected)

        sa = SArray([1,4.3,-1.4,0,3,5.6,8.9])
        index = [sa.argmax(),sa.argmin()]
        expected = [6,2]
        self.assertEqual(index,expected)

        #empty case
        sa = SArray([])
        index = [sa.argmax(),sa.argmin()]
        expected = [None,None]
        self.assertEqual(index,expected)

        # non-numeric type
        sa = SArray(["434","43"])
        with self.assertRaises(TypeError):
            sa.argmax()

        with self.assertRaises(TypeError):
            sa.argmin()

    def test_apply_with_recursion(self):
        sa = SArray(range(1000))
        sastr = sa.astype(str)
        rets = sa.apply(lambda x:sastr[x])
        self.assertEqual(list(rets), list(sastr))

    def test_save_sarray(self):
        '''save lazily evaluated SArray should not matrialize to target folder
        '''
        data = SArray(range(1000))
        data = data[data > 50]
        #lazy and good
        tmp_dir = tempfile.mkdtemp()
        data.save(tmp_dir)
        shutil.rmtree(tmp_dir)
        print(data)

    def test_to_numpy(self):
        X = SArray(range(100))
        import numpy as np
        import numpy.testing as nptest
        Y = np.array(range(100))
        nptest.assert_array_equal(X.to_numpy(), Y)

        X = X.astype(str)
        Y = np.array([str(i) for i in range(100)])
        nptest.assert_array_equal(X.to_numpy(), Y)

    def test_rolling_mean(self):
        data = SArray(range(1000))
        neg_data = SArray(range(-100,100,2))

        ### Small backward window including current
        res = data.rolling_mean(-3,0)
        expected = [None for i in range(3)] + [i + .5 for i in range(1,998)]
        self.__test_equal(res,expected,float)

        # Test float inputs as well
        res = data.astype(float).rolling_mean(-3,0)
        self.__test_equal(res,expected,float)

        # Test min observations
        res = data.rolling_mean(-3, 0, min_observations=5)
        self.__test_equal(res,expected,float)

        res = data.rolling_mean(-3, 0, min_observations=4)
        self.__test_equal(res,expected,float)

        res = data.rolling_mean(-3, 0, min_observations=3)
        expected[2] = 1.0
        self.__test_equal(res,expected,float)

        res = data.rolling_mean(-3, 0, min_observations=2)
        expected[1] = 0.5
        self.__test_equal(res,expected,float)

        res = data.rolling_mean(-3, 0, min_observations=1)
        expected[0] = 0.0
        self.__test_equal(res,expected,float)

        res = data.rolling_mean(-3, 0, min_observations=0)
        self.__test_equal(res,expected,float)

        with self.assertRaises(ValueError):
            res = data.rolling_mean(-3,0,min_observations=-1)

        res = neg_data.rolling_mean(-3,0)
        expected = [None for i in range(3)] + [float(i) for i in range(-97,96,2)]
        self.__test_equal(res,expected,float)

        # Test float inputs as well
        res = neg_data.astype(float).rolling_mean(-3,0)
        self.__test_equal(res,expected,float)

        # Test vector input
        res = SArray(self.vec_data).rolling_mean(-3,0)
        expected = [None for i in range(3)] + [array.array('d',[i+.5, i+1.5]) for i in range(2,9)]
        self.__test_equal(res,expected,array.array)

        ### Small forward window including current
        res = data.rolling_mean(0,4)
        expected = [float(i) for i in range(2,998)] + [None for i in range(4)]
        self.__test_equal(res,expected,float)

        res = neg_data.rolling_mean(0,4)
        expected = [float(i) for i in range(-96,95,2)] + [None for i in range(4)]
        self.__test_equal(res,expected,float)

        ### Small backward window not including current
        res = data.rolling_mean(-5,-1)
        expected = [None for i in range(5)] + [float(i) for i in range(2,997)]
        self.__test_equal(res,expected,float)

        res = neg_data.rolling_mean(-5,-1)
        expected = [None for i in range(5)] + [float(i) for i in range(-96,94,2)]
        self.__test_equal(res,expected,float)

        ### Small forward window not including current
        res = data.rolling_mean(1,5)
        expected = [float(i) for i in range(3,998)] + [None for i in range(5)]
        self.__test_equal(res,expected,float)

        res = neg_data.rolling_mean(1,5)
        expected = [float(i) for i in range(-94,96,2)] + [None for i in range(5)]
        self.__test_equal(res,expected,float)

        ### "Centered" rolling aggregate
        res = data.rolling_mean(-2,2)
        expected = [None for i in range(2)] + [float(i) for i in range(2,998)] + [None for i in range(2)]
        self.__test_equal(res,expected,float)

        res = neg_data.rolling_mean(-2,2)
        expected = [None for i in range(2)] + [float(i) for i in range(-96,96,2)] + [None for i in range(2)]
        self.__test_equal(res,expected,float)

        ### Lopsided rolling aggregate
        res = data.rolling_mean(-2,1)
        expected = [None for i in range(2)] + [i + .5 for i in range(1,998)] + [None for i in range(1)]
        self.__test_equal(res,expected,float)

        res = neg_data.rolling_mean(-2,1)
        expected = [None for i in range(2)] + [float(i) for i in range(-97,97,2)] + [None for i in range(1)]
        self.__test_equal(res,expected,float)

        ### A very forward window
        res = data.rolling_mean(500,502)
        expected = [float(i) for i in range(501,999)] + [None for i in range(502)]
        self.__test_equal(res,expected,float)

        res = neg_data.rolling_mean(50,52)
        expected = [float(i) for i in range(2,98,2)] + [None for i in range(52)]
        self.__test_equal(res,expected,float)

        ### A very backward window
        res = data.rolling_mean(-502,-500)
        expected = [None for i in range(502)] + [float(i) for i in range(1,499)]
        self.__test_equal(res,expected,float)

        res = neg_data.rolling_mean(-52,-50)
        expected = [None for i in range(52)] + [float(i) for i in range(-98,-2,2)]
        self.__test_equal(res,expected,float)

        ### A window size much larger than anticipated segment size
        res = data.rolling_mean(0,749)
        expected = [i + .5 for i in range(374,625)] + [None for i in range(749)]
        self.__test_equal(res,expected,float)

        ### A window size larger than the array
        res = data.rolling_mean(0,1000)
        expected = [None for i in range(1000)]
        self.__test_equal(res,expected,type(None))

        ### A window size of 1
        res = data.rolling_mean(0,0)
        self.__test_equal(res, list(data), float)

        res = data.rolling_mean(-2,-2)
        expected = [None for i in range(2)] + list(data[0:998])
        self.__test_equal(res, expected, float)

        res = data.rolling_mean(3,3)
        expected = list(data[3:1000]) + [None for i in range(3)]
        self.__test_equal(res, expected, float)

        ### A negative window size
        with self.assertRaises(RuntimeError):
            res = data.rolling_mean(4,2)

        ### Non-numeric
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.string_data).rolling_mean(0,1)

        ### Empty SArray
        sa = SArray()
        res = sa.rolling_mean(0,1)
        self.__test_equal(res, [], type(None))

        ### Small SArray
        sa = SArray([1,2,3])
        res = sa.rolling_mean(0,1)
        self.__test_equal(res, [1.5,2.5,None], float)

    def test_rolling_sum(self):
        data = SArray(range(1000))
        neg_data = SArray(range(-100,100,2))

        ### Small backward window including current
        res = data.rolling_sum(-3,0)
        expected = [None for i in range(3)] + [i for i in range(6,3994,4)]
        self.__test_equal(res,expected,int)

        # Test float inputs as well
        res = data.astype(float).rolling_sum(-3,0)
        self.__test_equal(res,expected,float)

        # Test min observations
        res = data.rolling_sum(-3, 0, min_observations=5)
        self.__test_equal(res,expected,int)

        res = data.rolling_sum(-3, 0, min_observations=4)
        self.__test_equal(res,expected,int)

        res = data.rolling_sum(-3, 0, min_observations=3)
        expected[2] = 3
        self.__test_equal(res,expected,int)

        res = data.rolling_sum(-3, 0, min_observations=2)
        expected[1] = 1
        self.__test_equal(res,expected,int)

        res = data.rolling_sum(-3, 0, min_observations=1)
        expected[0] = 0
        self.__test_equal(res,expected,int)

        res = data.rolling_sum(-3, 0, min_observations=0)
        self.__test_equal(res,expected,int)

        with self.assertRaises(ValueError):
            res = data.rolling_sum(-3,0,min_observations=-1)

        res = neg_data.rolling_sum(-3,0)
        expected = [None for i in range(3)] + [i for i in range(-388,388,8)]
        self.__test_equal(res,expected,int)

        # Test float inputs as well
        res = neg_data.astype(float).rolling_sum(-3,0)
        self.__test_equal(res,expected,float)

        # Test vector input
        res = SArray(self.vec_data).rolling_sum(-3,0)
        expected = [None for i in range(3)] + [array.array('d',[i, i+4]) for i in range(10,38,4)]
        self.__test_equal(res,expected,array.array)

        ### Small forward window including current
        res = data.rolling_sum(0,4)
        expected = [i for i in range(10,4990,5)] + [None for i in range(4)]
        self.__test_equal(res,expected,int)

        res = neg_data.rolling_sum(0,4)
        expected = [i for i in range(-480,480,10)] + [None for i in range(4)]
        self.__test_equal(res,expected,int)

        ### Small backward window not including current
        res = data.rolling_sum(-5,-1)
        expected = [None for i in range(5)] + [i for i in range(10,4985,5)]
        self.__test_equal(res,expected,int)

        res = neg_data.rolling_sum(-5,-1)
        expected = [None for i in range(5)] + [i for i in range(-480,470,10)]
        self.__test_equal(res,expected,int)

        ### Small forward window not including current
        res = data.rolling_sum(1,5)
        expected = [i for i in range(15,4990,5)] + [None for i in range(5)]
        self.__test_equal(res,expected,int)

        res = neg_data.rolling_sum(1,5)
        expected = [i for i in range(-470,480,10)] + [None for i in range(5)]
        self.__test_equal(res,expected,int)

        ### "Centered" rolling aggregate
        res = data.rolling_sum(-2,2)
        expected = [None for i in range(2)] + [i for i in range(10,4990,5)] + [None for i in range(2)]
        self.__test_equal(res,expected,int)

        res = neg_data.rolling_sum(-2,2)
        expected = [None for i in range(2)] + [i for i in range(-480,480,10)] + [None for i in range(2)]
        self.__test_equal(res,expected,int)

        ### Lopsided rolling aggregate
        res = data.rolling_sum(-2,1)
        expected = [None for i in range(2)] + [i for i in range(6,3994,4)] + [None for i in range(1)]
        self.__test_equal(res,expected,int)

        res = neg_data.rolling_sum(-2,1)
        expected = [None for i in range(2)] + [i for i in range(-388,388,8)] + [None for i in range(1)]
        self.__test_equal(res,expected,int)

        ### A very forward window
        res = data.rolling_sum(500,502)
        expected = [i for i in range(1503,2997,3)] + [None for i in range(502)]
        self.__test_equal(res,expected,int)

        res = neg_data.rolling_sum(50,52)
        expected = [i for i in range(6,294,6)] + [None for i in range(52)]
        self.__test_equal(res,expected,int)

        ### A very backward window
        res = data.rolling_sum(-502,-500)
        expected = [None for i in range(502)] + [i for i in range(3,1497,3)]
        self.__test_equal(res,expected,int)

        res = neg_data.rolling_sum(-52,-50)
        expected = [None for i in range(52)] + [i for i in range(-294,-6,6)]
        self.__test_equal(res,expected,int)

        ### A window size much larger than anticipated segment size
        res = data.rolling_sum(0,749)
        expected = [i for i in range(280875,469125,750)] + [None for i in range(749)]
        self.__test_equal(res,expected,int)

        ### A window size larger than the array
        res = data.rolling_sum(0,1000)
        expected = [None for i in range(1000)]
        self.__test_equal(res,expected,type(None))

        ### A window size of 1
        res = data.rolling_sum(0,0)
        self.__test_equal(res, list(data), int)

        res = data.rolling_sum(-2,-2)
        expected = [None for i in range(2)] + list(data[0:998])
        self.__test_equal(res, expected, int)

        res = data.rolling_sum(3,3)
        expected = list(data[3:1000]) + [None for i in range(3)]
        self.__test_equal(res, expected, int)

        ### A negative window size
        with self.assertRaises(RuntimeError):
            res = data.rolling_sum(4,2)

        ### Non-numeric
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.string_data).rolling_sum(0,1)

        ### Empty SArray
        sa = SArray()
        res = sa.rolling_sum(0,1)
        self.__test_equal(res, [], type(None))

        ### Small SArray
        sa = SArray([1,2,3])
        res = sa.rolling_sum(0,1)
        self.__test_equal(res, [3,5,None], int)

    def test_rolling_max(self):
        data = SArray(range(1000))

        ### Small backward window including current
        res = data.rolling_max(-3,0)
        expected = [None for i in range(3)] + [i for i in range(3,1000)]
        self.__test_equal(res,expected,int)

        # Test float inputs as well
        res = data.astype(float).rolling_max(-3,0)
        self.__test_equal(res,expected,float)

        # Test min observations
        res = data.rolling_max(-3, 0, min_observations=5)
        self.__test_equal(res,expected,int)

        res = data.rolling_max(-3, 0, min_observations=4)
        self.__test_equal(res,expected,int)

        res = data.rolling_max(-3, 0, min_observations=3)
        expected[2] = 2
        self.__test_equal(res,expected,int)

        with self.assertRaises(ValueError):
            res = data.rolling_max(-3,0,min_observations=-1)

        # Test vector input
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.vec_data).rolling_max(-3,0)

        ### Small forward window including current
        res = data.rolling_max(0,4)
        expected = [float(i) for i in range(4,1000)] + [None for i in range(4)]
        self.__test_equal(res,expected,int)

        ### A window size of 1
        res = data.rolling_max(0,0)
        self.__test_equal(res, list(data), int)

        res = data.rolling_max(-2,-2)
        expected = [None for i in range(2)] + list(data[0:998])
        self.__test_equal(res, expected, int)

        res = data.rolling_max(3,3)
        expected = list(data[3:1000]) + [None for i in range(3)]
        self.__test_equal(res, expected, int)

        ### A negative window size
        with self.assertRaises(RuntimeError):
            res = data.rolling_max(4,2)

        ### Non-numeric
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.string_data).rolling_max(0,1)

        ### Empty SArray
        sa = SArray()
        res = sa.rolling_max(0,1)
        self.__test_equal(res, [], type(None))

        ### Small SArray
        sa = SArray([1,2,3])
        res = sa.rolling_max(0,1)
        self.__test_equal(res, [2,3,None], int)

    def test_rolling_min(self):
        data = SArray(range(1000))

        ### Small backward window including current
        res = data.rolling_min(-3,0)
        expected = [None for i in range(3)] + [i for i in range(0,997)]
        self.__test_equal(res,expected,int)

        # Test float inputs as well
        res = data.astype(float).rolling_min(-3,0)
        self.__test_equal(res,expected,float)

        # Test min observations
        res = data.rolling_min(-3, 0, min_observations=5)
        self.__test_equal(res,expected,int)

        res = data.rolling_min(-3, 0, min_observations=4)
        self.__test_equal(res,expected,int)

        res = data.rolling_min(-3, 0, min_observations=3)
        expected[2] = 0
        self.__test_equal(res,expected,int)

        with self.assertRaises(ValueError):
            res = data.rolling_min(-3,0,min_observations=-1)

        # Test vector input
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.vec_data).rolling_min(-3,0)

        ### Small forward window including current
        res = data.rolling_min(0,4)
        expected = [float(i) for i in range(0,996)] + [None for i in range(4)]
        self.__test_equal(res,expected,int)

        ### A window size of 1
        res = data.rolling_min(0,0)
        self.__test_equal(res, list(data), int)

        res = data.rolling_min(-2,-2)
        expected = [None for i in range(2)] + list(data[0:998])
        self.__test_equal(res, expected, int)

        res = data.rolling_min(3,3)
        expected = list(data[3:1000]) + [None for i in range(3)]
        self.__test_equal(res, expected, int)

        ### A negative window size
        with self.assertRaises(RuntimeError):
            res = data.rolling_min(4,2)

        ### Non-numeric
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.string_data).rolling_min(0,1)

        ### Empty SArray
        sa = SArray()
        res = sa.rolling_min(0,1)
        self.__test_equal(res, [], type(None))

        ### Small SArray
        sa = SArray([1,2,3])
        res = sa.rolling_min(0,1)
        self.__test_equal(res, [1,2,None], int)

    def test_rolling_var(self):
        data = SArray(range(1000))

        ### Small backward window including current
        res = data.rolling_var(-3,0)
        expected = [None for i in range(3)] + [1.25 for i in range(997)]
        self.__test_equal(res,expected,float)

        # Test float inputs as well
        res = data.astype(float).rolling_var(-3,0)
        self.__test_equal(res,expected,float)

        # Test min observations
        res = data.rolling_var(-3, 0, min_observations=5)
        self.__test_equal(res,expected,float)

        res = data.rolling_var(-3, 0, min_observations=4)
        self.__test_equal(res,expected,float)

        res = data.rolling_var(-3, 0, min_observations=3)
        expected[2] = (2.0/3.0)
        self.__test_equal(res,expected,float)

        with self.assertRaises(ValueError):
            res = data.rolling_var(-3,0,min_observations=-1)

        # Test vector input
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.vec_data).rolling_var(-3,0)

        ### Small forward window including current
        res = data.rolling_var(0,4)
        expected = [2 for i in range(996)] + [None for i in range(4)]
        self.__test_equal(res,expected,float)

        ### A window size of 1
        res = data.rolling_var(0,0)
        self.__test_equal(res, [0 for i in range(1000)], float)

        res = data.rolling_var(-2,-2)
        self.__test_equal(res, [None,None] + [0 for i in range(998)], float)

        ### A negative window size
        with self.assertRaises(RuntimeError):
            res = data.rolling_var(4,2)

        ### Non-numeric
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.string_data).rolling_var(0,1)

        ### Empty SArray
        sa = SArray()
        res = sa.rolling_var(0,1)
        self.__test_equal(res, [], type(None))

        ### Small SArray
        sa = SArray([1,2,3])
        res = sa.rolling_var(0,1)
        self.__test_equal(res, [.25,.25,None], float)

    def test_rolling_stdv(self):
        data = SArray(range(1000))

        ### Small backward window including current
        res = data.rolling_stdv(-3,0)
        expected = [None for i in range(3)] + [1.118033988749895 for i in range(997)]
        self.__test_equal(res,expected,float)

        # Test float inputs as well
        res = data.astype(float).rolling_stdv(-3,0)
        self.__test_equal(res,expected,float)

        # Test min observations
        res = data.rolling_stdv(-3, 0, min_observations=5)
        self.__test_equal(res,expected,float)

        res = data.rolling_stdv(-3, 0, min_observations=4)
        self.__test_equal(res,expected,float)

        res = data.rolling_stdv(-3, 0, min_observations=3)
        expected[2] = math.sqrt(2.0/3.0)
        self.__test_equal(res,expected,float)

        with self.assertRaises(ValueError):
            res = data.rolling_stdv(-3,0,min_observations=-1)

        # Test vector input
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.vec_data).rolling_stdv(-3,0)

        ### Small forward window including current
        res = data.rolling_stdv(0,4)
        expected = [math.sqrt(2) for i in range(996)] + [None for i in range(4)]
        self.__test_equal(res,expected,float)

        ### A window size of 1
        res = data.rolling_stdv(0,0)
        self.__test_equal(res, [0 for i in range(1000)], float)

        res = data.rolling_stdv(-2,-2)
        self.__test_equal(res, [None,None] + [0 for i in range(998)], float)

        ### A negative window size
        with self.assertRaises(RuntimeError):
            res = data.rolling_stdv(4,2)

        ### Non-numeric
        with self.assertRaisesRegexp(RuntimeError, '.*support.*type.*'):
            res = SArray(self.string_data).rolling_stdv(0,1)

        ### Empty SArray
        sa = SArray()
        res = sa.rolling_stdv(0,1)
        self.__test_equal(res, [], type(None))

        ### Small SArray
        sa = SArray([1,2,3])
        res = sa.rolling_stdv(0,1)
        self.__test_equal(res, [.5,.5,None], float)

    def test_rolling_count(self):
        data = SArray(range(100))

        ### Small backward window including current
        res = data.rolling_count(-3,0)
        expected = [1,2,3] + [4 for i in range(97)]
        self.__test_equal(res,expected,int)

        # Test float inputs
        res = data.astype(float).rolling_count(-3,0)
        self.__test_equal(res,expected,int)

        # Test vector input
        res = SArray(self.vec_data).rolling_count(-3,0)
        expected = [1,2,3] + [4 for i in range(7)]
        self.__test_equal(res,expected,int)

        ### Test string input
        res = SArray(self.string_data).rolling_count(-3,0)
        self.__test_equal(res,expected[0:8],int)

        ### Small forward window including current
        res = data.rolling_count(0,4)
        expected = [5 for i in range(0,96)] + [4,3,2,1]
        self.__test_equal(res,expected,int)

        ### A window size of 1
        res = data.rolling_count(0,0)
        self.__test_equal(res, [1 for i in range(100)], int)

        res = data.rolling_count(-2,-2)
        self.__test_equal(res, [0,0] + [1 for i in range(98)], int)

        ### A negative window size
        with self.assertRaises(RuntimeError):
            res = data.rolling_count(4,2)

        ### Empty SArray
        sa = SArray()
        res = sa.rolling_count(0,1)
        self.__test_equal(res, [], type(None))

        ### Small SArray
        sa = SArray([1,2,3])
        res = sa.rolling_count(0,1)
        self.__test_equal(res, [2,2,1], int)

        sa = SArray([1,2,None])
        res = sa.rolling_count(0,1)
        self.__test_equal(res, [2,1,0], int)

    @nottest
    def cumulative_aggregate_comparison(self, out, ans):
        import array
        self.assertEqual(out.dtype(), ans.dtype())
        self.assertEqual(out.size(), ans.size())
        for i in range(len(out)):
            if out[i] is None:
                self.assertTrue(ans[i] is None)
            if ans[i] is None:
                self.assertTrue(out[i] is None)

            if type(out[i]) != array.array:
              self.assertAlmostEqual(out[i], ans[i])
            else:
              self.assertEqual(len(out[i]), len(ans[i]))
              oi = out[i]
              ansi = ans[i]
              for j in range(len(oi)):
                  self.assertAlmostEqual(oi, ansi)

    def test_cumulative_sum(self):

        def single_test(src, ans):
            out = src.cumulative_sum();
            self.cumulative_aggregate_comparison(out, ans)

        with self.assertRaises(RuntimeError):
            sa = SArray(["foo"]).cumulative_sum()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], ["foo"]]).cumulative_sum()
        with self.assertRaises(RuntimeError):
            sa = SArray([{"bar": 1}]).cumulative_sum()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1,1], [1], [1]]).cumulative_sum()

        single_test(
          SArray([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
          SArray([0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55])
        )
        single_test(
            SArray([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1]),
            SArray([0.1, 1.2, 3.3, 6.4, 10.5, 15.6, 21.7, 28.8])
        )
        single_test(
            SArray([[11.0, 2.0], [22.0, 1.0], [3.0, 4.0], [4.0, 4.0]]),
            SArray([[11.0, 2.0], [33.0, 3.0], [36.0, 7.0], [40.0, 11.0]])
        )
        single_test(
            SArray([None, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
            SArray([None, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55])
        )
        single_test(
            SArray([None, 1, None, 3, None, 5]),
            SArray([None, 1, 1, 4, 4, 9])
        )
        single_test(
            SArray([None, [33.0, 3.0], [3.0, 4.0], [4.0, 4.0]]),
            SArray([None, [33.0, 3.0], [36.0, 7.0], [40.0, 11.0]])
        )
        single_test(
            SArray([None, [33.0, 3.0], None, [4.0, 4.0]]),
            SArray([None, [33.0, 3.0], [33.0, 3.0], [37.0, 7.0]])
        )

    def test_cumulative_mean(self):

        def single_test(src, ans):
            out = src.cumulative_mean();
            self.cumulative_aggregate_comparison(out, ans)

        with self.assertRaises(RuntimeError):
            sa = SArray(["foo"]).cumulative_mean()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], ["foo"]]).cumulative_mean()
        with self.assertRaises(RuntimeError):
            sa = SArray([{"bar": 1}]).cumulative_mean()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1,1], [1], [1]]).cumulative_mean()

        single_test(
          SArray([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
          SArray([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0])
        )
        single_test(
            SArray([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1]),
            SArray([0.1, 0.6, 1.1, 1.6, 2.1, 2.6, 3.1, 3.6])
        )
        single_test(
            SArray([[11.0, 22.0], [33.0, 66.0], [4.0,   2.0],  [4.0,  2.0]]),
            SArray([[11.0, 22.0], [22.0, 44.0], [16.0, 30.0], [13.0, 23.0]])
        )
        single_test(
            SArray([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
            SArray([None, 0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0])
        )
        single_test(
            SArray([None, 1, None, 3, None, 5]),
            SArray([None, 1, 1.0, 2.0, 2.0, 3.0])
        )
        single_test(
            SArray([None, [11.0, 22.0], [33.0, 66.0], [4.0,   2.0]]),
            SArray([None, [11.0, 22.0], [22.0, 44.0], [16.0, 30.0]])
        )
        single_test(
            SArray([None, [11.0, 22.0], None, [33.0, 66.0], [4.0, 2.0]]),
            SArray([None, [11.0, 22.0], [11.0, 22.0], [22.0, 44.0], [16.0, 30.0]])
        )


    def test_cumulative_min(self):

        def single_test(src, ans):
            out = src.cumulative_min();
            self.cumulative_aggregate_comparison(out, ans)

        with self.assertRaises(RuntimeError):
            sa = SArray(["foo"]).cumulative_min()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], ["foo"]]).cumulative_min()
        with self.assertRaises(RuntimeError):
            sa = SArray([{"bar": 1}]).cumulative_min()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1,1], [1], [1]]).cumulative_min()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1], [1], [1]]).cumulative_min()

        single_test(
          SArray([0, 1, 2, 3, 4, 5, -1, 7, 8, -2, 10]),
          SArray([0, 0, 0, 0, 0, 0, -1, -1, -1, -2, -2])
        )
        single_test(
            SArray([7.1, 6.1, 3.1, 3.9, 4.1, 2.1, 2.9, 0.1]),
            SArray([7.1, 6.1, 3.1, 3.1, 3.1, 2.1, 2.1, 0.1])
        )
        single_test(
            SArray([None, 8, 6, 3, 4, None, 6, 2, 8, 9, 1]),
            SArray([None, 8, 6, 3, 3, 3,    3, 2, 2, 2, 1])
        )
        single_test(
            SArray([None, 5, None, 3, None, 10]),
            SArray([None, 5, 5, 3, 3, 3])
        )

    def test_cumulative_max(self):

        def single_test(src, ans):
            out = src.cumulative_max();
            self.cumulative_aggregate_comparison(out, ans)

        with self.assertRaises(RuntimeError):
            sa = SArray(["foo"]).cumulative_max()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], ["foo"]]).cumulative_max()
        with self.assertRaises(RuntimeError):
            sa = SArray([{"bar": 1}]).cumulative_max()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1,1], [1], [1]]).cumulative_max()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1], [1], [1]]).cumulative_max()

        single_test(
          SArray([0, 1, 0, 3, 5, 4, 1, 7, 6, 2, 10]),
          SArray([0, 1, 1, 3, 5, 5, 5, 7, 7, 7, 10])
        )
        single_test(
            SArray([2.1, 6.1, 3.1, 3.9, 2.1, 8.1, 8.9, 10.1]),
            SArray([2.1, 6.1, 6.1, 6.1, 6.1, 8.1, 8.9, 10.1])
        )
        single_test(
            SArray([None, 1, 6, 3, 4, None, 4, 2, 8, 9, 1]),
            SArray([None, 1, 6, 6, 6, 6,    6, 6, 8, 9, 9])
        )
        single_test(
            SArray([None, 2, None, 3, None, 10]),
            SArray([None, 2, 2, 3, 3, 10])
        )

    def test_cumulative_std(self):

        def single_test(src, ans):
            out = src.cumulative_std();
            self.cumulative_aggregate_comparison(out, ans)

        with self.assertRaises(RuntimeError):
            sa = SArray(["foo"]).cumulative_std()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], ["foo"]]).cumulative_std()
        with self.assertRaises(RuntimeError):
            sa = SArray([{"bar": 1}]).cumulative_std()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1,1], [1], [1]]).cumulative_std()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1], [1], [1]]).cumulative_std()

        single_test(
          SArray([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
          SArray([0.0, 0.5, 0.816496580927726, 1.118033988749895,
              1.4142135623730951, 1.707825127659933, 2.0, 2.29128784747792,
              2.581988897471611, 2.8722813232690143, 3.1622776601683795])
        )
        single_test(
            SArray([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1]),
            SArray([0.0, 0.5, 0.81649658092772603, 1.1180339887498949,
                1.4142135623730949, 1.707825127659933, 1.9999999999999998,
                2.2912878474779195])
        )
        single_test(
            SArray([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
            SArray([None, 0.0, 0.5, 0.816496580927726, 1.118033988749895,
                1.4142135623730951, 1.707825127659933, 2.0, 2.29128784747792,
                2.581988897471611, 2.8722813232690143, 3.1622776601683795])
        )
        single_test(
            SArray([None, 1,   None, 3, None, 5]),
            SArray([None, 0.0, 0.0, 1.0, 1.0, 1.6329931618554521])
        )

    def test_cumulative_var(self):

        def single_test(src, ans):
            out = src.cumulative_var();
            self.cumulative_aggregate_comparison(out, ans)

        with self.assertRaises(RuntimeError):
            sa = SArray(["foo"]).cumulative_var()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], ["foo"]]).cumulative_var()
        with self.assertRaises(RuntimeError):
            sa = SArray([{"bar": 1}]).cumulative_var()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1,1], [1], [1]]).cumulative_var()
        with self.assertRaises(RuntimeError):
            sa = SArray([[1], [1], [1], [1]]).cumulative_var()

        single_test(
          SArray([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
          SArray([0.0, 0.25, 0.6666666666666666, 1.25, 2.0, 2.9166666666666665,
              4.0, 5.25, 6.666666666666667, 8.25, 10.0])
        )
        single_test(
            SArray([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1]),
            SArray( [0.0, 0.25000000000000006, 0.6666666666666666, 1.25,
                1.9999999999999996, 2.916666666666666, 3.999999999999999,
                5.249999999999998])
        )
        single_test(
            SArray([None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
            SArray([None, 0.0, 0.25, 0.6666666666666666, 1.25, 2.0, 2.9166666666666665,
                4.0, 5.25, 6.666666666666667, 8.25, 10.0])
        )
        single_test(
            SArray([None, 1,   None, 3, None, 5]),
            SArray([None, 0.0, 0.0, 1.0, 1.0, 2.6666666666666665])
        )

    def test_numpy_datetime64(self):
        # Make all datetimes naive
        expected = [i.replace(tzinfo=GMT(0.0)) \
                if i is not None and i.tzinfo is None else i for i in self.datetime_data]

        # A regular list
        iso_str_list = [np.datetime64('2013-05-07T10:04:10Z'),
                        np.datetime64('1902-10-21T10:34:10Z'),
                        None]
        sa = SArray(iso_str_list)
        self.__test_equal(sa,expected,dt.datetime)

        iso_str_list[2] = np.datetime64('NaT')
        sa = SArray(iso_str_list)
        self.__test_equal(sa,expected,dt.datetime)

        # A numpy array
        np_ary = np.array(iso_str_list)
        sa = SArray(np_ary)
        self.__test_equal(sa,expected,dt.datetime)

        ### Every possible type of datetime64
        test_str = '1969-12-31T23:59:56Z'
        available_time_units = ['h','m','s','ms','us','ns','ps','fs','as']
        expected = [dt.datetime(1969,12,31,23,59,56,tzinfo=GMT(0.0)) for i in range(7)]
        expected.insert(0,dt.datetime(1969,12,31,23,59,0,tzinfo=GMT(0.0)))
        expected.insert(0,dt.datetime(1969,12,31,23,0,0,tzinfo=GMT(0.0)))
        for i in range(len(available_time_units)):
            sa = SArray([np.datetime64(test_str,available_time_units[i])])
            self.__test_equal(sa,[expected[i]],dt.datetime)

        test_str = '1908-06-01'
        available_date_units = ['Y','M','W','D']
        expected = [dt.datetime(1908,6,1,0,0,0,tzinfo=GMT(0.0)) for i in range(4)]
        expected[2] = dt.datetime(1908,5,28,0,0,0,tzinfo=GMT(0.0)) # weeks start on Thursday?
        expected[0] = dt.datetime(1908,1,1,0,0,0,tzinfo=GMT(0.0))
        for i in range(len(available_date_units)):
            sa = SArray([np.datetime64(test_str,available_date_units[i])])
            self.__test_equal(sa,[expected[i]],dt.datetime)

        # Daylight savings time (Just to be safe. datetime64 deals in UTC, and
        # we store times in UTC by default, so this shouldn't affect anything)
        sa = SArray([np.datetime64('2015-03-08T02:38:00-08')])
        expected = [dt.datetime(2015,3,8,10,38,tzinfo=GMT(0.0))]
        self.__test_equal(sa, expected, dt.datetime)

        # timezone considerations
        sa = SArray([np.datetime64('2016-01-01T05:45:00+0545')])
        expected = [dt.datetime(2016,1,1,0,0,0,tzinfo=GMT(0.0))]
        self.__test_equal(sa, expected, dt.datetime)

        ### Out of our datetime range
        with self.assertRaises(TypeError):
            sa = SArray([np.datetime64('1066-10-14T09:00:00Z')])

    def test_pandas_timestamp(self):
        iso_str_list = [pd.Timestamp('2013-05-07T10:04:10'),
                        pd.Timestamp('1902-10-21T10:34:10Z'),
                        None]
        sa = SArray(iso_str_list)
        self.__test_equal(sa,self.datetime_data,dt.datetime)

        iso_str_list[2] = pd.tslib.NaT
        sa = SArray(iso_str_list)
        self.__test_equal(sa,self.datetime_data,dt.datetime)

        sa = SArray([pd.Timestamp('2015-03-08T02:38:00-08')])
        expected = [dt.datetime(2015,3,8,2,38,tzinfo=GMT(-8.0))]
        self.__test_equal(sa, expected, dt.datetime)

        sa = SArray([pd.Timestamp('2016-01-01 05:45:00', tz=GMT(5.75))])
        expected =  [dt.datetime(2016,1,1,5,45,0,tzinfo=GMT(5.75))]
        self.__test_equal(sa, expected, dt.datetime)

    def test_decimal(self):
        import decimal
        test_val = decimal.Decimal(3.0)
        sa = SArray([test_val])
        expected = [3.0]
        self.__test_equal(sa, expected, float)

    def test_timedelta(self):
        test_val = dt.timedelta(1,1)
        sa = SArray([test_val])
        expected = [86401.0]
        self.__test_equal(sa, expected, float)

    def test_materialize(self):
        sa= SArray(range(100))
        sa = sa[sa > 10]
        self.assertFalse(sa.is_materialized())
        sa.materialize()
        self.assertTrue(sa.is_materialized())

    def test_ternary(self):
        lista = range(1000)
        a = SArray(lista)

        # identity
        self.__test_equal(SArray.where(a > 10, a, a), lista, int)

        # clip lower
        self.__test_equal(SArray.where(a > 10, a, 10),
                          [i if i > 10 else 10 for i in lista], int)

        # clip upper
        self.__test_equal(SArray.where(a > 10, 10, a),
                          [10 if i > 10 else i for i in lista], int)

        # constants
        self.__test_equal(SArray.where(a > 10, 10, 9),
                          [10 if i > 10 else 9 for i in lista], int)

        # constant float
        self.__test_equal(SArray.where(a > 10, 10.0, 9.0),
                          [10.0 if i > 10 else 9.0 for i in lista], float)

        # constant str
        self.__test_equal(SArray.where(a > 10, "10", "9"),
                          ["10" if i > 10 else "9" for i in lista], str)

        #inconsistent types
        with self.assertRaises(TypeError):
            SArray.where(a > 10, 10, "9") # 10 and "9" different types

        #inconsistent types
        with self.assertRaises(TypeError):
            SArray.where(a > 10, a, "9") # expecting an integer for "a"

        # technically different types but type coercion happened
        self.__test_equal(SArray.where(a > 10, a, 10.0),
                          [i if i > 10 else 10 for i in lista], int)

        # list types
        self.__test_equal(SArray.where(a > 10, [], [1], list),
                          [[] if i > 10 else [1] for i in lista], list)

        # really the same as the above, but using an SArray in place
        # of a constant in istrue. And hoping the type coercion
        # will take care of [1]
        b = SArray([[] for i in range(1000)])
        self.__test_equal(SArray.where(a > 10, b, [1]),
                          [[] if i > 10 else [1] for i in lista], list)

    def test_shape(self):
        sa = SArray()
        self.assertEqual(sa.shape, (0,))
        for i in [0,1,2,10,345]:
            sa = SArray(range(i))
            self.assertEqual(sa.shape, (i,))

    def test_random_split(self):
        sa = SArray(range(10))
        (train, test) = sa.random_split(0.8, seed=12423)
        self.assertEqual(list(train), [0, 1, 2, 3, 5, 7, 8, 9])
        self.assertEqual(list(test), [4,6])
        
    def test_copy(self):
        from copy import copy
        sa = SArray(range(1000))
        sa_copy = copy(sa)

        assert sa is not sa_copy

        assert (sa == sa_copy).all()

    def test_deepcopy(self):
        from copy import deepcopy
        sa = SArray(range(1000))
        sa_copy = deepcopy(sa)

        assert sa is not sa_copy

        assert (sa == sa_copy).all()
