"""
This module defines the SFrame class which provides the
ability to create, access and manipulate a remote scalable dataframe object.

SFrame acts similarly to pandas.DataFrame, but the data is completely immutable
and is stored column wise on the GraphLab Server side.
"""

'''
Copyright (C) 2015 Dato, Inc.
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the DATO-PYTHON-LICENSE file for details.
'''
import graphlab.connect as _mt
import graphlab.connect.main as glconnect
from graphlab.cython.cy_type_utils import infer_type_of_list
from graphlab.cython.context import debug_trace as cython_context
from graphlab.cython.cy_sframe import UnitySFrameProxy
from graphlab.util import _check_canvas_enabled, _make_internal_url, _is_callable
from graphlab.data_structures.sarray import SArray, _create_sequential_sarray
import graphlab.aggregate
import graphlab
import array
from prettytable import PrettyTable
from textwrap import wrap
import datetime
import inspect
from graphlab.deps import pandas, HAS_PANDAS
import time
import itertools
import os
import subprocess
import uuid
import platform

__all__ = ['SFrame']

SFRAME_GARBAGE_COLLECTOR = []

FOOTER_STRS = ['Note: Only the head of the SFrame is printed.',
               'You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.']

LAZY_FOOTER_STRS = ['Note: Only the head of the SFrame is printed. This SFrame is lazily evaluated.',
                    'You can use len(sf) to force materialization.']

SFRAME_ROOTS = [# Binary/lib location in production egg
                os.path.abspath(os.path.join(os.path.dirname(
                    os.path.realpath(__file__)), '..')),
                # Build tree location of SFrame binaries
                os.path.abspath(os.path.join(os.path.dirname(
                    os.path.realpath(__file__)),
                        '..', '..',  '..', '..', 'sframe')),
                # Location of python sources
                os.path.abspath(os.path.join(os.path.dirname(
                    os.path.realpath(__file__)),
                        '..', '..',  '..', '..', 'unity', 'python', 'graphlab')),
                # Build tree dependency location
                os.path.abspath(os.path.join(os.path.dirname(
                    os.path.realpath(__file__)),
                        '..', '..',  '..', '..', '..', '..', 'deps', 'local', 'lib'))
                ]

RDD_SFRAME_PICKLE = "rddtosf_pickle"
RDD_SFRAME_NONPICKLE = "rddtosf_nonpickle"
SFRAME_RDD_PICKLE = "sftordd_pickle"
HDFS_LIB = "libhdfs.so"
RDD_JAR_FILE = "graphlab-create-spark-integration.jar"
SYS_UTIL_PY = "sys_util.py"
RDD_SUPPORT_INITED = False
BINARY_PATHS = {}
STAGING_DIR = None
RDD_SUPPORT = True
PRODUCTION_RUN = False
YARN_OS = None
SPARK_SUPPORT_NAMES = {'RDD_SFRAME_PATH':'rddtosf_pickle',
        'RDD_SFRAME_NONPICKLE_PATH':'rddtosf_nonpickle',
        'SFRAME_RDD_PATH':'sftordd_pickle',
        'HDFS_LIB_PATH':'libhdfs.so',
        'RDD_JAR_PATH':'graphlab-create-spark-integration.jar',
        'SYS_UTIL_PY_PATH':'sys_util.py',
        'SPARK_PIPE_WRAPPER_PATH':'spark_pipe_wrapper'}

first = True
for i in SFRAME_ROOTS:
    for key,val in SPARK_SUPPORT_NAMES.iteritems():
        tmp_path = os.path.join(i, val)
        if key not in BINARY_PATHS and os.path.isfile(tmp_path):
            BINARY_PATHS[key] = tmp_path
    if all(name in BINARY_PATHS for name in SPARK_SUPPORT_NAMES.keys()):
        if first:
            PRODUCTION_RUN = True
        break
    first = False

if not all(name in BINARY_PATHS for name in SPARK_SUPPORT_NAMES.keys()):
    RDD_SUPPORT = False

def get_spark_integration_jar_path():
    """
    The absolute path of the jar file required to enable GraphLab Create's
    integration with Apache Spark.
    """
    if 'RDD_JAR_PATH' not in BINARY_PATHS:
        raise RuntimeError("Could not find a spark integration jar.  "\
            "Does your version of GraphLab Create support Spark Integration (is it >= 1.0)?")
    return BINARY_PATHS['RDD_JAR_PATH']

def __rdd_support_init__(sprk_ctx):
    global YARN_OS
    global RDD_SUPPORT_INITED
    global STAGING_DIR
    global BINARY_PATHS

    if not RDD_SUPPORT or RDD_SUPPORT_INITED:
        return

    # Make sure our GraphLabUtil scala functions are accessible from the driver
    try:
        tmp = sprk_ctx._jvm.org.graphlab.create.GraphLabUtil.EscapeString(sprk_ctx._jvm.java.lang.String("1,2,3,4"))
    except:
        raise RuntimeError("Could not execute RDD translation functions. "\
                           "Please make sure you have started Spark "\
                           "(either with spark-submit or pyspark) with the following flag set:\n"\
                           "'--driver-class-path " + BINARY_PATHS['RDD_JAR_PATH']+"'\n"\
                           "OR set the property spark.driver.extraClassPath in spark-defaults.conf")

    dummy_rdd = sprk_ctx.parallelize([1])

    if PRODUCTION_RUN and sprk_ctx.master == 'yarn-client':
        # Get cluster operating system
        os_rdd = dummy_rdd.map(lambda x: platform.system())
        YARN_OS = os_rdd.collect()[0]

        # Set binary path
        for i in BINARY_PATHS.keys():
            s = BINARY_PATHS[i]
            if os.path.basename(s) == SPARK_SUPPORT_NAMES['SYS_UTIL_PY_PATH']:
                continue
            if YARN_OS == 'Linux':
                BINARY_PATHS[i] = os.path.join(os.path.dirname(s), 'linux', os.path.basename(s))
            elif YARN_OS == 'Darwin':
                BINARY_PATHS[i] = os.path.join(os.path.dirname(s), 'osx', os.path.basename(s))
            else:
                raise RuntimeError("YARN cluster has unsupported operating system "\
                                    "(something other than Linux or Mac OS X). "\
                                    "Cannot convert RDDs on this cluster to SFrame.")

    # Create staging directory
    staging_dir = '.graphlabStaging'
    if sprk_ctx.master == 'yarn-client':
        tmp_loc = None

        # Get that staging directory's full name
        tmp_loc = dummy_rdd.map(
                lambda x: subprocess.check_output(
                    ["hdfs", "getconf", "-confKey", "fs.defaultFS"]).rstrip()).collect()[0]

        STAGING_DIR = os.path.join(tmp_loc, "user", sprk_ctx.sparkUser(), staging_dir)
        if STAGING_DIR is None:
            raise RuntimeError("Failed to create a staging directory on HDFS. "\
                               "Do your cluster nodes have a working hdfs client?")

        # Actually create the staging dir
        unity = glconnect.get_unity()
        unity.__mkdir__(STAGING_DIR)
        unity.__chmod__(STAGING_DIR, 0777)
    elif sprk_ctx.master[0:5] == 'local':
        # Save the output sframes to the same temp workspace this engine is
        # using
        #TODO: Consider cases where server and client aren't on the same machine
        unity = glconnect.get_unity()
        STAGING_DIR = unity.get_current_cache_file_location()
        if STAGING_DIR is None:
            raise RuntimeError("Could not retrieve local staging directory! \
                    Please contact us on http://forum.dato.com.")
    else:
        raise RuntimeError("Your spark context's master is '" +
                str(sprk_ctx.master) +
                "'. Only 'local' and 'yarn-client' are supported.")

    if sprk_ctx.master == 'yarn-client':
        sprk_ctx.addFile(BINARY_PATHS['RDD_SFRAME_PATH'])
        sprk_ctx.addFile(BINARY_PATHS['HDFS_LIB_PATH'])
        sprk_ctx.addFile(BINARY_PATHS['SFRAME_RDD_PATH'])
        sprk_ctx.addFile(BINARY_PATHS['RDD_SFRAME_NONPICKLE_PATH'])
        sprk_ctx.addFile(BINARY_PATHS['SYS_UTIL_PY_PATH'])
        sprk_ctx.addFile(BINARY_PATHS['SPARK_PIPE_WRAPPER_PATH'])
        sprk_ctx._jsc.addJar(BINARY_PATHS['RDD_JAR_PATH'])

    RDD_SUPPORT_INITED = True

def load_sframe(filename):
    """
    Load an SFrame. The filename extension is used to determine the format
    automatically. This function is particularly useful for SFrames previously
    saved in binary format. For CSV imports the ``SFrame.read_csv`` function
    provides greater control. If the SFrame is in binary format, ``filename`` is
    actually a directory, created when the SFrame is saved.

    Parameters
    ----------
    filename : string
        Location of the file to load. Can be a local path or a remote URL.

    Returns
    -------
    out : SFrame

    See Also
    --------
    SFrame.save, SFrame.read_csv

    Examples
    --------
    >>> sf = graphlab.SFrame({'id':[1,2,3], 'val':['A','B','C']})
    >>> sf.save('my_sframe')        # 'my_sframe' is a directory
    >>> sf_loaded = graphlab.load_sframe('my_sframe')
    """
    sf = SFrame(data=filename)
    return sf


class SFrame(object):
    """
    A tabular, column-mutable dataframe object that can scale to big data. The
    data in SFrame is stored column-wise on the GraphLab Server side, and is
    stored on persistent storage (e.g. disk) to avoid being constrained by
    memory size.  Each column in an SFrame is a size-immutable
    :class:`~graphlab.SArray`, but SFrames are mutable in that columns can be
    added and subtracted with ease.  An SFrame essentially acts as an ordered
    dict of SArrays.

    Currently, we support constructing an SFrame from the following data
    formats:

    * csv file (comma separated value)
    * sframe directory archive (A directory where an sframe was saved
      previously)
    * general text file (with csv parsing options, See :py:meth:`read_csv()`)
    * a Python dictionary
    * pandas.DataFrame
    * JSON
    * Apache Avro
    * PySpark RDD

    and from the following sources:

    * your local file system
    * the GraphLab Server's file system
    * HDFS
    * Amazon S3
    * HTTP(S).

    Only basic examples of construction are covered here. For more information
    and examples, please see the `User Guide <https://dato.com/learn/user
    guide/index.html#Working_with_data_Tabular_data>`_, `API Translator
    <https://dato.com/learn/translator>`_, `How-Tos
    <https://dato.com/learn/how-to>`_, and data science `Gallery
    <https://dato.com/learn/gallery>`_.

    Parameters
    ----------
    data : array | pandas.DataFrame | string | dict, optional
        The actual interpretation of this field is dependent on the ``format``
        parameter. If ``data`` is an array or Pandas DataFrame, the contents are
        stored in the SFrame. If ``data`` is a string, it is interpreted as a
        file. Files can be read from local file system or urls (local://,
        hdfs://, s3://, http://).

    format : string, optional
        Format of the data. The default, "auto" will automatically infer the
        input data format. The inference rules are simple: If the data is an
        array or a dataframe, it is associated with 'array' and 'dataframe'
        respectively. If the data is a string, it is interpreted as a file, and
        the file extension is used to infer the file format. The explicit
        options are:

        - "auto"
        - "array"
        - "dict"
        - "sarray"
        - "dataframe"
        - "csv"
        - "tsv"
        - "sframe".

    See Also
    --------
    read_csv:
        Create a new SFrame from a csv file. Preferred for text and CSV formats,
        because it has a lot more options for controlling the parser.

    save : Save an SFrame for later use.

    Notes
    -----
    - When working with the GraphLab EC2 instance (see
      :py:func:`graphlab.aws.launch_EC2()`), an SFrame cannot be constructed
      using local file path, because it involves a potentially large amount of
      data transfer from client to server. However, it is still okay to use a
      remote file path. See the examples below. A similar restriction applies to
      :py:class:`graphlab.SGraph` and :py:class:`graphlab.SArray`.

    - When reading from HDFS on Linux we must guess the location of your java
      installation. By default, we will use the location pointed to by the
      JAVA_HOME environment variable.  If this is not set, we check many common
      installation paths. You may use two environment variables to override
      this behavior.  GRAPHLAB_JAVA_HOME allows you to specify a specific java
      installation and overrides JAVA_HOME.  GRAPHLAB_LIBJVM_DIRECTORY
      overrides all and expects the exact directory that your preferred
      libjvm.so file is located.  Use this ONLY if you'd like to use a
      non-standard JVM.

    Examples
    --------

    >>> import graphlab
    >>> from graphlab import SFrame

    **Construction**

    Construct an SFrame from a dataframe and transfers the dataframe object
    across the network.

    >>> df = pandas.DataFrame()
    >>> sf = SFrame(data=df)

    Construct an SFrame from a local csv file (only works for local server).

    >>> sf = SFrame(data='~/mydata/foo.csv')

    Construct an SFrame from a csv file on Amazon S3. This requires the
    environment variables: *AWS_ACCESS_KEY_ID* and *AWS_SECRET_ACCESS_KEY* to be
    set before the python session started. Alternatively, you can use
    :py:func:`graphlab.aws.set_credentials()` to set the credentials after
    python is started and :py:func:`graphlab.aws.get_credentials()` to verify
    these environment variables.

    >>> sf = SFrame(data='s3://mybucket/foo.csv')

    Read from HDFS using a specific java installation (environment variable
    only applies when using Linux)

    >>> import os
    >>> os.environ['GRAPHLAB_JAVA_HOME'] = '/my/path/to/java'
    >>> from graphlab import SFrame
    >>> sf = SFrame("hdfs://mycluster.example.com:8020/user/myname/coolfile.txt")

    An SFrame can be constructed from a dictionary of values or SArrays:

    >>> sf = gl.SFrame({'id':[1,2,3],'val':['A','B','C']})
    >>> sf
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B
    2  3   C

    Or equivalently:

    >>> ids = SArray([1,2,3])
    >>> vals = SArray(['A','B','C'])
    >>> sf = SFrame({'id':ids,'val':vals})

    It can also be constructed from an array of SArrays in which case column
    names are automatically assigned.

    >>> ids = SArray([1,2,3])
    >>> vals = SArray(['A','B','C'])
    >>> sf = SFrame([ids, vals])
    >>> sf
    Columns:
        X1 int
        X2 str
    Rows: 3
    Data:
       X1  X2
    0  1   A
    1  2   B
    2  3   C

    If the SFrame is constructed from a list of values, an SFrame of a single
    column is constructed.

    >>> sf = SFrame([1,2,3])
    >>> sf
    Columns:
        X1 int
    Rows: 3
    Data:
       X1
    0  1
    1  2
    2  3

    **Parsing**

    The :py:func:`graphlab.SFrame.read_csv()` is quite powerful and, can be
    used to import a variety of row-based formats.

    First, some simple cases:

    >>> !cat ratings.csv
    user_id,movie_id,rating
    10210,1,1
    10213,2,5
    10217,2,2
    10102,1,3
    10109,3,4
    10117,5,2
    10122,2,4
    10114,1,5
    10125,1,1
    >>> gl.SFrame.read_csv('ratings.csv')
    Columns:
      user_id   int
      movie_id  int
      rating    int
    Rows: 9
    Data:
    +---------+----------+--------+
    | user_id | movie_id | rating |
    +---------+----------+--------+
    |  10210  |    1     |   1    |
    |  10213  |    2     |   5    |
    |  10217  |    2     |   2    |
    |  10102  |    1     |   3    |
    |  10109  |    3     |   4    |
    |  10117  |    5     |   2    |
    |  10122  |    2     |   4    |
    |  10114  |    1     |   5    |
    |  10125  |    1     |   1    |
    +---------+----------+--------+
    [9 rows x 3 columns]


    Delimiters can be specified, if "," is not the delimiter, for instance
    space ' ' in this case. Only single character delimiters are supported.

    >>> !cat ratings.csv
    user_id movie_id rating
    10210 1 1
    10213 2 5
    10217 2 2
    10102 1 3
    10109 3 4
    10117 5 2
    10122 2 4
    10114 1 5
    10125 1 1
    >>> gl.SFrame.read_csv('ratings.csv', delimiter=' ')

    By default, "NA" or a missing element are interpreted as missing values.

    >>> !cat ratings2.csv
    user,movie,rating
    "tom",,1
    harry,5,
    jack,2,2
    bill,,
    >>> gl.SFrame.read_csv('ratings2.csv')
    Columns:
      user  str
      movie int
      rating    int
    Rows: 4
    Data:
    +---------+-------+--------+
    |   user  | movie | rating |
    +---------+-------+--------+
    |   tom   |  None |   1    |
    |  harry  |   5   |  None  |
    |   jack  |   2   |   2    |
    | missing |  None |  None  |
    +---------+-------+--------+
    [4 rows x 3 columns]

    Furthermore due to the dictionary types and list types, can handle parsing
    of JSON-like formats.

    >>> !cat ratings3.csv
    business, categories, ratings
    "Restaurant 1", [1 4 9 10], {"funny":5, "cool":2}
    "Restaurant 2", [], {"happy":2, "sad":2}
    "Restaurant 3", [2, 11, 12], {}
    >>> gl.SFrame.read_csv('ratings3.csv')
    Columns:
    business    str
    categories  array
    ratings dict
    Rows: 3
    Data:
    +--------------+--------------------------------+-------------------------+
    |   business   |           categories           |         ratings         |
    +--------------+--------------------------------+-------------------------+
    | Restaurant 1 | array('d', [1.0, 4.0, 9.0, ... | {'funny': 5, 'cool': 2} |
    | Restaurant 2 |           array('d')           |  {'sad': 2, 'happy': 2} |
    | Restaurant 3 | array('d', [2.0, 11.0, 12.0])  |            {}           |
    +--------------+--------------------------------+-------------------------+
    [3 rows x 3 columns]

    The list and dictionary parsers are quite flexible and can absorb a
    variety of purely formatted inputs. Also, note that the list and dictionary
    types are recursive, allowing for arbitrary values to be contained.

    All these are valid lists:

    >>> !cat interesting_lists.csv
    list
    []
    [1,2,3]
    [1;2,3]
    [1 2 3]
    [{a:b}]
    ["c",d, e]
    [[a]]
    >>> gl.SFrame.read_csv('interesting_lists.csv')
    Columns:
      list  list
    Rows: 7
    Data:
    +-----------------+
    |       list      |
    +-----------------+
    |        []       |
    |    [1, 2, 3]    |
    |    [1, 2, 3]    |
    |    [1, 2, 3]    |
    |   [{'a': 'b'}]  |
    | ['c', 'd', 'e'] |
    |     [['a']]     |
    +-----------------+
    [7 rows x 1 columns]

    All these are valid dicts:

    >>> !cat interesting_dicts.csv
    dict
    {"classic":1,"dict":1}
    {space:1 seperated:1}
    {emptyvalue:}
    {}
    {:}
    {recursive1:[{a:b}]}
    {:[{:[a]}]}
    >>> gl.SFrame.read_csv('interesting_dicts.csv')
    Columns:
      dict  dict
    Rows: 7
    Data:
    +------------------------------+
    |             dict             |
    +------------------------------+
    |  {'dict': 1, 'classic': 1}   |
    | {'seperated': 1, 'space': 1} |
    |     {'emptyvalue': None}     |
    |              {}              |
    |         {None: None}         |
    | {'recursive1': [{'a': 'b'}]} |
    | {None: [{None: array('d')}]} |
    +------------------------------+
    [7 rows x 1 columns]

    **Saving**

    Save and load the sframe in native format.

    >>> sf.save('mysframedir')
    >>> sf2 = graphlab.load_sframe('mysframedir')

    **Column Manipulation **

    An SFrame is composed of a collection of columns of SArrays, and individual
    SArrays can be extracted easily. For instance given an SFrame:

    >>> sf = SFrame({'id':[1,2,3],'val':['A','B','C']})
    >>> sf
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B
    2  3   C

    The "id" column can be extracted using:

    >>> sf["id"]
    dtype: int
    Rows: 3
    [1, 2, 3]

    And can be deleted using:

    >>> del sf["id"]

    Multiple columns can be selected by passing a list of column names:

    >>> sf = SFrame({'id':[1,2,3],'val':['A','B','C'],'val2':[5,6,7]})
    >>> sf
    Columns:
        id   int
        val  str
        val2 int
    Rows: 3
    Data:
       id  val val2
    0  1   A   5
    1  2   B   6
    2  3   C   7
    >>> sf2 = sf[['id','val']]
    >>> sf2
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B
    2  3   C

    The same mechanism can be used to re-order columns:

    >>> sf = SFrame({'id':[1,2,3],'val':['A','B','C']})
    >>> sf
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B
    2  3   C
    >>> sf[['val','id']]
    >>> sf
    Columns:
        val str
        id  int
    Rows: 3
    Data:
       val id
    0  A   1
    1  B   2
    2  C   3

    **Element Access and Slicing**
    SFrames can be accessed by integer keys just like a regular python list.
    Such operations may not be fast on large datasets so looping over an SFrame
    should be avoided.

    >>> sf = SFrame({'id':[1,2,3],'val':['A','B','C']})
    >>> sf[0]
    {'id': 1, 'val': 'A'}
    >>> sf[2]
    {'id': 3, 'val': 'C'}
    >>> sf[5]
    IndexError: SFrame index out of range

    Negative indices can be used to access elements from the tail of the array

    >>> sf[-1] # returns the last element
    {'id': 3, 'val': 'C'}
    >>> sf[-2] # returns the second to last element
    {'id': 2, 'val': 'B'}

    The SFrame also supports the full range of python slicing operators:

    >>> sf[1000:] # Returns an SFrame containing rows 1000 to the end
    >>> sf[:1000] # Returns an SFrame containing rows 0 to row 999 inclusive
    >>> sf[0:1000:2] # Returns an SFrame containing rows 0 to row 1000 in steps of 2
    >>> sf[-100:] # Returns an SFrame containing last 100 rows
    >>> sf[-100:len(sf):2] # Returns an SFrame containing last 100 rows in steps of 2

    **Logical Filter**

    An SFrame can be filtered using

    >>> sframe[binary_filter]

    where sframe is an SFrame and binary_filter is an SArray of the same length.
    The result is a new SFrame which contains only rows of the SFrame where its
    matching row in the binary_filter is non zero.

    This permits the use of boolean operators that can be used to perform
    logical filtering operations. For instance, given an SFrame

    >>> sf
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B
    2  3   C

    >>> sf[(sf['id'] >= 1) & (sf['id'] <= 2)]
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B

    See :class:`~graphlab.SArray` for more details on the use of the logical
    filter.

    This can also be used more generally to provide filtering capability which
    is otherwise not expressible with simple boolean functions. For instance:

    >>> sf[sf['id'].apply(lambda x: math.log(x) <= 1)]
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B

    Or alternatively:

    >>> sf[sf.apply(lambda x: math.log(x['id']) <= 1)]

            Create an SFrame from a Python dictionary.

    >>> from graphlab import SFrame
    >>> sf = SFrame({'id':[1,2,3], 'val':['A','B','C']})
    >>> sf
    Columns:
        id  int
        val str
    Rows: 3
    Data:
       id  val
    0  1   A
    1  2   B
    2  3   C
    """

    __slots__ = ['shape', '__proxy__', '_proxy']

    def __init__(self, data=None,
                 format='auto',
                 _proxy=None):
        """__init__(data=list(), format='auto')
        Construct a new SFrame from a url or a pandas.DataFrame.
        """
        # emit metrics for num_rows, num_columns, and type (local://, s3, hdfs, http)
        tracker = _mt._get_metric_tracker()
        if (_proxy):
            self.__proxy__ = _proxy
        else:
            self.__proxy__ = UnitySFrameProxy(glconnect.get_client())
            _format = None
            if (format == 'auto'):
                if (HAS_PANDAS and isinstance(data, pandas.DataFrame)):
                    _format = 'dataframe'
                    tracker.track('sframe.location.memory', value=1)
                elif (isinstance(data, str) or isinstance(data, unicode)):

                    if data.find('://') == -1:
                        suffix = 'local'
                    else:
                        suffix = data.split('://')[0]
                    tracker.track(('sframe.location.%s' % (suffix)), value=1)

                    if data.endswith(('.csv', '.csv.gz')):
                        _format = 'csv'
                    elif data.endswith(('.tsv', '.tsv.gz')):
                        _format = 'tsv'
                    elif data.endswith(('.txt', '.txt.gz')):
                        print "Assuming file is csv. For other delimiters, " + \
                            "please use `SFrame.read_csv`."
                        _format = 'csv'
                    else:
                        _format = 'sframe'
                elif type(data) == SArray:
                    _format = 'sarray'

                elif isinstance(data, SFrame):
                    _format = 'sframe_obj'

                elif (hasattr(data, 'iteritems')):
                    _format = 'dict'
                    tracker.track('sframe.location.memory', value=1)

                elif hasattr(data, '__iter__'):
                    _format = 'array'
                    tracker.track('sframe.location.memory', value=1)
                elif data is None:
                    _format = 'empty'
                else:
                    raise ValueError('Cannot infer input type for data ' + str(data))
            else:
                _format = format

            tracker.track(('sframe.format.%s' % _format), value=1)

            with cython_context():
                if (_format == 'dataframe'):
                    self.__proxy__.load_from_dataframe(data)
                elif (_format == 'sframe_obj'):
                    for col in data.column_names():
                        self.__proxy__.add_column(data[col].__proxy__, col)
                elif (_format == 'sarray'):
                    self.__proxy__.add_column(data.__proxy__, "")
                elif (_format == 'array'):
                    if len(data) > 0:
                        unique_types = set([type(x) for x in data if x is not None])
                        if len(unique_types) == 1 and SArray in unique_types:
                            for arr in data:
                                self.add_column(arr)
                        elif SArray in unique_types:
                            raise ValueError("Cannot create SFrame from mix of regular values and SArrays")
                        else:
                            self.__proxy__.add_column(SArray(data).__proxy__, "")
                elif (_format == 'dict'):
                    for key,val in iter(sorted(data.iteritems())):
                        if (type(val) == SArray):
                            self.__proxy__.add_column(val.__proxy__, key)
                        else:
                            self.__proxy__.add_column(SArray(val).__proxy__, key)
                elif (_format == 'csv'):
                    url = _make_internal_url(data)
                    tmpsf = SFrame.read_csv(url, delimiter=',', header=True)
                    self.__proxy__ = tmpsf.__proxy__
                elif (_format == 'tsv'):
                    url = _make_internal_url(data)
                    tmpsf = SFrame.read_csv(url, delimiter='\t', header=True)
                    self.__proxy__ = tmpsf.__proxy__
                elif (_format == 'sframe'):
                    url = _make_internal_url(data)
                    self.__proxy__.load_from_sframe_index(url)
                elif (_format == 'empty'):
                    pass
                else:
                    raise ValueError('Unknown input type: ' + format)

        sframe_size = -1
        if self.__has_size__():
          sframe_size = self.num_rows()
        tracker.track('sframe.row.size', value=sframe_size)
        tracker.track('sframe.col.size', value=self.num_cols())

    @staticmethod
    def _infer_column_types_from_lines(first_rows):
        if (len(first_rows.column_names()) < 1):
          print "Insufficient number of columns to perform type inference"
          raise RuntimeError("Insufficient columns ")
        if len(first_rows) < 1:
          print "Insufficient number of rows to perform type inference"
          raise RuntimeError("Insufficient rows")
        # gets all the values column-wise
        all_column_values_transposed = [list(first_rows[col])
                for col in first_rows.column_names()]
        # transpose
        all_column_values = [list(x) for x in zip(*all_column_values_transposed)]
        all_column_type_hints = [[type(t) for t in vals] for vals in all_column_values]
        # collect the hints
        # if every line was inferred to have a different number of elements, die
        if len(set(len(x) for x in all_column_type_hints)) != 1:
            print "Unable to infer column types. Defaulting to str"
            return str

        import types

        column_type_hints = all_column_type_hints[0]
        # now perform type combining across rows
        for i in range(1, len(all_column_type_hints)):
          currow = all_column_type_hints[i]
          for j in range(len(column_type_hints)):
            # combine types
            d = set([currow[j], column_type_hints[j]])
            if (len(d) == 1):
              # easy case. both agree on the type
              continue
            if ((int in d) and (float in d)):
              # one is an int, one is a float. its a float
              column_type_hints[j] = float
            elif ((array.array in d) and (list in d)):
              # one is an array , one is a list. its a list
              column_type_hints[j] = list
            elif types.NoneType in d:
              # one is a NoneType. assign to other type
              if currow[j] != types.NoneType:
                  column_type_hints[j] = currow[j]
            else:
              column_type_hints[j] = str
        # final pass. everything whih is still NoneType is now a str
        for i in range(len(column_type_hints)):
          if column_type_hints[i] == types.NoneType:
            column_type_hints[i] = str

        return column_type_hints

    @classmethod
    def _read_csv_impl(cls,
                       url,
                       delimiter=',',
                       header=True,
                       error_bad_lines=False,
                       comment_char='',
                       escape_char='\\',
                       double_quote=True,
                       quote_char='\"',
                       skip_initial_space=True,
                       column_type_hints=None,
                       na_values=["NA"],
                       nrows=None,
                       verbose=True,
                       store_errors=True):
        """
        Constructs an SFrame from a CSV file or a path to multiple CSVs, and
        returns a pair containing the SFrame and optionally
        (if store_errors=True) a dict of filenames to SArrays
        indicating for each file, what are the incorrectly parsed lines
        encountered.

        Parameters
        ----------
        store_errors : bool
            If true, the output errors dict will be filled.

        See `read_csv` for the rest of the parameters.
        """
        parsing_config = dict()
        parsing_config["delimiter"] = delimiter
        parsing_config["use_header"] = header
        parsing_config["continue_on_failure"] = not error_bad_lines
        parsing_config["comment_char"] = comment_char
        parsing_config["escape_char"] = escape_char
        parsing_config["double_quote"] = double_quote
        parsing_config["quote_char"] = quote_char
        parsing_config["skip_initial_space"] = skip_initial_space
        parsing_config["store_errors"] = store_errors
        if type(na_values) is str:
          na_values = [na_values]
        if na_values is not None and len(na_values) > 0:
            parsing_config["na_values"] = na_values

        if nrows != None:
          parsing_config["row_limit"] = nrows

        proxy = UnitySFrameProxy(glconnect.get_client())
        internal_url = _make_internal_url(url)

        if (not verbose):
            glconnect.get_client().set_log_progress(False)

        # Attempt to automatically detect the column types. Either produce a
        # list of types; otherwise default to all str types.
        column_type_inference_was_used = False
        if column_type_hints is None:
            try:
                # Get the first 100 rows (using all the desired arguments).
                first_rows = graphlab.SFrame.read_csv(url, nrows=100,
                                 column_type_hints=type(None),
                                 header=header,
                                 delimiter=delimiter,
                                 comment_char=comment_char,
                                 escape_char=escape_char,
                                 double_quote=double_quote,
                                 quote_char=quote_char,
                                 skip_initial_space=skip_initial_space,
                                 na_values = na_values)
                column_type_hints = SFrame._infer_column_types_from_lines(first_rows)
                typelist = '[' + ','.join(t.__name__ for t in column_type_hints) + ']'
                print "------------------------------------------------------"
                print "Inferred types from first line of file as "
                print "column_type_hints="+ typelist
                print "If parsing fails due to incorrect types, you can correct"
                print "the inferred type list above and pass it to read_csv in"
                print "the column_type_hints argument"
                print "------------------------------------------------------"
                column_type_inference_was_used = True
            except Exception as e:
                if type(e) == RuntimeError and "CSV parsing cancelled" in e.message:
                    raise e
                # If the above fails, default back to str for all columns.
                column_type_hints = str
                print 'Could not detect types. Using str for each column.'

        if type(column_type_hints) is type:
            type_hints = {'__all_columns__': column_type_hints}
        elif type(column_type_hints) is list:
            type_hints = dict(zip(['__X%d__' % i for i in range(len(column_type_hints))], column_type_hints))
        elif type(column_type_hints) is dict:
            type_hints = column_type_hints
        else:
            raise TypeError("Invalid type for column_type_hints. Must be a dictionary, list or a single type.")


        _mt._get_metric_tracker().track('sframe.csv.parse')

        suffix=''
        if url.find('://') == -1:
            suffix = 'local'
        else:
            suffix = url.split('://')[0]

        _mt._get_metric_tracker().track(('sframe.location.%s' % (suffix)), value=1)
        try:
            with cython_context():
                errors = proxy.load_from_csvs(internal_url, parsing_config, type_hints)
        except Exception as e:
            if type(e) == RuntimeError and "CSV parsing cancelled" in e.message:
                raise e
            if column_type_inference_was_used:
                # try again
                print "Unable to parse the file with automatic type inference."
                print "Defaulting to column_type_hints=str"
                type_hints = {'__all_columns__': str}
                try:
                    with cython_context():
                        errors = proxy.load_from_csvs(internal_url, parsing_config, type_hints)
                except:
                    raise
            else:
                raise

        glconnect.get_client().set_log_progress(True)

        return (cls(_proxy=proxy), { f: SArray(_proxy = es) for (f, es) in errors.iteritems() })

    @classmethod
    def read_csv_with_errors(cls,
                             url,
                             delimiter=',',
                             header=True,
                             comment_char='',
                             escape_char='\\',
                             double_quote=True,
                             quote_char='\"',
                             skip_initial_space=True,
                             column_type_hints=None,
                             na_values=["NA"],
                             nrows=None,
                             verbose=True):
        """
        Constructs an SFrame from a CSV file or a path to multiple CSVs, and
        returns a pair containing the SFrame and a dict of filenames to SArrays
        indicating for each file, what are the incorrectly parsed lines
        encountered.

        Parameters
        ----------
        url : string
            Location of the CSV file or directory to load. If URL is a directory
            or a "glob" pattern, all matching files will be loaded.

        delimiter : string, optional
            This describes the delimiter used for parsing csv files.

        header : bool, optional
            If true, uses the first row as the column names. Otherwise use the
            default column names: 'X1, X2, ...'.

        comment_char : string, optional
            The character which denotes that the
            remainder of the line is a comment.

        escape_char : string, optional
            Character which begins a C escape sequence

        double_quote : bool, optional
            If True, two consecutive quotes in a string are parsed to a single
            quote.

        quote_char : string, optional
            Character sequence that indicates a quote.

        skip_initial_space : bool, optional
            Ignore extra spaces at the start of a field

        column_type_hints : None, type, list[type], dict[string, type], optional
            This provides type hints for each column. By default, this method
            attempts to detect the type of each column automatically.

            Supported types are int, float, str, list, dict, and array.array.

            * If a single type is provided, the type will be
              applied to all columns. For instance, column_type_hints=float
              will force all columns to be parsed as float.
            * If a list of types is provided, the types applies
              to each column in order, e.g.[int, float, str]
              will parse the first column as int, second as float and third as
              string.
            * If a dictionary of column name to type is provided,
              each type value in the dictionary is applied to the key it
              belongs to.
              For instance {'user':int} will hint that the column called "user"
              should be parsed as an integer, and the rest will default to
              string.

        na_values : str | list of str, optional
            A string or list of strings to be interpreted as missing values.

        nrows : int, optional
            If set, only this many rows will be read from the file.

        verbose : bool, optional
            If True, print the progress.

        Returns
        -------
        out : tuple
            The first element is the SFrame with good data. The second element
            is a dictionary of filenames to SArrays indicating for each file,
            what are the incorrectly parsed lines encountered.

        See Also
        --------
        read_csv, SFrame

        Examples
        --------
        >>> bad_url = 'https://s3.amazonaws.com/gl-testdata/bad_csv_example.csv'
        >>> (sf, bad_lines) = graphlab.SFrame.read_csv_with_errors(bad_url)
        >>> sf
        +---------+----------+--------+
        | user_id | movie_id | rating |
        +---------+----------+--------+
        |  25904  |   1663   |   3    |
        |  25907  |   1663   |   3    |
        |  25923  |   1663   |   3    |
        |  25924  |   1663   |   3    |
        |  25928  |   1663   |   2    |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [98 rows x 3 columns]

        >>> bad_lines
        {'https://s3.amazonaws.com/gl-testdata/bad_csv_example.csv': dtype: str
         Rows: 1
         ['x,y,z,a,b,c']}
       """
        return cls._read_csv_impl(url,
                                  delimiter=delimiter,
                                  header=header,
                                  error_bad_lines=False, # we are storing errors,
                                                         # thus we must not fail
                                                         # on bad lines
                                  comment_char=comment_char,
                                  escape_char=escape_char,
                                  double_quote=double_quote,
                                  quote_char=quote_char,
                                  skip_initial_space=skip_initial_space,
                                  column_type_hints=column_type_hints,
                                  na_values=na_values,
                                  nrows=nrows,
                                  verbose=verbose,
                                  store_errors=True)
    @classmethod
    def read_csv(cls,
                 url,
                 delimiter=',',
                 header=True,
                 error_bad_lines=False,
                 comment_char='',
                 escape_char='\\',
                 double_quote=True,
                 quote_char='\"',
                 skip_initial_space=True,
                 column_type_hints=None,
                 na_values=["NA"],
                 nrows=None,
                 verbose=True):
        """
        Constructs an SFrame from a CSV file or a path to multiple CSVs.

        Parameters
        ----------
        url : string
            Location of the CSV file or directory to load. If URL is a directory
            or a "glob" pattern, all matching files will be loaded.

        delimiter : string, optional
            This describes the delimiter used for parsing csv files.

        header : bool, optional
            If true, uses the first row as the column names. Otherwise use the
            default column names : 'X1, X2, ...'.

        error_bad_lines : bool
            If true, will fail upon encountering a bad line. If false, will
            continue parsing skipping lines which fail to parse correctly.
            A sample of the first 10 encountered bad lines will be printed.

        comment_char : string, optional
            The character which denotes that the remainder of the line is a
            comment.

        escape_char : string, optional
            Character which begins a C escape sequence

        double_quote : bool, optional
            If True, two consecutive quotes in a string are parsed to a single
            quote.

        quote_char : string, optional
            Character sequence that indicates a quote.

        skip_initial_space : bool, optional
            Ignore extra spaces at the start of a field

        column_type_hints : None, type, list[type], dict[string, type], optional
            This provides type hints for each column. By default, this method
            attempts to detect the type of each column automatically.

            Supported types are int, float, str, list, dict, and array.array.

            * If a single type is provided, the type will be
              applied to all columns. For instance, column_type_hints=float
              will force all columns to be parsed as float.
            * If a list of types is provided, the types applies
              to each column in order, e.g.[int, float, str]
              will parse the first column as int, second as float and third as
              string.
            * If a dictionary of column name to type is provided,
              each type value in the dictionary is applied to the key it
              belongs to.
              For instance {'user':int} will hint that the column called "user"
              should be parsed as an integer, and the rest will default to
              string.

        na_values : str | list of str, optional
            A string or list of strings to be interpreted as missing values.

        nrows : int, optional
            If set, only this many rows will be read from the file.

        verbose : bool, optional
            If True, print the progress.

        Returns
        -------
        out : SFrame

        See Also
        --------
        read_csv_with_errors, SFrame

        Examples
        --------

        Read a regular csv file, with all default options, automatically
        determine types:

        >>> url = 'http://s3.amazonaws.com/gl-testdata/rating_data_example.csv'
        >>> sf = graphlab.SFrame.read_csv(url)
        >>> sf
        Columns:
          user_id int
          movie_id  int
          rating  int
        Rows: 10000
        +---------+----------+--------+
        | user_id | movie_id | rating |
        +---------+----------+--------+
        |  25904  |   1663   |   3    |
        |  25907  |   1663   |   3    |
        |  25923  |   1663   |   3    |
        |  25924  |   1663   |   3    |
        |  25928  |   1663   |   2    |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [10000 rows x 3 columns]

        Read only the first 100 lines of the csv file:

        >>> sf = graphlab.SFrame.read_csv(url, nrows=100)
        >>> sf
        Columns:
          user_id int
          movie_id  int
          rating  int
        Rows: 100
        +---------+----------+--------+
        | user_id | movie_id | rating |
        +---------+----------+--------+
        |  25904  |   1663   |   3    |
        |  25907  |   1663   |   3    |
        |  25923  |   1663   |   3    |
        |  25924  |   1663   |   3    |
        |  25928  |   1663   |   2    |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [100 rows x 3 columns]

        Read all columns as str type

        >>> sf = graphlab.SFrame.read_csv(url, column_type_hints=str)
        >>> sf
        Columns:
          user_id  str
          movie_id  str
          rating  str
        Rows: 10000
        +---------+----------+--------+
        | user_id | movie_id | rating |
        +---------+----------+--------+
        |  25904  |   1663   |   3    |
        |  25907  |   1663   |   3    |
        |  25923  |   1663   |   3    |
        |  25924  |   1663   |   3    |
        |  25928  |   1663   |   2    |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [10000 rows x 3 columns]

        Specify types for a subset of columns and leave the rest to be str.

        >>> sf = graphlab.SFrame.read_csv(url,
        ...                               column_type_hints={
        ...                               'user_id':int, 'rating':float
        ...                               })
        >>> sf
        Columns:
          user_id str
          movie_id  str
          rating  float
        Rows: 10000
        +---------+----------+--------+
        | user_id | movie_id | rating |
        +---------+----------+--------+
        |  25904  |   1663   |  3.0   |
        |  25907  |   1663   |  3.0   |
        |  25923  |   1663   |  3.0   |
        |  25924  |   1663   |  3.0   |
        |  25928  |   1663   |  2.0   |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [10000 rows x 3 columns]

        Not treat first line as header:

        >>> sf = graphlab.SFrame.read_csv(url, header=False)
        >>> sf
        Columns:
          X1  str
          X2  str
          X3  str
        Rows: 10001
        +---------+----------+--------+
        |    X1   |    X2    |   X3   |
        +---------+----------+--------+
        | user_id | movie_id | rating |
        |  25904  |   1663   |   3    |
        |  25907  |   1663   |   3    |
        |  25923  |   1663   |   3    |
        |  25924  |   1663   |   3    |
        |  25928  |   1663   |   2    |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [10001 rows x 3 columns]

        Treat '3' as missing value:

        >>> sf = graphlab.SFrame.read_csv(url, na_values=['3'], column_type_hints=str)
        >>> sf
        Columns:
          user_id str
          movie_id  str
          rating  str
        Rows: 10000
        +---------+----------+--------+
        | user_id | movie_id | rating |
        +---------+----------+--------+
        |  25904  |   1663   |  None  |
        |  25907  |   1663   |  None  |
        |  25923  |   1663   |  None  |
        |  25924  |   1663   |  None  |
        |  25928  |   1663   |   2    |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [10000 rows x 3 columns]

        Throw error on parse failure:

        >>> bad_url = 'https://s3.amazonaws.com/gl-testdata/bad_csv_example.csv'
        >>> sf = graphlab.SFrame.read_csv(bad_url, error_bad_lines=True)
        RuntimeError: Runtime Exception. Unable to parse line "x,y,z,a,b,c"
        Set error_bad_lines=False to skip bad lines
        """

        return cls._read_csv_impl(url,
                                  delimiter=delimiter,
                                  header=header,
                                  error_bad_lines=error_bad_lines,
                                  comment_char=comment_char,
                                  escape_char=escape_char,
                                  double_quote=double_quote,
                                  quote_char=quote_char,
                                  skip_initial_space=skip_initial_space,
                                  column_type_hints=column_type_hints,
                                  na_values=na_values,
                                  nrows=nrows,
                                  verbose=verbose,
                                  store_errors=False)[0]


    def to_schema_rdd(self,sc,sql,number_of_partitions=4):
        """
        Convert the current SFrame to the Spark SchemaRDD.

        To enable this function, you must add the jar file bundled with GraphLab
        Create to the Spark driver's classpath.  This must happen BEFORE Spark
        launches its JVM, or else it will have no effect.  To do this, first get
        the location of the packaged jar with
        `graphlab.get_spark_integration_jar_path`. You then have two options:

            1. Add the path to the jar to your spark-defaults.conf file.  The
               property to set is 'spark.driver.extraClassPath'.

            OR

            2. Add the jar's path as a command line option to your favorite way to
               start pyspark (either spark-submit or pyspark).  For this, use the
               command line option '--driver-class-path'.

        Parameters
        ----------
        sc : SparkContext
            sc is an existing SparkContext.

        sql : SQLContext
            sql is an existing SQLContext.

        number_of_partitions : int
            number of partitions for the output rdd

        Returns
        ----------
        out: SchemaRDD

        Examples
        --------

        >>> from pyspark import SparkContext, SQLContext
        >>> from graphlab import SFrame
        >>> sc = SparkContext('local')
        >>> sqlc = SQLContext(sc)
        >>> sf = SFrame({'x': [1,2,3], 'y': ['fish', 'chips', 'salad']})
        >>> rdd = sf.to_schema_rdd(sc, sqlc)
        >>> rdd.collect()
        [Row(x=1, y=u'fish'), Row(x=2, y=u'chips'), Row(x=3, y=u'salad')]
        """

        def homogeneous_type(seq):
            if seq is None or len(seq) == 0:
                return True
            iseq = iter(seq)
            first_type = type(next(iseq))
            return True if all( (type(x) is first_type) for x in iseq ) else False

        if len(self) == 0:
            raise ValueError("SFrame is empty")

        column_names = self.column_names()
        first_row = self.head(1)[0]

        for name in column_names:
            if hasattr(first_row[name],'__iter__') and homogeneous_type(first_row[name]) is not True:
                raise TypeError("Support for translation to Spark SchemaRDD not enabled for heterogeneous iterable type (column: %s). Use SFrame.to_rdd()." % name)

        for _type in self.column_types():
                if(_type.__name__ == 'datetime'):
                    raise TypeError("Support for translation to Spark SchemaRDD not enabled for datetime type. Use SFrame.to_rdd() ")

        rdd = self.to_rdd(sc,number_of_partitions);
        from pyspark.sql import Row
        rowRdd = rdd.map(lambda x: Row(**x))
        return sql.inferSchema(rowRdd)

    def to_rdd(self, sc, number_of_partitions=4):
        """
        Convert the current SFrame to the Spark RDD.

        To enable this function, you must add the jar file bundled with GraphLab
        Create to the Spark driver's classpath.  This must happen BEFORE Spark
        launches its JVM, or else it will have no effect.  To do this, first get
        the location of the packaged jar with
        `graphlab.get_spark_integration_jar_path`. You then have two options:

            1. Add the path to the jar to your spark-defaults.conf file.  The
               property to set is 'spark.driver.extraClassPath'.

            OR

            2. Add the jar's path as a command line option to your favorite way to
               start pyspark (either spark-submit or pyspark).  For this, use the
               command line option '--driver-class-path'.

        Parameters
        ----------
        sc : SparkContext
            sc is an existing SparkContext.

        number_of_partitions: int
            number of partitions for the output rdd

        Returns
        ----------
        out: RDD

        Examples
        --------

        >>> from pyspark import SparkContext
        >>> from graphlab import SFrame
        >>> sc = SparkContext('local')
        >>> sf = SFrame({'x': [1,2,3], 'y': ['fish', 'chips', 'salad']})
        >>> rdd = sf.to_rdd(sc)
        >>> rdd.collect()
        [{'x': 1L, 'y': 'fish'}, {'x': 2L, 'y': 'chips'}, {'x': 3L, 'y': 'salad'}]
        """
        _mt._get_metric_tracker().track('sframe.to_rdd')
        if not RDD_SUPPORT:
            raise Exception("Support for translation to Spark RDDs not enabled.")

        for _type in self.column_types():
                if(_type.__name__ == 'Image'):
                    raise TypeError("Support for translation to Spark RDDs not enabled for Image type.")

        if type(number_of_partitions) is not int:
            raise ValueError("number_of_partitions parameter expects an integer type")
        if number_of_partitions == 0:
            raise ValueError("number_of_partitions can not be initialized to zero")

        # Save SFrame in a temporary place
        tmp_loc = self.__get_staging_dir__(sc)
        sf_loc = os.path.join(tmp_loc, str(uuid.uuid4()))
        self.save(sf_loc)

        # Keep track of the temporary sframe that is saved(). We need to delete it eventually.
        dummysf = load_sframe(sf_loc)
        dummysf.__proxy__.delete_on_close()
        SFRAME_GARBAGE_COLLECTOR.append(dummysf)

        sframe_len = self.__len__()
        small_partition_size = sframe_len/number_of_partitions
        big_partition_size = small_partition_size + 1
        num_big_partition_size = sframe_len % number_of_partitions
        num_small_partition_size = number_of_partitions - num_big_partition_size

        count = 0
        start_index = 0
        ranges = []
        while(count < number_of_partitions):
            if(count < num_big_partition_size):
                ranges.append((str(start_index)+":"+str(start_index + big_partition_size)))
                start_index = start_index + big_partition_size
            else:
                ranges.append((str(start_index)+":"+str(start_index + small_partition_size)))
                start_index = start_index + small_partition_size
            count+=1
        from pyspark import RDD
        rdd = sc.parallelize(ranges,number_of_partitions)
        if sc.master[0:5] == 'local':
            pipeRdd = sc._jvm.org.graphlab.create.GraphLabUtil.pythonToJava(
                rdd._jrdd).pipe(
                    BINARY_PATHS['SPARK_PIPE_WRAPPER_PATH'] + \
                        " " + BINARY_PATHS['SFRAME_RDD_PATH'] + " " + sf_loc)
        elif sc.master == 'yarn-client':
            pipeRdd = sc._jvm.org.graphlab.create.GraphLabUtil.pythonToJava(
                rdd._jrdd).pipe(
                    "./" + SPARK_SUPPORT_NAMES['SPARK_PIPE_WRAPPER_PATH'] + \
                    " " + "./" + SPARK_SUPPORT_NAMES['SFRAME_RDD_PATH'] + \
                    " " + sf_loc)
        serializedRdd = sc._jvm.org.graphlab.create.GraphLabUtil.stringToByte(pipeRdd)
        import pyspark
        output_rdd = RDD(serializedRdd,sc,pyspark.serializers.PickleSerializer())

        return output_rdd

    @classmethod
    def __get_staging_dir__(cls,cur_sc):
        if not RDD_SUPPORT_INITED:
            __rdd_support_init__(cur_sc)


        return STAGING_DIR

    @classmethod
    def from_rdd(cls, rdd):
        """
        Convert a Spark RDD into a GraphLab Create SFrame.

        To enable this function, you must add the jar file bundled with GraphLab
        Create to the Spark driver's classpath.  This must happen BEFORE Spark
        launches its JVM, or else it will have no effect.  To do this, first get
        the location of the packaged jar with
        `graphlab.get_spark_integration_jar_path`. You then have two options:

            1. Add the path to the jar to your spark-defaults.conf file.  The
               property to set is 'spark.driver.extraClassPath'.

            OR

            2. Add the jar's path as a command line option to your favorite way to
               start pyspark (either spark-submit or pyspark).  For this, use the
               command line option '--driver-class-path'.

        Parameters
        ----------
        rdd : pyspark.rdd.RDD

        Returns
        -------
        out : SFrame

        Examples
        --------

        >>> from pyspark import SparkContext
        >>> from graphlab import SFrame
        >>> sc = SparkContext('local')
        >>> rdd = sc.parallelize([1,2,3])
        >>> sf = SFrame.from_rdd(rdd)
        >>> sf
        Data:
        +-----+
        |  X1 |
        +-----+
        | 1.0 |
        | 2.0 |
        | 3.0 |
        +-----+
        [3 rows x 1 columns]
        """
        _mt._get_metric_tracker().track('sframe.from_rdd')
        if not RDD_SUPPORT:
            raise Exception("Support for translation to Spark RDDs not enabled.")

        checkRes = rdd.take(1);

        if len(checkRes) > 0 and checkRes[0].__class__.__name__ == 'Row' and rdd.__class__.__name__ not in {'SchemaRDD','DataFrame'}:
            raise Exception("Conversion from RDD(pyspark.sql.Row) to SFrame not supported. Please call inferSchema(RDD) first.")

        if(rdd._jrdd_deserializer.__class__.__name__ == 'UTF8Deserializer'):
            return SFrame.__from_UTF8Deserialized_rdd__(rdd)

        sf_names = None
        rdd_type = "rdd"
        if rdd.__class__.__name__ in {'SchemaRDD','DataFrame'}:
            rdd_type = "schemardd"
            first_row = rdd.take(1)[0]
            if hasattr(first_row, 'keys'):
              sf_names = first_row.keys()
            else:
              sf_names = first_row.__FIELDS__

            sf_names = [str(i) for i in sf_names]


        cur_sc = rdd.ctx

        tmp_loc = SFrame.__get_staging_dir__(cur_sc)

        if tmp_loc is None:
            raise RuntimeError("Could not determine staging directory for SFrame files.")

        mode = "batch"
        if(rdd._jrdd_deserializer.__class__.__name__ == 'PickleSerializer'):
            mode = "pickle"

        if cur_sc.master[0:5] == 'local':
            t = cur_sc._jvm.org.graphlab.create.GraphLabUtil.byteToString(
                rdd._jrdd).pipe(
                    BINARY_PATHS['SPARK_PIPE_WRAPPER_PATH'] + " " + \
                    BINARY_PATHS['RDD_SFRAME_PATH'] + " " + tmp_loc +\
                    " " + mode + " " + rdd_type)
        else:
            t = cur_sc._jvm.org.graphlab.create.GraphLabUtil.byteToString(
                rdd._jrdd).pipe(
                    "./" + SPARK_SUPPORT_NAMES['SPARK_PIPE_WRAPPER_PATH'] +\
                    " " + "./" + SPARK_SUPPORT_NAMES['RDD_SFRAME_PATH'] + " " +\
                    tmp_loc + " " + mode + " " + rdd_type)

        # We get the location of an SFrame index file per Spark partition in
        # the result.  We assume that this is in partition order.
        res = t.collect()

        out_sf = cls()
        sframe_list = []
        for url in res:
            sf = SFrame()
            sf.__proxy__.load_from_sframe_index(_make_internal_url(url))
            sf.__proxy__.delete_on_close()
            out_sf_coltypes = out_sf.column_types()
            if(len(out_sf_coltypes) != 0):
                sf_coltypes = sf.column_types()
                sf_temp_names = sf.column_names()
                out_sf_temp_names = out_sf.column_names()

                for i in range(len(sf_coltypes)):
                    if sf_coltypes[i] != out_sf_coltypes[i]:
                        print "mismatch for types %s and %s" % (sf_coltypes[i],out_sf_coltypes[i])
                        sf[sf_temp_names[i]] = sf[sf_temp_names[i]].astype(str)
                        out_sf[out_sf_temp_names[i]] = out_sf[out_sf_temp_names[i]].astype(str)
            out_sf = out_sf.append(sf)

        out_sf.__proxy__.delete_on_close()

        if sf_names is not None:
            out_names = out_sf.column_names()
            if(set(out_names) != set(sf_names)):
                out_sf = out_sf.rename(dict(zip(out_names, sf_names)))

        return out_sf

    @classmethod
    def __from_UTF8Deserialized_rdd__(cls, rdd):
        _mt._get_metric_tracker().track('sframe.__from_UTF8Deserialized_rdd__')
        if not RDD_SUPPORT:
            raise Exception("Support for translation to Spark RDDs not enabled.")
        cur_sc = rdd.ctx
        sf_names = None
        sf_types = None

        tmp_loc = SFrame.__get_staging_dir__(cur_sc)


        if tmp_loc is None:
            raise RuntimeError("Could not determine staging directory for SFrame files.")


        if(rdd.__class__.__name__ in {'SchemaRDD','DataFrame'}):
            first_row = rdd.take(1)[0]
            if hasattr(first_row, 'keys'):
              sf_names = first_row.keys()
              sf_types = [type(i) for i in first_row.values()]
            else:
              sf_names = first_row.__FIELDS__
              sf_types = [type(i) for i in first_row]

            sf_names = [str(i) for i in sf_names]

            for _type in sf_types:
                if(_type != int and _type != str and _type != float and _type != unicode):
                    raise TypeError("Only int, str, and float are supported for now")

            types = ""
            for i in sf_types:
                types += i.__name__ + ","
            if cur_sc.master[0:5] == 'local':
              t = rdd._jschema_rdd.toJavaStringOfValues().pipe(
                  BINARY_PATHS['SPARK_PIPE_WRAPPER_PATH'] + " " +\
                  BINARY_PATHS['RDD_SFRAME_NONPICKLE_PATH']  + " " + tmp_loc +\
                  " " + types)
            else:
              t = cur_sc._jvm.org.graphlab.create.GraphLabUtil.toJavaStringOfValues(
                  rdd._jschema_rdd).pipe(
                      "./" + SPARK_SUPPORT_NAMES['SPARK_PIPE_WRAPPER_PATH'] +\
                      " " + "./" +\
                      SPARK_SUPPORT_NAMES['RDD_SFRAME_NONPICKLE_PATH'] + " " +\
                      tmp_loc + " " + types)
        else:
            if cur_sc.master[0:5] == 'local':
              t = cur_sc._jvm.org.graphlab.create.GraphLabUtil.pythonToJava(
                  rdd._jrdd).pipe(
                      BINARY_PATHS['SPARK_PIPE_WRAPPER_PATH'] + " " +\
                      BINARY_PATHS['RDD_SFRAME_NONPICKLE_PATH'] +  " " +\
                      tmp_loc)
            else:
              t = cur_sc._jvm.org.graphlab.create.GraphLabUtil.pythonToJava(
                  rdd._jrdd).pipe(
                      "./" + SPARK_SUPPORT_NAMES['SPARK_PIPE_WRAPPER_PATH'] +\
                      " " + "./" +\
                      SPARK_SUPPORT_NAMES['RDD_SFRAME_NONPICKLE_PATH'] + " " +\
                      tmp_loc)

        # We get the location of an SFrame index file per Spark partition in
        # the result.  We assume that this is in partition order.
        res = t.collect()

        out_sf = cls()
        sframe_list = []
        for url in res:
            sf = SFrame()
            sf.__proxy__.load_from_sframe_index(_make_internal_url(url))
            sf.__proxy__.delete_on_close()
            out_sf = out_sf.append(sf)

        out_sf.__proxy__.delete_on_close()

        if sf_names is not None:
            out_names = out_sf.column_names()
            if(set(out_names) != set(sf_names)):
                out_sf = out_sf.rename(dict(zip(out_names, sf_names)))

        return out_sf

    @classmethod
    def from_odbc(cls, db, sql, verbose=False):
        """
        Convert a table or query from a database to an SFrame.

        This function does not do any checking on the given SQL query, and
        cannot know what effect it will have on the database.  Any side effects
        from the query will be reflected on the database.  If no result
        rows are returned, an empty SFrame is created.

        Keep in mind the default case your database stores table names in.  In
        some cases, you may need to add quotation marks (or whatever character
        your database uses to quote identifiers), especially if you created the
        table using `to_odbc`.

        Parameters
        ----------
        db :  `graphlab.extensions._odbc_connection.unity_odbc_connection`
          An ODBC connection object.  This can only be obtained by calling
          `graphlab.connect_odbc`.  Check that documentation for how to create
          this object.

        sql : str
          A SQL query.  The query must be acceptable by the ODBC driver used by
          `graphlab.extensions._odbc_connection.unity_odbc_connection`.

        Returns
        -------
        out : SFrame

        Notes
        -----
        This functionality is only supported when using GraphLab Create
        entirely on your local machine.  Therefore, GraphLab Create's EC2 and
        Hadoop execution modes will not be able to use ODBC.  Note that this
        does not apply to the machine your database is running, which can (and
        often will) be running on a separate machine.

        Examples
        --------
        >>> db = graphlab.connect_odbc("DSN=my_awesome_dsn;UID=user;PWD=mypassword")

        >>> a_table = graphlab.SFrame.from_odbc(db, "SELECT * FROM a_table")

        >>> join_result = graphlab.SFrame.from_odbc(db, 'SELECT * FROM "MyTable" a, "AnotherTable" b WHERE a.id=b.id')
        """
        result = db.execute_query(sql)
        if not isinstance(result, SFrame):
            raise RuntimeError("Cannot create an SFrame for query. No result set.")
        cls = result
        return cls

    def to_odbc(self, db, table_name, append_if_exists=False, verbose=True):
        """
        Convert an SFrame to a table in a database.

        By default, searches for a table in the database with the given name.
        If found, this will attempt to append all the rows of the SFrame to the
        end of the table.  If not, this will create a new table with the given
        name.  This behavior is toggled with the `append_if_exists` flag.

        When creating a new table, GraphLab Create uses a heuristic approach to
        pick a corresponding type for each column in the SFrame using the type
        information supplied by the database's ODBC driver.  Your driver must
        support giving this type information for GraphLab Create to support
        writing to the database.

        To allow more expressive and accurate naming, `to_odbc` puts quotes
        around each identifier (table names and column names).  Depending on
        your database, you may need to refer to the created table with quote
        characters around the name.  This character is not the same for all
        databases, but '"' is the most common.

        Parameters
        ----------
        db :  `graphlab.extensions._odbc_connection.unity_odbc_connection`
          An ODBC connection object.  This can only be obtained by calling
          `graphlab.connect_odbc`.  Check that documentation for how to create
          this object.

        table_name : str
          The name of the table you would like to create/append to.

        append_if_exists : bool
          If True, this will attempt to append to the table named `table_name`
          if it is found to exist in the database.

        verbose : bool
          Print progress updates on the insertion process.

        Notes
        -----
        This functionality is only supported when using GraphLab Create
        entirely on your local machine.  Therefore, GraphLab Create's EC2 and
        Hadoop execution modes will not be able to use ODBC.  Note that this
        "local machine" rule does not apply to the machine your database is
        running on, which can (and often will) be running on a separate
        machine.

        Examples
        --------
        >>> db = graphlab.connect_odbc("DSN=my_awesome_dsn;UID=user;PWD=mypassword")

        >>> sf = graphlab.SFrame({'a':[1,2,3],'b':['hi','pika','bye']})

        >>> sf.to_odbc(db, 'a_cool_table')
        """
        if (not verbose):
            glconnect.get_client().set_log_progress(False)

        db._insert_sframe(self, table_name, append_if_exists)

        if (not verbose):
            glconnect.get_client().set_log_progress(True)

    def __repr__(self):
        """
        Returns a string description of the frame
        """
        printed_sf = self._imagecols_to_stringcols()
        ret = self.__get_column_description__()
        if self.__has_size__():
            ret = ret + "Rows: " + str(len(self)) + "\n\n"
        else:
            ret = ret + "Rows: Unknown" + "\n\n"
        ret = ret + "Data:\n"
        if (len(printed_sf.head()) > 0):
            ret = ret + str(self)
        else:
            ret = ret + "\t[]"
        return ret

    def __get_column_description__(self):
        colnames = self.column_names()
        coltypes = self.column_types()
        ret = "Columns:\n"
        if len(colnames) > 0:
            for i in range(len(colnames)):
                ret = ret + "\t" + colnames[i] + "\t" + coltypes[i].__name__ + "\n"
            ret = ret + "\n"
        else:
            ret = ret + "\tNone\n\n"
        return ret

    def __get_pretty_tables__(self, wrap_text=False, max_row_width=80,
                              max_column_width=30, max_columns=20,
                              max_rows_to_display=60):
        """
        Returns a list of pretty print tables representing the current SFrame.
        If the number of columns is larger than max_columns, the last pretty
        table will contain an extra column of "...".

        Parameters
        ----------
        wrap_text : bool, optional

        max_row_width : int, optional
            Max number of characters per table.

        max_column_width : int, optional
            Max number of characters per column.

        max_columns : int, optional
            Max number of columns per table.

        max_rows_to_display : int, optional
            Max number of rows to display.

        Returns
        -------
        out : list[PrettyTable]
        """
        headsf = self.head(max_rows_to_display)
        if headsf.shape == (0, 0):
            return [PrettyTable()]

        # convert array.array column to list column so they print like [...]
        # and not array('d', ...)
        for col in headsf.column_names():
            if headsf[col].dtype() is array.array:
                headsf[col] = headsf[col].astype(list)

        def _value_to_str(value):
            if (type(value) is array.array):
                return str(list(value))
            elif (type(value) is list):
                return '[' + ", ".join(_value_to_str(x) for x in value) + ']'
            else:
                return str(value)

        def _escape_space(s):
            return "".join([ch.encode('string_escape') if ch.isspace() else ch for ch in s])

        def _truncate_respect_unicode(s, max_length):
            if (len(s) <= max_length):
                return s
            else:
                u = unicode(s, 'utf-8', errors='replace')
                return u[:max_length].encode('utf-8')

        def _truncate_str(s, wrap_str=False):
            """
            Truncate and optionally wrap the input string as unicode, replace
            unconvertible character with a diamond ?.
            """
            s = _escape_space(s)

            if len(s) <= max_column_width:
                return unicode(s, 'utf-8', errors='replace')
            else:
                ret = ''
                # if wrap_str is true, wrap the text and take at most 2 rows
                if wrap_str:
                    wrapped_lines = wrap(s, max_column_width)
                    if len(wrapped_lines) == 1:
                        return wrapped_lines[0]
                    last_line = wrapped_lines[1]
                    if len(last_line) >= max_column_width:
                        last_line = _truncate_respect_unicode(last_line, max_column_width - 4)
                    ret = wrapped_lines[0] + '\n' + last_line + ' ...'
                else:
                    ret = _truncate_respect_unicode(s, max_column_width - 4) + '...'
                return unicode(ret, 'utf-8', errors='replace')

        columns = self.column_names()[:max_columns]
        columns.reverse()  # reverse the order of columns and we will pop from the end

        num_column_of_last_table = 0
        row_of_tables = []
        # let's build a list of tables with max_columns
        # each table should satisfy, max_row_width, and max_column_width
        while len(columns) > 0:
            tbl = PrettyTable()
            table_width = 0
            num_column_of_last_table = 0
            while len(columns) > 0:
                col = columns.pop()
                # check the max length of element in the column
                if len(headsf) > 0:
                    col_width = min(max_column_width, max(len(str(x)) for x in headsf[col]))
                else:
                    col_width = max_column_width
                if (table_width + col_width < max_row_width):
                    # truncate the header if necessary
                    header = _truncate_str(col, wrap_text)
                    tbl.add_column(header, [_truncate_str(_value_to_str(x), wrap_text) for x in headsf[col]])
                    table_width = str(tbl).find('\n')
                    num_column_of_last_table += 1
                else:
                    # the column does not fit in the current table, push it back to columns
                    columns.append(col)
                    break
            tbl.align = 'c'
            row_of_tables.append(tbl)

        # add a column of all "..." if there are more columns than displayed
        if self.num_cols() > max_columns:
            row_of_tables[-1].add_column('...', ['...'] * len(headsf))
            num_column_of_last_table += 1

        # add a row of all "..." if there are more rows than displayed
        if self.__has_size__() and self.num_rows() > headsf.num_rows():
            row_of_tables[-1].add_row(['...'] * num_column_of_last_table)
        return row_of_tables

    def print_rows(self, num_rows=10, num_columns=40, max_column_width=30,
                   max_row_width=80):
        """
        Print the first M rows and N columns of the SFrame in human readable
        format.

        Parameters
        ----------
        num_rows : int, optional
            Number of rows to print.

        num_columns : int, optional
            Number of columns to print.

        max_column_width : int, optional
            Maximum width of a column. Columns use fewer characters if possible.

        max_row_width : int, optional
            Maximum width of a printed row. Columns beyond this width wrap to a
            new line. `max_row_width` is automatically reset to be the
            larger of itself and `max_column_width`.

        See Also
        --------
        head, tail
        """

        max_row_width = max(max_row_width, max_column_width + 1)

        printed_sf = self._imagecols_to_stringcols(num_rows)
        row_of_tables = printed_sf.__get_pretty_tables__(wrap_text=False,
                                                         max_rows_to_display=num_rows,
                                                         max_columns=num_columns,
                                                         max_column_width=max_column_width,
                                                         max_row_width=max_row_width)
        footer = "[%d rows x %d columns]\n" % self.shape
        print '\n'.join([str(tb) for tb in row_of_tables]) + "\n" + footer

    def _imagecols_to_stringcols(self, num_rows=10):
        # A list of column types
        types = self.column_types()
        # A list of indexable column names
        names = self.column_names()

        # Constructing names of sframe columns that are of image type
        image_column_names = [names[i] for i in range(len(names)) if types[i] == graphlab.Image]

        #If there are image-type columns, copy the SFrame and cast the top MAX_NUM_ROWS_TO_DISPLAY of those columns to string
        if len(image_column_names) > 0:
            printed_sf = SFrame()
            for t in names:
                if t in image_column_names:
                    printed_sf[t] = self[t]._head_str(num_rows)
                else:
                    printed_sf[t] = self[t].head(num_rows)
        else:
            printed_sf = self
        return printed_sf

    def __str__(self, num_rows=10, footer=True):
        """
        Returns a string containing the first 10 elements of the frame, along
        with a description of the frame.
        """
        MAX_ROWS_TO_DISPLAY = num_rows

        printed_sf = self._imagecols_to_stringcols(MAX_ROWS_TO_DISPLAY)

        row_of_tables = printed_sf.__get_pretty_tables__(wrap_text=False, max_rows_to_display=MAX_ROWS_TO_DISPLAY)
        if (not footer):
            return '\n'.join([str(tb) for tb in row_of_tables])

        if self.__has_size__():
            footer = '[%d rows x %d columns]\n' % self.shape
            if (self.num_rows() > MAX_ROWS_TO_DISPLAY):
                footer += '\n'.join(FOOTER_STRS)
        else:
            footer = '[? rows x %d columns]\n' % self.num_columns()
            footer += '\n'.join(LAZY_FOOTER_STRS)
        return '\n'.join([str(tb) for tb in row_of_tables]) + "\n" + footer

    def _repr_html_(self):
        MAX_ROWS_TO_DISPLAY = 10

        printed_sf = self._imagecols_to_stringcols(MAX_ROWS_TO_DISPLAY)

        row_of_tables = printed_sf.__get_pretty_tables__(wrap_text=True, max_row_width=120, max_columns=40, max_column_width=25, max_rows_to_display=MAX_ROWS_TO_DISPLAY)
        if self.__has_size__():
            footer = '[%d rows x %d columns]<br/>' % self.shape
            if (self.num_rows() > MAX_ROWS_TO_DISPLAY):
                footer += '<br/>'.join(FOOTER_STRS)
        else:
            footer = '[? rows x %d columns]<br/>' % self.num_columns()
            footer += '<br/>'.join(LAZY_FOOTER_STRS)
        begin = '<div style="max-height:1000px;max-width:1500px;overflow:auto;">'
        end = '\n</div>'
        return begin + '\n'.join([tb.get_html_string(format=True) for tb in row_of_tables]) + "\n" + footer + end

    def __nonzero__(self):
        """
        Returns true if the frame is not empty.
        """
        return self.num_rows() != 0

    def __len__(self):
        """
        Returns the number of rows of the sframe.
        """
        return self.num_rows()

    def __copy__(self):
        """
        Returns a shallow copy of the sframe.
        """
        return self.select_columns(self.column_names())

    def __eq__(self, other):
        raise NotImplementedError

    def __ne__(self, other):
        raise NotImplementedError

    def _row_selector(self, other):
        """
        Where other is an SArray of identical length as the current Frame,
        this returns a selection of a subset of rows in the current SFrame
        where the corresponding row in the selector is non-zero.
        """
        if type(other) is SArray:
            if len(other) != len(self):
                raise IndexError("Cannot perform logical indexing on arrays of different length.")
            with cython_context():
                return SFrame(_proxy=self.__proxy__.logical_filter(other.__proxy__))

    def dtype(self):
        """
        The type of each column.

        Returns
        -------
        out : list[type]
            Column types of the SFrame.

        See Also
        --------
        column_types
        """
        return self.column_types()

    def num_rows(self):
        """
        The number of rows in this SFrame.

        Returns
        -------
        out : int
            Number of rows in the SFrame.

        See Also
        --------
        num_columns
        """
        return self.__proxy__.num_rows()

    def num_cols(self):
        """
        The number of columns in this SFrame.

        Returns
        -------
        out : int
            Number of columns in the SFrame.

        See Also
        --------
        num_columns, num_rows
        """
        return self.__proxy__.num_columns()

    def num_columns(self):
        """
        The number of columns in this SFrame.

        Returns
        -------
        out : int
            Number of columns in the SFrame.

        See Also
        --------
        num_cols, num_rows
        """
        return self.__proxy__.num_columns()

    def column_names(self):
        """
        The name of each column in the SFrame.

        Returns
        -------
        out : list[string]
            Column names of the SFrame.

        See Also
        --------
        rename
        """
        return self.__proxy__.column_names()

    def column_types(self):
        """
        The type of each column in the SFrame.

        Returns
        -------
        out : list[type]
            Column types of the SFrame.

        See Also
        --------
        dtype
        """
        return self.__proxy__.dtype()

    def head(self, n=10):
        """
        The first n rows of the SFrame.

        Parameters
        ----------
        n : int, optional
            The number of rows to fetch.

        Returns
        -------
        out : SFrame
            A new SFrame which contains the first n rows of the current SFrame

        See Also
        --------
        tail, print_rows
        """
        return SFrame(_proxy=self.__proxy__.head(n))

    def to_dataframe(self):
        """
        Convert this SFrame to pandas.DataFrame.

        This operation will construct a pandas.DataFrame in memory. Care must
        be taken when size of the returned object is big.

        Returns
        -------
        out : pandas.DataFrame
            The dataframe which contains all rows of SFrame
        """
        assert HAS_PANDAS
        df = pandas.DataFrame()
        for i in range(self.num_columns()):
            column_name = self.column_names()[i]
            df[column_name] = list(self[column_name])
            if len(df[column_name]) == 0:
                df[column_name] = df[column_name].astype(self.column_types()[i])
        return df

    def tail(self, n=10):
        """
        The last n rows of the SFrame.

        Parameters
        ----------
        n : int, optional
            The number of rows to fetch.

        Returns
        -------
        out : SFrame
            A new SFrame which contains the last n rows of the current SFrame

        See Also
        --------
        head, print_rows
        """
        return SFrame(_proxy=self.__proxy__.tail(n))

    def apply(self, fn, dtype=None, seed=None):
        """
        Transform each row to an :class:`~graphlab.SArray` according to a
        specified function. Returns a new SArray of ``dtype`` where each element
        in this SArray is transformed by `fn(x)` where `x` is a single row in
        the sframe represented as a dictionary.  The ``fn`` should return
        exactly one value which can be cast into type ``dtype``. If ``dtype`` is
        not specified, the first 100 rows of the SFrame are used to make a guess
        of the target data type.

        Parameters
        ----------
        fn : function
            The function to transform each row of the SFrame. The return
            type should be convertible to `dtype` if `dtype` is not None.
            This can also be a toolkit extension function which is compiled
            as a native shared library using SDK.

        dtype : dtype, optional
            The dtype of the new SArray. If None, the first 100
            elements of the array are used to guess the target
            data type.

        seed : int, optional
            Used as the seed if a random number generator is included in `fn`.

        Returns
        -------
        out : SArray
            The SArray transformed by fn.  Each element of the SArray is of
            type ``dtype``

        Examples
        --------
        Concatenate strings from several columns:

        >>> sf = graphlab.SFrame({'user_id': [1, 2, 3], 'movie_id': [3, 3, 6],
                                  'rating': [4, 5, 1]})
        >>> sf.apply(lambda x: str(x['user_id']) + str(x['movie_id']) + str(x['rating']))
        dtype: str
        Rows: 3
        ['134', '235', '361']

        Using native toolkit extension function:

        .. code-block:: c++

            #include <graphlab/sdk/toolkit_function_macros.hpp>
            double mean(const std::map<flexible_type, flexible_type>& dict) {
              double sum = 0.0;
              for (const auto& kv: dict) sum += (double)kv.second;
              return sum / dict.size();
            }

            BEGIN_FUNCTION_REGISTRATION
            REGISTER_FUNCTION(mean, "row");
            END_FUNCTION_REGISTRATION

        compiled into example.so

        >>> import example

        >>> sf = graphlab.SFrame({'x0': [1, 2, 3], 'x1': [2, 3, 1],
        ...                       'x2': [3, 1, 2]})
        >>> sf.apply(example.mean)
        dtype: float
        Rows: 3
        [2.0,2.0,2.0]
        """
        assert _is_callable(fn), "Input must be a function"
        test_sf = self[:10]
        dryrun = [fn(row) for row in test_sf]
        if dtype is None:
            dtype = SArray(dryrun).dtype()

        if not seed:
            seed = int(time.time())

        _mt._get_metric_tracker().track('sframe.apply')

        nativefn = None
        try:
            import graphlab.extensions as extensions
            nativefn = extensions._build_native_function_call(fn)
        except:
            pass

        if nativefn is not None:
            # this is a toolkit lambda. We can do something about it
            with cython_context():
                return SArray(_proxy=self.__proxy__.transform_native(nativefn, dtype, seed))

        with cython_context():
            return SArray(_proxy=self.__proxy__.transform(fn, dtype, seed))

    def flat_map(self, column_names, fn, column_types='auto', seed=None):
        """
        Map each row of the SFrame to multiple rows in a new SFrame via a
        function.

        The output of `fn` must have type List[List[...]].  Each inner list
        will be a single row in the new output, and the collection of these
        rows within the outer list make up the data for the output SFrame.
        All rows must have the same length and the same order of types to
        make sure the result columns are homogeneously typed.  For example, if
        the first element emitted into in the outer list by `fn` is
        [43, 2.3, 'string'], then all other elements emitted into the outer
        list must be a list with three elements, where the first is an int,
        second is a float, and third is a string.  If column_types is not
        specified, the first 10 rows of the SFrame are used to determine the
        column types of the returned sframe.

        Parameters
        ----------
        column_names : list[str]
            The column names for the returned SFrame.

        fn : function
            The function that maps each of the sframe row into multiple rows,
            returning List[List[...]].  All outputted rows must have the same
            length and order of types.

        column_types : list[type], optional
            The column types of the output SFrame. Default value will be
            automatically inferred by running `fn` on the first 10 rows of the
            input. If the types cannot be inferred from the first 10 rows, an
            error is raised.

        seed : int, optional
            Used as the seed if a random number generator is included in `fn`.

        Returns
        -------
        out : SFrame
            A new SFrame containing the results of the flat_map of the
            original SFrame.

        Examples
        ---------
        Repeat each row according to the value in the 'number' column.

        >>> sf = graphlab.SFrame({'letter': ['a', 'b', 'c'],
        ...                       'number': [1, 2, 3]})
        >>> sf.flat_map(['number', 'letter'],
        ...             lambda x: [list(x.itervalues()) for i in range(0, x['number'])])
        +--------+--------+
        | number | letter |
        +--------+--------+
        |   1    |   a    |
        |   2    |   b    |
        |   2    |   b    |
        |   3    |   c    |
        |   3    |   c    |
        |   3    |   c    |
        +--------+--------+
        [6 rows x 2 columns]
        """
        assert inspect.isfunction(fn), "Input must be a function"
        if not seed:
            seed = int(time.time())

        _mt._get_metric_tracker().track('sframe.flat_map')

        # determine the column_types
        if column_types == 'auto':
            types = set()
            sample = self[0:10]
            results = [fn(row) for row in sample]

            for rows in results:
                if type(rows) is not list:
                    raise TypeError("Output type of the lambda function must be a list of lists")

                # note: this skips empty lists
                for row in rows:
                    if type(row) is not list:
                        raise TypeError("Output type of the lambda function must be a list of lists")
                    types.add(tuple([type(v) for v in row]))

            if len(types) == 0:
                raise TypeError, \
                    "Could not infer output column types from the first ten rows " +\
                    "of the SFrame. Please use the 'column_types' parameter to " +\
                    "set the types."

            if len(types) > 1:
                raise TypeError("Mapped rows must have the same length and types")

            column_types = list(types.pop())

        assert type(column_types) is list
        assert len(column_types) == len(column_names), "Number of output columns must match the size of column names"
        with cython_context():
            return SFrame(_proxy=self.__proxy__.flat_map(fn, column_names, column_types, seed))

    def sample(self, fraction, seed=None):
        """
        Sample the current SFrame's rows.

        Parameters
        ----------
        fraction : float
            Approximate fraction of the rows to fetch. Must be between 0 and 1.
            The number of rows returned is approximately the fraction times the
            number of rows.

        seed : int, optional
            Seed for the random number generator used to sample.

        Returns
        -------
        out : SFrame
            A new SFrame containing sampled rows of the current SFrame.

        Examples
        --------
        Suppose we have an SFrame with 6,145 rows.

        >>> import random
        >>> sf = SFrame({'id': range(0, 6145)})

        Retrieve about 30% of the SFrame rows with repeatable results by
        setting the random seed.

        >>> len(sf.sample(.3, seed=5))
        1783
        """
        if not seed:
            seed = int(time.time())

        if (fraction > 1 or fraction < 0):
            raise ValueError('Invalid sampling rate: ' + str(fraction))

        _mt._get_metric_tracker().track('sframe.sample')

        if (self.num_rows() == 0 or self.num_cols() == 0):
            return self
        else:
            with cython_context():
                return SFrame(_proxy=self.__proxy__.sample(fraction, seed))

    def random_split(self, fraction, seed=None):
        """
        Randomly split the rows of an SFrame into two SFrames. The first SFrame
        contains *M* rows, sampled uniformly (without replacement) from the
        original SFrame. *M* is approximately the fraction times the original
        number of rows. The second SFrame contains the remaining rows of the
        original SFrame.

        Parameters
        ----------
        fraction : float
            Approximate fraction of the rows to fetch for the first returned
            SFrame. Must be between 0 and 1.

        seed : int, optional
            Seed for the random number generator used to split.

        Returns
        -------
        out : tuple [SFrame]
            Two new SFrames.

        Examples
        --------
        Suppose we have an SFrame with 1,024 rows and we want to randomly split
        it into training and testing datasets with about a 90%/10% split.

        >>> sf = graphlab.SFrame({'id': range(1024)})
        >>> sf_train, sf_test = sf.random_split(.9, seed=5)
        >>> print len(sf_train), len(sf_test)
        922 102
        """
        if (fraction > 1 or fraction < 0):
            raise ValueError('Invalid sampling rate: ' + str(fraction))
        if (self.num_rows() == 0 or self.num_cols() == 0):
            return (SFrame(), SFrame())

        if not seed:
            seed = int(time.time())

        # The server side requires this to be an int, so cast if we can
        try:
            seed = int(seed)
        except ValueError:
            raise ValueError('The \'seed\' parameter must be of type int.')

        _mt._get_metric_tracker().track('sframe.random_split')

        with cython_context():
            proxy_pair = self.__proxy__.random_split(fraction, seed)
            return (SFrame(data=[], _proxy=proxy_pair[0]), SFrame(data=[], _proxy=proxy_pair[1]))

    def topk(self, column_name, k=10, reverse=False):
        """
        Get top k rows according to the given column. Result is according to and
        sorted by `column_name` in the given order (default is descending).
        When `k` is small, `topk` is more efficient than `sort`.

        Parameters
        ----------
        column_name : string
            The column to sort on

        k : int, optional
            The number of rows to return

        reverse : bool, optional
            If True, return the top k rows in ascending order, otherwise, in
            descending order.

        Returns
        -------
        out : SFrame
            an SFrame containing the top k rows sorted by column_name.

        See Also
        --------
        sort

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': range(1000)})
        >>> sf['value'] = -sf['id']
        >>> sf.topk('id', k=3)
        +--------+--------+
        |   id   |  value |
        +--------+--------+
        |   999  |  -999  |
        |   998  |  -998  |
        |   997  |  -997  |
        +--------+--------+
        [3 rows x 2 columns]

        >>> sf.topk('value', k=3)
        +--------+--------+
        |   id   |  value |
        +--------+--------+
        |   1    |  -1    |
        |   2    |  -2    |
        |   3    |  -3    |
        +--------+--------+
        [3 rows x 2 columns]
        """
        if type(column_name) is not str:
            raise TypeError("column_name must be a string")

        _mt._get_metric_tracker().track('sframe.topk')

        sf = self[self[column_name].topk_index(k, reverse)]
        return sf.sort(column_name, ascending=reverse)

    def save(self, filename, format=None):
        """
        Save the SFrame to a file system for later use.

        Parameters
        ----------
        filename : string
            The location to save the SFrame. Either a local directory or a
            remote URL. If the format is 'binary', a directory will be created
            at the location which will contain the sframe.

        format : {'binary', 'csv'}, optional
            Format in which to save the SFrame. Binary saved SFrames can be
            loaded much faster and without any format conversion losses. If not
            given, will try to infer the format from filename given. If file
            name ends with 'csv' or '.csv.gz', then save as 'csv' format,
            otherwise save as 'binary' format.

        See Also
        --------
        load_sframe, SFrame

        Examples
        --------
        >>> # Save the sframe into binary format
        >>> sf.save('data/training_data_sframe')

        >>> # Save the sframe into csv format
        >>> sf.save('data/training_data.csv', format='csv')
        """

        _mt._get_metric_tracker().track('sframe.save', properties={'format':format})
        if format == None:
            if filename.endswith(('.csv', '.csv.gz')):
                format = 'csv'
            else:
                format = 'binary'
        else:
            if format is 'csv':
                if not filename.endswith(('.csv', '.csv.gz')):
                    filename = filename + '.csv'
            elif format is not 'binary':
                raise ValueError("Invalid format: {}. Supported formats are 'csv' and 'binary'".format(format))

        ## Save the SFrame
        url = _make_internal_url(filename)

        with cython_context():
            if format is 'binary':
                self.__proxy__.save(url)

            elif format is 'csv':
                assert filename.endswith(('.csv', '.csv.gz'))
                self.__proxy__.save_as_csv(url, {})
            else:
                raise ValueError("Unsupported format: {}".format(format))

    def select_column(self, key):
        """
        Get a reference to the :class:`~graphlab.SArray` that corresponds with
        the given key. Throws an exception if the key is something other than a
        string or if the key is not found.

        Parameters
        ----------
        key : str
            The column name.

        Returns
        -------
        out : SArray
            The SArray that is referred by ``key``.

        See Also
        --------
        select_columns

        Examples
        --------
        >>> sf = graphlab.SFrame({'user_id': [1,2,3],
        ...                       'user_name': ['alice', 'bob', 'charlie']})
        >>> # This line is equivalent to `sa = sf['user_name']`
        >>> sa = sf.select_column('user_name')
        >>> sa
        dtype: str
        Rows: 3
        ['alice', 'bob', 'charlie']
        """
        if not isinstance(key, str):
            raise TypeError("Invalid key type: must be str")
        with cython_context():
            return SArray(data=[], _proxy=self.__proxy__.select_column(key))

    def select_columns(self, keylist):
        """
        Get SFrame composed only of the columns referred to in the given list of
        keys. Throws an exception if ANY of the keys are not in this SFrame or
        if ``keylist`` is anything other than a list of strings.

        Parameters
        ----------
        keylist : list[str]
            The list of column names.

        Returns
        -------
        out : SFrame
            A new SFrame that is made up of the columns referred to in
            ``keylist`` from the current SFrame.

        See Also
        --------
        select_column

        Examples
        --------
        >>> sf = graphlab.SFrame({'user_id': [1,2,3],
        ...                       'user_name': ['alice', 'bob', 'charlie'],
        ...                       'zipcode': [98101, 98102, 98103]
        ...                      })
        >>> # This line is equivalent to `sf2 = sf[['user_id', 'zipcode']]`
        >>> sf2 = sf.select_columns(['user_id', 'zipcode'])
        >>> sf2
        +---------+---------+
        | user_id | zipcode |
        +---------+---------+
        |    1    |  98101  |
        |    2    |  98102  |
        |    3    |  98103  |
        +---------+---------+
        [3 rows x 2 columns]
        """
        if not hasattr(keylist, '__iter__'):
            raise TypeError("keylist must be an iterable")
        if not all([isinstance(x, str) for x in keylist]):
            raise TypeError("Invalid key type: must be str")

        key_set = set(keylist)
        if (len(key_set)) != len(keylist):
            for key in key_set:
                if keylist.count(key) > 1:
                    raise ValueError("There are duplicate keys in key list: '" + key + "'")

        with cython_context():
            return SFrame(data=[], _proxy=self.__proxy__.select_columns(keylist))

    def add_column(self, data, name=""):
        """
        Add a column to this SFrame. The number of elements in the data given
        must match the length of every other column of the SFrame. This
        operation modifies the current SFrame in place and returns self. If no
        name is given, a default name is chosen.

        Parameters
        ----------
        data : SArray
            The 'column' of data to add.

        name : string, optional
            The name of the column. If no name is given, a default name is
            chosen.

        Returns
        -------
        out : SFrame
            The current SFrame.

        See Also
        --------
        add_columns

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': [1, 2, 3], 'val': ['A', 'B', 'C']})
        >>> sa = graphlab.SArray(['cat', 'dog', 'fossa'])
        >>> # This line is equivalant to `sf['species'] = sa`
        >>> sf.add_column(sa, name='species')
        >>> sf
        +----+-----+---------+
        | id | val | species |
        +----+-----+---------+
        | 1  |  A  |   cat   |
        | 2  |  B  |   dog   |
        | 3  |  C  |  fossa  |
        +----+-----+---------+
        [3 rows x 3 columns]
        """
        # Check type for pandas dataframe or SArray?
        if not isinstance(data, SArray):
            raise TypeError("Must give column as SArray")
        if not isinstance(name, str):
            raise TypeError("Invalid column name: must be str")
        with cython_context():
            self.__proxy__.add_column(data.__proxy__, name)
        return self

    def add_columns(self, data, namelist=None):
        """
        Adds multiple columns to this SFrame. The number of elements in all
        columns must match the length of every other column of the SFrame. This
        operation modifies the current SFrame in place and returns self.

        Parameters
        ----------
        data : list[SArray] or SFrame
            The columns to add.

        namelist : list of string, optional
            A list of column names. All names must be specified. ``namelist`` is
            ignored if data is an SFrame.

        Returns
        -------
        out : SFrame
            The current SFrame.

        See Also
        --------
        add_column

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': [1, 2, 3], 'val': ['A', 'B', 'C']})
        >>> sf2 = graphlab.SFrame({'species': ['cat', 'dog', 'fossa'],
        ...                        'age': [3, 5, 9]})
        >>> sf.add_columns(sf2)
        >>> sf
        +----+-----+-----+---------+
        | id | val | age | species |
        +----+-----+-----+---------+
        | 1  |  A  |  3  |   cat   |
        | 2  |  B  |  5  |   dog   |
        | 3  |  C  |  9  |  fossa  |
        +----+-----+-----+---------+
        [3 rows x 4 columns]
        """
        datalist = data
        if isinstance(data, SFrame):
            other = data
            datalist = [other.select_column(name) for name in other.column_names()]
            namelist = other.column_names()

            my_columns = set(self.column_names())
            for name in namelist:
                if name in my_columns:
                    raise ValueError("Column '" + name + "' already exists in current SFrame")
        else:
            if not hasattr(datalist, '__iter__'):
                raise TypeError("datalist must be an iterable")
            if not hasattr(namelist, '__iter__'):
                raise TypeError("namelist must be an iterable")

            if not all([isinstance(x, SArray) for x in datalist]):
                raise TypeError("Must give column as SArray")
            if not all([isinstance(x, str) for x in namelist]):
                raise TypeError("Invalid column name in list : must all be str")

        with cython_context():
            self.__proxy__.add_columns([x.__proxy__ for x in datalist], namelist)
        return self

    def remove_column(self, name):
        """
        Remove a column from this SFrame. This operation modifies the current
        SFrame in place and returns self.

        Parameters
        ----------
        name : string
            The name of the column to remove.

        Returns
        -------
        out : SFrame
            The SFrame with given column removed.

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': [1, 2, 3], 'val': ['A', 'B', 'C']})
        >>> # This is equivalent to `del sf['val']`
        >>> sf.remove_column('val')
        >>> sf
        +----+
        | id |
        +----+
        | 1  |
        | 2  |
        | 3  |
        +----+
        [3 rows x 1 columns]
        """
        if name not in self.column_names():
            raise KeyError('Cannot find column %s' % name)
        colid = self.column_names().index(name)
        with cython_context():
            self.__proxy__.remove_column(colid)
        return self

    def remove_columns(self, column_names):
        """
        Remove one or more columns from this SFrame. This operation modifies the current
        SFrame in place and returns self.

        Parameters
        ----------
        column_names : list or iterable
            A list or iterable of column names.

        Returns
        -------
        out : SFrame
            The SFrame with given columns removed.

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': [1, 2, 3], 'val1': ['A', 'B', 'C'], 'val2' : [10, 11, 12]})
        >>> sf.remove_columns(['val1', 'val2'])
        >>> sf
        +----+
        | id |
        +----+
        | 1  |
        | 2  |
        | 3  |
        +----+
        [3 rows x 1 columns]
        """
        column_names = list(column_names)
        existing_columns = dict((k, i) for i, k in enumerate(self.column_names()))

        for name in column_names:
            if name not in existing_columns:
                raise KeyError('Cannot find column %s' % name)

        # Delete it going backwards so we don't invalidate indices
        deletion_indices = sorted(existing_columns[name] for name in column_names)
        for colid in reversed(deletion_indices):
            with cython_context():
                self.__proxy__.remove_column(colid)
        return self


    def swap_columns(self, column_1, column_2):
        """
        Swap the columns with the given names. This operation modifies the
        current SFrame in place and returns self.

        Parameters
        ----------
        column_1 : string
            Name of column to swap

        column_2 : string
            Name of other column to swap

        Returns
        -------
        out : SFrame
            The SFrame with swapped columns.

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': [1, 2, 3], 'val': ['A', 'B', 'C']})
        >>> sf.swap_columns('id', 'val')
        >>> sf
        +-----+-----+
        | val | id  |
        +-----+-----+
        |  A  |  1  |
        |  B  |  2  |
        |  C  |  3  |
        +----+-----+
        [3 rows x 2 columns]
        """
        colnames = self.column_names()
        colid_1 = colnames.index(column_1)
        colid_2 = colnames.index(column_2)
        with cython_context():
            self.__proxy__.swap_columns(colid_1, colid_2)
        return self

    def rename(self, names):
        """
        Rename the given columns. ``names`` is expected to be a dict specifying
        the old and new names. This changes the names of the columns given as
        the keys and replaces them with the names given as the values.  This
        operation modifies the current SFrame in place and returns self.

        Parameters
        ----------
        names : dict [string, string]
            Dictionary of [old_name, new_name]

        Returns
        -------
        out : SFrame
            The current SFrame.

        See Also
        --------
        column_names

        Examples
        --------
        >>> sf = SFrame({'X1': ['Alice','Bob'],
        ...              'X2': ['123 Fake Street','456 Fake Street']})
        >>> sf.rename({'X1': 'name', 'X2':'address'})
        >>> sf
        +-------+-----------------+
        |  name |     address     |
        +-------+-----------------+
        | Alice | 123 Fake Street |
        |  Bob  | 456 Fake Street |
        +-------+-----------------+
        [2 rows x 2 columns]
        """
        if (type(names) is not dict):
            raise TypeError('names must be a dictionary: oldname -> newname')
        all_columns = set(self.column_names())
        for k in names:
            if not k in all_columns:
                raise ValueError('Cannot find column %s in the SFrame' % k)
        with cython_context():
            for k in names:
                colid = self.column_names().index(k)
                self.__proxy__.set_column_name(colid, names[k])
        return self

    def __getitem__(self, key):
        """
        This method does things based on the type of `key`.

        If `key` is:
            * str
                Calls `select_column` on `key`
            * SArray
                Performs a logical filter.  Expects given SArray to be the same
                length as all columns in current SFrame.  Every row
                corresponding with an entry in the given SArray that is
                equivalent to False is filtered from the result.
            * int
                Returns a single row of the SFrame (the `key`th one) as a dictionary.
            * slice
                Returns an SFrame including only the sliced rows.
        """
        if type(key) is SArray:
            return self._row_selector(key)
        elif type(key) is list:
            return self.select_columns(key)
        elif type(key) is str:
            return self.select_column(key)
        elif type(key) is int:
            if key < 0:
                key = len(self) + key
            if key >= len(self):
                raise IndexError("SFrame index out of range")
            return list(SFrame(_proxy = self.__proxy__.copy_range(key, 1, key+1)))[0]
        elif type(key) is slice:
            start = key.start
            stop = key.stop
            step = key.step
            if start is None:
                start = 0
            if stop is None:
                stop = len(self)
            if step is None:
                step = 1
            # handle negative indices
            if start < 0:
                start = len(self) + start
            if stop < 0:
                stop = len(self) + stop
            return SFrame(_proxy = self.__proxy__.copy_range(start, step, stop))
        else:
            raise TypeError("Invalid index type: must be SArray, list, or str")

    def __setitem__(self, key, value):
        """
        A wrapper around add_column(s).  Key can be either a list or a str.  If
        value is an SArray, it is added to the SFrame as a column.  If it is a
        constant value (int, str, or float), then a column is created where
        every entry is equal to the constant value.  Existing columns can also
        be replaced using this wrapper.
        """
        if type(key) is list:
            self.add_columns(value, key)
        elif type(key) is str:
            sa_value = None
            if (type(value) is SArray):
                sa_value = value
            elif hasattr(value, '__iter__'):  # wrap list, array... to sarray
                sa_value = SArray(value)
            else:  # create an sarray  of constant value
                sa_value = SArray.from_const(value, self.num_rows())

            # set new column
            if not key in self.column_names():
                with cython_context():
                    self.add_column(sa_value, key)
            else:
                # special case if replacing the only column.
                # server would fail the replacement if the new column has different
                # length than current one, which doesn't make sense if we are replacing
                # the only column. To support this, we first take out the only column
                # and then put it back if exception happens
                single_column = (self.num_cols() == 1)
                if (single_column):
                    tmpname = key
                    saved_column = self.select_column(key)
                    self.remove_column(key)
                else:
                    # add the column to a unique column name.
                    tmpname = '__' + '-'.join(self.column_names())
                try:
                    self.add_column(sa_value, tmpname)
                except Exception as e:
                    if (single_column):
                        self.add_column(saved_column, key)
                    raise

                if (not single_column):
                    # if add succeeded, remove the column name and rename tmpname->columnname.
                    self.swap_columns(key, tmpname)
                    self.remove_column(key)
                    self.rename({tmpname: key})

        else:
            raise TypeError('Cannot set column with key type ' + str(type(key)))

    def __delitem__(self, key):
        """
        Wrapper around remove_column.
        """
        self.remove_column(key)

    def __materialize__(self):
        """
        For an SFrame that is lazily evaluated, force the persistence of the
        SFrame to disk, committing all lazy evaluated operations.
        """
        with cython_context():
            self.__proxy__.materialize()

    def __is_materialized__(self):
        """
        Returns whether or not the SFrame has been materialized.
        """
        return self.__proxy__.is_materialized()

    def __has_size__(self):
        """
        Returns whether or not the size of the SFrame is known.
        """
        return self.__proxy__.has_size()

    def __iter__(self):
        """
        Provides an iterator to the rows of the SFrame.
        """

        _mt._get_metric_tracker().track('sframe.__iter__')

        def generator():
            elems_at_a_time = 262144
            self.__proxy__.begin_iterator()
            ret = self.__proxy__.iterator_get_next(elems_at_a_time)
            column_names = self.column_names()
            while(True):
                for j in ret:
                    yield dict(zip(column_names, j))

                if len(ret) == elems_at_a_time:
                    ret = self.__proxy__.iterator_get_next(elems_at_a_time)
                else:
                    break

        return generator()

    def append(self, other):
        """
        Add the rows of an SFrame to the end of this SFrame.

        Both SFrames must have the same set of columns with the same column
        names and column types.

        Parameters
        ----------
        other : SFrame
            Another SFrame whose rows are appended to the current SFrame.

        Returns
        -------
        out : SFrame
            The result SFrame from the append operation.

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': [4, 6, 8], 'val': ['D', 'F', 'H']})
        >>> sf2 = graphlab.SFrame({'id': [1, 2, 3], 'val': ['A', 'B', 'C']})
        >>> sf = sf.append(sf2)
        >>> sf
        +----+-----+
        | id | val |
        +----+-----+
        | 4  |  D  |
        | 6  |  F  |
        | 8  |  H  |
        | 1  |  A  |
        | 2  |  B  |
        | 3  |  C  |
        +----+-----+
        [6 rows x 2 columns]
        """
        _mt._get_metric_tracker().track('sframe.append')
        if type(other) is not SFrame:
            raise RuntimeError("SFrame append can only work with SFrame")

        left_empty = len(self.column_names()) == 0
        right_empty = len(other.column_names()) == 0

        if (left_empty and right_empty):
            return SFrame()

        if (left_empty or right_empty):
            non_empty_sframe = self if right_empty else other
            return non_empty_sframe

        my_column_names = self.column_names()
        my_column_types = self.column_types()
        other_column_names = other.column_names()
        if (len(my_column_names) != len(other_column_names)):
            raise RuntimeError("Two SFrames have to have the same number of columns")

        # check if the order of column name is the same
        column_name_order_match = True
        for i in range(len(my_column_names)):
            if other_column_names[i] != my_column_names[i]:
                column_name_order_match = False
                break;


        processed_other_frame = other
        if not column_name_order_match:
            # we allow name order of two sframes to be different, so we create a new sframe from
            # "other" sframe to make it has exactly the same shape
            processed_other_frame = SFrame()
            for i in range(len(my_column_names)):
                col_name = my_column_names[i]
                if(col_name not in other_column_names):
                    raise RuntimeError("Column " + my_column_names[i] + " does not exist in second SFrame")

                other_column = other.select_column(col_name);
                processed_other_frame.add_column(other_column, col_name)

                # check column type
                if my_column_types[i] != other_column.dtype():
                    raise RuntimeError("Column " + my_column_names[i] + " type is not the same in two SFrames, one is " + str(my_column_types[i]) + ", the other is " + str(other_column.dtype()))

        with cython_context():
            processed_other_frame.__materialize__()
            return SFrame(_proxy=self.__proxy__.append(processed_other_frame.__proxy__))

    def groupby(self, key_columns, operations, *args):
        """
        Perform a group on the key_columns followed by aggregations on the
        columns listed in operations.

        The operations parameter is a dictionary that indicates which
        aggregation operators to use and which columns to use them on. The
        available operators are SUM, MAX, MIN, COUNT, AVG, VAR, STDV, CONCAT,
        SELECT_ONE, ARGMIN, ARGMAX, and QUANTILE. For convenience, aggregators
        MEAN, STD, and VARIANCE are available as synonyms for AVG, STDV, and
        VAR. See :mod:`~graphlab.aggregate` for more detail on the aggregators.

        Parameters
        ----------
        key_columns : string | list[string]
            Column(s) to group by. Key columns can be of any type other than
            dictionary.

        operations : dict, list
            Dictionary of columns and aggregation operations. Each key is a
            output column name and each value is an aggregator. This can also
            be a list of aggregators, in which case column names will be
            automatically assigned.

        *args
            All other remaining arguments will be interpreted in the same
            way as the operations argument.

        Returns
        -------
        out_sf : SFrame
            A new SFrame, with a column for each groupby column and each
            aggregation operation.

        See Also
        --------
        aggregate

        Examples
        --------
        Suppose we have an SFrame with movie ratings by many users.

        >>> import graphlab.aggregate as agg
        >>> url = 'http://s3.amazonaws.com/gl-testdata/rating_data_example.csv'
        >>> sf = graphlab.SFrame.read_csv(url)
        >>> sf
        +---------+----------+--------+
        | user_id | movie_id | rating |
        +---------+----------+--------+
        |  25904  |   1663   |   3    |
        |  25907  |   1663   |   3    |
        |  25923  |   1663   |   3    |
        |  25924  |   1663   |   3    |
        |  25928  |   1663   |   2    |
        |  25933  |   1663   |   4    |
        |  25934  |   1663   |   4    |
        |  25935  |   1663   |   4    |
        |  25936  |   1663   |   5    |
        |  25937  |   1663   |   2    |
        |   ...   |   ...    |  ...   |
        +---------+----------+--------+
        [10000 rows x 3 columns]

        Compute the number of occurrences of each user.

        >>> user_count = sf.groupby(key_columns='user_id',
        ...                         operations={'count': agg.COUNT()})
        >>> user_count
        +---------+-------+
        | user_id | count |
        +---------+-------+
        |  62361  |   1   |
        |  30727  |   1   |
        |  40111  |   1   |
        |  50513  |   1   |
        |  35140  |   1   |
        |  42352  |   1   |
        |  29667  |   1   |
        |  46242  |   1   |
        |  58310  |   1   |
        |  64614  |   1   |
        |   ...   |  ...  |
        +---------+-------+
        [9852 rows x 2 columns]

        Compute the mean and standard deviation of ratings per user.

        >>> user_rating_stats = sf.groupby(key_columns='user_id',
        ...                                operations={
        ...                                    'mean_rating': agg.MEAN('rating'),
        ...                                    'std_rating': agg.STD('rating')
        ...                                })
        >>> user_rating_stats
        +---------+-------------+------------+
        | user_id | mean_rating | std_rating |
        +---------+-------------+------------+
        |  62361  |     5.0     |    0.0     |
        |  30727  |     4.0     |    0.0     |
        |  40111  |     2.0     |    0.0     |
        |  50513  |     4.0     |    0.0     |
        |  35140  |     4.0     |    0.0     |
        |  42352  |     5.0     |    0.0     |
        |  29667  |     4.0     |    0.0     |
        |  46242  |     5.0     |    0.0     |
        |  58310  |     2.0     |    0.0     |
        |  64614  |     2.0     |    0.0     |
        |   ...   |     ...     |    ...     |
        +---------+-------------+------------+
        [9852 rows x 3 columns]

        Compute the movie with the minimum rating per user.

        >>> chosen_movies = sf.groupby(key_columns='user_id',
        ...                            operations={
        ...                                'worst_movies': agg.ARGMIN('rating','movie_id')
        ...                            })
        >>> chosen_movies
        +---------+-------------+
        | user_id | worst_movies |
        +---------+-------------+
        |  62361  |     1663    |
        |  30727  |     1663    |
        |  40111  |     1663    |
        |  50513  |     1663    |
        |  35140  |     1663    |
        |  42352  |     1663    |
        |  29667  |     1663    |
        |  46242  |     1663    |
        |  58310  |     1663    |
        |  64614  |     1663    |
        |   ...   |     ...     |
        +---------+-------------+
        [9852 rows x 2 columns]

        Compute the movie with the max rating per user and also the movie with
        the maximum imdb-ranking per user.

        >>> sf['imdb-ranking'] = sf['rating'] * 10
        >>> chosen_movies = sf.groupby(key_columns='user_id',
        ...         operations={('max_rating_movie','max_imdb_ranking_movie'): agg.ARGMAX(('rating','imdb-ranking'),'movie_id')})
        >>> chosen_movies
        +---------+------------------+------------------------+
        | user_id | max_rating_movie | max_imdb_ranking_movie |
        +---------+------------------+------------------------+
        |  62361  |       1663       |          16630         |
        |  30727  |       1663       |          16630         |
        |  40111  |       1663       |          16630         |
        |  50513  |       1663       |          16630         |
        |  35140  |       1663       |          16630         |
        |  42352  |       1663       |          16630         |
        |  29667  |       1663       |          16630         |
        |  46242  |       1663       |          16630         |
        |  58310  |       1663       |          16630         |
        |  64614  |       1663       |          16630         |
        |   ...   |       ...        |          ...           |
        +---------+------------------+------------------------+
        [9852 rows x 3 columns]

        Compute the movie with the max rating per user.

        >>> chosen_movies = sf.groupby(key_columns='user_id',
                    operations={'best_movies': agg.ARGMAX('rating','movie')})

        Compute the movie with the max rating per user and also the movie with the maximum imdb-ranking per user.

        >>> chosen_movies = sf.groupby(key_columns='user_id',
                   operations={('max_rating_movie','max_imdb_ranking_movie'): agg.ARGMAX(('rating','imdb-ranking'),'movie')})

        Compute the count, mean, and standard deviation of ratings per (user,
        time), automatically assigning output column names.

        >>> sf['time'] = sf.apply(lambda x: (x['user_id'] + x['movie_id']) % 11 + 2000)
        >>> user_rating_stats = sf.groupby(['user_id', 'time'],
        ...                                [agg.COUNT(),
        ...                                 agg.AVG('rating'),
        ...                                 agg.STDV('rating')])
        >>> user_rating_stats
        +------+---------+-------+---------------+----------------+
        | time | user_id | Count | Avg of rating | Stdv of rating |
        +------+---------+-------+---------------+----------------+
        | 2006 |  61285  |   1   |      4.0      |      0.0       |
        | 2000 |  36078  |   1   |      4.0      |      0.0       |
        | 2003 |  47158  |   1   |      3.0      |      0.0       |
        | 2007 |  34446  |   1   |      3.0      |      0.0       |
        | 2010 |  47990  |   1   |      3.0      |      0.0       |
        | 2003 |  42120  |   1   |      5.0      |      0.0       |
        | 2007 |  44940  |   1   |      4.0      |      0.0       |
        | 2008 |  58240  |   1   |      4.0      |      0.0       |
        | 2002 |   102   |   1   |      1.0      |      0.0       |
        | 2009 |  52708  |   1   |      3.0      |      0.0       |
        | ...  |   ...   |  ...  |      ...      |      ...       |
        +------+---------+-------+---------------+----------------+
        [10000 rows x 5 columns]


        The groupby function can take a variable length list of aggregation
        specifiers so if we want the count and the 0.25 and 0.75 quantiles of
        ratings:

        >>> user_rating_stats = sf.groupby(['user_id', 'time'], agg.COUNT(),
        ...                                {'rating_quantiles': agg.QUANTILE('rating',[0.25, 0.75])})
        >>> user_rating_stats
        +------+---------+-------+------------------------+
        | time | user_id | Count |    rating_quantiles    |
        +------+---------+-------+------------------------+
        | 2006 |  61285  |   1   | array('d', [4.0, 4.0]) |
        | 2000 |  36078  |   1   | array('d', [4.0, 4.0]) |
        | 2003 |  47158  |   1   | array('d', [3.0, 3.0]) |
        | 2007 |  34446  |   1   | array('d', [3.0, 3.0]) |
        | 2010 |  47990  |   1   | array('d', [3.0, 3.0]) |
        | 2003 |  42120  |   1   | array('d', [5.0, 5.0]) |
        | 2007 |  44940  |   1   | array('d', [4.0, 4.0]) |
        | 2008 |  58240  |   1   | array('d', [4.0, 4.0]) |
        | 2002 |   102   |   1   | array('d', [1.0, 1.0]) |
        | 2009 |  52708  |   1   | array('d', [3.0, 3.0]) |
        | ...  |   ...   |  ...  |          ...           |
        +------+---------+-------+------------------------+
        [10000 rows x 4 columns]

        To put all items a user rated into one list value by their star rating:

        >>> user_rating_stats = sf.groupby(["user_id", "rating"],
        ...                                {"rated_movie_ids":agg.CONCAT("movie_id")})
        >>> user_rating_stats
        +--------+---------+----------------------+
        | rating | user_id |     rated_movie_ids  |
        +--------+---------+----------------------+
        |   3    |  31434  | array('d', [1663.0]) |
        |   5    |  25944  | array('d', [1663.0]) |
        |   4    |  38827  | array('d', [1663.0]) |
        |   4    |  51437  | array('d', [1663.0]) |
        |   4    |  42549  | array('d', [1663.0]) |
        |   4    |  49532  | array('d', [1663.0]) |
        |   3    |  26124  | array('d', [1663.0]) |
        |   4    |  46336  | array('d', [1663.0]) |
        |   4    |  52133  | array('d', [1663.0]) |
        |   5    |  62361  | array('d', [1663.0]) |
        |  ...   |   ...   |         ...          |
        +--------+---------+----------------------+
        [9952 rows x 3 columns]

        To put all items and rating of a given user together into a dictionary
        value:

        >>> user_rating_stats = sf.groupby("user_id",
        ...                                {"movie_rating":agg.CONCAT("movie_id", "rating")})
        >>> user_rating_stats
        +---------+--------------+
        | user_id | movie_rating |
        +---------+--------------+
        |  62361  |  {1663: 5}   |
        |  30727  |  {1663: 4}   |
        |  40111  |  {1663: 2}   |
        |  50513  |  {1663: 4}   |
        |  35140  |  {1663: 4}   |
        |  42352  |  {1663: 5}   |
        |  29667  |  {1663: 4}   |
        |  46242  |  {1663: 5}   |
        |  58310  |  {1663: 2}   |
        |  64614  |  {1663: 2}   |
        |   ...   |     ...      |
        +---------+--------------+
        [9852 rows x 2 columns]
        """
        # some basic checking first
        # make sure key_columns is a list
        if isinstance(key_columns, str):
            key_columns = [key_columns]
        # check that every column is a string, and is a valid column name
        my_column_names = self.column_names()
        key_columns_array = []
        for column in key_columns:
            if not isinstance(column, str):
                raise TypeError("Column name must be a string")
            if column not in my_column_names:
                raise KeyError("Column " + column + " does not exist in SFrame")
            if self[column].dtype() == dict:
                raise TypeError("Cannot group on a dictionary column.")
            key_columns_array.append(column)

        group_output_columns = []
        group_columns = []
        group_ops = []

        all_ops = [operations] + list(args)
        for op_entry in all_ops:
            # if it is not a dict, nor a list, it is just a single aggregator
            # element (probably COUNT). wrap it in a list so we can reuse the
            # list processing code
            operation = op_entry
            if not(isinstance(operation, list) or isinstance(operation, dict)):
              operation = [operation]
            if isinstance(operation, dict):
              # now sweep the dict and add to group_columns and group_ops
              for key in operation:
                  val = operation[key]
                  if type(val) is tuple:
                    (op, column) = val
                    if (op == '__builtin__avg__' and self[column[0]].dtype() is array.array):
                        op = '__builtin__vector__avg__'

                    if (op == '__builtin__sum__' and self[column[0]].dtype() is array.array):
                        op = '__builtin__vector__sum__'

                    if (op == '__builtin__argmax__' or op == '__builtin__argmin__') and ((type(column[0]) is tuple) != (type(key) is tuple)):
                        raise TypeError("Output column(s) and aggregate column(s) for aggregate operation should be either all tuple or all string.")

                    if (op == '__builtin__argmax__' or op == '__builtin__argmin__') and type(column[0]) is tuple:
                      for (col,output) in zip(column[0],key):
                        group_columns = group_columns + [[col,column[1]]]
                        group_ops = group_ops + [op]
                        group_output_columns = group_output_columns + [output]
                    else:
                      group_columns = group_columns + [column]
                      group_ops = group_ops + [op]
                      group_output_columns = group_output_columns + [key]
                  elif val == graphlab.aggregate.COUNT:
                    group_output_columns = group_output_columns + [key]
                    val = graphlab.aggregate.COUNT()
                    (op, column) = val
                    group_columns = group_columns + [column]
                    group_ops = group_ops + [op]
                  else:
                    raise TypeError("Unexpected type in aggregator definition of output column: " + key)
            elif isinstance(operation, list):
              # we will be using automatically defined column names
              for val in operation:
                  if type(val) is tuple:
                    (op, column) = val
                    if (op == '__builtin__avg__' and self[column[0]].dtype() is array.array):
                        op = '__builtin__vector__avg__'

                    if (op == '__builtin__sum__' and self[column[0]].dtype() is array.array):
                        op = '__builtin__vector__sum__'

                    if (op == '__builtin__argmax__' or op == '__builtin__argmin__') and type(column[0]) is tuple:
                      for col in column[0]:
                        group_columns = group_columns + [[col,column[1]]]
                        group_ops = group_ops + [op]
                        group_output_columns = group_output_columns + [""]
                    else:
                      group_columns = group_columns + [column]
                      group_ops = group_ops + [op]
                      group_output_columns = group_output_columns + [""]
                  elif val == graphlab.aggregate.COUNT:
                    group_output_columns = group_output_columns + [""]
                    val = graphlab.aggregate.COUNT()
                    (op, column) = val
                    group_columns = group_columns + [column]
                    group_ops = group_ops + [op]
                  else:
                    raise TypeError("Unexpected type in aggregator definition.")


        # let's validate group_columns and group_ops are valid
        for (cols, op) in zip(group_columns, group_ops):
            for col in cols:
                if not isinstance(col, str):
                    raise TypeError("Column name must be a string")

            if not isinstance(op, str):
                raise TypeError("Operation type not recognized.")

            if op is not graphlab.aggregate.COUNT()[0]:
                for col in cols:
                    if col not in my_column_names:
                        raise KeyError("Column " + col + " does not exist in SFrame")

            _mt._get_metric_tracker().track('sframe.groupby', properties={'operator':op})

        with cython_context():
            return SFrame(_proxy=self.__proxy__.groupby_aggregate(key_columns_array, group_columns,
                                                                  group_output_columns, group_ops))

    def join(self, right, on=None, how='inner'):
        """
        Merge two SFrames. Merges the current (left) SFrame with the given
        (right) SFrame using a SQL-style equi-join operation by columns.

        Parameters
        ----------
        right : SFrame
            The SFrame to join.

        on : None | str | list | dict, optional
            The column name(s) representing the set of join keys.  Each row that
            has the same value in this set of columns will be merged together.

            * If 'None' is given, join will use all columns that have the same
              name as the set of join keys.

            * If a str is given, this is interpreted as a join using one column,
              where both SFrames have the same column name.

            * If a list is given, this is interpreted as a join using one or
              more column names, where each column name given exists in both
              SFrames.

            * If a dict is given, each dict key is taken as a column name in the
              left SFrame, and each dict value is taken as the column name in
              right SFrame that will be joined together. e.g.
              {'left_col_name':'right_col_name'}.

        how : {'left', 'right', 'outer', 'inner'}, optional
            The type of join to perform.  'inner' is default.

            * inner: Equivalent to a SQL inner join.  Result consists of the
              rows from the two frames whose join key values match exactly,
              merged together into one SFrame.

            * left: Equivalent to a SQL left outer join. Result is the union
              between the result of an inner join and the rest of the rows from
              the left SFrame, merged with missing values.

            * right: Equivalent to a SQL right outer join.  Result is the union
              between the result of an inner join and the rest of the rows from
              the right SFrame, merged with missing values.

            * outer: Equivalent to a SQL full outer join. Result is
              the union between the result of a left outer join and a right
              outer join.

        Returns
        -------
        out : SFrame

        Examples
        --------
        >>> animals = graphlab.SFrame({'id': [1, 2, 3, 4],
        ...                           'name': ['dog', 'cat', 'sheep', 'cow']})
        >>> sounds = graphlab.SFrame({'id': [1, 3, 4, 5],
        ...                          'sound': ['woof', 'baa', 'moo', 'oink']})
        >>> animals.join(sounds, how='inner')
        +----+-------+-------+
        | id |  name | sound |
        +----+-------+-------+
        | 1  |  dog  |  woof |
        | 3  | sheep |  baa  |
        | 4  |  cow  |  moo  |
        +----+-------+-------+
        [3 rows x 3 columns]

        >>> animals.join(sounds, on='id', how='left')
        +----+-------+-------+
        | id |  name | sound |
        +----+-------+-------+
        | 1  |  dog  |  woof |
        | 3  | sheep |  baa  |
        | 4  |  cow  |  moo  |
        | 2  |  cat  |  None |
        +----+-------+-------+
        [4 rows x 3 columns]

        >>> animals.join(sounds, on=['id'], how='right')
        +----+-------+-------+
        | id |  name | sound |
        +----+-------+-------+
        | 1  |  dog  |  woof |
        | 3  | sheep |  baa  |
        | 4  |  cow  |  moo  |
        | 5  |  None |  oink |
        +----+-------+-------+
        [4 rows x 3 columns]

        >>> animals.join(sounds, on={'id':'id'}, how='outer')
        +----+-------+-------+
        | id |  name | sound |
        +----+-------+-------+
        | 1  |  dog  |  woof |
        | 3  | sheep |  baa  |
        | 4  |  cow  |  moo  |
        | 5  |  None |  oink |
        | 2  |  cat  |  None |
        +----+-------+-------+
        [5 rows x 3 columns]
        """
        _mt._get_metric_tracker().track('sframe.join', properties={'type':how})
        available_join_types = ['left','right','outer','inner']

        if not isinstance(right, SFrame):
            raise TypeError("Can only join two SFrames")

        if how not in available_join_types:
            raise ValueError("Invalid join type")

        join_keys = dict()
        if on is None:
            left_names = self.column_names()
            right_names = right.column_names()
            common_columns = [name for name in left_names if name in right_names]
            for name in common_columns:
                join_keys[name] = name
        elif type(on) is str:
            join_keys[on] = on
        elif type(on) is list:
            for name in on:
                if type(name) is not str:
                    raise TypeError("Join keys must each be a str.")
                join_keys[name] = name
        elif type(on) is dict:
            join_keys = on
        else:
            raise TypeError("Must pass a str, list, or dict of join keys")

        with cython_context():
            return SFrame(_proxy=self.__proxy__.join(right.__proxy__, how, join_keys))

    def filter_by(self, values, column_name, exclude=False):
        """
        Filter an SFrame by values inside an iterable object. Result is an
        SFrame that only includes (or excludes) the rows that have a column
        with the given ``column_name`` which holds one of the values in the
        given ``values`` :class:`~graphlab.SArray`. If ``values`` is not an
        SArray, we attempt to convert it to one before filtering.

        Parameters
        ----------
        values : SArray | list | numpy.ndarray | pandas.Series | str
            The values to use to filter the SFrame.  The resulting SFrame will
            only include rows that have one of these values in the given
            column.

        column_name : str
            The column of the SFrame to match with the given `values`.

        exclude : bool
            If True, the result SFrame will contain all rows EXCEPT those that
            have one of ``values`` in ``column_name``.

        Returns
        -------
        out : SFrame
            The filtered SFrame.

        Examples
        --------
        >>> sf = graphlab.SFrame({'id': [1, 2, 3, 4],
        ...                      'animal_type': ['dog', 'cat', 'cow', 'horse'],
        ...                      'name': ['bob', 'jim', 'jimbob', 'bobjim']})
        >>> household_pets = ['cat', 'hamster', 'dog', 'fish', 'bird', 'snake']
        >>> sf.filter_by(household_pets, 'animal_type')
        +-------------+----+------+
        | animal_type | id | name |
        +-------------+----+------+
        |     dog     | 1  | bob  |
        |     cat     | 2  | jim  |
        +-------------+----+------+
        [2 rows x 3 columns]
        >>> sf.filter_by(household_pets, 'animal_type', exclude=True)
        +-------------+----+--------+
        | animal_type | id |  name  |
        +-------------+----+--------+
        |    horse    | 4  | bobjim |
        |     cow     | 3  | jimbob |
        +-------------+----+--------+
        [2 rows x 3 columns]
        """
        _mt._get_metric_tracker().track('sframe.filter_by')
        if type(column_name) is not str:
            raise TypeError("Must pass a str as column_name")

        if type(values) is not SArray:
            # If we were given a single element, try to put in list and convert
            # to SArray
            if not hasattr(values, '__iter__'):
                values = [values]
            values = SArray(values)

        value_sf = SFrame()
        value_sf.add_column(values, column_name)

        # Make sure the values list has unique values, or else join will not
        # filter.
        value_sf = value_sf.groupby(column_name, {})

        existing_columns = self.column_names()
        if column_name not in existing_columns:
            raise KeyError("Column '" + column_name + "' not in SFrame.")

        existing_type = self.column_types()[self.column_names().index(column_name)]
        given_type = value_sf.column_types()[0]
        if given_type != existing_type:
            raise TypeError("Type of given values does not match type of column '" +
                column_name + "' in SFrame.")

        with cython_context():
            if exclude:
                id_name = "id"
                # Make sure this name is unique so we know what to remove in
                # the result
                while id_name in existing_columns:
                    id_name += "1"
                value_sf = value_sf.add_row_number(id_name)

                tmp = SFrame(_proxy=self.__proxy__.join(value_sf.__proxy__,
                                                     'left',
                                                     {column_name:column_name}))
                ret_sf = tmp[tmp[id_name] == None]
                del ret_sf[id_name]
                return ret_sf
            else:
                return SFrame(_proxy=self.__proxy__.join(value_sf.__proxy__,
                                                     'inner',
                                                     {column_name:column_name}))

    @_check_canvas_enabled
    def show(self, columns=None, view=None, x=None, y=None):
        """
        show(columns=None, view=None, x=None, y=None)

        Visualize the SFrame with GraphLab Create :mod:`~graphlab.canvas`. This function
        starts Canvas if it is not already running. If the SFrame has already been plotted,
        this function will update the plot.

        Parameters
        ----------
        columns : list of str, optional
            The columns of this SFrame to show in the SFrame view. In an
            interactive browser target of Canvas, the columns will be selectable
            and reorderable through the UI as well. If not specified, the
            SFrame view will use all columns of the SFrame.

        view : str, optional
            The name of the SFrame view to show. Can be one of:

            - None: Use the default (depends on which Canvas target is set).
            - 'Table': Show a scrollable, tabular view of the data in the
              SFrame.
            - 'Summary': Show a list of columns with some summary statistics
              and plots for each column.
            - 'Scatter Plot': Show a scatter plot of two numeric columns.
            - 'Heat Map': Show a heat map of two numeric columns.
            - 'Bar Chart': Show a bar chart of one numeric and one categorical
              column.
            - 'Line Chart': Show a line chart of one numeric and one
              categorical column.

        x : str, optional
            The column to use for the X axis in a Scatter Plot, Heat Map, Bar
            Chart, or Line Chart view. Must be the name of one of the columns
            in this SFrame. For Scatter Plot and Heat Map, the column must be
            numeric (int or float). If not set, defaults to the first available
            valid column.

        y : str, optional
            The column to use for the Y axis in a Scatter Plot, Heat Map, Bar
            Chart, or Line Chart view. Must be the name of one of the numeric
            columns in this SFrame. If not set, defaults to the second
            available numeric column.

        Returns
        -------
        view : graphlab.canvas.view.View
            An object representing the GraphLab Canvas view.

        See Also
        --------
        canvas

        Examples
        --------
        Suppose 'sf' is an SFrame, we can view it in GraphLab Canvas using:

        >>> sf.show()

        To choose a column filter (applied to all SFrame views):

        >>> sf.show(columns=["Foo", "Bar"]) # use only columns 'Foo' and 'Bar'
        >>> sf.show(columns=sf.column_names()[3:7]) # use columns 3-7

        To choose a specific view of the SFrame:

        >>> sf.show(view="Summary")
        >>> sf.show(view="Table")
        >>> sf.show(view="Bar Chart", x="col1", y="col2")
        >>> sf.show(view="Line Chart", x="col1", y="col2")
        >>> sf.show(view="Scatter Plot", x="col1", y="col2")
        >>> sf.show(view="Heat Map", x="col1", y="col2")
        """
        import graphlab.canvas
        import graphlab.canvas.inspect
        import graphlab.canvas.views.sframe

        graphlab.canvas.inspect.find_vars(self)
        return graphlab.canvas.show(graphlab.canvas.views.sframe.SFrameView(self, params={
            'view': view,
            'columns': columns,
            'x': x,
            'y': y
        }))

    def pack_columns(self, columns=None, column_prefix=None, dtype=list,
                     fill_na=None, remove_prefix=True, new_column_name=None):
        """
        Pack two or more columns of the current SFrame into one single
        column.The result is a new SFrame with the unaffected columns from the
        original SFrame plus the newly created column.

        The list of columns that are packed is chosen through either the
        ``columns`` or ``column_prefix`` parameter. Only one of the parameters
        is allowed to be provided. ``columns`` explicitly specifies the list of
        columns to pack, while ``column_prefix`` specifies that all columns that
        have the given prefix are to be packed.

        The type of the resulting column is decided by the ``dtype`` parameter.
        Allowed values for ``dtype`` are dict, array.array and list:

         - *dict*: pack to a dictionary SArray where column name becomes
           dictionary key and column value becomes dictionary value

         - *array.array*: pack all values from the packing columns into an array

         - *list*: pack all values from the packing columns into a list.

        Parameters
        ----------
        columns : list[str], optional
            A list of column names to be packed.  There needs to have at least
            two columns to pack.  If omitted and `column_prefix` is not
            specified, all columns from current SFrame are packed.  This
            parameter is mutually exclusive with the `column_prefix` parameter.

        column_prefix : str, optional
            Pack all columns with the given `column_prefix`.
            This parameter is mutually exclusive with the `columns` parameter.

        dtype : dict | array.array | list, optional
            The resulting packed column type. If not provided, dtype is list.

        fill_na : value, optional
            Value to fill into packed column if missing value is encountered.
            If packing to dictionary, `fill_na` is only applicable to dictionary
            values; missing keys are not replaced.

        remove_prefix : bool, optional
            If True and `column_prefix` is specified, the dictionary key will
            be constructed by removing the prefix from the column name.
            This option is only applicable when packing to dict type.

        new_column_name : str, optional
            Packed column name.  If not given and `column_prefix` is given,
            then the prefix will be used as the new column name, otherwise name
            is generated automatically.

        Returns
        -------
        out : SFrame
            An SFrame that contains columns that are not packed, plus the newly
            packed column.

        See Also
        --------
        unpack

        Notes
        -----
        - There must be at least two columns to pack.

        - If packing to dictionary, missing key is always dropped. Missing
          values are dropped if fill_na is not provided, otherwise, missing
          value is replaced by 'fill_na'. If packing to list or array, missing
          values will be kept. If 'fill_na' is provided, the missing value is
          replaced with 'fill_na' value.

        Examples
        --------
        Suppose 'sf' is an an SFrame that maintains business category
        information:

        >>> sf = graphlab.SFrame({'business': range(1, 5),
        ...                       'category.retail': [1, None, 1, None],
        ...                       'category.food': [1, 1, None, None],
        ...                       'category.service': [None, 1, 1, None],
        ...                       'category.shop': [1, 1, None, 1]})
        >>> sf
        +----------+-----------------+---------------+------------------+---------------+
        | business | category.retail | category.food | category.service | category.shop |
        +----------+-----------------+---------------+------------------+---------------+
        |    1     |        1        |       1       |       None       |       1       |
        |    2     |       None      |       1       |        1         |       1       |
        |    3     |        1        |      None     |        1         |      None     |
        |    4     |       None      |       1       |       None       |       1       |
        +----------+-----------------+---------------+------------------+---------------+
        [4 rows x 5 columns]

        To pack all category columns into a list:

        >>> sf.pack_columns(column_prefix='category')
        +----------+--------------------+
        | business |         X2         |
        +----------+--------------------+
        |    1     |  [1, 1, None, 1]   |
        |    2     |  [None, 1, 1, 1]   |
        |    3     | [1, None, 1, None] |
        |    4     | [None, 1, None, 1] |
        +----------+--------------------+
        [4 rows x 2 columns]

        To pack all category columns into a dictionary, with new column name:

        >>> sf.pack_columns(column_prefix='category', dtype=dict,
        ...                 new_column_name='category')
        +----------+--------------------------------+
        | business |            category            |
        +----------+--------------------------------+
        |    1     | {'food': 1, 'shop': 1, 're ... |
        |    2     | {'food': 1, 'shop': 1, 'se ... |
        |    3     |  {'retail': 1, 'service': 1}   |
        |    4     |     {'food': 1, 'shop': 1}     |
        +----------+--------------------------------+
        [4 rows x 2 columns]

        To keep column prefix in the resulting dict key:

        >>> sf.pack_columns(column_prefix='category', dtype=dict,
                            remove_prefix=False)
        +----------+--------------------------------+
        | business |               X2               |
        +----------+--------------------------------+
        |    1     | {'category.retail': 1, 'ca ... |
        |    2     | {'category.food': 1, 'cate ... |
        |    3     | {'category.retail': 1, 'ca ... |
        |    4     | {'category.food': 1, 'cate ... |
        +----------+--------------------------------+
        [4 rows x 2 columns]

        To explicitly pack a set of columns:

        >>> sf.pack_columns(columns = ['business', 'category.retail',
                                       'category.food', 'category.service',
                                       'category.shop'])
        +-----------------------+
        |           X1          |
        +-----------------------+
        |   [1, 1, 1, None, 1]  |
        |   [2, None, 1, 1, 1]  |
        | [3, 1, None, 1, None] |
        | [4, None, 1, None, 1] |
        +-----------------------+
        [4 rows x 1 columns]

        To pack all columns with name starting with 'category' into an array
        type, and with missing value replaced with 0:

        >>> sf.pack_columns(column_prefix="category", dtype=array.array,
        ...                 fill_na=0)
        +----------+--------------------------------+
        | business |               X2               |
        +----------+--------------------------------+
        |    1     | array('d', [1.0, 1.0, 0.0, ... |
        |    2     | array('d', [0.0, 1.0, 1.0, ... |
        |    3     | array('d', [1.0, 0.0, 1.0, ... |
        |    4     | array('d', [0.0, 1.0, 0.0, ... |
        +----------+--------------------------------+
        [4 rows x 2 columns]
        """
        if columns != None and column_prefix != None:
            raise ValueError("'columns' and 'column_prefix' parameter cannot be given at the same time.")

        if new_column_name == None and column_prefix != None:
            new_column_name = column_prefix

        if column_prefix != None:
            if type(column_prefix) != str:
                raise TypeError("'column_prefix' must be a string")
            columns = [name for name in self.column_names() if name.startswith(column_prefix)]
            if len(columns) == 0:
                raise ValueError("There is no column starts with prefix '" + column_prefix + "'")
        elif columns == None:
            columns = self.column_names()
        else:
            if not hasattr(columns, '__iter__'):
                raise TypeError("columns must be an iterable type")

            column_names = set(self.column_names())
            for column in columns:
                if (column not in column_names):
                    raise ValueError("Current SFrame has no column called '" + str(column) + "'.")

            # check duplicate names
            if len(set(columns)) != len(columns):
                raise ValueError("There is duplicate column names in columns parameter")

        if (len(columns) <= 1):
            raise ValueError("Please provide at least two columns to pack")

        if (dtype not in (dict, list, array.array)):
            raise ValueError("Resulting dtype has to be one of dict/array.array/list type")

        # fill_na value for array needs to be numeric
        if dtype == array.array:
            if (fill_na != None) and (type(fill_na) not in (int, float)):
                raise ValueError("fill_na value for array needs to be numeric type")
            # all columns have to be numeric type
            for column in columns:
                if self[column].dtype() not in (int, float):
                    raise TypeError("Column '" + column + "' type is not numeric, cannot pack into array type")

        # generate dict key names if pack to dictionary
        # we try to be smart here
        # if all column names are like: a.b, a.c, a.d,...
        # we then use "b", "c", "d", etc as the dictionary key during packing
        if (dtype == dict) and (column_prefix != None) and (remove_prefix == True):
            size_prefix = len(column_prefix)
            first_char = set([c[size_prefix:size_prefix+1] for c in columns])
            if ((len(first_char) == 1) and first_char.pop() in ['.','-','_']):
                dict_keys = [name[size_prefix+1:] for name in columns]
            else:
                dict_keys = [name[size_prefix:] for name in columns]

        else:
            dict_keys = columns

        rest_columns = [name for name in self.column_names() if name not in columns]
        if new_column_name != None:
            if type(new_column_name) != str:
                raise TypeError("'new_column_name' has to be a string")
            if new_column_name in rest_columns:
                raise KeyError("Current SFrame already contains a column name " + new_column_name)
        else:
            new_column_name = ""

        _mt._get_metric_tracker().track('sframe.pack_columns')

        ret_sa = None
        with cython_context():
            ret_sa = SArray(_proxy=self.__proxy__.pack_columns(columns, dict_keys, dtype, fill_na))

        new_sf = self.select_columns(rest_columns)
        new_sf.add_column(ret_sa, new_column_name)
        return new_sf


    def split_datetime(self, expand_column, column_name_prefix=None, limit=None, tzone=False):
        """
        Splits a datetime column of SFrame to multiple columns, with each value in a
        separate column. Returns a new SFrame with the expanded column replaced with
        a list of new columns. The expanded column must be of datetime type.

        For more details regarding name generation and
        other, refer to :py:func:`graphlab.SArray.split_datetim()`

        Parameters
        ----------
        expand_column : str
            Name of the unpacked column.

        column_name_prefix : str, optional
            If provided, expanded column names would start with the given prefix.
            If not provided, the default value is the name of the expanded column.

        limit : list[str], optional
            Limits the set of datetime elements to expand.
            Elements are 'year','month','day','hour','minute',
            and 'second'.

        tzone : bool, optional
            A boolean parameter that determines whether to show the timezone
            column or not. Defaults to False.

        Returns
        -------
        out : SFrame
            A new SFrame that contains rest of columns from original SFrame with
            the given column replaced with a collection of expanded columns.

        Examples
        ---------

        >>> sf
        Columns:
            id   int
            submission  datetime
        Rows: 2
        Data:
            +----+-------------------------------------------------+
            | id |               submission                        |
            +----+-------------------------------------------------+
            | 1  | datetime(2011, 1, 21, 7, 17, 21, tzinfo=GMT(+1))|
            | 2  | datetime(2011, 1, 21, 5, 43, 21, tzinfo=GMT(+1))|
            +----+-------------------------------------------------+

        >>> sf.split_datetime('submission',limit=['hour','minute'])
        Columns:
            id  int
            submission.hour int
            submission.minute int
        Rows: 2
        Data:
        +----+-----------------+-------------------+
        | id | submission.hour | submission.minute |
        +----+-----------------+-------------------+
        | 1  |        7        |        17         |
        | 2  |        5        |        43         |
        +----+-----------------+-------------------+

        """
        if expand_column not in self.column_names():
            raise KeyError("column '" + expand_column + "' does not exist in current SFrame")

        if column_name_prefix == None:
            column_name_prefix = expand_column

        new_sf = self[expand_column].split_datetime(column_name_prefix, limit, tzone)

        # construct return SFrame, check if there is conflict
        rest_columns =  [name for name in self.column_names() if name != expand_column]
        new_names = new_sf.column_names()
        while set(new_names).intersection(rest_columns):
            new_names = [name + ".1" for name in new_names]
        new_sf.rename(dict(zip(new_sf.column_names(), new_names)))

        _mt._get_metric_tracker().track('sframe.split_datetime')
        ret_sf = self.select_columns(rest_columns)
        ret_sf.add_columns(new_sf)
        return ret_sf

    def unpack(self, unpack_column, column_name_prefix=None, column_types=None,
               na_value=None, limit=None):
        """
        Expand one column of this SFrame to multiple columns with each value in
        a separate column. Returns a new SFrame with the unpacked column
        replaced with a list of new columns.  The column must be of
        list/array/dict type.

        For more details regarding name generation, missing value handling and
        other, refer to the SArray version of
        :py:func:`~graphlab.SArray.unpack()`.

        Parameters
        ----------
        unpack_column : str
            Name of the unpacked column

        column_name_prefix : str, optional
            If provided, unpacked column names would start with the given
            prefix. If not provided, default value is the name of the unpacked
            column.

        column_types : [type], optional
            Column types for the unpacked columns.
            If not provided, column types are automatically inferred from first
            100 rows. For array type, default column types are float.  If
            provided, column_types also restricts how many columns to unpack.

        na_value : flexible_type, optional
            If provided, convert all values that are equal to "na_value" to
            missing value (None).

        limit : list[str] | list[int], optional
            Control unpacking only a subset of list/array/dict value. For
            dictionary SArray, `limit` is a list of dictionary keys to restrict.
            For list/array SArray, `limit` is a list of integers that are
            indexes into the list/array value.

        Returns
        -------
        out : SFrame
            A new SFrame that contains rest of columns from original SFrame with
            the given column replaced with a collection of unpacked columns.

        See Also
        --------
        pack_columns, SArray.unpack

        Examples
        ---------
        >>> sf = graphlab.SFrame({'id': [1,2,3],
        ...                      'wc': [{'a': 1}, {'b': 2}, {'a': 1, 'b': 2}]})
        +----+------------------+
        | id |        wc        |
        +----+------------------+
        | 1  |     {'a': 1}     |
        | 2  |     {'b': 2}     |
        | 3  | {'a': 1, 'b': 2} |
        +----+------------------+
        [3 rows x 2 columns]

        >>> sf.unpack('wc')
        +----+------+------+
        | id | wc.a | wc.b |
        +----+------+------+
        | 1  |  1   | None |
        | 2  | None |  2   |
        | 3  |  1   |  2   |
        +----+------+------+
        [3 rows x 3 columns]

        To not have prefix in the generated column name:

        >>> sf.unpack('wc', column_name_prefix="")
        +----+------+------+
        | id |  a   |  b   |
        +----+------+------+
        | 1  |  1   | None |
        | 2  | None |  2   |
        | 3  |  1   |  2   |
        +----+------+------+
        [3 rows x 3 columns]

        To limit subset of keys to unpack:

        >>> sf.unpack('wc', limit=['b'])
        +----+------+
        | id | wc.b |
        +----+------+
        | 1  | None |
        | 2  |  2   |
        | 3  |  2   |
        +----+------+
        [3 rows x 3 columns]

        To unpack an array column:

        >>> sf = graphlab.SFrame({'id': [1,2,3],
        ...                       'friends': [array.array('d', [1.0, 2.0, 3.0]),
        ...                                   array.array('d', [2.0, 3.0, 4.0]),
        ...                                   array.array('d', [3.0, 4.0, 5.0])]})
        >>> sf
        +----+-----------------------------+
        | id |            friends          |
        +----+-----------------------------+
        | 1  | array('d', [1.0, 2.0, 3.0]) |
        | 2  | array('d', [2.0, 3.0, 4.0]) |
        | 3  | array('d', [3.0, 4.0, 5.0]) |
        +----+-----------------------------+
        [3 rows x 2 columns]

        >>> sf.unpack('friends')
        +----+-----------+-----------+-----------+
        | id | friends.0 | friends.1 | friends.2 |
        +----+-----------+-----------+-----------+
        | 1  |    1.0    |    2.0    |    3.0    |
        | 2  |    2.0    |    3.0    |    4.0    |
        | 3  |    3.0    |    4.0    |    5.0    |
        +----+-----------+-----------+-----------+
        [3 rows x 4 columns]
        """
        if unpack_column not in self.column_names():
            raise KeyError("column '" + unpack_column + "' does not exist in current SFrame")

        if column_name_prefix == None:
            column_name_prefix = unpack_column

        new_sf = self[unpack_column].unpack(column_name_prefix, column_types, na_value, limit)

        # construct return SFrame, check if there is conflict
        rest_columns =  [name for name in self.column_names() if name != unpack_column]
        new_names = new_sf.column_names()
        while set(new_names).intersection(rest_columns):
            new_names = [name + ".1" for name in new_names]
        new_sf.rename(dict(zip(new_sf.column_names(), new_names)))

        _mt._get_metric_tracker().track('sframe.unpack')
        ret_sf = self.select_columns(rest_columns)
        ret_sf.add_columns(new_sf)
        return ret_sf

    def stack(self, column_name, new_column_name=None, drop_na=False):
        """
        Convert a "wide" column of an SFrame to one or two "tall" columns by
        stacking all values.

        The stack works only for columns of dict, list, or array type.  If the
        column is dict type, two new columns are created as a result of
        stacking: one column holds the key and another column holds the value.
        The rest of the columns are repeated for each key/value pair.

        If the column is array or list type, one new column is created as a
        result of stacking. With each row holds one element of the array or list
        value, and the rest columns from the same original row repeated.

        The new SFrame includes the newly created column and all columns other
        than the one that is stacked.

        Parameters
        --------------
        column_name : str
            The column to stack. This column must be of dict/list/array type

        new_column_name : str | list of str, optional
            The new column name(s). If original column is list/array type,
            new_column_name must a string. If original column is dict type,
            new_column_name must be a list of two strings. If not given, column
            names are generated automatically.

        drop_na : boolean, optional
            If True, missing values and empty list/array/dict are all dropped
            from the resulting column(s). If False, missing values are
            maintained in stacked column(s).

        Returns
        -------
        out : SFrame
            A new SFrame that contains newly stacked column(s) plus columns in
            original SFrame other than the stacked column.

        See Also
        --------
        unstack

        Examples
        ---------
        Suppose 'sf' is an SFrame that contains a column of dict type:

        >>> sf = graphlab.SFrame({'topic':[1,2,3,4],
        ...                       'words': [{'a':3, 'cat':2},
        ...                                 {'a':1, 'the':2},
        ...                                 {'the':1, 'dog':3},
        ...                                 {}]
        ...                      })
        +-------+----------------------+
        | topic |        words         |
        +-------+----------------------+
        |   1   |  {'a': 3, 'cat': 2}  |
        |   2   |  {'a': 1, 'the': 2}  |
        |   3   | {'the': 1, 'dog': 3} |
        |   4   |          {}          |
        +-------+----------------------+
        [4 rows x 2 columns]

        Stack would stack all keys in one column and all values in another
        column:

        >>> sf.stack('words', new_column_name=['word', 'count'])
        +-------+------+-------+
        | topic | word | count |
        +-------+------+-------+
        |   1   |  a   |   3   |
        |   1   | cat  |   2   |
        |   2   |  a   |   1   |
        |   2   | the  |   2   |
        |   3   | the  |   1   |
        |   3   | dog  |   3   |
        |   4   | None |  None |
        +-------+------+-------+
        [7 rows x 3 columns]

        Observe that since topic 4 had no words, an empty row is inserted.
        To drop that row, set dropna=True in the parameters to stack.

        Suppose 'sf' is an SFrame that contains a user and his/her friends,
        where 'friends' columns is an array type. Stack on 'friends' column
        would create a user/friend list for each user/friend pair:

        >>> sf = graphlab.SFrame({'topic':[1,2,3],
        ...                       'friends':[[2,3,4], [5,6],
        ...                                  [4,5,10,None]]
        ...                      })
        >>> sf
        +-------+------------------+
        | topic |     friends      |
        +-------+------------------+
        |  1    |     [2, 3, 4]    |
        |  2    |      [5, 6]      |
        |  3    | [4, 5, 10, None] |
        +----- -+------------------+
        [3 rows x 2 columns]

        >>> sf.stack('friends', new_column_name='friend')
        +------+--------+
        | user | friend |
        +------+--------+
        |  1   |  2     |
        |  1   |  3     |
        |  1   |  4     |
        |  2   |  5     |
        |  2   |  6     |
        |  3   |  4     |
        |  3   |  5     |
        |  3   |  10    |
        |  3   |  None  |
        +------+--------+
        [9 rows x 2 columns]
        """
        # validate column_name
        column_name = str(column_name)
        if column_name not in self.column_names():
            raise ValueError("Cannot find column '" + str(column_name) + "' in the SFrame.")

        stack_column_type =  self[column_name].dtype()
        if (stack_column_type not in [dict, array.array, list]):
            raise TypeError("Stack is only supported for column of dict/list/array type.")

        if (new_column_name != None):
            if stack_column_type == dict:
                if (type(new_column_name) is not list):
                    raise TypeError("new_column_name has to be a list to stack dict type")
                elif (len(new_column_name) != 2):
                    raise TypeError("new_column_name must have length of two")
            else:
                if (type(new_column_name) != str):
                    raise TypeError("new_column_name has to be a str")
                new_column_name = [new_column_name]

            # check if the new column name conflicts with existing ones
            for name in new_column_name:
                if (name in self.column_names()) and (name != column_name):
                    raise ValueError("Column with name '" + name + "' already exists, pick a new column name")
        else:
            if stack_column_type == dict:
                new_column_name = ["",""]
            else:
                new_column_name = [""]

        # infer column types
        head_row = SArray(self[column_name].head(100)).dropna()
        if (len(head_row) == 0):
            raise ValueError("Cannot infer column type because there is not enough rows to infer value")
        if stack_column_type == dict:
            # infer key/value type
            keys = []; values = []
            for row in head_row:
                for val in row:
                    keys.append(val)
                    if val != None: values.append(row[val])

            new_column_type = [
                infer_type_of_list(keys),
                infer_type_of_list(values)
            ]
        else:
            values = [v for v in itertools.chain.from_iterable(head_row)]
            new_column_type = [infer_type_of_list(values)]

        _mt._get_metric_tracker().track('sframe.stack')

        with cython_context():
            return SFrame(_proxy=self.__proxy__.stack(column_name, new_column_name, new_column_type, drop_na))

    def unstack(self, column, new_column_name=None):
        """
        Concatenate values from one or two columns into one column, grouping by
        all other columns. The resulting column could be of type list, array or
        dictionary.  If ``column`` is a numeric column, the result will be of
        array.array type.  If ``column`` is a non-numeric column, the new column
        will be of list type. If ``column`` is a list of two columns, the new
        column will be of dict type where the keys are taken from the first
        column in the list.

        Parameters
        ----------
        column : str | [str, str]
            The column(s) that is(are) to be concatenated.
            If str, then collapsed column type is either array or list.
            If [str, str], then collapsed column type is dict

        new_column_name : str, optional
            New column name. If not given, a name is generated automatically.

        Returns
        -------
        out : SFrame
            A new SFrame containing the grouped columns as well as the new
            column.

        See Also
        --------
        stack : The inverse of unstack.

        groupby : ``unstack`` is a special version of ``groupby`` that uses the
          :mod:`~graphlab.aggregate.CONCAT` aggregator

        Notes
        -----
        - There is no guarantee the resulting SFrame maintains the same order as
          the original SFrame.

        - Missing values are maintained during unstack.

        - When unstacking into a dictionary, if there is more than one instance
          of a given key for a particular group, an arbitrary value is selected.

        Examples
        --------
        >>> sf = graphlab.SFrame({'count':[4, 2, 1, 1, 2, None],
        ...                       'topic':['cat', 'cat', 'dog', 'elephant', 'elephant', 'fish'],
        ...                       'word':['a', 'c', 'c', 'a', 'b', None]})
        >>> sf.unstack(column=['word', 'count'], new_column_name='words')
        +----------+------------------+
        |  topic   |      words       |
        +----------+------------------+
        | elephant | {'a': 1, 'b': 2} |
        |   dog    |     {'c': 1}     |
        |   cat    | {'a': 4, 'c': 2} |
        |   fish   |       None       |
        +----------+------------------+
        [4 rows x 2 columns]

        >>> sf = graphlab.SFrame({'friend': [2, 3, 4, 5, 6, 4, 5, 2, 3],
        ...                      'user': [1, 1, 1, 2, 2, 2, 3, 4, 4]})
        >>> sf.unstack('friend', new_column_name='friends')
        +------+-----------------------------+
        | user |           friends           |
        +------+-----------------------------+
        |  3   |      array('d', [5.0])      |
        |  1   | array('d', [2.0, 4.0, 3.0]) |
        |  2   | array('d', [5.0, 6.0, 4.0]) |
        |  4   |    array('d', [2.0, 3.0])   |
        +------+-----------------------------+
        [4 rows x 2 columns]
        """
        if (type(column) != str and len(column) != 2):
            raise TypeError("'column' parameter has to be either a string or a list of two strings.")

        _mt._get_metric_tracker().track('sframe.unstack')

        with cython_context():
            if type(column) == str:
                key_columns = [i for i in self.column_names() if i != column]
                if new_column_name != None:
                    return self.groupby(key_columns, {new_column_name : graphlab.aggregate.CONCAT(column)})
                else:
                    return self.groupby(key_columns, graphlab.aggregate.CONCAT(column))
            elif len(column) == 2:
                key_columns = [i for i in self.column_names() if i not in column]
                if new_column_name != None:
                    return self.groupby(key_columns, {new_column_name:graphlab.aggregate.CONCAT(column[0], column[1])})
                else:
                    return self.groupby(key_columns, graphlab.aggregate.CONCAT(column[0], column[1]))

    def unique(self):
        """
        Remove duplicate rows of the SFrame. Will not necessarily preserve the
        order of the given SFrame in the new SFrame.

        Returns
        -------
        out : SFrame
            A new SFrame that contains the unique rows of the current SFrame.

        Raises
        ------
        TypeError
          If any column in the SFrame is a dictionary type.

        See Also
        --------
        SArray.unique

        Examples
        --------
        >>> sf = graphlab.SFrame({'id':[1,2,3,3,4], 'value':[1,2,3,3,4]})
        >>> sf
        +----+-------+
        | id | value |
        +----+-------+
        | 1  |   1   |
        | 2  |   2   |
        | 3  |   3   |
        | 3  |   3   |
        | 4  |   4   |
        +----+-------+
        [5 rows x 2 columns]

        >>> sf.unique()
        +----+-------+
        | id | value |
        +----+-------+
        | 2  |   2   |
        | 4  |   4   |
        | 3  |   3   |
        | 1  |   1   |
        +----+-------+
        [4 rows x 2 columns]
        """
        return self.groupby(self.column_names(),{})

    def sort(self, sort_columns, ascending=True):
        """
        Sort current SFrame by the given columns, using the given sort order.
        Only columns that are type of str, int and float can be sorted.

        Parameters
        ----------
        sort_columns : str | list of str | list of (str, bool) pairs
            Names of columns to be sorted.  The result will be sorted first by
            first column, followed by second column, and so on. All columns will
            be sorted in the same order as governed by the `ascending`
            parameter. To control the sort ordering for each column
            individually, `sort_columns` must be a list of (str, bool) pairs.
            Given this case, the first value is the column name and the second
            value is a boolean indicating whether the sort order is ascending.

        ascending : bool, optional
            Sort all columns in the given order.

        Returns
        -------
        out : SFrame
            A new SFrame that is sorted according to given sort criteria

        See Also
        --------
        topk

        Examples
        --------
        Suppose 'sf' is an sframe that has three columns 'a', 'b', 'c'.
        To sort by column 'a', ascending

        >>> sf = graphlab.SFrame({'a':[1,3,2,1],
        ...                       'b':['a','c','b','b'],
        ...                       'c':['x','y','z','y']})
        >>> sf
        +---+---+---+
        | a | b | c |
        +---+---+---+
        | 1 | a | x |
        | 3 | c | y |
        | 2 | b | z |
        | 1 | b | y |
        +---+---+---+
        [4 rows x 3 columns]

        >>> sf.sort('a')
        +---+---+---+
        | a | b | c |
        +---+---+---+
        | 1 | a | x |
        | 1 | b | y |
        | 2 | b | z |
        | 3 | c | y |
        +---+---+---+
        [4 rows x 3 columns]

        To sort by column 'a', descending

        >>> sf.sort('a', ascending = False)
        +---+---+---+
        | a | b | c |
        +---+---+---+
        | 3 | c | y |
        | 2 | b | z |
        | 1 | a | x |
        | 1 | b | y |
        +---+---+---+
        [4 rows x 3 columns]

        To sort by column 'a' and 'b', all ascending

        >>> sf.sort(['a', 'b'])
        +---+---+---+
        | a | b | c |
        +---+---+---+
        | 1 | a | x |
        | 1 | b | y |
        | 2 | b | z |
        | 3 | c | y |
        +---+---+---+
        [4 rows x 3 columns]

        To sort by column 'a' ascending, and then by column 'c' descending

        >>> sf.sort([('a', True), ('c', False)])
        +---+---+---+
        | a | b | c |
        +---+---+---+
        | 1 | b | y |
        | 1 | a | x |
        | 2 | b | z |
        | 3 | c | y |
        +---+---+---+
        [4 rows x 3 columns]
        """
        sort_column_names = []
        sort_column_orders = []

        # validate sort_columns
        if (type(sort_columns) == str):
            sort_column_names = [sort_columns]
        elif (type(sort_columns) == list):
            if (len(sort_columns) == 0):
                raise ValueError("Please provide at least one column to sort")

            first_param_types = set([type(i) for i in sort_columns])
            if (len(first_param_types) != 1):
                raise ValueError("sort_columns element are not of the same type")

            first_param_type = first_param_types.pop()
            if (first_param_type == tuple):
                sort_column_names = [i[0] for i in sort_columns]
                sort_column_orders = [i[1] for i in sort_columns]
            elif(first_param_type == str):
                sort_column_names = sort_columns
            else:
                raise TypeError("sort_columns type is not supported")
        else:
            raise TypeError("sort_columns type is not correct. Supported types are str, list of str or list of (str,bool) pair.")

        # use the second parameter if the sort order is not given
        if (len(sort_column_orders) == 0):
            sort_column_orders = [ascending for i in sort_column_names]

        # make sure all column exists
        my_column_names = set(self.column_names())
        for column in sort_column_names:
            if (type(column) != str):
                raise TypeError("Only string parameter can be passed in as column names")
            if (column not in my_column_names):
                raise ValueError("SFrame has no column named: '" + str(column) + "'")
            if (self[column].dtype() not in (str, int, float,datetime.datetime)):
                raise TypeError("Only columns of type (str, int, float) can be sorted")

        _mt._get_metric_tracker().track('sframe.sort')

        with cython_context():
            return SFrame(_proxy=self.__proxy__.sort(sort_column_names, sort_column_orders))

    def dropna(self, columns=None, how='any'):
        """
        Remove missing values from an SFrame. A missing value is either ``None``
        or ``NaN``.  If ``how`` is 'any', a row will be removed if any of the
        columns in the ``columns`` parameter contains at least one missing
        value.  If ``how`` is 'all', a row will be removed if all of the columns
        in the ``columns`` parameter are missing values.

        If the ``columns`` parameter is not specified, the default is to
        consider all columns when searching for missing values.

        Parameters
        ----------
        columns : list or str, optional
            The columns to use when looking for missing values. By default, all
            columns are used.

        how : {'any', 'all'}, optional
            Specifies whether a row should be dropped if at least one column
            has missing values, or if all columns have missing values.  'any' is
            default.

        Returns
        -------
        out : SFrame
            SFrame with missing values removed (according to the given rules).

        See Also
        --------
        dropna_split :  Drops missing rows from the SFrame and returns them.

        Examples
        --------
        Drop all missing values.

        >>> sf = graphlab.SFrame({'a': [1, None, None], 'b': ['a', 'b', None]})
        >>> sf.dropna()
        +---+---+
        | a | b |
        +---+---+
        | 1 | a |
        +---+---+
        [1 rows x 2 columns]

        Drop rows where every value is missing.

        >>> sf.dropna(any="all")
        +------+---+
        |  a   | b |
        +------+---+
        |  1   | a |
        | None | b |
        +------+---+
        [2 rows x 2 columns]

        Drop rows where column 'a' has a missing value.

        >>> sf.dropna('a', any="all")
        +---+---+
        | a | b |
        +---+---+
        | 1 | a |
        +---+---+
        [1 rows x 2 columns]
        """
        _mt._get_metric_tracker().track('sframe.dropna')

        # If the user gives me an empty list (the indicator to use all columns)
        # NA values being dropped would not be the expected behavior. This
        # is a NOOP, so let's not bother the server
        if type(columns) is list and len(columns) == 0:
            return SFrame(_proxy=self.__proxy__)

        (columns, all_behavior) = self.__dropna_errchk(columns, how)

        with cython_context():
            return SFrame(_proxy=self.__proxy__.drop_missing_values(columns, all_behavior, False))

    def dropna_split(self, columns=None, how='any'):
        """
        Split rows with missing values from this SFrame. This function has the
        same functionality as :py:func:`~graphlab.SFrame.dropna`, but returns a
        tuple of two SFrames.  The first item is the expected output from
        :py:func:`~graphlab.SFrame.dropna`, and the second item contains all the
        rows filtered out by the `dropna` algorithm.

        Parameters
        ----------
        columns : list or str, optional
            The columns to use when looking for missing values. By default, all
            columns are used.

        how : {'any', 'all'}, optional
            Specifies whether a row should be dropped if at least one column
            has missing values, or if all columns have missing values.  'any' is
            default.

        Returns
        -------
        out : (SFrame, SFrame)
            (SFrame with missing values removed,
             SFrame with the removed missing values)

        See Also
        --------
        dropna

        Examples
        --------
        >>> sf = graphlab.SFrame({'a': [1, None, None], 'b': ['a', 'b', None]})
        >>> good, bad = sf.dropna_split()
        >>> good
        +---+---+
        | a | b |
        +---+---+
        | 1 | a |
        +---+---+
        [1 rows x 2 columns]

        >>> bad
        +------+------+
        |  a   |  b   |
        +------+------+
        | None |  b   |
        | None | None |
        +------+------+
        [2 rows x 2 columns]
        """
        _mt._get_metric_tracker().track('sframe.dropna_split')

        # If the user gives me an empty list (the indicator to use all columns)
        # NA values being dropped would not be the expected behavior. This
        # is a NOOP, so let's not bother the server
        if type(columns) is list and len(columns) == 0:
            return (SFrame(_proxy=self.__proxy__), SFrame())

        (columns, all_behavior) = self.__dropna_errchk(columns, how)

        sframe_tuple = self.__proxy__.drop_missing_values(columns, all_behavior, True)

        if len(sframe_tuple) != 2:
            raise RuntimeError("Did not return two SFrames!")

        with cython_context():
            return (SFrame(_proxy=sframe_tuple[0]), SFrame(_proxy=sframe_tuple[1]))

    def __dropna_errchk(self, columns, how):
        if columns is None:
            # Default behavior is to consider every column, specified to
            # the server by an empty list (to avoid sending all the column
            # in this case, since it is the most common)
            columns = list()
        elif type(columns) is str:
            columns = [columns]
        elif type(columns) is not list:
            raise TypeError("Must give columns as a list, str, or 'None'")
        else:
            # Verify that we are only passing strings in our list
            list_types = set([type(i) for i in columns])
            if (str not in list_types) or (len(list_types) > 1):
                raise TypeError("All columns must be of 'str' type")


        if how not in ['any','all']:
            raise ValueError("Must specify 'any' or 'all'")

        if how == 'all':
            all_behavior = True
        else:
            all_behavior = False

        return (columns, all_behavior)

    def fillna(self, column, value):
        """
        Fill all missing values with a given value in a given column. If the
        ``value`` is not the same type as the values in ``column``, this method
        attempts to convert the value to the original column's type. If this
        fails, an error is raised.

        Parameters
        ----------
        column : str
            The name of the column to modify.

        value : type convertible to SArray's type
            The value used to replace all missing values.

        Returns
        -------
        out : SFrame
            A new SFrame with the specified value in place of missing values.

        See Also
        --------
        dropna

        Examples
        --------
        >>> sf = graphlab.SFrame({'a':[1, None, None],
        ...                       'b':['13.1', '17.2', None]})
        >>> sf = sf.fillna('a', 0)
        >>> sf
        +---+------+
        | a |  b   |
        +---+------+
        | 1 | 13.1 |
        | 0 | 17.2 |
        | 0 | None |
        +---+------+
        [3 rows x 2 columns]
        """
        # Normal error checking
        if type(column) is not str:
            raise TypeError("Must give column name as a str")
        ret = self[self.column_names()]
        ret[column] = ret[column].fillna(value)
        return ret

    def add_row_number(self, column_name='id', start=0):
        """
        Returns a new SFrame with a new column that numbers each row
        sequentially. By default the count starts at 0, but this can be changed
        to a positive or negative number.  The new column will be named with
        the given column name.  An error will be raised if the given column
        name already exists in the SFrame.

        Parameters
        ----------
        column_name : str, optional
            The name of the new column that will hold the row numbers.

        start : int, optional
            The number used to start the row number count.

        Returns
        -------
        out : SFrame
            The new SFrame with a column name

        Notes
        -----
        The range of numbers is constrained by a signed 64-bit integer, so
        beware of overflow if you think the results in the row number column
        will be greater than 9 quintillion.

        Examples
        --------
        >>> sf = graphlab.SFrame({'a': [1, None, None], 'b': ['a', 'b', None]})
        >>> sf.add_row_number()
        +----+------+------+
        | id |  a   |  b   |
        +----+------+------+
        | 0  |  1   |  a   |
        | 1  | None |  b   |
        | 2  | None | None |
        +----+------+------+
        [3 rows x 3 columns]
        """
        _mt._get_metric_tracker().track('sframe.add_row_number')

        if type(column_name) is not str:
            raise TypeError("Must give column_name as strs")

        if type(start) is not int:
            raise TypeError("Must give start as int")

        if column_name in self.column_names():
            raise RuntimeError("Column '" + column_name + "' already exists in the current SFrame")

        the_col = _create_sequential_sarray(self.num_rows(), start)

        # Make sure the row number column is the first column
        new_sf = SFrame()
        new_sf.add_column(the_col, column_name)
        new_sf.add_columns(self)
        return new_sf

    @property
    def shape(self):
        """
        The shape of the SFrame, in a tuple. The first entry is the number of
        rows, the second is the number of columns.

        Examples
        --------
        >>> sf = graphlab.SFrame({'id':[1,2,3], 'val':['A','B','C']})
        >>> sf.shape
        (3, 2)
        """
        return (self.num_rows(), self.num_cols())

    @property
    def __proxy__(self):
        return self._proxy

    @__proxy__.setter
    def __proxy__(self, value):
        assert type(value) is UnitySFrameProxy
        self._proxy = value
