<file_sep>"""A flexible and expressive pandas validation library."""

from . import errors, constants
from .checks import Check
from .hypotheses import Hypothesis
from .decorators import check_input, check_output
from .dtypes import (
    PandasDtype,
    Bool,
    DateTime,
    Category,
    Float,
    Int,
    Object,
    String,
    Timedelta,
)
from .schemas import DataFrameSchema, SeriesSchema
from .schema_components import Column, Index, MultiIndex


__version__ = "0.3.0"
<file_sep>.. pandera documentation for seriesschemas

.. _SeriesSchemas:

Series Schemas
==============

``SeriesSchema``\s allow for the validation of ``pd.Series`` objects, and
are very similar to :ref:`columns<column>` and :ref:`indexes<index>` described
in :ref:`DataFrameSchemas<DataFrameSchemas>`.


.. testcode:: series_validation

    import pandas as pd
    import pandera as pa

    from pandera import Check, SeriesSchema

    # specify multiple validators
    schema = SeriesSchema(
        pa.String,
        checks=[
            Check(lambda s: s.str.startswith("foo")),
            Check(lambda s: s.str.endswith("bar")),
            Check(lambda x: len(x) > 3, element_wise=True)
        ],
        nullable=False,
        allow_duplicates=True,
        name="my_series")

    validated_series = schema.validate(
        pd.Series(["foobar", "foobar", "foobar"], name="my_series"))
    print(validated_series)

.. testoutput:: series_validation

    0    foobar
    1    foobar
    2    foobar
    Name: my_series, dtype: object
<file_sep>"""Tests a variety of python and pandas dtypes, and tests some specific
coercion examples."""

import pandas as pd
import pytest

from pandera import (
    Column, DataFrameSchema, Check, DateTime, Float, Int,
    String, Bool, Category, Object, Timedelta)
from pandera import dtypes
from pandera.errors import SchemaError


TESTABLE_DTYPES = [
    (Bool, "bool"),
    (DateTime, "datetime64[ns]"),
    (Category, "category"),
    (Float, "float64"),
    (Int, "int64"),
    (Object, "object"),
    (String, "object"),
    (Timedelta, "timedelta64[ns]"),
    ("bool", "bool"),
    ("datetime64[ns]", "datetime64[ns]"),
    ("category", "category"),
    ("float64", "float64"),
]


def test_numeric_dtypes():
    """Test every numeric type can be validated properly by schema.validate"""
    for dtype in [
            dtypes.Float,
            dtypes.Float16,
            dtypes.Float32,
            dtypes.Float64]:
        assert all(
            isinstance(
                schema.validate(
                    pd.DataFrame(
                        {"col": [-123.1, -7654.321, 1.0, 1.1, 1199.51, 5.1]},
                        dtype=dtype.value)),
                pd.DataFrame
            )
            for schema in [
                DataFrameSchema({"col": Column(dtype, nullable=False)}),
                DataFrameSchema({"col": Column(dtype.value, nullable=False)})
            ]
        )

    for dtype in [
            dtypes.Int,
            dtypes.Int8,
            dtypes.Int16,
            dtypes.Int32,
            dtypes.Int64]:
        assert all(
            isinstance(
                schema.validate(
                    pd.DataFrame(
                        {"col": [-712, -4, -321, 0, 1, 777, 5, 123, 9000]},
                        dtype=dtype.value)),
                pd.DataFrame
            )
            for schema in [
                DataFrameSchema({"col": Column(dtype, nullable=False)}),
                DataFrameSchema({"col": Column(dtype.value, nullable=False)})
            ]
        )

    for dtype in [
            dtypes.UInt8,
            dtypes.UInt16,
            dtypes.UInt32,
            dtypes.UInt64]:
        assert all(
            isinstance(
                schema.validate(
                    pd.DataFrame(
                        {"col": [1, 777, 5, 123, 9000]},
                        dtype=dtype.value)),
                pd.DataFrame
            )
            for schema in [
                DataFrameSchema({"col": Column(dtype, nullable=False)}),
                DataFrameSchema({"col": Column(dtype.value, nullable=False)})
            ]
        )


def test_category_dtype():
    """Test the category type can be validated properly by schema.validate"""
    schema = DataFrameSchema(
        columns={
            "col": Column(
                dtypes.Category,
                checks=[
                    Check(lambda s: set(s) == {"A", "B", "C"}),
                    Check(lambda s:
                          s.cat.categories.tolist() == ["A", "B", "C"]),
                    Check(lambda s: s.isin(["A", "B", "C"]))
                ],
                nullable=False
            ),
        },
        coerce=False
    )
    validated_df = schema.validate(
        pd.DataFrame(
            {"col": pd.Series(["A", "B", "A", "B", "C"], dtype="category")}
        )
    )
    assert isinstance(validated_df, pd.DataFrame)


def test_category_dtype_coerce():
    """Test coercion of the category type is validated properly by
    schema.validate and fails safely."""
    columns = {
        "col": Column(
            dtypes.Category,
            checks=Check(lambda s: set(s) == {"A", "B", "C"}),
            nullable=False
        ),
    }

    with pytest.raises(SchemaError):
        DataFrameSchema(columns=columns, coerce=False).validate(
            pd.DataFrame(
                {"col": pd.Series(["A", "B", "A", "B", "C"], dtype="object")}
            )
        )

    validated_df = DataFrameSchema(columns=columns, coerce=True).validate(
        pd.DataFrame(
            {"col": pd.Series(["A", "B", "A", "B", "C"], dtype="object")}
        )
    )
    assert isinstance(validated_df, pd.DataFrame)


def test_datetime():
    """Test datetime types can be validated properly by schema.validate"""
    schema = DataFrameSchema(
        columns={
            "col": Column(
                dtypes.DateTime,
                checks=Check(lambda s: s.min() > pd.Timestamp("2015")),
            )
        }
    )

    validated_df = schema.validate(
        pd.DataFrame(
            {"col": pd.to_datetime(["2019/01/01", "2018/05/21", "2016/03/10"])}
        )
    )

    assert isinstance(validated_df, pd.DataFrame)

    with pytest.raises(SchemaError):
        schema.validate(
            pd.DataFrame(
                {"col": pd.to_datetime(["2010/01/01"])}
            )
        )
<file_sep>.PHONY: docs tests upload-pypi conda-build-35 conda-build-36 conda-build-37

tests:
	pytest

clean:
	python setup.py clean

clean-pyc:
	find . -name '*.pyc' -exec rm {} \;

upload-pypi-test:
	python setup.py sdist bdist_wheel && \
		twine upload --repository-url https://test.pypi.org/legacy/ dist/* && \
		rm -rf dist

upload-pypi:
	python setup.py sdist bdist_wheel && \
		twine upload dist/* && \
		rm -rf dist

requirements:
	pip install -r requirements-dev.txt

docs:
	make -C docs doctest && python -m sphinx -E -W "docs/source" "docs/_build"

mock-ci-tests:
	. ./ci_tests.sh

conda-build: conda-build-35 conda-build-36 conda-build-37

conda-build-35:
	conda-build --python=3.5 conda.recipe

conda-build-36:
	conda-build --python=3.6 conda.recipe

conda-build-37:
	conda-build --python=3.7 conda.recipe
<file_sep>"""pandera-specific errors."""


class SchemaInitError(Exception):
    """Raised when schema initialization fails."""


class SchemaDefinitionError(Exception):
    """Raised when schema definition is invalid on object validation."""


class SchemaError(Exception):
    """Raised when object does not pass schema validation constraints."""
<file_sep>"""Schema datatypes."""
# pylint: disable=C0103

from enum import Enum


class PandasDtype(Enum):
    """Enumerate all valid pandas data types."""

    Bool = "bool"
    DateTime = "datetime64[ns]"
    Category = "category"
    Float = "float64"
    Float16 = "float16"
    Float32 = "float32"
    Float64 = "float64"
    Int = "int64"
    Int8 = "int8"
    Int16 = "int16"
    Int32 = "int32"
    Int64 = "int64"
    UInt8 = "uint8"
    UInt16 = "uint16"
    UInt32 = "uint32"
    UInt64 = "uint64"
    Object = "object"
    # the string datatype doesn't map to a unique string representation and is
    # representated as a numpy object array. This will change after pandas 1.0,
    # but for now will need to handle this as a special case.
    String = "string"
    Timedelta = "timedelta64[ns]"


Bool = PandasDtype.Bool
DateTime = PandasDtype.DateTime
Category = PandasDtype.Category
Float = PandasDtype.Float
Float16 = PandasDtype.Float16
Float32 = PandasDtype.Float32
Float64 = PandasDtype.Float64
Int = PandasDtype.Int
Int8 = PandasDtype.Int8
Int16 = PandasDtype.Int16
Int32 = PandasDtype.Int32
Int64 = PandasDtype.Int64
UInt8 = PandasDtype.UInt8
UInt16 = PandasDtype.UInt16
UInt32 = PandasDtype.UInt32
UInt64 = PandasDtype.UInt64
Object = PandasDtype.Object
String = PandasDtype.String
Timedelta = PandasDtype.Timedelta
<file_sep>.. pandera documentation for Checks

.. _checks:

Checks
======

Checking column properties
--------------------------

:class:`~pandera.checks.Check` objects accept a function as a required argument, which
is expected to have the following signature:

``pd.Series -> bool|pd.Series[bool]``.

For the :class:`~pandera.checks.Check` to pass, all of the elements in the boolean
series must evaluate to ``True``.


.. testcode:: checks

    import pandera as pa

    from pandera import Column, Check, DataFrameSchema

    schema = DataFrameSchema({"column1": Column(pa.Int, Check(lambda s: s <= 10))})


Multiple checks can be applied to a column:

.. testcode:: checks

  schema = DataFrameSchema({
      "column2": Column(pa.String, [
          Check(lambda s: s.str.startswith("value")),
          Check(lambda s: s.str.split("_", expand=True).shape[1] == 2)
      ]),
  })

Built-in Checks
---------------

For common validation tasks built-in checks are available in ``pandera``.
They are provided as factory methods in the :class:`~pandera.checks.Check` class.

This way comparison operations, string validations and whitelisting or blacklisting
of allowed values can be done more easily:

.. testcode:: builtin_checks

  import pandera as pa
  from pandera import Column, Check, DataFrameSchema

  schema = DataFrameSchema({
      "small_values": Column(pa.Float, [Check.less_than(100)]),
      "one_to_three": Column(pa.Int, [Check.isin([1, 2, 3])]),
      "phone_number": Column(pa.String, [Check.str_matches(r'^[a-z0-9-]+$')]),
  })

Vectorized vs. Element-wise Checks
----------------------------------

By default, :class:`~pandera.checks.Check` objects operate on ``pd.Series`` objects.
If you want to make atomic checks for each element in the Column, then you can provide
the ``element_wise=True`` keyword argument:

.. testcode:: vectorized_element_wise_checks

    import pandas as pd
    import pandera as pa

    from pandera import Column, Check, DataFrameSchema

    schema = DataFrameSchema({
        "a": Column(
            pa.Int,
            [
                # a vectorized check that returns a bool
                Check(lambda s: s.mean() > 5, element_wise=False),
                # a vectorized check that returns a boolean series
                Check(lambda s: s > 0, element_wise=False),
                # an element-wise check that returns a bool
                Check(lambda x: x > 0, element_wise=True),
            ]
        ),
    })
    df = pd.DataFrame({"a": [4, 4, 5, 6, 6, 7, 8, 9]})
    schema.validate(df)


``element_wise == False`` by default so that you can take advantage of the
speed gains provided by the ``pd.Series`` API by writing vectorized
checks.

.. _grouping:

Column Check Groups
-------------------

:class:`~pandera.schema_components.Column` checks support grouping by a different column so that
you can make assertions about subsets of the :class:`~pandera.schema_components.Column` of interest.
This changes the function signature of the :class:`~pandera.checks.Check` function so that
its input is a dict where keys are the group names and values are subsets
of the :class:`~pandera.schema_components.Column` series.

Specifying ``groupby`` as a column name, list of column names, or
callable changes the expected signature of the :class:`~pandera.checks.Check` function
argument to ``dict[Any|tuple[Any], Series] -> bool|Series[bool]`` where
the dict keys are the discrete keys in the ``groupby`` columns.

.. testcode:: column_check_groups

    import pandas as pd
    import pandera as pa

    from pandera import Column, Check, DataFrameSchema

    schema = DataFrameSchema({
        "height_in_feet": Column(
            pa.Float, [
                # groupby as a single column
                Check(lambda g: g[False].mean() > 6,
                      groupby="age_less_than_20"),
                # define multiple groupby columns
                Check(lambda g: g[(True, "F")].sum() == 9.1,
                      groupby=["age_less_than_20", "sex"]),
                # groupby as a callable with signature:
                # (DataFrame) -> DataFrameGroupBy
                Check(lambda g: g[(False, "M")].median() == 6.75,
                      groupby=lambda df: (
                        df
                        .assign(age_less_than_15=lambda d: d["age"] < 15)
                        .groupby(["age_less_than_15", "sex"]))),
            ]),
        "age": Column(pa.Int, Check(lambda s: s > 0)),
        "age_less_than_20": Column(pa.Bool),
        "sex": Column(pa.String, Check(lambda s: s.isin(["M", "F"])))
    })

    df = (
        pd.DataFrame({
            "height_in_feet": [6.5, 7, 6.1, 5.1, 4],
            "age": [25, 30, 21, 18, 13],
            "sex": ["M", "M", "F", "F", "F"]
        })
        .assign(age_less_than_20=lambda x: x["age"] < 20)
    )

    schema.validate(df)

In the above example we define a :class:`~pandera.schemas.DataFrameSchema` with column
checks for ``height_in_feet`` using a single column, multiple columns, and a
more complex groupby function that creates a new column
``age_less_than_15`` on the fly.


Wide Checks
-----------

``pandera`` is primarily designed to operate on long-form data (commonly known
as `tidy data <https://vita.had.co.nz/papers/tidy-data.pdf>`_), where each row
is an observation and each column is an attribute associated with an
observation.

However, ``pandera`` also supports checks on wide-form data to operate across
columns in a ``DataFrame``.

For example, if you want to make assertions about ``height`` across two groups,
the tidy dataset and schema might look like this:

.. testcode:: wide_checks

    import pandas as pd
    import pandera as pa

    from pandera import DataFrameSchema, Column, Check

    df = pd.DataFrame({
        "height": [5.6, 6.4, 4.0, 7.1],
        "group": ["A", "B", "A", "B"],
    })

    schema = DataFrameSchema({
        "height": Column(
            pa.Float,
            Check(lambda g: g["A"].mean() < g["B"].mean(), groupby="group")
        ),
        "group": Column(pa.String)
    })

    schema.validate(df)


The equivalent wide-form schema would look like this:

.. testcode:: wide_checks

    df = pd.DataFrame({
        "height_A": [5.6, 4.0],
        "height_B": [6.4, 7.1],
    })

    schema = DataFrameSchema(
        columns={
            "height_A": Column(pa.Float),
            "height_B": Column(pa.Float),
        },
        # define checks at the DataFrameSchema-level
        checks=Check(lambda df: df["height_A"].mean() < df["height_B"].mean())
    )

    schema.validate(df)
<file_sep><div align="left"><img src="https://raw.githubusercontent.com/pandera-dev/pandera/master/docs/source/_static/pandera-logo.png" width="140"></div>

# Pandera

A flexible and expressive [pandas](http://pandas.pydata.org) validation library.

<br>

[![Build Status](https://travis-ci.org/pandera-dev/pandera.svg?branch=master)](https://travis-ci.org/pandera-dev/pandera)
[![PyPI version shields.io](https://img.shields.io/pypi/v/pandera.svg)](https://pypi.org/project/pandera/)
[![PyPI license](https://img.shields.io/pypi/l/pandera.svg)](https://pypi.python.org/pypi/)
[![pyOpenSci](https://tinyurl.com/y22nb8up)](https://github.com/pyOpenSci/software-review/issues/12)
[![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
[![Documentation Status](https://readthedocs.org/projects/pandera/badge/?version=latest)](https://pandera.readthedocs.io/en/latest/?badge=latest)
[![codecov](https://codecov.io/gh/pandera-dev/pandera/branch/master/graph/badge.svg)](https://codecov.io/gh/pandera-dev/pandera)
[![PyPI pyversions](https://img.shields.io/pypi/pyversions/pandera.svg)](https://pypi.python.org/pypi/pandera/)
[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3385266.svg)](https://doi.org/10.5281/zenodo.3385266)
[![asv](http://img.shields.io/badge/benchmarked%20by-asv-green.svg?style=flat)](https://pandera-dev.github.io/pandera-asv-logs/)

`pandas` data structures contain information that `pandera` explicitly
validates at runtime. This is useful in production-critical or reproducible research
settings. `pandera` enables users to:

1. Check the types and properties of columns in a `DataFrame` or values in
   a `Series`.
1. Perform more complex statistical validation like hypothesis testing.
1. Seamlessly integrate with existing data analysis/processing pipelines
   via function decorators.

`pandera` provides a flexible and expressive API for performing data validation
on tidy (long-form) and wide data to make data processing pipelines more
readable and robust.


## Documentation

The official documentation is hosted on ReadTheDocs: https://pandera.readthedocs.io

## Install

Using pip:

```
pip install pandera
```

Using conda:

```
conda install -c conda-forge pandera
```

## Example Usage

### `DataFrameSchema`

```python
import pandas as pd
import pandera as pa

from pandera import Column, DataFrameSchema, Check, check_output


# validate columns
schema = DataFrameSchema({
    # the check function expects a series argument and should output a boolean
    # or a boolean Series.
    "column1": Column(pa.Int, Check(lambda s: s <= 10)),
    "column2": Column(pa.Float, Check(lambda s: s < -1.2)),
    # you can provide a list of validators
    "column3": Column(pa.String, [
        Check(lambda s: s.str.startswith("value_")),
        Check(lambda s: s.str.split("_", expand=True).shape[1] == 2)
    ]),
})

df = pd.DataFrame({
    "column1": [1, 4, 0, 10, 9],
    "column2": [-1.3, -1.4, -2.9, -10.1, -20.4],
    "column3": ["value_1", "value_2", "value_3", "value_2", "value_1"]
})

validated_df = schema.validate(df)
print(validated_df)

#     column1  column2  column3
#  0        1     -1.3  value_1
#  1        4     -1.4  value_2
#  2        0     -2.9  value_3
#  3       10    -10.1  value_2
#  4        9    -20.4  value_1

# If you have an existing data pipeline that uses pandas data structures, you can use the check_input and check_output decorators to check function arguments or returned variables from existing functions.

@check_output(schema)
def custom_function(df):
    return df
```

## Development Installation

```
git clone https://github.com/pandera-dev/pandera.git
cd pandera
pip install -r requirements-dev.txt
pip install -e .
```

## Tests

```
pip install pytest
pytest tests
```

## Contributing to pandera [![GitHub contributors](https://img.shields.io/github/contributors/pandera-dev/pandera.svg)](https://github.com/pandera-dev/pandera/graphs/contributors)

All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome.

A detailed overview on how to contribute can be found in the [contributing guide](https://github.com/pandera-dev/pandera/blob/master/.github/CONTRIBUTING.md) on GitHub.

## Issues

Go [here](https://github.com/pandera-dev/pandera-dev/issues) to submit feature
requests or bugfixes.

## Other Data Validation Libraries

Here are a few other alternatives for validating Python data structures.

**Generic Python object data validation**

- [voloptuous](https://github.com/alecthomas/voluptuous)
- [schema](https://github.com/keleshev/schema)

**`pandas`-specific data validation**

- [opulent-pandas](https://github.com/danielvdende/opulent-pandas)
- [PandasSchema](https://github.com/TMiguelT/PandasSchema)
- [pandas-validator](https://github.com/c-data/pandas-validator)
- [table_enforcer](https://github.com/xguse/table_enforcer)

**Other tools that include data validation**

- [great_expectations](https://github.com/great-expectations/great_expectations)

## Why `pandera`?

- `pandas`-centric data types, column nullability, and uniqueness are
  first-class concepts.
- `check_input` and `check_output` decorators enable seamless integration with
  existing code.
- `Check`s provide flexibility and performance by providing access to `pandas`
  API by design.
- `Hypothesis` class provides a tidy-first interface for statistical hypothesis
  testing.
- `Check`s and `Hypothesis` objects support both tidy and wide data validation.
- Comprehensive documentation on key functionality.


### Citation Information

```
@misc{niels_bantilan_2019_3385266,
  author       = {<NAME> and
                  <NAME> and
                  <NAME> and
                  chr1st1ank},
  title        = {pandera-dev/pandera: 0.2.0 pre-release 1},
  month        = sep,
  year         = 2019,
  doi          = {10.5281/zenodo.3385266},
  url          = {https://doi.org/10.5281/zenodo.3385266}
}
```
<file_sep>#!/bin/sh
pip install schema
$PYTHON setup.py install
<file_sep>.. pandera package index documentation toctree

API
===

pandera.schemas
---------------

.. automodule:: pandera.schemas
   :members:
   :undoc-members:
   :show-inheritance:


pandera.schema_components
-------------------------

.. automodule:: pandera.schema_components
   :members:
   :undoc-members:
   :show-inheritance:


pandera.checks
--------------

.. automodule:: pandera.checks
   :members:
   :undoc-members:
   :show-inheritance:


pandera.hypotheses
------------------

.. automodule:: pandera.hypotheses
   :members:
   :undoc-members:
   :show-inheritance:


pandera.decorators
------------------

.. automodule:: pandera.decorators
   :members:
   :undoc-members:
   :show-inheritance:


pandera.dtypes
--------------

.. automodule:: pandera.dtypes
   :members:
   :undoc-members:
   :show-inheritance:


pandera.errors
--------------

.. automodule:: pandera.errors
   :members:
   :undoc-members:
   :show-inheritance:
