diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt
index 3d82dd042da20..29b766e616b3b 100644
--- a/doc/source/whatsnew/v0.24.0.txt
+++ b/doc/source/whatsnew/v0.24.0.txt
@@ -48,7 +48,7 @@ Pandas has gained the ability to hold integer dtypes with missing values. This l
 Here is an example of the usage.

 We can construct a ``Series`` with the specified dtype. The dtype string ``Int64`` is a pandas ``ExtensionDtype``. Specifying a list or array using the traditional missing value
-marker of ``np.nan`` will infer to integer dtype. The display of the ``Series`` will also use the ``NaN`` to indicate missing values in string outputs. (:issue:`20700`, :issue:`20747`, :issue:`22441`)
+marker of ``np.nan`` will infer to integer dtype. The display of the ``Series`` will also use the ``NaN`` to indicate missing values in string outputs. (:issue:`20700`, :issue:`20747`, :issue:`22441`, :issue:`21789`, :issue:`22346`)

 .. ipython:: python

@@ -91,6 +91,13 @@ These dtypes can be merged & reshaped & casted.
    pd.concat([df[['A']], df[['B', 'C']]], axis=1).dtypes
    df['A'].astype(float)

+Reduction and groupby operations such as 'sum' work.
+
+.. ipython:: python
+
+   df.sum()
+   df.groupby('B').A.sum()
+
 .. warning::

    The Integer NA support currently uses the captilized dtype version, e.g. ``Int8`` as compared to the traditional ``int8``. This may be changed at a future date.
@@ -550,6 +557,7 @@ update the ``ExtensionDtype._metadata`` tuple to match the signature of your
 - Added :meth:`pandas.api.types.register_extension_dtype` to register an extension type with pandas (:issue:`22664`)
 - Series backed by an ``ExtensionArray`` now work with :func:`util.hash_pandas_object` (:issue:`23066`)
 - Updated the ``.type`` attribute for ``PeriodDtype``, ``DatetimeTZDtype``, and ``IntervalDtype`` to be instances of the dtype (``Period``, ``Timestamp``, and ``Interval`` respectively) (:issue:`22938`)
+- Support for reduction operations such as ``sum``, ``mean`` via opt-in base class method override (:issue:`22762`)

 .. _whatsnew_0240.api.incompatibilities:

diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py
index 627afd1b6f860..ef7e25033f24e 100644
--- a/pandas/core/arrays/base.py
+++ b/pandas/core/arrays/base.py
@@ -63,6 +63,10 @@ class ExtensionArray(object):
     as they only compose abstract methods. Still, a more efficient
     implementation may be available, and these methods can be overridden.

+    One can implement methods to handle array reductions.
+
+    * _reduce
+
     This class does not inherit from 'abc.ABCMeta' for performance reasons.
     Methods and properties required by the interface raise
     ``pandas.errors.AbstractMethodError`` and no ``register`` method is
@@ -675,6 +679,33 @@ def _ndarray_values(self):
         """
         return np.array(self)

+    def _reduce(self, name, skipna=True, **kwargs):
+        """
+        Return a scalar result of performing the reduction operation.
+
+        Parameters
+        ----------
+        name : str
+            Name of the function, supported values are:
+            { any, all, min, max, sum, mean, median, prod,
+            std, var, sem, kurt, skew }.
+        skipna : bool, default True
+            If True, skip NaN values.
+        **kwargs
+            Additional keyword arguments passed to the reduction function.
+            Currently, `ddof` is the only supported kwarg.
+
+        Returns
+        -------
+        scalar
+
+        Raises
+        ------
+        TypeError : subclass does not define reductions
+        """
+        raise TypeError("cannot perform {name} with type {dtype}".format(
+            name=name, dtype=self.dtype))
+

 class ExtensionOpsMixin(object):
     """
diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py
index 216bccf7d6309..79070bbbfd11a 100644
--- a/pandas/core/arrays/categorical.py
+++ b/pandas/core/arrays/categorical.py
@@ -2069,14 +2069,12 @@ def _reverse_indexer(self):
         return result

     # reduction ops #
-    def _reduce(self, op, name, axis=0, skipna=True, numeric_only=None,
-                filter_type=None, **kwds):
-        """ perform the reduction type operation """
+    def _reduce(self, name, axis=0, skipna=True, **kwargs):
         func = getattr(self, name, None)
         if func is None:
             msg = 'Categorical cannot perform the operation {op}'
             raise TypeError(msg.format(op=name))
-        return func(numeric_only=numeric_only, **kwds)
+        return func(**kwargs)

     def min(self, numeric_only=None, **kwargs):
         """ The minimum value of the object.
diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py
index e58109a25e1a5..9917045f2f7d2 100644
--- a/pandas/core/arrays/integer.py
+++ b/pandas/core/arrays/integer.py
@@ -8,6 +8,7 @@
 from pandas.compat import u, range, string_types
 from pandas.compat import set_function_name

+from pandas.core import nanops
 from pandas.core.dtypes.cast import astype_nansafe
 from pandas.core.dtypes.generic import ABCSeries, ABCIndexClass
 from pandas.core.dtypes.common import (
@@ -529,6 +530,31 @@ def cmp_method(self, other):
         name = '__{name}__'.format(name=op.__name__)
         return set_function_name(cmp_method, name, cls)

+    def _reduce(self, name, skipna=True, **kwargs):
+        data = self._data
+        mask = self._mask
+
+        # coerce to a nan-aware float if needed
+        if mask.any():
+            data = self._data.astype('float64')
+            data[mask] = self._na_value
+
+        op = getattr(nanops, 'nan' + name)
+        result = op(data, axis=0, skipna=skipna, mask=mask)
+
+        # if we have a boolean op, don't coerce
+        if name in ['any', 'all']:
+            pass
+
+        # if we have a preservable numeric op,
+        # provide coercion back to an integer type if possible
+        elif name in ['sum', 'min', 'max', 'prod'] and notna(result):
+            int_result = int(result)
+            if int_result == result:
+                result = int_result
+
+        return result
+
     def _maybe_mask_result(self, result, mask, other, op_name):
         """
         Parameters
diff --git a/pandas/core/series.py b/pandas/core/series.py
index a613b22ea9046..bff0f9fe25532 100644
--- a/pandas/core/series.py
+++ b/pandas/core/series.py
@@ -3392,16 +3392,25 @@ def _reduce(self, op, name, axis=0, skipna=True, numeric_only=None,

         """
         delegate = self._values
-        if isinstance(delegate, np.ndarray):
-            # Validate that 'axis' is consistent with Series's single axis.
-            if axis is not None:
-                self._get_axis_number(axis)
+
+        if axis is not None:
+            self._get_axis_number(axis)
+
+        # dispatch to ExtensionArray interface
+        if isinstance(delegate, ExtensionArray):
+            return delegate._reduce(name, skipna=skipna, **kwds)
+
+        # dispatch to numpy arrays
+        elif isinstance(delegate, np.ndarray):
             if numeric_only:
                 raise NotImplementedError('Series.{0} does not implement '
                                           'numeric_only.'.format(name))
             with np.errstate(all='ignore'):
                 return op(delegate, skipna=skipna, **kwds)

+        # TODO(EA) dispatch to Index
+        # remove once all internals extension types are
+        # moved to ExtensionArrays
         return delegate._reduce(op=op, name=name, axis=axis, skipna=skipna,
                                 numeric_only=numeric_only,
                                 filter_type=filter_type, **kwds)

diff --git a/pandas/conftest.py b/pandas/conftest.py
index 621de3ffd4b12..e84657a79b51a 100644
--- a/pandas/conftest.py
+++ b/pandas/conftest.py
@@ -131,6 +131,30 @@ def all_arithmetic_operators(request):
     return request.param

+_all_numeric_reductions = ['sum', 'max', 'min',
+                           'mean', 'prod', 'std', 'var', 'median',
+                           'kurt', 'skew']
+
+
+@pytest.fixture(params=_all_numeric_reductions)
+def all_numeric_reductions(request):
+    """
+    Fixture for numeric reduction names
+    """
+    return request.param
+
+
+_all_boolean_reductions = ['all', 'any']
+
+
+@pytest.fixture(params=_all_boolean_reductions)
+def all_boolean_reductions(request):
+    """
+    Fixture for boolean reduction names
+    """
+    return request.param
+
+
 _cython_table = pd.core.base.SelectionMixin._cython_table.items()

diff --git a/pandas/tests/arrays/test_integer.py b/pandas/tests/arrays/test_integer.py
index 349a6aee5701e..23ee8d217bd59 100644
--- a/pandas/tests/arrays/test_integer.py
+++ b/pandas/tests/arrays/test_integer.py
@@ -114,6 +114,13 @@ def _check_op(self, s, op_name, other, exc=None):
         # compute expected
         mask = s.isna()

+        # if s is a DataFrame, squeeze to a Series
+        # for comparison
+        if isinstance(s, pd.DataFrame):
+            result = result.squeeze()
+            s = s.squeeze()
+            mask = mask.squeeze()
+
         # other array is an Integer
         if isinstance(other, IntegerArray):
             omask = getattr(other, 'mask', None)
@@ -215,7 +222,6 @@ def test_arith_series_with_scalar(self, data, all_arithmetic_operators):
         s = pd.Series(data)
         self._check_op(s, op, 1, exc=TypeError)

-    @pytest.mark.xfail(run=False, reason="_reduce needs implementation")
     def test_arith_frame_with_scalar(self, data, all_arithmetic_operators):
         # frame & scalar
         op = all_arithmetic_operators
@@ -587,15 +593,23 @@ def test_cross_type_arithmetic():
     tm.assert_series_equal(result, expected)

-def test_groupby_mean_included():
+@pytest.mark.parametrize('op', ['sum', 'min', 'max', 'prod'])
+def test_preserve_dtypes(op):
+    # TODO(#22346): preserve Int64 dtype
+    # for ops that enable (mean would actually work here
+    # but generally it is a float return value)
     df = pd.DataFrame({
         "A": ['a', 'b', 'b'],
         "B": [1, None, 3],
         "C": integer_array([1, None, 3], dtype='Int64'),
     })

-    result = df.groupby("A").sum()
-    # TODO(#22346): preserve Int64 dtype
+    # op
+    result = getattr(df.C, op)()
+    assert isinstance(result, int)
+
+    # groupby
+    result = getattr(df.groupby("A"), op)()
     expected = pd.DataFrame({
         "B": np.array([1.0, 3.0]),
         "C": np.array([1, 3], dtype="int64")
@@ -603,6 +617,29 @@ def test_groupby_mean_included():
     tm.assert_frame_equal(result, expected)

+@pytest.mark.parametrize('op', ['mean'])
+def test_reduce_to_float(op):
+    # some reduce ops always return float, even if the result
+    # is a rounded number
+    df = pd.DataFrame({
+        "A": ['a', 'b', 'b'],
+        "B": [1, None, 3],
+        "C": integer_array([1, None, 3], dtype='Int64'),
+    })
+
+    # op
+    result = getattr(df.C, op)()
+    assert isinstance(result, float)
+
+    # groupby
+    result = getattr(df.groupby("A"), op)()
+    expected = pd.DataFrame({
+        "B": np.array([1.0, 3.0]),
+        "C": np.array([1, 3], dtype="float64")
+    }, index=pd.Index(['a', 'b'], name='A'))
+    tm.assert_frame_equal(result, expected)
+
+
 def test_astype_nansafe():
     # https://github.com/pandas-dev/pandas/pull/22343
     arr = integer_array([np.nan, 1, 2], dtype="Int8")
diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py
index f87c51a4ee16b..882b2c156478a 100644
--- a/pandas/tests/dtypes/test_common.py
+++ b/pandas/tests/dtypes/test_common.py
@@ -386,6 +386,8 @@ def test_is_datetime_or_timedelta_dtype():
     assert not com.is_datetime_or_timedelta_dtype(str)
     assert not com.is_datetime_or_timedelta_dtype(pd.Series([1, 2]))
     assert not com.is_datetime_or_timedelta_dtype(np.array(['a', 'b']))
+    assert not com.is_datetime_or_timedelta_dtype(
+        DatetimeTZDtype("ns", "US/Eastern"))

     assert com.is_datetime_or_timedelta_dtype(np.datetime64)
     assert com.is_datetime_or_timedelta_dtype(np.timedelta64)
diff --git a/pandas/tests/extension/arrow/test_bool.py b/pandas/tests/extension/arrow/test_bool.py
index e1afedcade3ff..12c37d1fdf895 100644
--- a/pandas/tests/extension/arrow/test_bool.py
+++ b/pandas/tests/extension/arrow/test_bool.py
@@ -39,6 +39,10 @@ def test_from_dtype(self, data):
         pytest.skip("GH-22666")

+class TestReduce(base.BaseNoReduceTests):
+    pass
+
+
 def test_is_bool_dtype(data):
     assert pd.api.types.is_bool_dtype(data)
     assert pd.core.common.is_bool_indexer(data)
diff --git a/pandas/tests/extension/base/__init__.py b/pandas/tests/extension/base/__init__.py
index b6b81bb941a59..d11bb8b6beb77 100644
--- a/pandas/tests/extension/base/__init__.py
+++ b/pandas/tests/extension/base/__init__.py
@@ -48,6 +48,7 @@ class TestMyDtype(BaseDtypeTests):
 from .interface import BaseInterfaceTests  # noqa
 from .methods import BaseMethodsTests  # noqa
 from .ops import BaseArithmeticOpsTests, BaseComparisonOpsTests, BaseOpsUtil  # noqa
+from .reduce import BaseNoReduceTests, BaseNumericReduceTests, BaseBooleanReduceTests  # noqa
 from .missing import BaseMissingTests  # noqa
 from .reshaping import BaseReshapingTests  # noqa
 from .setitem import BaseSetitemTests  # noqa
diff --git a/pandas/tests/extension/base/groupby.py b/pandas/tests/extension/base/groupby.py
index 174997c7d51e1..52c635d286df6 100644
--- a/pandas/tests/extension/base/groupby.py
+++ b/pandas/tests/extension/base/groupby.py
@@ -25,8 +25,8 @@ def test_groupby_extension_agg(self, as_index, data_for_grouping):
                            "B": data_for_grouping})
         result = df.groupby("B", as_index=as_index).A.mean()
         _, index = pd.factorize(data_for_grouping, sort=True)
-        # TODO(ExtensionIndex): remove astype
-        index = pd.Index(index.astype(object), name="B")
+
+        index = pd.Index(index, name="B")
         expected = pd.Series([3, 1, 4], index=index, name="A")
         if as_index:
             self.assert_series_equal(result, expected)
@@ -39,8 +39,8 @@ def test_groupby_extension_no_sort(self, data_for_grouping):
                            "B": data_for_grouping})
         result = df.groupby("B", sort=False).A.mean()
         _, index = pd.factorize(data_for_grouping, sort=False)
-        # TODO(ExtensionIndex): remove astype
-        index = pd.Index(index.astype(object), name="B")
+
+        index = pd.Index(index, name="B")
         expected = pd.Series([1, 3, 4], index=index, name="A")
         self.assert_series_equal(result, expected)

diff --git a/pandas/tests/extension/base/reduce.py b/pandas/tests/extension/base/reduce.py
new file mode 100644
index 0000000000000..4f6c7988314c0
--- /dev/null
+++ b/pandas/tests/extension/base/reduce.py
@@ -0,0 +1,58 @@
+import warnings
+import pytest
+import pandas.util.testing as tm
+import pandas as pd
+from .base import BaseExtensionTests
+
+
+class BaseReduceTests(BaseExtensionTests):
+    """
+    Reduction specific tests. Generally these only
+    make sense for numeric/boolean operations.
+    """
+    def check_reduce(self, s, op_name, skipna):
+        result = getattr(s, op_name)(skipna=skipna)
+        expected = getattr(s.astype('float64'), op_name)(skipna=skipna)
+        tm.assert_almost_equal(result, expected)
+
+
+class BaseNoReduceTests(BaseReduceTests):
+    """ we don't define any reductions """
+
+    @pytest.mark.parametrize('skipna', [True, False])
+    def test_reduce_series_numeric(self, data, all_numeric_reductions, skipna):
+        op_name = all_numeric_reductions
+        s = pd.Series(data)
+
+        with pytest.raises(TypeError):
+            getattr(s, op_name)(skipna=skipna)
+
+    @pytest.mark.parametrize('skipna', [True, False])
+    def test_reduce_series_boolean(self, data, all_boolean_reductions, skipna):
+        op_name = all_boolean_reductions
+        s = pd.Series(data)
+
+        with pytest.raises(TypeError):
+            getattr(s, op_name)(skipna=skipna)
+
+
+class BaseNumericReduceTests(BaseReduceTests):
+
+    @pytest.mark.parametrize('skipna', [True, False])
+    def test_reduce_series(self, data, all_numeric_reductions, skipna):
+        op_name = all_numeric_reductions
+        s = pd.Series(data)
+
+        # min/max with empty produce numpy warnings
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            self.check_reduce(s, op_name, skipna)
+
+
+class BaseBooleanReduceTests(BaseReduceTests):
+
+    @pytest.mark.parametrize('skipna', [True, False])
+    def test_reduce_series(self, data, all_boolean_reductions, skipna):
+        op_name = all_boolean_reductions
+        s = pd.Series(data)
+        self.check_reduce(s, op_name, skipna)
diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py
index a1ee3a4fefef2..53a598559393c 100644
--- a/pandas/tests/extension/decimal/array.py
+++ b/pandas/tests/extension/decimal/array.py
@@ -134,6 +134,18 @@ def _na_value(self):
     def _concat_same_type(cls, to_concat):
         return cls(np.concatenate([x._data for x in to_concat]))

+    def _reduce(self, name, skipna=True, **kwargs):
+
+        if skipna:
+            raise NotImplementedError("decimal does not support skipna=True")
+
+        try:
+            op = getattr(self.data, name)
+        except AttributeError:
+            raise NotImplementedError("decimal does not support "
+                                      "the {} operation".format(name))
+        return op(axis=0)
+

 def to_decimal(values, context=None):
     return DecimalArray([decimal.Decimal(x) for x in values], context=context)
diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py
index 6724e183a0606..f84d24295b049 100644
--- a/pandas/tests/extension/decimal/test_decimal.py
+++ b/pandas/tests/extension/decimal/test_decimal.py
@@ -131,6 +131,28 @@ class TestMissing(BaseDecimal, base.BaseMissingTests):
     pass

+class Reduce(object):
+
+    def check_reduce(self, s, op_name, skipna):
+
+        if skipna or op_name in ['median', 'skew', 'kurt']:
+            with pytest.raises(NotImplementedError):
+                getattr(s, op_name)(skipna=skipna)
+
+        else:
+            result = getattr(s, op_name)(skipna=skipna)
+            expected = getattr(np.asarray(s), op_name)()
+            tm.assert_almost_equal(result, expected)
+
+
+class TestNumericReduce(Reduce, base.BaseNumericReduceTests):
+    pass
+
+
+class TestBooleanReduce(Reduce, base.BaseBooleanReduceTests):
+    pass
+
+
 class TestMethods(BaseDecimal, base.BaseMethodsTests):
     @pytest.mark.parametrize('dropna', [True, False])
     @pytest.mark.xfail(reason="value_counts not implemented yet.")
diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json/test_json.py
index 6c8b12ed865fc..15d99f6c5d2fc 100644
--- a/pandas/tests/extension/json/test_json.py
+++ b/pandas/tests/extension/json/test_json.py
@@ -160,6 +160,10 @@ def test_fillna_frame(self):
                               reason="Dictionary order unstable")

+class TestReduce(base.BaseNoReduceTests):
+    pass
+
+
 class TestMethods(BaseJSON, base.BaseMethodsTests):
     @unhashable
     def test_value_counts(self, all_data, dropna):
diff --git a/pandas/tests/extension/test_categorical.py b/pandas/tests/extension/test_categorical.py
index f118279c4b915..a4518798aa400 100644
--- a/pandas/tests/extension/test_categorical.py
+++ b/pandas/tests/extension/test_categorical.py
@@ -164,6 +164,10 @@ def test_fillna_limit_backfill(self):
         pass

+class TestReduce(base.BaseNoReduceTests):
+    pass
+
+
 class TestMethods(base.BaseMethodsTests):
     pass

diff --git a/pandas/tests/extension/test_integer.py b/pandas/tests/extension/test_integer.py
index fa5c89d85e548..89c36bbe7b325 100644
--- a/pandas/tests/extension/test_integer.py
+++ b/pandas/tests/extension/test_integer.py
@@ -207,18 +207,12 @@ class TestCasting(base.BaseCastingTests):

 class TestGroupby(base.BaseGroupbyTests):
+    pass
+
+
+class TestNumericReduce(base.BaseNumericReduceTests):
+    pass
+

-    @pytest.mark.xfail(reason="groupby not working", strict=True)
-    def test_groupby_extension_no_sort(self, data_for_grouping):
-        super(TestGroupby, self).test_groupby_extension_no_sort(
-            data_for_grouping)
-
-    @pytest.mark.parametrize('as_index', [
-        pytest.param(True,
-                     marks=pytest.mark.xfail(reason="groupby not working",
-                                             strict=True)),
-        False
-    ])
-    def test_groupby_extension_agg(self, as_index, data_for_grouping):
-        super(TestGroupby, self).test_groupby_extension_agg(
-            as_index, data_for_grouping)
+class TestBooleanReduce(base.BaseBooleanReduceTests):
+    pass
diff --git a/pandas/tests/extension/test_interval.py b/pandas/tests/extension/test_interval.py
index 7302c5757d144..183ebea927b10 100644
--- a/pandas/tests/extension/test_interval.py
+++ b/pandas/tests/extension/test_interval.py
@@ -98,6 +98,10 @@ class TestInterface(BaseInterval, base.BaseInterfaceTests):
     pass

+class TestReduce(base.BaseNoReduceTests):
+    pass
+
+
 class TestMethods(BaseInterval, base.BaseMethodsTests):

     @pytest.mark.skip(reason='addition is not defined for intervals')

ENH: add reduce op (and groupby reduce) support to EA
after #21160
Reductions for ExtensionArray
Creeping up to this in https://github.com/pandas-dev/pandas/pull/22345

A few questions

1. What should this look like for EA authors? What helpers can / should we provide?
2. How does this affect users? Specifically, assuming IntegerArray implements reductions, what's the dtype here?

```python
In [11]: df = pd.DataFrame({"A": ['a', 'b', 'a'], "B": pd.core.arrays.IntegerArray([1, 2, 3])})

In [12]: df.groupby("A").B.min().dtype
```

Is it `int64`, or should we preserve `Int64`?

We can also discuss transforms ( cumulative operations like cumsum, maybe things like `.shift`).
ENH: add reduce op (and groupby reduce) support to EA
after #21160
Reductions for ExtensionArray
Creeping up to this in https://github.com/pandas-dev/pandas/pull/22345

A few questions

1. What should this look like for EA authors? What helpers can / should we provide?
2. How does this affect users? Specifically, assuming IntegerArray implements reductions, what's the dtype here?

```python
In [11]: df = pd.DataFrame({"A": ['a', 'b', 'a'], "B": pd.core.arrays.IntegerArray([1, 2, 3])})

In [12]: df.groupby("A").B.min().dtype
```

Is it `int64`, or should we preserve `Int64`?

We can also discuss transforms ( cumulative operations like cumsum, maybe things like `.shift`).


diff --git a/CHANGELOG.md b/CHANGELOG.md
index 978b7e7a..a6396c0b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,9 @@ to use the newest tag with new release
   to expel. It can be called second time to set up failover priority for
   replicasets where new instances were joined.
   As a result, `replicaset_healthy_timeout` is removed as unused.
+- Now list of instances for installing a package is selected once for all.
+  Before this patch, the complexity of calculating the list of instances was O(N^2),
+  now it is O(N). For 100 instances, it gives a 10x time reduction (60s -> 5s).

 ## [1.7.0] - 2020-11-24

diff --git a/filter_plugins/filter_hosts.py b/filter_plugins/filter_hosts.py
index aa103a74..099c41b6 100644
--- a/filter_plugins/filter_hosts.py
+++ b/filter_plugins/filter_hosts.py
@@ -1,5 +1,8 @@
-def get_machine_identifier(v):
-    return v['ansible_host'] if 'ansible_host' in v else v['inventory_hostname']
+def get_machine_hostname(v):
+    if 'ansible_host' in v:
+        return v['ansible_host']
+
+    raise Exception(f'Instance {v["inventory_hostname"]} has not "ansible_host" option!')

 def is_expelled(instance_vars):
@@ -18,19 +21,19 @@ def get_instance_fullname(app_name, inventory_hostname, stateboard):

 def get_one_not_expelled_instance_for_machine(hostvars, play_hosts):
-    res = []
-    added_machines = {}
+    machine_hostnames = set()
+    instance_names = []

-    for i in play_hosts:
-        if is_expelled(hostvars[i]):
+    for instance_name in play_hosts:
+        if is_expelled(hostvars[instance_name]):
             continue

-        machine_id = get_machine_identifier(hostvars[i])
-        if machine_id not in added_machines:
-            added_machines[machine_id] = True
-            res.append(i)
+        machine_hostname = get_machine_hostname(hostvars[instance_name])
+        if machine_hostname not in machine_hostnames:
+            machine_hostnames.add(machine_hostname)
+            instance_names.append(instance_name)

-    return res
+    return instance_names

 def get_one_not_expelled_instance(hostvars, play_hosts):
diff --git a/tasks/install_deb.yml b/tasks/install_deb.yml
index c33b5d59..144e87e8 100644
--- a/tasks/install_deb.yml
+++ b/tasks/install_deb.yml
@@ -52,7 +52,7 @@
     version: '{{ tnt_version[0] }}'
   when: '"tarantool" in deplist and cartridge_enable_tarantool_repo'

-- name: Install DEB
+- name: Install DEB package
   apt:
     deb: /tmp/{{ deb_filename }}
     update_cache: true
diff --git a/tasks/install_rpm.yml b/tasks/install_rpm.yml
index 9237094d..817e093c 100644
--- a/tasks/install_rpm.yml
+++ b/tasks/install_rpm.yml
@@ -57,7 +57,7 @@
     version: '{{ tnt_version[0] }}'
   when: 'cartridge_enable_tarantool_repo and "tarantool" in deplist.stdout'

-- name: Install RPM
+- name: Install RPM package
   yum:
     name: '/tmp/{{ rpm_filename }}'
     state: present
diff --git a/tasks/main.yml b/tasks/main.yml
index 0320c36d..f9fb11c6 100644
--- a/tasks/main.yml
+++ b/tasks/main.yml
@@ -50,12 +50,18 @@
     - cartridge-replicasets
     - cartridge-config

+- name: Get one instance from each physical machine
+  set_fact:
+    single_instances_for_each_machine: '{{ hostvars | get_one_not_expelled_instance_for_machine(play_hosts) }}'
+  run_once: true
+  tags: cartridge-instances
+
 # Rewrites cartridge_app_name from package info
 - name: Install package
   include_tasks: install_package.yml
   when: >-
     cartridge_package_path is defined and
-    inventory_hostname in hostvars | get_one_not_expelled_instance_for_machine(play_hosts)
+    inventory_hostname in single_instances_for_each_machine
   tags: cartridge-instances

 - name: Start instance

diff --git a/unit/test_tags.py b/unit/test_tags.py
index 01e1f68e..74281892 100644
--- a/unit/test_tags.py
+++ b/unit/test_tags.py
@@ -48,6 +48,7 @@ def test_without_tags(self):
             'Set remote_user for delegated tasks',
             'Validate config',
             'Set instance facts',
+            'Get one instance from each physical machine',
             'Install package',
             'Start instance',
             'Restart instances and reload systemd-daemon',
@@ -74,6 +75,7 @@ def test_tag_cartridge_instances(self):
             'Set remote_user for delegated tasks',
             'Validate config',
             'Set instance facts',
+            'Get one instance from each physical machine',
             'Install package',
             'Start instance',
             'Restart instances and reload systemd-daemon',

[2pt] Too long "Install package" task
I have 100 Tarantool instances in a cluster. When I deploy them via ansible-cartridge, it takes a very long time to complete the "Install package" task, since it is performed for each instance. Moreover, this task is performed even despite the fact that it is written in the logs that task is skipped for all instances.

Let's define the host list once instead of a hundred times.
[2pt] Too long "Install package" task
I have 100 Tarantool instances in a cluster. When I deploy them via ansible-cartridge, it takes a very long time to complete the "Install package" task, since it is performed for each instance. Moreover, this task is performed even despite the fact that it is written in the logs that task is skipped for all instances.

Let's define the host list once instead of a hundred times.


diff --git a/doc/whats_new/v1.2.rst b/doc/whats_new/v1.2.rst
index 832a4e4389b19..22aecf64a85b8 100644
--- a/doc/whats_new/v1.2.rst
+++ b/doc/whats_new/v1.2.rst
@@ -44,6 +44,11 @@ Changes impacting all modules
   NumPy's SIMD optimized primitives.
   :pr:`23446` by :user:`Meekail Zain <micky774>`

+- |Enhancement| Finiteness checks (detection of NaN and infinite values) in all
+  estimators are now faster by utilizing a more efficient stop-on-first
+  second-pass algorithm.
+  :pr:`23197` by :user:`Meekail Zain <micky774>`
+
 Changelog
 ---------

diff --git a/sklearn/utils/_isfinite.pyx b/sklearn/utils/_isfinite.pyx
new file mode 100644
index 0000000000000..bc6af6b8c24a1
--- /dev/null
+++ b/sklearn/utils/_isfinite.pyx
@@ -0,0 +1,40 @@
+# Author: jakirkham, Meekail Zain, Thomas Fan
+
+from libc.math cimport isnan, isinf
+cimport cython
+from cython cimport floating
+
+cpdef enum FiniteStatus:
+    all_finite = 0
+    has_nan = 1
+    has_infinite = 2
+
+def cy_isfinite(floating[::1] a, bint allow_nan=False):
+    cdef int result
+    with nogil:
+        result = _isfinite(a, allow_nan)
+    return result
+
+cdef inline int _isfinite(floating[::1] a, bint allow_nan) nogil:
+    cdef floating* a_ptr = &a[0]
+    cdef Py_ssize_t length = len(a)
+    if allow_nan:
+        return _isfinite_allow_nan(a_ptr, length)
+    else:
+        return _isfinite_disable_nan(a_ptr, length)
+
+cdef inline int _isfinite_allow_nan(floating* a_ptr, Py_ssize_t length) nogil:
+    cdef Py_ssize_t i
+    for i in range(length):
+        if isinf(a_ptr[i]) != 0:
+            return FiniteStatus.has_infinite
+    return FiniteStatus.all_finite
+
+cdef inline int _isfinite_disable_nan(floating* a_ptr, Py_ssize_t length) nogil:
+    cdef Py_ssize_t i
+    for i in range(length):
+        if isnan(a_ptr[i]) != 0:
+            return FiniteStatus.has_nan
+        if isinf(a_ptr[i]) != 0:
+            return FiniteStatus.has_infinite
+    return FiniteStatus.all_finite
diff --git a/sklearn/utils/setup.py b/sklearn/utils/setup.py
index eaf3a447389ea..915a8efeb2e01 100644
--- a/sklearn/utils/setup.py
+++ b/sklearn/utils/setup.py
@@ -116,6 +116,11 @@ def configuration(parent_package="", top_path=None):
         libraries=libraries,
         language="c++",
     )
+    config.add_extension(
+        "_isfinite",
+        sources=["_isfinite.pyx"],
+        libraries=libraries,
+    )

     config.add_subpackage("tests")

diff --git a/sklearn/utils/validation.py b/sklearn/utils/validation.py
index bb2c55d4beed5..6bfe99fdaffdc 100644
--- a/sklearn/utils/validation.py
+++ b/sklearn/utils/validation.py
@@ -29,6 +29,7 @@
 from ..exceptions import PositiveSpectrumWarning
 from ..exceptions import NotFittedError
 from ..exceptions import DataConversionWarning
+from ._isfinite import cy_isfinite, FiniteStatus

 FLOAT_DTYPES = (np.float64, np.float32, np.float16)

@@ -99,25 +100,33 @@ def _assert_all_finite(
         return
     X = np.asanyarray(X)
     # First try an O(n) time, O(1) space solution for the common case that
-    # everything is finite; fall back to O(n) space np.isfinite to prevent
-    # false positives from overflow in sum method.
+    # everything is finite; fall back to O(n) space `np.isinf/isnan` or custom
+    # Cython implementation to prevent false positives and provide a detailed
+    # error message.
     is_float = X.dtype.kind in "fc"
-    with np.errstate(over="ignore"):
-        first_pass_isfinite = is_float and np.isfinite(np.sum(X))
-    if first_pass_isfinite:
-        pass
-    elif is_float:
-        has_inf = np.isinf(X).any()
-        has_nan = np.isnan(X).any()
-        if has_inf or not allow_nan and has_nan:
-            if not allow_nan and has_nan:
+    if is_float:
+        with np.errstate(over="ignore"):
+            first_pass_isfinite = np.isfinite(np.sum(X))
+        if first_pass_isfinite:
+            return
+        # Cython implementation doesn't support FP16 or complex numbers
+        use_cython = X.data.contiguous and X.dtype.type in {np.float32, np.float64}
+        if use_cython:
+            out = cy_isfinite(X.reshape(-1), allow_nan=allow_nan)
+            has_nan_error = False if allow_nan else out == FiniteStatus.has_nan
+            has_inf = out == FiniteStatus.has_infinite
+        else:
+            has_inf = np.isinf(X).any()
+            has_nan_error = False if allow_nan else np.isnan(X).any()
+        if has_inf or has_nan_error:
+            if has_nan_error:
                 type_err = "NaN"
             else:
                 msg_dtype = msg_dtype if msg_dtype is not None else X.dtype
                 type_err = f"infinity or a value too large for {msg_dtype!r}"
             padded_input_name = input_name + " " if input_name else ""
             msg_err = f"Input {padded_input_name}contains {type_err}."
-            if not allow_nan and estimator_name and input_name == "X" and has_nan:
+            if estimator_name and input_name == "X" and has_nan_error:
                 # Improve the error message on how to handle missing values in
                 # scikit-learn.
                 msg_err += (

Optimizing assert_all_finite check
Have done some work to come up with a Cython implementation of [`_assert_all_finite`]( https://github.com/scikit-learn/scikit-learn/blob/6d15840432c08b2c628babe62d7bef6d6a01fcf6/sklearn/utils/validation.py#L40 ) as the current check is rather slow. Have come up with [this implementation]( https://gist.github.com/jakirkham/8bafcec842e92641885c91f79ef7f44f ). This simply uses the C `isfinite` (or `isinf` for `allow_nan=True`) function in a tight loop; returning `False` whenever `isfinite` returns `False` and returning `True` otherwise. As such it has a very small, constant memory overhead.

For smallish data (0-8000 element `float64` arrays), this is comparable to the naive NumPy implementation in time and faster than the current scikit-learn implementation. Should add that the Cython implementation uses considerably less memory than either implementation. For large sized arrays, scikit-learn's performance improves to ~2x faster than the NumPy or Cython implementation. Expect this is mostly due to how the caches are utilized. So this can be improved further.

Would be interested in getting some feedback about whether such a contribution would be interesting to scikit-learn. If so, can investigate further improvements to this implementation.

Because we are have check by default, and it can be relatively slow (https://github.com/scikit-learn/scikit-learn/pull/11487#issuecomment-405034867)  in some cases, it would be definitely nice to make it faster.

Ideally, it would be great to have such an improvement in numpy (with possible temporary backport in scikit-learn), there is related discussion in https://github.com/numpy/numpy/issues/6909 Maybe they could be interested by such a contribution?

Are you saying that the proposed implementation is faster than `np.isfinite(X.sum())` ? That would take too much memory, would it? If I understand correctly, things become slower when this evaluates to False (e.g. there are non finite elements) and we also have to run `np.isinf(X).any()` ?

Generally it seems like a good idea to contribute this stuff back to NumPy. Not sure they would take this implementation as it is customized for the case of searching the full array for the first invalid value and returning. Also while they do have a little bit of Cython code, they might prefer C for something like this in a prominent code path.

For small arrays (~1000 elements or less) it appears to be ~3.5x faster than the current scikit-learn implementation. As the array size gets larger, imagine other things like cache pressure, which this implementation does nothing with currently, become an issue and then NumPy-based implementations take over. Maybe using [NumPy iterators]( https://docs.scipy.org/doc/numpy-1.15.0/reference/c-api.iterator.html ) with buffering would improve the large array case. Generally open to suggestions here.

This behaves more like `l.index(...)` in the sense that it scans the array for a match to an unacceptable value (`inf` and possibly `nan`). If it finds a match, it returns `False` ending the scan. If it doesn't find a match, it will scan the full array and then return `True`. So the memory usage is constant and actually quite small (e.g. the index, the current value, length, step size, some pointers, some flags, etc.).

![is_finite_avg_runtime](https://user-images.githubusercontent.com/3019665/43241809-4b055272-906b-11e8-9a31-ffdd5712f0f4.png)
@jakirkham : I would be +1 for inclusion of this in scikit-learn (and of course, encourage contributing upstream)
A pull request here?
Hey there @jakirkham, just wanted to check if you intended to open a PR for your proposed solution.


diff --git a/doc/release/1.16.0-notes.rst b/doc/release/1.16.0-notes.rst
index 2b84bb90a311..6a73ed3548da 100644
--- a/doc/release/1.16.0-notes.rst
+++ b/doc/release/1.16.0-notes.rst
@@ -107,6 +107,12 @@ Previously, a ``LinAlgError`` would be raised when an empty matrix/empty
 matrices (with zero rows and/or columns) is/are passed in. Now outputs of
 appropriate shapes are returned.

+``np.diff`` Added kwargs prepend and append
+-------------------------------------------
+Add kwargs prepend and append, allowing for values to be inserted
+on either end of the differences.  Similar to options for ediff1d.
+Allows for the inverse of cumsum easily via prepend=0
+
 ARM support updated
 -------------------
 Support for ARM CPUs has been updated to accommodate 32 and 64 bit targets,
diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py
index 2992e92bb056..704e56514973 100644
--- a/numpy/lib/function_base.py
+++ b/numpy/lib/function_base.py
@@ -1090,7 +1090,7 @@ def gradient(f, *varargs, **kwargs):
         return outvals

-def diff(a, n=1, axis=-1):
+def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue):
     """
     Calculate the n-th discrete difference along the given axis.

@@ -1108,6 +1108,12 @@ def diff(a, n=1, axis=-1):
     axis : int, optional
         The axis along which the difference is taken, default is the
         last axis.
+    prepend, append : array_like, optional
+        Values to prepend or append to "a" along axis prior to
+        performing the difference.  Scalar values are expanded to
+        arrays with length 1 in the direction of axis and the shape
+        of the input array in along all other axes.  Otherwise the
+        dimension and shape must match "a" except along axis.

     Returns
     -------
@@ -1176,6 +1182,28 @@ def diff(a, n=1, axis=-1):
     nd = a.ndim
     axis = normalize_axis_index(axis, nd)

+    combined = []
+    if prepend is not np._NoValue:
+        prepend = np.asanyarray(prepend)
+        if prepend.ndim == 0:
+            shape = list(a.shape)
+            shape[axis] = 1
+            prepend = np.broadcast_to(prepend, tuple(shape))
+        combined.append(prepend)
+
+    combined.append(a)
+
+    if append is not np._NoValue:
+        append = np.asanyarray(append)
+        if append.ndim == 0:
+            shape = list(a.shape)
+            shape[axis] = 1
+            append = np.broadcast_to(append, tuple(shape))
+        combined.append(append)
+
+    if len(combined) > 1:
+        a = np.concatenate(combined, axis)
+
     slice1 = [slice(None)] * nd
     slice2 = [slice(None)] * nd
     slice1[axis] = slice(1, None)

diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
index d5faed6aea49..40cca1dbb4bb 100644
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -734,6 +734,58 @@ def test_subclass(self):
         assert_array_equal(out3.mask, [[], [], [], [], []])
         assert_(type(out3) is type(x))

+    def test_prepend(self):
+        x = np.arange(5) + 1
+        assert_array_equal(diff(x, prepend=0), np.ones(5))
+        assert_array_equal(diff(x, prepend=[0]), np.ones(5))
+        assert_array_equal(np.cumsum(np.diff(x, prepend=0)), x)
+        assert_array_equal(diff(x, prepend=[-1, 0]), np.ones(6))
+
+        x = np.arange(4).reshape(2, 2)
+        result = np.diff(x, axis=1, prepend=0)
+        expected = [[0, 1], [2, 1]]
+        assert_array_equal(result, expected)
+        result = np.diff(x, axis=1, prepend=[[0], [0]])
+        assert_array_equal(result, expected)
+
+        result = np.diff(x, axis=0, prepend=0)
+        expected = [[0, 1], [2, 2]]
+        assert_array_equal(result, expected)
+        result = np.diff(x, axis=0, prepend=[[0, 0]])
+        assert_array_equal(result, expected)
+
+        assert_raises(ValueError, np.diff, x, prepend=np.zeros((3,3)))
+
+        assert_raises(np.AxisError, diff, x, prepend=0, axis=3)
+
+    def test_append(self):
+        x = np.arange(5)
+        result = diff(x, append=0)
+        expected = [1, 1, 1, 1, -4]
+        assert_array_equal(result, expected)
+        result = diff(x, append=[0])
+        assert_array_equal(result, expected)
+        result = diff(x, append=[0, 2])
+        expected = expected + [2]
+        assert_array_equal(result, expected)
+
+        x = np.arange(4).reshape(2, 2)
+        result = np.diff(x, axis=1, append=0)
+        expected = [[1, -1], [1, -3]]
+        assert_array_equal(result, expected)
+        result = np.diff(x, axis=1, append=[[0], [0]])
+        assert_array_equal(result, expected)
+
+        result = np.diff(x, axis=0, append=0)
+        expected = [[2, 2], [-2, -3]]
+        assert_array_equal(result, expected)
+        result = np.diff(x, axis=0, append=[[0, 0]])
+        assert_array_equal(result, expected)
+
+        assert_raises(ValueError, np.diff, x, append=np.zeros((3,3)))
+
+        assert_raises(np.AxisError, diff, x, append=0, axis=3)
+

 class TestDelete(object):

Add "insert_zero" option for np.diff
I often find myself wanting the difference of an array along some axis, where the 0th element should just be the first entry of the array along that axis, so that the shape of the output matches that of the input.   This is a pretty common requirement for people working with finite differences in any way (eg all signal processing)

Currently, I do this with the awkward use of `np.insert`, as demonstrated:

```
a = np.random.RandomState(1234).randint(12, size=(3, 4))
array([[ 3,  6,  5,  4],
       [ 8,  9,  1,  7],
       [ 9, 10, 11, 10]])
# Want a diff along columns of a:
np.diff(np.insert(a, 0, 0, axis=0), axis=0)
> array([[ 3,  6,  5,  4],
       [ 5,  3, -4,  3],
       [ 1,  1, 10,  3]])
# Which is ugly and reallocates a whole intermediate array
# It would be more convenient to go:
np.diff(a, axis=0, insert_zero=True)
```

This would allow us to have an easy inverse of cumsum, so that

`np.all( np.cumsum(np.diff(arr, axis=1, insert_zero=True), axis=1) == arr)`

The inverse of cumsum problem would also be solved by adding `include_identity` to cumsum (#6044), which is something I've felt a stronger need for.

I prefer the ediff1d arguments to_begin and to_end, allowing for non zero and/or multiple elements to be inserted on either side.

Maybe the best solution is to add the padding options of ediff1d to diff, and then kill ediff1d.  The discussion [here](http://stackoverflow.com/questions/39014324/difference-between-numpy-ediff1d-and-diff) does kind of make it seem like ediff1d should go, as it's kind of violating the "one way to do things" rule.

I agree, that's basically the conclusion of part of the discussion of this [issue](https://github.com/numpy/numpy/issues/8175) also

A general initial condition would be better than a specific one.

Agreed.

01 - Enhancement
component: numpy.lib
56 - Needs Release Note.


diff --git a/doc/whats_new/v0.24.rst b/doc/whats_new/v0.24.rst
index a647de6dac628..ba137fb76698f 100644
--- a/doc/whats_new/v0.24.rst
+++ b/doc/whats_new/v0.24.rst
@@ -224,6 +224,11 @@ Changelog
   :meth:`tree.DecisionTreeRegressor.fit`, and has not effect.
   :pr:`17614` by :user:`Juan Carlos Alfaro Jiménez <alfaro96>`.

+:mod:`sklearn.datasets`
+.......................
+- |Feature| :func:`datasets.fetch_openml` now validates md5checksum of arff
+  files downloaded or cached to ensure data integrity.
+  :pr:`14800` by :user:`Shashank Singh <shashanksingh28>` and `Joel Nothman`_.

 Code and Documentation Contributors
 -----------------------------------
diff --git a/sklearn/datasets/_openml.py b/sklearn/datasets/_openml.py
index 6b537c09c59ad..d2ce5d471f8f7 100644
--- a/sklearn/datasets/_openml.py
+++ b/sklearn/datasets/_openml.py
@@ -2,6 +2,7 @@
 import json
 import os
 import shutil
+import hashlib
 from os.path import join
 from warnings import warn
 from contextlib import closing
@@ -492,7 +493,8 @@ def _load_arff_response(
     url: str,
     data_home: Optional[str],
     return_type, encode_nominal: bool,
-    parse_arff: Callable[[ArffContainerType], Tuple]
+    parse_arff: Callable[[ArffContainerType], Tuple],
+    md5_checksum: str
 ) -> Tuple:
     """Load arff data with url and parses arff response with parse_arff"""
     response = _open_openml_url(url, data_home)
@@ -500,10 +502,32 @@ def _load_arff_response(
     with closing(response):
         # Note that if the data is dense, no reading is done until the data
         # generator is iterated.
-        arff = _arff.load((line.decode('utf-8') for line in response),
+        actual_md5_checksum = hashlib.md5()
+
+        def _stream_checksum_generator(response):
+            for line in response:
+                actual_md5_checksum.update(line)
+                yield line.decode('utf-8')
+
+        stream = _stream_checksum_generator(response)
+
+        arff = _arff.load(stream,
                           return_type=return_type,
                           encode_nominal=encode_nominal)
-        return parse_arff(arff)
+
+        parsed_arff = parse_arff(arff)
+
+        # consume remaining stream, if early exited
+        for _ in stream:
+            pass
+
+        if actual_md5_checksum.hexdigest() != md5_checksum:
+            raise ValueError("md5 checksum of local file for " + url +
+                             " does not match description. "
+                             "Downloaded file could have been modified / "
+                             "corrupted, clean cache and retry...")
+
+        return parsed_arff

 def _download_data_to_bunch(
@@ -515,7 +539,8 @@ def _download_data_to_bunch(
     features_list: List,
     data_columns: List[int],
     target_columns: List,
-    shape: Optional[Tuple[int, int]]
+    shape: Optional[Tuple[int, int]],
+    md5_checksum: str
 ):
     """Download OpenML ARFF and convert to Bunch of data
     """
@@ -609,7 +634,8 @@ def postprocess(X, y, nominal_attributes):
         _load_arff_response)(url, data_home,
                              return_type=return_type,
                              encode_nominal=not as_frame,
-                             parse_arff=parse_arff)
+                             parse_arff=parse_arff,
+                             md5_checksum=md5_checksum)
     X, y, frame, nominal_attributes = postprocess(*out)

     return Bunch(data=X, target=y, frame=frame,
@@ -883,7 +909,9 @@ def fetch_openml(
                                     as_frame=as_frame,
                                     features_list=features_list, shape=shape,
                                     target_columns=target_columns,
-                                    data_columns=data_columns)
+                                    data_columns=data_columns,
+                                    md5_checksum=data_description[
+                                        "md5_checksum"])

     if return_X_y:
         return bunch.data, bunch.target

diff --git a/sklearn/datasets/tests/data/openml/1/api-v1-json-data-1.json.gz b/sklearn/datasets/tests/data/openml/1/api-v1-json-data-1.json.gz
index f75912bf2def7..ba544db491637 100644
Binary files a/sklearn/datasets/tests/data/openml/1/api-v1-json-data-1.json.gz and b/sklearn/datasets/tests/data/openml/1/api-v1-json-data-1.json.gz differ
diff --git a/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-1119.json.gz b/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-1119.json.gz
index 97ab0d1ce64f6..286e57672339a 100644
Binary files a/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-1119.json.gz and b/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-1119.json.gz differ
diff --git a/sklearn/datasets/tests/data/openml/2/api-v1-json-data-2.json.gz b/sklearn/datasets/tests/data/openml/2/api-v1-json-data-2.json.gz
index 22dfb6ff61c1b..921326973b212 100644
Binary files a/sklearn/datasets/tests/data/openml/2/api-v1-json-data-2.json.gz and b/sklearn/datasets/tests/data/openml/2/api-v1-json-data-2.json.gz differ
diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-292.json.gz b/sklearn/datasets/tests/data/openml/292/api-v1-json-data-292.json.gz
index 888140f92b360..17caf8a8dcc7b 100644
Binary files a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-292.json.gz and b/sklearn/datasets/tests/data/openml/292/api-v1-json-data-292.json.gz differ
diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-40981.json.gz b/sklearn/datasets/tests/data/openml/292/api-v1-json-data-40981.json.gz
index 888140f92b360..9fd04cf4a73cb 100644
Binary files a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-40981.json.gz and b/sklearn/datasets/tests/data/openml/292/api-v1-json-data-40981.json.gz differ
diff --git a/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-40589.json.gz b/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-40589.json.gz
index 9c71553ce5137..97089490a18c9 100644
Binary files a/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-40589.json.gz and b/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-40589.json.gz differ
diff --git a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-40675.json.gz b/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-40675.json.gz
index 42b876f0a4723..6f21ade9d3eba 100644
Binary files a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-40675.json.gz and b/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-40675.json.gz differ
diff --git a/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-40966.json.gz b/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-40966.json.gz
index 02b25d717f925..f1deb65592a9f 100644
Binary files a/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-40966.json.gz and b/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-40966.json.gz differ
diff --git a/sklearn/datasets/tests/test_openml.py b/sklearn/datasets/tests/test_openml.py
index b16a34727c2aa..4e1220807947f 100644
--- a/sklearn/datasets/tests/test_openml.py
+++ b/sklearn/datasets/tests/test_openml.py
@@ -149,6 +149,32 @@ def _fetch_dataset_from_openml(data_id, data_name, data_version,
     return data_by_id

+class _MockHTTPResponse:
+    def __init__(self, data, is_gzip):
+        self.data = data
+        self.is_gzip = is_gzip
+
+    def read(self, amt=-1):
+        return self.data.read(amt)
+
+    def close(self):
+        self.data.close()
+
+    def info(self):
+        if self.is_gzip:
+            return {'Content-Encoding': 'gzip'}
+        return {}
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+
 def _monkey_patch_webbased_functions(context,
                                      data_id,
                                      gzip_response):
@@ -163,37 +189,6 @@ def _monkey_patch_webbased_functions(context,
     path_suffix = '.gz'
     read_fn = gzip.open

-    class MockHTTPResponse:
-        def __init__(self, data, is_gzip):
-            self.data = data
-            self.is_gzip = is_gzip
-
-        def read(self, amt=-1):
-            return self.data.read(amt)
-
-        def tell(self):
-            return self.data.tell()
-
-        def seek(self, pos, whence=0):
-            return self.data.seek(pos, whence)
-
-        def close(self):
-            self.data.close()
-
-        def info(self):
-            if self.is_gzip:
-                return {'Content-Encoding': 'gzip'}
-            return {}
-
-        def __iter__(self):
-            return iter(self.data)
-
-        def __enter__(self):
-            return self
-
-        def __exit__(self, exc_type, exc_val, exc_tb):
-            return False
-
     def _file_name(url, suffix):
         return (re.sub(r'\W', '-', url[len("https://openml.org/"):])
                 + suffix + path_suffix)
@@ -206,10 +201,10 @@ def _mock_urlopen_data_description(url, has_gzip_header):

         if has_gzip_header and gzip_response:
             fp = open(path, 'rb')
-            return MockHTTPResponse(fp, True)
+            return _MockHTTPResponse(fp, True)
         else:
             fp = read_fn(path, 'rb')
-            return MockHTTPResponse(fp, False)
+            return _MockHTTPResponse(fp, False)

     def _mock_urlopen_data_features(url, has_gzip_header):
         assert url.startswith(url_prefix_data_features)
@@ -217,10 +212,10 @@ def _mock_urlopen_data_features(url, has_gzip_header):
                             _file_name(url, '.json'))
         if has_gzip_header and gzip_response:
             fp = open(path, 'rb')
-            return MockHTTPResponse(fp, True)
+            return _MockHTTPResponse(fp, True)
         else:
             fp = read_fn(path, 'rb')
-            return MockHTTPResponse(fp, False)
+            return _MockHTTPResponse(fp, False)

     def _mock_urlopen_download_data(url, has_gzip_header):
         assert (url.startswith(url_prefix_download_data))
@@ -230,10 +225,10 @@ def _mock_urlopen_download_data(url, has_gzip_header):

         if has_gzip_header and gzip_response:
             fp = open(path, 'rb')
-            return MockHTTPResponse(fp, True)
+            return _MockHTTPResponse(fp, True)
         else:
             fp = read_fn(path, 'rb')
-            return MockHTTPResponse(fp, False)
+            return _MockHTTPResponse(fp, False)

     def _mock_urlopen_data_list(url, has_gzip_header):
         assert url.startswith(url_prefix_data_list)
@@ -250,10 +245,10 @@ def _mock_urlopen_data_list(url, has_gzip_header):

         if has_gzip_header:
             fp = open(json_file_path, 'rb')
-            return MockHTTPResponse(fp, True)
+            return _MockHTTPResponse(fp, True)
         else:
             fp = read_fn(json_file_path, 'rb')
-            return MockHTTPResponse(fp, False)
+            return _MockHTTPResponse(fp, False)

     def _mock_urlopen(request):
         url = request.get_full_url()
@@ -1209,6 +1204,47 @@ def test_fetch_openml_with_ignored_feature(monkeypatch, gzip_response):
     assert 'animal' not in dataset['feature_names']

+@pytest.mark.parametrize('as_frame', [True, False])
+def test_fetch_openml_verify_checksum(monkeypatch, as_frame, cache, tmpdir):
+    if as_frame:
+        pytest.importorskip('pandas')
+
+    data_id = 2
+    _monkey_patch_webbased_functions(monkeypatch, data_id, True)
+
+    # create a temporary modified arff file
+    dataset_dir = os.path.join(currdir, 'data', 'openml', str(data_id))
+    original_data_path = os.path.join(dataset_dir,
+                                      'data-v1-download-1666876.arff.gz')
+    corrupt_copy = os.path.join(tmpdir, "test_invalid_checksum.arff")
+    with gzip.GzipFile(original_data_path, "rb") as orig_gzip, \
+            gzip.GzipFile(corrupt_copy, "wb") as modified_gzip:
+        data = bytearray(orig_gzip.read())
+        data[len(data)-1] = 37
+        modified_gzip.write(data)
+
+    # Requests are already mocked by monkey_patch_webbased_functions.
+    # We want to re-use that mock for all requests except file download,
+    # hence creating a thin mock over the original mock
+    mocked_openml_url = sklearn.datasets._openml.urlopen
+
+    def swap_file_mock(request):
+        url = request.get_full_url()
+        if url.endswith('data/v1/download/1666876'):
+            return _MockHTTPResponse(open(corrupt_copy, "rb"), is_gzip=True)
+        else:
+            return mocked_openml_url(request)
+
+    monkeypatch.setattr(sklearn.datasets._openml, 'urlopen', swap_file_mock)
+
+    # validate failed checksum
+    with pytest.raises(ValueError) as exc:
+        sklearn.datasets.fetch_openml(data_id=data_id, cache=False,
+                                      as_frame=as_frame)
+    # exception message should have file-path
+    assert exc.match("1666876")
+
+
 def test_convert_arff_data_type():
     pytest.importorskip('pandas')

Validate MD5 checksum of downloaded ARFF in fetch_openml
`fetch_openml` downloads the `data_set_description` from openml.org which includes an `md5_checksum` field. This should correspond to the MD5 checksum of the downloaded ARFF data. We should (optionally?) use `hashlib` to verify that the MD5 matches.

I'll give this a go.
thanks. you may need to hack the md5s currently in
sklearn/datasets/tests/data/openml to match the truncated ARFF data there

Picking this to work on while in NYC WiMLDS Sprint Aug 24, 2019
I think this check should be optional. What do other core developers think? Forcing the verification ensures security against man in middle mocking but but adds some processing overhead. Should users pay this overhead by default?

Also, forcing the check as default might make genuine mocking for tests difficult too
I think having it true by default seems good but I'm fine with being able to disable it with an option.


diff --git a/doc/source/reshaping.rst b/doc/source/reshaping.rst
index 1209c4a8d6be8..1b81d83bb76c7 100644
--- a/doc/source/reshaping.rst
+++ b/doc/source/reshaping.rst
@@ -240,7 +240,7 @@ values will be set to ``NaN``.
    df3
    df3.unstack()

-.. versionadded: 0.18.0
+.. versionadded:: 0.18.0

 Alternatively, unstack takes an optional ``fill_value`` argument, for specifying
 the value of missing data.
@@ -634,6 +634,17 @@ When a column contains only one level, it will be omitted in the result.

     pd.get_dummies(df, drop_first=True)

+By default new columns will have ``np.uint8`` dtype. To choose another dtype use ``dtype`` argument:
+
+.. ipython:: python
+
+    df = pd.DataFrame({'A': list('abc'), 'B': [1.1, 2.2, 3.3]})
+
+    pd.get_dummies(df, dtype=bool).dtypes
+
+.. versionadded:: 0.22.0
+
+
 .. _reshaping.factorize:

 Factorizing values
diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt
index 4f403ff8053a7..162a1ca2bfcb3 100644
--- a/doc/source/whatsnew/v0.22.0.txt
+++ b/doc/source/whatsnew/v0.22.0.txt
@@ -17,6 +17,21 @@ New features
 -
 -

+
+.. _whatsnew_0210.enhancements.get_dummies_dtype:
+
+``get_dummies`` now supports ``dtype`` argument
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :func:`get_dummies` now accepts a ``dtype`` argument, which specifies a dtype for the new columns. The default remains uint8. (:issue:`18330`)
+
+.. ipython:: python
+
+   df = pd.DataFrame({'a': [1, 2], 'b': [3, 4], 'c': [5, 6]})
+   pd.get_dummies(df, columns=['c']).dtypes
+   pd.get_dummies(df, columns=['c'], dtype=bool).dtypes
+
+
 .. _whatsnew_0220.enhancements.other:

 Other Enhancements
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
index cbc259ba7bceb..782971a742b54 100644
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -965,7 +965,7 @@ def _set_axis_name(self, name, axis=0, inplace=False):
         inplace : bool
             whether to modify `self` directly or return a copy

-            .. versionadded: 0.21.0
+            .. versionadded:: 0.21.0

         Returns
         -------
diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py
index 8b656d8ba25e9..5bb86885c0875 100644
--- a/pandas/core/reshape/reshape.py
+++ b/pandas/core/reshape/reshape.py
@@ -10,7 +10,7 @@
 from pandas.core.dtypes.common import (
     _ensure_platform_int,
     is_list_like, is_bool_dtype,
-    needs_i8_conversion, is_sparse)
+    needs_i8_conversion, is_sparse, is_object_dtype)
 from pandas.core.dtypes.cast import maybe_promote
 from pandas.core.dtypes.missing import notna

@@ -697,7 +697,7 @@ def _convert_level_number(level_num, columns):

 def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False,
-                columns=None, sparse=False, drop_first=False):
+                columns=None, sparse=False, drop_first=False, dtype=None):
     """
     Convert categorical variable into dummy/indicator variables

@@ -728,6 +728,11 @@ def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False,

         .. versionadded:: 0.18.0

+    dtype : dtype, default np.uint8
+        Data type for new columns. Only a single dtype is allowed.
+
+        .. versionadded:: 0.22.0
+
     Returns
     -------
     dummies : DataFrame or SparseDataFrame
@@ -783,6 +788,12 @@ def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False,
     3  0  0
     4  0  0

+    >>> pd.get_dummies(pd.Series(list('abc')), dtype=float)
+         a    b    c
+    0  1.0  0.0  0.0
+    1  0.0  1.0  0.0
+    2  0.0  0.0  1.0
+
     See Also
     --------
     Series.str.get_dummies
@@ -835,20 +846,29 @@ def check_len(item, name):

             dummy = _get_dummies_1d(data[col], prefix=pre, prefix_sep=sep,
                                     dummy_na=dummy_na, sparse=sparse,
-                                    drop_first=drop_first)
+                                    drop_first=drop_first, dtype=dtype)
             with_dummies.append(dummy)
         result = concat(with_dummies, axis=1)
     else:
         result = _get_dummies_1d(data, prefix, prefix_sep, dummy_na,
-                                 sparse=sparse, drop_first=drop_first)
+                                 sparse=sparse,
+                                 drop_first=drop_first,
+                                 dtype=dtype)
     return result

 def _get_dummies_1d(data, prefix, prefix_sep='_', dummy_na=False,
-                    sparse=False, drop_first=False):
+                    sparse=False, drop_first=False, dtype=None):
     # Series avoids inconsistent NaN handling
     codes, levels = _factorize_from_iterable(Series(data))

+    if dtype is None:
+        dtype = np.uint8
+    dtype = np.dtype(dtype)
+
+    if is_object_dtype(dtype):
+        raise ValueError("dtype=object is not a valid dtype for get_dummies")
+
     def get_empty_Frame(data, sparse):
         if isinstance(data, Series):
             index = data.index
@@ -903,18 +923,18 @@ def get_empty_Frame(data, sparse):
             sp_indices = sp_indices[1:]
             dummy_cols = dummy_cols[1:]
         for col, ixs in zip(dummy_cols, sp_indices):
-            sarr = SparseArray(np.ones(len(ixs), dtype=np.uint8),
+            sarr = SparseArray(np.ones(len(ixs), dtype=dtype),
                                sparse_index=IntIndex(N, ixs), fill_value=0,
-                               dtype=np.uint8)
+                               dtype=dtype)
             sparse_series[col] = SparseSeries(data=sarr, index=index)

         out = SparseDataFrame(sparse_series, index=index, columns=dummy_cols,
                               default_fill_value=0,
-                              dtype=np.uint8)
+                              dtype=dtype)
         return out

     else:
-        dummy_mat = np.eye(number_of_cols, dtype=np.uint8).take(codes, axis=0)
+        dummy_mat = np.eye(number_of_cols, dtype=dtype).take(codes, axis=0)

         if not dummy_na:
             # reset NaN GH4446

diff --git a/pandas/tests/reshape/test_reshape.py b/pandas/tests/reshape/test_reshape.py
index 2722c3e92d85a..5d4aa048ae303 100644
--- a/pandas/tests/reshape/test_reshape.py
+++ b/pandas/tests/reshape/test_reshape.py
@@ -218,35 +218,51 @@ def test_multiindex(self):

 class TestGetDummies(object):

-    sparse = False
-
-    def setup_method(self, method):
-        self.df = DataFrame({'A': ['a', 'b', 'a'],
-                             'B': ['b', 'b', 'c'],
-                             'C': [1, 2, 3]})
+    @pytest.fixture
+    def df(self):
+        return DataFrame({'A': ['a', 'b', 'a'],
+                          'B': ['b', 'b', 'c'],
+                          'C': [1, 2, 3]})
+
+    @pytest.fixture(params=['uint8', 'i8', np.float64, bool, None])
+    def dtype(self, request):
+        return np.dtype(request.param)
+
+    @pytest.fixture(params=['dense', 'sparse'])
+    def sparse(self, request):
+        # params are strings to simplify reading test results,
+        # e.g. TestGetDummies::test_basic[uint8-sparse] instead of [uint8-True]
+        return request.param == 'sparse'
+
+    def effective_dtype(self, dtype):
+        if dtype is None:
+            return np.uint8
+        return dtype
+
+    def test_raises_on_dtype_object(self, df):
+        with pytest.raises(ValueError):
+            get_dummies(df, dtype='object')

-    def test_basic(self):
+    def test_basic(self, sparse, dtype):
         s_list = list('abc')
         s_series = Series(s_list)
         s_series_index = Series(s_list, list('ABC'))

-        expected = DataFrame({'a': {0: 1,
-                                    1: 0,
-                                    2: 0},
-                              'b': {0: 0,
-                                    1: 1,
-                                    2: 0},
-                              'c': {0: 0,
-                                    1: 0,
-                                    2: 1}}, dtype=np.uint8)
-        assert_frame_equal(get_dummies(s_list, sparse=self.sparse), expected)
-        assert_frame_equal(get_dummies(s_series, sparse=self.sparse), expected)
+        expected = DataFrame({'a': [1, 0, 0],
+                              'b': [0, 1, 0],
+                              'c': [0, 0, 1]},
+                             dtype=self.effective_dtype(dtype))
+        result = get_dummies(s_list, sparse=sparse, dtype=dtype)
+        assert_frame_equal(result, expected)
+
+        result = get_dummies(s_series, sparse=sparse, dtype=dtype)
+        assert_frame_equal(result, expected)

         expected.index = list('ABC')
-        assert_frame_equal(
-            get_dummies(s_series_index, sparse=self.sparse), expected)
+        result = get_dummies(s_series_index, sparse=sparse, dtype=dtype)
+        assert_frame_equal(result, expected)

-    def test_basic_types(self):
+    def test_basic_types(self, sparse, dtype):
         # GH 10531
         s_list = list('abc')
         s_series = Series(s_list)
@@ -257,38 +273,43 @@ def test_basic_types(self):
         expected = DataFrame({'a': [1, 0, 0],
                               'b': [0, 1, 0],
                               'c': [0, 0, 1]},
-                             dtype='uint8',
+                             dtype=self.effective_dtype(dtype),
                              columns=list('abc'))
-        if not self.sparse:
+        if not sparse:
             compare = tm.assert_frame_equal
         else:
             expected = expected.to_sparse(fill_value=0, kind='integer')
             compare = tm.assert_sp_frame_equal

-        result = get_dummies(s_list, sparse=self.sparse)
+        result = get_dummies(s_list, sparse=sparse, dtype=dtype)
         compare(result, expected)

-        result = get_dummies(s_series, sparse=self.sparse)
+        result = get_dummies(s_series, sparse=sparse, dtype=dtype)
         compare(result, expected)

-        result = get_dummies(s_df, sparse=self.sparse, columns=s_df.columns)
+        result = get_dummies(s_df, columns=s_df.columns,
+                             sparse=sparse, dtype=dtype)
         tm.assert_series_equal(result.get_dtype_counts(),
-                               Series({'uint8': 8}))
+                               Series({dtype.name: 8}))

-        result = get_dummies(s_df, sparse=self.sparse, columns=['a'])
-        expected = Series({'uint8': 3, 'int64': 1, 'object': 1}).sort_values()
+        result = get_dummies(s_df, columns=['a'], sparse=sparse, dtype=dtype)
+        dtype_name = self.effective_dtype(dtype).name
+
+        expected_counts = {'int64': 1, 'object': 1}
+        expected_counts[dtype_name] = 3 + expected_counts.get(dtype_name, 0)
+
+        expected = Series(expected_counts).sort_values()
         tm.assert_series_equal(result.get_dtype_counts().sort_values(),
                                expected)

-    def test_just_na(self):
+    def test_just_na(self, sparse):
         just_na_list = [np.nan]
         just_na_series = Series(just_na_list)
         just_na_series_index = Series(just_na_list, index=['A'])

-        res_list = get_dummies(just_na_list, sparse=self.sparse)
-        res_series = get_dummies(just_na_series, sparse=self.sparse)
-        res_series_index = get_dummies(just_na_series_index,
-                                       sparse=self.sparse)
+        res_list = get_dummies(just_na_list, sparse=sparse)
+        res_series = get_dummies(just_na_series, sparse=sparse)
+        res_series_index = get_dummies(just_na_series_index, sparse=sparse)

         assert res_list.empty
         assert res_series.empty
@@ -298,216 +319,209 @@ def test_just_na(self):
         assert res_series.index.tolist() == [0]
         assert res_series_index.index.tolist() == ['A']

-    def test_include_na(self):
+    def test_include_na(self, sparse, dtype):
+        if sparse:
+            pytest.xfail(reason='nan in index is problematic (GH 16894)')
+
         s = ['a', 'b', np.nan]
-        res = get_dummies(s, sparse=self.sparse)
-        exp = DataFrame({'a': {0: 1, 1: 0, 2: 0},
-                         'b': {0: 0, 1: 1, 2: 0}}, dtype=np.uint8)
+        res = get_dummies(s, sparse=sparse, dtype=dtype)
+        exp = DataFrame({'a': [1, 0, 0],
+                         'b': [0, 1, 0]},
+                        dtype=self.effective_dtype(dtype))
         assert_frame_equal(res, exp)

         # Sparse dataframes do not allow nan labelled columns, see #GH8822
-        res_na = get_dummies(s, dummy_na=True, sparse=self.sparse)
-        exp_na = DataFrame({nan: {0: 0, 1: 0, 2: 1},
-                            'a': {0: 1, 1: 0, 2: 0},
-                            'b': {0: 0, 1: 1, 2: 0}},
-                           dtype=np.uint8)
+        res_na = get_dummies(s, dummy_na=True, sparse=sparse, dtype=dtype)
+        exp_na = DataFrame({nan: [0, 0, 1],
+                            'a': [1, 0, 0],
+                            'b': [0, 1, 0]},
+                           dtype=self.effective_dtype(dtype))
         exp_na = exp_na.reindex(['a', 'b', nan], axis=1)
         # hack (NaN handling in assert_index_equal)
         exp_na.columns = res_na.columns
         assert_frame_equal(res_na, exp_na)

-        res_just_na = get_dummies([nan], dummy_na=True, sparse=self.sparse)
+        res_just_na = get_dummies([nan], dummy_na=True,
+                                  sparse=sparse, dtype=dtype)
         exp_just_na = DataFrame(Series(1, index=[0]), columns=[nan],
-                                dtype=np.uint8)
+                                dtype=self.effective_dtype(dtype))
         tm.assert_numpy_array_equal(res_just_na.values, exp_just_na.values)

-    def test_unicode(self
-                     ):  # See GH 6885 - get_dummies chokes on unicode values
+    def test_unicode(self, sparse):
+        # See GH 6885 - get_dummies chokes on unicode values
         import unicodedata
         e = 'e'
         eacute = unicodedata.lookup('LATIN SMALL LETTER E WITH ACUTE')
         s = [e, eacute, eacute]
-        res = get_dummies(s, prefix='letter', sparse=self.sparse)
-        exp = DataFrame({'letter_e': {0: 1,
-                                      1: 0,
-                                      2: 0},
-                         u('letter_%s') % eacute: {0: 0,
-                                                   1: 1,
-                                                   2: 1}},
+        res = get_dummies(s, prefix='letter', sparse=sparse)
+        exp = DataFrame({'letter_e': [1, 0, 0],
+                         u('letter_%s') % eacute: [0, 1, 1]},
                         dtype=np.uint8)
         assert_frame_equal(res, exp)

-    def test_dataframe_dummies_all_obj(self):
-        df = self.df[['A', 'B']]
-        result = get_dummies(df, sparse=self.sparse)
+    def test_dataframe_dummies_all_obj(self, df, sparse):
+        df = df[['A', 'B']]
+        result = get_dummies(df, sparse=sparse)
         expected = DataFrame({'A_a': [1, 0, 1],
                               'A_b': [0, 1, 0],
                               'B_b': [1, 1, 0],
-                              'B_c': [0, 0, 1]}, dtype=np.uint8)
+                              'B_c': [0, 0, 1]},
+                             dtype=np.uint8)
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_mix_default(self):
-        df = self.df
-        result = get_dummies(df, sparse=self.sparse)
+    def test_dataframe_dummies_mix_default(self, df, sparse, dtype):
+        result = get_dummies(df, sparse=sparse, dtype=dtype)
         expected = DataFrame({'C': [1, 2, 3],
                               'A_a': [1, 0, 1],
                               'A_b': [0, 1, 0],
                               'B_b': [1, 1, 0],
                               'B_c': [0, 0, 1]})
         cols = ['A_a', 'A_b', 'B_b', 'B_c']
-        expected[cols] = expected[cols].astype(np.uint8)
+        expected[cols] = expected[cols].astype(dtype)
         expected = expected[['C', 'A_a', 'A_b', 'B_b', 'B_c']]
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_prefix_list(self):
+    def test_dataframe_dummies_prefix_list(self, df, sparse):
         prefixes = ['from_A', 'from_B']
-        df = DataFrame({'A': ['a', 'b', 'a'],
-                        'B': ['b', 'b', 'c'],
-                        'C': [1, 2, 3]})
-        result = get_dummies(df, prefix=prefixes, sparse=self.sparse)
+        result = get_dummies(df, prefix=prefixes, sparse=sparse)
         expected = DataFrame({'C': [1, 2, 3],
                               'from_A_a': [1, 0, 1],
                               'from_A_b': [0, 1, 0],
                               'from_B_b': [1, 1, 0],
-                              'from_B_c': [0, 0, 1]})
-        cols = expected.columns[1:]
-        expected[cols] = expected[cols].astype(np.uint8)
-        expected = expected[['C', 'from_A_a', 'from_A_b', 'from_B_b',
-                             'from_B_c']]
+                              'from_B_c': [0, 0, 1]},
+                             dtype=np.uint8)
+        expected[['C']] = df[['C']]
+        expected = expected[['C', 'from_A_a', 'from_A_b',
+                             'from_B_b', 'from_B_c']]
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_prefix_str(self):
+    def test_dataframe_dummies_prefix_str(self, df, sparse):
         # not that you should do this...
-        df = self.df
-        result = get_dummies(df, prefix='bad', sparse=self.sparse)
+        result = get_dummies(df, prefix='bad', sparse=sparse)
+        bad_columns = ['bad_a', 'bad_b', 'bad_b', 'bad_c']
         expected = DataFrame([[1, 1, 0, 1, 0],
                               [2, 0, 1, 1, 0],
                               [3, 1, 0, 0, 1]],
-                             columns=['C', 'bad_a', 'bad_b', 'bad_b', 'bad_c'],
+                             columns=['C'] + bad_columns,
                              dtype=np.uint8)
         expected = expected.astype({"C": np.int64})
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_subset(self):
-        df = self.df
+    def test_dataframe_dummies_subset(self, df, sparse):
         result = get_dummies(df, prefix=['from_A'], columns=['A'],
-                             sparse=self.sparse)
+                             sparse=sparse)
         expected = DataFrame({'from_A_a': [1, 0, 1],
                               'from_A_b': [0, 1, 0],
                               'B': ['b', 'b', 'c'],
-                              'C': [1, 2, 3]})
-        cols = ['from_A_a', 'from_A_b']
-        expected[cols] = expected[cols].astype(np.uint8)
+                              'C': [1, 2, 3]}, dtype=np.uint8)
+        expected[['C']] = df[['C']]
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_prefix_sep(self):
-        df = self.df
-        result = get_dummies(df, prefix_sep='..', sparse=self.sparse)
+    def test_dataframe_dummies_prefix_sep(self, df, sparse):
+        result = get_dummies(df, prefix_sep='..', sparse=sparse)
         expected = DataFrame({'C': [1, 2, 3],
                               'A..a': [1, 0, 1],
                               'A..b': [0, 1, 0],
                               'B..b': [1, 1, 0],
-                              'B..c': [0, 0, 1]})
+                              'B..c': [0, 0, 1]},
+                             dtype=np.uint8)
+        expected[['C']] = df[['C']]
         expected = expected[['C', 'A..a', 'A..b', 'B..b', 'B..c']]
-        cols = expected.columns[1:]
-        expected[cols] = expected[cols].astype(np.uint8)
         assert_frame_equal(result, expected)

-        result = get_dummies(df, prefix_sep=['..', '__'], sparse=self.sparse)
+        result = get_dummies(df, prefix_sep=['..', '__'], sparse=sparse)
         expected = expected.rename(columns={'B..b': 'B__b', 'B..c': 'B__c'})
         assert_frame_equal(result, expected)

-        result = get_dummies(df, prefix_sep={'A': '..',
-                                             'B': '__'}, sparse=self.sparse)
+        result = get_dummies(df, prefix_sep={'A': '..', 'B': '__'},
+                             sparse=sparse)
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_prefix_bad_length(self):
+    def test_dataframe_dummies_prefix_bad_length(self, df, sparse):
         with pytest.raises(ValueError):
-            get_dummies(self.df, prefix=['too few'], sparse=self.sparse)
+            get_dummies(df, prefix=['too few'], sparse=sparse)

-    def test_dataframe_dummies_prefix_sep_bad_length(self):
+    def test_dataframe_dummies_prefix_sep_bad_length(self, df, sparse):
         with pytest.raises(ValueError):
-            get_dummies(self.df, prefix_sep=['bad'], sparse=self.sparse)
+            get_dummies(df, prefix_sep=['bad'], sparse=sparse)

-    def test_dataframe_dummies_prefix_dict(self):
+    def test_dataframe_dummies_prefix_dict(self, sparse):
         prefixes = {'A': 'from_A', 'B': 'from_B'}
         df = DataFrame({'A': ['a', 'b', 'a'],
                         'B': ['b', 'b', 'c'],
                         'C': [1, 2, 3]})
-        result = get_dummies(df, prefix=prefixes, sparse=self.sparse)
+        result = get_dummies(df, prefix=prefixes, sparse=sparse)
+
         expected = DataFrame({'from_A_a': [1, 0, 1],
                               'from_A_b': [0, 1, 0],
                               'from_B_b': [1, 1, 0],
                               'from_B_c': [0, 0, 1],
                               'C': [1, 2, 3]})
-        cols = ['from_A_a', 'from_A_b', 'from_B_b', 'from_B_c']
-        expected[cols] = expected[cols].astype(np.uint8)
+
+        columns = ['from_A_a', 'from_A_b', 'from_B_b', 'from_B_c']
+        expected[columns] = expected[columns].astype(np.uint8)
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_with_na(self):
-        df = self.df
+    def test_dataframe_dummies_with_na(self, df, sparse, dtype):
         df.loc[3, :] = [np.nan, np.nan, np.nan]
-        result = get_dummies(df, dummy_na=True, sparse=self.sparse)
+        result = get_dummies(df, dummy_na=True,
+                             sparse=sparse, dtype=dtype).sort_index(axis=1)
         expected = DataFrame({'C': [1, 2, 3, np.nan],
                               'A_a': [1, 0, 1, 0],
                               'A_b': [0, 1, 0, 0],
                               'A_nan': [0, 0, 0, 1],
                               'B_b': [1, 1, 0, 0],
                               'B_c': [0, 0, 1, 0],
-                              'B_nan': [0, 0, 0, 1]})
-        cols = ['A_a', 'A_b', 'A_nan', 'B_b', 'B_c', 'B_nan']
-        expected[cols] = expected[cols].astype(np.uint8)
-        expected = expected[['C', 'A_a', 'A_b', 'A_nan',
-                             'B_b', 'B_c', 'B_nan']]
+                              'B_nan': [0, 0, 0, 1]}).sort_index(axis=1)
+
+        e_dtype = self.effective_dtype(dtype)
+        columns = ['A_a', 'A_b', 'A_nan', 'B_b', 'B_c', 'B_nan']
+        expected[columns] = expected[columns].astype(e_dtype)
         assert_frame_equal(result, expected)

-        result = get_dummies(df, dummy_na=False, sparse=self.sparse)
+        result = get_dummies(df, dummy_na=False, sparse=sparse, dtype=dtype)
         expected = expected[['C', 'A_a', 'A_b', 'B_b', 'B_c']]
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_with_categorical(self):
-        df = self.df
+    def test_dataframe_dummies_with_categorical(self, df, sparse, dtype):
         df['cat'] = pd.Categorical(['x', 'y', 'y'])
-        result = get_dummies(df, sparse=self.sparse)
+        result = get_dummies(df, sparse=sparse, dtype=dtype).sort_index(axis=1)
         expected = DataFrame({'C': [1, 2, 3],
                               'A_a': [1, 0, 1],
                               'A_b': [0, 1, 0],
                               'B_b': [1, 1, 0],
                               'B_c': [0, 0, 1],
                               'cat_x': [1, 0, 0],
-                              'cat_y': [0, 1, 1]})
-        cols = ['A_a', 'A_b', 'B_b', 'B_c', 'cat_x', 'cat_y']
-        expected[cols] = expected[cols].astype(np.uint8)
-        expected = expected[['C', 'A_a', 'A_b', 'B_b', 'B_c',
-                             'cat_x', 'cat_y']]
+                              'cat_y': [0, 1, 1]}).sort_index(axis=1)
+
+        columns = ['A_a', 'A_b', 'B_b', 'B_c', 'cat_x', 'cat_y']
+        effective_dtype = self.effective_dtype(dtype)
+        expected[columns] = expected[columns].astype(effective_dtype)
+        expected.sort_index(axis=1)
         assert_frame_equal(result, expected)

-    def test_basic_drop_first(self):
+    def test_basic_drop_first(self, sparse):
         # GH12402 Add a new parameter `drop_first` to avoid collinearity
         # Basic case
         s_list = list('abc')
         s_series = Series(s_list)
         s_series_index = Series(s_list, list('ABC'))

-        expected = DataFrame({'b': {0: 0,
-                                    1: 1,
-                                    2: 0},
-                              'c': {0: 0,
-                                    1: 0,
-                                    2: 1}}, dtype=np.uint8)
+        expected = DataFrame({'b': [0, 1, 0],
+                              'c': [0, 0, 1]},
+                             dtype=np.uint8)

-        result = get_dummies(s_list, sparse=self.sparse, drop_first=True)
+        result = get_dummies(s_list, drop_first=True, sparse=sparse)
         assert_frame_equal(result, expected)

-        result = get_dummies(s_series, sparse=self.sparse, drop_first=True)
+        result = get_dummies(s_series, drop_first=True, sparse=sparse)
         assert_frame_equal(result, expected)

         expected.index = list('ABC')
-        result = get_dummies(s_series_index, sparse=self.sparse,
-                             drop_first=True)
+        result = get_dummies(s_series_index, drop_first=True, sparse=sparse)
         assert_frame_equal(result, expected)

-    def test_basic_drop_first_one_level(self):
+    def test_basic_drop_first_one_level(self, sparse):
         # Test the case that categorical variable only has one level.
         s_list = list('aaa')
         s_series = Series(s_list)
@@ -515,53 +529,48 @@ def test_basic_drop_first_one_level(self):

         expected = DataFrame(index=np.arange(3))

-        result = get_dummies(s_list, sparse=self.sparse, drop_first=True)
+        result = get_dummies(s_list, drop_first=True, sparse=sparse)
         assert_frame_equal(result, expected)

-        result = get_dummies(s_series, sparse=self.sparse, drop_first=True)
+        result = get_dummies(s_series, drop_first=True, sparse=sparse)
         assert_frame_equal(result, expected)

         expected = DataFrame(index=list('ABC'))
-        result = get_dummies(s_series_index, sparse=self.sparse,
-                             drop_first=True)
+        result = get_dummies(s_series_index, drop_first=True, sparse=sparse)
         assert_frame_equal(result, expected)

-    def test_basic_drop_first_NA(self):
+    def test_basic_drop_first_NA(self, sparse):
         # Test NA hadling together with drop_first
         s_NA = ['a', 'b', np.nan]
-        res = get_dummies(s_NA, sparse=self.sparse, drop_first=True)
-        exp = DataFrame({'b': {0: 0,
-                               1: 1,
-                               2: 0}}, dtype=np.uint8)
+        res = get_dummies(s_NA, drop_first=True, sparse=sparse)
+        exp = DataFrame({'b': [0, 1, 0]}, dtype=np.uint8)
         assert_frame_equal(res, exp)

-        res_na = get_dummies(s_NA, dummy_na=True, sparse=self.sparse,
-                             drop_first=True)
-        exp_na = DataFrame({'b': {0: 0,
-                                  1: 1,
-                                  2: 0},
-                            nan: {0: 0,
-                                  1: 0,
-                                  2: 1}}, dtype=np.uint8).reindex(
-                                      ['b', nan], axis=1)
+        res_na = get_dummies(s_NA, dummy_na=True, drop_first=True,
+                             sparse=sparse)
+        exp_na = DataFrame(
+            {'b': [0, 1, 0],
+             nan: [0, 0, 1]},
+            dtype=np.uint8).reindex(['b', nan], axis=1)
         assert_frame_equal(res_na, exp_na)

-        res_just_na = get_dummies([nan], dummy_na=True, sparse=self.sparse,
-                                  drop_first=True)
+        res_just_na = get_dummies([nan], dummy_na=True, drop_first=True,
+                                  sparse=sparse)
         exp_just_na = DataFrame(index=np.arange(1))
         assert_frame_equal(res_just_na, exp_just_na)

-    def test_dataframe_dummies_drop_first(self):
-        df = self.df[['A', 'B']]
-        result = get_dummies(df, sparse=self.sparse, drop_first=True)
+    def test_dataframe_dummies_drop_first(self, df, sparse):
+        df = df[['A', 'B']]
+        result = get_dummies(df, drop_first=True, sparse=sparse)
         expected = DataFrame({'A_b': [0, 1, 0],
-                              'B_c': [0, 0, 1]}, dtype=np.uint8)
+                              'B_c': [0, 0, 1]},
+                             dtype=np.uint8)
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_drop_first_with_categorical(self):
-        df = self.df
+    def test_dataframe_dummies_drop_first_with_categorical(
+            self, df, sparse, dtype):
         df['cat'] = pd.Categorical(['x', 'y', 'y'])
-        result = get_dummies(df, sparse=self.sparse, drop_first=True)
+        result = get_dummies(df, drop_first=True, sparse=sparse)
         expected = DataFrame({'C': [1, 2, 3],
                               'A_b': [0, 1, 0],
                               'B_c': [0, 0, 1],
@@ -571,11 +580,10 @@ def test_dataframe_dummies_drop_first_with_categorical(self):
         expected = expected[['C', 'A_b', 'B_c', 'cat_y']]
         assert_frame_equal(result, expected)

-    def test_dataframe_dummies_drop_first_with_na(self):
-        df = self.df
+    def test_dataframe_dummies_drop_first_with_na(self, df, sparse):
         df.loc[3, :] = [np.nan, np.nan, np.nan]
-        result = get_dummies(df, dummy_na=True, sparse=self.sparse,
-                             drop_first=True)
+        result = get_dummies(df, dummy_na=True, drop_first=True,
+                             sparse=sparse).sort_index(axis=1)
         expected = DataFrame({'C': [1, 2, 3, np.nan],
                               'A_b': [0, 1, 0, 0],
                               'A_nan': [0, 0, 0, 1],
@@ -583,30 +591,34 @@ def test_dataframe_dummies_drop_first_with_na(self):
                               'B_nan': [0, 0, 0, 1]})
         cols = ['A_b', 'A_nan', 'B_c', 'B_nan']
         expected[cols] = expected[cols].astype(np.uint8)
-
-        expected = expected[['C', 'A_b', 'A_nan', 'B_c', 'B_nan']]
+        expected = expected.sort_index(axis=1)
         assert_frame_equal(result, expected)

-        result = get_dummies(df, dummy_na=False, sparse=self.sparse,
-                             drop_first=True)
+        result = get_dummies(df, dummy_na=False, drop_first=True,
+                             sparse=sparse)
         expected = expected[['C', 'A_b', 'B_c']]
         assert_frame_equal(result, expected)

     def test_int_int(self):
         data = Series([1, 2, 1])
         result = pd.get_dummies(data)
-        expected = DataFrame([[1, 0], [0, 1], [1, 0]], columns=[1, 2],
+        expected = DataFrame([[1, 0],
+                              [0, 1],
+                              [1, 0]],
+                             columns=[1, 2],
                              dtype=np.uint8)
         tm.assert_frame_equal(result, expected)

         data = Series(pd.Categorical(['a', 'b', 'a']))
         result = pd.get_dummies(data)
-        expected = DataFrame([[1, 0], [0, 1], [1, 0]],
+        expected = DataFrame([[1, 0],
+                              [0, 1],
+                              [1, 0]],
                              columns=pd.Categorical(['a', 'b']),
                              dtype=np.uint8)
         tm.assert_frame_equal(result, expected)

-    def test_int_df(self):
+    def test_int_df(self, dtype):
         data = DataFrame(
             {'A': [1, 2, 1],
              'B': pd.Categorical(['a', 'b', 'a']),
@@ -620,34 +632,28 @@ def test_int_df(self):
             [2, 2., 0, 1, 0, 1],
             [1, 1., 1, 0, 1, 0]
         ], columns=columns)
-        expected[columns[2:]] = expected[columns[2:]].astype(np.uint8)
-        result = pd.get_dummies(data, columns=['A', 'B'])
+        expected[columns[2:]] = expected[columns[2:]].astype(dtype)
+        result = pd.get_dummies(data, columns=['A', 'B'], dtype=dtype)
         tm.assert_frame_equal(result, expected)

-    def test_dataframe_dummies_preserve_categorical_dtype(self):
+    def test_dataframe_dummies_preserve_categorical_dtype(self, dtype):
         # GH13854
         for ordered in [False, True]:
             cat = pd.Categorical(list("xy"), categories=list("xyz"),
                                  ordered=ordered)
-            result = get_dummies(cat)
+            result = get_dummies(cat, dtype=dtype)

-            data = np.array([[1, 0, 0], [0, 1, 0]], dtype=np.uint8)
+            data = np.array([[1, 0, 0], [0, 1, 0]],
+                            dtype=self.effective_dtype(dtype))
             cols = pd.CategoricalIndex(cat.categories,
                                        categories=cat.categories,
                                        ordered=ordered)
-            expected = DataFrame(data, columns=cols)
+            expected = DataFrame(data, columns=cols,
+                                 dtype=self.effective_dtype(dtype))

             tm.assert_frame_equal(result, expected)

-class TestGetDummiesSparse(TestGetDummies):
-    sparse = True
-
-    @pytest.mark.xfail(reason='nan in index is problematic (GH 16894)')
-    def test_include_na(self):
-        super(TestGetDummiesSparse, self).test_include_na()
-
-
 class TestMakeAxisDummies(object):

     def test_preserve_categorical_dtype(self):

ENH: allow get_dummies to accept dtype argument
- [x] closes #18330 (there's no issue for this one)
- [x] tests added / passed
- [x] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
- [x] whatsnew entry

Update in version 0.19.0 made `get_dummies` return uint8 values instead of floats (#8725). While I agree with the argument that `get_dummies` should output integers __by default__ (to save some memory), in many cases it would be beneficial for user to choose other dtype.

In my case there was serious performance degradation between versions 0.18 and 0.19. After investigation, reason behind it turned out to be the change to `get_dummies` output type. DataFrame with dummy values was used as an argument to np.dot in an optimization function (second argument was matrix of floats). Since there were lots of iterations involved, and on each iteration np.dot was converting all uint8 values to float64, conversion overhead took unreasonably long time. It is possible to work around this issue by converting dummy columns "manually" afterwards, but it adds unnecessary complexity to the code and is clearly less convenient than calling `get_dummies` with `dtype=float`.

Apart from performance considerations, I can imagine `dtype=bool` to be a common use case.

`get_dummies(data, dtype=None)` is allowed and will return uint8 values to match the [DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html) interface (where None allows inferring datatype, which is default behavior).

I've extended the test suite to run all the `get_dummies` tests (except for those that don't deal with internal dtypes, like `test_just_na`) twice, once with `uint8` and once with `float64`.