diff --git a/dask_cuda/device_host_file.py b/dask_cuda/device_host_file.py
index 02c826f7..ac0eff00 100644
--- a/dask_cuda/device_host_file.py
+++ b/dask_cuda/device_host_file.py
@@ -1,28 +1,94 @@
 from zict import Buffer, File, Func
 from zict.common import ZictBase
-from distributed.protocol import deserialize_bytes, serialize_bytes
+from distributed.protocol import (
+    deserialize,
+    deserialize_bytes,
+    serialize,
+    serialize_bytelist,
+    dask_serialize,
+    dask_deserialize,
+)
+from dask.sizeof import sizeof
+from distributed.utils import nbytes
 from distributed.worker import weight

-from functools import partial
+from numba import cuda
 import os

 from .is_device_object import is_device_object

-def _serialize_if_device(obj):
-    """ Serialize an object if it's a device object """
-    if is_device_object(obj):
-        return serialize_bytes(obj, on_error="raise")
-    else:
-        return obj
+# Register sizeof for Numba DeviceNDArray while Dask doesn't add it
+if not hasattr(sizeof, "register_numba"):

+    @sizeof.register_lazy("numba")
+    def register_numba():
+        import numba

-def _deserialize_if_device(obj):
-    """ Deserialize an object if it's an instance of bytes """
-    if isinstance(obj, bytes):
-        return deserialize_bytes(obj)
-    else:
-        return obj
+        @sizeof.register(numba.cuda.cudadrv.devicearray.DeviceNDArray)
+        def sizeof_numba_devicearray(x):
+            return int(x.nbytes)
+
+
+class DeviceSerialized:
+    """ Store device object on the host
+
+    This stores a device-side object as
+
+    1.  A msgpack encodable header
+    2.  A list of objects that are returned by calling
+        `numba.cuda.as_cuda_array(f).copy_to_host()`
+        which are typically NumPy arrays
+    """
+
+    def __init__(self, header, parts, is_cuda):
+        self.header = header
+        self.parts = parts
+        self.is_cuda = is_cuda
+
+    def __sizeof__(self):
+        return sum(map(nbytes, self.parts))
+
+
+@dask_serialize.register(DeviceSerialized)
+def _(obj):
+    headers = []
+    all_frames = []
+    for part in obj.parts:
+        header, frames = serialize(part)
+        header["frame-start-stop"] = [len(all_frames), len(all_frames) + len(frames)]
+        headers.append(header)
+        all_frames.extend(frames)
+
+    header = {"sub-headers": headers, "is-cuda": obj.is_cuda, "main-header": obj.header}
+
+    return header, all_frames
+
+
+@dask_deserialize.register(DeviceSerialized)
+def _(header, frames):
+    parts = []
+    for sub_header in header["sub-headers"]:
+        start, stop = sub_header.pop("frame-start-stop")
+        part = deserialize(sub_header, frames[start:stop])
+        parts.append(part)
+
+    return DeviceSerialized(header["main-header"], parts, header["is-cuda"])
+
+
+def device_to_host(obj: object) -> DeviceSerialized:
+    header, frames = serialize(obj, serializers=["cuda"])
+    is_cuda = [hasattr(f, "__cuda_array_interface__") for f in frames]
+    frames = [
+        cuda.as_cuda_array(f).copy_to_host() if ic else f
+        for ic, f in zip(is_cuda, frames)
+    ]
+    return DeviceSerialized(header, frames, is_cuda)
+
+
+def host_to_device(s: DeviceSerialized) -> object:
+    frames = [cuda.to_device(f) if ic else f for ic, f in zip(s.is_cuda, s.parts)]
+    return deserialize(s.header, frames)

 class DeviceHostFile(ZictBase):
@@ -55,17 +121,14 @@ def __init__(
         path = os.path.join(local_directory, "storage")

         self.host_func = dict()
-        self.disk_func = Func(
-            partial(serialize_bytes, on_error="raise"), deserialize_bytes, File(path)
-        )
+        self.disk_func = Func(serialize_bytelist, deserialize_bytes, File(path))
         self.host_buffer = Buffer(
             self.host_func, self.disk_func, memory_limit, weight=weight
         )

+        self.device_keys = set()
         self.device_func = dict()
-        self.device_host_func = Func(
-            _serialize_if_device, _deserialize_if_device, self.host_buffer
-        )
+        self.device_host_func = Func(device_to_host, host_to_device, self.host_buffer)
         self.device_buffer = Buffer(
             self.device_func, self.device_host_func, device_memory_limit, weight=weight
         )
@@ -79,20 +142,18 @@ def __init__(

     def __setitem__(self, key, value):
         if is_device_object(value):
+            self.device_keys.add(key)
             self.device_buffer[key] = value
         else:
             self.host_buffer[key] = value

     def __getitem__(self, key):
-        if key in self.host_buffer:
-            obj = self.host_buffer[key]
-            del self.host_buffer[key]
-            self.device_buffer[key] = _deserialize_if_device(obj)
-
-        if key in self.device_buffer:
+        if key in self.device_keys:
             return self.device_buffer[key]
+        elif key in self.host_buffer:
+            return self.host_buffer[key]
         else:
-            raise KeyError
+            raise KeyError(key)

     def __len__(self):
         return len(self.device_buffer)
@@ -100,5 +161,6 @@ def __len__(self):
     def __iter__(self):
         return iter(self.device_buffer)

-    def __delitem__(self, i):
-        del self.device_buffer[i]
+    def __delitem__(self, key):
+        self.device_keys.discard(key)
+        del self.device_buffer[key]

diff --git a/dask_cuda/tests/test_device_host_file.py b/dask_cuda/tests/test_device_host_file.py
index d53cbc3c..4562a00e 100644
--- a/dask_cuda/tests/test_device_host_file.py
+++ b/dask_cuda/tests/test_device_host_file.py
@@ -1,10 +1,11 @@
 import numpy as np
 import cupy
-from dask_cuda.device_host_file import DeviceHostFile
+from dask_cuda.device_host_file import DeviceHostFile, host_to_device, device_to_host
+from distributed.protocol import deserialize_bytes, serialize_bytelist
 from random import randint
+import dask.array as da

 import pytest
-from cupy.testing import assert_array_equal

 @pytest.mark.parametrize("num_host_arrays", [1, 10, 100])
@@ -33,14 +34,15 @@ def test_device_host_file_short(
     full = host + device
     random.shuffle(full)

-    for i in full:
-        dhf[i[0]] = i[1]
+    for k, v in full:
+        dhf[k] = v

     random.shuffle(full)

-    for i in full:
-        assert_array_equal(i[1], dhf[i[0]])
-        del dhf[i[0]]
+    for k, original in full:
+        acquired = dhf[k]
+        da.assert_eq(original, acquired)
+        del dhf[k]

     assert set(dhf.device.keys()) == set()
     assert set(dhf.host.keys()) == set()
@@ -94,19 +96,48 @@ def test_device_host_file_step_by_step(tmp_path):
     assert set(dhf.host.keys()) == set(["a2", "b2"])
     assert set(dhf.disk.keys()) == set(["a1", "b1"])

-    assert_array_equal(dhf["a1"], a)
+    da.assert_eq(dhf["a1"], a)
     del dhf["a1"]
-    assert_array_equal(dhf["a2"], a)
+    da.assert_eq(dhf["a2"], a)
     del dhf["a2"]
-    assert_array_equal(dhf["b1"], b)
+    da.assert_eq(dhf["b1"], b)
     del dhf["b1"]
-    assert_array_equal(dhf["b2"], b)
+    da.assert_eq(dhf["b2"], b)
     del dhf["b2"]
-    assert_array_equal(dhf["b3"], b)
+    da.assert_eq(dhf["b3"], b)
     del dhf["b3"]
-    assert_array_equal(dhf["b4"], b)
+    da.assert_eq(dhf["b4"], b)
     del dhf["b4"]

     assert set(dhf.device.keys()) == set()
     assert set(dhf.host.keys()) == set()
     assert set(dhf.disk.keys()) == set()
+
+
+def test_serialize_cupy():
+    x = cupy.arange(10)
+
+    obj = device_to_host(x)
+    assert all(isinstance(part, (bytes, memoryview, np.ndarray)) for part in obj.parts)
+    btslst = serialize_bytelist(obj)
+
+    bts = deserialize_bytes(b"".join(btslst))
+    y = host_to_device(bts)
+
+    da.assert_eq(x, y)
+
+
+def test_serialize_cudf():
+    cudf = pytest.importorskip("cudf")
+    dd = pytest.importorskip("dask.dataframe")
+
+    df = cudf.DataFrame({"x": [1, 2, 3], "y": [4.0, 5.0, 6.0]})
+
+    obj = device_to_host(df)
+    assert all(isinstance(part, (bytes, memoryview, np.ndarray)) for part in obj.parts)
+    btslst = serialize_bytelist(obj)
+
+    bts = deserialize_bytes(b"".join(btslst))
+    df2 = host_to_device(bts)
+
+    dd.assert_eq(df, df2)
diff --git a/dask_cuda/tests/test_spill.py b/dask_cuda/tests/test_spill.py
index 46315297..9ff2e0ff 100644
--- a/dask_cuda/tests/test_spill.py
+++ b/dask_cuda/tests/test_spill.py
@@ -4,7 +4,7 @@
 from time import sleep

 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, loop, gen_test
+from distributed.utils_test import gen_cluster, loop, gen_test  # noqa: F401
 from distributed.worker import Worker
 from distributed import Client, get_worker, wait
 from dask_cuda import LocalCUDACluster
@@ -71,15 +71,17 @@ def delayed_worker_assert(total_size, device_chunk_overhead, serialized_chunk_ov
         {
             "device_memory_limit": 1e9,
             "memory_limit": 4e9,
-            "host_target": 0.6,
-            "host_spill": 0.7,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": False,
         },
         {
             "device_memory_limit": 1e9,
-            "memory_limit": 2e9,
-            "host_target": 0.6,
-            "host_spill": 0.6,
+            "memory_limit": 1e9,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": True,
         },
     ],
@@ -89,7 +91,7 @@ def test_cupy_device_spill(params):
         client=True,
         nthreads=[("127.0.0.1", 1)],
         Worker=Worker,
-        timeout=300,
+        timeout=60,
         worker_kwargs={
             "memory_limit": params["memory_limit"],
             "data": DeviceHostFile(
@@ -100,25 +102,25 @@ def test_cupy_device_spill(params):
         config={
             "distributed.worker.memory.target": params["host_target"],
             "distributed.worker.memory.spill": params["host_spill"],
+            "distributed.worker.memory.pause": params["host_pause"],
         },
     )
     def test_device_spill(client, scheduler, worker):
         rs = da.random.RandomState(RandomState=cupy.random.RandomState)
         x = rs.random(int(250e6), chunks=10e6)
-        yield wait(x)

         xx = x.persist()
         yield wait(xx)

         # Allow up to 1024 bytes overhead per chunk serialized
-        yield client.run(worker_assert, x.nbytes, 0, 1024)
+        yield client.run(worker_assert, x.nbytes, 1024, 1024)

         y = client.compute(x.sum())
         res = yield y

         assert (abs(res / x.size) - 0.5) < 1e-3

-        yield client.run(worker_assert, x.nbytes, 0, 1024)
+        yield client.run(worker_assert, x.nbytes, 1024, 1024)
         host_chunks = yield client.run(lambda: len(get_worker().data.host))
         disk_chunks = yield client.run(lambda: len(get_worker().data.disk))
         for hc, dc in zip(host_chunks.values(), disk_chunks.values()):
@@ -137,60 +139,64 @@ def test_device_spill(client, scheduler, worker):
         {
             "device_memory_limit": 1e9,
             "memory_limit": 4e9,
-            "host_target": 0.6,
-            "host_spill": 0.7,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": False,
         },
         {
             "device_memory_limit": 1e9,
-            "memory_limit": 2e9,
-            "host_target": 0.6,
-            "host_spill": 0.6,
+            "memory_limit": 1e9,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": True,
         },
     ],
 )
 @pytest.mark.asyncio
 async def test_cupy_cluster_device_spill(loop, params):
-    async with LocalCUDACluster(
-        1,
-        scheduler_port=0,
-        processes=True,
-        silence_logs=False,
-        dashboard_address=None,
-        asynchronous=True,
-        death_timeout=300,
-        device_memory_limit=params["device_memory_limit"],
-        memory_limit=params["memory_limit"],
-        memory_target_fraction=params["host_target"],
-        memory_spill_fraction=params["host_spill"],
-    ) as cluster:
-        async with Client(cluster, asynchronous=True) as client:
-
-            rs = da.random.RandomState(RandomState=cupy.random.RandomState)
-            x = rs.random(int(250e6), chunks=10e6)
-            await wait(x)
-
-            xx = x.persist()
-            await wait(xx)
-
-            # Allow up to 1024 bytes overhead per chunk serialized
-            await client.run(worker_assert, x.nbytes, 0, 1024)
-
-            y = client.compute(x.sum())
-            res = await y
-
-            assert (abs(res / x.size) - 0.5) < 1e-3
-
-            await client.run(worker_assert, x.nbytes, 0, 1024)
-            host_chunks = await client.run(lambda: len(get_worker().data.host))
-            disk_chunks = await client.run(lambda: len(get_worker().data.disk))
-            for hc, dc in zip(host_chunks.values(), disk_chunks.values()):
-                if params["spills_to_disk"]:
-                    assert dc > 0
-                else:
-                    assert hc > 0
-                    assert dc == 0
+    with dask.config.set({"distributed.worker.memory.terminate": False}):
+        async with LocalCUDACluster(
+            1,
+            scheduler_port=0,
+            processes=True,
+            silence_logs=False,
+            dashboard_address=None,
+            asynchronous=True,
+            death_timeout=60,
+            device_memory_limit=params["device_memory_limit"],
+            memory_limit=params["memory_limit"],
+            memory_target_fraction=params["host_target"],
+            memory_spill_fraction=params["host_spill"],
+            memory_pause_fraction=params["host_pause"],
+        ) as cluster:
+            async with Client(cluster, asynchronous=True) as client:
+
+                rs = da.random.RandomState(RandomState=cupy.random.RandomState)
+                x = rs.random(int(250e6), chunks=10e6)
+                await wait(x)
+
+                xx = x.persist()
+                await wait(xx)
+
+                # Allow up to 1024 bytes overhead per chunk serialized
+                await client.run(worker_assert, x.nbytes, 1024, 1024)
+
+                y = client.compute(x.sum())
+                res = await y
+
+                assert (abs(res / x.size) - 0.5) < 1e-3
+
+                await client.run(worker_assert, x.nbytes, 1024, 1024)
+                host_chunks = await client.run(lambda: len(get_worker().data.host))
+                disk_chunks = await client.run(lambda: len(get_worker().data.disk))
+                for hc, dc in zip(host_chunks.values(), disk_chunks.values()):
+                    if params["spills_to_disk"]:
+                        assert dc > 0
+                    else:
+                        assert hc > 0
+                        assert dc == 0

 @pytest.mark.parametrize(
@@ -199,15 +205,17 @@ async def test_cupy_cluster_device_spill(loop, params):
         {
             "device_memory_limit": 1e9,
             "memory_limit": 4e9,
-            "host_target": 0.6,
-            "host_spill": 0.7,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": False,
         },
         {
             "device_memory_limit": 1e9,
-            "memory_limit": 2e9,
-            "host_target": 0.6,
-            "host_spill": 0.6,
+            "memory_limit": 1e9,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": True,
         },
     ],
@@ -217,7 +225,7 @@ def test_cudf_device_spill(params):
         client=True,
         ncores=[("127.0.0.1", 1)],
         Worker=Worker,
-        timeout=300,
+        timeout=60,
         worker_kwargs={
             "memory_limit": params["memory_limit"],
             "data": DeviceHostFile(
@@ -229,13 +237,19 @@ def test_cudf_device_spill(params):
             "distributed.comm.timeouts.connect": "20s",
             "distributed.worker.memory.target": params["host_target"],
             "distributed.worker.memory.spill": params["host_spill"],
+            "distributed.worker.memory.pause": params["host_pause"],
         },
     )
     def test_device_spill(client, scheduler, worker):

-        cdf = dask.datasets.timeseries(
-            dtypes={"x": int, "y": float}, freq="30ms"
-        ).map_partitions(cudf.from_pandas)
+        # There's a known issue with datetime64:
+        # https://github.com/numpy/numpy/issues/4983#issuecomment-441332940
+        # The same error above happens when spilling datetime64 to disk
+        cdf = (
+            dask.datasets.timeseries(dtypes={"x": int, "y": float}, freq="20ms")
+            .reset_index(drop=True)
+            .map_partitions(cudf.from_pandas)
+        )

         sizes = yield client.compute(cdf.map_partitions(lambda df: df.__sizeof__()))
         sizes = sizes.tolist()
@@ -271,59 +285,70 @@ def test_device_spill(client, scheduler, worker):
         {
             "device_memory_limit": 1e9,
             "memory_limit": 4e9,
-            "host_target": 0.6,
-            "host_spill": 0.7,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": False,
         },
         {
             "device_memory_limit": 1e9,
-            "memory_limit": 2e9,
-            "host_target": 0.6,
-            "host_spill": 0.6,
+            "memory_limit": 1e9,
+            "host_target": 0.0,
+            "host_spill": 0.0,
+            "host_pause": None,
             "spills_to_disk": True,
         },
     ],
 )
 @pytest.mark.asyncio
 async def test_cudf_cluster_device_spill(loop, params):
-    async with LocalCUDACluster(
-        1,
-        device_memory_limit=params["device_memory_limit"],
-        memory_limit=params["memory_limit"],
-        memory_target_fraction=params["host_target"],
-        memory_spill_fraction=params["host_spill"],
-        death_timeout=300,
-        asynchronous=True,
-    ) as cluster:
-        async with Client(cluster, asynchronous=True) as client:
-
-            cdf = dask.datasets.timeseries(
-                dtypes={"x": int, "y": float}, freq="30ms"
-            ).map_partitions(cudf.from_pandas)
-
-            sizes = await client.compute(cdf.map_partitions(lambda df: df.__sizeof__()))
-            sizes = sizes.tolist()
-            nbytes = sum(sizes)
-            part_index_nbytes = (
-                await client.compute(cdf.partitions[0].index)
-            ).__sizeof__()
-
-            cdf2 = cdf.persist()
-            await wait(cdf2)
-
-            del cdf
-
-            await client.run(worker_assert, nbytes, 32, 2048 + part_index_nbytes)
-
-            host_chunks = await client.run(lambda: len(get_worker().data.host))
-            disk_chunks = await client.run(lambda: len(get_worker().data.disk))
-            for hc, dc in zip(host_chunks.values(), disk_chunks.values()):
-                if params["spills_to_disk"]:
-                    assert dc > 0
-                else:
-                    assert hc > 0
-                    assert dc == 0
-
-            del cdf2
-
-            await client.run(delayed_worker_assert, 0, 0, 0)
+    with dask.config.set({"distributed.worker.memory.terminate": False}):
+        async with LocalCUDACluster(
+            1,
+            device_memory_limit=params["device_memory_limit"],
+            memory_limit=params["memory_limit"],
+            memory_target_fraction=params["host_target"],
+            memory_spill_fraction=params["host_spill"],
+            memory_pause_fraction=params["host_pause"],
+            death_timeout=60,
+            asynchronous=True,
+        ) as cluster:
+            async with Client(cluster, asynchronous=True) as client:
+
+                # There's a known issue with datetime64:
+                # https://github.com/numpy/numpy/issues/4983#issuecomment-441332940
+                # The same error above happens when spilling datetime64 to disk
+                cdf = (
+                    dask.datasets.timeseries(dtypes={"x": int, "y": float}, freq="20ms")
+                    .reset_index(drop=True)
+                    .map_partitions(cudf.from_pandas)
+                )
+
+                sizes = await client.compute(
+                    cdf.map_partitions(lambda df: df.__sizeof__())
+                )
+                sizes = sizes.tolist()
+                nbytes = sum(sizes)
+                part_index_nbytes = (
+                    await client.compute(cdf.partitions[0].index)
+                ).__sizeof__()
+
+                cdf2 = cdf.persist()
+                await wait(cdf2)
+
+                del cdf
+
+                await client.run(worker_assert, nbytes, 32, 2048 + part_index_nbytes)
+
+                host_chunks = await client.run(lambda: len(get_worker().data.host))
+                disk_chunks = await client.run(lambda: len(get_worker().data.disk))
+                for hc, dc in zip(host_chunks.values(), disk_chunks.values()):
+                    if params["spills_to_disk"]:
+                        assert dc > 0
+                    else:
+                        assert hc > 0
+                        assert dc == 0
+
+                del cdf2
+
+                await client.run(delayed_worker_assert, 0, 0, 0)

Spill quickly from device to host
I'm curious how fast we are at moving data back and forth between device and host memory.  I suspect that currently we end up serializing with standard Dask serialization, which may not be aware of device memory.  We should take a look at using the `"cuda"` serialization family within Dask, that was added as part of the UCX work.

```python
from distributed.protocol import serialize, deserialize
x = cupy.arange(100)
header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle",    "error"))
```

(more here https://github.com/dask/distributed/tree/master/distributed/protocol/tests)

And then we should probably do a bit of profiling.  I would focus this on just the `DeviceHostDisk` class, rather than engaging the full dask.distributed cluster machinery.

This is relevant for a couple of applications that people care about today

1.  Larger than memory image processing (a bit)
2.  Out-of-core sorts and joins (cc @randerzander @kkraus14) (more relevant)

@pentschev , you seem like the obvious person to work on this if you have some time.  I think it'd be useful to know how fast we are today, then look at serialization with the new "cuda" Dask serializer, and then see how fast we can get things relative to theoretical hardware speeds.

Yes, I haven't done any profiling, and I'm sure this is something that could be improved. Thanks for pointing me to the CUDA serialization, I wasn't aware of that specialization.

If the is not of utmost urgency, I can take a look at this next week, this week I'm mostly busy with EuroPython presentation and at the conference.
Alrighty, I think that we can probably wait until next week (although @randerzander please jump in if not).  Enjoy the conference !
I did some quick benchmarking:

```python
In [1]: import cupy

In [2]: from distributed.protocol import serialize, deserialize

In [3]: x = cupy.arange(1.25e8)

In [4]: %%timeit
   ...: y = serialize(x)
   ...:
   ...:
1.73 s ± 17.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [5]: x.nbytes / 1.73 / 1e6
Out[5]: 578.0346820809249

In [6]: %%timeit
   ...: y = serialize(serialize(x, serializers=("cuda", "dask", "pickle")))
   ...:
   ...:
1.77 s ± 2.19 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [7]: x.nbytes / 1.77 / 1e6
Out[7]: 564.9717514124293
```

The way we serialize nowadays (in [4]) does roughly 576MB/s, whereas the suggested here does 565MB/s. The new way is slightly slower, and both are _very_ slow. Copying data back to CPU in form of a NumPy array is faster, about 2GB/s:

```python
In [8]: %%timeit
   ...: y = x.get()
   ...:
   ...:
494 ms ± 639 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [9]: x.nbytes / .494 / 1e6
Out[9]: 2024.2914979757086
```

From my experience, we should be able to get about 6GB/s for copying data from GPU to CPU, or 12GB/s when using pinned memory (which I'm not sure we can do with CuPy under all circumstances).

I suspect serialization adds a considerable overhead, but 2GB/s feels still too slow, Numba seems to give us similar performance:

```python
In [1]: import numpy as np

In [2]: from numba import cuda

In [3]: x = np.arange(1.25e8)

In [4]: d_x = cuda.to_device(x)

In [5]: %%timeit
   ...: d_x.copy_to_host()
   ...:
   ...:
514 ms ± 1.41 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [6]: 1e9 / .514 / 1e6
Out[6]: 1945.5252918287938
```

I'm not yet sure how we can improve any of those numbers, but I'll investigate further.
Serializing a NumPy array takes 1.11 s:

```python
In [1]: import numpy as np

In [2]: from distributed.protocol import serialize, deserialize

In [3]: x = np.arange(1.25e8)

In [4]: %%timeit
   ...: y = serialize(x)
   ...:
   ...:
9.26 µs ± 140 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

In [5]: %%timeit
   ...: y = serialize(serialize(x))
   ...:
   ...:
1.11 s ± 448 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)
```

If we split the CuPy copy (from my previous comment, 494 ms) and serialization (1.11 s), we get 1.604 s, almost the same as we do for serializing a CuPy array (1.73 s in my previous comment). Therefore, pickling dominates the serialization. Are there faster ways for pickling data in Python? Since the arrays are basically just a bunch of bytes anyway (assuming we don't need to transpose the array, etc.), I'm a bit surprised that it's so slow.
Should this

```python
y = serialize(serialize(x))
```
Instead be this?

```python
y = deserialize(*serialize(x, serializers=...), serializers=...)
```

I would expect to call serialize to get header and frames, and then move those into host memory and write them to disk

```python
header, frames = serialize(x, serializers=["cuda", "dask", "pickle", "msgpack"])
frames = [ensure_on_host(frame) for frame in frames]
```

> `header, frames = serialize(x, serializers=["cuda", "dask", "pickle", "msgpack"])`

This basically just creates a header and wraps the array `x` in a list, but doesn't really serialize anything. The second `serialize` call actually pickles the array, and that's what I wanted to time. In other words, calling `serialize` once, doesn't really do anything, thus it's really fast.
I see.  I suspect that things will be a bit faster if we move each device-side bytes object to a host-side-bytes object independently, rather than bundle them up into a single bytes-like object with serialize/pickle.
The things is that both copying data back to host and pickling are slow. Pickling still dominates, so that should probably be the first one to improve, but the outlook of copying memory back from device is not so good either.
Right, mostly I'm trying to separate overhead from using pickle relative to using whatever numba to/from_device function there is, and also trying to remove the overhead of concatenating all of the results together when we pickle the list.

Then, once we have a nice number that uses pure numba operations (which it looks like you have above already) we can ask the Numba devs.  Given that this is already the case, maybe we should raise an issue there and ask for help?  I suspect that they have run into this before and can help.
> Right, mostly I'm trying to separate overhead from using pickle relative to using whatever numba to/from_device function there is, and also trying to remove the overhead of concatenating all of the results together when we pickle the list.

The list is a single element, unlike the NumPy implementation, the CuPy one is not creating views of the object to break them down into various blocks. Doing that may help pickling (if that can be done in parallel), but copying will likely perform poorly.

> Then, once we have a nice number that uses pure numba operations (which it looks like you have above already) we can ask the Numba devs. Given that this is already the case, maybe we should raise an issue there and ask for help? I suspect that they have run into this before and can help.

I can do that, but I will first write a small C++ sample to illustrate that and make sure I have numbers for the same machine and compare apples to apples.
@pentschev can I ask you to quickly try an implementation that looks like the following:

```python
def _serialize_if_device(obj):
    """ Serialize an object if it's a device object """
    assert is_device_object(obj)  # this used to be a conditional, under what situations would this not be a device object?
    header, frames = cuda_serialize(obj)
    frames = [numba.DeviceArray(frame).copy_to_host() for frame in frames]
    return header, frames

def _deserialize_if_device(obj):
    header, frames = obj
    frames = [numba.cuda.to_device(frame) for frame in frames]
    return cuda_deserialize(header, frames)
```

Previously these functions would check to see if the object was device-like.  Why this condition?  Was this for things like lists and dicts?  If so, lets just implement `cuda_serialize` operations for lists and dicts that concatenate the headers and frames of their inputs as necessary.


diff --git a/django/db/backends/oracle/compiler.py b/django/db/backends/oracle/compiler.py
index 3ae567669f1b..9aa4acc0fe57 100644
--- a/django/db/backends/oracle/compiler.py
+++ b/django/db/backends/oracle/compiler.py
@@ -31,10 +31,17 @@ def as_sql(self, with_limits=True, with_col_aliases=False):
             high_where = ''
             if self.query.high_mark is not None:
                 high_where = 'WHERE ROWNUM <= %d' % (self.query.high_mark,)
-            sql = (
-                'SELECT * FROM (SELECT "_SUB".*, ROWNUM AS "_RN" FROM (%s) '
-                '"_SUB" %s) WHERE "_RN" > %d' % (sql, high_where, self.query.low_mark)
-            )
+
+            if self.query.low_mark:
+                sql = (
+                    'SELECT * FROM (SELECT "_SUB".*, ROWNUM AS "_RN" FROM (%s) '
+                    '"_SUB" %s) WHERE "_RN" > %d' % (sql, high_where, self.query.low_mark)
+                )
+            else:
+                # Simplify the query to support subqueries if there's no offset.
+                sql = (
+                    'SELECT * FROM (SELECT "_SUB".* FROM (%s) "_SUB" %s)' % (sql, high_where)
+                )

         return sql, params

diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py
index 225436231c90..8ab11b098ad6 100644
--- a/django/db/models/__init__.py
+++ b/django/db/models/__init__.py
@@ -6,7 +6,8 @@
     CASCADE, DO_NOTHING, PROTECT, SET, SET_DEFAULT, SET_NULL, ProtectedError,
 )
 from django.db.models.expressions import (
-    Case, Expression, ExpressionWrapper, F, Func, Value, When,
+    Case, Exists, Expression, ExpressionWrapper, F, Func, OuterRef, Subquery,
+    Value, When,
 )
 from django.db.models.fields import *  # NOQA
 from django.db.models.fields import __all__ as fields_all
@@ -62,7 +63,8 @@ def inner(*args, **kwargs):
     'ObjectDoesNotExist', 'signals',
     'CASCADE', 'DO_NOTHING', 'PROTECT', 'SET', 'SET_DEFAULT', 'SET_NULL',
     'ProtectedError',
-    'Case', 'Expression', 'ExpressionWrapper', 'F', 'Func', 'Value', 'When',
+    'Case', 'Exists', 'Expression', 'ExpressionWrapper', 'F', 'Func',
+    'OuterRef', 'Subquery', 'Value', 'When',
     'FileField', 'ImageField', 'OrderWrt', 'Lookup', 'Transform', 'Manager',
     'Prefetch', 'Q', 'QuerySet', 'prefetch_related_objects', 'DEFERRED', 'Model',
     'ForeignKey', 'ForeignObject', 'OneToOneField', 'ManyToManyField',
diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py
index 1ff4cd77353d..36c2b969db12 100644
--- a/django/db/models/expressions.py
+++ b/django/db/models/expressions.py
@@ -477,6 +477,33 @@ def desc(self, **kwargs):
         return OrderBy(self, descending=True, **kwargs)

+class ResolvedOuterRef(F):
+    """
+    An object that contains a reference to an outer query.
+
+    In this case, the reference to the outer query has been resolved because
+    the inner query has been used as a subquery.
+    """
+    def as_sql(self, *args, **kwargs):
+        raise ValueError(
+            'This queryset contains a reference to an outer query and may '
+            'only be used in a subquery.'
+        )
+
+    def _prepare(self, output_field=None):
+        return self
+
+
+class OuterRef(F):
+    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
+        if isinstance(self.name, self.__class__):
+            return self.name
+        return ResolvedOuterRef(self.name)
+
+    def _prepare(self, output_field=None):
+        return self
+
+
 class Func(Expression):
     """
     An SQL function call.
@@ -873,6 +900,128 @@ def as_sql(self, compiler, connection, template=None, case_joiner=None, **extra_
         return sql, sql_params

+class Subquery(Expression):
+    """
+    An explicit subquery. It may contain OuterRef() references to the outer
+    query which will be resolved when it is applied to that query.
+    """
+    template = '(%(subquery)s)'
+
+    def __init__(self, queryset, output_field=None, **extra):
+        self.queryset = queryset
+        self.extra = extra
+        if output_field is None and len(self.queryset.query.select) == 1:
+            output_field = self.queryset.query.select[0].field
+        super(Subquery, self).__init__(output_field)
+
+    def copy(self):
+        clone = super(Subquery, self).copy()
+        clone.queryset = clone.queryset.all()
+        return clone
+
+    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
+        clone = self.copy()
+        clone.is_summary = summarize
+        clone.queryset.query.bump_prefix(query)
+
+        # Need to recursively resolve these.
+        def resolve_all(child):
+            if hasattr(child, 'children'):
+                [resolve_all(_child) for _child in child.children]
+            if hasattr(child, 'rhs'):
+                child.rhs = resolve(child.rhs)
+
+        def resolve(child):
+            if hasattr(child, 'resolve_expression'):
+                return child.resolve_expression(
+                    query=query, allow_joins=allow_joins, reuse=reuse,
+                    summarize=summarize, for_save=for_save,
+                )
+            return child
+
+        resolve_all(clone.queryset.query.where)
+
+        for key, value in clone.queryset.query.annotations.items():
+            if isinstance(value, Subquery):
+                clone.queryset.query.annotations[key] = resolve(value)
+
+        return clone
+
+    def get_source_expressions(self):
+        return [
+            x for x in [
+                getattr(expr, 'lhs', None)
+                for expr in self.queryset.query.where.children
+            ] if x
+        ]
+
+    def relabeled_clone(self, change_map):
+        clone = self.copy()
+        clone.queryset.query = clone.queryset.query.relabeled_clone(change_map)
+        clone.queryset.query.external_aliases.update(
+            alias for alias in change_map.values()
+            if alias not in clone.queryset.query.tables
+        )
+        return clone
+
+    def as_sql(self, compiler, connection, template=None, **extra_context):
+        connection.ops.check_expression_support(self)
+        template_params = self.extra.copy()
+        template_params.update(extra_context)
+        template_params['subquery'], sql_params = self.queryset.query.get_compiler(connection=connection).as_sql()
+
+        template = template or template_params.get('template', self.template)
+        sql = template % template_params
+        sql = connection.ops.unification_cast_sql(self.output_field) % sql
+        return sql, sql_params
+
+    def _prepare(self, output_field):
+        # This method will only be called if this instance is the "rhs" in an
+        # expression: the wrapping () must be removed (as the expression that
+        # contains this will provide them). SQLite evaluates ((subquery))
+        # differently than the other databases.
+        if self.template == '(%(subquery)s)':
+            clone = self.copy()
+            clone.template = '%(subquery)s'
+            return clone
+        return self
+
+
+class Exists(Subquery):
+    template = 'EXISTS(%(subquery)s)'
+
+    def __init__(self, *args, **kwargs):
+        self.negated = kwargs.pop('negated', False)
+        super(Exists, self).__init__(*args, **kwargs)
+
+    def __invert__(self):
+        return type(self)(self.queryset, self.output_field, negated=(not self.negated), **self.extra)
+
+    @property
+    def output_field(self):
+        return fields.BooleanField()
+
+    def resolve_expression(self, query=None, **kwargs):
+        # As a performance optimization, remove ordering since EXISTS doesn't
+        # care about it, just whether or not a row matches.
+        self.queryset = self.queryset.order_by()
+        return super(Exists, self).resolve_expression(query, **kwargs)
+
+    def as_sql(self, compiler, connection, template=None, **extra_context):
+        sql, params = super(Exists, self).as_sql(compiler, connection, template, **extra_context)
+        if self.negated:
+            sql = 'NOT {}'.format(sql)
+        return sql, params
+
+    def as_oracle(self, compiler, connection, template=None, **extra_context):
+        # Oracle doesn't allow EXISTS() in the SELECT list, so wrap it with a
+        # CASE WHEN expression. Change the template since the When expression
+        # requires a left hand side (column) to compare against.
+        sql, params = self.as_sql(compiler, connection, template, **extra_context)
+        sql = 'CASE WHEN {} THEN 1 ELSE 0 END'.format(sql)
+        return sql, params
+
+
 class OrderBy(BaseExpression):
     template = '%(expression)s %(ordering)s'

diff --git a/docs/ref/models/expressions.txt b/docs/ref/models/expressions.txt
index e46df22f9828..01db10375878 100644
--- a/docs/ref/models/expressions.txt
+++ b/docs/ref/models/expressions.txt
@@ -450,6 +450,178 @@ Conditional expressions allow you to use :keyword:`if` ... :keyword:`elif` ...
 :keyword:`else` logic in queries. Django natively supports SQL ``CASE``
 expressions. For more details see :doc:`conditional-expressions`.

+``Subquery()`` expressions
+--------------------------
+
+.. class:: Subquery(queryset, output_field=None)
+
+.. versionadded:: 1.11
+
+You can add an explicit subquery to a ``QuerySet`` using the ``Subquery``
+expression.
+
+For example, to annotate each post with the email address of the author of the
+newest comment on that post::
+
+    >>> from django.db.models import OuterRef, Subquery
+    >>> newest = Comment.objects.filter(post=OuterRef('pk')).order_by('-created_at')
+    >>> Post.objects.annotate(newest_commenter_email=Subquery(newest.values('email')[:1]))
+
+On PostgreSQL, the SQL looks like:
+
+.. code-block:: sql
+
+    SELECT "post"."id", (
+        SELECT U0."email"
+        FROM "comment" U0
+        WHERE U0."post_id" = ("post"."id")
+        ORDER BY U0."created_at" DESC LIMIT 1
+    ) AS "newest_commenter_email" FROM "post"
+
+.. note::
+
+    The examples in this section are designed to show how to force
+    Django to execute a subquery. In some cases it may be possible to
+    write an equivalent queryset that performs the same task more
+    clearly or efficiently.
+
+Referencing columns from the outer queryset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. class:: OuterRef(field)
+
+.. versionadded:: 1.11
+
+Use ``OuterRef`` when a queryset in a ``Subquery`` needs to refer to a field
+from the outer query. It acts like an :class:`F` expression except that the
+check to see if it refers to a valid field isn't made until the outer queryset
+is resolved.
+
+Instances of ``OuterRef`` may be used in conjunction with nested instances
+of ``Subquery`` to refer to a containing queryset that isn't the immediate
+parent. For example, this queryset would need to be within a nested pair of
+``Subquery`` instances to resolve correctly::
+
+    >>> Book.objects.filter(author=OuterRef(OuterRef('pk')))
+
+Limiting a subquery to a single column
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are times when a single column must be returned from a ``Subquery``, for
+instance, to use a ``Subquery`` as the target of an ``__in`` lookup. To return
+all comments for posts published within the last day::
+
+    >>> from datetime import timedelta
+    >>> from django.utils import timezone
+    >>> one_day_ago = timezone.now() - timedelta(days=1)
+    >>> posts = Post.objects.filter(published_at__gte=one_day_ago)
+    >>> Comment.objects.filter(post__in=Subquery(posts.values('pk')))
+
+In this case, the subquery must use :meth:`~.QuerySet.values`
+to return only a single column: the primary key of the post.
+
+Limiting the subquery to a single row
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To prevent a subquery from returning multiple rows, a slice (``[:1]``) of the
+queryset is used::
+
+    >>> subquery = Subquery(newest.values('email')[:1])
+    >>> Post.objects.annotate(newest_commenter_email=subquery)
+
+In this case, the subquery must only return a single column *and* a single
+row: the email address of the most recently created comment.
+
+(Using :meth:`~.QuerySet.get` instead of a slice would fail because the
+``OuterRef`` cannot be resolved until the queryset is used within a
+``Subquery``.)
+
+``Exists()`` subqueries
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. class:: Exists(queryset)
+
+.. versionadded:: 1.11
+
+``Exists`` is a ``Subquery`` subclass that uses an SQL ``EXISTS`` statement. In
+many cases it will perform better than a subquery since the database is able to
+stop evaluation of the subquery when a first matching row is found.
+
+For example, to annotate each post with whether or not it has a comment from
+within the last day::
+
+    >>> from django.db.models import Exists, OuterRef
+    >>> from datetime import timedelta
+    >>> from django.utils import timezone
+    >>> one_day_ago = timezone.now() - timedelta(days=1)
+    >>> recent_comments = Comment.objects.filter(
+    ...     post=OuterRef('pk'),
+    ...     created_at__gte=one_day_ago,
+    ... )
+    >>> Post.objects.annotate(recent_comment=Exists(recent_comments)
+
+On PostgreSQL, the SQL looks like:
+
+.. code-block:: sql
+
+    SELECT "post"."id", "post"."published_at", EXISTS(
+        SELECT U0."id", U0."post_id", U0."email", U0."created_at"
+        FROM "comment" U0
+        WHERE (
+            U0."created_at" >= YYYY-MM-DD HH:MM:SS AND
+            U0."post_id" = ("post"."id")
+        )
+    ) AS "recent_comment" FROM "post"
+
+It's unnecessary to force ``Exists`` to refer to a single column, since the
+columns are discarded and a boolean result is returned. Similarly, since
+ordering is unimportant within an SQL ``EXISTS`` subquery and would only
+degrade performance, it's automatically removed.
+
+You can query using ``NOT EXISTS`` with ``~Exists()``.
+
+Filtering on a ``Subquery`` expression
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It's not possible to filter directly using ``Subquery`` and ``Exists``, e.g.::
+
+    >>> Post.objects.filter(Exists(recent_comments))
+    ...
+    TypeError: 'Exists' object is not iterable
+
+
+You must filter on a subquery expression by first annotating the queryset
+and then filtering based on that annotation::
+
+    >>> Post.objects.annotate(
+    ...     recent_comment=Exists(recent_comments),
+    ... ).filter(recent_comment=True)
+
+Using aggregates within a ``Subquery`` expression
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Aggregates may be used within a ``Subquery``, but they require a specific
+combination of :meth:`~.QuerySet.filter`, :meth:`~.QuerySet.values`, and
+:meth:`~.QuerySet.annotate` to get the subquery grouping correct.
+
+Assuming both models have a ``length`` field, to find posts where the post
+length is greater than the total length of all combined comments::
+
+    >>> from django.db.models import OuterRef, Subquery, Sum
+    >>> comments = Comment.objects.filter(post=OuterRef('pk')).values('post')
+    >>> total_comments = comments.annotate(total=Sum('length')).values('total')
+    >>> Post.objects.filter(length__gt=Subquery(total_comments))
+
+The initial ``filter(...)`` limits the subquery to the relevant parameters.
+``values('post')`` aggregates comments by ``Post``. Finally, ``annotate(...)``
+performs the aggregation. The order in which these queryset methods are applied
+is important. In this case, since the subquery must be limited to a single
+column, ``values('total')`` is required.
+
+This is the only way to perform an aggregation within a ``Subquery``, as
+using :meth:`~.QuerySet.aggregate` attempts to evaluate the queryset (and if
+there is an ``OuterRef``, this will not be possible to resolve).
+
 Raw SQL expressions
 -------------------

diff --git a/docs/releases/1.11.txt b/docs/releases/1.11.txt
index edbd2b0be310..e5b0efffef06 100644
--- a/docs/releases/1.11.txt
+++ b/docs/releases/1.11.txt
@@ -70,6 +70,14 @@ template system rather than in Python. See :doc:`/ref/forms/renderers`.
 You may need to adjust any custom widgets that you've written for a few
 :ref:`backwards incompatible changes <template-widget-incompatibilities-1-11>`.

+``Subquery`` expressions
+------------------------
+
+The new :class:`~django.db.models.Subquery` and
+:class:`~django.db.models.Exists` database expressions allow creating
+explicit subqueries. Subqueries may refer to fields from the outer queryset
+using the :class:`~django.db.models.OuterRef` class.
+
 Minor features
 --------------

diff --git a/tests/expressions/tests.py b/tests/expressions/tests.py
index 18d003d57d74..8399e3c0a92c 100644
--- a/tests/expressions/tests.py
+++ b/tests/expressions/tests.py
@@ -12,8 +12,8 @@
     Avg, Count, Max, Min, StdDev, Sum, Variance,
 )
 from django.db.models.expressions import (
-    Case, Col, ExpressionWrapper, F, Func, OrderBy, Random, RawSQL, Ref, Value,
-    When,
+    Case, Col, Exists, ExpressionWrapper, F, Func, OrderBy, OuterRef, Random,
+    RawSQL, Ref, Subquery, Value, When,
 )
 from django.db.models.functions import (
     Coalesce, Concat, Length, Lower, Substr, Upper,
@@ -32,15 +32,15 @@
 class BasicExpressionsTests(TestCase):
     @classmethod
     def setUpTestData(cls):
-        Company.objects.create(
+        cls.example_inc = Company.objects.create(
             name="Example Inc.", num_employees=2300, num_chairs=5,
             ceo=Employee.objects.create(firstname="Joe", lastname="Smith", salary=10)
         )
-        Company.objects.create(
+        cls.foobar_ltd = Company.objects.create(
             name="Foobar Ltd.", num_employees=3, num_chairs=4,
             ceo=Employee.objects.create(firstname="Frank", lastname="Meyer", salary=20)
         )
-        Company.objects.create(
+        cls.gmbh = Company.objects.create(
             name="Test GmbH", num_employees=32, num_chairs=1,
             ceo=Employee.objects.create(firstname="Max", lastname="Mustermann", salary=30)
         )
@@ -387,6 +387,136 @@ def test_ticket_18375_chained_filters(self):
         )
         self.assertEqual(str(qs.query).count('JOIN'), 2)

+    def test_outerref(self):
+        inner = Company.objects.filter(point_of_contact=OuterRef('pk'))
+        msg = (
+            'This queryset contains a reference to an outer query and may only '
+            'be used in a subquery.'
+        )
+        with self.assertRaisesMessage(ValueError, msg):
+            inner.exists()
+
+        outer = Employee.objects.annotate(is_point_of_contact=Exists(inner))
+        self.assertIs(outer.exists(), True)
+
+    def test_subquery(self):
+        Company.objects.filter(name='Example Inc.').update(
+            point_of_contact=Employee.objects.get(firstname='Joe', lastname='Smith'),
+            ceo=Employee.objects.get(firstname='Max', lastname='Mustermann'),
+        )
+        Employee.objects.create(firstname='Bob', lastname='Brown', salary=40)
+        qs = Employee.objects.annotate(
+            is_point_of_contact=Exists(Company.objects.filter(point_of_contact=OuterRef('pk'))),
+            is_not_point_of_contact=~Exists(Company.objects.filter(point_of_contact=OuterRef('pk'))),
+            is_ceo_of_small_company=Exists(Company.objects.filter(num_employees__lt=200, ceo=OuterRef('pk'))),
+            is_ceo_small_2=~~Exists(Company.objects.filter(num_employees__lt=200, ceo=OuterRef('pk'))),
+            largest_company=Subquery(Company.objects.order_by('-num_employees').filter(
+                models.Q(ceo=OuterRef('pk')) | models.Q(point_of_contact=OuterRef('pk'))
+            ).values('name')[:1], output_field=models.CharField())
+        ).values(
+            'firstname',
+            'is_point_of_contact',
+            'is_not_point_of_contact',
+            'is_ceo_of_small_company',
+            'is_ceo_small_2',
+            'largest_company',
+        ).order_by('firstname')
+
+        results = list(qs)
+        # Could use Coalesce(subq, Value('')) instead except for the bug in
+        # cx_Oracle mentioned in #23843.
+        bob = results[0]
+        if bob['largest_company'] == '' and connection.features.interprets_empty_strings_as_nulls:
+            bob['largest_company'] = None
+
+        self.assertEqual(results, [
+            {
+                'firstname': 'Bob',
+                'is_point_of_contact': False,
+                'is_not_point_of_contact': True,
+                'is_ceo_of_small_company': False,
+                'is_ceo_small_2': False,
+                'largest_company': None,
+            },
+            {
+                'firstname': 'Frank',
+                'is_point_of_contact': False,
+                'is_not_point_of_contact': True,
+                'is_ceo_of_small_company': True,
+                'is_ceo_small_2': True,
+                'largest_company': 'Foobar Ltd.',
+            },
+            {
+                'firstname': 'Joe',
+                'is_point_of_contact': True,
+                'is_not_point_of_contact': False,
+                'is_ceo_of_small_company': False,
+                'is_ceo_small_2': False,
+                'largest_company': 'Example Inc.',
+            },
+            {
+                'firstname': 'Max',
+                'is_point_of_contact': False,
+                'is_not_point_of_contact': True,
+                'is_ceo_of_small_company': True,
+                'is_ceo_small_2': True,
+                'largest_company': 'Example Inc.'
+            }
+        ])
+        # A less elegant way to write the same query: this uses a LEFT OUTER
+        # JOIN and an IS NULL, inside a WHERE NOT IN which is probably less
+        # efficient than EXISTS.
+        self.assertCountEqual(
+            qs.filter(is_point_of_contact=True).values('pk'),
+            Employee.objects.exclude(company_point_of_contact_set=None).values('pk')
+        )
+
+    def test_in_subquery(self):
+        # This is a contrived test (and you really wouldn't write this query),
+        # but it is a succinct way to test the __in=Subquery() construct.
+        small_companies = Company.objects.filter(num_employees__lt=200).values('pk')
+        subquery_test = Company.objects.filter(pk__in=Subquery(small_companies))
+        self.assertCountEqual(subquery_test, [self.foobar_ltd, self.gmbh])
+        subquery_test2 = Company.objects.filter(pk=Subquery(small_companies.filter(num_employees=3)))
+        self.assertCountEqual(subquery_test2, [self.foobar_ltd])
+
+    def test_nested_subquery(self):
+        inner = Company.objects.filter(point_of_contact=OuterRef('pk'))
+        outer = Employee.objects.annotate(is_point_of_contact=Exists(inner))
+        contrived = Employee.objects.annotate(
+            is_point_of_contact=Subquery(
+                outer.filter(pk=OuterRef('pk')).values('is_point_of_contact'),
+                output_field=models.BooleanField(),
+            ),
+        )
+        self.assertCountEqual(contrived.values_list(), outer.values_list())
+
+    def test_nested_subquery_outer_ref_2(self):
+        first = Time.objects.create(time='09:00')
+        second = Time.objects.create(time='17:00')
+        third = Time.objects.create(time='21:00')
+        SimulationRun.objects.bulk_create([
+            SimulationRun(start=first, end=second, midpoint='12:00'),
+            SimulationRun(start=first, end=third, midpoint='15:00'),
+            SimulationRun(start=second, end=first, midpoint='00:00'),
+        ])
+        inner = Time.objects.filter(time=OuterRef(OuterRef('time')), pk=OuterRef('start')).values('time')
+        middle = SimulationRun.objects.annotate(other=Subquery(inner)).values('other')[:1]
+        outer = Time.objects.annotate(other=Subquery(middle, output_field=models.TimeField()))
+        # This is a contrived example. It exercises the double OuterRef form.
+        self.assertCountEqual(outer, [first, second, third])
+
+    def test_annotations_within_subquery(self):
+        Company.objects.filter(num_employees__lt=50).update(ceo=Employee.objects.get(firstname='Frank'))
+        inner = Company.objects.filter(
+            ceo=OuterRef('pk')
+        ).values('ceo').annotate(total_employees=models.Sum('num_employees')).values('total_employees')
+        outer = Employee.objects.annotate(total_employees=Subquery(inner)).filter(salary__lte=Subquery(inner))
+        self.assertSequenceEqual(
+            outer.order_by('-total_employees').values('salary', 'total_employees'),
+            [{'salary': 10, 'total_employees': 2300}, {'salary': 20, 'total_employees': 35}],
+        )
+

 class IterableLookupInnerExpressionsTests(TestCase):
     @classmethod

Allow using a subquery in QuerySet.filter()
Description

(last modified by MikiSoft)
The following function is used for filtering by generic relation (and also by one column in the model where it is) which isn't natively supported by Django.
APP_LABEL = os.path.basename(os.path.dirname(__file__))
def generic_rel_filter(model, target, column, id):
	return model.objects.extra(where=['''
		{app_label}_{model}.id in (select object_id
		from {app_label}_{target}
		where content_type_id = (select id from django_content_type where model = '{model}')
			and {column} = {id})'''.format(app_label=APP_LABEL, model=model.__name__.lower(), target=target, column=column, id=id)])
Example: If I have Event and Like model, and the second one has generic relation to the first one (i.e. it has content_type, object_id and content_object fields), then if I want to get all events which current user liked, I would just make this call in a view: generic_rel_filter(Event, 'like', 'person', self.request.user.pk)
Note that this function isn't intended to be used with user specified parameters, otherwise it's prone to SQL injection attacks.
P.S. It can be done with ORM but then it would go with three queries, which is much slower than the method above (which uses only one query to do the same): Event.objects.filter(pk__in=Like.objects.filter(content_type=ContentType.objects.get(model='event'), person=self.request.user).values_list('object_id', flat=True))
Allow using a subquery in QuerySet.filter()
Description

(last modified by MikiSoft)
The following function is used for filtering by generic relation (and also by one column in the model where it is) which isn't natively supported by Django.
APP_LABEL = os.path.basename(os.path.dirname(__file__))
def generic_rel_filter(model, target, column, id):
	return model.objects.extra(where=['''
		{app_label}_{model}.id in (select object_id
		from {app_label}_{target}
		where content_type_id = (select id from django_content_type where model = '{model}')
			and {column} = {id})'''.format(app_label=APP_LABEL, model=model.__name__.lower(), target=target, column=column, id=id)])
Example: If I have Event and Like model, and the second one has generic relation to the first one (i.e. it has content_type, object_id and content_object fields), then if I want to get all events which current user liked, I would just make this call in a view: generic_rel_filter(Event, 'like', 'person', self.request.user.pk)
Note that this function isn't intended to be used with user specified parameters, otherwise it's prone to SQL injection attacks.
P.S. It can be done with ORM but then it would go with three queries, which is much slower than the method above (which uses only one query to do the same): Event.objects.filter(pk__in=Like.objects.filter(content_type=ContentType.objects.get(model='event'), person=self.request.user).values_list('object_id', flat=True))


diff --git a/core/domain/opportunity_services.py b/core/domain/opportunity_services.py
index 71a8c378ca4c..5a0be0a9da4c 100644
--- a/core/domain/opportunity_services.py
+++ b/core/domain/opportunity_services.py
@@ -18,6 +18,7 @@

 from __future__ import annotations

+import collections
 import logging

 from core.constants import constants
@@ -25,6 +26,7 @@
 from core.domain import opportunity_domain
 from core.domain import question_fetchers
 from core.domain import story_fetchers
+from core.domain import suggestion_services
 from core.domain import topic_fetchers
 from core.platform import models

@@ -93,43 +95,6 @@ def get_exploration_opportunity_summary_from_model(model):
         {})

-def get_exp_opportunity_summary_with_in_review_translations_from_model(
-    model, translations_in_review):
-    """Returns the ExplorationOpportunitySummary object out of the model when
-    there are translations that are in review.
-
-    Args:
-        model: ExplorationOpportunitySummaryModel. The exploration opportunity
-            summary model.
-        translations_in_review: list(SuggestionModel). The list of translations
-            which are in review.
-
-    Returns:
-        ExplorationOpportunitySummary. The corresponding
-        ExplorationOpportunitySummary object.
-    """
-    translation_opportunity = get_exploration_opportunity_summary_from_model(
-        model)
-    translation_in_review_counts = {}
-
-    for language_code in constants.SUPPORTED_CONTENT_LANGUAGES:
-        in_review_count = 0
-        for suggestion in translations_in_review:
-            if (
-                suggestion is not None and
-                suggestion.language_code == language_code['code'] and
-                suggestion.target_id == model.id):
-                in_review_count = in_review_count + 1
-        if in_review_count > 0:
-            translation_in_review_counts[
-                language_code['code']] = in_review_count
-
-    translation_opportunity.translation_in_review_counts = (
-        translation_in_review_counts)
-
-    return translation_opportunity
-
-
 def _save_multi_exploration_opportunity_summary(
         exploration_opportunity_summary_list):
     """Stores multiple ExplorationOpportunitySummary into datastore as a
@@ -497,28 +462,55 @@ def get_translation_opportunities(language_code, topic_name, cursor):
         opportunity_models
         .ExplorationOpportunitySummaryModel.get_all_translation_opportunities(
             page_size, cursor, language_code, topic_name))
-    opportunities = []
-    suggestion_ids = []
-    opportunity_ids = []
-    translations_in_review = []
-    opportunity_ids = [
+    opportunity_summaries = []
+    opportunity_summary_exp_ids = [
         opportunity.id for opportunity in exp_opportunity_summary_models]
-    if len(opportunity_ids) > 0:
-        suggestion_ids = (
-            suggestion_models
-            .GeneralSuggestionModel
-            .get_translation_suggestions_in_review_ids_with_exp_id(
-                opportunity_ids))
-        translations_in_review = (
-            suggestion_models
-            .GeneralSuggestionModel
-            .get_multiple_suggestions_from_suggestion_ids(suggestion_ids))
+    exp_id_to_in_review_count = {}
+    if len(opportunity_summary_exp_ids) > 0:
+        exp_id_to_in_review_count = (
+            _build_exp_id_to_translation_suggestion_in_review_count(
+                opportunity_summary_exp_ids, language_code))
     for exp_opportunity_summary_model in exp_opportunity_summary_models:
-        exp_opportunity_summary = (
-            get_exp_opportunity_summary_with_in_review_translations_from_model(
-                exp_opportunity_summary_model, translations_in_review))
-        opportunities.append(exp_opportunity_summary)
-    return opportunities, cursor, more
+        opportunity_summary = (
+            get_exploration_opportunity_summary_from_model(
+                exp_opportunity_summary_model))
+        if opportunity_summary.id in exp_id_to_in_review_count:
+            # Compute the translation_in_review_counts domain object field
+            # adhoc. Note that this field is not persisted and is only used in
+            # the frontend.
+            # TODO(#14833): Compute this value in the backend controller
+            # instead.
+            opportunity_summary.translation_in_review_counts = {
+                language_code: exp_id_to_in_review_count[opportunity_summary.id]
+            }
+        opportunity_summaries.append(opportunity_summary)
+    return opportunity_summaries, cursor, more
+
+
+def _build_exp_id_to_translation_suggestion_in_review_count(
+        exp_ids, language_code):
+    """Returns a dict mapping exploration ID to the count of corresponding
+    translation suggestions that are currently in review.
+
+    Args:
+        exp_ids: list(str). List of exploration IDs for which to count
+            corresponding translations suggestions.
+        language_code: str. The language for which translation suggestions
+            should be fetched.
+
+    Returns:
+        dict(str, int). Dict of exploration IDs to counts of corresponding
+        translation suggestions currently in review.
+    """
+    exp_id_to_in_review_count = collections.defaultdict(int)
+    suggestions_in_review = (
+        suggestion_services
+        .get_translation_suggestions_in_review_by_exp_ids(
+            exp_ids, language_code))
+    for suggestion in suggestions_in_review:
+        if suggestion is not None:
+            exp_id_to_in_review_count[suggestion.target_id] += 1
+    return exp_id_to_in_review_count

 def get_voiceover_opportunities(language_code, cursor):
diff --git a/core/domain/suggestion_services.py b/core/domain/suggestion_services.py
index a758fe622a5c..9c49486a6b3b 100644
--- a/core/domain/suggestion_services.py
+++ b/core/domain/suggestion_services.py
@@ -680,6 +680,31 @@ def get_translation_suggestions_in_review_by_exploration(exp_id, language_code):
     ]

+def get_translation_suggestions_in_review_by_exp_ids(exp_ids, language_code):
+    """Returns translation suggestions in review by exploration ID and language
+    code.
+
+    Args:
+        exp_ids: str. Exploration IDs matching the target ID of the
+            translation suggestions.
+        language_code: str. The ISO 639-1 language code of the translation
+            suggestions.
+
+    Returns:
+        list(Suggestion). A list of translation suggestions in review with
+        target_id in exp_ids and language_code == language_code.
+    """
+    suggestion_models_in_review = (
+        suggestion_models.GeneralSuggestionModel
+        .get_in_review_translation_suggestions_by_exp_ids(
+            exp_ids, language_code)
+    )
+    return [
+        get_suggestion_from_model(model) if model else None
+        for model in suggestion_models_in_review
+    ]
+
+
 def _get_plain_text_from_html_content_string(html_content_string):
     """Retrieves the plain text from the given html content string. RTE element
     occurrences in the html are replaced by their corresponding rte component
diff --git a/core/storage/suggestion/gae_models.py b/core/storage/suggestion/gae_models.py
index cdd00eccb427..fa7917085d25 100644
--- a/core/storage/suggestion/gae_models.py
+++ b/core/storage/suggestion/gae_models.py
@@ -485,6 +485,30 @@ def get_in_review_translation_suggestions(
             cls.language_code.IN(language_codes)
         )).fetch(feconf.DEFAULT_SUGGESTION_QUERY_LIMIT)

+    @classmethod
+    def get_in_review_translation_suggestions_by_exp_ids(
+        cls, exp_ids: List[str], language_code: str
+    ) -> Sequence[GeneralSuggestionModel]:
+        """Gets all in-review translation suggestions matching the supplied
+        exp_ids and language_code.
+
+        Args:
+            exp_ids: list(str). Exploration IDs matching the target ID of the
+                translation suggestions.
+            language_code: str. The ISO 639-1 language code of the translation
+                suggestions.
+
+        Returns:
+            list(SuggestionModel). A list of suggestions matching the supplied
+            exp_ids and language_code.
+        """
+        return cls.get_all().filter(datastore_services.all_of(
+            cls.status == STATUS_IN_REVIEW,
+            cls.suggestion_type == feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT,
+            cls.target_id.IN(exp_ids),
+            cls.language_code == language_code
+        )).fetch(feconf.DEFAULT_SUGGESTION_QUERY_LIMIT)
+
     @classmethod
     def get_in_review_question_suggestions(
         cls, user_id: str
diff --git a/core/templates/pages/contributor-dashboard-page/opportunities-list-item/opportunities-list-item.component.html b/core/templates/pages/contributor-dashboard-page/opportunities-list-item/opportunities-list-item.component.html
index bcf8aa7073f9..96e213bf9f3a 100644
--- a/core/templates/pages/contributor-dashboard-page/opportunities-list-item/opportunities-list-item.component.html
+++ b/core/templates/pages/contributor-dashboard-page/opportunities-list-item/opportunities-list-item.component.html
@@ -22,7 +22,7 @@

-

+

       <lazy-loading [hidden]="!opportunityDataIsLoading"></lazy-loading>

@@ -31,7 +31,7 @@
             *ngIf="translationProgressBar"
             [hidden]="opportunityDataIsLoading"
             [disableTooltip]="!translationProgressBar"
-            ngbTooltip="Percentage of accepted translated cards. {{opportunity.totalCount - (opportunity.translationsCount + opportunity.inReviewCount)}}/{{opportunity.totalCount}} cards remaining."
+            ngbTooltip="Percentage of accepted translated cards"
             placement="bottom">
         ({{ progressPercentage }})

diff --git a/core/domain/opportunity_services_test.py b/core/domain/opportunity_services_test.py
index e09248ae769b..680026817561 100644
--- a/core/domain/opportunity_services_test.py
+++ b/core/domain/opportunity_services_test.py
@@ -191,35 +191,33 @@ def test_get_translation_opportunities_with_translations_in_review(
             opportunity_services.get_translation_opportunities(
                 'hi', 'topic', None))
         self.assertEqual(len(translation_opportunities), 0)
-
         self.add_exploration_0_to_story()
         self.create_translation_suggestion_for_exploration_0_and_verify()

         translation_opportunities, _, _ = (
             opportunity_services.get_translation_opportunities(
                 'hi', 'topic', None))
+
         self.assertEqual(len(translation_opportunities), 1)
         opportunity = translation_opportunities[0]
-        languages_of_translations_in_review = (
-            opportunity.translation_in_review_counts.keys())
-        self.assertEqual(len(languages_of_translations_in_review), 1)
+        self.assertEqual(
+            opportunity.translation_in_review_counts,
+            {'hi': 1})

     def test_get_translation_opportunities_with_no_translations_in_review(self):
         translation_opportunities, _, _ = (
             opportunity_services.get_translation_opportunities(
                 'hi', 'topic', None))
         self.assertEqual(len(translation_opportunities), 0)
-
         self.add_exploration_0_to_story()

         translation_opportunities, _, _ = (
             opportunity_services.get_translation_opportunities(
                 'hi', 'topic', None))
+
         self.assertEqual(len(translation_opportunities), 1)
         opportunity = translation_opportunities[0]
-        languages_of_translations_in_review = (
-            opportunity.translation_in_review_counts.keys())
-        self.assertEqual(len(languages_of_translations_in_review), 0)
+        self.assertEqual(opportunity.translation_in_review_counts, {})

     def test_opportunity_get_deleted_with_removing_exploration_from_story_node(
             self):

Slow load times for translation Opportunities
<!--
  - Thanks for taking the time to report a bug in the Oppia project.
  - Before filing a new issue, please do a quick search to check that it hasn't
  - already been filed on the [issue tracker](https://github.com/oppia/oppia/issues)._
  -->

**Describe the bug**
opportunitiessummaryhandler/translation takes 6-7 seconds to execute on production

**To Reproduce**
Steps to reproduce the behavior:
 1. Go to 'Oppia.org'
 2. Click on 'Translate text'
 3. Open network tab in inspector
 4. Select a language/ or click on the pagination icons
 4. See error

**Observed behavior**
Slow load times for translation opportunities

**Expected behavior**
Should have similar load times to question opportunities which are aprx 400 ms to 1 second vs 6- 7 seconds we are seeing here

**Screenshots**
![Screen Shot 2021-12-01 at 12 56 27 PM](https://user-images.githubusercontent.com/35385366/144312712-8388cf71-29a3-4000-83fa-0c93396f035d.png)

Seeing the same issue for displaying reviewable suggestions, which is probably due to fetching all available reviewable suggestions up to the query limit of 1000 in core/storage/suggestion/gae_models.py.
@sagangwee Hmm, It's strange when i tested this change I had about 200 suggestions, and it seemed to display pretty quickly. Can you post a screenshot of the network tab when loading the reviewable suggestions.

Sure, thing:

<img width="762" alt="Screen Shot 2021-12-16 at 5 58 27 PM" src="https://user-images.githubusercontent.com/9004726/146461417-fd1c09bc-d0d2-4386-bb38-458db5c12c7a.png">

That's for what I believe is the max 1000 suggestions.

The slow translation opportunity load times are probably due to the translations in review calculation in [opportunity_services.py.get_exp_opportunity_summary_with_in_review_translations_from_model](https://github.com/oppia/oppia/blame/develop/core/domain/opportunity_services.py#L96)
since we are iterating through all the translations_in_review for every opportunity. Original change in https://github.com/oppia/oppia/pull/13835.

Specifically, we:
1. Fetch the IDs of all translation suggestions in review matching the opportunity IDs.
2. Fetch all suggestions from the IDs found in 1.
3. For every opportunity, iterate through all the suggestions from 2 and update the opportunity's in review counts.

Optimizations:
1. We can probably combine the suggestion fetches in 1 and 2 into on call. We also only care about the current language code, so we can apply a filter on language as well.
2. Instead of iterating through all the current in review suggestions for every opportunity in step 3, we can iterate through the suggestions and create a mapping of exploration ID -> in review suggestion count and use the mapping to update the opportunities.
Will fix as part of https://github.com/oppia/oppia/issues/7571.


diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst
index d0cd7d6f7e2..432cb095537 100644
--- a/docs/troubleshooting.rst
+++ b/docs/troubleshooting.rst
@@ -72,5 +72,127 @@ Restart your interpreter or notebook kernel.
 Avoid starting many Modin notebooks or interpreters in quick succession. Wait 2-3
 seconds before starting the next one.

+Importing heterogeneous data by ``read_csv``
+""""""""""""""""""""""""""""""""""""""""""""
+
+Since Modin ``read_csv`` imports data in parallel, it can occur that data read by
+different partitions can have different type (this happens when columns contains
+heterogeneous data, i.e. column values are of different types), which are handled
+differntly. Example of such behaviour is shown below.
+
+.. code-block:: python
+
+  import os
+  import pandas
+  import modin.pandas as pd
+  from modin.config import NPartitions
+
+  NPartitions.put(2)
+
+  test_filename = "test.csv"
+  # data with heterogeneous values in the first column
+  data = """one,2
+  3,4
+  5,6
+  7,8
+  9.0,10
+  """
+  kwargs = {
+      # names of the columns to set, if `names` parameter is set,
+      # header inffering from the first data row/rows will be disabled
+      "names": ["col1", "col2"],
+
+      # explicit setting of data type of column/columns with heterogeneous
+      # data will force partitions to read data with correct dtype
+      # "dtype": {"col1": str},
+  }
+
+
+  try :
+      with open(test_filename, "w") as f:
+          f.write(data)
+
+      pandas_df = pandas.read_csv(test_filename, **kwargs)
+      pd_df = pd.read_csv(test_filename, **kwargs)
+
+      print(pandas_df)
+      print(pd_df)
+  finally:
+      os.remove(test_filename)
+
+  Output:
+
+  pandas_df:
+    col1  col2
+  0  one     2
+  1    3     4
+  2    5     6
+  3    7     8
+  4  9.0    10
+
+  pd_df:
+    col1  col2
+  0  one     2
+  1    3     4
+  2    5     6
+  3  7.0     8
+  4  9.0    10
+
+
+In this case `DataFrame` read by pandas in the column ``col1`` contain only ``str`` data
+because of the first string value ("one"), that forced pandas to handle full column
+data as strings. Modin the fisrt partition (the first three rows) read data similary
+to pandas, but the second partition (the last two rows) doesn't contain any strings
+in the first column and it's data is read as floats because of the last column
+value and as a result `7` value was read as `7.0`, that differs from pandas output.
+
+The above example showed the mechanism of occurence of pandas and Modin ``read_csv``
+outputs discrepancy during heterogeneous data import. Please note, that similar
+situations can occur during different data/parameters combinations.
+
+**Solution**
+
+In the case if heterogeneous data is detected, corresponding warning will be showed in
+the user's console. Currently, the discrepancies of such type doesn't properly handled
+by Modin, and to avoid this issue, it is needed to set ``dtype`` parameter of ``read_csv``
+function manually to force correct data type definition during data import by
+partitions. Note, that to avoid excessive performance degradation, ``dtype`` value should
+be set fine-grained as it possible (specify ``dtype`` parameter only for columns with
+heterogeneous data).
+
+Setting of ``dtype`` parameter works well for most of the cases, but, unfortunetely, it is
+ineffective if data file contain column which should be interpreted as index
+(``index_col`` parameter is used) since ``dtype`` parameter is responsible only for data
+fields. For example, if in the above example, ``kwargs`` will be set in the next way:
+
+.. code-block:: python
+
+  kwargs = {
+      "names": ["col1", "col2"],
+      "dtype": {"col1": str},
+      "index_col": "col1",
+  }
+
+Resulting Modin DataFrame will contain incorrect value as in the case if ``dtype``
+is not set:
+
+.. code-block:: python
+
+  col1
+  one      2
+  3        4
+  5        6
+  7.0      8
+  9.0     10
+
+In this case data should be imported without setting of ``index_col`` parameter
+and only then index column should be set as index (by using ``DataFrame.set_index``
+funcion for example) as it is shown in the example below:
+
+.. code-block:: python
+
+  pd_df = pd.read_csv(filename, dtype=data_dtype, index_col=None)
+  pd_df = pd_df.set_index(index_col_name)
+  pd_df.index.name = None

 .. _issue: https://github.com/modin-project/modin/issues
diff --git a/modin/backends/pandas/parsers.py b/modin/backends/pandas/parsers.py
index 7173e810cfc..336a2f978dc 100644
--- a/modin/backends/pandas/parsers.py
+++ b/modin/backends/pandas/parsers.py
@@ -149,18 +149,36 @@ def get_dtypes(cls, dtypes_ids):

         Returns
         -------
-        pandas.Series
-            pandas.Series where index is columns names and values are
-            columns dtypes.
+        frame_dtypes : pandas.Series or dtype
+            Resulting dtype or pandas.Series where column names are used as
+            index and types of columns are used as values for full resulting
+            frame.
         """
+        # each element in `partitions_dtypes` is a Series, where column names are
+        # used as index and types of columns for different partitions are used as values
         partitions_dtypes = cls.materialize(dtypes_ids)
         if all([len(dtype) == 0 for dtype in partitions_dtypes]):
             return None
-        return (
-            pandas.concat(partitions_dtypes, axis=1)
-            .apply(lambda row: find_common_type_cat(row.values), axis=1)
-            .squeeze(axis=0)
-        )
+
+        combined_part_dtypes = pandas.concat(partitions_dtypes, axis=1)
+        frame_dtypes = combined_part_dtypes.iloc[:, 0]
+
+        if not combined_part_dtypes.eq(frame_dtypes, axis=0).all(axis=None):
+            ErrorMessage.missmatch_with_pandas(
+                operation="read_*",
+                message="Data types of partitions are different! "
+                "Please refer to the troubleshooting section of the Modin documentation "
+                "to fix this issue",
+            )
+
+            # concat all elements of `partitions_dtypes` and find common dtype
+            # for each of the column among all partitions
+            frame_dtypes = combined_part_dtypes.apply(
+                lambda row: find_common_type_cat(row.values),
+                axis=1,
+            ).squeeze(axis=0)
+
+        return frame_dtypes

     @classmethod
     def single_worker_read(cls, fname, **kwargs):

Warn user about heterogeneous data presence during `read_csv` call and propose workaround
Subpart of https://github.com/modin-project/modin/issues/3278.


diff --git a/securedrop_client/gui/widgets.py b/securedrop_client/gui/widgets.py
index 21209e303..ccbe4d443 100644
--- a/securedrop_client/gui/widgets.py
+++ b/securedrop_client/gui/widgets.py
@@ -709,10 +709,15 @@ def show_sources(self, sources: List[Source]):
             self.empty_conversation_view.show_no_sources_message()
             self.empty_conversation_view.show()

-        deleted_sources = self.source_list.update(sources)
-        for source_uuid in deleted_sources:
-            # Then call the function to remove the wrapper and its children.
-            self.delete_conversation(source_uuid)
+        if self.source_list.source_widgets:
+            # The source list already contains sources.
+            deleted_sources = self.source_list.update(sources)
+            for source_uuid in deleted_sources:
+                # Then call the function to remove the wrapper and its children.
+                self.delete_conversation(source_uuid)
+        else:
+            # We have an empty source list, so do an initial update.
+            self.source_list.initial_update(sources)

     def on_source_changed(self):
         """
@@ -977,6 +982,43 @@ def update(self, sources: List[Source]) -> List[str]:

         return deleted_uuids

+    def initial_update(self, sources: List[Source]):
+        """
+        Initialise the list with the passed in list of sources.
+        """
+        self.add_source(sources)
+
+    def add_source(self, sources, slice_size=1):
+        """
+        Add a slice of sources, and if necessary, reschedule the addition of
+        more sources.
+        """
+
+        def schedule_source_management(slice_size=slice_size):
+            if not sources:
+                # Nothing more to do.
+                return
+            # Process the remaining "slice_size" number of sources.
+            sources_slice = sources[:slice_size]
+            for source in sources_slice:
+                new_source = SourceWidget(self.controller, source)
+                self.source_widgets[source.uuid] = new_source
+                list_item = QListWidgetItem(self)
+                list_item.setSizeHint(new_source.sizeHint())
+
+                self.insertItem(0, list_item)
+                self.setItemWidget(list_item, new_source)
+            # ATTENTION! 32 is an arbitrary number arrived at via
+            # experimentation. It adds plenty of sources, but doesn't block
+            # for a noticable amount of time.
+            new_slice_size = min(32, slice_size * 2)
+            # Call add_source again for the remaining sources.
+            self.add_source(sources[slice_size:], new_slice_size)
+
+        # Schedule the closure defined above in the next iteration of the
+        # Qt event loop (thus unblocking the UI).
+        QTimer.singleShot(1, schedule_source_management)
+
     def get_selected_source(self):
         if not self.selectedItems():
             return None
diff --git a/securedrop_client/logic.py b/securedrop_client/logic.py
index 38f79d837..694339a01 100644
--- a/securedrop_client/logic.py
+++ b/securedrop_client/logic.py
@@ -520,8 +520,6 @@ def update_sources(self):
         Display the updated list of sources with those found in local storage.
         """
         sources = list(storage.get_local_sources(self.session))
-        if sources:
-            sources.sort(key=lambda x: x.last_updated)
         self.gui.show_sources(sources)

     def on_update_star_success(self, source_uuid: str) -> None:
diff --git a/securedrop_client/storage.py b/securedrop_client/storage.py
index 8c1707933..667859faa 100644
--- a/securedrop_client/storage.py
+++ b/securedrop_client/storage.py
@@ -27,7 +27,7 @@
 from dateutil.parser import parse
 from typing import List, Tuple, Type, Union

-from sqlalchemy import and_, or_
+from sqlalchemy import and_, desc, or_
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.orm.session import Session

@@ -44,9 +44,9 @@

 def get_local_sources(session: Session) -> List[Source]:
     """
-    Return all source objects from the local database.
+    Return all source objects from the local database, newest first.
     """
-    return session.query(Source).all()
+    return session.query(Source).order_by(desc(Source.last_updated)).all()

 def delete_local_source_by_uuid(session: Session, uuid: str, data_dir: str) -> None:

diff --git a/tests/functional/test_download_file.py b/tests/functional/test_download_file.py
index cba5409fe..9a5e02d98 100644
--- a/tests/functional/test_download_file.py
+++ b/tests/functional/test_download_file.py
@@ -28,7 +28,7 @@ def check_for_sources():

     qtbot.waitUntil(check_for_sources, timeout=10000)
     source_ids = list(gui.main_view.source_list.source_widgets.keys())
-    first_source_id = source_ids[2]
+    first_source_id = source_ids[0]
     first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)

diff --git a/tests/functional/test_export_dialog.py b/tests/functional/test_export_dialog.py
index 7d49b6335..fb76d3e97 100644
--- a/tests/functional/test_export_dialog.py
+++ b/tests/functional/test_export_dialog.py
@@ -28,7 +28,7 @@ def check_for_sources():

     qtbot.waitUntil(check_for_sources, timeout=10000)
     source_ids = list(gui.main_view.source_list.source_widgets.keys())
-    first_source_id = source_ids[2]
+    first_source_id = source_ids[0]
     first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)

diff --git a/tests/functional/test_receive_message.py b/tests/functional/test_receive_message.py
index 7520cf1e4..8014f2287 100644
--- a/tests/functional/test_receive_message.py
+++ b/tests/functional/test_receive_message.py
@@ -28,7 +28,7 @@ def check_for_sources():

     qtbot.waitUntil(check_for_sources, timeout=10000)
     source_ids = list(gui.main_view.source_list.source_widgets.keys())
-    first_source_id = source_ids[2]
+    first_source_id = source_ids[0]
     first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)

diff --git a/tests/functional/test_star_source.py b/tests/functional/test_star_source.py
index 0e2c5235c..cd3cf711f 100644
--- a/tests/functional/test_star_source.py
+++ b/tests/functional/test_star_source.py
@@ -26,7 +26,7 @@ def check_for_sources():

     qtbot.waitUntil(check_for_sources, timeout=10000)
     source_ids = list(gui.main_view.source_list.source_widgets.keys())
-    first_source_id = source_ids[1]
+    first_source_id = source_ids[0]
     first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)

diff --git a/tests/functional/test_unstar_source.py b/tests/functional/test_unstar_source.py
index b8fe81018..a700972e3 100644
--- a/tests/functional/test_unstar_source.py
+++ b/tests/functional/test_unstar_source.py
@@ -26,7 +26,7 @@ def check_for_sources():

     qtbot.waitUntil(check_for_sources, timeout=10000)
     source_ids = list(gui.main_view.source_list.source_widgets.keys())
-    first_source_id = source_ids[1]
+    first_source_id = source_ids[0]
     first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)

diff --git a/tests/gui/test_widgets.py b/tests/gui/test_widgets.py
index e4eb17fe7..f0666c0a9 100644
--- a/tests/gui/test_widgets.py
+++ b/tests/gui/test_widgets.py
@@ -11,6 +11,7 @@
 from PyQt5.QtTest import QTest
 from PyQt5.QtWidgets import QWidget, QApplication, QVBoxLayout, QMessageBox, QMainWindow, \
     QLineEdit
+import sqlalchemy
 import sqlalchemy.orm.exc
 from sqlalchemy.orm import attributes, scoped_session, sessionmaker

@@ -496,6 +497,20 @@ def test_MainView_show_sources_with_none_selected(mocker):
     mv.empty_conversation_view.show.assert_called_once_with()

+def test_MainView_show_sources_from_cold_start(mocker):
+    """
+    Ensure the initial_update is called if no sources exist in the UI.
+    """
+    mv = MainView(None)
+    mv.source_list = mocker.MagicMock()
+    mv.source_list.source_widgets = []
+    mv.empty_conversation_view = mocker.MagicMock()
+
+    mv.show_sources([])
+
+    mv.source_list.initial_update.assert_called_once_with([])
+
+
 def test_MainView_show_sources_with_no_sources_at_all(mocker):
     """
     Ensure the sources list is passed to the source list widget to be updated.
@@ -770,6 +785,24 @@ def test_SourceList_update_adds_new_sources(mocker):
     mock_sw.deleteLater.assert_not_called()

+def test_SourceList_initial_update_adds_new_sources(mocker):
+    """
+    Check a new SourceWidget for each passed-in source is created and no widgets are cleared or
+    removed.
+    """
+    sl = SourceList()
+
+    sl.clear = mocker.MagicMock()
+    sl.add_source = mocker.MagicMock()
+    sl.setItemWidget = mocker.MagicMock()
+    sl.currentRow = mocker.MagicMock(return_value=0)
+    sl.item = mocker.MagicMock()
+    sl.item().isSelected.return_value = True
+    sources = [mocker.MagicMock(), mocker.MagicMock(), mocker.MagicMock(), ]
+    sl.initial_update(sources)
+    sl.add_source.assert_called_once_with(sources)
+
+
 def test_SourceList_update_when_source_deleted(mocker, session, session_maker, homedir):
     """
     Test that SourceWidget.update raises an exception when its source has been deleted.
@@ -810,6 +843,19 @@ def test_SourceList_update_when_source_deleted(mocker, session, session_maker, h
     assert len(sl.source_widgets) == 0

+def test_SourceList_add_source_starts_timer(mocker, session_maker, homedir):
+    """
+    When the add_source method is called it schedules the addition of a source
+    to the source list via a single-shot QTimer.
+    """
+    sl = SourceList()
+    sources = [mocker.MagicMock(), mocker.MagicMock(), mocker.MagicMock(), ]
+    mock_timer = mocker.MagicMock()
+    with mocker.patch("securedrop_client.gui.widgets.QTimer", mock_timer):
+        sl.add_source(sources)
+    assert mock_timer.singleShot.call_count == 1
+
+
 def test_SourceList_update_when_source_deleted_crash(mocker, session, session_maker, homedir):
     """
     When SourceList.update calls SourceWidget.update and that
@@ -949,6 +995,76 @@ def test_SourceList_update_removes_item_from_beginning_of_list(mocker):
     assert len(sl.source_widgets) == 2

+def test_SourceList_add_source_closure_adds_sources(mocker):
+    """
+    The closure (function created within the add_source function) behaves in
+    the expected way given the context of the call to add_source.
+    """
+    sl = SourceList()
+    sl.controller = mocker.MagicMock()
+    sl.insertItem = mocker.MagicMock()
+    sl.setItemWidget = mocker.MagicMock()
+    sl.setCurrentItem = mocker.MagicMock()
+
+    mock_sw = mocker.MagicMock()
+    mock_lwi = mocker.MagicMock()
+    mocker.patch('securedrop_client.gui.widgets.SourceWidget', mock_sw)
+    mocker.patch('securedrop_client.gui.widgets.QListWidgetItem', mock_lwi)
+    sources = [mocker.MagicMock(), mocker.MagicMock(), mocker.MagicMock(), ]
+    mock_timer = mocker.MagicMock()
+    with mocker.patch("securedrop_client.gui.widgets.QTimer", mock_timer):
+        sl.add_source(sources, 1)
+        # Now grab the function created within add_source:
+        inner_fn = mock_timer.singleShot.call_args_list[0][0][1]
+        # Ensure add_source is mocked to avoid recursion in the test and so we
+        # can spy on how it's called.
+        sl.add_source = mocker.MagicMock()
+        # Call the inner function (as if the timer had completed).
+        inner_fn()
+        assert mock_sw.call_count == 1
+        assert mock_lwi.call_count == 1
+        assert sl.insertItem.call_count == 1
+        assert sl.setItemWidget.call_count == 1
+        assert len(sl.source_widgets) == 1
+        assert sl.setCurrentItem.call_count == 0
+        sl.add_source.assert_called_once_with(sources[1:], 2)
+
+
+def test_SourceList_add_source_closure_exits_on_no_more_sources(mocker):
+    """
+    The closure (function created within the add_source function) behaves in
+    the expected way given the context of the call to add_source.
+    """
+    sl = SourceList()
+    sl.controller = mocker.MagicMock()
+    sl.addItem = mocker.MagicMock()
+    sl.setItemWidget = mocker.MagicMock()
+    sl.setCurrentItem = mocker.MagicMock()
+
+    mock_sw = mocker.MagicMock()
+    mock_lwi = mocker.MagicMock()
+    mocker.patch('securedrop_client.gui.widgets.SourceWidget', mock_sw)
+    mocker.patch('securedrop_client.gui.widgets.QListWidgetItem', mock_lwi)
+    sources = []
+    mock_timer = mocker.MagicMock()
+    with mocker.patch("securedrop_client.gui.widgets.QTimer", mock_timer):
+        sl.add_source(sources, 1)
+        # Now grab the function created within add_source:
+        inner_fn = mock_timer.singleShot.call_args_list[0][0][1]
+        # Ensure add_source is mocked to avoid recursion in the test and so we
+        # can spy on how it's called.
+        sl.add_source = mocker.MagicMock()
+        # Call the inner function (as if the timer had completed).
+        assert inner_fn() is None
+        assert mock_sw.call_count == 0
+        assert mock_lwi.call_count == 0
+        assert sl.addItem.call_count == 0
+        assert sl.setItemWidget.call_count == 0
+        assert len(sl.source_widgets) == 0
+        assert sl.setCurrentItem.call_count == 0
+        assert sl.add_source.call_count == 0
+
+
 def test_SourceList_set_snippet(mocker):
     """
     Handle the emitted event in the expected manner.
@@ -1042,6 +1158,26 @@ def test_SourceWidget_update_attachment_icon(mocker):
     assert sw.paperclip.isHidden()

+def test_SourceWidget_update_raises_InvalidRequestError(mocker):
+    """
+    If the source no longer exists in the local data store, ensure the expected
+    exception is logged and re-raised.
+    """
+    controller = mocker.MagicMock()
+    source = factory.Source(document_count=1)
+    sw = SourceWidget(controller, source)
+    ex = sqlalchemy.exc.InvalidRequestError()
+    controller.session.refresh.side_effect = ex
+    mock_logger = mocker.MagicMock()
+    mocker.patch(
+        "securedrop_client.gui.widgets.logger",
+        mock_logger,
+    )
+    with pytest.raises(sqlalchemy.exc.InvalidRequestError):
+        sw.update()
+        assert mock_logger.error.call_count == 1
+
+
 def test_SourceWidget_set_snippet(mocker, session_maker, session, homedir):
     """
     Snippets are set as expected.
diff --git a/tests/test_storage.py b/tests/test_storage.py
index 7c8c55fe5..ab67307c5 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -203,7 +203,9 @@ def test_update_local_storage(homedir, mocker, session_maker):
     local_reply = mocker.MagicMock()
     mock_session.query().all = mocker.Mock()
     mock_session.query().all.side_effect = [
-        [local_source], [local_file], [local_message], [local_reply]]
+        [local_file], [local_message], [local_reply]]
+    mock_session.query().order_by().all = mocker.Mock()
+    mock_session.query().order_by().all.side_effect = [[local_source]]
     src_fn = mocker.patch('securedrop_client.storage.update_sources')
     rpl_fn = mocker.patch('securedrop_client.storage.update_replies')
     file_fn = mocker.patch('securedrop_client.storage.update_files')

Client starts with blacked out / blocked UI with many sources
Steps to reproduce:

1. Create a server with 1000 source (`$ NUM_SOURCES=1000 make dev`)
2. Start the client and let it download the sources.
3. Stop the client.
4. Restart the client.

At step 4 the drawing of the UI appears to be blocked (I speculate this is perhaps the client loading locally stored sources and associated messages in a blocking manner). Once the UI is drawn it is completely responsive. See the following screenie for an example of this behaviour:

![blocked_startup](https://user-images.githubusercontent.com/37602/75795529-5e67d100-5d6a-11ea-8f48-a904833df6e6.gif)