text:
import functools
import io
import logging
import pickle
import warnings
from queue import Queue
from unittest import TestCase

import pytest
import torch
import torch.nn as nn

import pyro
import pyro.distributions as dist
import pyro.poutine as poutine
from pyro.distributions import Bernoulli, Categorical, Normal
from pyro.poutine.runtime import _DIM_ALLOCATOR, NonlocalExit
from pyro.poutine.util import all_escape, discrete_escape
from tests.common import assert_close, assert_equal, assert_not_equal

logger = logging.getLogger(__name__)

def eq(x, y, prec=1e-10):
    return torch.norm(x - y).item() < prec

# XXX name is a bit silly
class NormalNormalNormalHandlerTestCase(TestCase):
    def setUp(self):
        pyro.clear_param_store()

        def model():
            latent1 = pyro.sample("latent1", Normal(torch.zeros(2), torch.ones(2)))
            latent2 = pyro.sample("latent2", Normal(latent1, 5 * torch.ones(2)))
            x_dist = Normal(latent2, torch.ones(2))
            pyro.sample("obs", x_dist, obs=torch.ones(2))
            return latent1

        def guide():
            loc1 = pyro.param("loc1", torch.randn(2, requires_grad=True))
            scale1 = pyro.param("scale1", torch.ones(2, requires_grad=True))
            pyro.sample("latent1", Normal(loc1, scale1))

            loc2 = pyro.param("loc2", torch.randn(2, requires_grad=True))
            scale2 = pyro.param("scale2", torch.ones(2, requires_grad=True))
            latent2 = pyro.sample("latent2", Normal(loc2, scale2))
            return latent2

        self.model = model
        self.guide = guide

        self.model_sites = ["latent1", "latent2", "obs", "_INPUT", "_RETURN"]

        self.guide_sites = [
            "latent1",
            "latent2",
            "loc1",
            "scale1",
            "loc2",
            "scale2",
            "_INPUT",
            "_RETURN",
        ]

        self.full_sample_sites = {"latent1": "latent1", "latent2": "latent2"}
        self.partial_sample_sites = {"latent1": "latent1"}

class TraceHandlerTests(NormalNormalNormalHandlerTestCase):
    def test_trace_full(self):
        guide_trace = poutine.trace(self.guide).get_trace()
        model_trace = poutine.trace(self.model).get_trace()
        for name in model_trace.nodes.keys():
            assert name in self.model_sites

        for name in guide_trace.nodes.keys():
            assert name in self.guide_sites
            assert guide_trace.nodes[name]["type"] in (
                "args",
                "return",
                "sample",
                "param",
            )
            if guide_trace.nodes[name]["type"] == "sample":
                assert not guide_trace.nodes[name]["is_observed"]

    def test_trace_return(self):
        model_trace = poutine.trace(self.model).get_trace()
        assert_equal(
            model_trace.nodes["latent1"]["value"], model_trace.nodes["_RETURN"]["value"]
        )

    def test_trace_param_only(self):
        model_trace = poutine.trace(self.model, param_only=True).get_trace()
        assert all(site["type"] == "param" for site in model_trace.nodes.values())

class ReplayHandlerTests(NormalNormalNormalHandlerTestCase):
    def test_replay_full(self):
        guide_trace = poutine.trace(self.guide).get_trace()
        model_trace = poutine.trace(
            poutine.replay(self.model, trace=guide_trace)
        ).get_trace()
        for name in self.full_sample_sites.keys():
            assert_equal(
                model_trace.nodes[name]["value"], guide_trace.nodes[name]["value"]
            )

    def test_replay_full_repeat(self):
        model_trace = poutine.trace(self.model).get_trace()
        ftr = poutine.trace(poutine.replay(self.model, trace=model_trace))
        tr11 = ftr.get_trace()
        tr12 = ftr.get_trace()
        tr2 = poutine.trace(poutine.replay(self.model, trace=model_trace)).get_trace()
        for name in self.full_sample_sites.keys():
            assert_equal(tr11.nodes[name]["value"], tr12.nodes[name]["value"])
            assert_equal(tr11.nodes[name]["value"], tr2.nodes[name]["value"])
            assert_equal(model_trace.nodes[name]["value"], tr11.nodes[name]["value"])
            assert_equal(model_trace.nodes[name]["value"], tr2.nodes[name]["value"])

class BlockHandlerTests(NormalNormalNormalHandlerTestCase):
    def test_block_hide_fn(self):
        model_trace = poutine.trace(
            poutine.block(
                self.model,
                hide_fn=lambda msg: "latent" in msg["name"],
                expose=["latent1"],
            )
        ).get_trace()
        assert "latent1" not in model_trace
        assert "latent2" not in model_trace
        assert "obs" in model_trace

    def test_block_expose_fn(self):
        model_trace = poutine.trace(
            poutine.block(
                self.model,
                expose_fn=lambda msg: "latent" in msg["name"],
                hide=["latent1"],
            )
        ).get_trace()
        assert "latent1" in model_trace
        assert "latent2" in model_trace
        assert "obs" not in model_trace

    def test_block_full(self):
        model_trace = poutine.trace(poutine.block(self.model)).get_trace()
        guide_trace = poutine.trace(poutine.block(self.guide)).get_trace()
        for name in model_trace.nodes.keys():
            assert model_trace.nodes[name]["type"] in ("args", "return")
        for name in guide_trace.nodes.keys():
            assert guide_trace.nodes[name]["type"] in ("args", "return")

    def test_block_full_hide(self):
        model_trace = poutine.trace(
            poutine.block(self.model, hide=self.model_sites)
        ).get_trace()
        guide_trace = poutine.trace(
            poutine.block(self.guide, hide=self.guide_sites)
        ).get_trace()
        for name in model_trace.nodes.keys():
            assert model_trace.nodes[name]["type"] in ("args", "return")
        for name in guide_trace.nodes.keys():
            assert guide_trace.nodes[name]["type"] in ("args", "return")

    def test_block_full_expose(self):
        model_trace = poutine.trace(
            poutine.block(self.model, expose=self.model_sites)
        ).get_trace()
        guide_trace = poutine.trace(
            poutine.block(self.guide, expose=self.guide_sites)
        ).get_trace()
        for name in self.model_sites:
            assert name in model_trace
        for name in self.guide_sites:
            assert name in guide_trace

    def test_block_full_hide_expose(self):
        try:
            poutine.block(
                self.model,
                hide=self.partial_sample_sites.keys(),
                expose=self.partial_sample_sites.keys(),
            )()
            assert False
        except AssertionError:
            assert True

    def test_block_partial_hide(self):
        model_trace = poutine.trace(
            poutine.block(self.model, hide=self.partial_sample_sites.keys())
        ).get_trace()
        guide_trace = poutine.trace(
            poutine.block(self.guide, hide=self.partial_sample_sites.keys())
        ).get_trace()
        for name in self.full_sample_sites.keys():
            if name in self.partial_sample_sites:
                assert name not in model_trace
                assert name not in guide_trace
            else:
                assert name in model_trace
                assert name in guide_trace

    def test_block_partial_expose(self):
        model_trace = poutine.trace(
            poutine.block(self.model, expose=self.partial_sample_sites.keys())
        ).get_trace()
        guide_trace = poutine.trace(
            poutine.block(self.guide, expose=self.partial_sample_sites.keys())
        ).get_trace()
        for name in self.full_sample_sites.keys():
            if name in self.partial_sample_sites:
                assert name in model_trace
                assert name in guide_trace
            else:
                assert name not in model_trace
                assert name not in guide_trace

    def test_block_tutorial_case(self):
        model_trace = poutine.trace(self.model).get_trace()
        guide_trace = poutine.trace(
            poutine.block(self.guide, hide_types=["observe"])
        ).get_trace()

        assert "latent1" in model_trace
        assert "latent1" in guide_trace
        assert "obs" in model_trace
        assert "obs" not in guide_trace

class QueueHandlerDiscreteTest(TestCase):
    def setUp(self):

        # simple Gaussian-mixture HMM
        def model():
            probs = pyro.param("probs", torch.tensor([[0.8], [0.3]]))
            loc = pyro.param("loc", torch.tensor([[-0.1], [0.9]]))
            scale = torch.ones(1, 1)

            latents = [torch.ones(1)]
            observes = []
            for t in range(3):

                latents.append(
                    pyro.sample(
                        "latent_{}".format(str(t)),
                        Bernoulli(probs[latents[-1][0].long().data]),
                    )
                )

                observes.append(
                    pyro.sample(
                        "observe_{}".format(str(t)),
                        Normal(loc[latents[-1][0].long().data], scale),
                        obs=torch.ones(1),
                    )
                )
            return latents

        self.sites = (
            ["observe_{}".format(str(t)) for t in range(3)]
            + ["latent_{}".format(str(t)) for t in range(3)]
            + ["_INPUT", "_RETURN"]
        )
        self.model = model
        self.queue = Queue()
        self.queue.put(poutine.Trace())

    def test_queue_single(self):
        f = poutine.trace(poutine.queue(self.model, queue=self.queue))
        tr = f.get_trace()
        for name in self.sites:
            assert name in tr

    def test_queue_enumerate(self):
        f = poutine.trace(poutine.queue(self.model, queue=self.queue))
        trs = []
        while not self.queue.empty():
            trs.append(f.get_trace())
        assert len(trs) == 2 ** 3

        true_latents = set()
        for i1 in range(2):
            for i2 in range(2):
                for i3 in range(2):
                    true_latents.add((i1, i2, i3))

        tr_latents = []
        for tr in trs:
            tr_latents.append(
                tuple(
                    [
                        int(tr.nodes[name]["value"].view(-1).item())
                        for name in tr
                        if tr.nodes[name]["type"] == "sample"
                        and not tr.nodes[name]["is_observed"]
                    ]
                )
            )

        assert true_latents == set(tr_latents)

    def test_queue_max_tries(self):
        f = poutine.queue(self.model, queue=self.queue, max_tries=3)
        with pytest.raises(ValueError):
            f()

class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc = nn.Linear(2, 1)

    def forward(self, x):
        return self.fc(x)

class LiftHandlerTests(TestCase):
    def setUp(self):
        pyro.clear_param_store()

        def loc1_prior(tensor, *args, **kwargs):
            flat_tensor = tensor.view(-1)
            m = torch.zeros(flat_tensor.size(0))
            s = torch.ones(flat_tensor.size(0))
            return Normal(m, s).sample().view(tensor.size())

        def scale1_prior(tensor, *args, **kwargs):
            flat_tensor = tensor.view(-1)
            m = torch.zeros(flat_tensor.size(0))
            s = torch.ones(flat_tensor.size(0))
            return Normal(m, s).sample().view(tensor.size()).exp()

        def loc2_prior(tensor, *args, **kwargs):
            flat_tensor = tensor.view(-1)
            m = torch.zeros(flat_tensor.size(0))
            return Bernoulli(m).sample().view(tensor.size())

        def scale2_prior(tensor, *args, **kwargs):
            return scale1_prior(tensor)

        def bias_prior(tensor, *args, **kwargs):
            return loc2_prior(tensor)

        def weight_prior(tensor, *args, **kwargs):
            return scale1_prior(tensor)

        def stoch_fn(tensor, *args, **kwargs):
            loc = torch.zeros(tensor.size())
            scale = torch.ones(tensor.size())
            return pyro.sample("sample", Normal(loc, scale))

        def guide():
            loc1 = pyro.param("loc1", torch.randn(2, requires_grad=True))
            scale1 = pyro.param("scale1", torch.ones(2, requires_grad=True))
            pyro.sample("latent1", Normal(loc1, scale1))

            loc2 = pyro.param("loc2", torch.randn(2, requires_grad=True))
            scale2 = pyro.param("scale2", torch.ones(2, requires_grad=True))
            latent2 = pyro.sample("latent2", Normal(loc2, scale2))
            return latent2

        def dup_param_guide():
            a = pyro.param("loc")
            b = pyro.param("loc")
            assert a == b

        self.model = Model()
        self.guide = guide
        self.dup_param_guide = dup_param_guide
        self.prior = scale1_prior
        self.prior_dict = {
            "loc1": loc1_prior,
            "scale1": scale1_prior,
            "loc2": loc2_prior,
            "scale2": scale2_prior,
        }
        self.partial_dict = {"loc1": loc1_prior, "scale1": scale1_prior}
        self.nn_prior = {"fc.bias": bias_prior, "fc.weight": weight_prior}
        self.fn = stoch_fn
        self.data = torch.randn(2, 2)

    def test_splice(self):
        tr = poutine.trace(self.guide).get_trace()
        lifted_tr = poutine.trace(
            poutine.lift(self.guide, prior=self.prior)
        ).get_trace()
        for name in tr.nodes.keys():
            if name in ("loc1", "loc2", "scale1", "scale2"):
                assert name not in lifted_tr
            else:
                assert name in lifted_tr

    def test_memoize(self):
        poutine.trace(poutine.lift(self.dup_param_guide, prior=dist.Normal(0, 1)))()

    def test_prior_dict(self):
        tr = poutine.trace(self.guide).get_trace()
        lifted_tr = poutine.trace(
            poutine.lift(self.guide, prior=self.prior_dict)
        ).get_trace()
        for name in tr.nodes.keys():
            assert name in lifted_tr
            if name in {"scale1", "loc1", "scale2", "loc2"}:
                assert name + "_prior" == lifted_tr.nodes[name]["fn"].__name__
            if tr.nodes[name]["type"] == "param":
                assert lifted_tr.nodes[name]["type"] == "sample"
                assert not lifted_tr.nodes[name]["is_observed"]

    def test_unlifted_param(self):
        tr = poutine.trace(self.guide).get_trace()
        lifted_tr = poutine.trace(
            poutine.lift(self.guide, prior=self.partial_dict)
        ).get_trace()
        for name in tr.nodes.keys():
            assert name in lifted_tr
            if name in ("scale1", "loc1"):
                assert name + "_prior" == lifted_tr.nodes[name]["fn"].__name__
                assert lifted_tr.nodes[name]["type"] == "sample"
                assert not lifted_tr.nodes[name]["is_observed"]
            if name in ("scale2", "loc2"):
                assert lifted_tr.nodes[name]["type"] == "param"

    @pytest.mark.filterwarnings("ignore::FutureWarning")
    def test_random_module(self):
        pyro.clear_param_store()
        with pyro.validation_enabled():
            lifted_tr = poutine.trace(
                pyro.random_module("name", self.model, prior=self.prior)
            ).get_trace()
        for name in lifted_tr.nodes.keys():
            if lifted_tr.nodes[name]["type"] == "param":
                assert lifted_tr.nodes[name]["type"] == "sample"
                assert not lifted_tr.nodes[name]["is_observed"]

    @pytest.mark.filterwarnings("ignore::FutureWarning")
    def test_random_module_warn(self):
        pyro.clear_param_store()
        bad_prior = {"foo": None}
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            with pyro.validation_enabled():
                poutine.trace(
                    pyro.random_module("name", self.model, prior=bad_prior)
                ).get_trace()
            assert len(w), "No warnings were raised"
            for warning in w:
                logger.info(warning)

    @pytest.mark.filterwarnings("ignore::FutureWarning")
    def test_random_module_prior_dict(self):
        pyro.clear_param_store()
        lifted_nn = pyro.random_module("name", self.model, prior=self.nn_prior)
        lifted_tr = poutine.trace(lifted_nn).get_trace()
        for key_name in lifted_tr.nodes.keys():
            name = pyro.params.user_param_name(key_name)
            if name in {"fc.weight", "fc.prior"}:
                dist_name = name[3:]
                assert dist_name + "_prior" == lifted_tr.nodes[key_name]["fn"].__name__
                assert lifted_tr.nodes[key_name]["type"] == "sample"
                assert not lifted_tr.nodes[key_name]["is_observed"]

class QueueHandlerMixedTest(TestCase):
    def setUp(self):

        # Simple model with 1 continuous + 1 discrete + 1 continuous variable.
        def model():
            p = torch.tensor([0.5])
            loc = torch.zeros(1)
            scale = torch.ones(1)

            x = pyro.sample("x", Normal(loc, scale))  # Before the discrete variable.
            y = pyro.sample("y", Bernoulli(p))
            z = pyro.sample("z", Normal(loc, scale))  # After the discrete variable.
            return dict(x=x, y=y, z=z)

        self.sites = ["x", "y", "z", "_INPUT", "_RETURN"]
        self.model = model
        self.queue = Queue()
        self.queue.put(poutine.Trace())

    def test_queue_single(self):
        f = poutine.trace(poutine.queue(self.model, queue=self.queue))
        tr = f.get_trace()
        for name in self.sites:
            assert name in tr

    def test_queue_enumerate(self):
        f = poutine.trace(poutine.queue(self.model, queue=self.queue))
        trs = []
        while not self.queue.empty():
            trs.append(f.get_trace())
        assert len(trs) == 2

        values = [
            {
                name: tr.nodes[name]["value"].view(-1).item()
                for name in tr.nodes.keys()
                if tr.nodes[name]["type"] == "sample"
            }
            for tr in trs
        ]

        expected_ys = set([0, 1])
        actual_ys = set([value["y"] for value in values])
        assert actual_ys == expected_ys

        # Check that x was sampled the same on all each paths.
        assert values[0]["x"] == values[1]["x"]

        # Check that y was sampled differently on each path.
        assert values[0]["z"] != values[1]["z"]  # Almost surely true.

class IndirectLambdaHandlerTests(TestCase):
    def setUp(self):
        def model(batch_size_outer=2, batch_size_inner=2):
            data = [[torch.ones(1)] * 2] * 2
            loc_latent = pyro.sample(
                "loc_latent", dist.Normal(torch.zeros(1), torch.ones(1))
            )
            for i in pyro.plate("plate_outer", 2, batch_size_outer):
                for j in pyro.plate("plate_inner_%d" % i, 2, batch_size_inner):
                    pyro.sample(
                        "z_%d_%d" % (i, j),
                        dist.Normal(loc_latent + data[i][j], torch.ones(1)),
                    )

        self.model = model
        self.expected_nodes = set(
            ["z_0_0", "z_0_1", "z_1_0", "z_1_1", "loc_latent", "_INPUT", "_RETURN"]
        )
        self.expected_edges = set(
            [
                ("loc_latent", "z_0_0"),
                ("loc_latent", "z_0_1"),
                ("loc_latent", "z_1_0"),
                ("loc_latent", "z_1_1"),
            ]
        )

    def test_graph_structure(self):
        tracegraph = poutine.trace(self.model, graph_type="dense").get_trace()
        # Ignore structure on plate_* nodes.
        actual_nodes = set(n for n in tracegraph.nodes if not n.startswith("plate_"))
        actual_edges = set(
            (n1, n2)
            for n1, n2 in tracegraph.edges
            if not n1.startswith("plate_")
            if not n2.startswith("plate_")
        )
        assert actual_nodes == self.expected_nodes
        assert actual_edges == self.expected_edges

    def test_scale_factors(self):
        def _test_scale_factor(batch_size_outer, batch_size_inner, expected):
            trace = poutine.trace(self.model, graph_type="dense").get_trace(
                batch_size_outer=batch_size_outer, batch_size_inner=batch_size_inner
            )
            scale_factors = []
            for node in ["z_0_0", "z_0_1", "z_1_0", "z_1_1"]:
                if node in trace:
                    scale_factors.append(trace.nodes[node]["scale"])
            assert scale_factors == expected

        _test_scale_factor(1, 1, [4.0])
        _test_scale_factor(2, 2, [1.0] * 4)
        _test_scale_factor(1, 2, [2.0] * 2)
        _test_scale_factor(2, 1, [2.0] * 2)

class ConditionHandlerTests(NormalNormalNormalHandlerTestCase):
    def test_condition(self):
        data = {"latent2": torch.randn(2)}
        tr2 = poutine.trace(poutine.condition(self.model, data=data)).get_trace()
        assert "latent2" in tr2
        assert (
            tr2.nodes["latent2"]["type"] == "sample"
            and tr2.nodes["latent2"]["is_observed"]
        )
        assert tr2.nodes["latent2"]["value"] is data["latent2"]

    def test_trace_data(self):
        tr1 = poutine.trace(
            poutine.block(self.model, expose_types=["sample"])
        ).get_trace()
        tr2 = poutine.trace(poutine.condition(self.model, data=tr1)).get_trace()
        assert (
            tr2.nodes["latent2"]["type"] == "sample"
            and tr2.nodes["latent2"]["is_observed"]
        )
        assert tr2.nodes["latent2"]["value"] is tr1.nodes["latent2"]["value"]

    def test_stack_overwrite_behavior(self):
        data1 = {"latent2": torch.randn(2)}
        data2 = {"latent2": torch.randn(2)}
        with poutine.trace() as tr:
            cm = poutine.condition(
                poutine.condition(self.model, data=data1), data=data2
            )
            cm()
        assert tr.trace.nodes["latent2"]["value"] is data2["latent2"]

    def test_stack_success(self):
        data1 = {"latent1": torch.randn(2)}
        data2 = {"latent2": torch.randn(2)}
        tr = poutine.trace(
            poutine.condition(poutine.condition(self.model, data=data1), data=data2)
        ).get_trace()
        assert (
            tr.nodes["latent1"]["type"] == "sample"
            and tr.nodes["latent1"]["is_observed"]
        )
        assert tr.nodes["latent1"]["value"] is data1["latent1"]
        assert (
            tr.nodes["latent2"]["type"] == "sample"
            and tr.nodes["latent2"]["is_observed"]
        )
        assert tr.nodes["latent2"]["value"] is data2["latent2"]

class UnconditionHandlerTests(NormalNormalNormalHandlerTestCase):
    def test_uncondition(self):
        unconditioned_model = poutine.uncondition(self.model)
        unconditioned_trace = poutine.trace(unconditioned_model).get_trace()
        conditioned_trace = poutine.trace(self.model).get_trace()
        assert_equal(conditioned_trace.nodes["obs"]["value"], torch.ones(2))
        assert_not_equal(unconditioned_trace.nodes["obs"]["value"], torch.ones(2))

    def test_undo_uncondition(self):
        unconditioned_model = poutine.uncondition(self.model)
        reconditioned_model = pyro.condition(
            unconditioned_model, {"obs": torch.ones(2)}
        )
        reconditioned_trace = poutine.trace(reconditioned_model).get_trace()
        assert_equal(reconditioned_trace.nodes["obs"]["value"], torch.ones(2))

class EscapeHandlerTests(TestCase):
    def setUp(self):

        # Simple model with 1 continuous + 1 discrete + 1 continuous variable.
        def model():
            p = torch.tensor([0.5])
            loc = torch.zeros(1)
            scale = torch.ones(1)

            x = pyro.sample("x", Normal(loc, scale))  # Before the discrete variable.
            y = pyro.sample("y", Bernoulli(p))
            z = pyro.sample("z", Normal(loc, scale))  # After the discrete variable.
            return dict(x=x, y=y, z=z)

        self.sites = ["x", "y", "z", "_INPUT", "_RETURN"]
        self.model = model

    def test_discrete_escape(self):
        try:
            poutine.escape(
                self.model,
                escape_fn=functools.partial(discrete_escape, poutine.Trace()),
            )()
            assert False
        except NonlocalExit as e:
            assert e.site["name"] == "y"

    def test_all_escape(self):
        try:
            poutine.escape(
                self.model, escape_fn=functools.partial(all_escape, poutine.Trace())
            )()
            assert False
        except NonlocalExit as e:
            assert e.site["name"] == "x"

    def test_trace_compose(self):
        tm = poutine.trace(self.model)
        try:
            poutine.escape(
                tm, escape_fn=functools.partial(all_escape, poutine.Trace())
            )()
            assert False
        except NonlocalExit:
            assert "x" in tm.trace
            try:
                tem = poutine.trace(
                    poutine.escape(
                        self.model,
                        escape_fn=functools.partial(all_escape, poutine.Trace()),
                    )
                )
                tem()
                assert False
            except NonlocalExit:
                assert "x" not in tem.trace

class InferConfigHandlerTests(TestCase):
    def setUp(self):
        def model():
            pyro.param("p", torch.zeros(1, requires_grad=True))
            pyro.sample(
                "a", Bernoulli(torch.tensor([0.5])), infer={"enumerate": "parallel"}
            )
            pyro.sample("b", Bernoulli(torch.tensor([0.5])))

        self.model = model

        def config_fn(site):
            if site["type"] == "sample":
                return {"blah": True}
            else:
                return {}

        self.config_fn = config_fn

    def test_infer_config_sample(self):
        cfg_model = poutine.infer_config(self.model, config_fn=self.config_fn)

        tr = poutine.trace(cfg_model).get_trace()

        assert tr.nodes["a"]["infer"] == {"enumerate": "parallel", "blah": True}
        assert tr.nodes["b"]["infer"] == {"blah": True}
        assert tr.nodes["p"]["infer"] == {}

@pytest.mark.parametrize("first_available_dim", [-1, -2, -3])
@pytest.mark.parametrize("depth", [0, 1, 2])
def test_enumerate_poutine(depth, first_available_dim):
    num_particles = 2

    def model():
        pyro.sample("x", Bernoulli(0.5))
        for i in range(depth):
            pyro.sample(
                "a_{}".format(i), Bernoulli(0.5), infer={"enumerate": "parallel"}
            )

    model = poutine.enum(model, first_available_dim=first_available_dim)
    model = poutine.trace(model)

    for i in range(num_particles):
        tr = model.get_trace()
        tr.compute_log_prob()
        log_prob = sum(site["log_prob"] for name, site in tr.iter_stochastic_nodes())
        actual_shape = log_prob.shape
        expected_shape = (2,) * depth
        if depth:
            expected_shape = expected_shape + (1,) * (-1 - first_available_dim)
        assert actual_shape == expected_shape, "error on iteration {}".format(i)

@pytest.mark.parametrize("first_available_dim", [-1, -2, -3])
@pytest.mark.parametrize("depth", [0, 1, 2])
def test_replay_enumerate_poutine(depth, first_available_dim):
    num_particles = 2
    y_dist = Categorical(torch.tensor([0.5, 0.25, 0.25]))

    def guide():
        pyro.sample("y", y_dist, infer={"enumerate": "parallel"})

    guide = poutine.enum(guide, first_available_dim=first_available_dim - depth)
    guide = poutine.trace(guide)
    guide_trace = guide.get_trace()

    def model():
        pyro.sample("x", Bernoulli(0.5))
        for i in range(depth):
            pyro.sample(
                "a_{}".format(i), Bernoulli(0.5), infer={"enumerate": "parallel"}
            )
        pyro.sample("y", y_dist, infer={"enumerate": "parallel"})
        for i in range(depth):
            pyro.sample(
                "b_{}".format(i), Bernoulli(0.5), infer={"enumerate": "parallel"}
            )

    model = poutine.enum(model, first_available_dim=first_available_dim)
    model = poutine.replay(model, trace=guide_trace)
    model = poutine.trace(model)

    for i in range(num_particles):
        tr = model.get_trace()
        assert tr.nodes["y"]["value"] is guide_trace.nodes["y"]["value"]
        tr.compute_log_prob()
        log_prob = sum(site["log_prob"] for name, site in tr.iter_stochastic_nodes())
        actual_shape = log_prob.shape
        expected_shape = (
            (2,) * depth + (3,) + (2,) * depth + (1,) * (-1 - first_available_dim)
        )
        assert actual_shape == expected_shape, "error on iteration {}".format(i)

@pytest.mark.parametrize("has_rsample", [False, True])
@pytest.mark.parametrize("depth", [0, 1, 2])
def test_plate_preserves_has_rsample(has_rsample, depth):
    def guide():
        loc = pyro.param("loc", torch.tensor(0.0))
        with pyro.plate_stack("plates", (2,) * depth):
            return pyro.sample("x", dist.Normal(loc, 1).has_rsample_(has_rsample))

    x = guide()
    assert x.dim() == depth
    assert x.requires_grad == has_rsample

def test_plate_error_on_enter():
    def model():
        with pyro.plate("foo", 0):
            pass

    assert len(_DIM_ALLOCATOR._stack) == 0
    with pytest.raises(ZeroDivisionError):
        poutine.trace(model)()
    assert len(_DIM_ALLOCATOR._stack) == 0, "stack was not cleaned on error"

def test_decorator_interface_primitives():
    @poutine.trace
    def model():
        pyro.param("p", torch.zeros(1, requires_grad=True))
        pyro.sample(
            "a", Bernoulli(torch.tensor([0.5])), infer={"enumerate": "parallel"}
        )
        pyro.sample("b", Bernoulli(torch.tensor([0.5])))

    tr = model.get_trace()
    assert isinstance(tr, poutine.Trace)
    assert tr.graph_type == "flat"

    @poutine.trace(graph_type="dense")
    def model():
        pyro.param("p", torch.zeros(1, requires_grad=True))
        pyro.sample(
            "a", Bernoulli(torch.tensor([0.5])), infer={"enumerate": "parallel"}
        )
        pyro.sample("b", Bernoulli(torch.tensor([0.5])))

    tr = model.get_trace()
    assert isinstance(tr, poutine.Trace)
    assert tr.graph_type == "dense"

    tr2 = poutine.trace(poutine.replay(model, trace=tr)).get_trace()

    assert_equal(tr2.nodes["a"]["value"], tr.nodes["a"]["value"])

def test_decorator_interface_queue():

    sites = ["x", "y", "z", "_INPUT", "_RETURN"]
    queue = Queue()
    queue.put(poutine.Trace())

    @poutine.queue(queue=queue)
    def model():
        p = torch.tensor([0.5])
        loc = torch.zeros(1)
        scale = torch.ones(1)

        x = pyro.sample("x", Normal(loc, scale))
        y = pyro.sample("y", Bernoulli(p))
        z = pyro.sample("z", Normal(loc, scale))
        return dict(x=x, y=y, z=z)

    tr = poutine.trace(model).get_trace()
    for name in sites:
        assert name in tr

def test_method_decorator_interface_condition():
    class cls_model:
        @poutine.condition(data={"b": torch.tensor(1.0)})
        def model(self, p):
            self._model(p)

        def _model(self, p):
            pyro.sample("a", Bernoulli(p))
            pyro.sample("b", Bernoulli(torch.tensor([0.5])))

    tr = poutine.trace(cls_model().model).get_trace(0.5)
    assert isinstance(tr, poutine.Trace)
    assert tr.graph_type == "flat"
    assert tr.nodes["b"]["is_observed"] and tr.nodes["b"]["value"].item() == 1.0

def test_trace_log_prob_err_msg():
    def model(v):
        pyro.sample("test_site", dist.Beta(1.0, 1.0), obs=v)

    tr = poutine.trace(model).get_trace(torch.tensor(2.0))
    exp_msg = (
        r"Error while computing log_prob at site 'test_site':\s*"
        r"The value argument must be within the support"
    )
    with pytest.raises(ValueError, match=exp_msg):
        tr.compute_log_prob()

def test_trace_log_prob_sum_err_msg():
    def model(v):
        pyro.sample("test_site", dist.Beta(1.0, 1.0), obs=v)

    tr = poutine.trace(model).get_trace(torch.tensor(2.0))
    exp_msg = (
        r"Error while computing log_prob_sum at site 'test_site':\s*"
        r"The value argument must be within the support"
    )
    with pytest.raises(ValueError, match=exp_msg):
        tr.log_prob_sum()

def test_trace_score_parts_err_msg():
    def guide(v):
        pyro.sample("test_site", dist.Beta(1.0, 1.0), obs=v)

    tr = poutine.trace(guide).get_trace(torch.tensor(2.0))
    exp_msg = (
        r"Error while computing score_parts at site 'test_site':\s*"
        r"The value argument must be within the support"
    )
    with pytest.raises(ValueError, match=exp_msg):
        tr.compute_score_parts()

def _model(a=torch.tensor(1.0), b=torch.tensor(1.0)):
    latent = pyro.sample("latent", dist.Beta(a, b))
    return pyro.sample("test_site", dist.Bernoulli(latent), obs=torch.tensor(1))

@pytest.mark.parametrize(
    "wrapper",
    [
        lambda fn: poutine.block(fn),
        lambda fn: poutine.condition(fn, {"latent": 0.9}),
        lambda fn: poutine.enum(fn, -1),
        lambda fn: poutine.replay(fn, poutine.trace(fn).get_trace()),
    ],
)
def test_pickling(wrapper):
    wrapped = wrapper(_model)
    buffer = io.BytesIO()
    # default protocol cannot serialize torch.Size objects (see https://github.com/pytorch/pytorch/issues/20823)
    torch.save(wrapped, buffer, pickle_protocol=pickle.HIGHEST_PROTOCOL)
    buffer.seek(0)
    deserialized = torch.load(buffer)
    obs = torch.tensor(0.5)
    pyro.set_rng_seed(0)
    actual_trace = poutine.trace(deserialized).get_trace(obs)
    pyro.set_rng_seed(0)
    expected_trace = poutine.trace(wrapped).get_trace(obs)
    assert tuple(actual_trace) == tuple(expected_trace.nodes)
    assert_close(
        [actual_trace.nodes[site]["value"] for site in actual_trace.stochastic_nodes],
        [
            expected_trace.nodes[site]["value"]
            for site in expected_trace.stochastic_nodes
        ],
    )

def test_arg_kwarg_error():
    def model():
        pyro.param("p", torch.zeros(1, requires_grad=True))
        pyro.sample(
            "a", Bernoulli(torch.tensor([0.5])), infer={"enumerate": "parallel"}
        )
        pyro.sample("b", Bernoulli(torch.tensor([0.5])))

    with pytest.raises(ValueError, match="not callable"):
        with poutine.mask(False):
            model()

    with poutine.mask(mask=False):
        model()


text:
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import { combineReducers } from 'redux';

import {
  DIALOG_ABOUT_CLOSE,
  DIALOG_ABOUT_OPEN,
} from '../../../constants/actions';

const open = (state = false, action) => {
  switch (action.type) {
    case DIALOG_ABOUT_CLOSE: return false;
    case DIALOG_ABOUT_OPEN: return true;
    default: return state;
  }
};

export default combineReducers({ open });

redpajama_set_name: RedPajamaGithub


text:
#include "vertexdatainterpreter.h"

#include <QListWidget>
#include <QStringList>

#include <QDebug>

#include <qmath.h>

static int
sizeForType(int type)
{
    switch(type) {
    case DT_INT8:
    case DT_UINT8:
        return 1;
    case DT_INT16:
    case DT_UINT16:
        return 2;
    case DT_INT32:
    case DT_UINT32:
        return 4;
    case DT_FLOAT:
        return sizeof(float);
    case DT_DOUBLE:
        return sizeof(double);
    default:
        return sizeof(int);
    }
}

template <typename T>
static QStringList
convertData(const QByteArray &dataArray,
            int type,
            int stride,
            int numComponents,
            int startingOffset)
{
    QStringList strings;

    int dataSize = dataArray.size() - startingOffset;
    const char *data = dataArray.constData() + startingOffset;
    int typeSize = sizeForType(type);
    int elementSize = numComponents * typeSize;

    if (!stride)
        stride = elementSize;

    int numElements = dataSize / stride;

    if ((numElements % numComponents) != 0) {
        int temp = qFloor(dataSize / (float)stride);
        int fullElemSize = temp * stride;
        if (fullElemSize + numComponents * typeSize <= dataSize) {
            /* num full elements plus the part of the buffer in which we fit */
            numElements = temp + 1;
        } else {
            numElements = temp;
        }
    }

#if 0
    qDebug() << "numElements = "<<numElements;
    qDebug() << "elementSize = "<<elementSize;
    qDebug() << "stride = "<<stride;
    qDebug() << "numComponents = "<<numComponents;
    qDebug() << "typeSize = "<<typeSize;
#endif

    for (int i = 0; i < numElements; ++i) {
        QString vectorString = QString::fromLatin1("%1) [").arg(i);
        for (int j = 0; j < numComponents; ++j) {
            int offset = i*stride + j*typeSize;
            const T *elementPtr =
                (const T*)(data + offset);
            T elem = *elementPtr;
            vectorString += QString::number(elem);
            if ((j + 1) < numComponents)
                vectorString += QLatin1String(", ");
        }
        vectorString += "]";
        strings += vectorString;
    }

    return strings;
}

VertexDataInterpreter::VertexDataInterpreter(QObject *parent)
    : QObject(parent),
      m_listWidget(0),
      m_type(DT_FLOAT),
      m_stride(16),
      m_components(4),
      m_startingOffset(0)
{
}

void VertexDataInterpreter::setData(const QByteArray &data)
{
    m_data = data;
    if (m_listWidget)
        m_listWidget->clear();
}

QByteArray VertexDataInterpreter::data() const
{
    return m_data;
}

void VertexDataInterpreter::setType(int type)
{
    m_type = type;
}

int VertexDataInterpreter::type() const
{
    return m_type;
}

void VertexDataInterpreter::setStride(int stride)
{
    m_stride = stride;
}

int VertexDataInterpreter::stride() const
{
    return m_stride;
}

void VertexDataInterpreter::setComponents(int num)
{
    m_components = num;
}

int VertexDataInterpreter::components() const
{
    return m_components;
}

void VertexDataInterpreter::setListWidget(QListWidget *listWidget)
{
    m_listWidget = listWidget;
}

void VertexDataInterpreter::interpretData()
{
    if (!m_listWidget)
        return;

    m_listWidget->clear();

    if (m_data.isEmpty() || !m_components)
        return;

    QStringList lst;
    switch(m_type) {
    case DT_INT8:
        lst = convertData<qint8>(m_data, m_type, m_stride, m_components,
                                 m_startingOffset);
        break;
    case DT_UINT8:
        lst = convertData<quint8>(m_data, m_type, m_stride, m_components,
                                  m_startingOffset);
        break;
    case DT_INT16:
        lst = convertData<qint16>(m_data, m_type, m_stride, m_components,
                                  m_startingOffset);
        break;
    case DT_UINT16:
        lst = convertData<quint16>(m_data, m_type, m_stride, m_components,
                                   m_startingOffset);
        break;
    case DT_INT32:
        lst = convertData<qint32>(m_data, m_type, m_stride, m_components,
                                  m_startingOffset);
        break;
    case DT_UINT32:
        lst = convertData<quint32>(m_data, m_type, m_stride, m_components,
                                   m_startingOffset);
        break;
    case DT_FLOAT:
        lst = convertData<float>(m_data, m_type, m_stride, m_components,
                                 m_startingOffset);
        break;
    case DT_DOUBLE:
        lst = convertData<double>(m_data, m_type, m_stride, m_components,
                                  m_startingOffset);
        break;
    default:
        qDebug()<<"unkown gltype = "<<m_type;
    }
    //qDebug()<<"list is "<<lst;
    m_listWidget->addItems(lst);
}

void VertexDataInterpreter::setTypeFromString(const QString &str)
{
    if (str == QLatin1String("GL_FLOAT")) {
        setType(DT_FLOAT);
    } else if (str == QLatin1String("GL_INT")) {
        setType(DT_INT32);
    } else if (str == QLatin1String("GL_UNSIGNED_INT")) {
        setType(DT_UINT32);
    } else if (str == QLatin1String("GL_SHORT")) {
        setType(DT_INT16);
    } else if (str == QLatin1String("GL_UNSIGNED_SHORT")) {
        setType(DT_UINT16);
    } else if (str == QLatin1String("GL_BYTE")) {
        setType(DT_INT8);
    } else if (str == QLatin1String("GL_UNSIGNED_BYTE")) {
        setType(DT_UINT8);
    } else if (str == QLatin1String("GL_DOUBLE")) {
        setType(DT_DOUBLE);
    } else {
        qDebug()<<"unknown vertex data type";
    }
}

int VertexDataInterpreter::startingOffset() const
{
    return m_startingOffset;
}

void VertexDataInterpreter::setStartingOffset(int offset)
{
    m_startingOffset = offset;
}


text:
package org.semanticweb.elk.owl.interfaces;

import org.semanticweb.elk.owl.visitors.ElkCardinalityRestrictionQualifiedVisitor;

/**
 * A generic interface for restrictions on properties with fillers (qualifiers)
 * and cardinality constraints.
 *
 * @author "Yevgeny Kazakov"
 *
 * @param <P>
 *            the type of the property of this restriction
 * @param <F>
 *            the type of the filler of this restriction
 */
public interface ElkCardinalityRestrictionQualified<P, F> extends
		ElkCardinalityRestriction<P>, ElkPropertyRestrictionQualified<P, F> {

	/**
	 * Accept an {@link ElkCardinalityRestrictionQualifiedVisitor}.
	 *
	 * @param visitor
	 *            the visitor that can work with this object type
	 * @return the output of the visitor
	 */
	public <O> O accept(ElkCardinalityRestrictionQualifiedVisitor<O> visitor);

	/**
	 * A factory for creating instances
	 *
	 * @author Yevgeny Kazakov
	 *
	 */
	interface Factory extends ElkDataExactCardinalityQualified.Factory,
			ElkDataMaxCardinalityQualified.Factory,
			ElkDataMinCardinalityQualified.Factory,
			ElkObjectExactCardinalityQualified.Factory,
			ElkObjectMaxCardinalityQualified.Factory,
			ElkObjectMinCardinalityQualified.Factory {

		// combined interface

	}

}


text:
id: segments
title: "Segments"
---

<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one
  ~ or more contributor license agreements.  See the NOTICE file
  ~ distributed with this work for additional information
  ~ regarding copyright ownership.  The ASF licenses this file
  ~ to you under the Apache License, Version 2.0 (the
  ~ "License"); you may not use this file except in compliance
  ~ with the License.  You may obtain a copy of the License at
  ~
  ~   http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing,
  ~ software distributed under the License is distributed on an
  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  ~ KIND, either express or implied.  See the License for the
  ~ specific language governing permissions and limitations
  ~ under the License.
  -->

Apache Druid (incubating) stores its index in *segment files*, which are partitioned by
time. In a basic setup, one segment file is created for each time
interval, where the time interval is configurable in the
`segmentGranularity` parameter of the
[`granularitySpec`](../ingestion/index.md#granularityspec).  For Druid to
operate well under heavy query load, it is important for the segment
file size to be within the recommended range of 300mb-700mb. If your
segment files are larger than this range, then consider either
changing the granularity of the time interval or partitioning your
data and tweaking the `targetPartitionSize` in your `partitionsSpec`
(a good starting point for this parameter is 5 million rows).  See the
sharding section below and the 'Partitioning specification' section of
the [Batch ingestion](../ingestion/hadoop.md#partitionsspec) documentation
for more information.

### A segment file's core data structures

Here we describe the internal structure of segment files, which is
essentially *columnar*: the data for each column is laid out in
separate data structures. By storing each column separately, Druid can
decrease query latency by scanning only those columns actually needed
for a query.  There are three basic column types: the timestamp
column, dimension columns, and metric columns, as illustrated in the
image below:

![Druid column types](../assets/druid-column-types.png "Druid Column Types")

The timestamp and metric columns are simple: behind the scenes each of
these is an array of integer or floating point values compressed with
LZ4. Once a query knows which rows it needs to select, it simply
decompresses these, pulls out the relevant rows, and applies the
desired aggregation operator. As with all columns, if a query doesn't
require a column, then that column's data is just skipped over.

Dimensions columns are different because they support filter and
group-by operations, so each dimension requires the following
three data structures:

1. A dictionary that maps values (which are always treated as strings) to integer IDs,
2. A list of the column's values, encoded using the dictionary in 1, and
3. For each distinct value in the column, a bitmap that indicates which rows contain that value.

Why these three data structures? The dictionary simply maps string
values to integer ids so that the values in 2 and 3 can be
represented compactly. The bitmaps in 3 -- also known as *inverted
indexes* allow for quick filtering operations (specifically, bitmaps
are convenient for quickly applying AND and OR operators). Finally,
the list of values in 2 is needed for *group by* and *TopN*
queries. In other words, queries that solely aggregate metrics based
on filters do not need to touch the list of dimension values stored in
2.

To get a concrete sense of these data structures, consider the 'page'
column from the example data above.  The three data structures that
represent this dimension are illustrated in the diagram below.

```
1: Dictionary that encodes column values
  {
    "Justin Bieber": 0,
    "Ke$ha":         1
  }

2: Column data
  [0,
   0,
   1,
   1]

3: Bitmaps - one for each unique value of the column
  value="Justin Bieber": [1,1,0,0]
  value="Ke$ha":         [0,0,1,1]
```

Note that the bitmap is different from the first two data structures:
whereas the first two grow linearly in the size of the data (in the
worst case), the size of the bitmap section is the product of data
size * column cardinality. Compression will help us here though
because we know that for each row in 'column data', there will only be a
single bitmap that has non-zero entry. This means that high cardinality
columns will have extremely sparse, and therefore highly compressible,
bitmaps. Druid exploits this using compression algorithms that are
specially suited for bitmaps, such as roaring bitmap compression.

### Multi-value columns

If a data source makes use of multi-value columns, then the data
structures within the segment files look a bit different. Let's
imagine that in the example above, the second row were tagged with
both the 'Ke$ha' *and* 'Justin Bieber' topics. In this case, the three
data structures would now look as follows:

```
1: Dictionary that encodes column values
  {
    "Justin Bieber": 0,
    "Ke$ha":         1
  }

2: Column data
  [0,
   [0,1],  <--Row value of multi-value column can have array of values
   1,
   1]

3: Bitmaps - one for each unique value
  value="Justin Bieber": [1,1,0,0]
  value="Ke$ha":         [0,1,1,1]
                            ^
                            |
                            |
    Multi-value column has multiple non-zero entries
```

Note the changes to the second row in the column data and the Ke$ha
bitmap. If a row has more than one value for a column, its entry in
the 'column data' is an array of values. Additionally, a row with *n*
values in 'column data' will have *n* non-zero valued entries in
bitmaps.

## Naming Convention

Identifiers for segments are typically constructed using the segment datasource, interval start time (in ISO 8601 format), interval end time (in ISO 8601 format), and a version. If data is additionally sharded beyond a time range, the segment identifier will also contain a partition number.

An example segment identifier may be:
datasource_intervalStart_intervalEnd_version_partitionNum

## Segment Components

Behind the scenes, a segment is comprised of several files, listed below.

* `version.bin`

    4 bytes representing the current segment version as an integer. E.g., for v9 segments, the version is 0x0, 0x0, 0x0, 0x9

* `meta.smoosh`

    A file with metadata (filenames and offsets) about the contents of the other `smoosh` files

* `XXXXX.smoosh`

    There are some number of these files, which are concatenated binary data

    The `smoosh` files represent multiple files "smooshed" together in order to minimize the number of file descriptors that must be open to house the data. They are files of up to 2GB in size (to match the limit of a memory mapped ByteBuffer in Java). The `smoosh` files house individual files for each of the columns in the data as well as an `index.drd` file with extra metadata about the segment.

    There is also a special column called `__time` that refers to the time column of the segment. This will hopefully become less and less special as the code evolves, but for now it's as special as my Mommy always told me I am.

In the codebase, segments have an internal format version. The current segment format version is `v9`.

## Format of a column

Each column is stored as two parts:

1.  A Jackson-serialized ColumnDescriptor
2.  The rest of the binary for the column

A ColumnDescriptor is essentially an object that allows us to use Jackson's polymorphic deserialization to add new and interesting methods of serialization with minimal impact to the code. It consists of some metadata about the column (what type is it, is it multi-value, etc.) and then a list of serialization/deserialization logic that can deserialize the rest of the binary.

## Sharding Data to Create Segments

### Sharding

Multiple segments may exist for the same interval of time for the same datasource. These segments form a `block` for an interval.
Depending on the type of `shardSpec` that is used to shard the data, Druid queries may only complete if a `block` is complete. That is to say, if a block consists of 3 segments, such as:

`sampleData_2011-01-01T02:00:00:00Z_2011-01-01T03:00:00:00Z_v1_0`

`sampleData_2011-01-01T02:00:00:00Z_2011-01-01T03:00:00:00Z_v1_1`

`sampleData_2011-01-01T02:00:00:00Z_2011-01-01T03:00:00:00Z_v1_2`

All 3 segments must be loaded before a query for the interval `2011-01-01T02:00:00:00Z_2011-01-01T03:00:00:00Z` completes.

The exception to this rule is with using linear shard specs. Linear shard specs do not force 'completeness' and queries can complete even if shards are not loaded in the system.
For example, if your real-time ingestion creates 3 segments that were sharded with linear shard spec, and only two of the segments were loaded in the system, queries would return results only for those 2 segments.

## Schema changes

## Replacing segments

Druid uniquely
identifies segments using the datasource, interval, version, and partition number. The partition number is only visible in the segment id if
there are multiple segments created for some granularity of time. For example, if you have hourly segments, but you
have more data in an hour than a single segment can hold, you can create multiple segments for the same hour. These segments will share
the same datasource, interval, and version, but have linearly increasing partition numbers.

```
foo_2015-01-01/2015-01-02_v1_0
foo_2015-01-01/2015-01-02_v1_1
foo_2015-01-01/2015-01-02_v1_2
```

In the example segments above, the dataSource = foo, interval = 2015-01-01/2015-01-02, version = v1, partitionNum = 0.
If at some later point in time, you reindex the data with a new schema, the newly created segments will have a higher version id.

```
foo_2015-01-01/2015-01-02_v2_0
foo_2015-01-01/2015-01-02_v2_1
foo_2015-01-01/2015-01-02_v2_2
```

Druid batch indexing (either Hadoop-based or IndexTask-based) guarantees atomic updates on an interval-by-interval basis.
In our example, until all `v2` segments for `2015-01-01/2015-01-02` are loaded in a Druid cluster, queries exclusively use `v1` segments.
Once all `v2` segments are loaded and queryable, all queries ignore `v1` segments and switch to the `v2` segments.
Shortly afterwards, the `v1` segments are unloaded from the cluster.

Note that updates that span multiple segment intervals are only atomic within each interval. They are not atomic across the entire update.
For example, you have segments such as the following:

```
foo_2015-01-01/2015-01-02_v1_0
foo_2015-01-02/2015-01-03_v1_1
foo_2015-01-03/2015-01-04_v1_2
```

`v2` segments will be loaded into the cluster as soon as they are built and replace `v1` segments for the period of time the
segments overlap. Before v2 segments are completely loaded, your cluster may have a mixture of `v1` and `v2` segments.

```
foo_2015-01-01/2015-01-02_v1_0
foo_2015-01-02/2015-01-03_v2_1
foo_2015-01-03/2015-01-04_v1_2
```

In this case, queries may hit a mixture of `v1` and `v2` segments.

## Different schemas among segments

Druid segments for the same datasource may have different schemas. If a string column (dimension) exists in one segment but not
another, queries that involve both segments still work. Queries for the segment missing the dimension will behave as if the dimension has only null values.
Similarly, if one segment has a numeric column (metric) but another does not, queries on the segment missing the
metric will generally "do the right thing". Aggregations over this missing metric behave as if the metric were missing.


text:
#include <SDRP/Utilities/Statistics.h>

namespace Radicle { namespace SDRP
{
	Statistics* Statistics::m_instance = NULL;

	Statistics* Statistics::Instance()
	{
		if( m_instance == NULL )
		{
			m_instance = new Statistics();
		}

		return m_instance;
	}

	Statistics::Statistics() :
	m_firstPacket( 0 ),
	m_lastPacket( 0 ),
	m_packetBytes( 0 ),
	m_packetCount( 0 )
	{}

	void Statistics::PacketSent( const RDSize packetSize )
	{
		if( m_firstPacket == 0.0 )
		{
			m_firstPacket = Logger::Time();
		}

		m_lastPacket = Logger::Time();
		m_packetBytes += packetSize;
		m_packetCount++;
	}

	const RDSize Statistics::PacketCount() const
	{
		return m_packetCount;
	}

	const RDSize Statistics::PacketBytes() const
	{
		return m_packetBytes;
	}

	const RDSize Statistics::Overhead() const
	{
		if( m_firstPacket != 0.0 )
		{
			return m_packetBytes / ( m_lastPacket - m_firstPacket );
		}

		return 0;
	}

	void Statistics::ReportRaw( std::ostream& out ) const
	{

	}
} }


text:
package com.wenming.library.processutil.models;

import android.os.Parcel;
import android.os.Parcelable;
import android.text.TextUtils;

import java.io.IOException;

public class AndroidProcess implements Parcelable {

  /**
   * Get the name of a running process.
   *
   * @param pid
   *     the process id.
   * @return the name of the process.
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  static String getProcessName(int pid) throws IOException {
    String cmdline = null;
    try {
      cmdline = ProcFile.readFile(String.format("/proc/%d/cmdline", pid)).trim();
    } catch (IOException ignored) {
    }
    if (TextUtils.isEmpty(cmdline)) {
      return Stat.get(pid).getComm();
    }
    return cmdline;
  }

  /** the process name */
  public final String name;

  /** the process id */
  public final int pid;

  /**
   * AndroidProcess constructor
   *
   * @param pid
   *     the process id
   * @throws IOException
   *     if /proc/[pid] does not exist or we don't have read access.
   */
  public AndroidProcess(int pid) throws IOException {
    this.pid = pid;
    this.name = getProcessName(pid);
  }

  /**
   * Read the contents of a file in /proc/[pid]/[filename].
   *
   * @param filename
   *     the relative path to the file.
   * @return the contents of the file.
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public String read(String filename) throws IOException {
    return ProcFile.readFile(String.format("/proc/%d/%s", pid, filename));
  }

  /**
   *
/proc/[pid]/attr/current (since Linux 2.6.0)

   *
   *
The contents of this file represent the current security attributes of the process.

   *
   *
In SELinux, this file is used to get the security context of a process. Prior to Linux
   * 2.6.11, this file could not be used to set the security context (a write was always denied),
   * since SELinux limited process security transitions to execve(2) (see the description of
   * /proc/[pid]/attr/exec, below).  ince Linux 2.6.11, SELinux lifted this restriction and began
   * supporting "set" operations via writes to this node if authorized by policy, although use of
   * this operation is only suitable for applications that are trusted to maintain any desired
   * separation between the old and new security contexts.  Prior to Linux 2.6.28, SELinux did not
   * allow threads within a multi- threaded process to set their security context via this node as
   * it would yield an inconsistency among the security contexts of the threads sharing the same
   * memory space. Since Linux 2.6.28, SELinux lifted this restriction and began supporting "set"
   * operations for threads within a multithreaded process if the new security context is bounded
   * by the old security context, where the bounded relation is defined in policy and guarantees
   * that the new security context has a subset of the permissions of the old security context.
   * Other security modules may choose to support "set" operations via writes to this node.

   *
   * @return the contents of /proc/[pid]/attr/current
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public String attr_current() throws IOException {
    return read("attr/current");
  }

  /**
   *
/proc/[pid]/cmdline

   *
   *
This read-only file holds the complete command line for the process, unless the process is
   * a zombie. In the latter case, there is nothing in this file: that is, a read on this file will
   * return 0 characters. The command-line arguments appear in this file as a set of strings
   * separated by null bytes ('\0'), with a further null byte after the last string.

   *
   * @return the name of the process. (note: process name may be empty. In case it is empty get
   * the process name from /proc/[pid]/stat).
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   * @see #name
   */
  public String cmdline() throws IOException {
    return read("cmdline");
  }

  /**
   *
/proc/[pid]/cgroup (since Linux 2.6.24)

   *
   *
This file describes control groups to which the process/task belongs. For each cgroup
   * hierarchy there is one entry containing colon-separated fields of the form:

   *
   *
5:cpuacct,cpu,cpuset:/daemons

   *
   *
The colon-separated fields are, from left to right:

   *
   *

   *
hierarchy ID number

   *
set of subsystems bound to the hierarchy

   *
control group in the hierarchy to which the process belongs

   *

   *
   *
This file is present only if the CONFIG_CGROUPS kernel configuration option is enabled.

   *
   * @return the {@link Cgroup} for this process
   * @throws IOException
   */
  public Cgroup cgroup() throws IOException {
    return Cgroup.get(pid);
  }

  /**
   *
/proc/[pid]/oom_score (since Linux 2.6.11)

   *
   *
This file displays the current score that the kernel gives to this process for the
   * purpose of selecting a process for the OOM-killer. A higher score means that the
   * process is more likely to be selected by the OOM-killer.

   *
   *
The basis for this score is the amount of memory used by the process, with
   * increases (+) or decreases (-) for factors including:

   *
   *

   *
whether the process creates a lot of children using fork(2)(+);

   *
whether the process has been running a long time, or has used a lot of CPU time (-);

   *
whether the process has a low nice value (i.e., > 0) (+);

   *
whether the process is privileged (-); and

   *
whether the process is making direct hardware access (-).

   *

   *
   *
The oom_score also reflects the adjustment specified by the oom_score_adj
   * or oom_adj setting for the process.

   *
   * @return the oom_score value for this process
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public int oom_score() throws IOException {
    return Integer.parseInt(read("oom_score"));
  }

  /**
   *
/proc/[pid]/oom_adj (since Linux 2.6.11)

   *
   *
This file can be used to adjust the score used to select which process should be killed in
   * an out-of-memory (OOM) situation. The kernel uses this value for a bit-shift operation of the
   * process's oom_score value: valid values are in the* range -16 to +15, plus the special value
   * -17, which disables OOM-killing altogether for this process. A positive score increases the
   * likelihood of this process being killed by the OOM-killer; a negative score decreases the
   * likelihood.

   *
   *
The default value for this file is 0; a new process inherits its parent's oom_adj setting.
   * A process must be privileged (CAP_SYS_RESOURCE) to update this file.

   *
   *
Since Linux 2.6.36, use of this file is deprecated in favor of
   * /proc/[pid]/oom_score_adj.

   *
   * @return the oom_adj value for this process
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public int oom_adj() throws IOException {
    return Integer.parseInt(read("oom_adj"));
  }

  /**
   *
/proc/[pid]/oom_score_adj (since Linux 2.6.36)

   *
   *
This file can be used to adjust the badness heuristic used to select which process gets
   * killed in out-of-memory conditions.

   *
   *
The badness heuristic assigns a value to each candidate task ranging from 0 (never kill) to
   * 1000 (always kill) to determine which process is targeted. The units are roughly a proportion
   * along that range of allowed memory the process may allocate from, based on an estimation of
   * its current memory and swap use. For example, if a task is using all allowed memory, its
   * badness score will be 1000.  If it is using half of its allowed memory, its score will be
   * 500.

   *
   *
There is an additional factor included in the badness score: root processes are given 3%
   * extra memory over other tasks.

   *
   *
The amount of "allowed" memory depends on the context in which the OOM-killer was called.
   * If it is due to the memory assigned to the allocating task's cpuset being exhausted, the
   * allowed memory represents the set of mems assigned to that cpuset (see cpuset(7)). If it is
   * due to a mempolicy's node(s) being exhausted, the allowed memory represents the set of
   * mempolicy nodes. If it is due to a memory limit (or swap limit) being reached, the allowed
   * memory is that configured limit. Finally, if it is due to the entire system being out of
   * memory, the allowed memory represents all allocatable resources.

   *
   *
The value of oom_score_adj is added to the badness score before it is used to determine
   * which task to kill.  Acceptable values range from -1000 (OOM_SCORE_ADJ_MIN) to +1000
   * (OOM_SCORE_ADJ_MAX). This allows user space to control the preference for OOM-killing, ranging
   * from always preferring a certain task or completely disabling it from OOM killing.  The lowest
   * possible value, -1000, is equivalent to disabling OOM- killing entirely for that task, since
   * it will always report a badness score of 0.

   *
   *
Consequently, it is very simple for user space to define the amount of memory to consider
   * for each task.  Setting a oom_score_adj value of +500, for example, is roughly equivalent to
   * allowing the remainder of tasks sharing the same system, cpuset, mempolicy, or memory
   * controller resources to use at least 50% more memory.  A value of -500, on the other hand,
   * would be roughly equivalent to discounting 50% of the task's allowed memory from being
   * considered as scoring against the task.

   *
   *
For backward compatibility with previous kernels, /proc/[pid]/oom_adj can still be used to
   * tune the badness score.  Its value is scaled linearly with oom_score_adj.

   *
   *
Writing to /proc/[pid]/oom_score_adj or /proc/[pid]/oom_adj will change the other with its
   * scaled value.

   *
   * @return the oom_score_adj value for this process
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public int oom_score_adj() throws IOException {
    return Integer.parseInt(read("oom_score_adj"));
  }

  /**
   *
/proc/[pid]/stat

   *
   *
Status information about the process.  This is used by ps(1). It is defined in the kernel
   * source file fs/proc/array.c.

   *
   *
The fields, in order, with their proper scanf(3) format specifiers, are:

   *
   *

   *
   *
pid  %d The process ID.

   *
   *
comm  %s The filename of the executable, in parentheses. This is visible whether or not
   * the executable is swapped out.

   *
   *
state  %c One of the following characters, indicating process state:
   *

   *
R  Running

   *
S  Sleeping in an interruptible wait

   *
D  Waiting in uninterruptible disk sleep

   *
Z  Zombie

   *
T  Stopped (on a signal) or (before Linux 2.6.33) trace stopped

   *
t  Tracing stop (Linux 2.6.33 onward)

   *
W  Paging (only before Linux 2.6.0)

   *
X  Dead (from Linux 2.6.0 onward)

   *
x  Dead (Linux 2.6.33 to 3.13 only)

   *
K  Wakekill (Linux 2.6.33 to 3.13 only)

   *
W  Waking (Linux 2.6.33 to 3.13 only)

   *
P  Parked (Linux 3.9 to 3.13 only)

   *

   *

   *
   *
ppid %d The PID of the parent of this process.

   *
   *
pgrp %d The process group ID of the process.

   *
   *
session %d The session ID of the process.

   *
   *
tty_nr %d The controlling terminal of the process.  (The minor device number is contained
   * in the combination of bits 31 to 20 and 7 to 0; the major device number is in bits 15 to 8.)
   *

   *
   *
tpgid %d The ID of the foreground process group of the controlling terminal of the
   * process.

   *
   *
flags %u The kernel flags word of the process.  For bit meanings, see the PF_* defines in
   * the Linux kernel source file include/linux/sched.h.  Details depend on the kernel version.
   * The format for this field was %lu before Linux 2.6.

   *
   *
minflt %lu The number of minor faults the process has made which have not required
   * loading a memory page from disk.

   *
   *
cminflt %lu The number of minor faults that the process's waited-for children have
   * made

   *
   *
majflt  %lu The number of major faults the process has made which have required loading a
   * memory page from disk.

   *
   *
cmajflt  %lu The number of major faults that the process's waited-for children have
   * made

   *
   *
utime  %lu Amount of time that this process has been scheduled in user mode, measured in
   * clock ticks (divide by sysconf(_SC_CLK_TCK)).  This includes guest time,   guest_time (time
   * spent running a virtual CPU, see below), so that applications that are not aware of the guest
   * time field do not lose that time from their calculations.

   *
   *
stime  %lu Amount of time that this process has been scheduled in kernel mode, measured
   * in clock ticks (divide by sysconf(_SC_CLK_TCK)).

   *
   *
cutime  %ld Amount of time that this process's waited-for children have been scheduled in
   * user mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)). (See also times(2).)
   * This includes guest time, cguest_time (time spent running a virtual CPU, see below).

   *
   *
cstime  %ld Amount of time that this process's waited-for children have been scheduled in
   * kernel mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)).

   *
   *
priority  %ld (Explanation for Linux 2.6) For processes running a real-time scheduling
   * policy (policy below; see sched_setscheduler(2)), this is the negated scheduling priority,
   * minus one; that is, a number in the range -2 to -100, corresponding to real-time priorities 1
   * to 99.  For processes running under a non-real-time scheduling policy, this is the raw nice
   * value (setpriority(2)) as represented in the kernel.  The kernel stores nice values as numbers
   * in the range 0 (high) to 39 (low), corresponding to the user-visible nice range of -20 to 19.
   * Before Linux 2.6, this was a scaled value based on the scheduler weighting given to this
   * process

   *
   *
nice  %ld The nice value (see setpriority(2)), a value in the range 19 (low priority) to
   * -20 (high priority).

   *
   *
num_threads  %ld Number of threads in this process (since Linux 2.6). Before kernel 2.6,
   * this field was hard coded to 0 as a placeholder for an earlier removed field.

   *
   *
itrealvalue  %ld The time in jiffies before the next SIGALRM is sent to the process due
   * to an interval timer.  Since kernel 2.6.17, this field is no longer maintained, and is hard
   * coded as 0.

   *
   *
starttime  %llu The time the process started after system boot.  In kernels before Linux
   * 2.6, this value was expressed in jiffies.  Since Linux 2.6, the value is expressed in clock
   * ticks (divide by sysconf(_SC_CLK_TCK)).

   *
   *
The format for this field was %lu before Linux 2.6.  (23) vsize  %lu Virtual memory size
   * in bytes.

   *
   *
rss  %ld Resident Set Size: number of pages the process has in real memory.  This is just
   * the pages which count toward text, data, or stack space.  This does not include pages which
   * have not been demand-loaded in, or which are swapped out.

   *
   *
rsslim  %lu Current soft limit in bytes on the rss of the process; see the description of
   * RLIMIT_RSS in getrlimit(2).

   *
   *
startcode  %lu The address above which program text can run.

   *
   *
endcode  %lu The address below which program text can run.

   *
   *
startstack  %lu The address of the start (i.e., bottom) of the stack.

   *
   *
kstkesp  %lu The current value of ESP (stack pointer), as found in the kernel stack page
   * for the process.

   *
   *
kstkeip  %lu The current EIP (instruction pointer).

   *
   *
signal  %lu The bitmap of pending signals, displayed as a decimal number.  Obsolete,
   * because it does not provide information on real-time signals; use /proc/[pid]/status
   * instead

   *
   *
blocked  %lu The bitmap of blocked signals, displayed as a decimal number.  Obsolete,
   * because it does not provide information on real-time signals; use /proc/[pid]/status
   * instead

   *
   *
sigignore  %lu The bitmap of ignored signals, displayed as a decimal number. Obsolete,
   * because it does not provide information on real-time signals; use /proc/[pid]/status
   * instead

   *
   *
sigcatch  %lu The bitmap of caught signals, displayed as a decimal number. Obsolete,
   * because it does not provide information on real-time signals; use /proc/[pid]/status
   * instead.

   *
   *
wchan  %lu This is the "channel" in which the process is waiting.  It is the address of a
   * location in the kernel where the process is sleeping.  The corresponding symbolic name can be
   * found in /proc/[pid]/wchan.

   *
   *
nswap  %lu Number of pages swapped (not maintained).

   *
   *
cnswap  %lu Cumulative nswap for child processes (not maintained).

   *
   *
exit_signal  %d  (since Linux 2.1.22) Signal to be sent to parent when we die.

   *
   *
processor  %d  (since Linux 2.2.8) CPU number last executed on.

   *
   *
rt_priority  %u  (since Linux 2.5.19) Real-time scheduling priority, a number in the
   * range 1 to 99 for processes scheduled under a real-time policy, or 0, for non-real-time
   * processes (see sched_setscheduler(2)).

   *
   *
policy  %u  (since Linux 2.5.19) Scheduling policy (see sched_setscheduler(2)). Decode
   * using the SCHED_* constants in linux/sched.h.  The format for this field was %lu before Linux
   * 2.6.22.

   *
   *
delayacct_blkio_ticks  %llu  (since Linux 2.6.18) Aggregated block I/O delays, measured
   * in clock ticks (centiseconds).

   *
   *
guest_time  %lu  (since Linux 2.6.24) Guest time of the process (time spent running a
   * virtual CPU for a guest operating system), measured in clock ticks (divide by
   * sysconf(_SC_CLK_TCK)).

   *
   *
cguest_time  %ld  (since Linux 2.6.24) Guest time of the process's children, measured in
   * clock ticks (divide by sysconf(_SC_CLK_TCK)).

   *
   *
start_data  %lu  (since Linux 3.3) Address above which program initialized and
   * uninitialized (BSS) data are placed.

   *
   *
end_data  %lu  (since Linux 3.3) Address below which program initialized and
   * uninitialized (BSS) data are placed.

   *
   *
start_brk  %lu  (since Linux 3.3) Address above which program heap can be expanded with
   * brk(2).

   *
   *
arg_start  %lu  (since Linux 3.5) Address above which program command-line arguments
   * (argv) are placed.

   *
   *
arg_end  %lu  (since Linux 3.5) Address below program command-line arguments (argv) are
   * placed.

   *
   *
env_start  %lu  (since Linux 3.5) Address above which program environment is placed.

   *
   *
env_end  %lu  (since Linux 3.5) Address below which program environment is placed.

   *
   *
exit_code  %d  (since Linux 3.5) The thread's exit status in the form reported by
   * waitpid(2).

   *
   *

   *
   * @return the {@link Stat} for this process
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public Stat stat() throws IOException {
    return Stat.get(pid);
  }

  /**
   *
Provides information about memory usage, measured in pages.

   *
   *
The columns are:

   *
   *

   *
size       (1) total program size (same as VmSize in /proc/[pid]/status)

   *
resident   (2) resident set size (same as VmRSS in /proc/[pid]/status)

   *
share      (3) shared pages (i.e., backed by a file)

   *
text       (4) text (code)

   *
lib        (5) library (unused in Linux 2.6)

   *
data       (6) data + stack

   *
dt         (7) dirty pages (unused in Linux 2.6)

   *

   *
   * @return the {@link Statm} for this process
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public Statm statm() throws IOException {
    return Statm.get(pid);
  }

  /**
   *
/proc/[pid]/status

   *
   *
Provides much of the information in /proc/[pid]/stat and /proc/[pid]/statm in a format
   * that's
   * easier for humans to parse.

   *
   *
Here's an example:

   *
   *

   * $ cat /proc/$$/status
   * Name:   bash
   * State:  S (sleeping)
   * Tgid:   3515
   * Pid:    3515
   * PPid:   3452
   * TracerPid:      0
   * Uid:    1000    1000    1000    1000
   * Gid:    100     100     100     100
   * FDSize: 256
   * Groups: 16 33 100
   * VmPeak:     9136 kB
   * VmSize:     7896 kB
   * VmLck:         0 kB
   * VmPin:         0 kB
   * VmHWM:      7572 kB
   * VmRSS:      6316 kB
   * VmData:     5224 kB
   * VmStk:        88 kB
   * VmExe:       572 kB
   * VmLib:      1708 kB
   * VmPMD:         4 kB
   * VmPTE:        20 kB
   * VmSwap:        0 kB
   * Threads:        1
   * SigQ:   0/3067
   * SigPnd: 0000000000000000
   * ShdPnd: 0000000000000000
   * SigBlk: 0000000000010000
   * SigIgn: 0000000000384004
   * SigCgt: 000000004b813efb
   * CapInh: 0000000000000000
   * CapPrm: 0000000000000000
   * CapEff: 0000000000000000
   * CapBnd: ffffffffffffffff
   * Seccomp:        0
   * Cpus_allowed:   00000001
   * Cpus_allowed_list:      0
   * Mems_allowed:   1
   * Mems_allowed_list:      0
   * voluntary_ctxt_switches:        150
   * nonvoluntary_ctxt_switches:     545
   *

   *
   *
The fields are as follows:

   *
   *

   *
Name: Command run by this process.

   *
State: Current state of the process.  One of "R (running)", "S (sleeping)", "D (disk
   * sleep)",
   * "T (stopped)", "T (tracing stop)", "Z (zombie)", or "X (dead)".

   *
Tgid: Thread group ID (i.e., Process ID).

   *
Pid: Thread ID (see gettid(2)).

   *
PPid: PID of parent process.

   *
TracerPid: PID of process tracing this process (0 if not being traced).

   *
Uid, Gid: Real, effective, saved set, and filesystem UIDs (GIDs).

   *
FDSize: Number of file descriptor slots currently allocated.

   *
Groups: Supplementary group list.

   *
VmPeak: Peak virtual memory size.

   *
VmSize: Virtual memory size.

   *
VmLck: Locked memory size (see mlock(3)).

   *
VmPin: Pinned memory size (since Linux 3.2).  These are pages that can't be moved because
   * something needs to directly access physical memory.

   *
VmHWM: Peak resident set size ("high water mark").

   *
VmRSS: Resident set size.

   *
VmData, VmStk, VmExe: Size of data, stack, and text segments.

   *
VmLib: Shared library code size.

   *
VmPTE: Page table entries size (since Linux 2.6.10).

   *
VmPMD: Size of second-level page tables (since Linux 4.0).

   *
VmSwap: Swapped-out virtual memory size by anonymous private pages; shmem swap usage is
   * not
   * included (since Linux 2.6.34).

   *
Threads: Number of threads in process containing this thread.

   *
SigQ: This field contains two slash-separated numbers that relate to queued signals for
   * the
   * real user ID of this process.  The first of these is the number of currently queued signals
   * for
   * this real user ID, and the second is the resource limit on the number of queued signals for
   * this
   * process (see the description of RLIMIT_SIGPENDING in getrlimit(2)).

   *
SigPnd, ShdPnd: Number of signals pending for thread and for process as a whole (see
   * pthreads(7) and signal(7)).

   *
SigBlk, SigIgn, SigCgt: Masks indicating signals being blocked, ignored, and caught (see
   * signal(7)).

   *
CapInh, CapPrm, CapEff: Masks of capabilities enabled in inheritable, permitted, and
   * effective sets (see capabilities(7)).

   *
CapBnd: Capability Bounding set (since Linux 2.6.26, see capabilities(7)).

   *
Seccomp: Seccomp mode of the process (since Linux 3.8, see seccomp(2)). 0 means
   * SECCOMP_MODE_DISABLED; 1 means SECCOMP_MODE_STRICT; 2 means SECCOMP_MODE_FILTER. This field is
   * provided only if the kernel was built with the CONFIG_SECCOMP kernel configuration option
   * enabled.

   *
Cpus_allowed: Mask of CPUs on which this process may run (since Linux 2.6.24, see
   * cpuset(7)).

   *
Cpus_allowed_list: Same as previous, but in "list format" (since Linux 2.6.26, see
   * cpuset(7)).

   *
Mems_allowed: Mask of memory nodes allowed to this process (since Linux 2.6.24, see
   * cpuset(7)).

   *
Mems_allowed_list: Same as previous, but in "list format" (since Linux 2.6.26, see
   * cpuset(7)).
   * voluntary_ctxt_switches, nonvoluntary_ctxt_switches: Number of voluntary and involuntary
   * context
   * switches (since Linux 2.6.23).

   *

   *
   * @return the {@link Status} for this process
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public Status status() throws IOException {
    return Status.get(pid);
  }

  /**
   * The symbolic name corresponding to the location in the kernel where the process is sleeping.
   *
   * @return the contents of /proc/[pid]/wchan
   * @throws IOException
   *     if the file does not exist or we don't have read permissions.
   */
  public String wchan() throws IOException {
    return read("wchan");
  }

  @Override public int describeContents() {
    return 0;
  }

  @Override public void writeToParcel(Parcel dest, int flags) {
    dest.writeString(this.name);
    dest.writeInt(this.pid);
  }

  protected AndroidProcess(Parcel in) {
    this.name = in.readString();
    this.pid = in.readInt();
  }

  public static final Creator<AndroidProcess> CREATOR = new Creator<AndroidProcess>() {

    @Override public AndroidProcess createFromParcel(Parcel source) {
      return new AndroidProcess(source);
    }

    @Override public AndroidProcess[] newArray(int size) {
      return new AndroidProcess[size];
    }
  };

}


text:
<html class="reftest-wait">
<head>
<script>
setTimeout('document.documentElement.className = ""', 1000);
</script>
<body>
<iframe src="389326-1-inner.xhtml"></iframe>
</body>
</html>


text:
#ifndef BYTEQ_H
#define BYTEQ_H 1

#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

/* Maximum number of bytes in a byteq. */
#define BYTEQ_SIZE 512

/* General-purpose circular queue of bytes. */
struct byteq {
    uint8_t buffer[BYTEQ_SIZE]; /* Circular queue. */
    unsigned int head;          /* Head of queue. */
    unsigned int tail;          /* Chases the head. */
};

void byteq_init(struct byteq *);
int byteq_used(const struct byteq *);
int byteq_avail(const struct byteq *);
bool byteq_is_empty(const struct byteq *);
bool byteq_is_full(const struct byteq *);
void byteq_put(struct byteq *, uint8_t c);
void byteq_putn(struct byteq *, const void *, size_t n);
void byteq_put_string(struct byteq *, const char *);
uint8_t byteq_get(struct byteq *);
int byteq_write(struct byteq *, int fd);
int byteq_read(struct byteq *, int fd);

uint8_t *byteq_head(struct byteq *);
int byteq_headroom(const struct byteq *);
void byteq_advance_head(struct byteq *, unsigned int n);
int byteq_tailroom(const struct byteq *);
const uint8_t *byteq_tail(const struct byteq *);
void byteq_advance_tail(struct byteq *, unsigned int n);

#endif /* byteq.h */


text:
require "cases/encryption/helper"

class ActiveRecord::Encryption::EnvelopeEncryptionKeyProviderTest < ActiveRecord::EncryptionTestCase
  setup do
    @key_provider = ActiveRecord::Encryption::EnvelopeEncryptionKeyProvider.new
  end

  test "encryption_key returns random encryption keys" do
    keys = 5.times.collect { @key_provider.encryption_key }
    assert_equal 5, keys.group_by(&:secret).length
  end

  test "generate_random_encryption_key generates keys of 32 bytes" do
    assert_equal 32, @key_provider.encryption_key.secret.bytesize
  end

  test "generated random keys carry their secret encrypted with the master key" do
    key = @key_provider.encryption_key
    encrypted_secret = key.public_tags.encrypted_data_key
    assert_equal key.secret, ActiveRecord::Encryption.cipher.decrypt(encrypted_secret, key: @key_provider.active_primary_key.secret)
  end

  test "decryption_key_for returns the decryption key for a message that was encrypted with a generated encryption key" do
    key = @key_provider.encryption_key
    encrypted_encoded_message = ActiveRecord::Encryption.encryptor.encrypt("some message", key_provider: ActiveRecord::Encryption::KeyProvider.new(key))
    encrypted_message = ActiveRecord::Encryption.message_serializer.load encrypted_encoded_message
    assert_equal key.secret, @key_provider.decryption_keys(encrypted_message).first.secret
  end

  test "work with multiple keys when config.store_key_references is false" do
    ActiveRecord::Encryption.config.primary_key = ["key 1", "key 2"]

    assert_encryptor_works_with @key_provider
  end

  test "work with multiple keys when config.store_key_references is true" do
    ActiveRecord::Encryption.config.primary_key = ["key 1", "key 2"]
    ActiveRecord::Encryption.config.store_key_references = true

    assert_encryptor_works_with @key_provider
  end

  private
    def assert_multiple_primary_keys
      assert Rails.application.credentials.dig(:active_record_encryption, :primary_key).length > 1
    end
end


text:
namespace android_webview {

// Handles the database operations necessary to implement the autocomplete
// functionality. This includes creating and initializing the components that
// handle the database backend, and providing a synchronous interface when
// needed (the chromium database components have an async. interface).
class AwFormDatabaseService : public WebDataServiceConsumer {
 public:
  AwFormDatabaseService(const base::FilePath path);

  virtual ~AwFormDatabaseService();

  void Shutdown();

  // Returns whether the database has any data stored. May do
  // IO access and block.
  bool HasFormData();

  // Clear any saved form data. Executes asynchronously.
  void ClearFormData();

  scoped_refptr<autofill::AutofillWebDataService>
      get_autofill_webdata_service();

  // WebDataServiceConsumer implementation.
  virtual void OnWebDataServiceRequestDone(
      WebDataServiceBase::Handle h,
      const WDTypedResult* result) OVERRIDE;

 private:
  // Cancels the currently pending WebDataService query, if there is one.
  void CancelPendingQuery();

  void HasFormDataImpl();

  // Stores the query handle when an async database query is executed.
  WebDataServiceBase::Handle pending_query_handle_;
  bool has_form_data_;
  base::WaitableEvent completion_;

  scoped_refptr<autofill::AutofillWebDataService> autofill_data_;
  scoped_refptr<WebDatabaseService> web_database_;

  DISALLOW_COPY_AND_ASSIGN(AwFormDatabaseService);
};

}  // namespace android_webview

#endif  // ANDROID_WEBVIEW_BROWSER_AW_FORM_DATABASE_SERVICE_H_


text:
from __future__ import absolute_import

from sentry.api.serializers import Serializer, register
from sentry.models import UserReport

@register(UserReport)
class UserReportSerializer(Serializer):
    def serialize(self, obj, attrs, user):
        # TODO(dcramer): add in various context from the event
        # context == user / http / extra interfaces
        return {
            'id': str(obj.id),
            'eventID': obj.event_id,
            'name': obj.name,
            'email': obj.email,
            'comments': obj.comments,
            'dateCreated': obj.date_added,
        }


text:
class QHBoxLayout;

namespace csapex
{
class ParameterContextMenu;

class CSAPEX_QT_EXPORT TriggerParameterAdapter : public ParameterAdapter
{
public:
    TriggerParameterAdapter(param::TriggerParameter::Ptr p);

    QWidget* setup(QBoxLayout* layout, const std::string& display_name) override;
    void setupContextMenu(ParameterContextMenu* context_handler) override;

private:
    param::TriggerParameterPtr trigger_p_;
};

}  // namespace csapex

#endif  // TRIGGER_PARAM_ADAPTER_H


text:
package de.escidocng.frontend;

import javax.annotation.PostConstruct;
import javax.servlet.MultipartConfigElement;

import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.context.embedded.MultipartConfigFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.env.Environment;
import org.springframework.web.multipart.commons.CommonsMultipartResolver;
import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;

import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationConfig;
import com.fasterxml.jackson.datatype.jsr310.JSR310Module;

import de.escidocng.frontend.util.HttpHelper;
import de.escidocng.frontend.util.EscidocngExceptionHandler;

/**
 * General JavaConfig class for the escidocng frontend containing all the necessary beans for a escidocng frontend
 */
@Configuration
@ComponentScan(basePackages = "de.escidocng.frontend.controller")
@EnableAutoConfiguration
public class EscidocngFrontendConfiguration {

    @Autowired
    public Environment env;

    @PostConstruct
    public void init() {
        if (StringUtils.isNotBlank(env.getProperty("escidocng.proxy.name"))) {
            System.setProperty("http.proxyHost", env.getProperty("escidocng.proxy.name"));
            System.setProperty("https.proxyHost", env.getProperty("escidocng.proxy.name"));
            if (StringUtils.isNotBlank(env.getProperty("escidocng.proxy.port"))) {
                System.setProperty("http.proxyPort", env.getProperty("escidocng.proxy.port"));
                System.setProperty("https.proxyPort", env.getProperty("escidocng.proxy.port"));
            }
            if (StringUtils.isNotBlank(env.getProperty("escidocng.proxy.none"))) {
                System.setProperty("http.nonProxyHosts", env.getProperty("escidocng.proxy.none"));
                System.setProperty("https.nonProxyHosts", env.getProperty("escidocng.proxy.none"));
            }
        }
    }

    /**
     * Get a {@link org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping} Spring bean
     *
     * @return a RequestMappingHandlerMapping implementation
     */
    @Bean
    public RequestMappingHandlerMapping handlerMapping() {
        return new RequestMappingHandlerMapping();
    }

    /**
     * Get a Jackson {@link com.fasterxml.jackson.databind.ObjectMapper} Spring bean for JSON
     * serialization/deserialization
     *
     * @return the {@link com.fasterxml.jackson.databind.ObjectMapper} implementation used by various services
     */
    @Bean
    @Primary
    public ObjectMapper objectMapper() {
        final ObjectMapper mapper = new ObjectMapper();
        mapper.registerModule(new JSR310Module());
        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
        return mapper;
    }

    /**
     * Get the {@link com.fasterxml.jackson.databind.SerializationConfig} Spring bean for the Jackson
     * {@link com.fasterxml.jackson.databind.ObjectMapper}
     *
     * @return the {@link com.fasterxml.jackson.databind.SerializationConfig} that should be used by the Jackson
     *         mapper
     */
    @Bean
    public SerializationConfig serializationConfig() {
        return objectMapper().getSerializationConfig();
    }

    /**
     * A commons-multipart {@link org.springframework.web.multipart.commons.CommonsMultipartResolver} for resolving
     * files in a HTTP multipart request
     *
     * @return a {@link org.springframework.web.multipart.commons.CommonsMultipartResolver} object used by Spring MVC
     */
    @Bean
    public CommonsMultipartResolver multipartResolver() {
        return new CommonsMultipartResolver();
    }

    /**
     * A commons-multipart {@link javax.servlet.MultipartConfigElement} for resolving
     * files in a HTTP multipart request
     *
     * @return a {@link javax.servlet.MultipartConfigElement} object used by Spring MVC
     */
    @Bean
    MultipartConfigElement multipartConfigElement() {
        MultipartConfigFactory factory = new MultipartConfigFactory();
        return factory.createMultipartConfig();
    }

    /**
     * A HttpHelper {@link de.escidocng.frontend.util.HttpHelper} for requesting urls
     *
     * @return a {@link de.escidocng.frontend.util.HttpHelper} object
     */
    @Bean
    public HttpHelper httpHelper() {
        return new HttpHelper();
    }

    /**
     * Get a {@link de.escidocng.frontend.util.EscidocngExceptionHandler} implementation for use by the repository
     *
     * @return a {@link de.escidocng.frontend.util.EscidocngExceptionHandler} implementation
     */
    @Bean
    public EscidocngExceptionHandler escidocngExceptionHandler() {
        return new EscidocngExceptionHandler();
    }

}


text:
/** Euphoria API focused on Beam coders. */
@Internal
package org.apache.beam.sdk.extensions.kryo;

import org.apache.beam.sdk.annotations.Internal;


text:
import * as mongoose from "mongoose";
import BaseModel from "../../BaseModel";
import { Member, Method, Model } from "../../Decorators";
import ModelRepo from "../../ModelRepo";

@Model<User>()
export abstract class User extends BaseModel {

    @Member({ type: String })
    public name: string;

    @Member({ type: Date })
    public dob: Date;

    @Member({ type: Boolean })
    public isMale: boolean;

    @Method()
    public getName(): string {
        return this.name;
    }

    @Method()
    public async updateName(name): Promise<string> {
        this.name = name;
        await this.save();
        return name;
    }

}

export class RUser extends ModelRepo<User> {

    public async getAllObjects(): Promise<User[]> {
        return this.query.find({});
    }

}

export default new RUser(User);


text:
using ImageQuery.Canvas;
using ImageQuery.Environment;
using ImageQuery.Query.Expressions;
using ImageQuery.Query.Value;

namespace ImageQuery.Query.Statements
{
    public class DefineColorStatement : IQueryStatement
    {
        public string Name { get; set; }
        public IExpression Value { get; set; }
        public void Run(IEnvironment env)
        {
            var variable = new ColorValue();
            env.CreateVariable(Name, variable);

            if (Value != null)
            {
                variable.Color = Value.Evaluate(env).Color;
            }
            else
            {
                variable.Color = new Color(0, 0, 0, 1);
            }
        }
    }
}