text:
import asyncio
import datetime

import a_sync
import asynctest
import mock
import pytest

import paasta_tools.instance.kubernetes as pik
from paasta_tools import utils
from paasta_tools.utils import DeploymentVersion
from tests.conftest import Struct
from tests.conftest import wrap_value_in_task

@pytest.fixture
def mock_pod():
    return Struct(
        metadata=Struct(
            owner_references=[Struct(kind="ReplicaSet", name="replicaset_1")],
            name="pod_1",
            namespace="paasta",
            creation_timestamp=datetime.datetime(2021, 3, 6),
            deletion_timestamp=None,
            labels={
                "paasta.yelp.com/git_sha": "aaa000",
                "paasta.yelp.com/config_sha": "config000",
                "paasta.yelp.com/service": "service",
                "paasta.yelp.com/instance": "instance",
            },
        ),
        status=Struct(
            pod_ip="1.2.3.4",
            host_ip="4.3.2.1",
            phase="Running",
            reason=None,
            message=None,
            conditions=[
                Struct(
                    type="Ready",
                    status="True",
                ),
                Struct(
                    type="PodScheduled",
                    status="True",
                ),
            ],
            container_statuses=[
                Struct(
                    name="main_container",
                    restart_count=0,
                    state=Struct(
                        running=dict(
                            reason="a_state_reason",
                            message="a_state_message",
                            started_at=datetime.datetime(2021, 3, 6),
                        ),
                        waiting=None,
                        terminated=None,
                    ),
                    last_state=Struct(
                        running=None,
                        waiting=None,
                        terminated=dict(
                            reason="a_last_state_reason",
                            message="a_last_state_message",
                            started_at=datetime.datetime(2021, 3, 4),
                            finished_at=datetime.datetime(2021, 3, 5),
                        ),
                    ),
                ),
            ],
        ),
        spec=Struct(
            containers=[
                Struct(
                    name="main_container",
                    liveness_probe=Struct(
                        initial_delay_seconds=1,
                        failure_threshold=2,
                        period_seconds=3,
                        timeout_seconds=4,
                        http_get=Struct(
                            port=8080,
                            path="/healthcheck",
                        ),
                    ),
                )
            ]
        ),
    )

def test_instance_types_integrity():
    for it in pik.INSTANCE_TYPES:
        assert it in utils.INSTANCE_TYPES
    for it in pik.INSTANCE_TYPES_WITH_SET_STATE:
        assert it in utils.INSTANCE_TYPES

def instance_status_kwargs():
    return dict(
        service="",
        instance="",
        instance_type="",
        verbose=0,
        include_smartstack=False,
        include_envoy=False,
        settings=mock.Mock(),
        use_new=False,
    )

@mock.patch("paasta_tools.instance.kubernetes.cr_status", autospec=True)
@mock.patch("paasta_tools.instance.kubernetes.kubernetes_status", autospec=True)
def test_instance_status_invalid_instance_type(mock_kubernetes_status, mock_cr_status):
    kwargs = instance_status_kwargs()
    with pytest.raises(RuntimeError) as excinfo:
        pik.instance_status(**kwargs)
    assert "Unknown instance type" in str(excinfo.value)
    assert len(mock_cr_status.mock_calls) == 0
    assert len(mock_kubernetes_status.mock_calls) == 0

@mock.patch("paasta_tools.instance.kubernetes.cr_status", autospec=True)
@mock.patch("paasta_tools.instance.kubernetes.kubernetes_status", autospec=True)
def test_instance_status_kubernetes_only(mock_kubernetes_status, mock_cr_status):
    kwargs = instance_status_kwargs()
    kwargs.update(instance_type="kubernetes")
    pik.instance_status(**kwargs)
    assert len(mock_cr_status.mock_calls) == 0
    assert len(mock_kubernetes_status.mock_calls) == 1

@mock.patch("paasta_tools.instance.kubernetes.cr_status", autospec=True)
@mock.patch("paasta_tools.instance.kubernetes.kubernetes_status", autospec=True)
def test_instance_status_cr_only(mock_kubernetes_status, mock_cr_status):
    kwargs = instance_status_kwargs()
    kwargs.update(instance_type="flink")
    pik.instance_status(**kwargs)
    assert len(mock_cr_status.mock_calls) == 1
    assert len(mock_kubernetes_status.mock_calls) == 0

@mock.patch("paasta_tools.instance.kubernetes.cr_status", autospec=True)
@mock.patch("paasta_tools.instance.kubernetes.kubernetes_status", autospec=True)
def test_instance_status_cr_and_kubernetes(mock_kubernetes_status, mock_cr_status):
    kwargs = instance_status_kwargs()
    kwargs.update(instance_type="cassandracluster")
    pik.instance_status(**kwargs)
    assert len(mock_cr_status.mock_calls) == 1
    assert len(mock_kubernetes_status.mock_calls) == 1

def test_kubernetes_status():
    with asynctest.patch(
        "paasta_tools.instance.kubernetes.job_status",
        autospec=True,
    ), asynctest.patch(
        "paasta_tools.kubernetes_tools.replicasets_for_service_instance",
        autospec=True,
    ) as mock_replicasets_for_service_instance, asynctest.patch(
        "paasta_tools.kubernetes_tools.pods_for_service_instance",
        autospec=True,
    ) as mock_pods_for_service_instance, asynctest.patch(
        "paasta_tools.kubernetes_tools.get_kubernetes_app_by_name",
        autospec=True,
    ), asynctest.patch(
        "paasta_tools.instance.kubernetes.LONG_RUNNING_INSTANCE_TYPE_HANDLERS",
        autospec=True,
    ) as mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS:
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS["flink"] = mock.Mock()
        mock_pods_for_service_instance.return_value = []
        mock_replicasets_for_service_instance.return_value = []
        status = pik.kubernetes_status(
            service="",
            instance="",
            verbose=0,
            include_smartstack=False,
            include_envoy=False,
            instance_type="flink",
            settings=mock.Mock(),
        )
        assert "app_count" in status
        assert "evicted_count" in status
        assert "bounce_method" in status
        assert "desired_state" in status

class TestKubernetesStatusV2:
    @pytest.fixture
    def mock_pods_for_service_instance(self):
        with asynctest.patch(
            "paasta_tools.kubernetes_tools.pods_for_service_instance", autospec=True
        ) as mock_pods_for_service_instance:
            yield mock_pods_for_service_instance

    @pytest.fixture
    def mock_replicasets_for_service_instance(self):
        with asynctest.patch(
            "paasta_tools.kubernetes_tools.replicasets_for_service_instance",
            autospec=True,
        ) as mock_replicasets_for_service_instance:
            yield mock_replicasets_for_service_instance

    @pytest.fixture
    def mock_mesh_status(self):
        with asynctest.patch(
            "paasta_tools.instance.kubernetes.mesh_status",
            autospec=True,
        ) as mock_mesh_status:
            yield mock_mesh_status

    @pytest.fixture
    def mock_load_service_namespace_config(self):
        with asynctest.patch(
            "paasta_tools.kubernetes_tools.load_service_namespace_config",
            autospec=True,
        ) as mock_load_service_namespace_config:
            yield mock_load_service_namespace_config

    @pytest.fixture
    def mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS(self):
        with asynctest.patch(
            "paasta_tools.instance.kubernetes.LONG_RUNNING_INSTANCE_TYPE_HANDLERS",
            autospec=True,
        ) as mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS:
            yield mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS

    @pytest.fixture
    def mock_controller_revisions_for_service_instance(self):
        with asynctest.patch(
            "paasta_tools.kubernetes_tools.controller_revisions_for_service_instance",
            autospec=True,
        ) as mock_controller_revisions_for_service_instance:
            yield mock_controller_revisions_for_service_instance

    @pytest.fixture
    def mock_get_pod_event_messages(self):
        with asynctest.patch(
            "paasta_tools.instance.kubernetes.get_pod_event_messages",
            autospec=True,
        ) as mock_get_pod_event_messages:
            yield mock_get_pod_event_messages

    def test_replicaset(
        self,
        mock_replicasets_for_service_instance,
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS,
        mock_load_service_namespace_config,
        mock_pods_for_service_instance,
        mock_mesh_status,
        mock_get_pod_event_messages,
        mock_pod,
    ):
        mock_job_config = mock.Mock(
            get_persistent_volumes=mock.Mock(return_value=[]),
        )
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS[
            "kubernetes"
        ].loader.return_value = mock_job_config
        mock_replicasets_for_service_instance.return_value = [
            Struct(
                spec=Struct(replicas=1),
                metadata=Struct(
                    name="replicaset_1",
                    creation_timestamp=datetime.datetime(2021, 3, 5),
                    deletion_timestamp=None,
                    labels={
                        "paasta.yelp.com/git_sha": "aaa000",
                        "paasta.yelp.com/config_sha": "config000",
                    },
                ),
            ),
        ]
        mock_pods_for_service_instance.return_value = [mock_pod]

        mock_load_service_namespace_config.return_value = {}
        mock_job_config.get_registrations.return_value = ["service.instance"]
        mock_get_pod_event_messages.return_value = []
        status = pik.kubernetes_status_v2(
            service="service",
            instance="instance",
            verbose=0,
            include_smartstack=False,
            include_envoy=False,
            instance_type="kubernetes",
            settings=mock.Mock(),
        )

        assert status == {
            "app_name": mock_job_config.get_sanitised_deployment_name.return_value,
            "desired_state": mock_job_config.get_desired_state.return_value,
            "desired_instances": mock_job_config.get_instances.return_value,
            "bounce_method": mock_job_config.get_bounce_method.return_value,
            "versions": [
                {
                    "type": "ReplicaSet",
                    "name": "replicaset_1",
                    "replicas": 1,
                    "ready_replicas": 0,
                    "create_timestamp": datetime.datetime(2021, 3, 5).timestamp(),
                    "git_sha": "aaa000",
                    "image_version": None,
                    "config_sha": "config000",
                    "pods": [
                        {
                            "name": "pod_1",
                            "ip": "1.2.3.4",
                            "create_timestamp": datetime.datetime(
                                2021, 3, 6
                            ).timestamp(),
                            "delete_timestamp": None,
                            "host": "4.3.2.1",
                            "phase": "Running",
                            "reason": None,
                            "message": None,
                            "scheduled": True,
                            "ready": True,
                            "mesh_ready": None,
                            "events": [],
                            "containers": [
                                {
                                    "healthcheck_grace_period": 1,
                                    "healthcheck_cmd": {
                                        "http_url": "http://1.2.3.4:8080/healthcheck"
                                    },
                                    "name": "main_container",
                                    "restart_count": 0,
                                    "state": "running",
                                    "reason": "a_state_reason",
                                    "message": "a_state_message",
                                    "last_state": "terminated",
                                    "last_reason": "a_last_state_reason",
                                    "last_message": "a_last_state_message",
                                    "last_duration": 86400.0,
                                    "last_timestamp": datetime.datetime(
                                        2021, 3, 4
                                    ).timestamp(),
                                    "previous_tail_lines": None,
                                    "timestamp": datetime.datetime(
                                        2021, 3, 6
                                    ).timestamp(),
                                    "tail_lines": {
                                        "error_message": "",
                                        "stderr": [],
                                        "stdout": [],
                                    },
                                },
                            ],
                        },
                    ],
                }
            ],
        }

    def test_statefulset(
        self,
        mock_controller_revisions_for_service_instance,
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS,
        mock_load_service_namespace_config,
        mock_pods_for_service_instance,
        mock_mesh_status,
        mock_pod,
    ):
        mock_job_config = mock.Mock(
            get_persistent_volumes=mock.Mock(return_value=[mock.Mock]),
        )
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS[
            "kubernetes"
        ].loader.return_value = mock_job_config
        mock_controller_revisions_for_service_instance.return_value = [
            Struct(
                metadata=Struct(
                    name="controller_revision_1",
                    creation_timestamp=datetime.datetime(2021, 4, 1),
                    labels={
                        "paasta.yelp.com/git_sha": "aaa000",
                        "paasta.yelp.com/config_sha": "config000",
                    },
                ),
            ),
        ]

        mock_pod.metadata.owner_references = []
        mock_pods_for_service_instance.return_value = [mock_pod]

        with asynctest.patch(
            "paasta_tools.instance.kubernetes.get_pod_status", autospec=True
        ) as mock_get_pod_status:
            mock_get_pod_status.return_value = {}
            status = pik.kubernetes_status_v2(
                service="service",
                instance="instance",
                verbose=0,
                include_smartstack=False,
                include_envoy=False,
                instance_type="kubernetes",
                settings=mock.Mock(),
            )

        assert len(status["versions"]) == 1
        assert status["versions"][0] == {
            "name": "controller_revision_1",
            "type": "ControllerRevision",
            "replicas": 1,
            "ready_replicas": 1,
            "create_timestamp": datetime.datetime(2021, 4, 1).timestamp(),
            "git_sha": "aaa000",
            "image_version": None,
            "config_sha": "config000",
            "pods": [mock.ANY],
        }

    def test_statefulset_with_image_version(
        self,
        mock_controller_revisions_for_service_instance,
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS,
        mock_load_service_namespace_config,
        mock_pods_for_service_instance,
        mock_mesh_status,
        mock_pod,
    ):
        mock_job_config = mock.Mock(
            get_persistent_volumes=mock.Mock(return_value=[mock.Mock]),
        )
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS[
            "kubernetes"
        ].loader.return_value = mock_job_config
        mock_controller_revisions_for_service_instance.return_value = [
            Struct(
                metadata=Struct(
                    name="controller_revision_1",
                    creation_timestamp=datetime.datetime(2021, 4, 1),
                    labels={
                        "paasta.yelp.com/git_sha": "aaa000",
                        "paasta.yelp.com/config_sha": "config000",
                        "paasta.yelp.com/image_version": "extrastuff",
                    },
                ),
            ),
        ]

        mock_pod.metadata.owner_references = []
        mock_pods_for_service_instance.return_value = [mock_pod]

        with asynctest.patch(
            "paasta_tools.instance.kubernetes.get_pod_status", autospec=True
        ) as mock_get_pod_status:
            mock_get_pod_status.return_value = {}
            status = pik.kubernetes_status_v2(
                service="service",
                instance="instance",
                verbose=0,
                include_smartstack=False,
                include_envoy=False,
                instance_type="kubernetes",
                settings=mock.Mock(),
            )

        assert len(status["versions"]) == 1
        assert status["versions"][0] == {
            "name": "controller_revision_1",
            "type": "ControllerRevision",
            "replicas": 1,
            "ready_replicas": 1,
            "create_timestamp": datetime.datetime(2021, 4, 1).timestamp(),
            "git_sha": "aaa000",
            "image_version": "extrastuff",
            "config_sha": "config000",
            "pods": [mock.ANY],
        }

    def test_event_timeout(
        self,
        mock_replicasets_for_service_instance,
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS,
        mock_load_service_namespace_config,
        mock_pods_for_service_instance,
        mock_mesh_status,
        mock_get_pod_event_messages,
        mock_pod,
    ):
        mock_job_config = mock.Mock(
            get_persistent_volumes=mock.Mock(return_value=[]),
        )
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS[
            "kubernetes"
        ].loader.return_value = mock_job_config
        mock_replicasets_for_service_instance.return_value = [
            Struct(
                spec=Struct(replicas=1),
                metadata=Struct(
                    name="replicaset_1",
                    creation_timestamp=datetime.datetime(2021, 3, 5),
                    deletion_timestamp=None,
                    labels={
                        "paasta.yelp.com/git_sha": "aaa000",
                        "paasta.yelp.com/config_sha": "config000",
                    },
                ),
            ),
        ]
        mock_pods_for_service_instance.return_value = [mock_pod]
        mock_load_service_namespace_config.return_value = {}
        mock_job_config.get_registrations.return_value = ["service.instance"]
        mock_get_pod_event_messages.side_effect = asyncio.TimeoutError

        status = pik.kubernetes_status_v2(
            service="service",
            instance="instance",
            verbose=0,
            include_smartstack=False,
            include_envoy=False,
            instance_type="kubernetes",
            settings=mock.Mock(),
        )

        # Verify  we did not throw an exception
        assert status
        assert all(
            p["events"] == [{"error": "Could not retrieve events. Please try again."}]
            for p in status["versions"][0]["pods"]
        )

    def test_pod_timeout(
        self,
        mock_replicasets_for_service_instance,
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS,
        mock_load_service_namespace_config,
        mock_pods_for_service_instance,
        mock_mesh_status,
        mock_get_pod_event_messages,
        mock_pod,
    ):
        mock_job_config = mock.Mock(
            get_persistent_volumes=mock.Mock(return_value=[]),
        )
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS[
            "kubernetes"
        ].loader.return_value = mock_job_config
        mock_replicasets_for_service_instance.return_value = [
            Struct(
                spec=Struct(replicas=1),
                metadata=Struct(
                    name="replicaset_1",
                    creation_timestamp=datetime.datetime(2021, 3, 5),
                    deletion_timestamp=None,
                    labels={
                        "paasta.yelp.com/git_sha": "aaa000",
                        "paasta.yelp.com/config_sha": "config000",
                    },
                ),
            ),
        ]
        mock_load_service_namespace_config.return_value = {}
        mock_job_config.get_registrations.return_value = ["service.instance"]
        mock_get_pod_event_messages.return_value = []
        mock_pods_for_service_instance.side_effect = asyncio.TimeoutError

        status = pik.kubernetes_status_v2(
            service="service",
            instance="instance",
            verbose=0,
            include_smartstack=False,
            include_envoy=False,
            instance_type="kubernetes",
            settings=mock.Mock(),
        )

        # Verify  we did not throw an exception
        assert status
        assert "Could not fetch instance data" in status["error_message"]

@mock.patch("paasta_tools.kubernetes_tools.get_kubernetes_app_by_name", autospec=True)
def test_job_status_include_replicaset_non_verbose(mock_get_kubernetes_app_by_name):
    kstatus = {}
    a_sync.block(
        pik.job_status,
        kstatus=kstatus,
        client=mock.Mock(),
        job_config=mock.Mock(),
        pod_list=[],
        replicaset_list=[mock.Mock(), mock.Mock(), mock.Mock()],
        verbose=0,
        namespace=mock.Mock(),
    )

    assert len(kstatus["replicasets"]) == 3

def test_kubernetes_status_include_smartstack():
    with asynctest.patch(
        "paasta_tools.instance.kubernetes.job_status",
        autospec=True,
    ), asynctest.patch(
        "paasta_tools.kubernetes_tools.load_service_namespace_config", autospec=True
    ) as mock_load_service_namespace_config, asynctest.patch(
        "paasta_tools.instance.kubernetes.mesh_status",
        autospec=True,
    ) as mock_mesh_status, asynctest.patch(
        "paasta_tools.kubernetes_tools.replicasets_for_service_instance", autospec=True
    ) as mock_replicasets_for_service_instance, asynctest.patch(
        "paasta_tools.kubernetes_tools.pods_for_service_instance",
        autospec=True,
    ) as mock_pods_for_service_instance, asynctest.patch(
        "paasta_tools.kubernetes_tools.get_kubernetes_app_by_name",
        autospec=True,
    ), asynctest.patch(
        "paasta_tools.instance.kubernetes.LONG_RUNNING_INSTANCE_TYPE_HANDLERS",
        autospec=True,
    ) as mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS:
        mock_load_service_namespace_config.return_value = {"proxy_port": 1234}
        mock_LONG_RUNNING_INSTANCE_TYPE_HANDLERS["flink"] = mock.Mock()
        mock_pods_for_service_instance.return_value = []
        mock_replicasets_for_service_instance.return_value = []
        mock_service = mock.Mock()
        status = pik.kubernetes_status(
            service=mock_service,
            instance="",
            verbose=0,
            include_smartstack=True,
            include_envoy=False,
            instance_type="flink",
            settings=mock.Mock(),
        )
        assert (
            mock_load_service_namespace_config.mock_calls[0][2]["service"]
            is mock_service
        )
        assert mock_mesh_status.mock_calls[0][2]["service"] is mock_service
        assert "app_count" in status
        assert "evicted_count" in status
        assert "bounce_method" in status
        assert "desired_state" in status

def test_cr_status_bad_instance_type():
    with pytest.raises(RuntimeError) as excinfo:
        pik.cr_status(
            service="",
            instance="",
            verbose=0,
            instance_type="marathon",
            kube_client=mock.Mock(),
        )
    assert "Unknown instance type" in str(excinfo.value)

@mock.patch("paasta_tools.kubernetes_tools.get_cr", autospec=True)
def test_cr_status_happy_path(mock_get_cr):
    mock_status = mock.Mock()
    mock_metadata = mock.Mock()
    mock_return = dict(status=mock_status, metadata=mock_metadata)
    mock_get_cr.return_value = mock_return
    status = pik.cr_status(
        service="",
        instance="",
        verbose=0,
        instance_type="flink",
        kube_client=mock.Mock(),
    )
    assert status == mock_return

def test_set_cr_desired_state_invalid_instance_type():
    with pytest.raises(RuntimeError) as excinfo:
        pik.set_cr_desired_state(
            kube_client=mock.Mock(),
            service=mock.Mock(),
            instance=mock.Mock(),
            instance_type="marathon",
            desired_state=mock.Mock(),
        )
    assert "Unknown instance type" in str(excinfo.value)

@mock.patch("paasta_tools.kubernetes_tools.set_cr_desired_state", autospec=True)
def test_set_cr_desired_state_calls_k8s_tools(mock_set_cr_desired_state):
    pik.set_cr_desired_state(
        kube_client=mock.Mock(),
        service=mock.Mock(),
        instance=mock.Mock(),
        instance_type="flink",
        desired_state=mock.Mock(),
    )
    assert len(mock_set_cr_desired_state.mock_calls) == 1

def test_can_set_state():
    for it in pik.INSTANCE_TYPES_WITH_SET_STATE:
        assert pik.can_set_state(it)

    assert not pik.can_set_state("marathon")

def test_can_handle():
    for it in pik.INSTANCE_TYPES:
        assert pik.can_handle(it)

    assert not pik.can_handle("marathon")

def test_filter_actually_running_replicasets():
    replicaset_list = [
        mock.Mock(),
        mock.Mock(),
        mock.Mock(),
        mock.Mock(),
    ]
    # the `spec` kwarg is special to Mock so we have to set it this way.
    replicaset_list[0].configure_mock(
        **{"spec.replicas": 5, "status.ready_replicas": 5}
    )
    replicaset_list[1].configure_mock(
        **{"spec.replicas": 5, "status.ready_replicas": 0}
    )
    replicaset_list[2].configure_mock(
        **{"spec.replicas": 0, "status.ready_replicas": 0}
    )
    replicaset_list[3].configure_mock(
        **{"spec.replicas": 0, "status.ready_replicas": 5}
    )

    expected = [
        replicaset_list[0],
        replicaset_list[1],
        replicaset_list[3],
    ]
    assert pik.filter_actually_running_replicasets(replicaset_list) == expected

@pytest.mark.asyncio
async def test_get_pod_status_mesh_ready(event_loop):
    with asynctest.patch(
        "paasta_tools.instance.kubernetes.get_pod_containers", autospec=True
    ) as mock_get_pod_containers, asynctest.patch(
        "paasta_tools.kubernetes_tools.is_pod_scheduled", autospec=True
    ) as mock_is_pod_scheduled, asynctest.patch(
        "paasta_tools.kubernetes_tools.get_pod_event_messages", autospec=True
    ) as mock_get_pod_event_messages:
        mock_get_pod_containers.return_value = []
        mock_get_pod_event_messages.return_value = []
        mock_is_pod_scheduled.return_value = True
        mock_pod = mock.MagicMock()
        mock_pod.status.pod_ip = "1.2.3.4"
        mock_ready_condition = mock.MagicMock()
        mock_ready_condition.type = "Ready"
        mock_ready_condition.status = "True"
        mock_kube_client = mock.MagicMock()
        mock_pod.status.conditions = [mock_ready_condition]
        backends_task = wrap_value_in_task([{"address": "0.0.0.0"}])
        status = await pik.get_pod_status(mock_pod, backends_task, mock_kube_client, 10)
    assert status["ready"]
    assert not status["mesh_ready"]

@pytest.mark.parametrize(
    "include_smartstack,include_envoy,expected",
    [
        (True, True, ("smartstack", "envoy")),
        (True, False, ("smartstack",)),
        (False, True, ("envoy",)),
    ],
)
def test_kubernetes_mesh_status(
    include_smartstack,
    include_envoy,
    expected,
):
    with asynctest.patch(
        "paasta_tools.kubernetes_tools.load_service_namespace_config", autospec=True
    ) as mock_load_service_namespace_config, asynctest.patch(
        "paasta_tools.instance.kubernetes.mesh_status", autospec=True
    ) as mock_mesh_status, asynctest.patch(
        "paasta_tools.kubernetes_tools.pods_for_service_instance", autospec=True
    ) as mock_pods_for_service_instance, asynctest.patch(
        "paasta_tools.instance.kubernetes.LONG_RUNNING_INSTANCE_TYPE_HANDLERS",
        {"flink": mock.Mock()},
        autospec=False,
    ):
        mock_load_service_namespace_config.return_value = {"proxy_port": 1234}
        mock_pods_for_service_instance.return_value = ["pod_1"]
        mock_job_config = pik.LONG_RUNNING_INSTANCE_TYPE_HANDLERS[
            "flink"
        ].loader.return_value
        mock_settings = mock.Mock()

        kmesh = pik.kubernetes_mesh_status(
            service="fake_service",
            instance="fake_instance",
            instance_type="flink",
            settings=mock_settings,
            include_smartstack=include_smartstack,
            include_envoy=include_envoy,
        )

        assert len(kmesh) == len(expected)
        for i in range(len(expected)):
            mesh_type = expected[i]
            assert kmesh.get(mesh_type) == mock_mesh_status.return_value
            assert mock_mesh_status.call_args_list[i] == mock.call(
                service="fake_service",
                instance=mock_job_config.get_nerve_namespace.return_value,
                job_config=mock_job_config,
                service_namespace_config={"proxy_port": 1234},
                pods_task=mock.ANY,
                should_return_individual_backends=True,
                settings=mock_settings,
                service_mesh=getattr(pik.ServiceMesh, mesh_type.upper()),
            )
            _, kwargs = mock_mesh_status.call_args_list[i]
            assert kwargs["pods_task"].result() == ["pod_1"]

@mock.patch(
    "paasta_tools.kubernetes_tools.load_service_namespace_config", autospec=True
)
@mock.patch("paasta_tools.instance.kubernetes.mesh_status", autospec=True)
@mock.patch(
    "paasta_tools.kubernetes_tools.pods_for_service_instance",
    mock.Mock(return_value=("pod_1")),
    autospec=False,
)
@mock.patch(
    "paasta_tools.instance.kubernetes.LONG_RUNNING_INSTANCE_TYPE_HANDLERS",
    {"flink": mock.Mock()},
    autospec=False,
)
@pytest.mark.parametrize(
    "include_mesh,inst_type,service_ns_conf,expected_msg",
    [
        (False, "flink", {"proxy_port": 1234}, "No mesh types"),
        (True, "tron", {"proxy_port": 1234}, "not supported"),
        (True, "flink", {}, "not configured"),
    ],
)
def test_kubernetes_mesh_status_error(
    mock_mesh_status,
    mock_load_service_namespace_config,
    include_mesh,
    inst_type,
    service_ns_conf,
    expected_msg,
):
    mock_load_service_namespace_config.return_value = service_ns_conf
    mock_settings = mock.Mock()

    with pytest.raises(RuntimeError) as excinfo:
        pik.kubernetes_mesh_status(
            service="fake_service",
            instance="fake_instance",
            instance_type=inst_type,
            settings=mock_settings,
            include_smartstack=include_mesh,
            include_envoy=include_mesh,
        )

    assert expected_msg in excinfo.value.args[0]
    assert mock_mesh_status.call_args_list == []

def test_bounce_status():
    with asynctest.patch(
        "paasta_tools.instance.kubernetes.kubernetes_tools", autospec=True
    ) as mock_kubernetes_tools:
        mock_config = mock_kubernetes_tools.load_kubernetes_service_config.return_value
        mock_kubernetes_tools.get_kubernetes_app_deploy_status.return_value = (
            "deploy_status",
            "message",
        )
        mock_kubernetes_tools.get_active_versions_for_service.return_value = [
            (DeploymentVersion("aaa", None), "config_aaa"),
            (DeploymentVersion("bbb", None), "config_bbb"),
            (DeploymentVersion("ccc", "extrastuff"), "config_ccc"),
        ]

        mock_settings = mock.Mock()
        status = pik.bounce_status("fake_service", "fake_instance", mock_settings)
        assert status == {
            "expected_instance_count": mock_config.get_instances.return_value,
            "desired_state": mock_config.get_desired_state.return_value,
            "running_instance_count": mock_kubernetes_tools.get_kubernetes_app_by_name.return_value.status.ready_replicas,
            "deploy_status": mock_kubernetes_tools.KubernetesDeployStatus.tostring.return_value,
            "active_shas": [
                ("aaa", "config_aaa"),
                ("bbb", "config_bbb"),
                ("ccc", "config_ccc"),
            ],
            "active_versions": [
                ("aaa", None, "config_aaa"),
                ("bbb", None, "config_bbb"),
                ("ccc", "extrastuff", "config_ccc"),
            ],
            "app_count": 3,
        }

@pytest.mark.asyncio
async def test_get_pod_containers(mock_pod):
    mock_client = mock.Mock()

    with asynctest.patch(
        "paasta_tools.instance.kubernetes.get_tail_lines_for_kubernetes_container",
        side_effect=[["current"], ["previous"], ["current"], ["previous"]],
        autospec=None,
    ), mock.patch(
        "paasta_tools.kubernetes_tools.recent_container_restart",
        return_value=True,
        autospec=None,
    ):
        containers = await pik.get_pod_containers(mock_pod, mock_client, 10)
        mock_pod.status.container_statuses[0].state.running["started_at"] = None
        no_start_containers = await pik.get_pod_containers(mock_pod, mock_client, 10)

    assert containers == [
        dict(
            name="main_container",
            restart_count=0,
            state="running",
            reason="a_state_reason",
            message="a_state_message",
            last_state="terminated",
            last_reason="a_last_state_reason",
            last_message="a_last_state_message",
            last_duration=86400.0,
            last_timestamp=datetime.datetime(2021, 3, 4).timestamp(),
            previous_tail_lines=["previous"],
            timestamp=datetime.datetime(2021, 3, 6).timestamp(),
            healthcheck_grace_period=1,
            healthcheck_cmd={"http_url": "http://1.2.3.4:8080/healthcheck"},
            tail_lines=["current"],
        ),
    ]

    assert no_start_containers[0]["timestamp"] is None


text:
module Azure::ServiceFabric::V7_0_0_42
  module Models
    #
    # Describes a reference to a volume resource.
    #
    class VolumeReference

      include MsRestAzure

      # @return [String] Name of the volume being referenced.
      attr_accessor :name

      # @return [Boolean] The flag indicating whether the volume is read only.
      # Default is 'false'.
      attr_accessor :read_only

      # @return [String] The path within the container at which the volume
      # should be mounted. Only valid path characters are allowed.
      attr_accessor :destination_path

      #
      # Mapper for VolumeReference class as Ruby Hash.
      # This will be used for serialization/deserialization.
      #
      def self.mapper()
        {
          client_side_validation: true,
          required: false,
          serialized_name: 'VolumeReference',
          type: {
            name: 'Composite',
            class_name: 'VolumeReference',
            model_properties: {
              name: {
                client_side_validation: true,
                required: true,
                serialized_name: 'name',
                type: {
                  name: 'String'
                }
              },
              read_only: {
                client_side_validation: true,
                required: false,
                serialized_name: 'readOnly',
                type: {
                  name: 'Boolean'
                }
              },
              destination_path: {
                client_side_validation: true,
                required: true,
                serialized_name: 'destinationPath',
                type: {
                  name: 'String'
                }
              }
            }
          }
        }
      end
    end
  end
end

redpajama_set_name: RedPajamaGithub


text:
var gulp        = require('gulp');
var browserSync = require('browser-sync');
var less        = require('gulp-less')
var browserify  = require('browserify');
var babelify    = require('babelify');
var source      = require('vinyl-source-stream');
var watch       = require('gulp-watch');

//  first arguement is the files I want to watch, the second is the task to run
gulp.task('watch', function(){
  gulp.watch(['./clientReact/*.jsx'], ['react'])
  gulp.watch(['./server/public/styles/*.less'], ['compile-less'])
  // gulp.watch("./server/views/*.html").on('change', browserSync.reload);
})

gulp.task('react', function(){

  return browserify({
            entries: './clientReact/app.jsx',
            extensions: ['.jsx'],
            debug: true
          })
          .transform('babelify', {presets: ["react", "es2015"]})

          .bundle()
          .pipe(source('build.js'))
          .pipe(gulp.dest('./server/public'))
})

gulp.task('compile-less', function(){
  gulp.src('./server/public/styles/main.less')
  .pipe(less())
  .pipe(gulp.dest('./server/public/styles'));
})

gulp.task('default', ['compile-less', 'react', 'watch'])


text:
<?xml version="1.0" encoding="UTF-8"?>
<!--
 Copyright 2009 The Kuali Foundation

 Licensed under the Educational Community License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.opensource.org/licenses/ecl2.php

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-->
<!DOCTYPE struts-config PUBLIC "-//Apache Software Foundation//DTD Struts Configuration 1.3//EN" "http://jakarta.apache.org/struts/dtds/struts-config_1_3.dtd">

<!--
	This file is an example Rice client application struts configuration file.
     It includes various form beans, forwards, and mappings that are needed
     by the KNS web application framework
-->

<struts-config>

  <form-beans>
    <!-- Begin required KNS forms -->
    <form-bean name="BackdoorForm" type="org.kuali.rice.kns.web.struts.form.BackdoorForm" />
    <form-bean name="KualiForm" type="org.kuali.rice.krad.web.struts.form.KualiForm" />
    <form-bean name="KualiLogoutForm" type="org.kuali.rice.kns.web.struts.form.KualiLogoutForm" />
    <!-- End required KNS forms -->
  </form-beans>

  <global-exceptions>
    <exception type="java.lang.Throwable"
		handler="org.kuali.rice.kns.web.struts.form.pojo.StrutsExceptionIncidentHandler"
		key="meaningless" />
	<exception type="org.kuali.rice.krad.exception.AuthorizationException"
    	handler="org.kuali.rice.kns.web.struts.form.pojo.AuthorizationExceptionHandler"
        key="meaningless" />
  </global-exceptions>

  <global-forwards>
    <!-- Begin required KNS forwards -->
    <forward name="save" path="/portal.jsp" redirect="true" />
    <forward name="cancel" path="/portal.jsp" redirect="true" />
    <forward name="close" path="/portal.jsp" redirect="true" />
    <forward name="portal" path="/portal.jsp" redirect="true" />
    <forward name="route_report" path="/WEB-INF/jsp/RoutingReportForward.jsp"/>
    <forward name="updateTextArea" path="/kr/WEB-INF/jsp/TextArea.jsp" />
    <forward name="exceptionIncidentHandler" path="/kr/kualiExceptionIncidentReport.do"/>
	<forward name="authorizationExceptionHandler" path="/kr/authorizationExceptionReport.do"/>
    <!-- End required KNS forwards -->
  </global-forwards>

  <action-mappings>
    <!-- Begin required KNS mappings -->
    <action path="/portal" name="KualiForm"
     type="org.kuali.rice.kns.web.struts.action.KualiSimpleAction">
      <forward name="basic" path="/portal.jsp" />
    </action>

    <action path="/backdoorlogin" name="BackdoorForm" scope="request"
            parameter="methodToCall"
            input="/WEB-INF/jsp/backdoor/index.jsp"
            type="org.kuali.rice.kns.web.struts.action.BackdoorAction">
      <forward name="basic" path="/portal.jsp" />
      <forward name="viewPortal" path="/portal.jsp" />
      <forward name="logout" path="/logout.do" />
    </action>

    <action path="/logout" name="KualiLogoutForm" type="org.kuali.rice.kns.web.struts.action.KualiLogoutAction" />

    <action path="/index" name="KualiForm"
            type="org.kuali.rice.kns.web.struts.action.KualiSimpleAction">
      <forward name="basic" path="/portal.jsp" />
    </action>
    <!-- An action mapping to "/updateTextArea" is needed here when Javascript is enabled. -->
    <action path="/updateTextArea" name="KualiForm"
            type="org.kuali.rice.kns.web.struts.action.KualiDocumentActionBase">
      <forward name="basic" path="/kr/WEB-INF/jsp/TextArea.jsp" />
    </action>
    <!-- End required KNS mappings -->
  </action-mappings>

  <!-- The standard Struts Request Processor for the KNS -->
  <controller
    processorClass="org.kuali.rice.kns.web.struts.action.KualiRequestProcessor" />

  <!-- Replace this with the path to your own ApplicationResources.properties file which
  	   contains externalized messages for internationalization purposes

  	   This file must contain, at the very least, an app.title property

   -->
  <message-resources
   factory="org.kuali.rice.kns.web.struts.action.KualiPropertyMessageResourcesFactory"
   parameter="my.app.package.ApplicationResources" />
</struts-config>


text:
namespace Azure.AI.AnomalyDetector.Models
{
    /// <summary> The AlignPolicy. </summary>
    public partial class AlignPolicy
    {
        /// <summary> Initializes a new instance of AlignPolicy. </summary>
        public AlignPolicy()
        {
        }

        /// <summary> Initializes a new instance of AlignPolicy. </summary>
        /// <param name="alignMode"> An optional field, indicates how we align different variables into the same time-range which is required by the model.{Inner, Outer}. </param>
        /// <param name="fillNAMethod"> An optional field, indicates how missed values will be filled with. Can not be set to NotFill, when alignMode is Outer.{Previous, Subsequent, Linear, Zero, Fix, NotFill}. </param>
        /// <param name="paddingValue"> optional field, only be useful if FillNAMethod is set to Pad. </param>
        internal AlignPolicy(AlignMode? alignMode, FillNAMethod? fillNAMethod, int? paddingValue)
        {
            AlignMode = alignMode;
            FillNAMethod = fillNAMethod;
            PaddingValue = paddingValue;
        }

        /// <summary> An optional field, indicates how we align different variables into the same time-range which is required by the model.{Inner, Outer}. </summary>
        public AlignMode? AlignMode { get; set; }
        /// <summary> An optional field, indicates how missed values will be filled with. Can not be set to NotFill, when alignMode is Outer.{Previous, Subsequent, Linear, Zero, Fix, NotFill}. </summary>
        public FillNAMethod? FillNAMethod { get; set; }
        /// <summary> optional field, only be useful if FillNAMethod is set to Pad. </summary>
        public int? PaddingValue { get; set; }
    }
}


text:
/* Derived from Apache Ant. */
package org.dita.dost.invoker;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Project;
import org.dita.dost.util.Configuration;

import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;

/**
 * Command line arguments.
 *
 * @since 3.4
 */
abstract class Arguments {

    boolean useColor;
    File logFile;
    /**
     * Our current message output status. Follows Project.MSG_XXX.
     */
    int msgOutputLevel = Project.MSG_WARN;
    /**
     * File that we are using for configuration.
     */
    File buildFile;
    /**
     * Names of classes to add as listeners to project.
     */
    Vector<String> listeners;
    /**
     * Indicates whether this build is to support interactive input
     */
    boolean allowInput;
    /**
     * keep going mode
     */
    boolean keepGoingMode;
    /**
     * The Ant logger class. There may be only one logger. It will have the
     * right to use the 'out' PrintStream. The class must implements the
     * BuildLogger interface.
     */
    String loggerClassname;
    /**
     * The Ant InputHandler class. There may be only one input handler.
     */
    String inputHandlerClassname;
    /**
     * Whether or not output to the log is to be unadorned.
     */
    boolean emacsMode;
    /**
     * optional thread priority
     */
    Integer threadPriority;
    /**
     * proxy flag: default is false
     */
    boolean proxy;
    boolean justPrintUsage;
    boolean justPrintVersion;
    boolean justPrintDiagnostics;
    final Map<String, Object> definedProps = new HashMap<>();

    Arguments() {
        useColor = getUseColor();
    }

    abstract void printUsage(boolean compact);

    abstract Arguments parse(String[] arguments);

    boolean getUseColor() {
        final String os = System.getProperty("os.name");
        if (os != null && os.startsWith("Windows")) {
            return false;
        } else if (System.getenv("NO_COLOR") != null) {
            return false;
        } else if (Objects.equals(System.getenv("TERM"), "dumb")) {
            return false;
        }
        return Boolean.parseBoolean(Configuration.configuration.getOrDefault("cli.color", "true"));
    }

    boolean isLongForm(String arg, String property) {
        final String name = arg.contains("=") ? arg.substring(0, arg.indexOf('=')) : arg;
        return name.equals(property) || name.equals("-" + property);
    }

    void parseCommonOptions(final String arg, final Deque<String> args) {
        if (isLongForm(arg, "-help") || arg.equals("-h")) {
            justPrintUsage = true;
        } else if (isLongForm(arg, "-verbose") || arg.equals("-v")) {
            msgOutputLevel = Project.MSG_INFO;
        } else if (isLongForm(arg, "-debug") || arg.equals("-d")) {
            msgOutputLevel = Project.MSG_VERBOSE;
        } else if (isLongForm(arg, "-emacs") || arg.equals("-e")) {
            emacsMode = true;
        } else if (isLongForm(arg, "-logfile") || arg.equals("-l")) {
            handleArgLogFile(arg, args);
        } else if (isLongForm(arg, "-buildfile") || isLongForm(arg, "-file")) {
            handleArgBuildFile(args);
        } else if (isLongForm(arg, "-listener")) {
            handleArgListener(args);
        } else if (isLongForm(arg, "-logger")) {
            handleArgLogger(args);
        } else if (isLongForm(arg, "-no-color")) {
            useColor = false;
        } else {
            throw new BuildException("Unsupported argument: %s", arg);
        }
    }

    void handleArgLogFile(String arg, Deque<String> args) {
        final Map.Entry<String, String> entry = parse(arg, args);
        if (entry.getValue() == null) {
            throw new BuildException("Missing value for log file " + entry.getKey());
        }
        logFile = new File(entry.getValue());
        useColor = false;
    }

    Map.Entry<String, String> parse(final String arg, final Deque<String> args) {
        String name = arg;
        String value;
        final int posEq = name.indexOf("=");
        if (posEq > 0) {
            value = name.substring(posEq + 1);
            name = name.substring(0, posEq);
        } else {
            value = args.pop();
        }
        return new AbstractMap.SimpleEntry<>(name, value);
    }

    /**
     * Handle the --buildfile, --file, -f argument
     */
    void handleArgBuildFile(final Deque<String> args) {
        final String value = args.pop();
        if (value == null) {
            throw new BuildException("You must specify a buildfile when using the --buildfile argument");
        }
        buildFile = new File(value.replace('/', File.separatorChar));
    }

    /**
     * Handle --listener argument
     */
    void handleArgListener(final Deque<String> args) {
        final String value = args.pop();
        if (value == null) {
            throw new BuildException("You must specify a classname when using the --listener argument");
        }
        listeners.addElement(value);
    }

    /**
     * Handle the --logger argument.
     */
    private void handleArgLogger(final Deque<String> args) {
        if (loggerClassname != null) {
            throw new BuildException("Only one logger class may be specified.");
        }
        loggerClassname = args.pop();
        if (loggerClassname == null) {
            throw new BuildException("You must specify a classname when using the -logger argument");
        }
    }

    static abstract class Argument {
        final String property;
        final String desc;

        Argument(final String property, final String desc) {
            this.property = property;
            this.desc = desc;
        }

        abstract String getValue(final String value);
    }

    static class StringArgument extends Argument {
        StringArgument(final String property, final String desc) {
            super(property, desc);
        }

        @Override
        String getValue(final String value) {
            return value;
        }
    }

    static class BooleanArgument extends Argument {
        final String trueValue;
        final String falseValue;

        BooleanArgument(final String property, final String desc, final String trueValue, final String falseValue) {
            super(property, desc);
            this.trueValue = trueValue;
            this.falseValue = falseValue;
        }

        @Override
        String getValue(final String value) {
            switch (value.toLowerCase()) {
                case "true":
                case "yes":
                case "on":
                case "1":
                    return trueValue;
                default:
                    return falseValue;
            }
        }
    }

    static class EnumArgument extends Argument {
        final Set<String> values;

        EnumArgument(final String property, final String desc, final Set<String> values) {
            super(property, desc);
            this.values = values;
        }

        @Override
        String getValue(final String value) {
            if (!values.contains(value)) {
                throw new BuildException("Invalid value for property " + property + ": " + value);
            }
            return value;
        }
    }

    static class FileArgument extends Argument {
        FileArgument(final String property, final String desc) {
            super(property, desc);
        }

        @Override
        String getValue(final String value) {
            return Paths.get(value).normalize().toString();
        }
    }

    static class AbsoluteFileArgument extends Argument {
        AbsoluteFileArgument(final String property, final String desc) {
            super(property, desc);
        }

        @Override
        String getValue(final String value) {
            return Paths.get(value).toAbsolutePath().normalize().toString();
        }
    }

    static class AbsoluteFileListArgument extends Argument {
        AbsoluteFileListArgument(final String property, final String desc) {
            super(property, desc);
        }

        @Override
        String getValue(final String value) {
            return Arrays.stream(value.split(File.pathSeparator))
                    .map(oneFile -> new File(oneFile).getAbsolutePath())
                    .collect(Collectors.joining(File.pathSeparator));
        }
    }

    static class FileOrUriArgument extends Argument {
        FileOrUriArgument(final String property, final String desc) {
            super(property, desc);
        }

        @Override
        String getValue(final String value) {
            final Path f = Paths.get(value).toAbsolutePath().normalize();
            if (Files.exists(f)) {
                return f.toString();
            } else {
                return value;
            }
        }
    }

}


text:
layout: page
number: 8
title: "Evaluation of gene-obesity interaction effects on cholesterol levels: a genetic predisposition score on HDL-cholesterol is modified by obesity"
authors: Lamina C, Forer L, Schönherr S, Kollerits B, Ried JS, Gieger C, Peters A, Wichmann HE, Kronenberg F
journal: Atherosclerosis
volume: 225
issue: 2
pages: 363-369
pmid: 23058813
pubdate:
  year: 2012
  month: Dec
doi: 10.1016/j.atherosclerosis.2012.09.016
---


text:
package org.openkilda.wfm.topology.utils;

import static org.openkilda.messaging.Utils.CORRELATION_ID;
import static org.openkilda.messaging.Utils.DEFAULT_CORRELATION_ID;
import static org.openkilda.wfm.protocol.BoltToBoltMessage.FIELD_ID_CORRELATION_ID;
import static org.openkilda.wfm.protocol.JsonMessage.FIELD_ID_JSON;
import static org.openkilda.wfm.topology.AbstractTopology.MESSAGE_FIELD;

import org.openkilda.messaging.Message;
import org.openkilda.wfm.AbstractBolt;
import org.openkilda.wfm.CommandContext;

import com.google.common.base.Strings;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

/**
 * An aspect for IRichBolt / IStatefulBolt which decorates processing of a tuple with the logger context.
 */
@Aspect
public class LoggerContextInitializer {
    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerContextInitializer.class);

    /**
     * Wraps "execute" method of storm bolts to inject/cleanup fields in logger MDC.
     */
    @Around("execution(* org.apache.storm.topology.IRichBolt+.execute(org.apache.storm.tuple.Tuple)) && args(input)"
            + "|| execution(* org.apache.storm.topology.IStatefulBolt+.execute(org.apache.storm.tuple.Tuple)) "
            + "&& args(input)")
    public Object aroundAdvice(ProceedingJoinPoint joinPoint, Tuple input) throws Throwable {
        CommandContext context = extract(input)
                .orElseGet(() -> {
                    LOGGER.debug("CorrelationId was not sent or can't be extracted for tuple {}", input);
                    return new CommandContext(DEFAULT_CORRELATION_ID);
                });

        Map<String, String> fields = prepareFields(context);
        Map<String, String> current = inject(fields);

        try {
            return joinPoint.proceed(joinPoint.getArgs());
        } finally {
            revert(current);
        }
    }

    /**
     * Extract the context information (correlation id) from a tuple.
     *
     * @param input a tuple to process
     * @return the context if required information is found
     */
    public static Optional<CommandContext> extract(Tuple input) {
        final Fields fields = input.getFields();
        if (fields == null || fields.size() == 0) {
            return Optional.empty();
        }

        if (fields.contains(AbstractBolt.FIELD_ID_CONTEXT)) {
            Object context = input.getValueByField(AbstractBolt.FIELD_ID_CONTEXT);
            if (context instanceof CommandContext) {
                return Optional.of((CommandContext) context);
            } else {
                return Optional.empty();
            }
        }

        if (fields.contains(FIELD_ID_CORRELATION_ID)) {
            Optional<CommandContext> context = Optional.ofNullable(input.getStringByField(FIELD_ID_CORRELATION_ID))
                    .map(String::trim)
                    .map(Strings::emptyToNull)
                    .map(CommandContext::new);
            if (context.isPresent()) {
                return context;
            }
        }

        if (fields.contains(MESSAGE_FIELD)) {
            Object messageField = input.getValueByField(MESSAGE_FIELD);
            if (messageField instanceof Message) {
                Optional<CommandContext> context = Optional.ofNullable(((Message) messageField).getCorrelationId())
                        .map(String::trim)
                        .map(Strings::emptyToNull)
                        .map(CommandContext::new);
                if (context.isPresent()) {
                    return context;
                }
            } else if (messageField instanceof String) {
                String corrId = exractFromJson((String) messageField);
                if (corrId != null) {
                    return Optional.of(new CommandContext(corrId));
                }
            }
        }

        if (fields.contains(FIELD_ID_JSON)) {
            String corrId = exractFromJson(input.getStringByField(FIELD_ID_JSON));
            if (corrId != null) {
                return Optional.of(new CommandContext(corrId));
            }
        }

        return Optional.empty();
    }

    private static String exractFromJson(String jsonStr) {
        JsonParser jsonParser = new JsonParser();
        try {
            JsonElement json = jsonParser.parse(jsonStr);
            if (json.isJsonObject()) {
                JsonPrimitive corrIdObj = ((JsonObject) json).getAsJsonPrimitive(CORRELATION_ID);
                if (corrIdObj != null && corrIdObj.isString()) {
                    return corrIdObj.getAsString();
                }
            }
        } catch (JsonParseException ex) {
            LOGGER.warn("Unable to parse message payload as a JSON object.", ex);
        }
        return null;
    }

    private static Map<String, String> prepareFields(CommandContext context) {
        Map<String, String> fields = new HashMap<>();
        fields.put(CORRELATION_ID, context.getCorrelationId());
        if (context.getKafkaTopic() != null) {
            fields.put("kafka.partition", String.format("%s-%s", context.getKafkaTopic(), context.getKafkaPartition()));
            fields.put("kafka.offset", context.getKafkaOffset().toString());
        }
        return fields;
    }

    private static Map<String, String> inject(Map<String, String> fields) {
        Map<String, String> current = new HashMap<>();
        for (Map.Entry<String, String> entry : fields.entrySet()) {
            String key = entry.getKey();
            current.put(key, MDC.get(key));
            MDC.put(key, entry.getValue());
        }
        return current;
    }

    private static void revert(Map<String, String> fields) {
        for (Map.Entry<String, String> entry : fields.entrySet()) {
            String value = entry.getValue();
            if (value == null) {
                MDC.remove(entry.getKey());
            } else {
                MDC.put(entry.getKey(), entry.getValue());
            }
        }
    }
}


text:
"""
Tests for editing Wikibase items.

Tests which should fail should instead be in the TestWikibaseSaveTest
class in edit_failiure_tests.py
"""
#
# (C) Pywikibot team, 2014-2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

import time

import pywikibot
from pywikibot.tools import MediaWikiVersion

from tests.aspects import unittest, WikibaseTestCase

class TestWikibaseWriteGeneral(WikibaseTestCase):

    """Run general wikibase write tests."""

    family = 'wikidata'
    code = 'test'

    user = True
    write = True

    def test_label_set(self):
        """Test setting an English label."""
        testsite = self.get_repo()
        item = pywikibot.ItemPage(testsite, 'Q68')
        self.assertIsInstance(item, pywikibot.ItemPage)
        self.assertEqual(item.getID(), 'Q68')
        item.editLabels({'en': 'Test123'})
        item.get(force=True)
        self.assertEqual(item.labels['en'], 'Test123')

    def test_label_remove(self):
        """Test adding a Farsi and English label and removing the Farsi one."""
        testsite = self.get_repo()
        item = pywikibot.ItemPage(testsite, 'Q68')
        # These two should be additive
        item.editLabels({'en': 'Test123'})
        item.editLabels({'fa': 'Test123'})
        item.get(force=True)
        self.assertIn('en', item.labels.keys())
        self.assertIn('fa', item.labels.keys())

        # This should remove the 'fa' label
        item.editLabels({'en': 'Test123', 'fa': ''})

        # Check 'fa' label is removed
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.get()
        self.assertNotIn('fa', item.labels.keys())

    def test_alias_set(self):
        """Test setting an English alias."""
        testsite = self.get_repo()
        ts = str(time.time())
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.editAliases({'en': [ts]})

    def test_add_claim_with_qualifier(self):
        """Test adding a claim with a qualifier to an item and a property."""
        testsite = self.get_repo()
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.get()
        if 'P115' in item.claims:
            item.removeClaims(item.claims['P115'])

        claim = pywikibot.page.Claim(testsite, 'P115', datatype='wikibase-item')
        target = pywikibot.ItemPage(testsite, 'Q271')
        claim.setTarget(target)

        item.addClaim(claim)

        item.get(force=True)

        end_date = pywikibot.page.Claim(testsite, 'P88', isQualifier=True)
        end_date.setTarget(pywikibot.WbTime(year=2012))
        item.claims['P115'][0].addQualifier(end_date)

        # Testing all again but this time in properties
        item = pywikibot.PropertyPage(testsite, 'P115')
        item.get()
        if 'P115' in item.claims:
            to_remove = []
            for claim in item.claims['P115']:
                to_remove.append({'id': claim.toJSON()['id'], 'remove': ''})
            item.editEntity({'claims': to_remove})

        claim = pywikibot.page.Claim(testsite, 'P115', datatype='wikibase-item')
        target = pywikibot.ItemPage(testsite, 'Q271')
        claim.setTarget(target)
        item.editEntity({'claims': [claim.toJSON()]})

        item.get(force=True)

        end_date = pywikibot.page.Claim(testsite, 'P88', isQualifier=True)
        end_date.setTarget(pywikibot.WbTime(year=2012))
        item.claims['P115'][0].addQualifier(end_date)

    def test_edit_entity_new_item(self):
        """Test creating a new item using C{ItemPage.editEntity}."""
        testsite = self.get_repo()
        ts = str(time.time())
        data = {
            'labels': {
                'en': {
                    'language': 'en',
                    'value': 'Pywikibot test new item',
                }
            },
            'descriptions': {
                'en': {
                    'language': 'en',
                    'value': 'Pywikibot test new item - ' + ts,
                }
            }
        }
        item = pywikibot.ItemPage(testsite)
        item.editEntity(data)

    def test_edit_entity_new_linked_item(self):
        """Test linking a page using a new item."""
        ts = str(time.time())

        # Create a new page, which is unlinked
        site = self.get_site()
        title = 'Wikidata:Test ' + ts
        page = pywikibot.Page(site, title)
        page.text = ts
        page.save()

        data = {
            'labels': {
                'en': {
                    'language': 'en',
                    'value': 'Pywikibot test new linked item',
                }
            },
            'sitelinks': {
                page.site.dbName(): {
                    'site': page.site.dbName(),
                    'title': page.title()
                }
            },
        }

        repo = self.get_repo()
        item = pywikibot.ItemPage(repo)
        self.assertEqual(item._defined_by(), dict())
        item.editEntity(data)

    def test_set_redirect_target(self):
        """Test set_redirect_target method."""
        testsite = self.get_repo()
        item = pywikibot.ItemPage(testsite, 'Q1107')
        target_id = 'Q68'
        if not item.isRedirectPage():
            item.editEntity(data={}, clear=True)
        elif item.getRedirectTarget().getID() == 'Q68':
            target_id = 'Q67'
        target_item = pywikibot.ItemPage(testsite, target_id)
        item.set_redirect_target(target_id, force=True)
        self.assertTrue(item.isRedirectPage())
        new_item = pywikibot.ItemPage(testsite, item.getID())
        self.assertTrue(new_item.isRedirectPage())
        self.assertEqual(new_item.getRedirectTarget(), target_item)

class TestWikibaseMakeClaim(WikibaseTestCase):

    """Run wikibase write tests for claims."""

    family = 'wikidata'
    code = 'test'

    user = True
    write = True

    def _clean_item(self, repo, prop):
        """
        Return an item without any existing claims of the given property.

        @param repo: repository to fetch item from
        @type: pywikibot.site.DataSite
        @param prop: P-value of the property to scrub
        @type prop: str
        @return: scrubbed item
        @rtype: pywikibot.ItemPage
        """
        item = pywikibot.ItemPage(repo, 'Q68')
        item.get()
        if prop in item.claims:
            item.removeClaims(item.claims[prop])
        item.get(force=True)
        return item

    def test_math_edit(self):
        """Attempt adding a math claim with valid input."""
        testsite = self.get_repo()
        item = self._clean_item(testsite, 'P717')

        # set new claim
        claim = pywikibot.page.Claim(testsite, 'P717', datatype='math')
        target = 'a^2 + b^2 = c^2'
        claim.setTarget(target)
        item.addClaim(claim)

        # confirm new claim
        item.get(force=True)
        claim = item.claims['P717'][0]
        self.assertEqual(claim.getTarget(), target)

    def test_WbMonolingualText_edit(self):
        """Attempt adding a monolingual text with valid input."""
        # Clean the slate in preparation for test.
        testsite = self.get_repo()
        item = self._clean_item(testsite, 'P271')

        # set new claim
        claim = pywikibot.page.Claim(testsite, 'P271', datatype='monolingualtext')
        target = pywikibot.WbMonolingualText(text='Test this!', language='en')
        claim.setTarget(target)
        item.addClaim(claim)

        # confirm new claim
        item.get(force=True)
        claim = item.claims['P271'][0]
        self.assertEqual(claim.getTarget(), target)

    def test_Coordinate_edit(self):
        """Attempt adding a Coordinate with globe set via item."""
        testsite = self.get_repo()
        item = self._clean_item(testsite, 'P20480')

        # Make sure the wiki supports wikibase-conceptbaseuri
        version = testsite.version()
        if MediaWikiVersion(version) < MediaWikiVersion('1.29.0-wmf.2'):
            raise unittest.SkipTest('Wiki version must be 1.29.0-wmf.2 or '
                                    'newer to support unbound uncertainties.')

        # set new claim
        claim = pywikibot.page.Claim(testsite, 'P20480',
                                     datatype='globe-coordinate')
        target = pywikibot.Coordinate(site=testsite, lat=12.0, lon=13.0,
                                      globe_item=item)
        claim.setTarget(target)
        item.addClaim(claim)

        # confirm new claim
        item.get(force=True)
        claim = item.claims['P20480'][0]
        self.assertEqual(claim.getTarget(), target)

    def test_WbQuantity_edit_unbound(self):
        """Attempt adding a quantity with unbound errors."""
        # Clean the slate in preparation for test.
        testsite = self.get_repo()
        item = self._clean_item(testsite, 'P69')

        # Make sure the wiki supports unbound uncertainties
        version = testsite.version()
        if MediaWikiVersion(version) < MediaWikiVersion('1.29.0-wmf.2'):
            raise unittest.SkipTest('Wiki version must be 1.29.0-wmf.2 or '
                                    'newer to support unbound uncertainties.')

        # set new claim
        claim = pywikibot.page.Claim(testsite, 'P69', datatype='quantity')
        target = pywikibot.WbQuantity(amount=1234)
        claim.setTarget(target)
        item.addClaim(claim)

        # confirm new claim
        item.get(force=True)
        claim = item.claims['P69'][0]
        self.assertEqual(claim.getTarget(), target)

    def test_WbQuantity_edit(self):
        """Attempt adding a quantity with valid input."""
        # Clean the slate in preparation for test.
        testsite = self.get_repo()
        item = self._clean_item(testsite, 'P69')

        # Make sure the wiki supports wikibase-conceptbaseuri
        version = testsite.version()
        if MediaWikiVersion(version) < MediaWikiVersion('1.28-wmf.23'):
            raise unittest.SkipTest('Wiki version must be 1.28-wmf.23 or '
                                    'newer to expose wikibase-conceptbaseuri.')

        # set new claim
        claim = pywikibot.page.Claim(testsite, 'P69', datatype='quantity')
        target = pywikibot.WbQuantity(amount=1234, error=1, unit=item)
        claim.setTarget(target)
        item.addClaim(claim)

        # confirm new claim
        item.get(force=True)
        claim = item.claims['P69'][0]
        self.assertEqual(claim.getTarget(), target)

    def test_identifier_edit(self):
        """Attempt adding a math claim with valid input."""
        testsite = self.get_repo()
        item = self._clean_item(testsite, 'P718')

        # set new claim
        claim = pywikibot.page.Claim(testsite, 'P718', datatype='external-id')
        target = 'CrazyURI123_:)'
        claim.setTarget(target)
        item.addClaim(claim)

        # confirm new claim
        item.get(force=True)
        claim = item.claims['P718'][0]
        self.assertEqual(claim.getTarget(), target)

    def test_WbGeoShape_edit(self):
        """Attempt adding a geo-shape with valid input."""
        # Clean the slate in preparation for test.
        testsite = self.get_repo()
        item = self._clean_item(testsite, 'P27199')

        # set new claim
        claim = pywikibot.page.Claim(testsite, 'P27199', datatype='geo-shape')
        commons_site = pywikibot.Site('commons', 'commons')
        page = pywikibot.Page(commons_site, 'Data:Lyngby Hovedgade.map')
        target = pywikibot.WbGeoShape(page)
        claim.setTarget(target)
        item.addClaim(claim)

        # confirm new claim
        item.get(force=True)
        claim = item.claims['P27199'][0]
        self.assertEqual(claim.getTarget(), target)

class TestWikibaseRemoveQualifier(WikibaseTestCase):

    """Run wikibase write tests to remove qualifiers."""

    family = 'wikidata'
    code = 'test'

    user = True
    write = True

    def setUp(self):
        """Add a claim with two qualifiers."""
        super(TestWikibaseRemoveQualifier, self).setUp()
        testsite = self.get_repo()
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.get()
        # Create claim with qualifier
        if 'P115' in item.claims:
            item.removeClaims(item.claims['P115'])

        claim = pywikibot.page.Claim(testsite, 'P115', datatype='wikibase-item')
        target = pywikibot.ItemPage(testsite, 'Q271')
        claim.setTarget(target)
        item.addClaim(claim)

        item.get(force=True)

        qual_1 = pywikibot.page.Claim(testsite, 'P88', isQualifier=True)
        qual_1.setTarget(pywikibot.WbTime(year=2012))
        item.claims['P115'][0].addQualifier(qual_1)

        qual_2 = pywikibot.page.Claim(testsite, 'P580', isQualifier=True)
        qual_2.setTarget(pywikibot.ItemPage(testsite, 'Q67'))
        item.claims['P115'][0].addQualifier(qual_2)

    def test_remove_single(self):
        """Test adding a claim with two qualifiers, then removing one."""
        self.setUp()
        testsite = self.get_repo()
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.get(force=True)

        # Remove qualifier
        claim = item.claims['P115'][0]
        qual_3 = claim.qualifiers[u'P580'][0]
        claim.removeQualifier(qual_3)

        # Check P580 qualifier removed but P88 qualifier remains
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.get(force=True)
        claim = item.claims['P115'][0]
        self.assertNotIn('P580', claim.qualifiers.keys())
        self.assertIn('P88', claim.qualifiers.keys())

    def test_remove_multiple(self):
        """Test adding a claim with two qualifiers, then removing both."""
        self.setUp()
        testsite = self.get_repo()
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.get(force=True)

        # Remove qualifiers
        item.get(force=True)
        claim = item.claims['P115'][0]
        qual_3 = claim.qualifiers[u'P580'][0]
        qual_4 = claim.qualifiers[u'P88'][0]
        claim.removeQualifiers([qual_3, qual_4])

        # Check P580 and P88 qualifiers are removed
        item = pywikibot.ItemPage(testsite, 'Q68')
        item.get(force=True)
        claim = item.claims['P115'][0]
        self.assertNotIn('P580', claim.qualifiers.keys())
        self.assertNotIn('P88', claim.qualifiers.keys())

if __name__ == '__main__':  # pragma: no cover
    try:
        unittest.main()
    except SystemExit:
        pass


text:
This small tool provides the ability to sync files or directories while ensuring durability.

It supports:

* sync to disk a file or a directory while making sure the parent directory is fsync'ed too
* sync to disk all the buffered modifications
* sync to disk the filesystem on which a specific file resides

## Install

Do you like Arch Linux? You're in luck: https://aur.archlinux.org/packages/syncit/ . Cheers to <gergely [at] imreh [dot] net>!

Otherwise:

```
make
make install
```

## How to use

When ran without file argument it will behave as a normal [sync](https://linux.die.net/man/8/sync).
When a file argument is provided the tool will fsync the file (which can be a directory too) and the parent directory to ensure durability of data.
When `-f` option is used, it will sync the entire filesystem the specified file resides on.

Check help:

```
syncit --help
```

## Contributions
Want to contribute? Great! Throw pull requests at us.


text:
#ifndef SkDashImpl_DEFINED
#define SkDashImpl_DEFINED

#include "src/core/SkPathEffectBase.h"

class SkDashImpl : public SkPathEffectBase {
public:
    SkDashImpl(const SkScalar intervals[], int count, SkScalar phase);

protected:
    ~SkDashImpl() override;
    void flatten(SkWriteBuffer&) const override;
    bool onFilterPath(SkPath* dst, const SkPath& src, SkStrokeRec*, const SkRect*,
                      const SkMatrix&) const override;

    bool onAsPoints(PointData* results, const SkPath& src, const SkStrokeRec&, const SkMatrix&,
                    const SkRect*) const override;

    DashType onAsADash(DashInfo* info) const override;

private:
    SK_FLATTENABLE_HOOKS(SkDashImpl)

    bool computeFastBounds(SkRect* bounds) const override {
        // Dashing a path returns a subset of the input path so just return true and leave
        // bounds unmodified
        return true;
    }

    SkScalar*   fIntervals;
    int32_t     fCount;
    SkScalar    fPhase;
    // computed from phase

    SkScalar    fInitialDashLength;
    int32_t     fInitialDashIndex;
    SkScalar    fIntervalLength;

    using INHERITED = SkPathEffectBase;
};

#endif


text:
package org.spongepowered.api.data.manipulator.mutable.block;

import org.spongepowered.api.block.BlockTypes;
import org.spongepowered.api.data.manipulator.DataManipulator;
import org.spongepowered.api.data.manipulator.immutable.block.ImmutableWireAttachmentData;
import org.spongepowered.api.data.type.WireAttachmentType;
import org.spongepowered.api.data.value.mutable.MapValue;
import org.spongepowered.api.data.value.mutable.Value;
import org.spongepowered.api.util.Direction;

/**
 * Represents the {@link WireAttachmentType}s for the four cardinal directions of a
 * piece of {@link BlockTypes#REDSTONE_WIRE}.
 */
public interface WireAttachmentData extends DataManipulator<WireAttachmentData, ImmutableWireAttachmentData> {

    /**
     * Gets the current {@link MapValue} for the available {@link Direction}s
     * and associated{@link WireAttachmentType}s.
     *
     * @return The map value for the directions and their associated
     *     wire attachment types
     */
    MapValue<Direction, WireAttachmentType> wireAttachments();

    /**
     * Gets the {@link Value} for the {@link Direction#NORTH}'s
     * {@link WireAttachmentType}.
     *
     * @return The value for the northern wire attachment type
     */
    Value<WireAttachmentType> wireAttachmentNorth();

    /**
     * Gets the {@link Value} for the {@link Direction#SOUTH}'s
     * {@link WireAttachmentType}.
     *
     * @return The value for the southern wire attachment type
     */
    Value<WireAttachmentType> wireAttachmentSouth();

    /**
     * Gets the {@link Value} for the {@link Direction#EAST}'s
     * {@link WireAttachmentType}.
     *
     * @return The value for the eastern wire attachment type
     */
    Value<WireAttachmentType> wireAttachmentEast();

    /**
     * Gets the {@link Value} for the {@link Direction#WEST}'s
     * {@link WireAttachmentType}.
     *
     * @return The value for the western wire attachment type
     */
    Value<WireAttachmentType> wireAttachmentWest();

}


text: class SnmpsimError(Exception): pass


text:
<?xml version="1.0" encoding="UTF-8"?>

<document xmlns="http://maven.apache.org/XDOC/2.0"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
  <properties>
    <title>Axis-admin Task</title>
  </properties>

Task to administer a local or remote Axis server. Remember, for remote admin, the server has to be accept remote management calls.

    Attribute
    Description
    Type

    debug
    set a debug flag
    boolean

    failonerror
    Whether or not the build should halt if this task fails. Defaults to true.
    boolean

    fileprotocol
    specifies that a simple file protocol be used
    String

    hostname
    name the host to admin
    String

    newpassword
    set a new password; only valid if action=passwd
    String

    password
    the admin password
    String

    port
    the port to connect to
    int

    servletpath
    the path to the AxisAdmin servlet
    String

    transportchain
    set the transport chain to use
    String

    url
    full url to the admin endpoint
    String

    username
    username to log in as
    String

    xmlfile
    the name of the XML file containing deployment information
    File

</document>


text:
OpinionLabs is a 3rd-party site which collects survey-style user feedback. Feedback is entered into a popup window accessed through certain pages on www.paypal.com. The popup window is served by opinionlabs.com so is not hosted by paypal.com (paypal.com pages only create the link to the survey popup window).

Additional information can be found at https://dev.paypal.com/wiki/General/UVLOpinionLab.

This component is included in FooterMarketing, FooterProduct, and FooterMinimal

| Attributes                    | Expected Values   | Description      																		|
| -----------------             |:------------------| :-------------------------------------------------------------------------------------|
| showSiteFeedbackIcon          | -- 				| Boolean indicating whether to show or hide [+] icon image next to site feedback link	|
| miniBrowser      	            | --  				| Boolean indicating if opinionLab is used inside a miniBrowser						|
| commentCardCmd				| --     			| PayPal command link used to first direct the comment card link to a PayPal page which then redirects to the OpinionLab site	 																				|
| sfRedirect				    | "true"      			| Boolean to open OpinionLab popup after redirection through Paypal server. When disabled, popup opens directly on 'secure.opinionlab.com'																					|
| showOpinionLabOnBrowserClose	| --      			| Boolean indicating whether to open opinion lab pop up on closing PP window																																|
| siteFeedbackLabelKey			| --    			| Content key to find correct site feed back label																																|

## Prerequisite
CoreComponents are distributed through Bower. For more information on Bower & Bower installation please follow [Getting Started](/pages/CoreComponents/corecomponents.github.com/gettingStarted.html)

## Installation
### Usage under Node-Webcore
##### Navigate to Public/templates folder under Node-Webcore project in Command Prompt
```
> bower install UVLOpinionLab
```
##### Wrap OpinionLab partial with an HTML element with id 'siteFeedback' as below.
```html
<div id='siteFeedback'>
	{>"components/UVLOpinionLab/templates/opinionLab"
    		commentCardCmd=""
		showOpinionLabOnBrowserClose="false"
		sfRedirect="false"

		showSiteFeedbackIcon="false"
		miniBrowser="false"
		siteFeedbackLabelKey="CoreComponents.Footer.siteFeedbackLabel" /}
</div>
```
##### Add following under paths in main.js
```javascript
"opinionLab"            : "../templates/components/UVLOpinionLab/js/opinionLab",
"opinionLabComponent"   : "../templates/components/UVLOpinionLab/js/opinionLabComponent",
"onlineOpinionPopup"    : "../templates/components/UVLOpinionLab/js/onlineOpinionPopup"
```
##### Add following JS snippet to load and initialize OpinionLab
```javascript
define(['opinionLabComponent']);
```
#### Adding to buildconfig.js for JS Optimization with r.js
```javascript
"opinionLab"            : "../templates/components/UVLOpinionLab/js/opinionLab",
"opinionLabComponent"   : "../templates/components/UVLOpinionLab/js/opinionLabComponent",
"onlineOpinionPopup"    : "../templates/components/UVLOpinionLab/js/onlineOpinionPopup",
"opinionLabVars"	: "empty:"
```
##### Populate Mock Data
Load [Mock Data](https://github.paypal.com/CoreComponents/UVLOpinionLab/blob/master/mock/data/default.json) to populate OpinionLab.

### Porting to Sparta
| Copy              | From Webcore                                      | To Sparta        																																|
| ----------------- |:-------------------------------------------       | :---------------------------------------------------------|
| js                | Public/templates/components/UVLOpinionLab/js         | webapp/js/components/UVLOpinionLab/							|
| dust				| Public/templates/components/UVLOpinionLab/templates  | webapp/WEB-INF/tmpl/dust/components/UVLOpinionLab/templates/	|

##### Populate Data in Sparta
  * Everything under [sys](https://github.paypal.com/CoreComponents/UVLOpinionLab/blob/master/mock/data/default.json#L7) is populated automatically.
  * To populate [encryptedAccountNumber](https://github.paypal.com/CoreComponents/UVLOpinionLab/blob/master/mock/data/default.json#L4) add the following in Controller to populate TemplateModel with Logged-in user information.

```css
	public String render(ModelMap model) throws Exception {
		...
		model.put("user", DustCoreUtils.getBasicUserData());
		...
```
## Screenshot
Feedback link along with OpinionLab Popup
![textInput Screenshot](https://github.paypal.com/pages/CoreComponents/corecomponents.github.com/images/components/OpinionLab/OpinionLab.png)

## Example Usage
```dust
<li id="siteFeedback">
	{>"components/UVLOpinionLab/templates/opinionLab"
    					commentCardCmd="""
					showOpinionLabOnBrowserClose="false"
					sfRedirect="false"

					showSiteFeedbackIcon="false"
					miniBrowser="false"
					siteFeedbackLabelKey="CoreComponents.Footer.siteFeedbackLabel" /}
</li>
```

## Known Issues
No known issue

## Locale Support
All Locales

## Versioning
All Core Components utilize the Semantic Versioning guidelines.

Releases will be numbered with the following format:

<major>.<minor>.<patch>

And constructed with the following guidelines:

Breaking backward compatibility bumps the major (and resets the minor and patch)
New additions without breaking backward compatibility bumps the minor (and resets the patch)
Bug fixes and misc changes bumps the patch

For more information on SemVer, please visit http://semver.org/.

## Contributions

We welcome contributions. If you have an enhancement that could benefit other teams or have discovered and resolved a bug, we encourage you to submit a Git Pull request to make your changes available to all users of the component.

Please review our contribution guidelines prior to submitting your request - https://github.paypal.com/Sparta/SpartaCoreComponentsJSP/wiki/Pull-Request-Acceptance-Criteria


text:
using Internal.TypeSystem;
using System;
using System.Text;

namespace Internal.IL
{
    internal sealed class MethodILDebugView
    {
        private readonly MethodIL _methodIL;

        public MethodILDebugView(MethodIL methodIL)
        {
            _methodIL = methodIL;
        }

        public string Disassembly
        {
            get
            {
                byte[] ilBytes = _methodIL.GetILBytes() ?? Array.Empty<byte>();

                StringBuilder sb = new StringBuilder();

                sb.Append("// Code size: ");
                sb.Append(ilBytes.Length);
                sb.AppendLine();
                sb.Append(".maxstack ");
                sb.Append(_methodIL.MaxStack);
                sb.AppendLine();

                LocalVariableDefinition[] locals = _methodIL.GetLocals();
                if (locals != null && locals.Length > 0)
                {
                    sb.Append(".locals ");
                    if (_methodIL.IsInitLocals)
                        sb.Append("init ");

                    sb.Append("(");

                    for (int i = 0; i < locals.Length; i++)
                    {
                        if (i != 0)
                        {
                            sb.AppendLine(",");
                            sb.Append(' ', 4);
                        }
                        sb.Append(locals[i].Type.ToString());
                        sb.Append(" ");
                        if (locals[i].IsPinned)
                            sb.Append("pinned ");
                        sb.Append("V_");
                        sb.Append(i);
                    }
                    sb.AppendLine(")");
                }
                sb.AppendLine();

                int offset = 0;

                Func<int, string> resolver = token => _methodIL.GetObject(token).ToString();

                while (offset < ilBytes.Length)
                {
                    sb.Append(ILDisassember.FormatOffset(offset));
                    sb.Append(": ");
                    sb.AppendLine(ILDisassember.Disassemble(resolver, ilBytes, ref offset));
                }

                return sb.ToString();
            }
        }
    }
}


text:
#include <vector>
#include <fstream>
#include <algorithm>

#include "Cyclades_sparse_sgd.cpp"

using namespace std;

int loadData(string filename, vector<vector<double> > &data, vector<vector<long> > &dataCols){
  ifstream datafile;
  datafile.open(filename);

  int counter = 0;
  string line;
  while ( getline(datafile, line) ){
    size_t matchPos = -1;
    size_t startPos = -1;
    vector<double> d;
    vector<long> dc;
    long num = 0;
    while (true){
      startPos = matchPos + 1;
      matchPos = line.find('\t', startPos);
      if (matchPos != string::npos){
        num = stoi(line.substr(startPos, matchPos - startPos));
        if (startPos == 0){
          // First number in line; number of features.
          // We do nothing with this for now.
        }else{
          dc.push_back(num);
          d.push_back(drand48());
        }
      }else{
        break;
      }
    }
    data.push_back(d);
    dataCols.push_back(dc);
    counter ++;
  }
  datafile.close();
  return counter;
}

void loadDataAccess(string filename, int numThreads, int numBatches, vector<vector<vector<long> > > &examples){
  ifstream dataAccessFile;
  dataAccessFile.open(filename);

  string line;
  for (int batchID = 0; batchID < numBatches; batchID++){
    vector<vector<long> > examplesByBatch;

    for (int threadID = 0; threadID < numThreads; threadID++){

      if (getline(dataAccessFile, line)){

        vector<long> examplesByBatchThread;
        size_t matchPos = -1;
        size_t startPos = -1;
        long num = 0;

        startPos = matchPos + 1;
        matchPos = line.find('\t', startPos);
        num = stoi(line.substr(startPos, matchPos - startPos));
        // Sanity check
        if (num != batchID){
          cerr << "ERROR: read batchID as " << num << " but expected " << batchID << endl;
        }

        startPos = matchPos + 1;
        matchPos = line.find('\t', startPos);
        num = stoi(line.substr(startPos, matchPos - startPos));
        // Sanity check
        if (num != threadID){
          cerr << "ERROR: read threadID as " << num << " but expected " << threadID << endl;
        }

        // Read examples
        while (true){
          startPos = matchPos + 1;
          matchPos = line.find('\t', startPos);
          if (matchPos != string::npos){
            num = stoi(line.substr(startPos, matchPos - startPos));
            examplesByBatchThread.push_back(num);
          }else{
            break;
          }
        }

        examplesByBatch.push_back(examplesByBatchThread);
      }
    }

    examples.push_back(examplesByBatch);
  }

  dataAccessFile.close();
}

std::vector<SparseVector<double>*> allocateDataToNUMA(int numBatches, int numThreads, int nfeat, vector<vector<double> > data, vector<vector<long> > dataCols, vector<vector<vector<long> > > examples, vector<SparseVector<double>*>& row_pointers_all, vector<long>& batchNumRows, vector <long>& batchNumCols, vector<long>& batchNumElems){

  long noopDensity = 10;

  std::vector<SparseVector<double>*> allpointers;

  for (int ibatch=0; ibatch < numBatches; ibatch++){
    long nrows = 0;
    long ncols = nfeat+1;
    long nelems = 0;

    // Count max number of examples in batch across threads
    int maxCount = 0;
    for (int ithread=0; ithread < numThreads; ithread++){
      if (examples[ibatch][ithread].size() > maxCount){
        maxCount = examples[ibatch][ithread].size();
      }
    }

    int nexp = maxCount * numThreads;

    SparseVector<double>* row_pointers =
      (SparseVector<double>*) ::operator new(nexp * sizeof(SparseVector<double>));

    for (int ithread=0; ithread < numThreads; ithread++){
      numa_run_on_node(ithread);
      numa_set_localalloc();

      for (int iexp = 0; iexp < examples[ibatch][ithread].size(); iexp++){
        long expID = examples[ibatch][ithread][iexp];
        long length = data[expID].size();
        double* content = new double[length + 1];
        long* cols = new long[length + 1];
        for (int ii = 0; ii < length; ii++){
          content[ii] = data[expID][ii];
          cols[ii]    = dataCols[expID][ii];
          nelems++;
        }
        content[length] = drand48() > 0.8 ? 0 : 1.0;
        cols[length] = nfeat;
        nelems++;
        row_pointers[nrows++] = SparseVector<double>(content, cols, length+1);
        allpointers.push_back(&row_pointers[nrows-1]);
      }

      for (int iexp = examples[ibatch][ithread].size(); iexp < maxCount; iexp++){
        long length = 0;
        double* content = new double[length + 1];
        long* cols = new long[length + 1];
        content[length] = drand48() > 0.8 ? 0 : 1.0;
        cols[length] = nfeat;
        nelems++;
        row_pointers[nrows++] = SparseVector<double>(content, cols, length+1);
        allpointers.push_back(&row_pointers[nrows-1]);
      }

    }

    // Sanity check that nrows = nexp
    if (nrows != nexp) cout << "nrows = " << nrows << ", nexp = " << nexp << endl;

    row_pointers_all.push_back(row_pointers);
    batchNumRows.push_back(nrows);
    batchNumCols.push_back(ncols);
    batchNumElems.push_back(nelems);
  }
  return allpointers;

}

/**
 * \brief This is one example of running SGD for logistic regression
 * in DimmWitted. You can find more examples in test/glm_dense.cc
 * and test/glm_sparse.cc, and the documented code in
 * app/glm_dense_sgd.h
 */
// int main_dataaccess(int argc, char** argv){
int main(int argc, char** argv){

  bool ISDW = false;

  int numEpochs = 1;
  int numBatches = 2000;
  int numThreads = 8;
  int nfeat = 1000000;

  vector<vector<double> > data;
  vector<vector<long  > > dataCols;
  vector<vector<vector<long> > > examples;
  vector<long> batchNumRows, batchNumCols, batchNumElems;
  vector<SparseVector<double>*> row_pointers_all;

  loadData("data/dataaccess_data_multinomialLogisticRegression.txt", data, dataCols);
  loadDataAccess("data/dataaccess_nthreads8_multinomialLogisticRegression.txt", numThreads, numBatches, examples);
  std::vector<SparseVector<double>*> allpointers =
    allocateDataToNUMA(numBatches, numThreads, nfeat, data, dataCols, examples, row_pointers_all, batchNumRows, batchNumCols, batchNumElems);

  if (false){
    cout << row_pointers_all.size() << " / " << numBatches << endl;
    for (int ibatch = 0; ibatch < numBatches; ibatch++){
      for (int irow = 0; irow < batchNumRows[ibatch]; irow++){
        cout << ibatch << '\t' << irow;
        for (int icol = 0; icol < row_pointers_all[ibatch][irow].n; icol ++){
          cout << "\t(" << row_pointers_all[ibatch][irow].idxs[icol] << ":" << row_pointers_all[ibatch][irow].p[icol] << ")";
        }
        cout << endl;
      }
    }
  }

  if(ISDW){
    SparseVector<double>* allrowpointers =
      (SparseVector<double>*) ::operator new(allpointers.size() * sizeof(SparseVector<double>));
    std::cout << allpointers.size() << std::endl;
    for(int i=0;i<allpointers.size();i++){
      allrowpointers[i] = SparseVector<double>(allpointers[i]->p, allpointers[i]->idxs, allpointers[i]->n);
    }
    random_shuffle(&allrowpointers[0],&allrowpointers[allpointers.size()]);
    std::cout << "Shuffled" << std::endl;

    int nelem = 0;
    for(int i=0;i<batchNumElems.size();i++){
    nelem += batchNumElems[i];
    }
    std::cout << "NELEM " << nelem << std::endl;

    double rs = test_cyc_sparse_sgd_hogwild<DW_MODELREPL_PERMACHINE, DW_DATAREPL_SHARDING>
      (numEpochs, numBatches, numThreads, nfeat, allrowpointers, allpointers.size(), nelem);
    std::cout << "SUM OF MODEL (Should be ~1.3-1.4): " << rs << std::endl;
    return 0;

  }else{
    double rs = 0.0;
    rs = test_cyc_sparse_sgd<DW_MODELREPL_PERMACHINE, DW_DATAREPL_SHARDING>(numEpochs, numBatches, numThreads, nfeat, row_pointers_all, batchNumRows, batchNumCols, batchNumElems);
    std::cout << "SUM OF MODEL (Should be ~1.3-1.4): " << rs << std::endl;
    return 0;
  }
}

void allocateSyntheticData(int numBatches, int numThreads, int nfeat, int density, int numExamplesPerBatchPerThread, vector<SparseVector<double>*>& row_pointers_all, vector<long>& batchNumRows, vector <long>& batchNumCols, vector<long>& batchNumElems){
  for (int ibatch=0; ibatch < numBatches; ibatch++){
    SparseVector<double>* row_pointers = (SparseVector<double>*) ::operator new(numExamplesPerBatchPerThread * numThreads * sizeof(SparseVector<double>));
    int nrows = 0;
    int nelems = 0;
    for (int ithread = 0; ithread < numThreads; ithread++){
      numa_run_on_node(ithread);
      numa_set_localalloc();
      for (int iexp = 0; iexp < numExamplesPerBatchPerThread; iexp++){
        double* content = new double[density + 1];
        long*   cols    = new long[density + 1];
        for (int idensity = 0; idensity < density; idensity++){
          cols[idensity] = (int)(drand48() * nfeat);
          content[idensity] = drand48();
          nelems ++;
        }
        cols[density] = nfeat;
        content[density] = drand48() > 0.8 ? 0 : 1;
        nelems++;
        row_pointers[nrows++] = SparseVector<double>(content, cols, density+1);
      }
    }
    row_pointers_all.push_back(row_pointers);
    batchNumRows.push_back(nrows);
    batchNumCols.push_back(nfeat+1);
    batchNumElems.push_back(nelems);
  }
}

int main_synthetic(int argc, char** argv){
// int main(int argc, char** argv){
  int numEpochs = 1;
  int numBatches = 100;
  int numExamplesPerBatchPerThread = 200;
  int numThreads = 8;
  int nfeat = 10000;
  int density = 10;

  vector<long> batchNumRows, batchNumCols, batchNumElems;
  vector<SparseVector<double>*> row_pointers_all;

  allocateSyntheticData(numBatches, numThreads, nfeat, density, numExamplesPerBatchPerThread, row_pointers_all, batchNumRows, batchNumCols, batchNumElems);

  double rs = 0.0;
  rs = test_cyc_sparse_sgd<DW_MODELREPL_PERMACHINE, DW_DATAREPL_SHARDING>(numEpochs, numBatches, numThreads, nfeat, row_pointers_all, batchNumRows, batchNumCols, batchNumElems);
  std::cout << "SUM OF MODEL (Should be ~1.3-1.4): " << rs << std::endl;
  return 0;

}


text:
package org.terasology.utilities.concurrency;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.terasology.engine.GameThread;
import org.terasology.monitoring.ThreadActivity;
import org.terasology.monitoring.ThreadMonitor;

import java.util.concurrent.BlockingQueue;

/**
 */
final class TaskProcessor<T extends Task> implements Runnable {

    private static final Logger logger = LoggerFactory.getLogger(TaskProcessor.class);

    private String name;
    private BlockingQueue<T> queue;

    public TaskProcessor(String name, BlockingQueue<T> taskQueue) {
        this.queue = taskQueue;
        this.name = name;
    }

    @Override
    public void run() {
        boolean running = true;
        Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
        Thread.currentThread().setName(name);
        while (running) {
            try {
                T task = queue.take();
                try (ThreadActivity ignored = ThreadMonitor.startThreadActivity(task.getName())) {
                    task.run();
                }
                if (task.isTerminateSignal()) {
                    running = false;
                }
            } catch (InterruptedException e) {
                ThreadMonitor.addError(e);
                logger.error("Thread interrupted", e);
            } catch (RuntimeException e) {
                ThreadMonitor.addError(e);
                logger.error("Error in thread {}", Thread.currentThread().getName(), e);
            } catch (Error e) {
                GameThread.asynch(() -> {
                    throw e;  // re-throw on game thread to terminate the entire application
                });
            }
        }
        logger.debug("Thread shutdown safely");
    }
}