text:
import json
import os.path
import threading
import time
import unittest

import mock
import pytest
from mesos.interface.mesos_pb2 import TaskState
from twitter.common.contextutil import temporary_dir
from twitter.common.dirutil import chmod_plus_x
from twitter.common.exceptions import ExceptionalThread
from twitter.common.testing.clock import ThreadedClock

from apache.aurora.common.health_check.http_signaler import HttpSignaler
from apache.aurora.config.schema.base import (
    HealthCheckConfig,
    HealthCheckerConfig,
    HttpHealthChecker,
    ShellHealthChecker
)
from apache.aurora.executor.common.health_checker import (
    HealthChecker,
    HealthCheckerProvider,
    NoopHealthChecker
)
from apache.aurora.executor.common.sandbox import SandboxInterface
from apache.aurora.executor.common.status_checker import StatusResult

from .fixtures import HELLO_WORLD, MESOS_JOB

from gen.apache.aurora.api.ttypes import AssignedTask, ExecutorConfig, JobKey, TaskConfig

FAKE_MESOS_CONTAINERIZER_BINARY = '''#!/bin/sh
if [[ $# == 1 ]]; then
  echo "command_info" >&2
else
  echo "$@"
fi'''

class TestHealthChecker(unittest.TestCase):
  def setUp(self):
    self._clock = ThreadedClock(0)
    self._checker = mock.Mock(spec=HttpSignaler)
    self.initial_interval_secs = 15
    self.interval_secs = 10
    self.fake_health_checks = []
    def mock_health_check():
      return self.fake_health_checks.pop(0)
    self._checker.health = mock.Mock(spec=self._checker.__call__)
    self._checker.health.side_effect = mock_health_check

  def append_health_checks(self, status, num_calls=1):
    for i in range(num_calls):
      self.fake_health_checks.append((status, 'reason'))

  def test_grace_period_2x_success(self):
    '''Grace period is 2 x interval and health checks succeed.'''

    self.append_health_checks(True, num_calls=2)
    hct = HealthChecker(
              self._checker.health,
              interval_secs=self.interval_secs,
              clock=self._clock)
    hct.start()
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    hct.stop()
    assert self._checker.health.call_count == 1

  def test_grace_period_2x_failure_then_success(self):
    '''Grace period is 2 x interval and health checks fail then succeed.'''

    self.append_health_checks(False)
    self.append_health_checks(True)
    hct = HealthChecker(
              self._checker.health,
              interval_secs=self.interval_secs,
              clock=self._clock)
    hct.start()
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
    assert hct.threaded_health_checker.running is False
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    hct.stop()
    assert self._checker.health.call_count == 2

  def test_ignore_failures_after_running_inside_grace_period(self):
    '''Grace period is 2 x interval and health checks succeed then fail.'''

    self.append_health_checks(True)
    self.append_health_checks(False)
    hct = HealthChecker(
              self._checker.health,
              interval_secs=self.interval_secs,
              clock=self._clock)
    hct.start()
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    assert hct.threaded_health_checker.current_consecutive_failures == 0
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    assert hct.threaded_health_checker.current_consecutive_failures == 0
    hct.stop()
    assert self._checker.health.call_count == 2

  def test_does_not_ignores_failures_after_running_outside_grace_period(self):
    '''Grace period is 2 x interval and health checks succeed then fail.'''

    self.append_health_checks(True)
    self.append_health_checks(False, num_calls=2)
    hct = HealthChecker(
              self._checker.health,
              interval_secs=self.interval_secs,
              clock=self._clock)
    hct.start()
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    assert hct.threaded_health_checker.current_consecutive_failures == 0
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    assert hct.threaded_health_checker.current_consecutive_failures == 0
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
    assert hct.threaded_health_checker.running is True
    assert hct.threaded_health_checker.current_consecutive_failures == 1
    hct.stop()
    assert self._checker.health.call_count == 3

  def test_grace_period_2x_failure(self):
    '''
      Grace period is 2 x interval and all health checks fail.
      Failures are ignored when in grace period.
    '''

    self.append_health_checks(False, num_calls=3)
    hct = HealthChecker(
              self._checker.health,
              interval_secs=self.interval_secs,
              clock=self._clock)
    hct.start()
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
    assert hct.threaded_health_checker.running is False
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
    assert hct.threaded_health_checker.running is False
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
    assert hct.threaded_health_checker.running is False
    hct.stop()
    assert self._checker.health.call_count == 3

  def test_success_outside_grace_period(self):
    '''
    Health checks fail inside grace period, but pass outside and leads to success
    '''

    self.append_health_checks(False, num_calls=2)
    self.append_health_checks(True)
    hct = HealthChecker(
              self._checker.health,
              interval_secs=self.interval_secs,
              clock=self._clock)
    hct.start()
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
    assert hct.threaded_health_checker.running is False
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
    assert hct.threaded_health_checker.running is False
    self._clock.tick(self.interval_secs)
    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    hct.stop()
    assert self._checker.health.call_count == 3

  def test_include_max_failure_to_forgiving_attempts(self):
    '''
    Health checks fail but never breaches `max_consecutive_failures`
    '''
    max_consecutive_failures = 4

    # health checks fail within grace period
    self.append_health_checks(False, num_calls=2)

    # health checks fails max_consecutive_failures times and then succeeds
    self.append_health_checks(False, num_calls=max_consecutive_failures)
    self.append_health_checks(True)

    # health checks fails max_consecutive_failures times and then succeeds
    self.append_health_checks(False, num_calls=max_consecutive_failures)
    self.append_health_checks(False)

    hct = HealthChecker(
              self._checker.health,
              interval_secs=self.interval_secs,
              max_consecutive_failures=max_consecutive_failures,
              clock=self._clock)
    hct.start()

    # failures ignored inside grace period
    for _ in range(2):
      assert self._clock.converge(threads=[hct.threaded_health_checker])
      self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
      assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
      assert hct.threaded_health_checker.running is False
      self._clock.tick(self.interval_secs)

    # failures never breach max
    for _ in range(max_consecutive_failures):
      assert self._clock.converge(threads=[hct.threaded_health_checker])
      self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
      assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
      assert hct.threaded_health_checker.running is False
      self._clock.tick(self.interval_secs)

    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.threaded_health_checker.running is True
    self._clock.tick(self.interval_secs)

    # failures breach max, causes task failure
    for _ in range(max_consecutive_failures):
      assert self._clock.converge(threads=[hct.threaded_health_checker])
      self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
      assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
      assert hct.threaded_health_checker.running is True
      self._clock.tick(self.interval_secs)

    assert self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
    assert hct.threaded_health_checker.running is True

    hct.stop()
    assert self._checker.health.call_count == 12

  def test_initial_interval_whatev(self):
    self.append_health_checks(False, 2)
    hct = HealthChecker(
        self._checker.health,
        interval_secs=self.interval_secs,
        grace_period_secs=0,
        clock=self._clock)
    hct.start()
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
    assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
    hct.stop()
    assert self._checker.health.call_count == 1

  def test_consecutive_failures_max_failures(self):
    '''Verify that a task is unhealthy after max_consecutive_failures is exceeded'''
    grace_period_secs = self.initial_interval_secs
    interval_secs = self.interval_secs
    self.append_health_checks(True, num_calls=2)
    self.append_health_checks(False, num_calls=3)
    hct = HealthChecker(
        self._checker.health,
        interval_secs=interval_secs,
        grace_period_secs=grace_period_secs,
        max_consecutive_failures=2,
        min_consecutive_successes=2,
        clock=self._clock)
    hct.start()

    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
    assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
    assert hct.metrics.sample()['consecutive_failures'] == 0
    self._clock.tick(interval_secs)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.metrics.sample()['consecutive_failures'] == 0
    assert hct.threaded_health_checker.running is True
    self._clock.tick(interval_secs)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.metrics.sample()['consecutive_failures'] == 1
    self._clock.tick(interval_secs)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
    assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
    assert hct.metrics.sample()['consecutive_failures'] == 2
    self._clock.tick(interval_secs)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
    assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
    assert hct.metrics.sample()['consecutive_failures'] == 3
    hct.stop()
    assert self._checker.health.call_count == 5

  def test_consecutive_failures_failfast(self):
    '''Verify that health check is failed fast'''
    grace_period_secs = self.initial_interval_secs
    interval_secs = self.interval_secs
    self.append_health_checks(False, num_calls=5)
    hct = HealthChecker(
        self._checker.health,
        interval_secs=interval_secs,
        grace_period_secs=grace_period_secs,
        max_consecutive_failures=2,
        min_consecutive_successes=4,
        clock=self._clock)
    hct.start()

    # failure is ignored inside grace_period_secs
    for _ in range(2):
      self._clock.converge(threads=[hct.threaded_health_checker])
      self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
      assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
      assert hct.metrics.sample()['consecutive_failures'] == 0
      self._clock.tick(interval_secs)

    # 3 consecutive health check failures causes fail-fast
    for attempt in range(2):
      self._clock.converge(threads=[hct.threaded_health_checker])
      self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
      assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
      # failure is not ignored outside grace_period_secs
      assert hct.metrics.sample()['consecutive_failures'] == (attempt + 1)
      self._clock.tick(interval_secs)

    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
    assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
    assert hct.metrics.sample()['consecutive_failures'] == 3
    hct.stop()
    assert self._checker.health.call_count == 5

  @pytest.mark.skipif('True', reason='Flaky test (AURORA-1182)')
  def test_health_checker_metrics(self):
    def slow_check():
      self._clock.sleep(0.5)
      return (True, None)
    hct = HealthChecker(slow_check, interval_secs=1, grace_period_secs=1, clock=self._clock)
    hct.start()
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, amount=1)

    assert hct._total_latency == 0
    assert hct.metrics.sample()['total_latency_secs'] == 0

    # start the health check (during health check it is still 0)
    epsilon = 0.001
    self._clock.tick(1.0 + epsilon)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
    assert hct._total_latency == 0
    assert hct.metrics.sample()['total_latency_secs'] == 0
    assert hct.metrics.sample()['checks'] == 0

    # finish the health check
    self._clock.tick(0.5 + epsilon)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, amount=1)  # interval_secs
    assert hct._total_latency == 0.5
    assert hct.metrics.sample()['total_latency_secs'] == 0.5
    assert hct.metrics.sample()['checks'] == 1

    # tick again
    self._clock.tick(1.0 + epsilon)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.tick(0.5 + epsilon)
    self._clock.converge(threads=[hct.threaded_health_checker])
    self._clock.assert_waiting(hct.threaded_health_checker, amount=1)  # interval_secs
    assert hct._total_latency == 1.0
    assert hct.metrics.sample()['total_latency_secs'] == 1.0
    assert hct.metrics.sample()['checks'] == 2

class TestHealthCheckerProvider(unittest.TestCase):
  def test_from_assigned_task_http(self):
    interval_secs = 17
    initial_interval_secs = 3
    max_consecutive_failures = 2
    min_consecutive_successes = 2
    task_config = TaskConfig(
        executorConfig=ExecutorConfig(
            name='thermos',
            data=MESOS_JOB(
                task=HELLO_WORLD,
                health_check_config=HealthCheckConfig(
                    interval_secs=interval_secs,
                    initial_interval_secs=initial_interval_secs,
                    max_consecutive_failures=max_consecutive_failures,
                    min_consecutive_successes=min_consecutive_successes,
                    timeout_secs=7
                )
            ).json_dumps()
        )
    )
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'health': 9001})
    health_checker = HealthCheckerProvider().from_assigned_task(assigned_task, None)
    hc = health_checker.threaded_health_checker
    assert hc.interval == interval_secs
    assert hc.grace_period_secs == initial_interval_secs
    assert hc.max_consecutive_failures == max_consecutive_failures
    assert hc.min_consecutive_successes == min_consecutive_successes

  def test_from_assigned_task_http_endpoint_style_config(self):
    interval_secs = 17
    initial_interval_secs = 3
    max_consecutive_failures = 2
    min_consecutive_successes = 2
    http_config = HttpHealthChecker(
      endpoint='/foo',
      expected_response='bar',
      expected_response_code=201
    )
    task_config = TaskConfig(
        executorConfig=ExecutorConfig(
            name='thermos',
            data=MESOS_JOB(
                task=HELLO_WORLD,
                health_check_config=HealthCheckConfig(
                    health_checker=HealthCheckerConfig(http=http_config),
                    interval_secs=interval_secs,
                    initial_interval_secs=initial_interval_secs,
                    max_consecutive_failures=max_consecutive_failures,
                    min_consecutive_successes=min_consecutive_successes,
                    timeout_secs=7
                )
            ).json_dumps()
        )
    )
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'health': 9001})
    execconfig_data = json.loads(assigned_task.task.executorConfig.data)
    http_exec_config = execconfig_data['health_check_config']['health_checker']['http']
    assert http_exec_config['endpoint'] == '/foo'
    assert http_exec_config['expected_response'] == 'bar'
    assert http_exec_config['expected_response_code'] == 201
    health_checker = HealthCheckerProvider().from_assigned_task(assigned_task, None)
    hc = health_checker.threaded_health_checker
    assert hc.interval == interval_secs
    assert hc.grace_period_secs == initial_interval_secs
    assert hc.max_consecutive_failures == max_consecutive_failures
    assert hc.min_consecutive_successes == min_consecutive_successes

  @mock.patch('pwd.getpwnam')
  def test_from_assigned_task_shell(self, mock_getpwnam):
    interval_secs = 17
    initial_interval_secs = 3
    max_consecutive_failures = 2
    min_consecutive_successes = 2
    timeout_secs = 5
    shell_config = ShellHealthChecker(shell_command='failed command')
    task_config = TaskConfig(
        job=JobKey(role='role', environment='env', name='name'),
        executorConfig=ExecutorConfig(
            name='thermos-generic',
            data=MESOS_JOB(
                task=HELLO_WORLD,
                health_check_config=HealthCheckConfig(
                    health_checker=HealthCheckerConfig(shell=shell_config),
                    interval_secs=interval_secs,
                    initial_interval_secs=initial_interval_secs,
                    max_consecutive_failures=max_consecutive_failures,
                    min_consecutive_successes=min_consecutive_successes,
                    timeout_secs=timeout_secs
                )
            ).json_dumps()
        )
    )
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001})
    execconfig_data = json.loads(assigned_task.task.executorConfig.data)
    assert execconfig_data[
             'health_check_config']['health_checker']['shell']['shell_command'] == 'failed command'

    mock_sandbox = mock.Mock(spec_set=SandboxInterface)
    type(mock_sandbox).root = mock.PropertyMock(return_value='/some/path')
    type(mock_sandbox).is_filesystem_image = mock.PropertyMock(return_value=False)

    health_checker = HealthCheckerProvider().from_assigned_task(assigned_task, mock_sandbox)
    hc = health_checker.threaded_health_checker
    assert hc.interval == interval_secs
    assert hc.grace_period_secs == initial_interval_secs
    assert hc.max_consecutive_failures == max_consecutive_failures
    assert hc.min_consecutive_successes == min_consecutive_successes
    mock_getpwnam.assert_called_once_with(task_config.job.role)

  @mock.patch('pwd.getpwnam')
  def test_from_assigned_task_shell_no_demotion(self, mock_getpwnam):
    interval_secs = 17
    initial_interval_secs = 3
    max_consecutive_failures = 2
    min_consecutive_successes = 2
    timeout_secs = 5
    shell_config = ShellHealthChecker(shell_command='failed command')
    task_config = TaskConfig(
        job=JobKey(role='role', environment='env', name='name'),
        executorConfig=ExecutorConfig(
            name='thermos-generic',
            data=MESOS_JOB(
                task=HELLO_WORLD,
                health_check_config=HealthCheckConfig(
                    health_checker=HealthCheckerConfig(shell=shell_config),
                    interval_secs=interval_secs,
                    initial_interval_secs=initial_interval_secs,
                    max_consecutive_failures=max_consecutive_failures,
                    min_consecutive_successes=min_consecutive_successes,
                    timeout_secs=timeout_secs
                )
            ).json_dumps()
        )
    )
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001})
    execconfig_data = json.loads(assigned_task.task.executorConfig.data)
    assert execconfig_data[
             'health_check_config']['health_checker']['shell']['shell_command'] == 'failed command'

    mock_sandbox = mock.Mock(spec_set=SandboxInterface)
    type(mock_sandbox).root = mock.PropertyMock(return_value='/some/path')
    type(mock_sandbox).is_filesystem_image = mock.PropertyMock(return_value=False)

    health_checker = HealthCheckerProvider(nosetuid_health_checks=True).from_assigned_task(
        assigned_task,
        mock_sandbox)
    hc = health_checker.threaded_health_checker
    assert hc.interval == interval_secs
    assert hc.grace_period_secs == initial_interval_secs
    assert hc.max_consecutive_failures == max_consecutive_failures
    assert hc.min_consecutive_successes == min_consecutive_successes
    # Should not be trying to access role's user info.
    assert not mock_getpwnam.called

  def make_fake_mesos_containerizer(self, temp_dir):
    # We use a fake version of the mesos-containerizer binary that just echoes out its args so
    # we can assert on them in the process's output. Also imitates a failure when there are not
    # enough arguments, this is used to find the version of the binary (by checking the failure
    # message)
    fake_mesos_containerizer_path = os.path.join(temp_dir, 'fake-mesos-containerizer')
    with open(fake_mesos_containerizer_path, 'w') as fd:
      fd.write(FAKE_MESOS_CONTAINERIZER_BINARY)
    chmod_plus_x(fake_mesos_containerizer_path)
    return fake_mesos_containerizer_path

  @mock.patch.dict(os.environ, {'MESOS_DIRECTORY': '/some/path'})
  @mock.patch('apache.aurora.executor.common.health_checker.ShellHealthCheck')
  def test_from_assigned_task_shell_filesystem_image(self, mock_shell):
    shell_config = ShellHealthChecker(shell_command='run health check')
    task_config = TaskConfig(
      job=JobKey(role='role', environment='env', name='name'),
      executorConfig=ExecutorConfig(
        name='thermos-generic',
        data=MESOS_JOB(
          task=HELLO_WORLD,
          health_check_config=HealthCheckConfig(
            health_checker=HealthCheckerConfig(shell=shell_config),
            timeout_secs=5
          )
        ).json_dumps()
      )
    )
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001})
    execconfig_data = json.loads(assigned_task.task.executorConfig.data)
    assert execconfig_data['health_check_config'][
             'health_checker']['shell']['shell_command'] == 'run health check'

    with temporary_dir() as td:
      fake_mesos_containerizer_path = self.make_fake_mesos_containerizer(td)

      mock_sandbox = mock.Mock(spec_set=SandboxInterface)
      type(mock_sandbox).root = mock.PropertyMock(return_value=td)
      type(mock_sandbox).is_filesystem_image = mock.PropertyMock(return_value=True)
      container_root_path = os.path.join(td, 'container-root')
      type(mock_sandbox).container_root = mock.PropertyMock(return_value=container_root_path)

      HealthCheckerProvider(
        nosetuid_health_checks=False,
        mesos_containerizer_path=fake_mesos_containerizer_path
      ).from_assigned_task(
        assigned_task,
        mock_sandbox
      )

      class ValidateWrapperCmd(object):
        def __eq__(self, other):
          assert "--user=role" in other
          return True

      assert mock_shell.mock_calls == [
        mock.call(
          raw_cmd='run health check',
          wrapped_cmd=ValidateWrapperCmd(),
          preexec_fn=None,
          timeout_secs=5.0
        )
      ]

  @mock.patch.dict(os.environ, {'MESOS_DIRECTORY': '/some/path'})
  @mock.patch('getpass.getuser', return_value='user')
  @mock.patch('apache.aurora.executor.common.health_checker.ShellHealthCheck')
  def test_from_assigned_task_shell_filesystem_image_no_demotion(self, mock_shell, mock_getuser):
    shell_config = ShellHealthChecker(shell_command='run health check')
    task_config = TaskConfig(
      job=JobKey(role='role', environment='env', name='name'),
      executorConfig=ExecutorConfig(
        name='thermos-generic',
        data=MESOS_JOB(
          task=HELLO_WORLD,
          health_check_config=HealthCheckConfig(
            health_checker=HealthCheckerConfig(shell=shell_config),
            timeout_secs=5
          )
        ).json_dumps()
      )
    )
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001})
    execconfig_data = json.loads(assigned_task.task.executorConfig.data)
    assert execconfig_data['health_check_config'][
             'health_checker']['shell']['shell_command'] == 'run health check'

    with temporary_dir() as td:
      fake_mesos_containerizer_path = self.make_fake_mesos_containerizer(td)

      mock_sandbox = mock.Mock(spec_set=SandboxInterface)
      type(mock_sandbox).root = mock.PropertyMock(return_value=td)
      type(mock_sandbox).is_filesystem_image = mock.PropertyMock(return_value=True)
      container_root_path = os.path.join(td, 'container-root')
      type(mock_sandbox).container_root = mock.PropertyMock(return_value=container_root_path)

      HealthCheckerProvider(
        nosetuid_health_checks=True,
        mesos_containerizer_path=fake_mesos_containerizer_path
      ).from_assigned_task(
        assigned_task,
        mock_sandbox
      )

      class ValidateWrapperCmd(object):
        def __eq__(self, other):
          assert "--user=user" in other
          return True

      assert mock_shell.mock_calls == [
        mock.call(
          raw_cmd='run health check',
          wrapped_cmd=ValidateWrapperCmd(),
          preexec_fn=None,
          timeout_secs=5.0
        )
      ]

  def test_interpolate_cmd(self):
    """Making sure thermos.ports[foo] gets correctly substituted with assignedPorts info."""
    interval_secs = 17
    initial_interval_secs = 3
    max_consecutive_failures = 2
    min_consecutive_successes = 2
    timeout_secs = 5
    shell_cmd = 'FOO_PORT={{thermos.ports[foo]}} failed command'
    shell_config = ShellHealthChecker(shell_command=shell_cmd)
    task_config = TaskConfig(
        executorConfig=ExecutorConfig(
            name='thermos-generic',
            data=MESOS_JOB(
                task=HELLO_WORLD,
                health_check_config=HealthCheckConfig(
                    health_checker=HealthCheckerConfig(shell=shell_config),
                    interval_secs=interval_secs,
                    initial_interval_secs=initial_interval_secs,
                    max_consecutive_failures=max_consecutive_failures,
                    min_consecutive_successes=min_consecutive_successes,
                    timeout_secs=timeout_secs
                )
            ).json_dumps()
        )
    )
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001})
    interpolated_cmd = HealthCheckerProvider.interpolate_cmd(
      assigned_task,
      cmd=shell_cmd
    )
    assert interpolated_cmd == 'FOO_PORT=9001 failed command'

  def test_from_assigned_task_no_health_port(self):
    interval_secs = 17
    initial_interval_secs = 3
    max_consecutive_failures = 2
    min_consecutive_successes = 2
    timeout_secs = 5
    task_config = TaskConfig(
        executorConfig=ExecutorConfig(
            name='thermos-generic',
            data=MESOS_JOB(
                task=HELLO_WORLD,
                health_check_config=HealthCheckConfig(
                    interval_secs=interval_secs,
                    initial_interval_secs=initial_interval_secs,
                    max_consecutive_failures=max_consecutive_failures,
                    min_consecutive_successes=min_consecutive_successes,
                    timeout_secs=timeout_secs
                )
            ).json_dumps()
        )
    )
    # No health port and we don't have a shell_command.
    assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'http': 9001})
    health_checker = HealthCheckerProvider().from_assigned_task(assigned_task, None)
    assert isinstance(health_checker, NoopHealthChecker)

class TestThreadedHealthChecker(unittest.TestCase):
  def setUp(self):
    self.health = mock.Mock()
    self.health.return_value = (True, 'Fake')

    self.sandbox = mock.Mock(spec_set=SandboxInterface)
    self.sandbox.exists.return_value = True
    self.sandbox.root = '/root'

    self.initial_interval_secs = 15
    self.interval_secs = 10
    self.max_consecutive_failures = 1
    self.min_consecutive_successes = 2
    self.clock = mock.Mock(spec=time)
    self.clock.time.return_value = 0

    self.health_checker = HealthChecker(
        self.health,
        None,
        self.interval_secs,
        self.initial_interval_secs,
        self.max_consecutive_failures,
        self.min_consecutive_successes,
        self.clock)
    self.health_checker_sandbox_exists = HealthChecker(
        self.health,
        self.sandbox,
        self.interval_secs,
        self.initial_interval_secs,
        self.max_consecutive_failures,
        self.min_consecutive_successes,
        self.clock)

  def test_perform_check_if_not_disabled_snooze_file_is_none(self):
    self.health_checker_sandbox_exists.threaded_health_checker.snooze_file = None
    assert self.health.call_count == 0
    assert self.health_checker_sandbox_exists.metrics.sample()['snoozed'] == 0
    self.health_checker.threaded_health_checker._perform_check_if_not_disabled()
    assert self.health.call_count == 1
    assert self.health_checker_sandbox_exists.metrics.sample()['snoozed'] == 0

  @mock.patch('os.path', spec_set=os.path)
  def test_perform_check_if_not_disabled_no_snooze_file(self, mock_os_path):
    mock_os_path.isfile.return_value = False
    assert self.health.call_count == 0
    assert self.health_checker_sandbox_exists.metrics.sample()['snoozed'] == 0
    self.health_checker_sandbox_exists.threaded_health_checker._perform_check_if_not_disabled()
    assert self.health.call_count == 1
    assert self.health_checker_sandbox_exists.metrics.sample()['snoozed'] == 0

  @mock.patch('os.path', spec_set=os.path)
  def test_perform_check_if_not_disabled_snooze_file_exists(self, mock_os_path):
    mock_os_path.isfile.return_value = True
    assert self.health.call_count == 0
    assert self.health_checker_sandbox_exists.metrics.sample()['snoozed'] == 0
    result = (
        self.health_checker_sandbox_exists.threaded_health_checker._perform_check_if_not_disabled())
    assert self.health.call_count == 0
    assert self.health_checker_sandbox_exists.metrics.sample()['snoozed'] == 1
    assert result == (True, None)

  def test_maybe_update_health_check_count_reset_count(self):
    hc = self.health_checker.threaded_health_checker
    hc.attempts = hc.forgiving_attempts

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason-1')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason-3')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1

  def test_maybe_update_health_check_count_ignore_failures_within_grace_period(self):
    hc = self.health_checker.threaded_health_checker

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-3')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-4')
    assert hc.current_consecutive_failures == 2
    assert hc.current_consecutive_successes == 0

  def test_maybe_update_health_check_count_dont_ignore_failures_after_grace_period(self):
    hc = self.health_checker.threaded_health_checker
    hc.attempts = hc.forgiving_attempts

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 2
    assert hc.current_consecutive_successes == 0

  def test_maybe_update_health_check_count_fail_fast(self):
    hc = self.health_checker.threaded_health_checker
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.healthy is True
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-3')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-4')
    assert hc.current_consecutive_failures == 2
    assert hc.current_consecutive_successes == 0
    assert hc.running is False
    assert hc.healthy is False
    assert hc.reason == 'reason-4'

  def test_maybe_update_health_check_count_max_failures_1(self):
    hc = self.health_checker.threaded_health_checker
    hc.current_consecutive_successes = 1
    hc.attempts = hc.forgiving_attempts

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1
    assert hc.healthy is True

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0
    assert hc.healthy is True

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 2
    assert hc.current_consecutive_successes == 0
    assert hc.healthy is False
    assert hc.reason == 'reason-2'

  def test_maybe_update_health_check_count_success(self):
    hc = self.health_checker.threaded_health_checker
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.running is False
    assert hc.healthy is True

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1
    assert hc.running is False
    assert hc.healthy is True

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 2
    assert hc.running is True
    assert hc.healthy is True

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 3
    assert hc.running is True
    assert hc.healthy is True

  def test_run_success(self):
    self.health.return_value = (True, 'success')
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert self.clock.sleep.call_count == 3
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 0
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 3
    assert self.health_checker.threaded_health_checker.running is True
    assert self.health_checker.threaded_health_checker.healthy is True
    assert self.health_checker.threaded_health_checker.reason is None

  def test_run_failure(self):
    self.health.return_value = (False, 'failure')
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert self.clock.sleep.call_count == 4
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 2
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 0
    assert self.health_checker.threaded_health_checker.running is False
    assert self.health_checker.threaded_health_checker.healthy is False
    assert self.health_checker.threaded_health_checker.reason == 'failure'

  def test_run_failure_unhealthy_when_failfast(self):
    health_status = [(False, 'failure-1'), (True, None), (False, 'failure-3'), (False, 'failure-4')]
    self.health.side_effect = lambda: health_status.pop(0)
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert self.clock.sleep.call_count == 4
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 2
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 0
    assert self.health_checker.threaded_health_checker.running is False
    assert self.health_checker.threaded_health_checker.healthy is False
    assert self.health_checker.threaded_health_checker.reason == 'failure-4'

  def test_run_unhealthy_after_callback(self):
    health_status = [(True, None), (True, None), (False, 'failure-4'), (False, 'failure-5')]
    self.health.side_effect = lambda: health_status.pop(0)
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert self.clock.sleep.call_count == 4
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 2
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 0
    assert self.health_checker.threaded_health_checker.running is True
    assert self.health_checker.threaded_health_checker.healthy is False
    assert self.health_checker.threaded_health_checker.reason == 'failure-5'

  @mock.patch('apache.aurora.executor.common.health_checker.ExceptionalThread.start',
      spec=ExceptionalThread.start)
  def test_start(self, mock_start):
    assert mock_start.call_count == 0
    self.health_checker.threaded_health_checker.start()
    mock_start.assert_called_once_with(self.health_checker.threaded_health_checker)

  def test_stop(self):
    assert not self.health_checker.threaded_health_checker.dead.is_set()
    self.health_checker.threaded_health_checker.stop()
    assert self.health_checker.threaded_health_checker.dead.is_set()

class TestThreadedHealthCheckerWithDefaults(unittest.TestCase):
  '''
    Similar tests as above but with the default health check configuration. This
    will ensure that the defaults are always valid.
  '''

  def setUp(self):
    self.health = mock.Mock()
    self.health.return_value = (True, 'Fake')

    self.sandbox = mock.Mock(spec_set=SandboxInterface)
    self.sandbox.exists.return_value = True
    self.sandbox.root = '/root'

    self.health_checker = HealthCheckerProvider().from_assigned_task(
        AssignedTask(
            task=TaskConfig(
                executorConfig=ExecutorConfig(
                    name='thermos',
                    data=MESOS_JOB(task=HELLO_WORLD).json_dumps())),
            instanceId=1,
            assignedPorts={'health': 9001}),
        self.sandbox)

    self.health_checker.threaded_health_checker.checker = self.health

  def test_perform_check_if_not_disabled_snooze_file_is_none(self):
    self.health_checker.threaded_health_checker.snooze_file = None
    assert self.health.call_count == 0
    assert self.health_checker.metrics.sample()['snoozed'] == 0
    self.health_checker.threaded_health_checker._perform_check_if_not_disabled()
    assert self.health.call_count == 1
    assert self.health_checker.metrics.sample()['snoozed'] == 0

  @mock.patch('os.path', spec_set=os.path)
  def test_perform_check_if_not_disabled_no_snooze_file(self, mock_os_path):
    mock_os_path.isfile.return_value = False
    assert self.health.call_count == 0
    assert self.health_checker.metrics.sample()['snoozed'] == 0
    self.health_checker.threaded_health_checker._perform_check_if_not_disabled()
    assert self.health.call_count == 1
    assert self.health_checker.metrics.sample()['snoozed'] == 0

  @mock.patch('os.path', spec_set=os.path)
  def test_perform_check_if_not_disabled_snooze_file_exists(self, mock_os_path):
    mock_os_path.isfile.return_value = True
    assert self.health.call_count == 0
    assert self.health_checker.metrics.sample()['snoozed'] == 0
    result = (
        self.health_checker.threaded_health_checker._perform_check_if_not_disabled())
    assert self.health.call_count == 0
    assert self.health_checker.metrics.sample()['snoozed'] == 1
    assert result == (True, None)

  def test_maybe_update_health_check_count_reset_count(self):
    hc = self.health_checker.threaded_health_checker
    hc.attempts = hc.forgiving_attempts

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason-1')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason-3')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1

  def test_maybe_update_health_check_count_ignore_failures_inside_grace_period(self):
    hc = self.health_checker.threaded_health_checker

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

  def test_maybe_update_health_check_count_dont_ignore_failures_after_grace_period(self):
    hc = self.health_checker.threaded_health_checker
    hc.attempts = hc.forgiving_attempts

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 2
    assert hc.current_consecutive_successes == 0

  def test_maybe_update_health_check_count_fail_fast(self):
    hc = self.health_checker.threaded_health_checker
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.healthy is True
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-2')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-3')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0
    assert hc.running is False
    assert hc.healthy is False
    assert hc.reason == 'reason-3'

  def test_maybe_update_health_check_count_max_failures(self):
    hc = self.health_checker.threaded_health_checker
    hc.attempts = hc.forgiving_attempts
    hc.current_consecutive_successes = 1

    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1
    assert hc.healthy is True

    hc.attempts += 1
    hc._maybe_update_health_check_count(False, 'reason-1')
    assert hc.current_consecutive_failures == 1
    assert hc.current_consecutive_successes == 0
    assert hc.healthy is False
    assert hc.reason == 'reason-1'

  def test_maybe_update_health_check_count_success(self):
    hc = self.health_checker.threaded_health_checker
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 0
    assert hc.healthy is True
    assert hc.running is False

    hc.attempts += 1
    hc._maybe_update_health_check_count(True, 'reason')
    assert hc.current_consecutive_failures == 0
    assert hc.current_consecutive_successes == 1
    assert hc.running is True
    assert hc.healthy is True

  @mock.patch('apache.aurora.executor.common.health_checker.time.sleep', spec=time.sleep)
  def test_run_success(self, mock_sleep):
    mock_sleep.return_value = None
    self.health.return_value = (True, 'success')
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert mock_sleep.call_count == 3
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 0
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 3
    assert self.health_checker.threaded_health_checker.running is True
    assert self.health_checker.threaded_health_checker.healthy is True
    assert self.health_checker.threaded_health_checker.reason is None

  @mock.patch('apache.aurora.executor.common.health_checker.time.sleep', spec=time.sleep)
  def test_run_failure(self, mock_sleep):
    mock_sleep.return_value = None
    self.health.return_value = (False, 'failure')
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert mock_sleep.call_count == 3
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 1
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 0
    assert self.health_checker.threaded_health_checker.running is False
    assert self.health_checker.threaded_health_checker.healthy is False
    assert self.health_checker.threaded_health_checker.reason == 'failure'

  @mock.patch('apache.aurora.executor.common.health_checker.time.sleep', spec=time.sleep)
  def test_first_success_after_grace_period_and_max_consecutive_failures(self, mock_sleep):
    mock_sleep.return_value = None
    health_status = [(False, 'failure-1'), (False, 'failure-2'), (True, None)]
    self.health.side_effect = lambda: health_status.pop(0)
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert mock_sleep.call_count == 3
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 0
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 1
    assert self.health_checker.threaded_health_checker.running is True
    assert self.health_checker.threaded_health_checker.healthy is True
    assert self.health_checker.threaded_health_checker.reason is None

  @mock.patch('apache.aurora.executor.common.health_checker.time.sleep', spec=time.sleep)
  def test_success_then_failures_ignored_till_grace_period_ends(self, mock_sleep):
    mock_sleep.return_value = None
    health_status = [(True, None), (False, 'failure-2'), (False, 'failure-3')]
    self.health.side_effect = lambda: health_status.pop(0)
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert mock_sleep.call_count == 3
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 1
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 0
    assert self.health_checker.threaded_health_checker.running is True
    assert self.health_checker.threaded_health_checker.healthy is False
    assert self.health_checker.threaded_health_checker.reason == 'failure-3'

  @mock.patch('apache.aurora.executor.common.health_checker.time.sleep', spec=time.sleep)
  def test_run_failure_unhealthy_when_failfast(self, mock_sleep):
    mock_sleep.return_value = None
    health_status = [(False, 'failure-1'), (False, 'failure-2'), (False, 'failure-3')]
    self.health.side_effect = lambda: health_status.pop(0)
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert mock_sleep.call_count == 3
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 1
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 0
    assert self.health_checker.threaded_health_checker.running is False
    assert self.health_checker.threaded_health_checker.healthy is False
    assert self.health_checker.threaded_health_checker.reason == 'failure-3'

  @mock.patch('apache.aurora.executor.common.health_checker.time.sleep', spec=time.sleep)
  def test_run_unhealthy_after_callback(self, mock_sleep):
    mock_sleep.return_value = None
    health_status = [(True, None), (True, None), (False, 'failure-4'), (False, 'failure-5')]
    self.health.side_effect = lambda: health_status.pop(0)
    mock_is_set = mock.Mock(spec=threading._Event.is_set)
    liveness = [False, False, False, False, True]
    mock_is_set.side_effect = lambda: liveness.pop(0)
    self.health_checker.threaded_health_checker.dead.is_set = mock_is_set
    self.health_checker.threaded_health_checker.run()
    assert mock_sleep.call_count == 4
    assert self.health_checker.threaded_health_checker.current_consecutive_failures == 2
    assert self.health_checker.threaded_health_checker.current_consecutive_successes == 0
    assert self.health_checker.threaded_health_checker.running is True
    assert self.health_checker.threaded_health_checker.healthy is False
    assert self.health_checker.threaded_health_checker.reason == 'failure-5'

  @mock.patch('apache.aurora.executor.common.health_checker.ExceptionalThread.start',
      spec=ExceptionalThread.start)
  def test_start(self, mock_start):
    assert mock_start.call_count == 0
    self.health_checker.threaded_health_checker.start()
    mock_start.assert_called_once_with(self.health_checker.threaded_health_checker)

  def test_stop(self):
    assert not self.health_checker.threaded_health_checker.dead.is_set()
    self.health_checker.threaded_health_checker.stop()
    assert self.health_checker.threaded_health_checker.dead.is_set()


text:
Home News A glimmer of hope for Britain? 11 TIMES fewer Covid patients in Omicron-ravaged South Africa are now being admitted to hospital than during summer Delta wave - as country's health minister hails signs that crisis HAS peaked in worst-hit province
A glimmer of hope for Britain? 11 TIMES fewer Covid patients in Omicron-ravaged South Africa are now being admitted to hospital than during summer Delta wave - as country's health minister hails signs that crisis HAS peaked in worst-hit province
Eleven times fewer patients diagnosed with Covid in Omicron-stricken South Africa are being admitted to hospital now than in the summer, experts claimed today.
The country's health minister Joe Phaahla revealed 1.7 per cent of Covid cases went on to be hospitalised during the second week of the current wave. For comparison, he said the equivalent figure was 19 per cent in the second week of South Africa's Delta crisis.
It comes as 4,000 fewer new cases were reported in the country in the past 24 hours. A total of 20,713 new infections were reported today, according to the National Institute for Communicable Diseases.
A further 35 Covid-related deaths were also recorded on Friday, bringing total fatalities to 90,297.
Mr Phaahla told a press conference there were early indications that 'the peak has been reached' in Gauteng, which was first to feel the full force of the variant.
The province's daily infections have started to trend downwards, about three weeks after the ultra-infectious variant took hold.
Higher immunity levels due to vaccination and previous infection now than when Delta took off are thought to be behind the lower hospitalisation rate.
But Mr Phaahla suggested Omicron may have evolved to be milder, bolstering claims made by doctors treating patients on the frontline.
Public health official Wassila Jassat, who also attended the conference, said South Africa had fewer patients needing oxygen now than when Delta emerged. She added that patients were hospitalised for a shorter period.
It is the latest glimmer of hope for Britain that Omicron may be milder than first feared, suggesting that its rapid rise won't cripple the health service in the coming weeks – despite gloomy projections.
But Professor Chris Whitty has warned against making comparisons between the two countries, pointing out that a far larger proportion of South Africa's population is young and less vulnerable compared to the UK.
Data from the National Institute for Communicable Diseases showed that less than two per cent of patients were being hospitalised in the second week of the Omicron wave in South Africa. For comparison, when the Delta wave struck it was 12 per cent
Graph shows: the proportion of Omicron patients admitted to intensive care units in Guateng, the epicentre of South Africa's fourth wave, during the Omicron wave (left) and Delta wave (right)
Graph shows: the proportion of Omicron patients put on ventilators in Guateng, the epicentre of South Africa's fourth wave, during the Omicron wave (left) and Delta wave (right)
Daily Covid cases in South Africa have risen 10 per cent in a week today. But the country is currently enjoying a public holiday, which likely skewed the figures
Hospitalisations in the country fell 23.4 per cent in a week today after another 374 people were admitted. But this could also be down to the holiday, with fewer people available to process data
Graph shows: The number of cases (left), hospitalisations (central) and intensive care admissions (right) from the start of the Delta (blue lines) and Omicron (red lines) waves in South Africa
Health minister Joe Phaahla said in a press conference today that 1.7 per cent of Covid cases were currently being admitted to hospital. For comparison, he said it was 19 per cent during the second wave. Mr Phaahla is pictured above getting a Covid jab
Mr Phaahla did not present an analysis to back up his suggestion that a lower proportion of patients were being hospitalised now than when Delta took off.
But two-thirds more swabs were being carried out every day at this point during the Omicron wave than for Delta which may have skewed the analysis.
Officials figures show some 54,800 tests were carried out every day in the second week of the current wave, compared to 32,600 in the second week of the Delta wave.
Omicron was first detected in South Africa on November 25, and Delta was first detected on May 8.
NHS is forced to drop rules telling fully-vaxxed staff who... 'They were bright, caring, loveable boys': Agony of the... Around 936,000 people in England had Covid last week - one... Britain expects at least another six months of Covid chaos:... ALL major Covid vaccines are 'markedly' weaker at preventing... London tube strike WILL go ahead from TONIGHT with 24-hour...
Its cases rocketed when Omicron first struck the country, and earlier this week reached a record high of almost 26,000 infections reported in a single day. Yesterday they rose 10 per cent week-on-week after another 24,700 were recorded.
Hospitalisations in the country also dipped yesterday compared to the same time last week, but they are trending upwards to about 600 admissions a day from less than 100 before Omicron emerged.
But they are still below hospitalisations during the country's second wave, which surged above 700 a day.
Doctors on the frontlines say that fewer patients need to be admitted to ICU or receive oxygen than when Delta took off, suggesting it may be more mild.
Mr Phaahla told the conference: 'We believe that [the reduced severity] might not necessarily just be that Omicron is less virulent.'
He said it also depends 'coverage of vaccination' and 'natural immunity', adding: 'That's why we are seeing mild illness.'
He also pointed to a slowdown in cases in Gauteng, where the average number of daily infections has dipped from 10,448 at the peak four days ago to 9,235 yesterday.
He said: 'In terms of case numbers and percentage of testing positive, we are seeing a decrease in Gauteng, and early indications suggest that the peak has been reached, but there is rapid increase in the other provinces.'
Professor Whitty has warned against comparisons with South Africa, however, telling a gloomy Downing Street press conference on Wednesday that it was hard to compare the two.
He said: 'I think there has been a certain amount of commentary about the fact that doctors and scientists from South Africa... have indicated that there may be some reduction in the hospitalisation rates they are seeing with Omicron.
'I want to put a really serious caution on this because I think it has been overinterpreted.
'The amount of immunity for this wave because of prior Delta wave and vaccination is far higher than it was for their last wave and, therefore, the fact they have a lower hospitalisation rate this time is unsurprising.
'That doesn't mean there is some degree of milder disease — that is possible — but I think there is a danger that people have overinterpreted this to say this is not a problem and there's nothing to worry about. I want to be clear, I am afraid this is going to be a problem.'
A glimmer of hope for Britain? 11 TIMES fewer Covid patients in Omicron-ravaged South Africa are now being admitted to hospital than during summer Delta wave - as country's health minister hails signs that crisis HAS peaked in worst-hit province Reviewed by free heip on December 18, 2021 Rating: 5

redpajama_set_name: RedPajamaCommonCrawl


text:
All the Action at DU: AISA, SFI protest against ABVP's 'demonising effect' on syllabus, call it shameful
Teachers and left-leaning organisations took out a protest rally in the North Campus of DU to condemn the alleged interference of the ABVP
Prajanma Das
Image for representational purpose only
July 23, 2019 was an eventful day for the Delhi University — ABVP protested and urged the HODs to consider student representation in the syllabus committees to allow rational debate, the professors accused them of hooliganism and that the police was of no help and some teachers and left-leaning organisations took out a protest rally in the North Campus of DU "to save the academic freedom in the campus".
All India Students' Association called for a march in defence of academic freedom and against ABVP's interference in the syllabus making of various departments of DU, namely English, History, Sociology and Political Science. "Hundreds joined the march which included students of English Department, members of AISA, SFI, KYS, Pinjra Tod, DSU and teachers of Delhi University as well," said Kawalpreet Kaur, President of AISA. "The task of designing the syllabus is an exercise done by the professors of the departments who have full autonomy to make courses of their own. It is shameful that today DU administration allowed the ABVP to enter premises of Head of Departments of Social Science courses where they were seen threatening the professors to mould the syllabus according to the wishes of the ABVP. Such an atmosphere of fear defeats the very purpose of a democratic university."
Aseif Ali, a student of English Department, said "It saddens me to see our department being demonised by the ABVP. There is an atmosphere of threat in the entire department. Our department teaches students to develop a critical perspective that is why it has come under attack from the ABVP. We stand in support of our professors."
The administration's stand in the whole issue has also come under fire from both sides, "It is saddening to see ABVP's constant involvement in threats and attacks and the DU administration's deafening silence about it. It is high time that the students and teachers should come together and fight for their academic freedom," said Abha Dev Habib, who teaches at Miranda House said.
students science ABVP protest Delhi University campus AISA
Nine months on, final year Symbiosis Law School students rusticated for #MeToo claims out in the cold
DU Syllabus row: ABVP protests outside departments for 'rational debate', teachers say they've been 'chased away'
JNU's Yogi has now started a Hindu Defence League to stop students offering Namaz in library


The Administration of the United States as well as the authorities in Israel, day and night, never tire of repeating that "the State of Israel is the only democracy in the Middle East." However, some Israeli realities call into question this assessment. One of typical phenomenons of Israel was discussed in American Jewess Elana Sztokman's book "The War on Women in Israel. How Religious Radicalism is Stifling the Voice of a Nation". In many, she writes, primarily orthodox circles of Israel misogyny and discrimination against women thrive. And this discrimination is supported by the authorities, the state itself, where Judaism is the state religion.
In public transportation in Israel (primarily buses) there are "special compartments in the rear seats." Only women can sit there. In Israel, there are cities where women are required to walk only on certain sides of the street, and there are places where women are not allowed to be. Sections are allocated for women in post offices and banks, in parks, and at entertainment attractions… Segregation on buses, allocating special walkways for women, and especially the ban on performances by female artists in army units is increasingly becoming the norm in Israel. Gender segregation is increasingly spreading to other areas of life, where there was nothing like this previously. One and a half years ago in Israel there already were more than 150 bus routes, where the segregation policy was in effect. Public transport in the most populated areas of the Jewish state is forced to comply with the laws of the Sabbath, that is, it does not operate on Saturdays. There was a time when this sort of segregation mainly occurred in the synagogues, but today it is increasingly spreading to other areas of life, in the public sphere, in the army, and even in private homes. In houses that pay particular attention to religious precepts of Judaism today there are separate places allocated just for women. Women are only allowed to eat in the kitchen, separately from men.
Elana Sztokman in meeting with readers recalled the legendary Rosa Parks. Rosa's refusal to sit behind the segregated "whites only" and "blacks only" bus sections once kindled the flame of civil disobedience and was an important step in the struggle for civil rights in the United States. Sztokman said that for her, as an American religious Jew, to be in the back of a bus in Jerusalem was totally unacceptable. But in Israel, this situation has not yet caused mass protests. Sztokman remarked: "The War on Women in Israel" is not an invention of its own religious orthodoxy. Over the past forty years almost all American ideas and values have taken root in Israel: political, economic and social, religious and secular, right and left, conservative and liberal. Jewish fundamentalism, both ultra-Orthodox and right religious nationalism was also largely imported to Israel from the United States. Until now, this has largely been instigated by graduates of religious schools in Brooklyn – the "yeshivahs".
Like so much else in Israel, the policy of segregation takes a semi-official character. On paper the law is the same for everyone and does not recognise such a thing as an ultra-Orthodox sector or an ultra-Orthodox community. However, in reality, the authorities are constantly breaking the law. Even worse is the implementation of laws on labour and vacation. Here the power of the law is flipped like a switch, sometimes focusing on secular circles of society. In other cases, laws are bent in favour of religious policy. Like so much else in Israel, the policy of segregation takes place under the "telephone justice".
Israel declares itself to be a secular state, but the process of religious revival in Israel is identical to the processes occurring in the Middle East. Like most of its neighbouring Arab and Islamic Republics, Israel's religion is not separated from the state, acts of civil status come within the purview of religious courts.
One of the most sensitive spots and hot spots of the struggle for women's rights – and this is the holiest site of Judaism – is the Western Wall. In her book Sztokman describes the history of the women's struggle for the right to pray there beyond the special sector. The "Women of the Wall" movement was created in the 1980s within a much broader movement for the admission of women to the prayer halls of Orthodox synagogues. From the outset, the feminists in response to their demands were met with violence at the Western Wall. The author of article you are reading was fortunate enough to observe the first appearance of "Women of the Wall", when the indignant crowd of religious Jews, headed by Shmuel Rabinovitch, Rabbi of the synagogue of the Wailing Wall, attacked the worshipping women. Chairs, rocks and even bags of filth were thrown at the women. The case of admitting women into the male area at the Western Wall has come before the courts. In 2011, a new stage arose. The Israeli police did not shy from taking sides and began to arrest the "Women of the Wall". The courts also toughened their stance, and the fenced part of the area recognised as an "orthodox synagogue", even though the place was built as a result of statements after the victory of the Six Day War that "The Wall is ours!" and will be the property of the Israeli nation, and not of a specific religious community.
In the ultra-Orthodox community there are many unfortunate women. There are hotlines for victims of violence. The organization Bat Melech established two shelters for women from Orthodox communities. And the resistance of religious circles is great. Last year, it was reported that the Chief Rabbinate demanded cell-phone companies to block the phones of the emergency organisation Bat Melech and other psychological assistance services.
Preparedness to use violence is increasingly embraced by Orthodox Judaism, which is often supported by government structures.
Israeli society experienced ultra-Orthodox protest for the first time in 1986 in Jerusalem when almost all the bus stops were burned because advertisements for fashion and cosmetics were displayed there. It was then that the Jerusalemites waiting for buses in the heat, deprived of their usual shade suddenly felt anxiety about their future.
Authorities in "free" Israel are unlikely to side with Elana Sztokman. By their nature, they are focused on segregation, not only of the Palestinians, but as in this case, of women.
Alexander Efimov, PhD of Historical Sciences, an expert on Middle East problems, exclusively for the online magazine "New Eastern Outlook".