import pytest
import retrying
import shakedown
import time
import traceback

from datetime import timedelta
from dcos.mesos import DCOSClient
from dcos import mesos
from shakedown import *
from utils import *

MAX_CONSECUTIVE_SCALE_FAILS = 9
MAX_HOURS_OF_TEST = 4

EVENT_HEADER = '    event:'

ERROR_LAUNCH = 'Error (launch failure):'
ERROR_SCALING = 'Error (scaling error):'
ERROR_SCALE_TIMEOUT = 'Error (scale timeout):'
ERROR_MARATHON_TIMEOUT = 'Futures timed out:'
ERROR_DEPLOYMENT = 'Error (deployment error):'

FATAL_NOT_SCALING = 'Fatal (not scaling):'
FATAL_CONSECUTIVE_LAUNCH = 'Fatal (consecutive launch):'
FATAL_CONSECUTIVE_SCALING = 'Fatal (consecutive scaling)'
FATAL_CONSECUTIVE_DEPLOYMENT = 'Fatal (consecutive deployment):'
FATAL_CONSECUTIVE_UNDEPLOYMENT = 'Fatal (consecutive undeployment):'

ERRORS = [ERROR_LAUNCH, ERROR_SCALING, ERROR_SCALE_TIMEOUT, ERROR_MARATHON_TIMEOUT,
          ERROR_DEPLOYMENT, FATAL_CONSECUTIVE_LAUNCH, FATAL_NOT_SCALING,
          FATAL_CONSECUTIVE_DEPLOYMENT, FATAL_CONSECUTIVE_UNDEPLOYMENT, FATAL_CONSECUTIVE_SCALING]

SKIP_RESOURCES = 'Insufficient Resources'
SKIP_PREVIOUS_TEST_FAILED = 'Previous Scale Test Failed'


def app(id=1, instances=1):
    app_json = {
      "id": "",
      "instances":  1,
      "cmd": "for (( ; ; )); do sleep 100000000; done",
      "cpus": 0.01,
      "mem": 32,
      "disk": 0
    }
    if not str(id).startswith("/"):
        id = "/" + str(id)
    app_json['id'] = id
    app_json['instances'] = instances

    return app_json


def group(gcount=1, instances=1):
    id = "/2deep/group"
    group = {
        "id": id,
        "apps": []
    }

    for num in range(1, gcount + 1):
        app_json = app(id + "/" + str(num), instances)
        group['apps'].append(app_json)

    return group


def constraints(name, operator, value=None):
    constraints = [name, operator]
    if value is not None:
        constraints.append(value)
    return [constraints]


def unique_host_constraint():
    return constraints('hostname', 'UNIQUE')


def delete_all_apps():
    client = marathon.create_client()
    client.remove_group("/", True)


def time_deployment(test=""):
    client = marathon.create_client()
    start = time.time()
    deployment_count = 1
    while deployment_count > 0:
        # need protection when tearing down
        try:
            deployments = client.get_deployments()
            deployment_count = len(deployments)
            if deployment_count > 0:
                time.sleep(1)
        except:
            wait_for_service_endpoint('marathon-user')
            pass

    end = time.time()
    elapse = round(end - start, 3)
    return elapse


def delete_group(group="/2deep/group"):
    client = marathon.create_client()
    client.remove_group(group, True)


def deployment_less_than_predicate(count=10):
    client = marathon.create_client()
    return len(client.get_deployments()) < count


def launch_group(test_obj):
    """ Launches a "group" style test, which is 1 HTTP Post request for all
        of the apps defined by count.  It is common to launch X apps as a group
        with only 1 instance each.  It is possible to control the number of instances
        of an app.
    """
    client = marathon.create_client()
    client.create_group(group(test_obj.count, test_obj.instance))


def count_test_app(test_obj):
    """
    Runs the `count` scale test for apps in marathon.   This is for apps and not pods.
    The count test is defined as X number of apps with Y number of instances.
    Y is commonly 1 instance and the test is scaling up to X number of applications.
    The details of how many apps and how many instances are defined in the test_obj.
    This test will make X number of HTTP requests against Marathon.

    :param test_obj: Is of type ScaleTest and defines the criteria for the test and logs the results and events of the test.
    """

    with clean_marathon_state(test_obj):
        # launch
        test_obj.start_test()
        launch_results = test_obj.launch_results
        try:
            launch_apps(test_obj)
        except Exception as e:
            launch_results.failed(e)
            wait_for_marathon_up(test_obj)
        else:
            launch_results.completed()

        # deployment
        try:
            if launch_results.success:
                time_deployment2(test_obj)
            else:
                test_obj.deploy_results.failed("Unable to continue based on launch failure")

        except Exception as e:
            print(e)
            test_obj.deploy_results.failed(e)


def launch_apps(test_obj):
    """ This function launches (makes HTTP POST requests) for apps.  It is used
        for instance and count tests.   Instance test will only have 1 count with X
        instances and is the simple case.
        The group test uses a different launch function.
    """

    client = marathon.create_client()
    count = test_obj.count
    instances = test_obj.instance
    launch_results = test_obj.launch_results
    deploy_results = test_obj.deploy_results
    scale_failure_count = 0
    for num in range(1, count + 1):
        try:
            client.add_app(app(num, instances))
            scale_failure_count = 0

            # every 100 adds wait for scale up
            if not num % 100:
                target = num * instances
                deploy_results.set_current_scale(current_scale())
                # wait for target
                if count_deployment(test_obj, target):
                    abort_msg = 'Count test launch failure at {} out of {}'.format(num, test_obj.target)
                    test_obj.add_event(abort_msg)
                    raise Exception(abort_msg)

        except Exception as e:
            log_error_event(test_obj, e, ERROR_LAUNCH)

            scale_failure_count = scale_failure_count + 1

            # 9 tries to see if scale increases, if no abort
            if scale_failure_count > MAX_CONSECUTIVE_SCALE_FAILS:
                abort_msg = 'Aborting based on too many failures: {}'.format(scale_failure_count)
                log_error_event(test_obj, abort_msg, FATAL_CONSECUTIVE_DEPLOYMENT, True)
                raise Exception(abort_msg)
            # need some time
            else:
                time.sleep(calculate_scale_wait_time(test_obj, scale_failure_count))
                quiet_wait_for_marathon_up(test_obj)


def log_error_event(test_obj, message, message_type='', noisy=False):
    full_message = '{} {}'.format(message_type, message)
    if test_obj is not None:
        test_obj.add_event(full_message)
    if noisy:
        print(full_message)


def instance_test_app(test_obj):
    """
    Runs the `instance` scale test for apps in marathon.   This is for apps and not pods.
    The instance test is defined as 1 app with X number of instances.
    the test is scaling up to X number of instances of an application.
    The details of how many instances are defined in the test_obj.
    This test will make 1 HTTP requests against Marathon.

    :param test_obj: Is of type ScaleTest and defines the criteria for the test and logs the results and events of the test.
    """

    with clean_marathon_state(test_obj):
        # launch
        test_obj.start_test()
        launch_results = test_obj.launch_results
        try:
            launch_apps(test_obj)
        except:
            # service unavail == wait for marathon
            launch_results.failed('Failure to launched (but we still will wait for deploys)')
            wait_for_marathon_up(test_obj)
        else:
            launch_results.completed()

        # deployment
        try:
            test_obj.reset_loop_count()
            time_deployment2(test_obj)
        except Exception as e:
            print(e)
            test_obj.deploy_results.failed(e)


def group_test_app(test_obj):
    """
    Runs the `group` scale test for apps in marathon.   This is for apps and not pods.
    The group test is defined as X number of apps with Y number of instances.
    Y number of instances is commonly 1.  The test is scaling up to X number of application and instances as submitted as 1 request.
    The details of how many instances are defined in the test_obj.
    This test will make 1 HTTP requests against Marathon.

    :param test_obj: Is of type ScaleTest and defines the criteria for the test and logs the results and events of the test.
    """

    with clean_marathon_state(test_obj):
        # launch
        test_obj.start_test()
        launch_results = test_obj.launch_results
        try:
            launch_group(test_obj)
        except Exception as e:
            print(e)
            # service unavail == wait for marathon
            launch_results.failed(e)
            wait_for_marathon_up(test_obj)
        else:
            launch_results.completed()

        # deployment
        try:
            test_obj.reset_loop_count()
            time_deployment2(test_obj)
        except Exception as e:
            print(e)
            test_obj.deploy_results.failed(e)


def delete_all_apps_wait(test_obj=None, msg='undeployment failure'):
    """ Used to remove all instances of apps and wait until the deployment finishes
    """

    if test_obj is not None and test_obj.deploy_results.current_scale > 0:
        test_obj.add_event('Undeploying {} tasks'.format(test_obj.deploy_results.current_scale))

    try:
        delete_all_apps()
    except Exception as e:
        log_error_event(test_obj, e, noisy=True)

    # some deletes (group test deletes commonly) timeout on remove_app
    # however it is a marathon internal issue on getting a timely response
    # all tested situations the remove did succeed
    try:
        undeployment_wait(test_obj)
    except Exception as e:
        log_error_event(test_obj, e, noisy=True)
        assert False, e


def undeployment_wait(test_obj=None):
    client = marathon.create_client()
    start = time.time()
    deployment_count = 1
    failure_count = 0
    while deployment_count > 0:
        # need protection when tearing down
        try:
            deployments = client.get_deployments()
            deployment_count = len(deployments)

            if deployment_count > 0:
                time.sleep(1)
                failure_count = 0
        except:
            failure_count += 1
            # consecutive failures great than x
            if failure_count > 10 and test_obj is not None:
                test_obj.failed('Too many failures waiting for undeploy', FATAL_CONSECUTIVE_UNDEPLOYMENT)
                raise TestException()

            wait_for_marathon_up(test_obj)
            pass

    if test_obj is not None:
        test_obj.undeploy_complete(start)


def count_deployment(test_obj, step_target):

    deploy_results = test_obj.deploy_results
    client = marathon.create_client()

    deploying = True
    abort = False
    failure_count = 0

    scale_failure_count = 0
    while deploying and not abort:
        try:

            task_count = current_scale()
            deploy_results.set_current_scale(task_count)

            deploying = task_count < step_target

            if deploying:
                time.sleep(calculate_deployment_wait_time(test_obj))
                # reset failure count,  it is used for consecutive failures
                failure_count = 0
                quiet_wait_for_marathon_up(test_obj)

            abort = abort_deployment_check(test_obj)
            scale_failure_count = 0
        except DCOSScaleException as e:
            # current scale is lower than previous scale
            log_error_event(test_obj, e, ERROR_SCALING)
            scale_failure_count = scale_failure_count + 1

            # consecutive failures
            if scale_failure_count > MAX_CONSECUTIVE_SCALE_FAILS:
                deploy_results.failed(
                    'Aborting based on too many failures: {}'.format(scale_failure_count),
                    FATAL_CONSECUTIVE_SCALING)
                abort = True
            # need some time
            else:
                time.sleep(calculate_scale_wait_time(test_obj, scale_failure_count))
                quiet_wait_for_marathon_up(test_obj)

        except DCOSNotScalingException as e:
            deploy_results.failed(e, FATAL_NOT_SCALING)
            abort = True

        except Exception as e:
            log_error_event(test_obj, e, ERROR_DEPLOYMENT)
            failure_count = failure_count + 1

            # consecutive failures > x will fail test
            if failure_count > MAX_CONSECUTIVE_SCALE_FAILS:
                message = 'Too many failures query for deployments'
                deploy_results.failed(message, FATAL_CONSECUTIVE_DEPLOYMENT)
                raise TestException(message)

            time.sleep(calculate_deployment_wait_time(test_obj, failure_count))
            quiet_wait_for_marathon_up(test_obj)
            pass

    return abort


def time_deployment2(test_obj):
    """ Times the deployment of a launched set of applications for this test object.
        This function will wait until the following conditions are met or occur:
            * target scale is reached
            * 5 consecutive DCOSScaleException (current scale is less than previous scale)
            * 1 DCOSNotScalingException (scale hasn't increased for 20 mins)
            * Any other exception happens 10x consecutive (this happens at very high scale)
    """

    deploy_results = test_obj.deploy_results
    client = marathon.create_client()

    deploying = True