<repo_name>isabella232/canoe<file_sep>/tests/unit/test_handler.py
# coding: utf-8

import io
import os
import sys
import pytest
from unittest.mock import Mock
import xml.etree.ElementTree as ElementTree

CWD = os.path.dirname(os.path.realpath(__file__)) + "/../../"
sys.path.insert(0, os.path.join(CWD, ''))

from canoe import app # noqa


@pytest.fixture()
def kayako():
    client = Mock()
    departments = """<?xml version="1.0" encoding="UTF-8"?>
    <departments>
        <department>
            <id><![CDATA[1]]></id>
            <title><![CDATA[Project Name]]></title>
        </department>
        <department>
            <id><![CDATA[2]]></id>
            <title><![CDATA[Customer 2]]></title>
            <parentdepartmentid><![CDATA[1]]></parentdepartmentid>
        </department>
        <department>
            <id><![CDATA[3]]></id>
            <title><![CDATA[Customer 3]]></title>
            <parentdepartmentid><![CDATA[1]]></parentdepartmentid>
        </department>
        <department>
            <id><![CDATA[4]]></id>
            <title><![CDATA[Customer 4]]></title>
            <parentdepartmentid><![CDATA[1]]></parentdepartmentid>
        </department>
        <department>
            <id><![CDATA[5]]></id>
            <title><![CDATA[Other Main Department]]></title>
        </department>
    </departments>
    """
    client.list_departments.return_value = ElementTree.fromstring(departments)
    tickets = """<?xml version="1.0" encoding="UTF-8"?>
        <tickets>
        <ticket id="273" flagtype="5">
            <displayid><![CDATA[MAB-597-12345]]></displayid>
        </ticket>
        <ticket id="274" flagtype="5">
            <displayid><![CDATA[JAB-293-54321]]></displayid>
        </ticket>
    </tickets>
    """
    client.list_open_tickets.return_value = ElementTree.fromstring(tickets)
    posts = """<?xml version="1.0" encoding="UTF-8"?>
    <tickets>
        <ticket id="277" flagtype="5">
            <displayid><![CDATA[CYA-293-12345]]></displayid>
            <departmentid><![CDATA[20]]></departmentid>
            <userorganization><![CDATA[Customer Name]]></userorganization>
            <ownerstaffname><![CDATA[Staff Owner]]></ownerstaffname>
            <fullname><![CDATA[Customer]]></fullname>
            <email><![CDATA[<EMAIL>]]></email>
            <subject><![CDATA[Mayday Mayday]]></subject>
            <posts>
                <post>
                    <id><![CDATA[3]]></id>
                    <ticketpostid><![CDATA[1496]]></ticketpostid>
                    <ticketid><![CDATA[277]]></ticketid>
                    <dateline><![CDATA[1552419863]]></dateline>
                    <fullname><![CDATA[Sender FullName (customer)]]></fullname>
                    <email><![CDATA[<EMAIL>]]></email>
                    <contents><![CDATA[Thanks mate

                    ]]></contents>
                </post>
                <post>
                    <id><![CDATA[2]]></id>
                    <ticketpostid><![CDATA[1495]]></ticketpostid>
                    <ticketid><![CDATA[277]]></ticketid>
                    <dateline><![CDATA[1552418863]]></dateline>
                    <fullname><![CDATA[Sender FullName]]></fullname>
                    <email><![CDATA[<EMAIL>]]></email>
                    <contents><![CDATA[Any time!

                    Thank you,
                    Your support

                    ]]></contents>
                </post>
                <post>
                    <id><![CDATA[1]]></id>
                    <ticketpostid><![CDATA[1488]]></ticketpostid>
                    <ticketid><![CDATA[277]]></ticketid>
                    <dateline><![CDATA[1552317114]]></dateline>
                    <fullname><![CDATA[Sender FullName (customer)]]></fullname>
                    <email><![CDATA[<EMAIL>]]></email>
                    <contents><![CDATA[This fixed it,

                    Thank you!

                    ]]></contents>
                </post>
            </posts>
        </ticket>
    </tickets>
    """
    client.get_ticket.return_value = ElementTree.fromstring(posts)
    return client


@pytest.fixture()
def slack(monkeypatch):
    client = Mock()
    monkeypatch.setattr('canoe.app.slack_client', client)
    return client


@pytest.fixture()
def seed_event():
    return {'type': 'seed'}


@pytest.fixture()
def context():
    return {}


def test_list_relevant_department_ids(kayako):
    ids = app.list_relevant_department_ids(kayako, 'Project Name')
    assert ['1', '2', '3', '4'] == list(ids)


def test_seed_handler(seed_event, context, kayako, monkeypatch):
    session = Mock()
    monkeypatch.setattr('canoe.app.session', session)
    sqs = session.resource.return_value
    queue = sqs.Queue.return_value
    monkeypatch.setattr('canoe.app.kayako', kayako)
    monkeypatch.setenv('CANOE_ROOT_PROJECT_NAME', 'Project Name')
    app.seed_handler(seed_event, context)
    queue.send_messages.assert_called_with(
        Entries=[
            {'Id': '1', 'MessageBody': '{"department_id": "1"}'},
            {'Id': '2', 'MessageBody': '{"department_id": "2"}'},
            {'Id': '3', 'MessageBody': '{"department_id": "3"}'},
            {'Id': '4', 'MessageBody': '{"department_id": "4"}'}
        ]
    )


@pytest.fixture()
def sqs_departments_event():
    return {
        'Records': [
            {
                'body': '{"department_id": "2"}',
            }
        ]
    }


def test_distribute_departments_tickets_handler(
        sqs_departments_event, context, kayako, monkeypatch):
    session = Mock()
    monkeypatch.setattr('canoe.app.session', session)
    sqs = session.resource.return_value
    queue = sqs.Queue.return_value
    monkeypatch.setattr('canoe.app.kayako', kayako)
    app.distribute_departments_tickets_handler(sqs_departments_event, context)
    queue.send_messages.assert_called_with(
        Entries=[
            {'Id': '273', 'MessageBody': '{"ticket_id": "273"}'},
            {'Id': '274', 'MessageBody': '{"ticket_id": "274"}'}
        ]
    )


@pytest.fixture()
def sqs_check_tickets_event():
    return {
        'Records': [
            {
                'body': '{"ticket_id": "273"}',
            }
        ]
    }


def test_check_ticket_handler(
        sqs_check_tickets_event, context, kayako, monkeypatch):
    s3 = Mock()
    monkeypatch.setattr('canoe.app.s3', s3)
    state = """<?xml version="1.0" encoding="UTF-8"?>
    <tickets>
        <ticket id="277" flagtype="5">
            <displayid><![CDATA[CYA-293-12345]]></displayid>
            <departmentid><![CDATA[20]]></departmentid>
            <userorganization><![CDATA[Customer Name]]></userorganization>
            <ownerstaffname><![CDATA[Staff Owner]]></ownerstaffname>
            <fullname><![CDATA[Customer]]></fullname>
            <email><![CDATA[<EMAIL>]]></email>
            <subject><![CDATA[Mayday Mayday]]></subject>
            <posts>
                <post>
                    <id><![CDATA[2]]></id>
                    <ticketpostid><![CDATA[1495]]></ticketpostid>
                    <ticketid><![CDATA[277]]></ticketid>
                    <dateline><![CDATA[1552418863]]></dateline>
                    <fullname><![CDATA[Sender FullName]]></fullname>
                    <email><![CDATA[<EMAIL>]]></email>
                    <contents><![CDATA[Any time!

                    Thank you,
                    Your support

                    ]]></contents>
                </post>
                <post>
                    <id><![CDATA[1]]></id>
                    <ticketpostid><![CDATA[1488]]></ticketpostid>
                    <ticketid><![CDATA[277]]></ticketid>
                    <dateline><![CDATA[1552317114]]></dateline>
                    <fullname><![CDATA[Sender FullName (customer)]]></fullname>
                    <email><![CDATA[<EMAIL>]]></email>
                    <contents><![CDATA[This fixed it,

                    Thank you!

                    ]]></contents>
                </post>
            </posts>
        </ticket>
    </tickets>
    """
    s3_obj = {'Body': io.StringIO(state)}
    s3.get_object.return_value = s3_obj
    session = Mock()
    monkeypatch.setattr('canoe.app.session', session)
    sqs = session.resource.return_value
    queue = sqs.Queue.return_value
    monkeypatch.setattr('canoe.app.kayako', kayako)
    app.check_ticket_handler(sqs_check_tickets_event, context)
    queue.send_messages.assert_called_with(
        Entries=[
            {
                'Id': '0',
                'MessageBody': '{"type": "new_post", "object": {"dateline": "1552419863", "fullname": "Sender FullName (customer)", "email": "<EMAIL>", "contents": "Thanks mate\\n\\n                    ", "displayid": "CYA-293-12345", "userorganization": "Customer Name", "subject": "Mayday Mayday", "ticket_id": "273"}}'  # noqa: E501
            }
        ]
    )


def test_diff_new_posts_empty_state():
    posts = """<?xml version="1.0" encoding="UTF-8"?>
    <tickets>
        <ticket>
            <posts>
                <post>
                    <ticketpostid><![CDATA[1496]]></ticketpostid>
                    <dateline><![CDATA[1552419863]]></dateline>
                </post>
                <post>
                    <ticketpostid><![CDATA[1495]]></ticketpostid>
                    <dateline><![CDATA[1552418863]]></dateline>
                </post>
                <post>
                    <ticketpostid><![CDATA[1488]]></ticketpostid>
                    <dateline><![CDATA[1552317114]]></dateline>
                </post>
            </posts>
        </ticket>
    </tickets>
    """
    diff = app.diff_new_posts(ElementTree.fromstring(posts), None)
    diff_xml = [ElementTree.tostring(el, encoding="unicode") for el in diff]
    assert diff_xml == [
        '<post>\n'
        '                    <ticketpostid>1488</ticketpostid>\n'
        '                    <dateline>1552317114</dateline>\n'
        '                </post>\n'
        '            ',
        '<post>\n'
        '                    <ticketpostid>1495</ticketpostid>\n'
        '                    <dateline>1552418863</dateline>\n'
        '                </post>\n'
        '                ',
        '<post>\n'
        '                    <ticketpostid>1496</ticketpostid>\n'
        '                    <dateline>1552419863</dateline>\n'
        '                </post>\n'
        '                '
    ]


def test_diff_new_posts_empty_many_items():
    posts = """<?xml version="1.0" encoding="UTF-8"?>
    <tickets>
        <ticket>
            <posts>
                <post>
                    <ticketpostid><![CDATA[1496]]></ticketpostid>
                    <dateline><![CDATA[1552419863]]></dateline>
                </post>
                <post>
                    <ticketpostid><![CDATA[1495]]></ticketpostid>
                    <dateline><![CDATA[1552418863]]></dateline>
                </post>
                <post>
                    <ticketpostid><![CDATA[1488]]></ticketpostid>
                    <dateline><![CDATA[1552317114]]></dateline>
                </post>
            </posts>
        </ticket>
    </tickets>
    """
    state = """<?xml version="1.0" encoding="UTF-8"?>
    <tickets>
        <ticket>
            <posts>
                <post>
                    <ticketpostid><![CDATA[1488]]></ticketpostid>
                    <dateline><![CDATA[1552317114]]></dateline>
                </post>
            </posts>
        </ticket>
    </tickets>
    """
    diff = app.diff_new_posts(ElementTree.fromstring(posts), ElementTree.fromstring(state))
    diff_xml = [ElementTree.tostring(el, encoding="unicode") for el in diff]
    assert diff_xml == [
        '<post>\n'
        '                    <ticketpostid>1495</ticketpostid>\n'
        '                    <dateline>1552418863</dateline>\n'
        '                </post>\n'
        '                ',
        '<post>\n'
        '                    <ticketpostid>1496</ticketpostid>\n'
        '                    <dateline>1552419863</dateline>\n'
        '                </post>\n'
        '                '
    ]


@pytest.fixture()
def updates_event():
    return {
        'Records': [
            {
                'body': '{"type": "new_post", "object": {"dateline": "1552419863", "fullname": "Sender FullName (customer)", "email": "<EMAIL>", "contents": "Thanks mate\\n\\n                    ", "displayid": "CYA-293-12345", "userorganization": "Customer Name", "subject": "Mayday Mayday", "ticket_id": "273"}}'  # noqa: E501
            }
        ]
    }


def test_updates_notifications_handler(monkeypatch, slack, updates_event, context):
    app.updates_notifications_handler(updates_event, context)
    slack.chat_postMessage.assert_called_with(
        channel='PROJECTID',
        text='[CYA-293-12345]: Mayday Mayday\n@here Sender FullName (customer) left a comment on a ticket',
        blocks=[
            {
                'type': 'section',
                'text': {
                    'type': 'mrkdwn',
                    'text': '<https://kayako-srv.com?/Tickets/Ticket/View/273|[CYA-293-12345]: Mayday Mayday>'
                }
            },
            {
                'type': 'section',
                'text': {
                    'type': 'mrkdwn',
                    'text': '@here Sender FullName (customer) left a comment on a ticket'
                }
            }
        ]
    )
<file_sep>/canoe/app.py
import boto3
import json
import os
import logging
import sys
import bisect
import itertools

import slack

from xml.etree import ElementTree

CWD = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(CWD, 'lib'))

from kayako import Kayako       # noqa: E402

LOG_LEVEL = logging.INFO
boto3.set_stream_logger('', LOG_LEVEL)
logger = logging.getLogger()
logger.setLevel(LOG_LEVEL)

# Global variables are reused across execution contexts (if available)
session = boto3.Session()
s3 = session.client('s3')

kayako = Kayako(os.getenv('CANOE_KAYAKO_API_URL'),
                os.getenv('CANOE_KAYAKO_API_KEY'),
                os.getenv('CANOE_KAYAKO_SECRET_KEY'))

SLACK_CHANNEL_ID = os.getenv('CANOE_SLACK_CHANNEL_ID')
slack_client = slack.WebClient(token=os.getenv('CANOE_SLACK_API_TOKEN'))


def seed_handler(event, context):
    if event.get('type', None) != 'seed':
        logger.warning(f'unexpected event: {event}')
        return

    project_name = os.getenv('CANOE_ROOT_PROJECT_NAME')
    dep_ids = list_relevant_department_ids(kayako, project_name)
    queue_url = os.getenv('CANOE_CHECK_DEPARTMENT_QUEUE_URL')
    sqs = session.resource('sqs')
    queue = sqs.Queue(queue_url)
    messages = sqs_messages(dep_ids)
    if not messages:
        logger.warning('no messages to send')
        return

    send_messages(queue, messages)


def send_messages(queue, items, batch_size=10):
    iter_items = [iter(items)] * batch_size
    for batch in itertools.zip_longest(*iter_items, fillvalue=None):
        batch = list(filter(None, batch))
        queue.send_messages(Entries=batch)


def distribute_departments_tickets_handler(event, context):
    ticket_ids = []

    for record in event['Records']:
        message = json.loads(record['body'])
        department_id = message['department_id']
        tickets = kayako.list_open_tickets(department_id)
        for ticket in tickets.findall('.//ticket'):
            ticket_id = ticket.get('id')
            ticket_ids.append(ticket_id)

    queue_url = os.getenv('CANOE_CHECK_TICKET_QUEUE_URL')
    sqs = session.resource('sqs')
    queue = sqs.Queue(queue_url)
    messages = check_ticket_messages(ticket_ids)
    send_messages(queue, messages)


# we are including the top level department and sub departments as well
def list_relevant_department_ids(kayako, project_name):
    departments = kayako.list_departments()
    parent_el_ids = departments.findall(f".//department/title[.='{project_name}']../id")
    for parent_el_id in parent_el_ids:
        yield parent_el_id.text
        xpath = f".//department/parentdepartmentid[.='{parent_el_id.text}']../id"
        for dep_id_el in departments.findall(xpath):
            yield dep_id_el.text


def check_ticket_handler(event, context):
    tickets_updates = []
    for record in event['Records']:
        message = json.loads(record['body'])
        ticket_id = message['ticket_id']

        state = get_ticket_state(ticket_id)
        ticket = kayako.get_ticket(ticket_id)
        new_posts = diff_new_posts(ticket, state)
        updates = list(ticket_updates(ticket_id, ticket, new_posts))
        tickets_updates.extend(updates)
        if updates:
            save_ticket_state(ticket_id, ticket)

    if not is_in_learning_mode():
        queue_url = os.getenv('CANOE_TICKETS_UPDATES_QUEUE_URL')
        sqs = session.resource('sqs')
        queue = sqs.Queue(queue_url)
        messages = tickets_updates_messages(tickets_updates)
        send_messages(queue, messages)


def is_in_learning_mode():
    return os.getenv('CANOE_LEARNING_MODE') == 'true'


def updates_notifications_handler(event, context):
    for record in event['Records']:
        body = json.loads(record['body'])
        if body['type'] == 'new_post':
            new_post = body['object']
            text = message_text(new_post)
            blocks = message_blocks(new_post)
            slack_client.chat_postMessage(
                channel=SLACK_CHANNEL_ID,
                text=text,
                blocks=blocks)


def message_text(new_post):
    tmpl = ('[{displayid}]: {subject}\n'
            '@here {fullname} left a comment on a ticket')
    return tmpl.format(**new_post)


def message_blocks(new_post):
    portal_uri = os.getenv('CANOE_KAYAKO_UI_URL')
    ticket_link = '{portal_uri}?/Tickets/Ticket/View/{ticket_id}'.format(
        portal_uri=portal_uri, **new_post)
    return [
        {
            'type': 'section',
            'text': {
                'type': 'mrkdwn',
                'text': '<{link}|[{displayid}]: {subject}>'.format(link=ticket_link, **new_post)
            }
        },
        {
            'type': 'section',
            'text': {
                'type': 'mrkdwn',
                'text': '@here {fullname} left a comment on a ticket'.format(**new_post)
            }
        }
    ]


def save_ticket_state(ticket_id, ticket):
    bucket = tickets_state_bucket()
    key = ticket_state_key(ticket_id)
    xml = ElementTree.tostring(ticket, encoding="utf-8")
    s3.put_object(Bucket=bucket, Key=key, Body=xml)


def diff_new_posts(ticket, state):
    state_dateline = latest_post_dateline(state)
    posts = ticket.findall('.//posts/post')
    datelines = [int(post.find('dateline').text) for post in posts]
    datelined_posts = list(zip(datelines, posts))
    datelined_posts.sort(key=lambda p: p[0])
    datelines, posts = zip(*datelined_posts)
    pos = bisect.bisect_right(datelines, state_dateline)
    return posts[pos:]


def ticket_updates(ticket_id, ticket, posts):
    fields = ['displayid', 'userorganization', 'subject']
    update_base = extract_fields(ticket, fields, './/ticket/{}')
    update_base['ticket_id'] = ticket_id
    for post in posts:
        ticket_fields = ['dateline', 'fullname', 'email', 'contents']
        ticket_update = extract_fields(post, ticket_fields)
        ticket_update.update(update_base)
        yield ticket_update


def extract_fields(element, fields, path_tmpl=None):
    def format_path(p):
        return path_tmpl.format(p) if path_tmpl else p
    return {field: element.find(format_path(field)).text for field in fields}


def latest_post_dateline(state):
    if not state:
        return 0

    dateline_els = state.findall('.//post/dateline')
    return max([int(dl.text) for dl in dateline_els])


def get_ticket_state(ticket_id):
    state = read_ticket_state(ticket_id)
    if state:
        return ElementTree.parse(state)


def read_ticket_state(ticket_id):
    key = ticket_state_key(ticket_id)
    bucket = tickets_state_bucket()

    try:
        s3_object = s3.get_object(Bucket=bucket, Key=key)
        return s3_object['Body']
    except s3.exceptions.NoSuchKey:
        logger.info(f'No state found {bucket}/{key}')


def ticket_state_key(ticket_id):
    return f'tickets/{ticket_id}.xml'


def tickets_state_bucket():
    return os.getenv('CANOE_TICKETS_STATE_BUCKET')


def sqs_messages(department_ids):
    return [
        {
            'Id': dep_id,
            'MessageBody': json.dumps({'department_id': dep_id})
        }
        for dep_id in department_ids
    ]


def check_ticket_messages(ticket_ids):
    return [
        {
            'Id': ticket_id,
            'MessageBody': json.dumps({'ticket_id': ticket_id})
        }
        for ticket_id in ticket_ids
    ]


def tickets_updates_messages(updates):
    return [
        {
            'Id': str(index),
            'MessageBody': json.dumps({
                'type': 'new_post',
                'object': update
            })
        }
        for index, update in enumerate(updates)
    ]
<file_sep>/canoe/lib/kayako.py
import base64
import hmac
import random
import requests
from xml.etree import ElementTree
from requests.auth import AuthBase
from urllib.parse import urlsplit, parse_qsl, urlencode


class KayakoAuth(AuthBase):

    def __init__(self, api_key, secret_key):
        self._api_key = api_key
        self._secret_key = secret_key.encode('utf-8')

    def __call__(self, request):
        salt = str(random.getrandbits(32)).encode('utf-8')
        digest = hmac.digest(self._secret_key, msg=salt, digest='sha256')
        signature = base64.encodebytes(digest).replace(b'\n', b'')

        url_split = urlsplit(request.url)
        query = dict(parse_qsl(url_split.query))
        query['apikey'] = self._api_key
        query['salt'] = salt
        query['signature'] = signature
        auth_url_split = url_split._replace(query=urlencode(query))
        request.url = auth_url_split.geturl()
        return request


class Kayako:

    def __init__(self, url, api_key, secret_key):
        self._url = url
        self._session = requests.Session()
        self._session.auth = KayakoAuth(api_key, secret_key)

    def list_departments(self):
        text = self.request('get', '/Base/Department')
        return ElementTree.fromstring(text)

    def list_open_tickets(self, department_id):
        action = f'/Tickets/Ticket/ListAll/{department_id}/1/-1/-1/-1/-1/ticketid/ASC'
        text = self.request('get', action)
        return ElementTree.fromstring(text)

    def get_ticket(self, ticket_id):
        action = f'/Tickets/Ticket/{ticket_id}'
        text = self.request('get', action)
        return ElementTree.fromstring(text)

    def request(self, method, action, params=None, **kwargs):
        params = params or {}
        params['e'] = action
        response = self._session.request(
            method, self._url, params=params, **kwargs)
        response.raise_for_status()
        return response.text
<file_sep>/build.sh
STACK_NAME=canoe-helpdesk
S3_BUCKET=my-bucket #obtain it from internal docs

gmake build SERVICE=canoe
aws --region eu-west-2 cloudformation package \
    --template-file template.yaml     \
    --output-template-file packaged.yaml     \
    --s3-bucket $S3_BUCKET
aws --region eu-west-2 cloudformation deploy \
    --template-file $PWD/packaged.yaml \
    --stack-name $STACK_NAME \
    --parameter-overrides "file://$PWD/env.json" \
    --capabilities CAPABILITY_IAM
