#!/usr/bin/env python3.4
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import random, time
import hashlib
import os
from threading import Thread, Lock
import mailbox
import email.errors, email.utils, email.header
from urllib.request import urlopen
import re
import chardet
import datetime
import configparser
import argparse
from os import listdir
from os.path import isfile, join, isdir
import glob
import codecs
import multiprocessing
import tempfile
import gzip

try:
    from elasticsearch import Elasticsearch, helpers
    from formatflowed import convertToWrapped
except:
    print("Sorry, you need to install the elasticsearch and formatflowed modules from pip first.")
    sys.exit(-1)
    
y = 0
baddies = 0
block = Lock()
lists = []
start = time.time()
quickmode = False
lid_override = None
private = False
appender = "apache.org"


source = "./"
maildir = False
list_override = None
project = ""
recursive = False
filebased = False
fileToLID = {}
interactive = False
extension = "*.mbox"
attachments = False
piperWeirdness = False
parseHTML = False
iBody = None
resendTo = None
timeout = 600

# Fetch config
config = configparser.RawConfigParser()
config.read('ponymail.cfg')
auth = None
if config.has_option('elasticsearch', 'user'):
    auth = (config.get('elasticsearch','user'), config.get('elasticsearch','password'))



ssl = False
dbname = config.get("elasticsearch", "dbname")
if config.has_option("elasticsearch", "ssl") and config.get("elasticsearch", "ssl").lower() == 'true':
    ssl = True
    
cropout = None
if config.has_option("debug", "cropout") and config.get("debug", "cropout") != "":
    cropout = config.get("debug", "cropout")
    
uri = ""
if config.has_option("elasticsearch", "uri") and config.get("elasticsearch", "uri") != "":
    uri = config.get("elasticsearch", "uri")
es = Elasticsearch([
    {
        'host': config.get("elasticsearch", "hostname"),
        'port': int(config.get("elasticsearch", "port")),
        'use_ssl': ssl,
        'url_prefix': uri,
        'http_auth': auth
    }],
    max_retries=5,
    retry_on_timeout=True
    )

rootURL = ""

def parse_attachment(part):
    cd = part.get("Content-Disposition", None)
    if cd:
        dispositions = cd.strip().split(";")
        if dispositions[0].lower() == "attachment":
            try:
                fd = part.get_payload(decode=True)
                if fd:
                    attachment = {}
                    attachment['content_type'] = part.get_content_type()
                    attachment['size'] = len(fd)
                    attachment['filename'] = None
                    h = hashlib.sha256(fd).hexdigest()
                    b64 = codecs.encode(fd, "base64").decode('ascii')
                    attachment['hash'] = h
                    for param in dispositions[1:]:
                        key,val = param.split("=", 1)
                        if key.lower().strip() == "filename":
                            val = val.strip(' "')
                            print("Found attachment: %s" % val)
                            attachment['filename'] = val
                    if attachment['filename']:
                        return attachment, b64 # Return meta data and contents separately
            except:
                pass
    return None, None

def msgfiles(msg):
        attachments = []
        contents = {}
        if msg.is_multipart():    
            for part in msg.walk():
                part_meta, part_file = parse_attachment(part)
                if part_meta:
                    attachments.append(part_meta)
                    contents[part_meta['hash']] = part_file
        return attachments, contents
    
def pm_charsets(msg):
    charsets = set({})
    for c in msg.get_charsets():
        if c is not None:
            charsets.update([c])
    return charsets

def msgbody(msg):
    global parseHTML, iBody
    firstHTML = None
    body = None
    if msg.is_multipart():
        for part in msg.walk():
            try:
                if part.is_multipart(): 
                    for subpart in part.walk():
                        if subpart.get_content_type() == 'text/plain' and not body:
                                body = subpart.get_payload(decode=True)
                        elif subpart.get_content_type() == 'text/html' and parseHTML and not firstHTML:
                             firstHTML = subpart.get_payload(decode=True)
        
                elif part.get_content_type() == 'text/plain' and not body:
                    body = part.get_payload(decode=True)
                elif part.get_content_type() == 'text/html' and parseHTML and not firstHTML:
                    firstHTML = part.get_payload(decode=True)
            except Exception as err:
                print("Body parser error: %s" % err)
    elif msg.get_content_type() == 'text/plain':
        body = msg.get_payload(decode=True)
    elif msg.get_content_type() == 'text/html' and parseHTML and not firstHTML:
        firstHTML = msg.get_payload(decode=True)
        
    # this requires a GPL lib, user will have to install it themselves
    if firstHTML and (not body or len(body) <= 1 or (iBody and str(body).find(str(iBody)) != -1)):
        print("No body found as text/plain, converting HTML to text")
        body = html2text.html2text(firstHTML.decode("utf-8", errors='replace') if type(firstHTML) is bytes else firstHTML)

    for charset in pm_charsets(msg):
        try:
            body = body.decode(charset) if type(body) is bytes else body
        except:
            body = body.decode('utf-8', errors='replace') if type(body) is bytes else body
            
    return body  


def msgfactory(fp):
    try:
        return email.message_from_file(fp)
    except Exception as err:
        # Don't return None since that will
        # stop the mailbox iterator
        print("hmm: %s" % err)
        return None


class BulkThread(Thread):
    def assign(self, json, xes, dtype = 'mbox', wc = 'quorum'):
        self.json = json
        self.xes = xes
        self.dtype = dtype
        self.wc = wc

    def insert(self):
        global config
        sys.stderr.flush()
        iname = config.get("elasticsearch", "dbname")
        if not self.xes.indices.exists(iname):
            self.xes.indices.create(index = iname)

        js_arr = []
        i = 0
        for entry in self.json:
            js = entry
            js['@version'] = 1
            #js['@import_timestamp'] = time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime())
            js_arr.append({
                '_op_type': 'index',
                '_consistency': self.wc,
                '_index': iname,
                '_type': self.dtype,
                '_id': js['mid'],
                'doc': js,
                '_source': js
            })
        try:
            helpers.bulk(self.xes, js_arr)
        except Exception as err:
            print("Warning: Could not bulk insert: %s" % err)
        #print("Inserted %u entries" % len(js_arr))


class SlurpThread(Thread):

    def run(self):
        global block, y, es, lists, baddies, config, resendTo, timeout
        ja = []
        jas = []
        print("Thread started")
        mla = None
        ml = ""
        mboxfile = ""
        filename = ""
        xlist_override = None
    
        while len(lists) > 0:
            print("%u elements left to slurp" % len(lists))
            block.acquire()
            try:
                mla = lists.pop(0)
            except Exception as err:
                print("Could not pop list: %s" % err)
                block.release()
                return
            if not mla:
                print("Nothing more to do here")
                block.release()
                return
            block.release()
            y += 1
            EY = 1980
            EM = 1
            stime = time.time()
            dFile = False
            if filebased:
                
                tmpname = mla[0]
                filename = mla[0]
                xlist_override = mla[1]
                if filename.find(".gz") != -1:
                    print("Decompressing %s..." % filename)
                    try:
                        with open(filename, "rb") as bf:
                            bmd = bf.read()
                            bf.close()
                            bmd = gzip.decompress(bmd)
                            tmpfile = tempfile.NamedTemporaryFile(mode='w+b', buffering=1, delete=False)
                            tmpfile.write(bmd)
                            tmpfile.flush()
                            tmpfile.close()
                            tmpname = tmpfile.name
                            filename = tmpname
                            dFile = True # Slated for deletion upon having been read
                            print("%s -> %u bytes" % (tmpname, len(bmd)))
                    except Exception as err:
                        print("This wasn't a gzip file: %s" % err )
                print("Slurping %s" % filename)
            else:
                ml = mla[0]
                mboxfile = mla[1]
                xlist_override = list_override
                print("Slurping %s/%s" % (ml, mboxfile))
                m = re.match(r"(\d\d\d\d)(\d\d)", mboxfile)
                EY = 1997
                EM = 1
                if m:
                    EY = int(m.group(1))
                    EM = int(m.group(2))
                ctx = urlopen("%s%s/%s" % (source, ml, mboxfile ))
                inp = ctx.read().decode(ctx.headers.get_content_charset() or 'utf-8', errors='ignore')
    
                tmpname = hashlib.sha224(("%f-%f-%s-%s.mbox" % (random.random(), time.time(), ml, mboxfile)).encode('utf-8') ).hexdigest()
                with open(tmpname, "w") as f:
                    f.write(inp)
                    f.close()

            count = 0
            LEY = EY

            if maildir: