from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
from future.builtins import *  # NOQA @UnusedWildImport

import warnings
from collections import defaultdict
import datetime
import os
import io

from obspy import UTCDateTime, read
from obspy.geodetics import kilometers2degrees, degrees2kilometers
from obspy.core.event import (
    Event, Origin, Magnitude, Comment, Catalog, EventDescription, CreationInfo,
    OriginQuality, Pick, WaveformStreamID, Arrival, Amplitude,
    FocalMechanism, MomentTensor, NodalPlane, NodalPlanes, QuantityError,
    Tensor, ResourceIdentifier)

mag_mapping = {"ML": "L", "MLv": "L", "mB": "B", "Ms": "s", "MS": "S",
               "MW": "W", "MbLg": "G", "Mc": "C"}

onsets = {'I': 'impulsive', 'E': 'emergent'}
# List of currently implemented line-endings, which in Nordic mark what format
# info in that line will be.
accepted_tags = ['1', '6', '7', 'E', ' ', 'F', 'M', '3']

class NordicParsingError(Exception):
    """
    Internal general error for IO operations in obspy.core.io.nordic.
    """
    def __init__(self, value):
        self.value = value

def _is_sfile(sfile, encoding='latin-1'):
    """
    Basic test of whether the file is nordic format or not.

    Not exhaustive, but checks some of the basics.

    :type sfile: str
    :param sfile: Path to sfile
    :type encoding: str
    :param encoding: Encoding of the file.

    :rtype: bool
    """
    if not hasattr(sfile, "readline"):
        try:
            with open(sfile, 'r', encoding=encoding) as f:
                tags = _get_line_tags(f=f, report=False)
        except Exception:
            return False
    else:
        try:
            tags = _get_line_tags(f=sfile, report=False)
        except Exception:
            return False
    if tags is not None:
        try:
            head_line = tags['1'][0][0]
        except IndexError:
            return False
        try:
            sfile_seconds = int(head_line[16:18])
        except ValueError:
            return False
        if sfile_seconds == 60:
            sfile_seconds = 0
        try:
            UTCDateTime(
                int(head_line[1:5]), int(head_line[6:8]), int(head_line[8:10]),
                int(head_line[11:13]), int(head_line[13:15]), sfile_seconds,
                int(head_line[19:20]) * 100000)
            return True
        except Exception:
            return False
    else:
        return False

def _get_line_tags(f, report=True):
    """
    Associate lines with a known line-type
    :param f: File open in read
    :param report: Whether to report warnings about lines not implemented
    """
    f.seek(0)
    line = f.readline()
    if len(line.rstrip()) != 80:
        # Cannot be Nordic
        raise NordicParsingError(
            "Lines are not 80 characters long: not a nordic file")
    f.seek(0)
    tags = defaultdict(list)
    for i, line in enumerate(f):
        try:
            line_id = line.rstrip()[79]
        except IndexError:
            line_id = ' '
        if line_id in accepted_tags:
            tags[line_id].append((line, i))
        elif report:
            warnings.warn("Lines of type %s have not been implemented yet, "
                          "please submit a development request" % line_id)
    return tags

def _int_conv(string):
    """
    Convenience tool to convert from string to integer.

    If empty string return None rather than an error.

    >>> _int_conv('12')
    12
    >>> _int_conv('')

    """
    try:
        intstring = int(string)
    except Exception:
        intstring = None
    return intstring

def _float_conv(string):
    """
    Convenience tool to convert from string to float.

    If empty string return None rather than an error.

    >>> _float_conv('12')
    12.0
    >>> _float_conv('')
    >>> _float_conv('12.324')
    12.324
    """
    try:
        floatstring = float(string)
    except Exception:
        floatstring = None
    return floatstring

def _str_conv(number, rounded=False):
    """
    Convenience tool to convert a number, either float or int into a string.

    If the int or float is None, returns empty string.

    >>> print(_str_conv(12.3))
    12.3
    >>> print(_str_conv(12.34546, rounded=1))
    12.3
    >>> print(_str_conv(None))
    <BLANKLINE>
    >>> print(_str_conv(1123040))
    11.2e5
    """
    if not number:
        return str(' ')
    if not rounded and isinstance(number, (float, int)):
        if number < 100000:
            string = str(number)
        else:
            exponant = int('{0:.2E}'.format(number).split('E+')[-1]) - 1
            divisor = 10 ** exponant
            string = '{0:.1f}'.format(number / divisor) + 'e' + str(exponant)
    elif rounded == 2 and isinstance(number, (float, int)):
        if number < 100000:
            string = '{0:.2f}'.format(number)
        else:
            exponant = int('{0:.2E}'.format(number).split('E+')[-1]) - 1
            divisor = 10 ** exponant
            string = '{0:.2f}'.format(number / divisor) + 'e' + str(exponant)
    elif rounded == 1 and isinstance(number, (float, int)):
        if number < 100000:
            string = '{0:.1f}'.format(number)
        else:
            exponant = int('{0:.2E}'.format(number).split('E+')[-1]) - 1
            divisor = 10 ** exponant
            string = '{0:.1f}'.format(number / divisor) + 'e' + str(exponant)
    else:
        return str(number)
    return string

def _evmagtonor(mag_type):
    """
    Switch from obspy event magnitude types to seisan syntax.

    >>> print(_evmagtonor('mB'))  # doctest: +SKIP
    B
    >>> print(_evmagtonor('M'))  # doctest: +SKIP
    W
    >>> print(_evmagtonor('bob'))  # doctest: +SKIP
    <BLANKLINE>
    """
    if mag_type == 'M':
        warnings.warn('Converting generic magnitude to moment magnitude')
        return "W"
    try:
        mag = mag_mapping[mag_type]
    except KeyError:
        warnings.warn(mag_type + ' is not convertible')
        return ''
    return mag

def _nortoevmag(mag_type):
    """
    Switch from nordic type magnitude notation to obspy event magnitudes.

    >>> print(_nortoevmag('B'))  # doctest: +SKIP
    mB
    >>> print(_nortoevmag('bob'))  # doctest: +SKIP
    <BLANKLINE>
    """
    if mag_type.upper() == "L":
        return "ML"
    inv_mag_mapping = {item: key for key, item in mag_mapping.items()}
    try:
        mag = inv_mag_mapping[mag_type]
    except KeyError:
        warnings.warn(mag_type + ' is not convertible')
        return ''
    return mag

def readheader(sfile, encoding='latin-1'):
    """
    Read header information from a seisan nordic format S-file.

    :type sfile: str
    :param sfile: Path to the s-file
    :type encoding: str
    :param encoding: Encoding for file, used to decode from bytes to string

    :returns: :class:`~obspy.core.event.event.Event`
    """
    with open(sfile, 'r', encoding=encoding) as f:
        tagged_lines = _get_line_tags(f)
        if len(tagged_lines['1']) == 0:
            raise NordicParsingError("No header lines found")
        header = _readheader(head_lines=tagged_lines['1'])
    return header

def _readheader(head_lines):
    """
    Internal header reader.
    :type head_lines: list
    :param head_lines:
        List of tuples of (strings, line-number) of the header lines.

    :returns: :class:`~obspy.core.event.event.Event`
    """
    # There are two possible types of origin line, one with all info, and a
    # subsequent one for additional magnitudes.
    head_lines.sort(key=lambda tup: tup[1])
    # Construct a rough catalog, then merge events together to cope with
    # multiple origins
    _cat = Catalog()
    for line in head_lines:
        _cat.append(_read_origin(line=line[0]))
    new_event = _cat.events.pop(0)
    for event in _cat:
        matched = False
        origin_times = [origin.time for origin in new_event.origins]
        if event.origins[0].time in origin_times:
            origin_index = origin_times.index(event.origins[0].time)
            agency = new_event.origins[origin_index].creation_info.agency_id
            if event.creation_info.agency_id == agency:
                event_desc = new_event.event_descriptions[origin_index].text
                if event.event_descriptions[0].text == event_desc:
                    matched = True
        new_event.magnitudes.extend(event.magnitudes)
        if not matched:
            new_event.origins.append(event.origins[0])
            new_event.event_descriptions.append(event.event_descriptions[0])
    # Set the useful things like preferred magnitude and preferred origin
    new_event.preferred_origin_id = new_event.origins[0].resource_id
    try:
        # Select moment first, then local, then
        mag_filter = ['MW', 'Mw', 'ML', 'Ml', 'MB', 'Mb',
                      'MS', 'Ms', 'MC', 'Mc']
        _magnitudes = [(m.magnitude_type, m.resource_id)
                       for m in new_event.magnitudes]
        preferred_magnitude = sorted(_magnitudes,
                                     key=lambda x: mag_filter.index(x[0]))[0]
        new_event.preferred_magnitude_id = preferred_magnitude[1]
    except (ValueError, IndexError):
        # If there is a magnitude not specified in filter
        try:
            new_event.preferred_magnitude_id = new_event.magnitudes[0].\
                resource_id
        except IndexError:
            pass
    return new_event

def _read_origin(line):
    """
    Read one origin (type 1) line.

    :param str line: Origin format (type 1) line
    :return: `~obspy.core.event.Event`
    """
    new_event = Event()
    try:
        sfile_seconds = line[16:20].strip()
        if len(sfile_seconds) == 0:
            sfile_seconds = 0.0
        else:
            sfile_seconds = float(sfile_seconds)
        new_event.origins.append(Origin())
        new_event.origins[0].time = UTCDateTime(
            int(line[1:5]), int(line[6:8]), int(line[8:10]),
            int(line[11:13]), int(line[13:15]), 0, 0) + sfile_seconds
    except Exception:
        raise NordicParsingError("Couldn't read a date from sfile")
    # new_event.loc_mod_ind=line[20]
    new_event.event_descriptions.append(EventDescription(text=line[21:23]))
    for key, _slice in [('latitude', slice(23, 30)),
                        ('longitude', slice(30, 38)),
                        ('depth', slice(38, 43))]:
        try:
            new_event.origins[0].__dict__[key] = float(line[_slice])
        except ValueError:
            new_event.origins[0].__dict__[key] = None
    if new_event.origins[0].depth:
        new_event.origins[0].depth *= 1000
    if line[43].strip():
        warnings.warn("Depth indicator {0} has not been mapped "
                      "to the event".format(line[43]))
    if line[44].strip():
        warnings.warn("Origin location indicator {0} has not been mapped "
                      "to the event".format(line[44]))
    new_event.creation_info = CreationInfo(agency_id=line[45:48].strip())
    new_event.origins[0].creation_info = CreationInfo(
        agency_id=line[45:48].strip())
    ksta = Comment(text='Number of stations=' + line[49:51].strip())
    new_event.origins[0].comments.append(ksta)
    if _float_conv(line[51:55]) is not None:
        new_event.origins[0].quality = OriginQuality(
            standard_error=_float_conv(line[51:55]))
    # Read in magnitudes if they are there.
    magnitudes = []
    magnitudes.extend(_read_mags(line, new_event))
    new_event.magnitudes = magnitudes
    return new_event

def _read_mags(line, event):
    """
    Read the magnitude info from a Nordic header line. Convenience function
    """
    magnitudes = []
    for index in [59, 67, 75]:
        if not line[index].isspace():
            magnitudes.append(Magnitude(
                mag=_float_conv(line[index - 3:index]),
                magnitude_type=_nortoevmag(line[index]),
                creation_info=CreationInfo(
                    agency_id=line[index + 1:index + 4].strip()),
                origin_id=event.origins[0].resource_id))
    return magnitudes

def read_spectral_info(sfile, encoding='latin-1'):
    """
    Read spectral info from an sfile.

    :type sfile: str
    :param sfile: Sfile to read from.
    :type encoding: str
    :param encoding: Encoding for file, used to decode from bytes to string

    :returns:
        list of dictionaries of spectral information, units as in seisan
        manual, expect for logs which have been converted to floats.
    """
    with open(sfile, 'r', encoding=encoding) as f:
        tagged_lines = _get_line_tags(f=f)
        spec_inf = _read_spectral_info(tagged_lines=tagged_lines)
    return spec_inf

def _read_spectral_info(tagged_lines, event=None):
    """
    Internal spectral reader.

    :type tagged_lines: dict
    :param tagged_lines: dictionary of tagged lines
    :type event: :class:`~obspy.core.event.Event`
    :param event: Event to associate spectral info with

    :returns:
        list of dictionaries of spectral information, units as in
        seisan manual, expect for logs which have been converted to floats.
    """
    if '3' not in tagged_lines.keys():
        return {}
    if event is None:
        event = _readheader(head_lines=tagged_lines['1'])
    origin_date = UTCDateTime(event.origins[0].time.date)
    relevant_lines = []
    for line in tagged_lines['3']:
        if line[0][1:5] == 'SPEC':
            relevant_lines.append(line)
    relevant_lines = [line[0] for line in
                      sorted(relevant_lines, key=lambda tup: tup[1])]
    spec_inf = {}
    if not relevant_lines:
        return spec_inf
    for line in relevant_lines:
        spec_str = line.strip()
        if spec_str[5:12] in ['AVERAGE', 'STANDARD_DEVIATION']:
            info = {}
            station = spec_str[5:12]
            channel = ''
            info['moment'] = _float_conv(spec_str[16:22])
            if info['moment'] is not None:
                info['moment'] = 10 ** info['moment']
            info['stress_drop'] = _float_conv(spec_str[24:30])
            info['spectral_level'] = _float_conv(spec_str[32:38])
            if info['spectral_level'] is not None:
                info['spectral_level'] = 10 ** info['spectral_level']
            info['corner_freq'] = _float_conv(spec_str[40:46])
            info['source_radius'] = _float_conv(spec_str[47:54])
            info['decay'] = _float_conv(spec_str[56:62])
            info['window_length'] = _float_conv(spec_str[64:70])
            info['moment_mag'] = _float_conv(spec_str[72:78])
            spec_inf[(station, channel)] = info
            continue
        station = spec_str[4:9].strip()
        channel = ''.join(spec_str[9:13].split())
        try:
            info = spec_inf[(station, channel)]
        except KeyError:
            info = {}
        if spec_str[14] == 'T':
            info['starttime'] = origin_date + \
                (int(spec_str[15:17]) * 3600) +\
                (int(spec_str[17:19]) * 60) + int(spec_str[19:21])
            if info['starttime'] < event.origins[0].time:
                # Wrong day, case of origin at end of day
                info['starttime'] += 86400
            info['kappa'] = _float_conv(spec_str[23:30])
            info['distance'] = _float_conv(spec_str[32:38])
            if spec_str[38:40] == 'VS':
                info['velocity'] = _float_conv(spec_str[40:46])
                info['velocity_type'] = 'S'
            elif spec_str[38:40] == 'VP':
                info['velocity'] = _float_conv(spec_str[40:46])
                info['velocity_type'] = 'P'
            else:
                warnings.warn('Only VP and VS spectral information'
                              ' implemented')
            info['density'] = _float_conv(spec_str[48:54])
            info['Q0'] = _float_conv(spec_str[56:62])
            info['QA'] = _float_conv(spec_str[64:70])
        elif spec_str[14] == 'M':
            info['moment'] = _float_conv(spec_str[16:22])
            if info['moment']:
                info['moment'] = 10 ** info['moment']
            info['stress_drop'] = _float_conv(spec_str[24:30])
            info['spectral_level'] = _float_conv(spec_str[32:38])
            if info['spectral_level']:
                info['spectral_level'] = 10 ** info['spectral_level']
            info['corner_freq'] = _float_conv(spec_str[40:46])
            info['source_radius'] = _float_conv(spec_str[47:54])
            info['decay'] = _float_conv(spec_str[56:62])
            info['window_length'] = _float_conv(spec_str[64:70])
            info['moment_mag'] = _float_conv(spec_str[72:78])
        spec_inf[(station, channel)] = info
    return spec_inf

def read_nordic(select_file, return_wavnames=False, encoding='latin-1'):
    """
    Read a catalog of events from a Nordic formatted select file.

    Generates a series of temporary files for each event in the select file.

    :type select_file: str
    :param select_file: Nordic formatted select.out file to open
    :type return_wavnames: bool
    :param return_wavnames:
        If True, will return the names of the waveforms that the events
        are associated with.
    :type encoding: str
    :param encoding: Encoding for file, used to decode from bytes to string

    :return: catalog of events
    :rtype: :class:`~obspy.core.event.event.Event`
    """
    if not hasattr(select_file, "readline"):
        try:
            f = open(select_file, 'r', encoding=encoding)
        except Exception:
            try:
                f = select_file.decode(encoding)
            except Exception:
                f = str(select_file)
    else:
        f = select_file
    wav_names = []
    event_str = []
    catalog = Catalog()
    for line in f:
        if len(line.rstrip()) > 0:
            event_str.append(line)
        elif len(event_str) > 0:
            catalog, wav_names = _extract_event(
                event_str=event_str, catalog=catalog, wav_names=wav_names,
                return_wavnames=return_wavnames)
            event_str = []
    f.close()
    if len(event_str) > 0:
        # May occur if the last line of the file is not blank as it should be
        catalog, wav_names = _extract_event(
            event_str=event_str, catalog=catalog, wav_names=wav_names,
            return_wavnames=return_wavnames)
    if return_wavnames:
        return catalog, wav_names
    for event in catalog:
        event.scope_resource_ids()
    return catalog

def _extract_event(event_str, catalog, wav_names, return_wavnames=False):
    """
    Helper to extract event info from a list of line strings.

    :param event_str: List of lines from sfile
    :type event_str: list of str
    :param catalog: Catalog to append the event to
    :type catalog: `obspy.core.event.Catalog`
    :param wav_names: List of waveform names
    :type wav_names: list
    :param return_wavnames: Whether to extract the waveform name or not.
    :type return_wavnames: bool

    :return: Adds event to catalog and returns. Works in place on catalog.
    """
    tmp_sfile = io.StringIO()
    for event_line in event_str:
        tmp_sfile.write(event_line)
    tagged_lines = _get_line_tags(f=tmp_sfile)
    new_event = _readheader(head_lines=tagged_lines['1'])
    new_event = _read_uncertainty(tagged_lines, new_event)
    new_event = _read_focal_mechanisms(tagged_lines, new_event)
    new_event = _read_moment_tensors(tagged_lines, new_event)
    if return_wavnames:
        wav_names.append(_readwavename(f=tmp_sfile))
    new_event = _read_picks(tagged_lines=tagged_lines, new_event=new_event)
    catalog += new_event
    return catalog, wav_names

def _read_uncertainty(tagged_lines, event):
    """
    Read hyp uncertainty line.
    """
    if 'E' not in tagged_lines.keys():
        return event
    # In principle there shouldn't be more than one error line, but I think
    # there can be - need to associate the correct error
    line = tagged_lines['E'][0][0]
    # TODO: Convert this to ConfidenceEllipsoid
    errors = {'x_err': None}
    try:
        errors = {'x_err': float(line[24:30]), 'y_err': float(line[32:38]),
                  'z_err': float(line[38:43]), 'xy_cov': float(line[43:55]),
                  'xz_cov': float(line[55:67]), 'yz_cov': float(line[67:79])}
    except ValueError:
        pass
    if errors['x_err']:
        warnings.warn("Hypocentral Error estimates found, but not converted.")
    try:
        event.origins[0].quality = OriginQuality(
            azimuthal_gap=int(line[5:8]), standard_error=float(line[14:20]))
    except ValueError:
        pass
    return event

def _read_focal_mechanisms(tagged_lines, event):
    """
    Read focal mechanism info from s-file
    """
    if 'F' not in tagged_lines.keys():
        return event
    UserWarning("Found focal-mechanism info: reading amplitude-ratio fit,"
                "number of bad polarities and number of bad amplitude ratios"
                "is not implemented.")
    for line, line_num in tagged_lines['F']:
        nodal_p = NodalPlane(strike=float(line[0:10]), dip=float(line[10:20]),
                             rake=float(line[20:30]))
        try:
            # Apparently these don't have to be filled.
            nodal_p.strike_errors = QuantityError(float(line[30:35]))
            nodal_p.dip_errors = QuantityError(float(line[35:40]))
            nodal_p.rake_errors = QuantityError(float(line[40:45]))
        except ValueError:
            pass
        fm = FocalMechanism(nodal_planes=NodalPlanes(nodal_plane_1=nodal_p))
        try:
            fm.method_id = ResourceIdentifier(
                "smi:nc.anss.org/focalMehcanism/" + line[70:77].strip())
            fm.creation_info = CreationInfo(agency_id=line[66:69].strip)
            fm.misfit = float(line[45:50])
            fm.station_distribution_ratio = float(line[50:55])
        except ValueError:
            pass
        event.focal_mechanisms.append(fm)
    return event

def _read_moment_tensors(tagged_lines, event):
    if 'M' not in tagged_lines.keys():
        return event
    # Group moment tensor lines together
    mt_lines = sorted(tagged_lines['M'], key=lambda tup: tup[1])
    for mt_ind in range(len(mt_lines) // 2):
        mt_line_1 = mt_lines[mt_ind * 2][0]
        mt_line_2 = mt_lines[(mt_ind * 2) + 1][0]
        if not str(mt_line_2[1:3]) == str('MT'):
            raise NordicParsingError("Matching moment tensor lines not found.")
        sfile_seconds = int(mt_line_1[16:18])
        if sfile_seconds == 60:
            sfile_seconds = 0
            add_seconds = 60
        else:
            add_seconds = 0
        ori_time = UTCDateTime(
            int(mt_line_1[1:5]), int(mt_line_1[6:8]), int(mt_line_1[8:10]),
            int(mt_line_1[11:13]), int(mt_line_1[13:15]), sfile_seconds,
            int(mt_line_1[19:20]) * 100000) + add_seconds
        event.origins.append(Origin(
            time=ori_time, latitude=float(mt_line_1[23:30]),
            longitude=float(mt_line_1[30:38]),
            depth=float(mt_line_1[38:43]) * 1000,
            creation_info=CreationInfo(agency_id=mt_line_1[45:48].strip())))
        event.magnitudes.append(Magnitude(
            mag=float(mt_line_1[55:59]),
            magnitude_type=_nortoevmag(mt_line_1[59]),
            creation_info=CreationInfo(agency_id=mt_line_1[60:63].strip()),
            origin_id=event.origins[-1].resource_id))
        event.focal_mechanisms.append(FocalMechanism(
            moment_tensor=MomentTensor(
                derived_origin_id=event.origins[-1].resource_id,
                moment_magnitude_id=event.magnitudes[-1].resource_id,
                scalar_moment=float(mt_line_2[52:62]), tensor=Tensor(
                    m_rr=float(mt_line_2[3:9]), m_tt=float(mt_line_2[10:16]),
                    m_pp=float(mt_line_2[17:23]), m_rt=float(mt_line_2[24:30]),
                    m_rp=float(mt_line_2[31:37]),
                    m_tp=float(mt_line_2[38:44])),
                method_id=ResourceIdentifier(
                    "smi:nc.anss.org/momentTensor/" + mt_line_1[70:77].strip()
                ))))
    return event

def _read_picks(tagged_lines, new_event):
    """
    Internal pick reader. Use read_nordic instead.

    :type tagged_lines: dict
    :param tagged_lines: Lines keyed by line type
    :type new_event: :class:`~obspy.core.event.event.Event`
    :param new_event: event to associate picks with.

    :returns: :class:`~obspy.core.event.event.Event`
    """
    evtime = new_event.origins[0].time
    pickline = []
    # Set a default, ignored later unless overwritten
    snr = None
    # pick-lines can be tagged by either ' ' or '4'
    tags = [' ', '4']
    for tag in tags:
        try:
            pickline.extend(
                [tup[0] for tup in sorted(
                    tagged_lines[tag], key=lambda tup: tup[1])])
        except KeyError:
            pass
    header = sorted(tagged_lines['7'], key=lambda tup: tup[1])[0][0]
    for line in pickline:
        if line[18:28].strip() == '':  # If line is empty miss it
            continue
        if len(line) < 80:
            line = line.ljust(80)  # Pick-lines without a tag may be short.
        weight = line[14]
        if weight == '_':
            phase = line[10:17]
            weight = 0
            polarity = ''
        else:
            phase = line[10:14].strip()
            polarity = line[16]
            if weight == ' ':
                weight = 0
        polarity_maps = {"": "undecidable", "C": "positive", "D": "negative"}
        try:
            polarity = polarity_maps[polarity]
        except KeyError:
            polarity = "undecidable"
        # It is valid nordic for the origin to be hour 23 and picks to be hour
        # 00 or 24: this signifies a pick over a day boundary.
        if int(line[18:20]) == 0 and evtime.hour == 23:
            day_add = 86400
            pick_hour = 0
        elif int(line[18:20]) == 24:
            day_add = 86400
            pick_hour = 0
        else:
            day_add = 0
            pick_hour = int(line[18:20])
        try:
            time = UTCDateTime(evtime.year, evtime.month, evtime.day,
                               pick_hour, int(line[20:22]),
                               float(line[23:28])) + day_add
        except ValueError:
            time = UTCDateTime(evtime.year, evtime.month, evtime.day,
                               int(line[18:20]), pick_hour,
                               float("0." + line[23:38].split('.')[1])) +\
                60 + day_add
            # Add 60 seconds on to the time, this copes with s-file
            # preference to write seconds in 1-60 rather than 0-59 which
            # datetime objects accept
        if header[57:60] == 'AIN':
            ain = _float_conv(line[57:60])
            warnings.warn('AIN: %s in header, currently unsupported' % ain)
        elif header[57:60] == 'SNR':
            snr = _float_conv(line[57:60])
        else:
            warnings.warn('%s is not currently supported' % header[57:60])
        # finalweight = _int_conv(line[68:70])
        # Create a new obspy.event.Pick class for this pick
        _waveform_id = WaveformStreamID(station_code=line[1:6].strip(),
                                        channel_code=line[6:8].strip(),
                                        network_code='NA')
        pick = Pick(waveform_id=_waveform_id, phase_hint=phase,
                    polarity=polarity, time=time)
        try:
            pick.onset = onsets[line[9]]
        except KeyError:
            pass
        if line[15] == 'A':
            pick.evaluation_mode = 'automatic'
        else:
            pick.evaluation_mode = 'manual'
        # Note these two are not always filled - velocity conversion not yet
        # implemented, needs to be converted from km/s to s/deg
        # if not velocity == 999.0:
            # new_event.picks[pick_index].horizontal_slowness = 1.0 / velocity
        if _float_conv(line[46:51]) is not None:
            pick.backazimuth = _float_conv(line[46:51])
        # Create new obspy.event.Amplitude class which references above Pick
        # only if there is an amplitude picked.
        if _float_conv(line[33:40]) is not None:
            _amplitude = Amplitude(generic_amplitude=_float_conv(line[33:40]),
                                   period=_float_conv(line[41:45]),
                                   pick_id=pick.resource_id,
                                   waveform_id=pick.waveform_id)
            if pick.phase_hint == 'IAML':
                # Amplitude for local magnitude
                _amplitude.type = 'AML'
                # Set to be evaluating a point in the trace
                _amplitude.category = 'point'
                # Default AML unit in seisan is nm (Page 139 of seisan
                # documentation, version 10.0)
                _amplitude.generic_amplitude /= 1e9
                _amplitude.unit = 'm'
                _amplitude.magnitude_hint = 'ML'
            else:
                # Generic amplitude type
                _amplitude.type = 'A'
            if snr:
                _amplitude.snr = snr
            new_event.amplitudes.append(_amplitude)
        elif _int_conv(line[28:33]) is not None:
            # Create an amplitude instance for code duration also
            _amplitude = Amplitude(generic_amplitude=_int_conv(line[28:33]),
                                   pick_id=pick.resource_id,
                                   waveform_id=pick.waveform_id)
            # Amplitude for coda magnitude
            _amplitude.type = 'END'
            # Set to be evaluating a point in the trace
            _amplitude.category = 'duration'
            _amplitude.unit = 's'
            _amplitude.magnitude_hint = 'Mc'
            if snr is not None:
                _amplitude.snr = snr
            new_event.amplitudes.append(_amplitude)
        # Create new obspy.event.Arrival class referencing above Pick
        if _float_conv(line[33:40]) is None:
            arrival = Arrival(phase=pick.phase_hint, pick_id=pick.resource_id)
            if weight is not None:
                arrival.time_weight = weight
            if _int_conv(line[60:63]) is not None:
                arrival.backazimuth_residual = _int_conv(line[60:63])
            if _float_conv(line[63:68]) is not None:
                arrival.time_residual = _float_conv(line[63:68])
            if _float_conv(line[70:75]) is not None:
                arrival.distance = kilometers2degrees(_float_conv(line[70:75]))
            if _int_conv(line[76:79]) is not None:
                arrival.azimuth = _int_conv(line[76:79])
            new_event.origins[0].arrivals.append(arrival)
        new_event.picks.append(pick)
    return new_event

def readwavename(sfile, encoding='latin-1'):
    """
    Extract the waveform filename from the s-file.

    Returns a list of waveform names found in the s-file as multiples can
    be present.

    :type sfile: str
    :param sfile: Path to the sfile
    :type encoding: str
    :param encoding: Encoding for file, used to decode from bytes to string

    :returns: List of strings of wave paths
    :rtype: list
    """
    with open(sfile, 'r', encoding=encoding) as f:
        wavenames = _readwavename(f=f)
    return wavenames

def _readwavename(f):
    """
    Internal wave-name reader.

    :type f: file
    :param f: File open in read-mode

    :return: list of wave-file names
    """
    wavename = []
    for line in f:
        if len(line) == 81 and line[79] == '6':
            wavename.append(line[1:79].strip())
    return wavename

def blanksfile(wavefile, evtype, userid, overwrite=False, evtime=None):
    """
    Generate an empty s-file with a populated header for a given waveform.

    :type wavefile: str
    :param wavefile: Wave-file to associate with this S-file, the timing of \
        the S-file will be taken from this file if evtime is not set.
    :type evtype: str
    :param evtype: Event type letter code, e.g. L, R, D
    :type userid: str
    :param userid: 4-character SEISAN USER ID
    :type overwrite: bool
    :param overwrite: Overwrite an existing S-file, default=False
    :type evtime: :class:`~obspy.core.utcdatetime.UTCDateTime`
    :param evtime: If given this will set the timing of the S-file

    :returns: str, S-file name
    """
    if evtime is None:
        try:
            st = read(wavefile)
            evtime = st[0].stats.starttime
        except Exception:
            raise NordicParsingError('Wavefile: ' + wavefile +
                                     ' is invalid, try again with real data.')
    # Check that user ID is the correct length
    if len(userid) != 4:
        raise NordicParsingError('User ID must be 4 characters long')
    # Check that evtype is one of L,R,D
    if evtype not in ['L', 'R', 'D']:
        raise NordicParsingError('Event type must be either L, R or D')

    # Generate s-file name in the format dd-hhmm-ss[L,R,D].Syyyymm
    sfile = str(evtime.day).zfill(2) + '-' + str(evtime.hour).zfill(2) +\
        str(evtime.minute).zfill(2) + '-' + str(evtime.second).zfill(2) +\
        evtype + '.S' + str(evtime.year) + str(evtime.month).zfill(2)
    # Check is sfile exists
    if os.path.isfile(sfile) and not overwrite:
        warnings.warn('Desired sfile: ' + sfile + ' exists, will not ' +
                      'overwrite')
        for i in range(1, 10):
            sfile = str(evtime.day).zfill(2) + '-' +\
                str(evtime.hour).zfill(2) +\
                str(evtime.minute).zfill(2) + '-' +\
                str(evtime.second + i).zfill(2) + evtype + '.S' +\
                str(evtime.year) + str(evtime.month).zfill(2)
            if not os.path.isfile(sfile):
                break
        else:
            msg = ('Tried generated files up to 20s in advance and found ' +
                   'all exist.')
            raise NordicParsingError(msg)
    with open(sfile, 'w') as f:
        # Write line 1 of s-file
        f.write(' ' + str(evtime.year) + ' ' + str(evtime.month).rjust(2) +
                str(evtime.day).rjust(2) + ' ' +
                str(evtime.hour).rjust(2) +
                str(evtime.minute).rjust(2) + ' ' +
                str(float(evtime.second)).rjust(4) + ' ' +
                evtype + '1'.rjust(58) + '\n')
        # Write line 2 of s-file
        output_time = datetime.datetime.now()
        f.write(' ACTION:ARG ' + str(output_time.year)[2:4] +
                '-' + str(output_time.month).zfill(2) + '-' +
                str(output_time.day).zfill(2) + ' ' +
                str(output_time.hour).zfill(2) + ':' +
                str(output_time.minute).zfill(2) + ' OP:' +
                userid.ljust(4) + ' STATUS:' + 'ID:'.rjust(18) +
                str(evtime.year) + str(evtime.month).zfill(2) +
                str(evtime.day).zfill(2) + str(evtime.hour).zfill(2) +
                str(evtime.minute).zfill(2) + str(evtime.second).zfill(2) +
                'I'.rjust(6) + '\n')
        # Write line 3 of s-file
        write_wavfile = os.path.basename(wavefile)
        f.write(' ' + write_wavfile + '6'.rjust(79 - len(write_wavfile)) +
                '\n')
        # Write final line of s-file
        f.write(" STAT SP IPHASW D HRMM SECON CODA AMPLIT PERI AZIMU" +
                " VELO AIN AR TRES W  DIS CAZ7\n")
    return sfile

def write_select(catalog, filename, userid='OBSP', evtype='L',
                 wavefiles=None):
    """
    Function to write a catalog to a select file in nordic format.

    :type catalog: :class:`~obspy.core.event.event.Catalog`
    :param catalog: A catalog of obspy events
    :type filename: str
    :param filename: Path to write to
    :type userid: str
    :param userid: Up to 4 character user ID
    :type evtype: str
    :param evtype:
        Single character string to describe the event, either L, R or D.
    :type wavefiles: list
    :param wavefiles:
        Waveforms to associate the events with, must be ordered in the same
         way as the events in the catalog.
    """
    if not wavefiles:
        wavefiles = ['DUMMY' for _i in range(len(catalog))]
    with open(filename, 'w') as fout:
        for event, wavfile in zip(catalog, wavefiles):
            select = io.StringIO()
            _write_nordic(event=event, filename=None, userid=userid,
                          evtype=evtype, wavefiles=wavfile,
                          string_io=select)
            select.seek(0)
            for line in select:
                fout.write(line)
            fout.write('\n')

def _write_nordic(event, filename, userid='OBSP', evtype='L', outdir='.',
                  wavefiles='DUMMY', explosion=False,
                  overwrite=True, string_io=None):
    """
    Write an :class:`~obspy.core.event.Event` to a nordic formatted s-file.

    :type event: :class:`~obspy.core.event.event.Event`
    :param event: A single obspy event
    :type filename: str
    :param filename:
        Filename to write to, can be None, and filename will be generated from
        the origin time in nordic format.
    :type userid: str
    :param userid: Up to 4 character user ID
    :type evtype: str
    :param evtype:
        Single character string to describe the event, either L, R or D.
    :type outdir: str
    :param outdir: Path to directory to write to
    :type wavefiles: list
    :param wavefiles: Waveforms to associate the nordic file with
    :type explosion: bool
    :param explosion:
        Note if the event is an explosion, will be marked by an E.
    :type overwrite: bool
    :param overwrite: force to overwrite old files, defaults to False
    :type string_io: io.StringIO
    :param string_io:
        If given, will write to the StringIO object in memory rather than to
        the filename.

    :returns: str: name of nordic file written

    .. note::

        Seisan can find waveforms either by their relative or absolute path, or
        by looking for the file recursively in directories within the WAV
        directory in your seisan install.  Because all lines need to be less
        than 79 characters long (fortran hangover) in the s-files, you will
        need to determine whether the full-path is okay or not.
    """
    # First we need to work out what to call the s-file and open it
    # Check that user ID is the correct length
    if len(userid) != 4:
        raise NordicParsingError('%s User ID must be 4 characters long'
                                 % userid)
    # Check that outdir exists
    if not os.path.isdir(outdir):
        raise NordicParsingError('Out path does not exist, I will not '
                                 'create this: ' + outdir)
    # Check that evtype is one of L,R,D
    if evtype not in ['L', 'R', 'D']:
        raise NordicParsingError('Event type must be either L, R or D')
    if explosion:
        evtype += 'E'
    # Check that there is one event
    if isinstance(event, Catalog) and len(event) == 1:
        event = event[0]
    elif isinstance(event, Event):
        event = event
    else:
        raise NordicParsingError('Needs a single event')
    if not isinstance(wavefiles, list):
        wavefiles = [str(wavefiles)]
    # Determine name from origin time
    try:
        origin = event.preferred_origin() or event.origins[0]
    except IndexError:
        msg = 'Need at least one origin with at least an origin time'
        raise NordicParsingError(msg)
    evtime = origin.time
    if not evtime:
        msg = ('event has an origin, but time is not populated.  ' +
               'This is required!')
        raise NordicParsingError(msg)
    # Attempt to cope with possible pre-existing files
    if not filename:
        range_list = []
        for i in range(30):  # Look +/- 30 seconds around origin time
            range_list.append(i)
            range_list.append(-1 * i)
        range_list = range_list[1:]
        for add_secs in range_list:
            sfilename = (evtime + add_secs).datetime.strftime('%d-%H%M-%S') +\
                evtype[0] + '.S' + (evtime + add_secs).\
                datetime.strftime('%Y%m')
            if not os.path.isfile(os.path.join(outdir, sfilename)):
                sfile_path = os.path.join(outdir, sfilename)
                break
            elif overwrite:
                sfile_path = os.path.join(outdir, sfilename)
                break
        else:
            raise NordicParsingError(os.path.join(outdir, sfilename) +
                                     ' already exists, will not overwrite')
    else:
        sfile_path = os.path.join(outdir, filename)
        sfilename = filename
    # Write the header info.
    if origin.latitude is not None:
        lat = '{0:.3f}'.format(origin.latitude)
    else:
        lat = ''
    if origin.longitude is not None:
        lon = '{0:.3f}'.format(origin.longitude)
    else:
        lon = ''
    if origin.depth is not None:
        depth = '{0:.1f}'.format(origin.depth / 1000.0)
    else:
        depth = ''
    if event.creation_info:
        try:
            agency = event.creation_info.get('agency_id')
            # If there is creation_info this may not raise an error annoyingly
            if agency is None:
                agency = ''
        except AttributeError:
            agency = ''
    else:
        agency = ''
    if len(agency) > 3:
        agency = agency[0:3]
    # Cope with differences in event uncertainty naming
    if origin.quality and origin.quality['standard_error']:
        timerms = '{0:.1f}'.format(origin.quality['standard_error'])
    else:
        timerms = '0.0'
    conv_mags = []
    # Get up to six magnitudes
    mt_ids = []
    if hasattr(event, 'focal_mechanisms'):
        for fm in event.focal_mechanisms:
            if hasattr(fm, 'moment_tensor') and hasattr(
               fm.moment_tensor, 'moment_magnitude_id'):
                mt_ids.append(fm.moment_tensor.moment_magnitude_id)
    for mag_ind in range(6):
        mag_info = {}
        try:
            if event.magnitudes[mag_ind].resource_id in mt_ids:
                raise IndexError("Repeated magnitude")
                # This magnitude will get put in with the moment tensor
            mag_info['mag'] = '{0:.1f}'.format(
                event.magnitudes[mag_ind].mag) or ''
            mag_info['type'] = _evmagtonor(event.magnitudes[mag_ind].
                                           magnitude_type) or ''
            if event.magnitudes[0].creation_info:
                mag_info['agency'] = event.magnitudes[mag_ind].\
                    creation_info.agency_id or ''
            else:
                mag_info['agency'] = ''
        except IndexError:
            mag_info.update({'mag': '', 'type': '', 'agency': ''})
        conv_mags.append(mag_info)
    conv_mags.sort(key=lambda d: d['mag'], reverse=True)
    if conv_mags[3]['mag'] == '':
        conv_mags = conv_mags[0:3]
    # Work out how many stations were used
    if len(event.picks) > 0:
        stations = [pick.waveform_id.station_code for pick in event.picks]
        ksta = str(len(set(stations)))
    else:
        ksta = ''
    if not string_io:
        sfile = open(sfile_path, 'w')
    else:
        sfile = string_io
    sfile.write(
        " {0} {1}{2} {3}{4} {5}.{6} {7}{8}{9}{10}  {11}{12}{13}{14}{15}{16}"
        "{17}{18}{19}{20}{21}{22}1\n".format(
            evtime.year, str(evtime.month).rjust(2), str(evtime.day).rjust(2),
            str(evtime.hour).rjust(2), str(evtime.minute).rjust(2),
            str(evtime.second).rjust(2), str(evtime.microsecond).ljust(1)[0:1],
            evtype.ljust(2), lat.rjust(7), lon.rjust(8), depth.rjust(5),
            agency.rjust(3)[0:3], ksta.rjust(3), timerms.rjust(4),
            conv_mags[0]['mag'].rjust(4), conv_mags[0]['type'].rjust(1),
            conv_mags[0]['agency'][0:3].rjust(3),
            conv_mags[1]['mag'].rjust(4), conv_mags[1]['type'].rjust(1),
            conv_mags[1]['agency'][0:3].rjust(3),
            conv_mags[2]['mag'].rjust(4), conv_mags[2]['type'].rjust(1),
            conv_mags[2]['agency'][0:3].rjust(3)))
    if len(conv_mags) > 3:
        sfile.write(
            " {0} {1}{2} {3}{4} {5}.{6} {7}                      {8}       "
            "{9}{10}{11}{12}{13}{14}{15}{16}{17}1\n".format(
                evtime.year, str(evtime.month).rjust(2),
                str(evtime.day).rjust(2), str(evtime.hour).rjust(2),
                str(evtime.minute).rjust(2), str(evtime.second).rjust(2),
                str(evtime.microsecond).ljust(1)[0:1], evtype.ljust(2),
                agency.rjust(3)[0:3],
                conv_mags[3]['mag'].rjust(4), conv_mags[3]['type'].rjust(1),
                conv_mags[3]['agency'][0:3].rjust(3),
                conv_mags[4]['mag'].rjust(4), conv_mags[4]['type'].rjust(1),
                conv_mags[4]['agency'][0:3].rjust(3),
                conv_mags[5]['mag'].rjust(4), conv_mags[5]['type'].rjust(1),
                conv_mags[5]['agency'][0:3].rjust(3)))
    # Write hyp error line
    try:
        sfile.write(_write_hyp_error_line(origin) + '\n')
    except (NordicParsingError, TypeError):
        pass
    # Write fault plane solution
    if hasattr(event, 'focal_mechanisms') and len(event.focal_mechanisms) > 0:
        for focal_mechanism in event.focal_mechanisms:
            try:
                sfile.write(
                    _write_focal_mechanism_line(focal_mechanism) + '\n')
            except AttributeError:
                pass
        # Write moment tensor solution
        for focal_mechanism in event.focal_mechanisms:
            try:
                sfile.write(
                    _write_moment_tensor_line(focal_mechanism) + '\n')
            except AttributeError:
                pass
    # Write line 2 (type: I) of s-file
    sfile.write(
        " Action:ARG {0} OP:{1} STATUS:               ID:{2}     I\n".format(
            datetime.datetime.now().strftime("%y-%m-%d %H:%M"),
            userid.ljust(4)[0:4], evtime.strftime("%Y%m%d%H%M%S")))
    # Write line-type 6 of s-file
    for wavefile in wavefiles:
        sfile.write(' ' + os.path.basename(wavefile) +
                    '6'.rjust(79 - len(os.path.basename(wavefile))) + '\n')
    # Write final line of s-file
    sfile.write(' STAT SP IPHASW D HRMM SECON CODA AMPLIT PERI AZIMU' +
                ' VELO AIN AR TRES W  DIS CAZ7\n')
    # Now call the populate sfile function
    if len(event.picks) > 0:
        newpicks = '\n'.join(nordpick(event))
        sfile.write(newpicks + '\n')
        sfile.write('\n'.rjust(81))
    if not string_io:
        sfile.close()
        return str(sfilename)
    else:
        return

def _write_moment_tensor_line(focal_mechanism):
    """
    Generate the two lines required for moment tensor solutions in Nordic.
    """
    # First line contains hypocenter info, second contains tensor info
    lines = [list(' ' * 79 + 'M'), list(' ' * 79 + 'M')]
    # Get the origin associated with the moment tensor
    origin = focal_mechanism.moment_tensor.derived_origin_id.\
        get_referred_object()
    magnitude = focal_mechanism.moment_tensor.moment_magnitude_id.\
        get_referred_object()
    # Sort out the first line
    lines[0][1:5] = str(origin.time.year).rjust(4)
    lines[0][6:8] = str(origin.time.month).rjust(2)
    lines[0][8:10] = str(origin.time.day).rjust(2)
    lines[0][11:13] = str(origin.time.hour).rjust(2)
    lines[0][13:15] = str(origin.time.minute).rjust(2)
    lines[0][16:20] = str(origin.time.second).rjust(2) + '.' +\
        str(origin.time.microsecond).ljust(1)[0:1]
    lines[0][23:30] = _str_conv(origin.latitude, 3).rjust(7)
    lines[0][30:38] = _str_conv(origin.longitude, 3).rjust(8)
    lines[0][38:43] = _str_conv(origin.depth / 1000.0, 1).rjust(5)
    if hasattr(origin, 'creation_info') and hasattr(
            origin.creation_info, 'agency_id'):
        lines[0][45:48] = origin.creation_info.agency_id.rjust(3)[0:3]
    lines[0][55:59] = _str_conv(magnitude.mag, 1).rjust(4)
    lines[0][59] = _evmagtonor(magnitude.magnitude_type)
    if hasattr(magnitude, 'creation_info') and hasattr(
            magnitude.creation_info, 'agency_id'):
        lines[0][60:63] = magnitude.creation_info.agency_id.rjust(3)[0:3]
    lines[0][70:77] = (str(
        focal_mechanism.moment_tensor.method_id).split('/')[-1]).rjust(7)
    # Sort out the second line
    lines[1][1:3] = 'MT'
    lines[1][3:9] = _str_conv(
        focal_mechanism.moment_tensor.tensor.m_rr, 3).rjust(6)
    lines[1][10:16] = _str_conv(
        focal_mechanism.moment_tensor.tensor.m_tt, 3).rjust(6)
    lines[1][17:23] = _str_conv(
        focal_mechanism.moment_tensor.tensor.m_pp, 3).rjust(6)
    lines[1][24:30] = _str_conv(
        focal_mechanism.moment_tensor.tensor.m_rt, 3).rjust(6)
    lines[1][31:37] = _str_conv(
        focal_mechanism.moment_tensor.tensor.m_rp, 3).rjust(6)
    lines[1][38:44] = _str_conv(
        focal_mechanism.moment_tensor.tensor.m_tp, 3).rjust(6)
    if hasattr(magnitude, 'creation_info') and hasattr(
            magnitude.creation_info, 'agency_id'):
        lines[1][45:48] = magnitude.creation_info.agency_id.rjust(3)[0:3]
    lines[1][48] = 'S'
    lines[1][52:62] = (
        "%.3e" % focal_mechanism.moment_tensor.scalar_moment).rjust(10)
    lines[1][70:77] = (
        str(focal_mechanism.moment_tensor.method_id).split('/')[-1]).rjust(7)
    return '\n'.join([''.join(line) for line in lines])

def _write_focal_mechanism_line(focal_mechanism):
    """
    Get the line for a focal-mechanism
    """
    nodal_plane = focal_mechanism.nodal_planes.nodal_plane_1
    line = list(' ' * 79 + 'F')
    line[0:10] = (_str_conv(nodal_plane.strike, 1)).rjust(10)
    line[10:20] = (_str_conv(nodal_plane.dip, 1)).rjust(10)
    line[20:30] = (_str_conv(nodal_plane.rake, 1)).rjust(10)
    try:
        line[30:35] = (_str_conv(nodal_plane.strike_errors.uncertainty,
                                 1)).rjust(5)
        line[35:40] = (_str_conv(nodal_plane.dip_errors.uncertainty,
                                 1)).rjust(5)
        line[40:45] = (_str_conv(nodal_plane.rake_errors.uncertainty,
                                 1)).rjust(5)
    except AttributeError:
        pass
    if hasattr(focal_mechanism, 'misfit'):
        line[45:50] = (_str_conv(focal_mechanism.misfit, 1)).rjust(5)
    if hasattr(focal_mechanism, 'station_distribution_ratio'):
        line[50:55] = (_str_conv(
            focal_mechanism.station_distribution_ratio, 1)).rjust(5)
    if hasattr(focal_mechanism, 'creation_info') and hasattr(
            focal_mechanism.creation_info, 'agency_id'):
        line[66:69] = (str(
            focal_mechanism.creation_info.agency_id)).rjust(3)[0:3]
    if hasattr(focal_mechanism, 'method_id'):
        line[70:77] = (str(focal_mechanism.method_id).split('/')[-1]).rjust(7)
    return ''.join(line)

def _write_hyp_error_line(origin):
    """
    Generate hypocentral error line.

    format:
     GAP=126        0.64       1.3     1.7  2.3 -0.9900E+00 -0.4052E+00  \
     0.2392E+00E
    """
    error_line = list(' ' * 79 + 'E')
    if not hasattr(origin, 'quality'):
        raise NordicParsingError("Origin has no quality associated")
    error_line[1:5] = 'GAP='
    error_line[5:8] = str(int(origin.quality['azimuthal_gap'])).ljust(3)
    error_line[14:20] = (_str_conv(
        origin.quality['standard_error'], 2)).rjust(6)
    try:
        errors = origin.origin_uncertainty[
            'confidence_ellipsoid'].confidence_ellipsoid_to_xyz()
    except AttributeError:
        return ''.join(error_line)
    error_line[24:30] = (_str_conv(errors['y_err'] / 1000.0, 1)).rjust(6)
    error_line[32:38] = (_str_conv(errors['x_err'] / 1000.0, 1)).rjust(6)
    error_line[38:43] = (_str_conv(errors['z_err'] / 1000.0, 1)).rjust(5)
    error_line[43:55] = ("%.4e" % (errors['xy_cov'] / 1E06)).rjust(12)
    error_line[55:67] = ("%.4e" % (errors['xz_cov'] / 1E06)).rjust(12)
    error_line[67:79] = ("%.4e" % (errors['yz_cov'] / 1E06)).rjust(12)
    return ''.join(error_line)

def nordpick(event):
    """
    Format picks in an :class:`~obspy.core.event.event.Event` to nordic.

    :type event: :class:`~obspy.core.event.event.Event`
    :param event: A single obspy event.

    :returns: List of String

    .. note::

        Currently finalweight is unsupported, nor is velocity, or
        angle of incidence.  This is because
        :class:`~obspy.core.event.event.Event` stores slowness
        in s/deg and takeoff angle, which would require computation
        from the values stored in seisan.  Multiple weights are also
        not supported.
    """
    pick_strings = []
    for pick in event.picks:
        if not pick.waveform_id:
            msg = ('No waveform id for pick at time %s, skipping' % pick.time)
            warnings.warn(msg)
            continue
        # Convert string to short sting
        if pick.onset == 'impulsive':
            impulsivity = 'I'
        elif pick.onset == 'emergent':
            impulsivity = 'E'
        else:
            impulsivity = ' '

        # Convert string to short string
        if pick.polarity == 'positive':
            polarity = 'C'
        elif pick.polarity == 'negative':
            polarity = 'D'
        else:
            polarity = ' '
        # Extract velocity: Note that horizontal slowness in quakeML is stored
        # as s/deg
        if pick.horizontal_slowness is not None:
            # velocity = 1.0 / pick.horizontal_slowness
            velocity = ' '  # Currently this conversion is unsupported.
        else:
            velocity = ' '
        # Extract azimuth
        if pick.backazimuth is not None:
            azimuth = pick.backazimuth
        else:
            azimuth = ' '
        # Extract the correct arrival info for this pick - assuming only one
        # arrival per pick...
        try:
            origin = event.preferred_origin() or event.origins[0]
            arrival = [arrival for arrival in origin.arrivals
                       if arrival.pick_id == pick.resource_id]
        except IndexError:
            arrival = []
        if len(arrival) > 0:
            arrival = arrival[0]
            # Extract weight - should be stored as 0-4, or 9 for seisan.
            if arrival.time_weight is not None:
                weight = int(arrival.time_weight)
            else:
                weight = '0'
            # Extract azimuth residual
            if arrival.backazimuth_residual is not None:
                azimuthres = int(arrival.backazimuth_residual)
            else:
                azimuthres = ' '
            # Extract time residual
            if arrival.time_residual is not None:
                timeres = arrival.time_residual
            else:
                timeres = ' '
            # Extract distance
            if arrival.distance is not None:
                distance = degrees2kilometers(arrival.distance)
                if distance >= 100.0:
                    distance = str(_int_conv(distance))
                elif 10.0 < distance < 100.0:
                    distance = _str_conv(round(distance, 1), 1)
                elif distance < 10.0:
                    distance = _str_conv(round(distance, 2), 2)
                else:
                    distance = _str_conv(distance, False)
            else:
                distance = ' '
            # Extract CAZ
            if arrival.azimuth is not None:
                caz = int(arrival.azimuth)
            else:
                caz = ' '
        else:
            caz = ' '
            distance = ' '
            timeres = ' '
            azimuthres = ' '
            azimuth = ' '
            weight = 0
        if not pick.phase_hint:
            # Cope with some authorities not providing phase hints :(
            phase_hint = ' '
        else:
            phase_hint = pick.phase_hint
        # Extract amplitude: note there can be multiple amplitudes, but they
        # should be associated with different picks.
        amplitude = [amplitude for amplitude in event.amplitudes
                     if amplitude.pick_id == pick.resource_id]
        if len(amplitude) > 0:
            if len(amplitude) > 1:
                msg = 'Nordic files need one pick for each amplitude, ' + \
                      'using the first amplitude only'
                warnings.warn(msg)
            amplitude = amplitude[0]
            # Determine type of amplitude
            if amplitude.type != 'END':
                # Extract period
                if amplitude.period is not None:
                    peri = amplitude.period
                    if peri < 10.0:
                        peri_round = 2
                    elif peri >= 10.0:
                        peri_round = 1
                    else:
                        peri_round = False
                else:
                    peri = ' '
                    peri_round = False
                # Extract amplitude and convert units
                if amplitude.generic_amplitude is not None:
                    amp = amplitude.generic_amplitude
                    if amplitude.unit in ['m', 'm/s', 'm/(s*s)', 'm*s']:
                        amp *= 1e9
                    # Otherwise we will assume that the amplitude is in counts
                else:
                    amp = None
                coda = ' '
                mag_hint = (amplitude.magnitude_hint or amplitude.type)
                if mag_hint is not None and mag_hint.upper() in ['AML', 'ML']:
                    phase_hint = 'IAML'
                    impulsivity = ' '
            else:
                coda = int(amplitude.generic_amplitude)
                peri = ' '
                peri_round = False
                amp = None
        else:
            peri = ' '
            peri_round = False
            amp = None
            coda = ' '
        # If the weight is 0 then we don't need to print it
        if weight == 0 or weight == '0':
            weight = None  # this will return an empty string using _str_conv
        elif weight is not None:
            weight = int(weight)
        if pick.evaluation_mode == "automatic":
            eval_mode = "A"
        elif pick.evaluation_mode == "manual":
            eval_mode = " "
        else:
            warnings.warn("Evaluation mode %s is not supported"
                          % pick.evaluation_mode)
            eval_mode = " "
        # Generate a print string and attach it to the list
        channel_code = pick.waveform_id.channel_code or '   '
        pick_strings.append(
            " {0}{1}{2} {3}{4}{5}{6}{7} {8}{9}{10}.{11}{12}"
            "{13}{14}{15}{16}    {17}{18}  {19}{20} ".format(
                pick.waveform_id.station_code.ljust(5), channel_code[0],
                channel_code[-1], impulsivity, phase_hint.ljust(4),
                _str_conv(weight).rjust(1), eval_mode, polarity.rjust(1),
                str(pick.time.hour).rjust(2), str(pick.time.minute).rjust(2),
                str(pick.time.second).rjust(3),
                str(float(pick.time.microsecond) /
                    (10 ** 4)).split('.')[0].zfill(2),
                _str_conv(coda).rjust(5)[0:5],
                _str_conv(amp, rounded=1).rjust(7)[0:7],
                _str_conv(peri, rounded=peri_round).rjust(5),
                _str_conv(azimuth).rjust(6), _str_conv(velocity).rjust(5),
                _str_conv(azimuthres).rjust(3),
                _str_conv(timeres, rounded=2).rjust(5)[0:5], distance.rjust(5),
                _str_conv(caz).rjust(4)))
        # Note that currently finalweight is unsupported, nor is velocity, or
        # angle of incidence.  This is because obspy.event stores slowness in
        # s/deg and takeoff angle, which would require computation from the
        # values stored in seisan.  Multiple weights are also not supported in
        # Obspy.event
    return pick_strings

if __name__ == "__main__":
    import doctest
    doctest.testmod()


#!/bin/env python

import dockergc

if __name__ == "__main__":
    #Instantiate the GetArgs() class
    cli_args = dockergc.GetArgs()
    remove_image_ret, remove_containers_ret, branch_version = cli_args.getargs()

    #Instantiate Git
    if branch_version[0] != 'None':
        gitc = dockergc.Git(branch_version[0], '</PATH/TO/BRANCH>')
        gitc.git_clone()

    #Manage containers
    if remove_containers_ret[0] != 'None':
        container = dockergc.Container(remove_containers_ret[0], '/PATH/TO/containers.docker')
        container.fetch_containers()
        container.docker_remove_containers()

    #Manage images
    if remove_image_ret[0] != 'None':
        image = dockergc.Image(remove_image_ret[0], '/PATH/TO/images.docker')
        image.fetch_images()
        image.docker_remove_images()


#--------------------------------------------------------------------------------
# Name:      astar solver.py
#
# Purpose:   This .py file contains all required functions except the function that prints
#            the solution. This can be find in the puzzle.py file.

#   legal_moves function: creates a list with all the moves 0 is allowed to do given its
#   coordinates in the form (row,column)
#
#   generate_child function: it generates children for every node, where node is a state and
#   ech child is a possible next state.
#
#   find_path function: return a list with the steps.
#
#   manhattan function: return the heuristic of a state using manhattan distance
#
#   f function: state-evaluation function
#
#   astar function: astar search algorithm.
#
#Required libs: heapq (should not need installation)
#
#Author:    Ioannis Mastoras
#
#Created:   4 April 2020
#
#-----------------------------------------------------------------------------------

import heapq

#------USER INPUT--------------------
n = int(input("Enter size of tile puzzle (integer greater that 0): "))

state = []  #store the state as a list
print("\n Enter start state row by row (numbers delimited by white space): \n")

for i in range(0,n):
    s = list(map(int,input("Enter Start state: row %a : " % str(i+1)).strip().split()))[:n**2]  #strip() add a comma, and split() deletes the space
    for k in s:
        state.append(k) #adds everynumber in the list
print("\n Enter goal state row by row (numbers delimited by white space): \n")
goal_state = []
for i in range(0,n):
    s = list(map(int,input("Enter Goal state: row %a : " % str(i+1)).strip().split()))[:n**2]
    for k in s:
        goal_state.append(k)
#########################################################

######--returns a list with legal moves of 0 for a given state--####
def legal_moves(row,col):

    legal_action = ['Down', 'Left', 'Up','Right'] #list with all possible moves

    if row == 0:  # up is not not allowed at the top row
        legal_action.remove('Up')
    elif row == n-1:  # down is not allowed at the bottom row
        legal_action.remove('Down')
    if col == 0:      #Left is not allowed at the leftost column
        legal_action.remove('Left')
    elif col == n-1:  #Right is not allowed at the rightmost column
        legal_action.remove('Right')
    return legal_action #list with moves 0 can make
#########################################################

####--retuns the children, i.e. next states, for a given node/state, based on the legal moves---##
def generate_child(node,g):

    children = []  #list of children

    x = node.index(0)  #index of zero in the list, i.e. where is the zero.
                       #Note that the NxN matrix has been converted to 1D matix, i.e. a list

    i = int(x / n)  #row coordinate
    j = int(x % n)   #column coordinate

    legal_actions=legal_moves(i,j) #calls the function legal_moves

    for action in legal_actions:

        new_state = node.copy()     #loop always starts with the curent state, and then find the child

        if action == 'Down':
            new_state[x], new_state[x+n] = new_state[x+n], new_state[x]

        elif action == 'Left':
            new_state[x], new_state[x-1] = new_state[x-1], new_state[x]
                                                                            #for each move in the list, it changes the index of 0 accordingly
        elif action == 'Up':
            new_state[x], new_state[x-n] = new_state[x-n], new_state[x]

        elif action == 'Right':
            new_state[x], new_state[x+1] = new_state[x+1], new_state[x]

        children.append([f(g,manhattan(new_state)),new_state,action]) #puts in the chidren list the child, the action that resulted to that state, and the cost

    return children #a list of children, i.e. next states
#########################################################

##--returns the heuristic for a state using manhattan distance--##
def manhattan(mstate):

    manhattan = 0

    for i in mstate: #for tile in the board

        if i != 0: #we need only calculate it for n-1 tiles, so I exlcluded 0
            t = mstate.index(i) #index of the tile in the current state
            s = goal_state.index(i) #index of the tile in the goal state

            drow = abs(int(t/n)-int(s/n)) #difference in row
            dcol = abs(int(t%n)-int(s%n)) #difference in columns

            manhattan += drow + dcol #add the difs. to get the manhattan distance

    return manhattan #the heuristic of the state
#########################################################

##--State-Evaluation Function--##
def f(g,h):
    f = g+h #g is the distance for the current state back to start and h is the heuristic/manhattan
    return f
#########################################################

##--A*--##
def astar(initialState,goalState):

    openList = [] #list of states, their cost, and step to creat them
    closedList = [] #list of states with lowest cost

    openList.append([0,initialState,''])

    g = 1 #g is the distance for the current state back to start
    while openList:

        heapq.heapify(openList) #prioritizes the open list according to the cost of the states
                                #from lowest to highest

        node = heapq.heappop(openList) #chooses the swallowest node in frontier; the one at the front
        closedList.append(node) #puts node in the closedList since it has the smallest cost value

        if node[1] == goalState: #test if the node is the goal node

            return find_path(closedList) #calls find_path function to return the path
            break #stops the loop

        ch = generate_child(node[1],g) #calls the generate_child function to return a list of children of the node

        for child in ch: #child has the form [cost,child node],'action']

            if child not in openList: #puts all chilren on the openlist
                openList.append(child)
        g += 1

    return
#########################################################

###--returns a list with the moves 0 needs to make for the puzzle to be solved
def find_path(lst): #lst is a list with all nodes, actions, and cost

#it has the form [ [0,[start node],None],...[cost,[node],'action'],..[cost,[goal node],'action']]

    steps = []
    for i in range(1,len(lst)): #exclude initial state
        steps.append(lst[i][2]) #put all steps in a list
    return steps #return the list

###################################################
#
#End of solver.py
#You shoud run puzzle.py to see the output
#
###################################################


from mkmsdk.mkm import mkm

MTG_ID = 1
EN_ID = 1
FR_ID = 2
DE_ID = 3

def get_prices(card_name, lang):
	result = []
	for x in mkm.market_place.products(name=card_name, game=MTG_ID, language=lang, match="false").json()["product"]:
		if (x['category']['idCategory'] == 1):
			result.append({'expansion': x['expansion'], 'number': x['number'], 'priceGuide': x['priceGuide']})
	return result

if __name__ == '__main__':
	for x in get_prices('Hazoret', 1):
		print(x)
	print()
	for x in get_prices('Kitchen Fin', 1):
		print(x)


from pyspark.sql import SparkSession
from pyspark.sql.functions import avg, col, count, when

# Inicializar sessão do Spark
spark = SparkSession.builder.appName("LargeDataAnalysis").getOrCreate()

# Carregar dados em um DataFrame do Spark
df = spark.read.csv("large_data.csv", header=True, inferSchema=True)

# Adicionar uma nova coluna ao DataFrame com base em uma condição
df = df.withColumn("new_column", when(col("column_name") > 5, 1).otherwise(0))

# Agrupar dados por uma coluna e calcular a média de outra coluna
grouped = df.groupby("group_column_name").agg(avg(col("column_to_average")), count("*"))

# Filtrar linhas com base em uma condição
filtered = grouped.filter(col("avg(column_to_average)") > 10)

# Mostrar as primeiras linhas do DataFrame filtrado
filtered.show(10)

# Parar a sessão do Spark
spark.stop()


#!/usr/bin/python3

import matplotlib.pyplot as plt
import matplotlib as mpl
import simulation.plotting as pt
import matplotlib.gridspec as gridspec
import simulation.fio as io
import simulation.measures as ms
import numpy as np
import h5py

from matplotlib import animation

font = {'size':10, 'weight' : 'normal'}
mpl.rcParams['mathtext.fontset'] = 'stix'
mpl.rcParams['font.family'] = 'STIXGeneral'
mpl.rc('font',**font)

# First set up the figure, the axis, and the plot element we want to animate

# initialization function: plot the background of each frame
def init():
        grd.set_data([[],[]])
        return grd,

# animation function.  This is called sequentially
def animate(i):
        page = grid_list[i]
        grd.set_data(page)
        return grd,

def make_V_name(V):
    if V in ['X', 'Y', 'Z']:
        name = '\sigma^{}'.format(V.lower())
    else:
        name = V
    if len(name.split('_')) == 2:
        name = '('.join(name.split('_'))+'^\circ)'
    else:
        name = '('.join(name.split('_'))
    return name

def make_mode_name(mode):
    if mode is 'sweep':
        name = '\mathrm{SWP}'
    elif mode is 'alt':
        name = '\mathrm{ALT}'
    elif mode is 'block':
        name = '\mathrm{BLK}'
    else:
        name =''
    return name

def make_U_name(mode, S, V):
    S = str(S)
    name = r'$U^{%s}_{%s}(%s)$' % (make_mode_name(mode), S, make_V_name(V))

    return name

output_dir = 'Hphase'
#data_repo = '/mnt/ext0/qca_output/'+output_dir+'/data/'
data_repo = None

deg_list = range(0, 185, 5)
fixed_params_dict = {
            'output_dir' : [output_dir],
            'L' : [21],
            'T' : [60],
            'IC': ['f0'],
            'BC': ['1_00'],
            'mode': ['alt'],
            'S' : [6]
             }

var_params_dict = {
            'V' : ['HP_' + str(deg) for deg in deg_list]
             }

params_list_list = io.make_params_list_list(fixed_params_dict, var_params_dict)

def plot_grid(grid, ax,  span=[0,60], n_xticks = 4, n_yticks = 6):
    im = ax.imshow( grid,
                    origin = 'lower',
                    cmap=plt.cm.jet,
                    vmin = np.mean(grid) - 1/2*np.std(grid),
                    vmax = np.mean(grid) + np.std(grid),
                    interpolation = 'none',
                    aspect = '1',
                    rasterized = True)

    x_tick_labels = range(len(grid[0]))
    y_tick_labels = range(*span)

    ylabel = 'Iteration'
    xlabel = 'Site'
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    delta = max(1, int(len(x_tick_labels)/n_xticks))
    ax.set_xticks(range(0, len(x_tick_labels), delta ))
    ax.set_xticklabels(x_tick_labels[::delta])

    delta = max(1, int(len(y_tick_labels)/n_yticks))
    ax.set_yticks(range(0, len(y_tick_labels), delta ))
    ax.set_yticklabels(y_tick_labels[::delta])

    #box = ax.get_position()
    #cax = plt.axes([box.x1-0.1, box.y0+0.1, 0.04, box.height - 0.19])
    #cb = plt.colorbar(im, cax = cax, ticks = [-1.0, -0.5, 0.0, 0.5, 1.0])
    #cb.ax.tick_params(labelsize=12)
    #cb.set_label(r'$\langle \sigma_j^z \rangle$', rotation=0, labelpad = -22,
    #        y=1.12)

    return im

grid_list = []
title_list = []

n_xticks = 4
n_yticks = 5
t_span=[0,1000]
slider = 100
dt=1
span = [0, dt]
for params_list in params_list_list:
    for params in params_list:
        output_dir = params['output_dir']
        mode = params['mode']
        S = params['S']
        V = params['V']
        T = params['T']
        L = params['L']
        IC = params['IC']
        title = make_U_name(mode, S, V)
        if data_repo is not None:
            sname = io.sim_name(params)
            res_path = data_repo + sname + '_v0.hdf5'
        else:
            res_path = io.default_file_name(params, 'data', '.hdf5')
        res = h5py.File(res_path)
        exp = ms.get_diag_vecs(res['zz'][::])
        exp = 0.5*(1-exp)
        s = res['s'][::]
        M = res['m']
        g2 = res['gxx'][::]
        grid = exp
        grid_list.append(grid)
        title_list.append(title)

fig = plt.figure(figsize=(2,3))
ax = fig.add_subplot(111)
im=plot_grid(grid_list[0], ax, span=span)
title = ax.set_title(title_list[0], fontsize=10)
fig.subplots_adjust(bottom=0.2)

def init():
    im.set_data(grid_list[0])
    title.set_text(title_list[0])
    return (im, title)

# animation function.  This is called sequentially
def animate(i):
    a=im.get_array()
    a=grid_list[i]
    im.set_array(a)
    im.set_clim(vmin=np.min(a), vmax=np.max(a))
    title.set_text(title_list[i])
    return (im, title)

anim = animation.FuncAnimation(fig, animate, init_func=init,
                                       frames=len(grid_list), interval=150)

bn = io.base_name(output_dir, 'plots')
anim.save(bn + 'L21_S6_alt_phase.avi', fps=8,
        dpi=500)

#plt.show()


# encoding=utf-8
class ShortInputException(Exception):
    """A user-defined exception class."""

    def __init__(self, length, at_least):
        Exception.__init__(self)
        self.length = length
        self.at_least = at_least

try:
    text = input('Plz input something:')
    if len(text) < 3:
        raise ShortInputException(len(text), 3)
    # Other work could go on here
    # ...
except EOFError:
    print('\nThis is a EOFError')
except KeyboardInterrupt:
    print('\nThis is a KeyboardInterrupt')
except ShortInputException as sie:
    print('ShortInputException: Your input consists only {} characters, and it is too short, plz enter at least {} characters.'.format(sie.length, sie.at_least))
else:
    print('Your input is', text)


################################################################################################
# rhml_runner
#
# This will coordinate a set of model creation and test runs based on passed in config
###############################################################################################
from os import system
import sys
import configparser
import math
import random
import numpy as np
from typing import List
from sklearn.model_selection import train_test_split
from sklearn.model_selection import ParameterGrid
import rhml_dataloader,rhml_reporter,rhml_configreader
from RHML.DTrees import ClassificationTree as CTree
from RHML.DTrees import RegressionTree as RTree
from RHML.Ensembles.RandomForest import ClassificationRandomForest
from RHML.Ensembles.RandomForest import RegressionRandomForest

from RHML.Ensembles.Baggers import ClassificationBagger,RegressionBagger
from RHML.DTrees.ClassificationTree import RHMLClassificationDecisionTree
from RHML.DTrees.RegressionTree import RHMLRegressionDecisionTree
from RHML.Ensembles.Boosters import AdaBooster
from RHML.Ensembles.Boosters import RegressionGradBooster

# Some public statics for runtypes supported
RUN_SINGLE  = 0
RUN_GRID    = 1
RUN_MULTI   = 2

#****************************************************************************************
# run
# Main entry point . Load required data and initiate the appropriate run type
# ***************************************************************************************
def run(configReader,runType):
    # Load the data we want to use :
    dataName = rhml_configreader.findDataToLoad(configReader)[0][0]
    featuresToSkip = rhml_configreader.findFeaturesToSkip(configReader)
    theData,theLabels,theFeatures = rhml_dataloader.loadDataSet(dataName,featuresToSkip)

    # What type of problem are we dealing with : classification or regression -  This is driven the dataset selected
    problemType = rhml_dataloader.getDataConfig(dataName,rhml_dataloader.DATA_PROBLEM_TYPE)

    # Set random see so get similar results to compare - can be set in config
    random.seed(getRandomStateSeed(configReader))

    if runType==RUN_GRID:
        runGridSearch(dataName,featuresToSkip,theData,theLabels,theFeatures,problemType,configReader)
    elif runType==RUN_MULTI:
        runMultiModel(dataName,featuresToSkip,theData,theLabels,theFeatures,problemType,configReader)
    else:
        runSingleSplit(dataName,featuresToSkip,theData,theLabels,theFeatures,problemType,configReader)

#****************************************************************************************
# runSingleSplit
# Supports : one data source, multiple models, (no moving paramters)
# Creates : one report per model
# ***************************************************************************************
def runSingleSplit(dataName,featuresToSkip,theData,theLabels,theFeatures,problemType,configReader):
    # data split for training and test
    X_train, X_test, y_train, y_test = train_test_split(theData, theLabels, test_size=0.33, random_state=getRandomStateSeed(configReader),shuffle=True)

    # get list of models we want to run
    modelsToRun = rhml_configreader.findWhichModelsToRun(configReader)

    reportTitleRoot= "Title : "

    # get title from config if we have one set
    reportTitleRoot = rhml_configreader.getTitleFromConfig(configReader)
    reportDesc = rhml_configreader.getDescriptionFromConfig(configReader)

    for model in modelsToRun:

        # get model specific params here : combine with defaults too
        modelParamsString = model[0].upper()+"_PARAMS"
        modelKwargs = rhml_configreader.getModelKwargs(configReader,modelParamsString)

        # create the appropraite model with provided and default params
        theModel = createModel(problemType,model[0],X_train,y_train,modelKwargs)

        # create a report for this model - this gets training and test based results
        # note - this will open it in a browser asa get the report
        if theModel is not None:

            modelSummary,featureImp,training_results,testing_results,modelOOBResults = runTests(theModel,X_train, X_test, y_train, y_test)

            # Some things report more things .... eg. proximity matrix for RF and Bags ...
            proximityMatrix= calcProximityMatrix(theModel,X_train)

            reportTitle = reportTitleRoot+" - "+theModel.getModelType()
            rhml_reporter.buildReport(  dataName,
                                        problemType,
                                        reportTitle,
                                        reportDesc,
                                        theFeatures,
                                        featuresToSkip,
                                        modelKwargs,
                                        modelSummary,
                                        featureImp,
                                        training_results,
                                        testing_results,
                                        proximityMatrix,
                                        y_train,
                                        modelOOBResults)

#****************************************************************************************
# runGridSearch
# Supports : one model type, but one or more hyperparams
#   Uses cross-validation to find the best setting for those hyperparams.
#   The config needs to give a set of values  e.g[1,2,3] for a param that will get tested here
# Creates : a single report
# ***************************************************************************************
def runGridSearch(dataName,featuresToSkip,theData,theLabels,theFeatures,problemType,configReader):

    # For grid search we need to get some specific config things from config file

    # Find the model to use (inside gridsearch config section)and check we only have one model to run for grid search!
    modelsToRun = rhml_configreader.findWhichGridModelsToRun(configReader)

    if len(modelsToRun)==0:
        sys.exit(f"There are no models listed in the grid search config section")

    if len(modelsToRun)>1:
        sys.exit(f"There are too many models listed in the grid search config section {modelsToRun} : Grid Search only supports one model at a time.")

    modelCode = (modelsToRun[0][0]).lower()
    print(f"Starting Grid Search for model {modelCode}")

    # get grid specific params: merge grid and standadrd settings for given model
    modelParamsString = modelCode.upper()+"_PARAMS"
    modelKwargs = rhml_configreader.getModelKwargs(configReader,modelParamsString)
    modelKwargs =rhml_configreader.adjustKwargsForGridSearch(configReader,modelKwargs)

    # Since we use these for reporting ensure kwargs carry defaults at this point too
    amendForDefaults(problemType,modelCode,modelKwargs)
    print(f"kwargs set to: {modelKwargs}")

    # Process the settings that have some 'ranges'
    searchParamGrid = {}
    for key,value in modelKwargs.items():
        if isinstance(value,List):
            print(f"We have a range for : {key} and it looks like this  {value}")
            searchParamGrid[key]=value

    numberOfGridSearchParams = len(searchParamGrid)
    if numberOfGridSearchParams==0:
        sys.exit("For a grid search there needs to be at least one ranged hyperparamter")

    # Now calc all the pos combos of params we need to test in the grid search :
    grid = ParameterGrid(searchParamGrid)

    # Split out data - use training data for CV, and eventually test on test only!
    X_train, X_test, y_train, y_test = train_test_split(theData, theLabels, test_size=0.33, random_state=getRandomStateSeed(configReader),shuffle=True)

    grid_results = []
    # We spin for as many combos in our grid ....
    for index, gridItem in enumerate(grid):
        # The gridItem gives us the settings we need to change to our modelKwargs for this run
        runParams = modelKwargs.copy()   # start with x's keys and values
        runParams.update(gridItem)
        print(f"index:{index}: This run will use the following kwargs {runParams}")

        # start each step in the grid at the same place so compare like with like
        random.seed(getRandomStateSeed(configReader))

        cross_validation_folds = 5
        # split the data into cross_validation_folds : this just creates the index of the data in each fold!
        foldDataIndexes = createCVFoldDataIndexes(X_train.shape[0],cross_validation_folds)

        crossValScores = []
        for iteration in range(cross_validation_folds):
            trainingIndexs, testIndexes = getTrainTestIndexesForCVIteration(iteration, foldDataIndexes)
            cv_trainingData = X_train[trainingIndexs,:]
            cv_testData = X_train[testIndexes,:]
            cv_trainingLabels = y_train[trainingIndexs]
            cv_testLabels = y_train[testIndexes]

            # create the appropraite model with provided and default params
            theModel = createModel(problemType,modelCode,cv_trainingData,cv_trainingLabels,runParams)
            if theModel is not None:
                test_results = theModel.test(cv_testData,cv_testLabels)

                # stash the score (which score to use?) in a 'results' table, with the key is the index of the ParamterGrid being used here!
                score = test_results.score[0]
                crossValScores.append(score)

        # now add the average across all cross validation runs as the final score for this combo of params
        grid_results.append(np.mean(crossValScores))

    print(f"The finals scores are  {grid_results}, sizee: {len(grid_results)} max : {max(grid_results)}")
    # want to know the index of the max, so we can get the settings that are best
    best_index = grid_results.index(max(grid_results))
    print(f"The best is at index = {best_index}")
    print(f"Settings to use are : {grid[best_index]}")

    # Report and plots .....
    reportTitleRoot = "Grid Search Report - "
    # get title from config if we have one set
    reportDesc = rhml_configreader.getDescriptionFromConfig(configReader)
    configTitle = rhml_configreader.getTitleFromConfig(configReader)
    if configTitle !="":
        reportTitleRoot = configTitle+" - "+reportTitleRoot

    reportTitle = reportTitleRoot+theModel.getModelType()
    rhml_reporter.buildGridSearchReport( dataName,featuresToSkip,reportTitle,reportDesc,searchParamGrid,grid,grid_results,best_index,modelKwargs)

#****************************************************************************************
# runMultiModel : Targetting commpare multiple models
# Supports : one dataset, multiple models with a single moving parameter
#   Only one moving paramter allowed at once
#   One or more models allowed
#   Train a model based on base params and the moving parameter, and output results based on train/test data
# Creates : a single report
# ***************************************************************************************
def runMultiModel(dataName,featuresToSkip,theData,theLabels,theFeatures,problemType,configReader):
    # For multi model we need to get some specific config things from config file

    # Find the models to use (inside multi model config section)
    modelsToRun = rhml_configreader.findWhichMultiModelsToRun(configReader)
    numberOfModels = len(modelsToRun)

    if numberOfModels==0:
        sys.exit(f"There are no models listed in the grid search config section")

    modelKwargs = rhml_configreader.getMultiModelKwargs(configReader)
    # Need pay special attention to which config params are 'moving' i.e have a list of values ...
    movingParameters,movingParamaterValues = getListParams(modelKwargs)

    if movingParamaterValues is None or len(movingParamaterValues)==0:
        sys.exit(f"There are no ranged paramters in the multi-model section")

    numberParameterValues = len(movingParamaterValues[0])

    numberLists = len(movingParameters)
    if numberLists==0:
        sys.exit("For a multi model report there needs to be at a ranged hyperparamter e.g config with [1,2,3] format")
    elif numberLists > 1:
        sys.exit("For a multi model report there can only be one hyperparamter that has a range/list value e.g config with [1,2,3] format")

    # split the data in train / test
    X_train, X_test, y_train, y_test = train_test_split(theData, theLabels, test_size=0.33, random_state=getRandomStateSeed(configReader),shuffle=True)

    # set up some empty cache to  store the results
    training_results_cache = np.zeros((numberOfModels,numberParameterValues))
    testing_results_cache = np.zeros((numberOfModels,numberParameterValues))
    oob_results_cache = np.zeros((numberOfModels,numberParameterValues))
    models_run=[]
    model_kwargs=[] #cache the actual kwargs used so we can report on them later

    # loop for each model
    for i,model in enumerate(modelsToRun):

        # start each step in the grid at the same place so compare like with like
        random.seed(getRandomStateSeed(configReader))

        # get raw params together for the model type
        modelCode = (model[0]).lower()
        models_run.append(modelCode)
        print(f"Starting Multi Model for model {modelCode} - model index is {i}")

        # get grid specific params: merge multi and standadrd settings for given model
        modelParamsString = modelCode.upper()+"_PARAMS"
        modelKwargs = rhml_configreader.getModelKwargs(configReader,modelParamsString)

        # renove the param we will override, as just want to record the 'other' params for reports
        adjusted_kwargs = dict(modelKwargs) #make deep copy
        adjusted_kwargs.pop(movingParameters[0],None)
        model_kwargs.append(adjusted_kwargs)

        print(f"kwargs is set to: {modelKwargs}")

        # now loop over the moving paramter values, and replace where needed
        for p,param_value in enumerate(movingParamaterValues[0]):
            # Update kwargs to use the current value of the moving param
            modelKwargs[movingParameters[0]]=param_value

            theModel = createModel(problemType,modelCode,X_train,y_train,modelKwargs)
            if theModel is not None:
                train_results = theModel.test(X_train,y_train)
                test_results = theModel.test(X_test,y_test)

                # stash the score (which score to use?) in a 'results' table, with the key is the index of the ParamterGrid being used here!
                train_score = train_results.score[0]
                test_score = test_results.score[0]
                training_results_cache[i,p]=train_score
                testing_results_cache[i,p]=test_score

                # may have OOB results for this model too : if supported and if switched on
                modelOOBResults = getattr(theModel,"OOBScore",None)
                if modelOOBResults is not None:
                    oob_results_cache[i,p]=modelOOBResults[0]
                else:
                    oob_results_cache[i,p]=-1

    print(f"Results are in ..... ")
    print(f"Training results : {training_results_cache}")
    print(f"Testing results : {testing_results_cache}")

    # now plot it : note this would be in a report at some point instead but just plot it for now
    # plotMultiModel(models_run,movingParameters[0],movingParamaterValues[0],training_results_cache,testing_results_cache)

    reportTitle = "Multi Model Report - "
    reportDesc = rhml_configreader.getDescriptionFromConfig(configReader)
    configTitle = rhml_configreader.getTitleFromConfig(configReader)
    if configTitle !="":
        reportTitle = reportTitle+" "+configTitle
    rhml_reporter.buildMultiModelReport( dataName,featuresToSkip,reportTitle,reportDesc,models_run,model_kwargs,movingParameters[0],movingParamaterValues[0],training_results_cache,testing_results_cache,oob_results_cache)
#****************************************************************************************#****************************************************************************************

#****************************************************************************************
# Supporting functions .....
#****************************************************************************************
# Depending on config, create the required model to run the job
def createModel(problemType, modelCode, data, labels,modelKwargs):
    if problemType == rhml_dataloader.CLASSIFICATION:
        return createClassificationModel(modelCode, data, labels,modelKwargs)
    else:
        return createRegressionModel(modelCode, data, labels,modelKwargs)

def createClassificationModel(modelCode, data, labels,modelKwargs):

    # All models except DT need 'number of trees' set , so check here
    numberTrees=modelKwargs.get(rhml_configreader.MODEL_NUM_TREES,0)
    if numberTrees == 0 and modelCode != rhml_configreader.MODEL_DTREE:
        print(f"MISSING CONFIG : Required config missing for model {modelCode} : {rhml_configreader.MODEL_NUM_TREES} must be > 0")
        return None

    if modelCode == rhml_configreader.MODEL_DTREE:
        model = CTree.RHMLClassificationDecisionTree(data,labels,**modelKwargs)
        return model

    elif modelCode == rhml_configreader.MODEL_BAG:
        model = ClassificationBagger.RHMLClassificationBagger(RHMLClassificationDecisionTree,numberTrees,data,labels,**modelKwargs)
        return model

    elif modelCode == rhml_configreader.MODEL_RF:
        # check the must have params : could do this as validation step before run at all?
        numberFeatures=modelKwargs.get(rhml_configreader.MODEL_RF_NUM_FEATURES,0)
        if numberFeatures == 0:
            print(f"Required config missing for model {modelCode} : {rhml_configreader.MODEL_RF_NUM_FEATURES} must be > 0")
            return None
        myRandomForest = ClassificationRandomForest.RHMLClassificationRandomForest(numberFeatures,numberTrees,data,labels,**modelKwargs)
        return myRandomForest

    elif modelCode == rhml_configreader.MODEL_BOOST:
        if not modelKwargs.get('max_depth'):
            modelKwargs['max_depth'] = AdaBooster.RHMLAdaBooster.DEFAULT_MAX_DEPTH
        model = AdaBooster.RHMLAdaBooster(numberTrees,data,labels,**modelKwargs)
        return model

    return None

def createRegressionModel(modelCode, data, labels,modelKwargs):

    # All models except DT need 'number of trees' set , so check here
    numberTrees=modelKwargs.get(rhml_configreader.MODEL_NUM_TREES,0)
    if numberTrees == 0 and modelCode != rhml_configreader.MODEL_DTREE:
        print(f"MISSING CONFIG : Required config missing for model {modelCode} : {rhml_configreader.MODEL_NUM_TREES} must be > 0")
        return None

    if modelCode == rhml_configreader.MODEL_DTREE:
        model = RTree.RHMLRegressionDecisionTree(data,labels,**modelKwargs)
        return model

    elif modelCode == rhml_configreader.MODEL_BAG:
        model = RegressionBagger.RHMLRegressionBagger(RHMLRegressionDecisionTree,numberTrees,data,labels,**modelKwargs)
        return model

    elif modelCode == rhml_configreader.MODEL_RF:
        # check the must have params
        numberFeatures=modelKwargs.get(rhml_configreader.MODEL_RF_NUM_FEATURES,0)
        if numberFeatures == 0:
            print(f"Required config missing for model {modelCode} : {rhml_configreader.MODEL_RF_NUM_FEATURES} must be > 0")
            return None
        myRandomForest = RegressionRandomForest.RHMLRegressionRandomForest(numberFeatures,numberTrees,data,labels,**modelKwargs)
        return myRandomForest

    elif modelCode == rhml_configreader.MODEL_BOOST:
        boostedModel = RegressionGradBooster.RHMLRegressionGradBooster(numberTrees,data,labels,**modelKwargs) #{"min_split":25,"max_depth":6}
        return boostedModel
    return None

# Assumes we have got data split into train and test :
def runTests(theModel,X_train, X_test, y_train, y_test):

    modelSummary = theModel.getSummary()

    # get feature importance
    featureImp = theModel.calcFeatureImportance()

    # run the training data through :
    training_results = theModel.test(X_train,y_train)

    # run through the test data:
    testing_results = theModel.test(X_test,y_test)

    # OOB results may be available too ... if not set it to None
    modelOOBResults = getattr(theModel,"OOBScore",None)

    return modelSummary,featureImp,training_results,testing_results,modelOOBResults

# Enforce any required defaults here
def amendForDefaults(problemType,modelCode,modelKwargs):
    if problemType == rhml_dataloader.CLASSIFICATION:
        if modelCode == rhml_configreader.MODEL_BOOST:
            if not modelKwargs.get('max_depth'):
                modelKwargs['max_depth'] = AdaBooster.RHMLAdaBooster.DEFAULT_MAX_DEPTH

# If the underlying model supports the calcProximity method, call it and return results
def calcProximityMatrix(theModel,X_train):
    proximityMatrix = None
    modelProximityMatrixMethod = getattr(theModel,"calcProximity",None)
    if modelProximityMatrixMethod is not None:
        print("Model supports proximity")
        proximityMatrix = theModel.calcProximity(X_train)
    else:
        print("Model DOES NOT support Proximity")

    return proximityMatrix

# For cross validation, create a set of folds each of which contain some random indexs
def createCVFoldDataIndexes(numberOfDataItems,numberOfFolds):
    # we want some random indexs in each of the folds, but only one index in each fold
    numberPerFold = int(math.ceil(numberOfDataItems/numberOfFolds))
    allIndexes = list(range(numberOfDataItems))
    random.shuffle(allIndexes)

    allFolds=[]
    for i in range(numberOfFolds):
        theFold = allIndexes[(i*numberPerFold):(((i+1)*numberPerFold))-1]
        allFolds.append(theFold)

    return allFolds

# For a given cross validation iteration, work out the indexs to be used for training and for test
# for example, for iteration 3, you would keep the 3rd one for test and merge the others for training
def getTrainTestIndexesForCVIteration(iteration, folds):
    mergedTraining =[]
    for i,fold in enumerate(folds):
        if i != iteration:
            mergedTraining.extend(fold)

    return mergedTraining,folds[iteration]

# getListParams
# Find paramaters that have a list form ie use sqaure brqaces [ ]
# This returns the key names in one array and the list of values in a separate array
def getListParams(modelKwargs):
    paramNames = []
    paramValues = []
    for key,value in modelKwargs.items():
        if isinstance(value,List):
            print(f"We have a range for : {key} and it looks like this  {value}")
            paramNames.append(key)
            paramValues.append(value)
    return paramNames,paramValues

# Get a seed value - uses default but can be set in config too
def getRandomStateSeed(configReader):
    random_state_seed=101
    config_seed = rhml_configreader.getRandomSeedFromConfig(configReader)
    if config_seed != "":
        random_state_seed = int(config_seed)
    return random_state_seed


# -*- coding: utf-8 -*-
"""
Get relevant references and, if url is a scopus link, clear it.
"""
import pandas as pd
import bibtexparser

filepath = "relevant_references.bib"
with open(filepath, 'r', encoding='utf-8') as f:
    bib = bibtexparser.load(f)

df = pd.DataFrame.from_dict(bib.entries)
for i, url in enumerate(df.loc[:, ('url')]):
    try:
        if 'scopus' in url:
            df.loc[:, ('url')][i] = ""

    except TypeError:
        next

with open('relevant_references_clean.bib', 'w') as bibtex_file:
    bibtexparser.dump(bibtex_database, bibtex_file)


"""https://www.hackerrank.com/challenges/2d-array/problem?h_l=interview&playlist_slugs%5B%5D=interview-preparation-kit&playlist_slugs%5B%5D=arrays

Complete the function hourglassSum in the editor below. It should return an integer, the maximum hourglass sum in the array.
"""

def hourglassSum(arr):
    """given the 6x6 array extract all the hourglasses avalible and retrieve the max
    """
    # special case: empty
    if not arr or not arr[0]: return
    # special case: not hourglass possible
    if len(arr) < 3 or len(arr[0]) < 3: return
    # general case
    hourglasses =[]
    for i in range(len(arr)-2):
        for j in range(len(arr[0])-2):
            up = [arr[i][j],arr[i][j+1],arr[i][j+2]]
            middle = [arr[i+1][j+1]]
            down = [arr[i+2][j],arr[i+2][j+1],arr[i+2][j+2]]
            hourglasses.append(sum(up + middle + down))
    return max(hourglasses)

# test
test = [[1, 1, 1, 0, 0, 0],
        [0, 1, 0, 0, 0, 0],
        [1, 1, 1, 0, 0, 0],
        [0, 0, 2, 4, 4, 0],
        [0, 0, 0, 2, 0, 0],
        [0, 0, 1, 2, 4, 0]]
print(f'input: ')
for line in test: print(line)
print(f'ans: {hourglassSum(test)}')