import pandas as pd
import os
import pickle
import numpy as np
from collections.abc import Iterable
from scipy.stats import norm
from .base import _BaseSumStats, _H5SSConnection
from .utils import _MultiWindowPlot, _recompute_maf
from .plot import (manhattan, qqplot, afplot, zzplot, pzplot)
from .exceptions import sumstatswarn

pd.options.mode.chained_assignment = None

class SumStats(_BaseSumStats):
    """Class for summary statistics of a single GWAS.

    :param path: Path to the file containing summary statistics. Should be a csv, or tab-delimited txt-file (.gz supported).
    :type path: str
    :param phenotype: Phenotype name
    :type phenotype: str
    :param gwas_n: Optional N subjects in the GWAS, for N-based meta analysis (if there is no N column in the summary statistics)
    :type gwas_n: int
    :param column_names: Optional dictionary of column names, if these are not automatically recognised. Keys should be: ['rsid', 'chr', 'bp', 'ea', 'oa', 'maf', 'b', 'se', 'p', 'hwe', 'info', 'n', 'eaf', 'oaf']
    :type column_names: dict
    :param data: Dataset for the new SumStats object, in general, don't specify this.
    :type data: dict
    :param low_ram: Whether to use the low_ram option for this SumStats object. Use this only when running into MemoryErrors. Enabling this option will read/write data from local storage rather then RAM. It will save lots of RAM, but it will significantly decrease processing speed.
    :type low_ram: bool
    :param tmpdir: Which directory to store the temporary files if low_ram is enabled.
    :type tmpdir: str
    :param kwargs: other keyword arguments to be passed to pandas.read_csv() method

    """

    def __init__(self, path, phenotype=None, gwas_n=None, column_names=None, data=None, low_ram=False,
                 tmpdir='sumstats_temporary', **kwargs):
        assert isinstance(path, str) or (path is None),  "path should be str"
        assert isinstance(phenotype, str) or (phenotype is None), "phenotype should be str"
        assert isinstance(gwas_n, int) or gwas_n is None, "gwas_n should be int"
        assert isinstance(low_ram, bool), "low_ram should be True or False"
        assert isinstance(tmpdir, str), "tmpdir should be str"
        super().__init__()
        self.low_ram = low_ram
        self.tmpdir = tmpdir
        self.plot_funcs = {'manhattan': self.manhattan, 'qqplot': self.qqplot, 'pzplot': self.pzplot}
        if phenotype is None:
            self.phenotype_name = path.split('/')[-1].split('.')[0]
        else:
            self.phenotype_name = phenotype
        self.column_names = column_names
        if path is not None:
            if path.endswith('.pickle'):
                with open(path, 'rb') as f:
                    sumstatsobj = pickle.load(f)
                if sumstatsobj.low_ram:
                    if not os.path.isfile(sumstatsobj.data.path):
                        raise ImportError(
                            'The data that was stored in {} using low_ram does not exist anymore.'.format(path))
                if self.__version__ != sumstatsobj.__version__:
                    if self.__version__.split('.')[0] != sumstatsobj.__version__.split('.')[0]:
                        pass  # What to do in case of major version differences: No problem yet.
                    else:
                        sumstatswarn('Imported sumstats version {} does not match current version {}'.format(
                            sumstatsobj.__version__, self.__version__))
                for k, v in {m: getattr(sumstatsobj, m) for m in dir(sumstatsobj) if
                             (not m.startswith('__'))}.items():
                    setattr(self, k, v)
                self.__version__ = sumstatsobj.__version__
            else:
                if 'sep' not in kwargs.keys():
                    if path.endswith('.txt') or path.endswith('.txt.gz') or path.endswith('.tsv') or path.endswith('.tsv.gz'):
                            self.data = pd.read_csv(path, sep='\t', **kwargs)
                    elif path.endswith('.csv') or path.endswith('.csv.gz'):
                        self.data = pd.read_csv(path, **kwargs)
                    else:
                        raise ImportError('Only .txt(.gz), .tsv(.gz) or .csv(.gz) files allowed when \'sep\' is not specified.')
                else:
                    self.data = pd.read_csv(path, **kwargs)
            self.gwas_n = gwas_n

            self._sync_columns()
            self._split()
        else:
            self.phenotype_name = phenotype
            self.data = data
            self.reset_index()
        self.qc_result = {}
        self.columns = self.data[1].columns

    def _sync_columns(self):
        """Internal function to synchronize column names and types with the defaults.

        :return: None

        """
        self.data.columns = [x.lower() for x in self.data.columns]
        if (len(self.data.columns)) != len(list(set(self.data.columns))):
            raise KeyError('Duplicate column names not allowed! (not case-sensitive)')
        required = ['rsid', 'chr', 'bp', 'ea', 'oa', 'maf', 'b', 'se', 'p']
        column_name_variants = {
            'rsid': ['rsid', 'rs', 'rsnumber'],
            'chr': ['chr', 'chromosome'],
            'bp': ['bp', 'pos'],
            'ea': ['a1', 'ea', 'effectallele', 'allele1', 'ref', 'refallele'],
            'oa': ['a2', 'oa', 'otherallele', 'allele0', 'allele2', 'alt', 'altallele'],
            'maf': ['maf', 'minorallelefrequency', 'minfreq'],
            'b': ['b', 'beta', 'effect'],
            'se': ['se', 'stderr'],
            'p': ['p', 'pval', 'pvalue', 'p_bolt_lmm_inf', 'p-value'],
            'hwe': ['p_hwe', 'hwe', 'hwe_p'],
            'info': ['info', 'rsq', 'r2'],
            'n': ['n', 'nchrobs', 'samplesize', 'totalsamplesize']
        }
        column_name_variants_af = {
            'eaf': ['a1freq', 'a1frq', 'a1f', 'eaf', 'freq1'],
            'oaf': ['a2freq', 'a2frq', 'a2f', 'oaf', 'freq2'],
        }
        column_types = {'rsid': str, 'chr': np.uint8, 'bp': np.uint32, 'ea': str, 'oa': str, 'maf': np.float32,
                        'b': np.float64, 'se': np.float64, 'p': np.float64, 'hwe': np.float32, 'info': np.float32,
                        'n': np.uint32, 'eaf': np.float32, 'oaf': np.float32}
        if self.column_names is not None:
            for k, v in self.column_names:
                if (k not in column_name_variants) and (k not in column_name_variants_af):
                    raise KeyError('{} not recognized as column type'.format(k))
                elif k in column_name_variants:
                    column_name_variants[k] = [v]
                elif k in column_name_variants_af:
                    column_name_variants_af[k] = [v]
        found, found_af, not_found = [], [], []
        for target, options in column_name_variants.items():
            for opt in options:
                if opt in self.data.columns:
                    self.data.rename(columns={opt: target}, inplace=True)
                    found += [target]
                    break
            if (target == 'rsid') and ('rsid' not in found):
                for opt in ['snpname', 'snp', 'snpid', 'cptid', 'markername']:
                    if opt in self.data.columns:
                        self.data.rename(columns={opt: target}, inplace=True)
                        found.append(target)
                        sumstatswarn(
                            'Assuming column {} represents rsid as no other rsid column was found.'.format(opt))
                        break
        if self.data['rsid'].isnull().values.any():
            sumstatswarn('Missing RSIDs were replaced with CHR:BP')
            midx = self.data['rsid'].isnull()
            self.data.loc[midx, 'rsid'] = self.data.loc[midx, 'chr'].map(str) + ':' + self.data.loc[midx, 'bp'].map(str)
        for target, options in column_name_variants_af.items():
            for opt in options:
                if opt in self.data.columns:
                    self.data.rename(columns={opt: target}, inplace=True)
                    found_af.append(target)
        if 'maf' not in found:
            if len(found_af) == 0:
                raise KeyError('No allele frequency column found')
            if ('eaf' in found_af) and ('oaf' not in found_af):
                self.data['oaf'] = 1 - self.data['eaf']
            elif ('eaf' not in found_af) and ('oaf' in found_af):
                self.data['eaf'] = 1 - self.data['oaf']
            self.data['maf'] = self.data[['eaf', 'oaf']].min(axis=1)
            found.append('maf')
        for target in required:
            if target not in found:
                not_found.append(target)
        if 'n' not in self.data.columns:
            if self.gwas_n is None:
                sumstatswarn('No N found or specified, samplesize based meta analysis not possible')
            else:
                self.data['n'] = self.gwas_n
        else:
            if np.any(self.data['n'].isna()):
                nnna = np.sum(self.data['n'].isna())
                sumstatswarn('{} missing values in sample size column were mean imputed'.format(nnna))
                self.data.loc[self.data['n'].isna(), 'n'] = self.data['n'].mean()
        if np.any(self.data['bp'].isna()):
            naidx = np.where(self.data['bp'].isna())
            self.data.loc[naidx[0], 'bp'] = (self.data.loc[naidx[0] - 1, 'bp'] + 1).tolist()
            if np.any(self.data['bp'].isna()):
                raise ValueError('Too many missing values in basepair column')
            else:
                sumstatswarn('{} missing values in basepair column were imputed to be (bp at the previous row)+1'.format(len(naidx)))
        if np.any(self.data['chr'].isna()):
            naidx = np.where(self.data['chr'].isna())
            self.data.loc[naidx[0], 'chr'] = (self.data.loc[naidx[0] - 1, 'chr']).tolist()
            if np.any(self.data['chr'].isna()):
                raise ValueError('Too many missing values in chromosome column')
            else:
                sumstatswarn('{} missing values in chromosome column were imputed to be the same as chromsome of the previous row'.format(len(naidx)))
        try:
            self.data['chr'] = self.data['chr'].astype(int)
        except ValueError:
            self.data['chr'] = self.data['chr'].str.replace('X', '23')
            try:
                self.data['chr'] = self.data['chr'].astype(int)
            except ValueError:
                raise ValueError('Could not convert chromosome column to integer')
        if len(not_found) > 0:
            raise KeyError('Could not find columns: {}'.format(', '.join(not_found)))
        self.data['ea'] = self.data['ea'].str.upper()
        self.data['oa'] = self.data['oa'].str.upper()
        self.variables = list(self.data.columns)
        for column, targettype in column_types.items():
            if column in self.data.columns:
                self.data[column].astype(targettype, copy=False)

    def _split(self):
        """ Internal function to split the initial large dataset into chunks based on chromosome

        :return: None

        """
        if not self.low_ram:
            new_data = {}
        else:
            new_data = _H5SSConnection(self.phenotype_name, tmpdir=self.tmpdir)
        for c in range(1, 24):
            new_data[c] = self.data.loc[self.data['chr'] == c, :].copy()
            if len(new_data[c]) == 0:
                sumstatswarn('No data found for chromosome {}'.format(c))
        self.data = new_data

    def qc(self, maf=None, hwe=None, info=None, **kwargs):
        """Basic GWAS quality control function.

        :param maf: Minor allele frequency cutoff, will drop SNPs where MAF < cutoff. Default: 0.1
        :type maf: float or None
        :param hwe: Hardy-Weinberg Equilibrium cutoff, will drop SNPs where HWE < cutoff, if specified and HWE column is present in the data.
        :type hwe: float or None
        :param info: Imputation quality cutoff, will drop SNPs where Info < cutoff, if specified and Info column is present in the data.
        :type info: float or None
        :param kwargs: Other columns to filter on, keyword should be column name, SNPs whill be dropped where the value < argument.
        :return: None

        """
        assert isinstance(maf, float) or (maf is None), "maf should be float"
        assert isinstance(hwe, float) or (hwe is None), "hwe should be float"
        assert isinstance(info, float) or (info is None), "info should be float"
        qc_vals = dict(maf=.01)
        qc_info = dict(org_len=0, maf=0, new_len=0)
        if maf is not None:
            qc_vals['maf'] = maf
        if hwe is not None:
            if 'hwe' not in self.variables:
                sumstatswarn('HWE qc cutoff specified but HWE column was not found, skipping QC step.')
            else:
                qc_vals['hwe'] = hwe
                qc_info['hwe'] = 0
        if info is not None:
            if 'info' not in self.variables:
                sumstatswarn('Info qc cutoff specified but info column was not found, skipping QC step.')
            else:
                qc_vals['info'] = info
                qc_info['info'] = 0
        for k, v in kwargs.items():
            if k not in self.variables:
                raise KeyError('Column {} not found.'.format(k))
            else:
                qc_vals[k] = v
                qc_info[k] = 0
        for c in range(1, 24):
            data = self.data[c]
            qc_info['org_len'] += len(data)
            for var, cutoff in qc_vals.items():
                qc_info[var] += len(data.loc[data[var] >= cutoff, :])
            for var, cutoff in qc_vals.items():
                data = data.loc[data[var] >= cutoff, :].copy()
            qc_info['new_len'] += len(data)
            self.data[c] = data
        self.qc_result = qc_info

    def merge(self, other, how='inner', low_memory=False):
        """Merge with other SumStats object(s).

        :param other: Other sumstats object, or list of other SumStats objects.
        :type other: :class:`pysumstats.plot.SumStats` or list
        :param how: Type of merge.
        :type how: str
        :param low_memory: Enable to use a more RAM-efficient merging method (WARNING: still untested)
        :type low_memory: bool
        :return: :class:`pysumstats.plot.MergedSumStats` object

        """
        assert isinstance(other, SumStats) or isinstance(other, list), "other should be SumStats or list"
        if isinstance(other, list):
            for o in other:
                assert isinstance(o, SumStats), "items in other should be SumStats"
            merged = self.merge(other[0], how=how, low_memory=low_memory)
            merged.merge(other[1:], how=how, inplace=True, low_memory=low_memory)
            return merged
        else:
            if self.phenotype_name == other.phenotype_name:
                sumstatswarn('Phenotype names were equal, converting to {0}_x and {0}_y'.format(self.phenotype_name))
                self.phenotype_name = self.phenotype_name + '_x'
                other.phenotype_name = other.phenotype_name + '_y'
            merge_info = dict(xlen=0, ylen=0, newlen=0)
            joined = [x for x in self.variables if x in other.variables]
            for x in [x for x in self.variables if x not in joined]:
                sumstatswarn(
                    'Could not find column {} in pysumstats for {}, column dropped'.format(x, other.phenotype_name))
            for x in [x for x in other.variables if x not in joined]:
                sumstatswarn(
                    'Could not find column {} in pysumstats for {}, column dropped'.format(x, self.phenotype_name))
            if not self.low_ram:
                merged_data = {}
            else:
                merged_data = _H5SSConnection(
                    '{}{}{}{}{}'.format(self.phenotype_name[0], self.phenotype_name[-1], other.phenotype_name[0],
                                        other.phenotype_name[1], len(os.listdir('sumstats_temporary'))),
                    tmpdir=self.tmpdir)
            if 'n' not in self.columns:
                sumstatswarn('No sample size column found for {}, using 1 for calculation of overall MAF'.format(self.phenotype_name))
            if 'n' not in other.columns:
                sumstatswarn('No sample size column found for {}, using 1 for calculation of overall MAF'.format(other.phenotype_name))
            if low_memory and (not how == 'inner'):
                raise NotImplementedError('low_memory only allows for an inner merge.')
            for c in range(1, 24):
                data_x = self.data[c].copy()
                merge_info['xlen'] += len(data_x)
                data_y = other.data[c].copy()
                merge_info['ylen'] += len(data_y)
                data_m_x = data_x.rename(columns={k: '{}_{}'.format(k, self.phenotype_name) for k in data_x.columns if
                                                  k not in ['rsid', 'chr', 'bp']})
                data_m_y = data_y.rename(columns={k: '{}_{}'.format(k, other.phenotype_name) for k in data_y.columns if
                                                  k not in ['rsid', 'chr', 'bp']})
                if low_memory:
                    if np.any(data_m_x.duplicated(subset='rsid')) or np.any(data_m_y.duplicated(subset='rsid')):
                        raise KeyError('Duplicated rsids in either dataset will cause low_memory merge to fail')
                    merged_data[c] = data_m_x.loc[data_m_x['rsid'].isin(data_m_y['rsid'].tolist()), :]
                    data_m_y = data_m_y.loc[data_m_y['rsid'].isin(data_m_x['rsid'].tolist()), :]
                    merged_data[c].sort_values(by='rsid', inplace=True)
                    data_m_y.sort_values(by='rsid', inplace=True)
                    merged_data[c].reset_index(inplace=True, drop=True)
                    data_m_y.reset_index(inplace=True, drop=True)
                    data_m_y = data_m_y[
                        [x + '_' + other.phenotype_name for x in joined if x not in ['rsid', 'chr', 'bp']]]
                    for x in data_m_y.columns:
                        merged_data[c][x] = data_m_y[x]
                        data_m_y.drop(axis=1, labels=[x], inplace=True)
                    merged_data[c].sort_values(by='bp', inplace=True)
                else:
                    joined_x = ['rsid', 'chr', 'bp'] + [x + '_' + self.phenotype_name for x in joined if
                                                        x not in ['rsid', 'chr', 'bp']]
                    joined_y = ['rsid', 'chr_y', 'bp_y'] + [x + '_' + other.phenotype_name for x in joined if
                                           x not in ['rsid', 'chr', 'bp']]
                    data_m_y = data_m_y.rename(columns={'chr': 'chr_y', 'bp': 'bp_y'})
                    merged_data[c] = data_m_x[joined_x].merge(data_m_y[joined_y], on='rsid', how=how)
                    merged_data[c].loc[merged_data[c]['chr'].isna(), 'chr'] = merged_data[c].loc[merged_data[c]['chr'].isna(), 'chr_y']
                    merged_data[c].loc[merged_data[c]['bp'].isna(), 'bp'] = merged_data[c].loc[
                        merged_data[c]['bp'].isna(), 'bp_y']
                    merged_data[c] = merged_data[c].drop(axis=1, labels=['chr_y', 'bp_y'])
                merged_data[c]['maf'] = _recompute_maf(merged_data[c], [self.phenotype_name, other.phenotype_name])
                merge_info['newlen'] += len(merged_data[c])
            return MergedSumStats(data=merged_data, phenotypes=[self.phenotype_name, other.phenotype_name],
                                  merge_info=merge_info, variables=joined,
                                  xy=[self.phenotype_name, other.phenotype_name], low_ram=self.low_ram,
                                  tmpdir=self.tmpdir)

    def describe(self, columns=None, per_chromosome=False):
        """Get a summary of the data.

        :param columns: List of column names to print summary for (default: ['b', 'se', 'p'])
        :type columns: list.
        :param per_chromosome: Enable to return a list of summary dataframes per chromosome
        :type per_chromosome: bool.
        :return: pd.Dataframe, or list

        """
        assert isinstance(columns, list) or (columns is None), "columns should be a list"
        assert isinstance(per_chromosome, bool), "per_chromosome should be True or False"
        if columns is None:
            columns = ['b', 'se', 'p']
        if (not isinstance(columns, list)) and isinstance(columns, str):
            columns = [columns]
        sum_cols = []
        for c in columns:
            if c in self.data[1].columns:
                sum_cols.append(c)
            else:
                sumstatswarn('{} not found in columns, skipping.'.format(c))
        return self._get_summary(sum_cols, per_chromosome)

    def manhattan(self, **kwargs):
        """Generate a manhattan plot using this sumstats data

        :param kwargs: keyworded arguments to be passed to :func:`pysumstats.plot.manhattan`
        :return: None, or (fig, ax)

        """
        manhattan(self[['rsid', 'chr', 'bp', 'p']], **kwargs)

    def qqplot(self, **kwargs):
        """Generate a QQ-plot using this sumstats data

        :param kwargs: keyworded arguments to be passed to :func:`pysumstats.plot.qqplot`
        :return: None, or (fig, ax)

        """
        qqplot(self['p'].values, **kwargs)

    def pzplot(self, **kwargs):
        """Generate a PZ-plot using this sumstats data

        :param kwargs: keyworded arguments to be passed to :func:`pysumstats.plot.pzplot`
        :return: None, or (fig, ax)

        """
        pzplot(self[['b', 'se', 'p']], **kwargs)

class MergedSumStats(_BaseSumStats):
    """Class containing merged summary statistics. In general you will not create a MergedSumStats object manually.

    :param data: dataset containing merged summary statistics
    :type data: dict
    :param phenotypes: list of phenotype names.
    :type phenotypes: list
    :param merge_info: Dict with information on the merge
    :type merge_info: dict
    :param variables: list of variables contained in the data.
    :type variables: list
    :param xy: x and y suffixes (to be used in _allign)
    :type xy: list
    :param low_ram: Whether to use the low_ram option for this MergedSumStats object (passed down from SumStats). Use this only when running into MemoryErrors. Enabling this option will read/write data from local storage rather then RAM. It will save lots of RAM, but it will gratly decrease processing speed.
    :type low_ram: bool
    :param tmpdir: Which directory to store the temporary files if low_ram is enabled (passed down from SumStats).
    :type tmpdir: str
    :param allign: Enable to auto-allign SNPs
    :type allign: bool

    """

    def __init__(self, data, phenotypes, merge_info, variables, xy, low_ram=False, tmpdir='sumstats_temporary',
                 allign=True):
        super().__init__()
        self.pheno_names = phenotypes
        self.data = data
        self.info = merge_info
        self.variables = variables
        self.suffixes = xy
        self.columns = self.data[1].columns
        if allign:
            self._allign()
        self.reset_index()
        self.phenotype_name = None
        self.low_ram = low_ram
        self.tmpdir = tmpdir
        self.plot_funcs = {'manhattan': self.manhattan, 'qqplot': self.qqplot, 'pzplot': self.pzplot,
                           'afplot': self.afplot, 'zzplot': self.zzplot}

    def _allign(self, ynames=None):
        """Function to allign SNPs to the first phenotype.

        :param ynames: Optional argument of multiple phenotypes that should be alligned.
        :type ynames: list
        :return: None

        """
        if self.suffixes[0] is None:
            suffix_x = ''
        else:
            suffix_x = '_' + self.suffixes[0]
        if self.suffixes[1] is None:
            suffix_y = ''
        else:
            suffix_y = '_' + self.suffixes[1]
        eax, oax = 'ea{}'.format(suffix_x), 'oa{}'.format(suffix_x)
        if ynames is None:
            eay, oay, bys = 'ea{}'.format(suffix_y), 'oa{}'.format(suffix_y), ['b{}'.format(suffix_y)]
            eafys, oafys = ['eaf{}'.format(suffix_y)], ['oaf{}'.format(suffix_y)]
        else:
            eay, oay, bys = 'ea{}'.format(suffix_y), 'oa{}'.format(suffix_y), ['b{}'.format(y) for y in ynames]
            eafys, oafys = ['eaf{}'.format(y) for y in ynames], ['oaf{}'.format(y) for y in ynames]
        dropped = 0
        for c in self.data.keys():
            data = self.data[c].copy()
            data.loc[data[eax].isna(), eax] = data.loc[data[eax].isna(), eay]
            data.loc[data[oax].isna(), oax] = data.loc[data[oax].isna(), oay]
            data.loc[data[eay].isna(), eay] = data.loc[data[eay].isna(), eax]
            data.loc[data[oay].isna(), oay] = data.loc[data[oay].isna(), oax]
            org_len = len(data)
            for a in [eax, oax, eay, oay]:
                data[a] = data[a].astype(str)
            flip = data.loc[(data[eax] == data[oay]) & (data[oax] == data[eay]), bys[0]].index
            for by in bys:
                data.loc[flip, by] *= -1
            data.loc[flip, eay] = data.loc[flip, eax]
            data.loc[flip, oay] = data.loc[flip, oax]
            for fy in eafys + oafys:
                if fy in data.columns:
                    data.loc[flip, fy] = abs(1 - data.loc[flip, fy])
            data2 = data.loc[(data[eax] == data[eay]), :].copy()
            data2 = data2.loc[(data2[oax] == data2[oay]), :].copy()
            dropped += (org_len - len(data2))
            data2.drop(axis=1, labels=[eay, oay], inplace=True)
            data2.rename(columns={eax: 'ea', oax: 'oa'}, inplace=True)
            self.data[c] = data2
        self.columns = self.data[1].columns
        if dropped > 0:
            sumstatswarn('Dropped {} SNPs due to allele mismatch'.format(dropped))
        self.suffixes = [None, None]

    def meta_analyze(self, name='meta', method='ivw', debug=False):
        """Meta analyze all GWAS summary statistics contained in this object.
        WARNING: There appears to be an error somewhere in this function that causes incorrect result.
        For now running .meta_analyze() will instead run .gwama() with an identity matrix (functionally identical to an ivw meta_analysis)
        :param name: New phenotype name to use for the new SumStats object (default: 'meta')
        :type name: str
        :param method: Meta-analysis method to use, should be one of ['ivw', 'samplesize'], default: 'ivw'
        :type method: str
        :param debug: Run the meta_analyze function instead of .gwama() for debugging purposes
        :type debug: bool
        :return: :class:`pysumstats.SumStats` object.

        """
        assert isinstance(name, str), "name should be str"
        if not self.low_ram:
            new_data = {}
        else:
            new_data = _H5SSConnection(name, tmpdir=self.tmpdir)
        if method not in ['ivw', 'samplesize']:
            raise KeyError('method should be one of [\'ivw\', \'samplesize\']')
        if not debug:
            sumstatswarn("The meta_analyze function is currently bugged. .gwama() with an identity matrix is now performed instead.")
            id_mat = pd.DataFrame(np.identity(len(self.pheno_names)), columns=self.pheno_names, index=self.pheno_names)
            return self.gwama(cov_matrix=id_mat, name=name)
        else:
            sumstatswarn("Warning: Please do not use the results of this function without extensive verification!")
            columns = self.data[1].columns
            missing_n = []
            for p in self.pheno_names:
                if 'n_{}'.format(p) not in columns:
                    missing_n.append(p)
            if len(missing_n) > 0:
                if method == 'samplesize':
                    raise KeyError('Missing sample size column for {}.'.format(', '.join(missing_n)))
                else:
                    sumstatswarn('Missing sample size column for {}, output N will be incorrect.'.format(', '.join(missing_n)))
            for c in self.data.keys():
                data = self.data[c]

                n_dat = data[['rsid', 'chr', 'bp', 'ea', 'oa', 'maf']]
                if method == 'samplesize':
                    for p in self.pheno_names:
                        data.loc[data['b_{}'.format(p)].isna(), 'b_{}'.format(p)] = 0
                        data.loc[data['se_{}'.format(p)].isna(), 'se_{}'.format(p)] = 1
                        data.loc[:, 'z_{}'.format(p)] = data['b_{}'.format(p)] / data['se_{}'.format(p)]
                        data.loc[:, 'w_{}'.format(p)] = np.sqrt(data['n_{}'.format(p)])
                        data.loc[:, 'z_{}'.format(p)] = data['z_{}'.format(p)] * data['w_'.format(p)]
                    zsums = data[['z_'.format(x) for x in self.pheno_names]].sum(axis=1)
                    wsums = data[['w_'.format(x) for x in self.pheno_names]].sum(axis=1)
                    n_dat.loc[:, 'z'] = zsums / np.sqrt(np.sum(wsums ** 2))
                else:
                    for p in self.pheno_names:
                        data.loc[:, 'w_{}'.format(p)] = 1 / (data.loc[:, 'se_{}'.format(p)] ** 2)
                        data.loc[:, 'bw_{}'.format(p)] = data.loc[:, 'b_{}'.format(p)] * data.loc[:, 'w_{}'.format(p)]
                    bwsums = data[['bw_{}'.format(x) for x in self.pheno_names]].sum(axis=1)
                    wsums = data[['w_{}'.format(x) for x in self.pheno_names]].sum(axis=1)
                    n_dat.loc[:, 'se'] = np.sqrt(1 / wsums)
                    n_dat.loc[:, 'b'] = bwsums / wsums
                    n_dat.loc[:, 'z'] = n_dat.loc[:, 'b'] / n_dat.loc[:, 'se']
                n_dat.loc[:, 'p'] = norm.sf(abs(n_dat.loc[:, 'z'])) * 2
                n_dat['n'] = 0
                for p in self.pheno_names:
                    if 'n_{}'.format(p) in data.columns:
                        n_dat['n'] += data['n_{}'.format(p)]
                new_data[c] = n_dat
            new_columns = list(new_data[c].columns)
            new_data = SumStats(path=None, phenotype=name, data=new_data)
            new_data.variables = new_columns
            return new_data

    def gwama(self, cov_matrix=None, h2_snp=None, name='gwama'):
        """Multivariate meta analysis as described in Baselmans, et al. 2019.

        :param cov_matrix: Covariance matrix, defaults to generating a correlation matrix of Z-scores
        :type cov_matrix: pd.Dataframe
        :param h2_snp: Dict of SNP heritabilities per GWAS, to use as additional weights. Defaults to all 1's.
        :type h2_snp: dict
        :param name: New phenotype name to use in the new SumStats object (default: 'gwama')
        :return: :class:`pysumstats.SumStats` object

        """
        assert isinstance(cov_matrix, pd.DataFrame), "cov_matrix should be a pd.DataFrame"
        assert isinstance(name, str), "name should be a string"
        if h2_snp is None:
            sumstatswarn('h2-snp not specified, using ones instead. This will bias the estimates')
            h2_snp = {x: 1 for x in self.pheno_names}
        elif (not isinstance(h2_snp, pd.DataFrame)) or isinstance(h2_snp, dict) or (not isinstance(h2_snp, pd.Series)):
            raise KeyError('h2-snp should be a dataframe or dictionary with phenotype names as keys or column names.')
        else:
            for p in self.pheno_names:
                try:
                    _ = h2_snp[p]
                except KeyError:
                    raise KeyError('Could not find h2_snp value for {}'.format(p))
        if cov_matrix is None:
            cov_matrix = pd.DataFrame(index=self.pheno_names, columns=self.pheno_names)
            vars = {}
            for p in self.pheno_names:
                vars[p] = []
                cov_matrix.loc[p, p] = 1
                for c in range(1, 24):
                    data = self.data[c]
                    data['z_{}'.format(p)] = data['b_{}'.format(p)] / data['se_{}'.format(p)]
                    vars[p].append(self.data[1]['z_{}'.format(p)].var() * (len(data) / len(self)))
                    self.data[c] = data
            for p1, p2 in [(i, i2) for n, i in enumerate(self.pheno_names) for i2 in self.pheno_names[(n + 1):]]:
                tmp_statistics = {}
                for c in range(1, 24):
                    if len(self.data[c]) > 0:
                        covar = self.data[c][['z_{}'.format(p1), 'z_{}'.format(p2)]].cov().iloc[0, 1] * (
                                len(self.data[c]) / len(self))
                        tmp_statistics[c] = dict(p1_var=vars[p1][c], p2_var=vars[p2][c], covar=covar)
                tmp_df = pd.DataFrame.from_dict(tmp_statistics)
                cov_matrix.loc[p1, p2] = tmp_df.loc['covar', :].sum() / (
                        np.sqrt(tmp_df.loc['p1_var', :].sum()) * np.sqrt(tmp_df.loc['p2_var', :].sum()))
                cov_matrix.loc[p2, p1] = cov_matrix.loc[p1, p2]
        else:
            if not isinstance(cov_matrix, pd.DataFrame):
                raise KeyError(
                    'Incorrect cov_matrix specified, should be pandas dataframe with phenotypes as columns and indices.')
            if (not sorted(list(cov_matrix.index)) == sorted(list(self.pheno_names))) or (
                    not sorted(list(cov_matrix.columns)) == sorted(list(self.pheno_names))):
                raise IndexError(
                    'Incorrect cov_matrix specified, should be pandas dataframe with phenotypes as columns and indices.')
        if not self.low_ram:
            new_data = {}
        else:
            new_data = _H5SSConnection(name, tmpdir=self.tmpdir)
        i_cov_matrix = pd.DataFrame(np.linalg.inv(cov_matrix.to_numpy(dtype=np.float64)), index=cov_matrix.index,
                                    columns=cov_matrix.columns)
        for c in range(1, 24):
            n_dat = self.data[c][['rsid', 'chr', 'bp', 'ea', 'oa', 'maf']].copy()
            data = self.data[c]
            for p in self.pheno_names:
                data.loc[data['n_{}'.format(p)].isna(), 'n_{}'.format(p)] = 0
                data.loc[data['b_{}'.format(p)].isna(), 'b_{}'.format(p)] = 0
                data.loc[data['se_{}'.format(p)].isna(), 'se_{}'.format(p)] = 1
                data.loc[data['eaf_{}'.format(p)].isna(), 'eaf_{}'.format(p)] = 0
            for p in self.pheno_names:
                data['z_{}'.format(p)] = data['b_{}'.format(p)] / data['se_{}'.format(p)]
                data['w_{}'.format(p)] = np.sqrt(data['n_{}'.format(p)] * h2_snp[p])
                data['wz_{}'.format(p)] = data['w_{}'.format(p)] * data['z_{}'.format(p)]
            data['cw'] = np.array(
                [data['w_{}'.format(p)] * data['w_{}'.format(p2)] * cov_matrix.loc[p, p2]
                 for p in self.pheno_names for p2 in [x for x in self.pheno_names]]).sum(axis=0)
            n_dat['z'] = data[['wz_{}'.format(x) for x in self.pheno_names]].sum(axis=1) / np.sqrt(
                data['cw'])
            n_dat.loc[:, 'p'] = norm.sf(abs(n_dat.loc[:, 'z'])) * 2
            n_dat['n'] = np.array([np.sqrt(data['n_{}'.format(p)]) * np.sqrt(
                data['n_{}'.format(p2)]) * i_cov_matrix.loc[p, p2] for p in self.pheno_names for p2 in
                                   [x for x in self.pheno_names]]).sum(axis=0)
            n_obs = data[['n_{}'.format(p) for p in self.pheno_names]].sum(axis=1)
            n_dat['eaf'] = np.array(
                [data['n_{}'.format(p)] * data['eaf_{}'.format(p)] for p in self.pheno_names]).sum(
                axis=0) / n_obs
            n_dat['maf'] = pd.concat([n_dat['eaf'], (1 - n_dat['eaf'])], axis=1).min(axis=1)
            n_dat['b'] = n_dat['z'] / np.sqrt(n_obs * 2 * n_dat['eaf'] * (1 - n_dat['eaf']))
            n_dat['se'] = (1 / np.sqrt(n_obs)) * (1 / np.sqrt(2 * n_dat['eaf'] * (1 - n_dat['eaf'])))
            new_data[c] = n_dat
        new_columns = list(new_data[1].columns)
        new_data = SumStats(path=None, phenotype=name, data=new_data)
        new_data.variables = new_columns
        return new_data

    def merge(self, other, inplace=False, how='inner', low_memory=False):
        """ Merge with other SumStats or MergedSumstats object(s).

        :param other: :class:`pysumstats.SumStats`, or :class:`pysumstats.MergedSumStats` object, or a list of SumStats, MergedSumstats objects
        :type other: :class:`pysumstats.SumStats`, :class:`pysumstats.MergedSumStats`, or list.
        :param inplace: Enable to store the new data in the current MergedSumStats object. (currently not supported when low_ram is enabled)
        :type inplace: bool
        :param how: Type of merge, for now only implemented for merges with :class:`pysumstats.SumStats` objects
        :type how: str
        :param low_memory: Enable to use a more RAM-efficient merging method (WARNING: still untested)
        :type low_memory: bool
        :return: None, or :class:`pysumstats.MergedSumStats` object.

        """
        assert isinstance(other, SumStats) or isinstance(other, MergedSumStats) or isinstance(other, list), "other should be SumStats, MergedSumStats or list"
        assert isinstance(inplace, bool), "inplace should be True or False"
        assert isinstance(how, str), "how should be a string"
        assert isinstance(low_memory, bool), "low_memory should be True or False"
        if isinstance(other, list):
            for o in other:
                assert isinstance(o, SumStats) or isinstance(o, MergedSumStats), "items in other should be SumStats or MergedSumStats"
        if self.low_ram and inplace:
            sumstatswarn('Inplace merging with low_ram currently does not yield any performance benefits.')
        if isinstance(other, list):
            if inplace:
                for o in other:
                    self.merge(o, True, how=how, low_memory=low_memory)
            else:
                newmerged = self.merge(other[0], False, how=how, low_memory=low_memory)
                for o in other[1:]:
                    newmerged.merge(o, True, how=how, low_memory=low_memory)
                return newmerged
        else:
            if low_memory and (not how == 'inner'):
                raise NotImplementedError('low_memory only allows for an inner merge.')
            if other.phenotype_name is not None:
                if other.phenotype_name in self.pheno_names:
                    sumstatswarn(
                        'Phenotype name already in the dataset, converting to and {0}2'.format(other.phenotype_name))
                    other.phenotype_name = other.phenotype_name + '2'
                self.suffixes[1] = other.phenotype_name
                merge_info = dict(xlen=0, ylen=0, newlen=0)
                joined = [x for x in other.variables if x in self.variables]
                for x in [x for x in other.variables if x not in joined]:
                    sumstatswarn(
                        'Could not find column {} in pysumstats for {}, column dropped'.format(x, other.phenotype_name))
                merged_data = {}
                for c in range(1, 24):
                    data_x = self.data[c].copy()
                    merge_info['xlen'] += len(data_x)
                    data_y = other.data[c].copy()
                    merge_info['ylen'] += len(data_y)
                    data_m_y = data_y.rename(
                        columns={k: '{}_{}'.format(k, other.phenotype_name) for k in data_y.columns if
                                 k not in ['rsid', 'chr', 'bp']})
                    if low_memory:
                        if np.any(data_x.duplicated(subset='rsid')) or np.any(data_m_y.duplicated(subset='rsid')):
                            raise KeyError('Missing rsids in either dataset will cause low_memory merge to fail')
                        merged_data[c] = data_x.loc[data_x['rsid'].isin(data_m_y['rsid'].tolist()), :]
                        data_m_y = data_m_y.loc[data_m_y['rsid'].isin(data_x['rsid'].tolist()), :]
                        del data_y
                        merged_data[c].sort_values(by='rsid', inplace=True)
                        data_m_y.sort_values(by='rsid', inplace=True)
                        merged_data[c].reset_index(inplace=True, drop=True)
                        data_m_y.reset_index(inplace=True, drop=True)
                        data_m_y = data_m_y[
                            [x + '_' + other.phenotype_name for x in joined if x not in ['rsid', 'chr', 'bp']]]
                        for x in data_m_y.columns:
                            merged_data[c][x] = data_m_y[x]
                            data_m_y.drop(axis=1, labels=[x], inplace=True)
                        merged_data[c].sort_values(by='bp', inplace=True)
                    else:
                        joined_y = ['rsid', 'chr_y', 'bp_y'] + [x + '_' + other.phenotype_name for x in joined if
                                                                x not in ['rsid', 'chr', 'bp']]
                        data_m_y = data_m_y.rename(columns={'chr': 'chr_y', 'bp': 'bp_y'})
                        merged_data[c] = data_x.merge(data_m_y[joined_y], on='rsid', how=how)
                        merged_data[c].loc[merged_data[c]['chr'].isna(), 'chr'] = merged_data[c].loc[
                            merged_data[c]['chr'].isna(), 'chr_y']
                        merged_data[c].loc[merged_data[c]['bp'].isna(), 'bp'] = merged_data[c].loc[
                            merged_data[c]['bp'].isna(), 'bp_y']
                        merged_data[c] = merged_data[c].drop(axis=1, labels=['chr_y', 'bp_y'])
                    merged_data[c]['maf'] = _recompute_maf(merged_data[c], self.pheno_names + [other.phenotype_name])
                    merge_info['newlen'] += len(merged_data[c])
                    new_phenos = self.pheno_names + [other.phenotype_name]
                if not inplace:
                    return MergedSumStats(data=merged_data, phenotypes=new_phenos, merge_info=merge_info,
                                          variables=joined, xy=self.suffixes, low_ram=self.low_ram, tmpdir=self.tmpdir)
                else:
                    self.__init__(merged_data, new_phenos, merge_info, joined, self.suffixes, self.low_ram)
            else:
                for n, other_name in enumerate(other.pheno_names):
                    if other_name in self.pheno_names:
                        other.pheno_names[n] = other_name + '_y'
                        for c in range(1, 24):
                            for x in other.variables:
                                odat = other.data[c]
                                odat.rename(columns={x + '_' + other_name: x + '_' + other_name + '_y'}, inplace=True)
                                other.data[c] = odat
                if not self.low_ram:
                    merged_data = {}
                else:
                    tmpfilename = ''.join([''.join([j[0], j[1]]) for j in self.pheno_names]) + ''.join(
                        [''.join([j[0], j[1]]) for j in other.pheno_names])
                    merged_data = _H5SSConnection(
                        '{}{}'.format(tmpfilename, len(os.listdir('sumstats_temporary'))), tmpdir=self.tmpdir)
                for c in range(1, 24):
                    sdat = self.data[c]
                    odat = other.data[c]
                    s_id = pd.DataFrame(sdat['rsid'])
                    o_id = pd.DataFrame(odat['rsid'])
                    s_id['s_idx'] = s_id.index
                    o_id['o_idx'] = o_id.index
                    idm = s_id.merge(o_id, on='rsid')
                    sdat.rename(columns={'ea': 'ea_x', 'oa': 'oa_x'}, inplace=True)
                    odat.rename(columns={'ea': 'ea_y', 'oa': 'oa_y'}, inplace=True)
                    other_cols = [x for x in odat.columns if x not in ['rsid', 'chr', 'bp']]
                    sdat = sdat.loc[idm['s_idx'], :].reset_index(drop=True)
                    odat = odat.loc[idm['o_idx'], other_cols].reset_index(drop=True)
                    for oc in odat.columns:
                        sdat[oc] = odat[oc]
                    merged_data[c] = sdat
                    merged_data[c]['maf'] = _recompute_maf(merged_data[c], self.pheno_names + other.pheno_names)
                if inplace:
                    self.data = merged_data
                    self.columns = self.data[1].columns
                    self.suffixes = ['x', 'y']
                    self._allign(ynames=['_{}'.format(x) for x in other.pheno_names])
                else:
                    newmergedss = MergedSumStats(data=merged_data, phenotypes=self.pheno_names + other.pheno_names,
                                                 merge_info={}, xy=['x', 'y'], variables=self.variables,
                                                 low_ram=self.low_ram, tmpdir=self.tmpdir, allign=False)
                    newmergedss._allign(ynames=['_{}'.format(x) for x in other.pheno_names])
                    return newmergedss

    def describe(self, columns=None, per_chromosome=False):
        """ Get a summary of the data.

        :param columns: List of column names to print summary for (default: ['b', 'se', 'p'])
        :type columns: list
        :param per_chromosome: Enable to return a list of summary dataframes per chromosome
        :type per_chromosome: bool
        :return: pd.Dataframe, or list of pd.Dataframes

        """
        assert isinstance(columns, list) or (columns is None), "columns should be a list"
        assert isinstance(per_chromosome, bool), "per_chromosome should be True or False"
        if columns is None:
            columns = ['b', 'se', 'p']
        if (not isinstance(columns, list)) and isinstance(columns, str):
            columns = [columns]
        sum_cols = []
        for c in columns:
            if c not in self.data[1].columns:
                if '{}_{}'.format(c, self.pheno_names[0]) in self.data[1].columns:
                    for c2 in ['{}_{}'.format(c, p) for p in self.pheno_names]:
                        sum_cols.append(c2)
                else:
                    sumstatswarn('column {} not found'.format(c))
            else:
                sum_cols.append(c)
        return self._get_summary(sum_cols, per_chromosome)

    def prep_for_mr(self, exposure, outcome, filename=None, p_cutoff=None, bidirectional=False, **kwargs):
        """Save a pre-formatted file to use with the MendelianRandomization package in R.

        :param exposure: phenotype name to use as exposure.
        :type exposure: str
        :param outcome: phenotype name to use as outcome.
        :type outcome: str
        :param filename: Path to where the resulting file(s) should be stored, or list of paths if bidirectional=True
        :type filename: str, list or None
        :param p_cutoff: Optional p-value cut-off to apply. Will include SNPs where P > p_cutoff
        :type p_cutoff: float
        :param bidirectional: Enable to store two files (exposure=exposure, outcome=outcome), and (exposure=outcome, outcome=exposure)
        :type bidirectional: bool
        :param kwargs: Additional keyword arguments to be passed to pandas to_csv function.
        :return: None

        """
        assert isinstance(exposure, str), "exposure should be a string"
        assert isinstance(outcome, str), "exposure should be a string"
        assert isinstance(filename, str) or isinstance(filename, list) or (filename is None), "filename should be a string or list of strings"
        assert isinstance(p_cutoff, float), "p_cutoff should be a float"
        assert isinstance(bidirectional, bool), "bidirectional should be True or False"
        if isinstance(filename, list):
            for o in filename:
                assert isinstance(o, str), "filename should be a string or list of strings"
        if bidirectional and (not isinstance(filename, list)):
            raise ValueError(
                'If bidirectional, filename should be a list of filenames: (exp-outcome) name and (outcome-exp) name.')
        mr_data = []
        if exposure not in self.pheno_names:
            raise KeyError('{} not found in phenotypes'.format(exposure))
        if outcome not in self.pheno_names:
            raise KeyError('{} not found in phenotypes'.format(outcome))
        for c in self.data.keys():
            if p_cutoff is None:
                mr_data_ = self.data[c].copy()
            else:
                mr_data_ = self.data[c].loc[self.data[c]['p_{}'.format(exposure)] <= p_cutoff, :].copy()
            newcols = [['se', 'se'], ['b', 'b']]
            newcols2 = [[exposure, 'exposure'], [outcome, 'outcome']]
            renames = {'{}_{}'.format(old, x): '{}.{}'.format(x2, new) for x, x2 in newcols2 for old, new in newcols}
            mr_data_.rename(columns=renames, inplace=True)
            mr_data_.rename(columns={'ea': 'exposure.A1', 'oa': 'exposure.A2'}, inplace=True)
            mr_data_['outcome.A1'] = mr_data_['exposure.A1']
            mr_data_['outcome.A2'] = mr_data_['exposure.A2']
            mr_data.append(mr_data_[
                               ['rsid', 'chr', 'bp', 'exposure.A1', 'exposure.A2', 'outcome.A1', 'outcome.A2'] + list(
                                   renames.values())])
        mr_data = pd.concat(mr_data, axis=0)
        if filename is None:
            mr_data.to_csv('MR-prepped_{}_{}.csv'.format(exposure, outcome), **kwargs)
        else:
            if not bidirectional:
                mr_data.to_csv(filename, **kwargs)
            else:
                mr_data.to_csv(filename[0], **kwargs)
        if bidirectional:
            if filename is None:
                self.prep_for_mr(exposure=outcome, outcome=exposure, filename=None, p_cutoff=p_cutoff,
                                 bidirectional=False, **kwargs)
            else:
                self.prep_for_mr(exposure=outcome, outcome=exposure, filename=filename[1], p_cutoff=p_cutoff,
                                 bidirectional=False, **kwargs)

    def manhattan(self, filename=None, phenotypes=None, nrows=None, ncols=None, figsize=None, dpi=300, **kwargs):
        """Generates manhattan plots for each phenotype (or specified phenotypes) in merged GWAS data.

        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param phenotypes: List of phenotype names to plot manhattans for (defaults to plotting all phenotypes)
        :type phenotypes: list
        :param nrows: Specify number of rows in the figure ( defaults to int(ceil(len(phenotypes)/ncols)) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(log2(len(phenotypes)/2)) )
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*8, nrows*4) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.manhattan`
        :return: (fig, axes) or None.
        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(phenotypes, list) or (phenotypes is None), "phenotypes should be a list of strings or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if phenotypes is None:
            phenotypes = self.pheno_names
        for p in phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        plotwindow = _MultiWindowPlot(len(phenotypes), nrows=nrows, ncols=ncols, figsize=figsize, shape='rect')
        dat = self[['rsid', 'chr', 'bp']]
        for phenotype in phenotypes:
            fig, ax = plotwindow.get_next_ax()
            dat['p'] = self['p_{}'.format(phenotype)]
            if 'title' not in kwargs.keys():
                manhattan(dat, fig=fig, ax=ax, title=phenotype, **kwargs)
            else:
                manhattan(dat, fig=fig, ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def qqplot(self, filename=None, phenotypes=None, nrows=None, ncols=None, figsize=None, dpi=300, **kwargs):
        """Generates QQ-plots for each phenotype (or specified phenotypes) in merged GWAS data.

        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param phenotypes: List of phenotype names to plot QQ-plots for (defaults to plotting all phenotypes)
        :type phenotypes: list
        :param nrows: Specify number of rows in the figure ( defaults to int(sqrt(len(phenotypes))) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(ceil(len(phenotypes)/nrows)) )
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.qqplot`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(phenotypes, list) or (phenotypes is None), "phenotypes should be a list of strings or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if phenotypes is None:
            phenotypes = self.pheno_names
        for p in phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        plotwindow = _MultiWindowPlot(len(phenotypes), nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for phenotype in phenotypes:
            fig, ax = plotwindow.get_next_ax()
            if 'title' not in kwargs.keys():
                qqplot(self['p_{}'.format(phenotype)].values, fig=fig, ax=ax, title=phenotype, **kwargs)
            else:
                qqplot(self['p_{}'.format(phenotype)].values, fig=fig, ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def pzplot(self, filename=None, phenotypes=None, nrows=None, ncols=None, figsize=None, dpi=300, **kwargs):
        """Generates PZ-plots for each phenotype (or specified phenotypes) in merged GWAS data.

        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param phenotypes: List of phenotype names to plot PZ-plots for (defaults to plotting all phenotypes)
        :type phenotypes: list
        :param nrows: Specify number of rows in the figure ( defaults to int(sqrt(len(phenotypes))) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(ceil(len(phenotypes)/nrows)) )
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.pzplot`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(phenotypes, list) or (phenotypes is None), "phenotypes should be a list of strings or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if phenotypes is None:
            phenotypes = self.pheno_names
        for p in phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        plotwindow = _MultiWindowPlot(len(phenotypes), nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for phenotype in phenotypes:
            dat = self[['{}_{}'.format(v, phenotype) for v in ['b', 'se', 'p']]].copy()
            dat.rename(columns={'{}_{}'.format(v, phenotype): v for v in ['b', 'se', 'p']}, inplace=True)
            fig, ax = plotwindow.get_next_ax()
            if 'title' not in kwargs.keys():
                pzplot(dat, fig=fig, ax=ax, title='PZ_{}'.format(phenotype), **kwargs)
            else:
                pzplot(dat, fig=fig, ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def afplot(self, ref_phenotypes=None, other_phenotypes=None, filename=None, nrows=None, ncols=None, figsize=None,
               dpi=300, **kwargs):
        """Generates AF comparison plots for merged GWAS data.

        :param ref_phenotypes: List of phenotypes to use as reference (defaults to all phenotypes)
        :type ref_phenotypes: list
        :param other_phenotypes: List of phenotypes to compare reference to (defaults to all phenotypes, overlapping plots will be dropped)
        :type other_phenotypes: list
        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param nrows: Specify number of rows in the figure ( defaults to int(ceil(n_plots/ncols)) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(sqrt(n_plots)))
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.manhattan`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(ref_phenotypes, list) or (ref_phenotypes is None), "ref_phenotypes should be a string or None"
        assert isinstance(other_phenotypes, list) or (other_phenotypes is None), "other_phenotypes should be a string or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if ref_phenotypes is None:
            ref_phenotypes = self.pheno_names
        for p in ref_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        if other_phenotypes is None:
            other_phenotypes = self.pheno_names
        for p in other_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        n_plots = len(ref_phenotypes) * len(other_phenotypes)
        plotwindow = _MultiWindowPlot(n_plots, nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for p1 in ref_phenotypes:
            eafp1 = self['eaf_{}'.format(p1)].values
            for p2 in other_phenotypes:
                fig, ax = plotwindow.get_next_ax()
                if p1 != p2:
                    afplot(eafp1, self['eaf_{}'.format(p2)].values,
                           refname=r'$EAF_{}$'.format('{' + p1.replace('_', r'\_') + '}'),
                           othername=r'$EAF_{}$'.format('{' + p2.replace('_', r'\_') + '}'), fig=fig, ax=ax,
                           **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def zzplot(self, ref_phenotypes=None, other_phenotypes=None, filename=None, nrows=None, ncols=None, figsize=None,
               dpi=300, **kwargs):
        """Generates ZZ comparison plots for merged GWAS data.

        :param ref_phenotypes: List of phenotypes to use as reference (defaults to all phenotypes)
        :type ref_phenotypes: list
        :param other_phenotypes: List of phenotypes to compare reference to (defaults to all phenotypes, overlapping plots will be dropped)
        :type other_phenotypes: list
        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param nrows: Specify number of rows in the figure ( defaults to int(ceil(n_plots/ncols)) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(sqrt(n_plots)))
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.zzplot`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(ref_phenotypes, list) or (ref_phenotypes is None), "ref_phenotypes should be a string or None"
        assert isinstance(other_phenotypes, list) or (other_phenotypes is None), "other_phenotypes should be a string or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if ref_phenotypes is None:
            ref_phenotypes = self.pheno_names
        for p in ref_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        if other_phenotypes is None:
            other_phenotypes = self.pheno_names
        for p in other_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        n_plots = len(ref_phenotypes) * len(other_phenotypes)
        plotwindow = _MultiWindowPlot(n_plots, nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for p1 in ref_phenotypes:
            zz1 = self[['b_{}'.format(p1), 'se_{}'.format(p1)]].copy()
            zz1.rename(columns={'b_{}'.format(p1): 'b', 'se_{}'.format(p1): 'se'}, inplace=True)
            for p2 in other_phenotypes:
                fig, ax = plotwindow.get_next_ax()
                if p1 != p2:
                    zz2 = self[['b_{}'.format(p2), 'se_{}'.format(p2)]].copy()
                    zz2.rename(columns={'b_{}'.format(p2): 'b', 'se_{}'.format(p2): 'se'}, inplace=True)
                    zzplot(zz1, zz2, xname=r'$Z_{}$'.format('{' + p1.replace('_', r'\_') + '}'),
                           yname='r$Z_{}$'.format('{' + p2.replace('_', r'\_') + '}'),
                           fig=fig,
                           ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)


movesets = [
             ['L', "L'", 'L2', 'R', "R'", 'R2', 'U', "U'", 'U2', 'B', "B'", 'B2', 'D', "D'", 'D2', 'F', "F'", 'F2'],
             ['L', "L'", 'L2', 'R', "R'", 'R2', 'U', "U'", 'U2', 'D', "D'", 'D2', 'F2', 'B2'],
             ['U', "U'", 'U2', 'D', "D'", 'D2', 'L2', 'F2', 'R2', 'B2'],
             ['U2', 'L2', 'F2', 'R2', 'D2', 'B2']
           ]

masks = [{1: 'orange', 3: 'orange', 5: 'orange', 7: 'orange', 21: 'orange', 23: 'orange', 28: 'orange', 30: 'orange',
          32: 'orange', 34: 'orange', 48: 'orange', 50: 'orange'},

         {0: 'orange', 1: 'orange', 2: 'orange', 3: 'orange', 5: 'orange', 6: 'orange', 7: 'orange', 8: 'orange',
          27: 'orange', 28: 'orange', 29: 'orange', 30: 'orange', 32: 'orange', 33: 'orange', 34: 'orange',
          35: 'orange', 21: 'pink', 23: 'pink', 48: 'pink', 50: 'pink'},

         {0: 'original', 2: 'original', 6: 'original', 8: 'original', 9: 'original', 11: 'original', 15: 'original',
          17: 'original', 18: 'original', 20: 'original', 24: 'original', 26: 'original', 27: 'original',
          29: 'original', 33: 'original', 35: 'original', 36: 'original', 38: 'original', 42: 'original',
          44: 'original', 45: 'original', 47: 'original', 51: 'original', 53: 'original', 37: 'red', 39: 'red',
          41: 'red', 43: 'red', 19: 'original', 21: 'original', 23: 'original', 25: 'original', 10: 'original',
          12: 'original', 14: 'original', 16: 'original', 46: 'green', 48: 'green', 50: 'green', 52: 'green'},

         {0: 'original', 1: 'original', 2: 'original', 3: 'original', 4: 'original', 5: 'original', 6: 'original',
          7: 'original', 8: 'original', 9: 'original', 10: 'original', 11: 'original', 12: 'original', 13: 'original',
          14: 'original', 15: 'original', 16: 'original', 17: 'original', 18: 'original', 19: 'original',
          20: 'original', 21: 'original', 22: 'original', 23: 'original', 24: 'original', 25: 'original',
          26: 'original', 27: 'original', 28: 'original', 29: 'original', 30: 'original', 31: 'original',
          32: 'original', 33: 'original', 34: 'original', 35: 'original', 36: 'original', 37: 'original',
          38: 'original', 39: 'original', 40: 'original', 41: 'original', 42: 'original', 43: 'original',
          44: 'original', 45: 'original', 46: 'original', 47: 'original', 48: 'original', 49: 'original',
          50: 'original', 51: 'original', 52: 'original', 53: 'original'
          }
         ]

prune_depths = [10, 6, 5, 6]


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import tkinter as tk
import tkinter.font as tkFont
import os
import sys
import platform
import math
import random
import socket
import json

PATH = os.path.dirname(os.path.realpath(__file__))
MODE = 'debug' # 'normal / profile / debug'

try:
    import keyboard
except ImportError as e:
    os.system('python3 {}'.format(os.path.join(PATH, 'install.py')))

os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = 'hide'

from scripts import profile
from app import LocalGame, OnlineGame, SplashScreen, Settings

__author__     = "Dorian Beauchesne"
__copyright__  = "Copyright 2019, 2PQSTD "
__credits__    = ["Dorian Beauchesne", "Louis de Bussac"]
__sounds__     = ["Théo Bonneval", "Charles Lessueur"]
__license__    = "MPSI2 - MIT"
__version__    = "0.8.2"
__maintainer__ = "Dorian Beauchesne"
__email__      = "dorian.beauchesne@icloud.com"
__status__     = "Development"

def start_local_game(name, difficulty, role):
    app = LocalGame(name, difficulty, role)
    app.generate_target()
    app.start()

def start_online_game(name, ip, port, role):
    app = OnlineGame(name, role)
    app.connect(ip, port)
    app.start()

def toggle_settings(splash_screen):
    splash_screen.fen.attributes('-topmost', False)
    settings = Settings(splash_screen)

def parse_args(argv):
    global MODE
    for arg in argv:
        if '=' in arg:
            key, val = arg.split('=')
            if key == 'mode':
                MODE = val
        elif '-' in arg:
            pass

if __name__ == '__main__':
    if len(sys.argv) > 1: parse_args(sys.argv)
    splash_screen = SplashScreen(start_online_game, start_local_game, toggle_settings)
    splash_screen.create_window()
    splash_screen.start()
    sys.exit(0)


#!/usr/bin/python3
from scanconfig import ScanConfig
from scan import Scan
from colorama import Fore
import multiprocessing as multp

# Main block
if __name__ == '__main__':
    multp.set_start_method("spawn")
    conf = ScanConfig('conf.yaml')
    scan = Scan()

    # Checks if Nmap is installed on the system prior to operation
    if scan.checkInstall() == False:
        print(Fore.RED + 'nmap is not installed on the system.\n')
    else:
        print(Fore.GREEN + scan.checkVersion().decode('utf-8'))

    # Function to create a "hosts.txt" file if one does not yet exist
    conf.createCIDR_File(conf.getConf('cidr-range'))

    # Opens the file with targets in them and enumerates through them individually
    with open('hosts.txt', 'r') as file:
        agg_hosts = file.readlines()
        while len(agg_hosts) != 0:
            instr = multp.Process
            try:
                # This loop will create the processes ("threads") and subsequently executes them
                for process_count in range(int(conf.getConf('threads'))):
                    # Empty list of all the processes (scans) to be executed
                    processes = []
                    scan_attempt = instr(target=scan.scanTarget, args=(agg_hosts[process_count], conf.parseListArgs(conf.getConf('args')), conf.getConf('ports')))
                    processes.append(scan_attempt)
                    # Starts the "thread"
                    scan_attempt.start()
                    agg_hosts.pop(process_count)

                # Joins the processes
                for process in processes:
                    process.join()

            except IndexError:
                try:
                    multp.Process(target=scan.scanTarget(agg_hosts[0], conf.parseListArgs(conf.getConf('args')), conf.getConf('ports'))).start()
                    agg_hosts.pop(0)
                except IndexError:
                    print('No more hosts left to scan.')
                    break
                print('No more hosts left to scan.')
                break

    file.close()


"""Example implementation of the Coordinate Descent algorithm in Python

- The minimum value of F(x) is computed iteratively by minimizing individual coordinates in a cyclic manner
- On each iteration the active coordinate is updated as follows

  X_k+1 = X_k - alpha * [del_F(xk)] * e

  Where
  * alpha : step length,
  * e : the active co-ordinate vector(required to make vector addition to Xk possible),
  * del_F(xk) : component of the gradient corresponding to the active coordinate
"""

import logging
import numpy as np
import math

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)

__author__ = "@NdamuleloNemakh"
__version__ = 0.01

class CoordinateDescent:
    def __init__(self, max_iterations=500, no_of_dimensions=2, step_length=1, fixed_step_length=True):
        self.Dimensions = no_of_dimensions
        self.StepLength = step_length
        self.FixedStepLength = fixed_step_length
        self.MaxIterations = max_iterations
        self.CurrentIteration = 0
        self.Minimiser: np.ndarray = np.array([0, 0])
        self.ConvergenceThreshold = 10e-4

    def __preview_config(self):
        configs = f'''
                   Configuration Preview
                   ========================
                   n                   = {self.Dimensions}
                   Step Length         = {self.StepLength}
                   Fixed Step Length   = {self.FixedStepLength}
                   Max Iterations      = {self.MaxIterations}
                   '''
        log.debug(configs)
        return configs

    @property
    def _current_coordinate_idx(self):
        # Choose next cordinate to optimize, in a CYCLIC manner
        next_idx = lambda i: i % 2 + 1
        return next_idx(self.CurrentIteration)

    @property
    def _ith_coordinate_vector(self):
        v = np.zeros(self.Dimensions)
        active_idx = self._current_coordinate_idx - 1
        v[active_idx] = 1
        return v

    @staticmethod
    def __current_gradient(xk) -> np.ndarray:
        #  Example: F(X1, X2) = X1 - X2 + 2X1**2 + 2X1X2 + X2**2
        x1 = xk[0]
        x2 = xk[1]
        del_fx1 = 1 + 4 * x1 + 2 * x2
        del_fx2 = 2 * x1 + 2 * x2 - 1
        del_f = np.array([del_fx1, del_fx2])
        return del_f

    @staticmethod
    def f(xk, decimal_places=3):
        x1 = xk[0]
        x2 = xk[1]
        fn_value = x1 - x2 + 2 * math.pow(x1, 2) + 2 * x1 * x2 * math.pow(x2, 2)
        return round(fn_value, decimal_places)

    @property
    def __current_gradient_value(self):
        del_f = self.__current_gradient(self.Minimiser)
        # sqaured_sum = functools.reduce(lambda a, b: a ** 2 + b ** 2, del_f)
        return np.sqrt(del_f.dot(del_f))

    @property
    def __current_coordinate_gradient(self):
        del_f = self.__current_gradient(self.Minimiser)
        return del_f[self._current_coordinate_idx - 1]

    def __find_next_candidate(self):
        """Compute the next potential minimiser point, i.e. Xk+1"""
        xk = self.Minimiser
        del_f = self.__current_coordinate_gradient
        xk_plus_1 = xk - self.StepLength * del_f * self._ith_coordinate_vector
        log.debug(f"X**{self.CurrentIteration} = {xk} - {self.StepLength} * {del_f} * {self._ith_coordinate_vector}")
        return xk_plus_1

    def execute(self):
        log.info('Coordinate descent iteration started')
        self.__preview_config()
        for k in range(self.MaxIterations):
            log.debug(f"-----------Iteration {k}--------------")
            self.CurrentIteration = k
            self.Minimiser = self.__find_next_candidate()
            log.debug(f"------Minimiser = {self.Minimiser}. Gradient = {self.__current_gradient(self.Minimiser)} or "
                      f"{self.__current_gradient_value}--------\n")
            if self.__current_gradient_value <= self.ConvergenceThreshold:
                log.warning('Convergence condition satisfied, STOP!')
                break
        log.info(f"------Minimiser = {self.Minimiser}. Gradient = {self.__current_gradient_value}--------")
        log.info(f'Coordinate descent completed successfully. Total Iterations={self.CurrentIteration}')
        return self.Minimiser, self.f(self.Minimiser), round(self.__current_gradient_value, 3)


import heapq
import collections
class Solution:
    def networkDelayTime(self, times: List[List[int]], N: int, K: int) -> int:
        graph=collections.defaultdict(list)
        heap=[]
        heapq.heapify(heap)
        heapq.heappush(heap,[0,K])
        inf=float('inf')
        dist=[inf for _ in range(N+1)]
        dist[K]=0
        dist[0]=0
        for time in times:
            u,v,w=time
            graph[u].append([w,v])
        while len(heap)>0:
            u=heapq.heappop(heap)
            for ele in graph[u[1]]:
                w=ele[0]
                v=ele[1]
                if w+u[0]<dist[v]:
                    dist[v]=w+u[0]
                    heapq.heappush(heap,[w+u[0],v])

        for i in range(1,len(dist)):
            if dist[i]==inf:
                return -1

        return max(dist)


from typing import List

class Solution:
    def minimumAbsDifference(self, arr: List[int]) -> List[List[int]]:
        n = len(arr)
        arr.sort()

        min_diff, ans = float('inf'), []
        for i in range(n - 1):
            delta = arr[i + 1] - arr[i]
            if delta < min_diff:
                min_diff = delta
                ans = [[arr[i], arr[i+1]]]
            elif delta == min_diff:
                ans.append([arr[i], arr[i+1]])

        return ans


#!/usr/bin/env python3
import atexit
import os
import shutil
import click
from langdetect import detect

import pysrt
from smart import fit_subtitles_to_audio
from utils import SCRATCH
import better_exceptions; better_exceptions.hook()

def cleanup():
    shutil.rmtree(SCRATCH)

@click.command(help='Resync the subtitles of a movie file')
@click.argument('movie_file', type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True))
def main(movie_file):
    click.echo('Resyncing subtitles for movie file: {}'.format(movie_file))
    validate(movie_file)
    subtitles: pysrt.SubRipFile = read_subtitles(movie_file)
    language = detect_language(subtitles.text)
    fit_subtitles_to_audio(subtitles, movie_file)

def detect_language(text):
    language = detect(text)
    return language

def validate(movie_file):
    is_movie(movie_file)
    get_subtitle_file(movie_file)

def is_movie(movie_file):
    video_extensions = [".mp4", ".mkv", ".avi", ".flv", ".wmv", ".mov", ".3gp", ".m4v", ".ts"]
    return any(movie_file.endswith(extension) for extension in video_extensions)

def get_subtitle_file(movie_file):
    subtitle_extensions = [".srt"]
    base_file, _ = os.path.splitext(movie_file)
    for extension in subtitle_extensions:
        if os.path.exists(base_file + extension):
            return base_file + extension

def read_subtitles(movie_file):
    return pysrt.open(get_subtitle_file(movie_file))

if __name__ == '__main__':
    atexit.register(cleanup)
    main()


from betl.core import Transformer
from betl.extractors import Extractor
from betl.loaders import Loader

def main(
     extractors: list[Extractor],
     transformer: Transformer,
     loaders: list[Loader]
 ) -> None:
     datasets = {
         extractor.dataset_name: extractor.extract()
         for extractor in extractors
     }
     processed_data = transformer(**datasets)
     for loader in loaders:
         loader.load(processed_data)


class Solution:
    def genNext(self, word):  # KMP算法生成next数组，本题可以用KMP加速但是需要耗费大量的存储空间
        next = [0] * len(word)
        i, j = 1, 0
        while i < len(word):
            if word[i] == word[j]:
                next[i] = j+1
                i += 1
                j += 1
            elif j != 0:
                j = next[j-1]
            else:
                next[j] = 0
                i += 1

        return next

    def exist(self, board, word) -> bool:
        def dfs(i, j, k):
            if not 0 <= i < len(board) or not 0 <= j < len(board[0]) or board[i][j] != word[k]:
                return False
            if k == len(word) - 1:
                return True
            board[i][j] = ''
            res = dfs(i + 1, j, k + 1) or dfs(i - 1, j, k + 1) or dfs(i, j + 1, k + 1) or dfs(i, j - 1, k + 1)
            board[i][j] = word[k]
            return res

        for i in range(len(board)):
            for j in range(len(board[0])):
                if dfs(i, j, 0):
                    return True
        return False

if __name__ == "__main__":
    solution = Solution()
    board = [["A", "B", "C", "E"], ["S", "F", "C", "S"], ["A", "D", "E", "E"]]
    word = "ABCCED"
    print(solution.exist(board, word))


import uuid

from django.core.exceptions import ValidationError
from django.db import models

from sagepaypi.constants import COUNTRY_CHOICES, US_STATE_CHOICES
from sagepaypi.models import CardIdentifier

from tests.test_case import AppTestCase

class TestModel(AppTestCase):

    # fields

    def test_id(self):
        field = self.get_field(CardIdentifier, 'id')
        self.assertModelField(field, models.UUIDField)
        self.assertFalse(field.editable)
        self.assertTrue(field.primary_key)

    def test_created_at(self):
        field = self.get_field(CardIdentifier, 'created_at')
        self.assertModelField(field, models.DateTimeField, blank=True)
        self.assertTrue(field.auto_now_add)

    def test_first_name(self):
        field = self.get_field(CardIdentifier, 'first_name')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 100)

    def test_last_name(self):
        field = self.get_field(CardIdentifier, 'last_name')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 100)

    def test_billing_address_1(self):
        field = self.get_field(CardIdentifier, 'billing_address_1')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 255)

    def test_billing_address_2(self):
        field = self.get_field(CardIdentifier, 'billing_address_2')
        self.assertModelField(field, models.CharField, True, True)
        self.assertEqual(field.max_length, 255)

    def test_billing_city(self):
        field = self.get_field(CardIdentifier, 'billing_city')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 255)

    def test_billing_postal_code(self):
        field = self.get_field(CardIdentifier, 'billing_postal_code')
        self.assertModelField(field, models.CharField, True, True)
        self.assertEqual(field.max_length, 12)

    def test_billing_country(self):
        field = self.get_field(CardIdentifier, 'billing_country')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 2)

    def test_billing_state(self):
        field = self.get_field(CardIdentifier, 'billing_state')
        self.assertModelField(field, models.CharField, True, True)
        self.assertEqual(field.max_length, 2)

    def test_merchant_session_key(self):
        field = self.get_field(CardIdentifier, 'merchant_session_key')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 100)

    def test_card_type(self):
        field = self.get_field(CardIdentifier, 'card_type')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 255)

    def test_last_four_digits(self):
        field = self.get_field(CardIdentifier, 'last_four_digits')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 4)

    def test_expiry_date(self):
        field = self.get_field(CardIdentifier, 'expiry_date')
        self.assertModelField(field, models.CharField)

    def test_card_identifier(self):
        field = self.get_field(CardIdentifier, 'card_identifier')
        self.assertModelField(field, models.CharField)
        self.assertEqual(field.max_length, 100)

    # properties

    def test_str(self):
        pk = uuid.uuid4()
        self.assertEqual(CardIdentifier(pk=pk).__str__(), str(pk))

    def test_billing_address(self):
        card = CardIdentifier(
            billing_address_1='88 The Road',
            billing_address_2='Some Estate',
            billing_postal_code='412',
            billing_city='City',
            billing_country='US',
            billing_state='AL'
        )
        self.assertEqual(
            card.billing_address,
            {
                'address1': card.billing_address_1,
                'address2': card.billing_address_2,
                'city': card.billing_city,
                'country': card.billing_country,
                'postalCode': card.billing_postal_code,
                'state': card.billing_state
            }
        )

    def test_display_text(self):
        self.assertEqual(
            CardIdentifier(card_type='Visa', last_four_digits='1234').display_text,
            'Visa (1234)'
        )

    # validation

    def test_postal_code_required_when_country_not_IE(self):
        card = CardIdentifier(
            first_name='Foo',
            last_name='User',
            billing_address_1='88 The Road',
            billing_city='City',
            billing_country='GB'
        )
        with self.assertRaises(ValidationError) as e:
            card.clean()

        self.assertEqual(e.exception.args[0], {'billing_postal_code': 'This field is required.'})

    def test_postal_code_not_required_when_country_is_IE(self):
        card = CardIdentifier(
            first_name='Foo',
            last_name='User',
            billing_address_1='88 The Road',
            billing_city='City',
            billing_country='IE'
        )
        card.clean()

    def test_state_required_when_country_is_US(self):
        card = CardIdentifier(
            first_name='Foo',
            last_name='User',
            billing_address_1='88 The Road',
            billing_postal_code='412',
            billing_city='City',
            billing_country='US'
        )
        with self.assertRaises(ValidationError) as e:
            card.clean()

        self.assertEqual(e.exception.args[0], {'billing_state': 'This field is required.'})

    def test_state_not_required_when_country_is_not_US(self):
        card = CardIdentifier(
            first_name='Foo',
            last_name='User',
            billing_address_1='88 The Road',
            billing_city='City',
            billing_country='IE'
        )
        card.clean()


from django.conf.urls import url
from mysale import urls
from user import views,views_login,views_pub,views_reply

app_name = 'comments'
urlpatterns = [
    url(r'^test/$',views_pub.test),

    url(r'index/$',views_pub.index),
    url(r'publish/$',views_pub.publish),
    url(r'publish_handler/$',views_pub.publish_handler),

    url(r'reply/$', views_reply.reply),
    # url(r'^reply/post/(?P<publish_pk>[0-9]+)/$', views_reply.post_comment),
    url(r'comment/d(publish_pk)+/$', views_reply.post_comment),

]


import multiprocessing
import time
import os
import argparse
from multiprocessing.managers import SyncManager

from ..mine.classMiner import instMiner
from ..mine.classRedescription import Redescription
from ..mine.toolLog import Log

import pdb
# to clean up: sudo netstat -nlp | grep python

PORT = 55444
AUTHKEY = 'sesame'
MAXK = 4

def sendMessage(output, message, type_message, source):
    output.put({"message": message, "type_message": type_message, "source": source})

class WorkerProcess(multiprocessing.Process):
    def __init__(self, pid, data, preferences, queue_in, result_q, cust_params={}):
        multiprocessing.Process.__init__(self)
        verbosity = preferences.get("verbosity", 0)
        logger = Log("inter", verbosity, output=result_q, method_comm=sendMessage)
        self.miner = instMiner(data, preferences, logger, pid, qin=queue_in, cust_params=cust_params)
        self.cust_params = cust_params
        self.start()

    def run(self):
        pass

class MinerProcess(WorkerProcess):
    def run(self):
        self.miner.full_run(self.cust_params)

class ExpanderProcess(WorkerProcess):
    def run(self):
        self.miner.part_run(self.cust_params)

class ProjectorProcess(multiprocessing.Process):
    def __init__(self, pid, data, preferences, queue_in, result_q, proj={}):
        multiprocessing.Process.__init__(self)
        self.id = pid
        verbosity = preferences.get("verbosity", 0)
        self.logger = Log("inter", verbosity, output=result_q, method_comm=sendMessage)
        if proj is not None:
            self.proj = proj
            self.start()

    def getId(self):
        return self.id

    def stop(self):
        self.proj.stop()
        self.logger.printL(1, self.proj, "result", self.getId())
        self.logger.printL(1, None, "progress", self.getId())
        self.terminate()

    def run(self):
        print("\t--- Projection running...")
        try:
            self.proj.do()
        except Exception as e:
            print("\t--- Projection ERROR!")
            self.proj.clearCoords()
            self.logger.printL(1, "Projection Failed!\n[ %s ]" % e, "error", self.getId())
        finally:
            print("\t--- Projection done")
            self.logger.printL(1, self.proj, "result", self.getId())
            self.logger.printL(1, None, "progress", self.getId())

def make_server_manager(port, authkey):

    job_q = multiprocessing.Queue()
    reconnect_q = multiprocessing.Queue()
    ids_d = dict()

    class JobQueueManager(SyncManager):
        pass

    JobQueueManager.register('get_job_q', callable=lambda: job_q)
    JobQueueManager.register('get_reconnect_q', callable=lambda: reconnect_q)
    JobQueueManager.register('get_ids_d', callable=lambda: ids_d)

    manager = JobQueueManager(address=("", port), authkey=authkey.encode('ascii'))
    manager.start()

    print('Central server started at port %s' % port)
    return manager

class WorkServer(object):

    def __init__(self, port=PORT, authkey=AUTHKEY, max_k=MAXK):
        print("PID", os.getpid())
        # MAKES REDS TO HAVE NEGATIVE IDS SO THEY CAN BE RECOGNIZED HAS EZTERNAL
        Redescription.setUidGen(nv=-1, step=-1, mp_lock=True)
        self.handlers = {}
        self.manager = make_server_manager(port, authkey)
        self.shared_job_q = self.manager.get_job_q()  # queue
        self.shared_ids_d = self.manager.get_ids_d()  # dict
        self.shared_reconnect_q = self.manager.get_reconnect_q()  # queue

        self.max_k = max_k
        self.port = port
        self.authkey = authkey
        self.nextHandlerId = 0

    def run(self):
        if True:
            while True:
                # read tasks from queue
                job = self.getJobsQueue().get()
                if type(job) is dict:
                    if job.get("task") == "startup":
                        # create new handler
                        self.nextHandlerId += 1
                        hid = self.port + self.nextHandlerId
                        self.handlers[hid] = WorkHandler(self, hid, self.authkey, self.max_k)
                        self.shared_ids_d.update({job.get("cid"): hid})
                        # print(type(self.shared_ids_d))
                        print("--- Creating new handler\tHID+%s\t(%s)" % (hid, job.get("cid")))

                    elif job.get("task") == "info":
                        # create new handler
                        hid = self.port + self.nextHandlerId
                        ld_str = self.getLoadStr()
                        self.shared_ids_d.update({job.get("cid"): ld_str})
                        print("--- Sending info\t%s\t(%s)" % (ld_str, job.get("cid")))

                    elif job.get("task") == "reconnect":
                        if job.get("hid") in self.handlers:
                            hid = job.get("hid")
                            wids = [(k, v.get("task"), "pending") for (k, v) in self.handlers[hid].pending.items()]
                            wids += [(k, v.get("task"), "working") for (k, v) in self.handlers[hid].working.items()]
                            wids += [(k, v.get("task"), "retired") for (k, v) in self.handlers[hid].retired.items()]
                            rwids = [w for w in wids if w[1] in WorkHandler.types_reconnect]
                            nwids = [w for w in wids if w[1] not in WorkHandler.types_reconnect]
                            rwids_str = ", ".join([":".join(map(str, w)) for w in rwids])
                            nwids_str = ", ".join([":".join(map(str, w)) for w in nwids])
                            print("--- Reconnecting handler\tHID=%s\tworkers: %s (%s)\t(%s)" % (hid, rwids_str, nwids_str, job.get("cid")))  # details: rwids
                            self.shared_ids_d.update({job.get("cid"): hid})
                            self.shared_reconnect_q.put(rwids, False)
                        else:
                            h_str = ", ".join(["%s" % hh for hh in self.handlers.keys()])
                            print("--- Could not find handler to reconnect\tHID?%s\tamong: %s\t(%s)" % (hid, h_str, job.get("cid")))  # details: rwids

                    # other, handler specific task, forward
                    elif job.get("hid") in self.handlers:
                        print("--- Forwarding task to handler\tHID=%s WID=%s %s" % (job.get("hid"), job.get("wid", "?"), job["task"]))
                        for hdl in self.getHandlers(job.get("hid")):
                            hdl.handleJob(job)

    def getHandlers(self, hid=None):
        if hid == "all":
            return
        if hid in self.handlers:
            return [self.handlers[h] for h in self.handlers.keys()]
        return []

    def getLoadStr(self):
        return " ".join([hd.getLoadStr() for (hdid, hd) in self.handlers.items()])

    def __del__(self):
        hids = list(self.handlers.keys())
        for hid in hids:
            self.handlers[hid].shutdown()
        self.manager.shutdown()

    def unregister(self, hid):
        # unregister handler upon shutdown
        if hid in self.handlers:
            del self.handlers[hid]

    def getJobsQueue(self):
        return self.shared_job_q

    def getIdsDict(self):
        return self.shared_ids_d

def make_hs_manager(port, authkey):
    job_q = multiprocessing.Queue()
    result_q = multiprocessing.Queue()
    reconnect_q = multiprocessing.Queue()

    class HSQueueManager(SyncManager):
        pass

    HSQueueManager.register('get_job_q', callable=lambda: job_q)
    HSQueueManager.register('get_result_q', callable=lambda: result_q)

    manager = HSQueueManager(address=("", port), authkey=authkey.encode('ascii'))
    manager.start()
    print("Work server started at port %s" % port)
    return manager

class WorkHandler(object):

    type_workers = {"mine": {"launch": MinerProcess, "stop": "message"},
                    "expand": {"launch": ExpanderProcess, "stop": "message"},
                    "improve": {"launch": ExpanderProcess, "stop": "message"},
                    "project": {"launch": ProjectorProcess, "stop": "terminate"}}
    types_reconnect = ["mine", "expand", "improve"]

    @classmethod
    def getWorkClassForTask(tcl, task):
        if task in tcl.type_workers:
            return tcl.type_workers[task]["launch"]

    @classmethod
    def getDetForTask(tcl, task):
        if task in tcl.type_workers:
            return tcl.type_workers.get(task)
        return {}

    @classmethod
    def knownTaskWork(tcl, task):
        return task in tcl.type_workers

    def __init__(self, work_server, port, authkey, max_k=MAXK):
        self.manager = make_hs_manager(port, authkey)
        self.shared_result_q = self.manager.get_result_q()  # queue
        self.work_server = work_server
        self.id = port
        self.max_k = max_k
        self.pending = {}
        self.working = {}
        self.retired = {}

    def getId(self):
        return self.id

    def getResultsQueue(self):
        return self.shared_result_q

    def getLoadStr(self):
        return "%d:w%d+p%d+r%d" % (self.getId(), len(self.working), len(self.pending), len(self.retired))

    def handleJob(self, job):
        # retire and layoff are for workers, shutdown for the whole handler
        # if retire: stop work in process, move to retire, and launch any pending job
        if job.get("task") == "retire":
            if job.get("wid") in self.working:
                self.retired[job.get("wid")] = self.working.pop(job.get("wid"))
                self.launchPending()

        # if layoff: remove from pending or stop work in process
        elif job.get("task") == "layoff":
            if job.get("wid") in self.pending:
                self.pending.pop("wid")
            elif job.get("wid") in self.working:
                self.stopJob(job.get("wid"))
                self.retired[job.get("wid")] = self.working.pop(job.get("wid"))
                self.launchPending()

        # shutdown this handler
        elif job.get("task") == "shutdown":
            self.shutdown()

        # other, unknown task, notify, so the worker on client side don't hang waiting
        elif not self.knownTaskWork(job.get("task")):
            self.notifyFatalErrorWid("Fatal: Unknown task for worker!", job.get("wid"))
            return False

        # if acceptable task: launch work if there are free processes, else add job to pending
        # if already in pending or working, comes from attempt to reconnect, do nothing
        elif job.get("wid") not in self.pending and job.get("wid") not in self.working:
            if len(self.working) < self.max_k:
                tmp = self.launchJob(job)
                if tmp is not None:
                    self.working[job.get("wid")] = tmp
            else:
                self.pending[job.get("wid")] = job
        return True

    def notifyFatalErrorWid(self, msg, wid):
        sendMessage(self.getResultsQueue(), msg, "error", "%s-!" % wid)

    def launchPending(self):
        # if pending tasks launch oldest one
        while len(self.working) < self.max_k and len(self.pending) > 0:
            oldest_wid = min(self.pending.keys())
            tmp = self.launchJob(self.pending[oldest_wid])
            if tmp is not None:
                self.working[oldest_wid] = tmp
            self.pending.pop(oldest_wid)

    def launchJob(self, job):
        if self.knownTaskWork(job.get("task")):
            # print("--- Handler launching task\tHID=%s WID=%s %s" % (job.get("hid"), job.get("wid"), job["task"]))
            if self.getDetForTask(job.get("task")).get("stop") == "message":
                queue = multiprocessing.Queue()
            else:
                queue = None
            p = self.getWorkClassForTask(job.get("task"))(job.get("wid"), job.get("data"), job.get("preferences"), queue, self.getResultsQueue(), job.get("more"))
            return {"process": p, "queue": queue, "task": job.get("task")}

    def stopJob(self, wid):
        if self.working[wid]["queue"] is not None:
            self.working[wid]["queue"].put({"message": "stop", "type_message": "progress", "source": "plant"})
        else:
            self.working[wid]["process"].stop()
            # self.working[wid]["process"].terminate()

    # close this handler
    def shutdown(self):
        workers = list(self.working.keys())
        for wid in workers:
            self.stopJob(wid)
            self.working.pop(wid)
        time.sleep(5)
        try:
            del self.working
            del self.pending
            del self.retired
        except AttributeError:
            pass
        finally:
            self.manager.shutdown()
            self.work_server.unregister(self.getId())

# main entry point

def run_server(sargs):

    parser = argparse.ArgumentParser(description='Launch server for off-loading redescription mining computations.')
    parser.add_argument("-p", "--port", type=int, help="address on which the manager process listens for new connections", default=PORT)
    parser.add_argument("-a", "--authkey", type=str, help="authentication key which will be used to check the validity of incoming connections to the server process", default=AUTHKEY)
    parser.add_argument("-m", "--max_k", type=int, help="maximum number of computational tasks handled at once", default=MAXK)
    parser.add_argument("-c", "--chroot", type=str, help="change the root directory of the current process to this path", default=argparse.SUPPRESS)
    parser.add_argument("-u", "--uid", type=int, help="set the current process's user id", default=argparse.SUPPRESS)
    parser.add_argument("-g", "--gid", type=int, help="set the current process's group id", default=argparse.SUPPRESS)

    args = vars(parser.parse_args(sargs[1:]))
    if "uid" in args:
        if "gid" in args:
            # We drop GID before we drop UID
            os.setgid(args.pop("gid"))
        os.setuid(args.pop("uid"))
    else:  # uid not in args
        if "gid" in args:
            os.setgid(args.pop("gid"))
        if "chroot" in args:
            os.chroot(args.pop("chroot"))

    ws = WorkServer(**args)
    try:
        ws.run()
    except KeyboardInterrupt:
        print("Stopped...")
    finally:
        del ws


import math

n1 = float(input("Digite horas no formato hhh.mm: "))
vh = float(input("Digite valor hora: "))

n2 = math.floor(n1)
n3 = (n1 - n2)*100
res = (n3/60)+n2

res1 = res*vh

inss=res1*0.10
fgts=res1*0.11

if (res1<= 1500) and (res1>900):
    desc = 5
    ir=res1*0.05
    liq = res1-inss-ir

if (res1<= 2500) and (res1>1500):
    desc = 10
    ir=res1*0.10
    liq = res1-inss-ir

if (res1> 2500):
    desc = 20
    ir=res1*0.20
    liq = res1-inss-ir

if (res1<= 900):
    desc = 0
    ir="isento"
    liq = res1-inss

tdesc=ir+inss

print ("Salario Bruto    (",n1,"*",vh,"):     R$ {:.2f}".format(res1))
print ("    (-)IR (",desc,"%)       : R$", ir)
print ("    (-)INSS (10%)        : R$ {:.2f}".format(inss))
print ("    FGTS (11%)           : R$ {:.2f}".format(fgts))
print ("    toltal de descontos  : R$ {:.2f}".format(tdesc))
print ("    Salario Liquido      : R$ {:.2f}".format(liq))