"""Reader module contains only dataset loaders
that are used to iterate over items."""
import csv
import gzip
import io
import struct
import zipfile
from collections import OrderedDict
from os import path

import numpy as np

from .var import NP2STRUCT

__all__ = ["Dataset", "Resource", "Container", "READER_LIST"]

class Resource(object):

    """Dataset resource."""

    def __init__(self, attributes, outputs):
        assert isinstance(
            attributes, np.ndarray), "attributes have to be a numpy array"
        assert isinstance(
            outputs, np.ndarray), "outputs have to be a numpy array"
        self.attributes = attributes
        self.outputs = outputs
        self.__idx = 0

    def __len__(self):
        return len(self.attributes)

    def __iter__(self):
        self.__idx = 0
        while self.__idx < len(self.attributes):
            yield (self.attributes[self.__idx], self.outputs[self.__idx])
            self.__idx += 1

    def __getitem__(self, idx):
        return (self.attributes[idx].copy(), self.outputs[idx].copy())

    def __setitem__(self, idx, val):
        self.attributes[idx], self.outputs[idx] = val
        return (self.attributes[idx].copy(), self.outputs[idx].copy())

    def repeat(self, num_rep, target='attributes'):
        """Add a third dimension and duplicate data.

        Example:
            - with 3 repetitions:

                from [ [1, 2], [3, 4] ]

                to [
                    [ [1, 2], [3, 4] ],
                    [ [1, 2], [3, 4] ],
                    [ [1, 2], [3, 4] ]
                ]

        Note:
            np.array([tmp] * num_rep) is more performing than
            np.tile(tmp, (num_rep, 1, 1))

            tmp = np.array([ [1, 2], [3, 4] ])

            %time _ = [np.tile(tmp, (3, 1, 1)) for _ in range(1000000)]
            CPU times: user 6.9 s, sys: 189 ms, total: 7.09 s
            Wall time: 7.14 s

            %time _ = [np.array([tmp]*3) for _ in range(1000000)]
            CPU times: user 3.03 s, sys: 147 ms, total: 3.18 s
            Wall time: 3.19 s
        """
        if target == 'attributes':
            self.attributes = np.array([self.attributes] * num_rep)
        elif target == 'outputs':
            self.outputs = np.array([self.outputs] * num_rep)
        else:
            raise Exception("Unrecognized target '{}'".format(target))
        return self

    def shuffle(self, seed=None):
        """Shuffle resource items."""
        np.random.seed(seed)
        indexes = np.random.permutation(len(self.attributes))
        self.attributes = self.attributes[indexes]
        self.outputs = self.outputs[indexes]
        return self

    def extract(self, indexes):
        """Remove items from this resource."""
        attributes = self.attributes[indexes].copy()
        outputs = self.outputs[indexes].copy()
        self.attributes = np.delete(self.attributes, indexes, 0)
        self.outputs = np.delete(self.outputs, indexes, 0)
        return Resource(attributes, outputs)

    def split(self, num_of_splits):
        """Split the current resource."""
        attributes = np.array_split(self.attributes, num_of_splits)
        outputs = np.array_split(self.outputs, num_of_splits)
        return [Resource(attributes[idx], outputs[idx]) for idx in range(len(attributes))]

    def copy(self):
        """Copy all the items of this resource."""
        return Resource(self.attributes.copy(), self.outputs.copy())

    def convert_type(self, type_):
        """Convert data type."""
        assert type_ == 'double' or type_ == 'float', "Output data can be double or float"
        if type_ == 'double':
            self.attributes = self.attributes.astype(np.float64)
            self.outputs = self.outputs.astype(np.float64)
        elif type_ == 'float':
            self.attributes = self.attributes.astype(np.float32)
            self.outputs = self.outputs.astype(np.float32)

    def to_bin(self):
        """Convert to binary this resource."""
        values = [self.attributes.flatten(), self.outputs.flatten()]
        data = np.concatenate(values, axis=0)
        return struct.pack("{}{}".format(
            data.size,
            NP2STRUCT[self.attributes.dtype]
        ), *data)

    def __add__(self, other):
        if len(self.attributes) > 0:
            attributes = np.concatenate(
                [self.attributes, other.attributes], axis=0)
            outputs = np.concatenate([self.outputs, other.outputs], axis=0)
        else:
            attributes = other.attributes
            outputs = other.outputs
        return Resource(attributes, outputs)

    def __iadd__(self, other):
        if len(self.attributes) > 0:
            self.attributes = np.append(
                self.attributes, other.attributes, axis=0)
            self.outputs = np.append(self.outputs, other.outputs, axis=0)
        else:
            self.attributes = other.attributes
            self.outputs = other.outputs
        return self

class Container(object):

    """Special container for Train, Validation and Test sets."""

    def __init__(self, *args, **kwargs):
        self.resources = []

    def insert(self, source):
        """Insert a resource into this container."""
        if isinstance(source, list):
            self.resources.extend(source)
        else:
            self.resources.append(source)
        return self

    def delete(self, idx):
        """Remove a resource."""
        self.resources.pop(idx)
        return self

    def __iter__(self):
        for resource in self.resources:
            yield resource

    def __len__(self):
        return sum([len(resource) for resource in self.resources])

    def __getitem__(self, idx):
        target_idx = idx
        for resource in self.resources:
            if target_idx < len(resource):
                return resource[target_idx]
            target_idx -= len(resource)
        raise IndexError()

    def __setitem__(self, idx, val):
        target_idx = idx
        for resource in self.resources:
            if target_idx < len(resource):
                resource[target_idx] = val
                return resource[target_idx]
            target_idx -= len(resource)
        raise IndexError()

class Dataset(object):

    """Base dataset object."""

    def __init__(self, normalized=False, onehot=False, kind="reg"):
        assert kind == "cls" or kind == "reg", "kind can be 'cls' (classification) or 'reg' (regression)"
        self.kind = kind
        self.normalized = normalized
        self.onehot = onehot
        self._train = Container()
        self._validation = Container()
        self._test = Container()
        self.data = OrderedDict([
            ('train', self._train),
            ('validation', self._validation),
            ('test', self._test)
        ])

    @property
    def train(self):
        """Train set."""
        return self._train

    @property
    def validation(self):
        """Validation set."""
        return self._validation

    @property
    def test(self):
        """Test set."""
        return self._test

    def insert(self, source, target="train"):
        """Insert a resoure in a proper container."""
        assert isinstance(
            source, Resource), "source have to be a Resource object"
        assert target == "train" or target == "validation" or target == "test", "target can be 'train', 'validation' or 'test'"
        cur_target = self.data[target]
        cur_target.insert(source)

    def __iter__(self):
        for set_ in self.data.values():
            for resource in set_.resources:
                for data in resource:
                    yield data

    def __getitem__(self, idx):
        target_idx = idx
        for set_ in self.data.values():
            for resource in set_.resources:
                if target_idx < len(resource):
                    return resource[target_idx]
                target_idx -= len(resource)
        raise IndexError()

    def __len__(self):
        return sum(
            [len(set_) for set_ in self.data.values()]
        )

class MAGICGammaTelescopeDataset(Dataset):

    """MAGIC Gamma Telescope dataset."""

    def __init__(self, magic_source_folder=None, normalized=True, onehot=True):
        super(MAGICGammaTelescopeDataset, self).__init__(
            normalized=normalized, onehot=onehot, kind='cls')

        data = []
        labels = []

        attributes = [
            'fLength',
            'fWidth',
            'fSize',
            'fConc',
            'fConc1',
            'fAsym',
            'fM3Long',
            'fM3Trans',
            'fAlpha',
            'fDist',
            'class'
        ]

        with open(path.join(magic_source_folder, "magic04.data"), "rb") as csvfile:
            file_wrapper = io.TextIOWrapper(csvfile, newline='')
            reader = csv.DictReader(file_wrapper,
                                    delimiter=',',
                                    fieldnames=attributes
                                    )
            rows = list(reader)

            min_max = check_min_max(rows)

            for row in rows:
                new_data = []
                for attr in attributes[:-1]:
                    tmp = float(row[attr])
                    if normalized:
                        tmp = (
                            tmp - min_max[attr][0]) / (
                            min_max[attr][1] - min_max[attr][0]
                        )
                    new_data.append(tmp)
                data.append(np.array(new_data))

                if onehot:
                    if row['class'] == 'g':
                        labels.append(np.array([0, 1]))
                    else:
                        labels.append(np.array([1, 0]))
                else:
                    labels.append(1 if row['class'] == 'g' else 0)

        data = np.array(data)
        labels = np.array(labels)

        self.insert(Resource(data, labels))

class QSARDataset(Dataset):

    """QSAR dataset."""

    def __init__(self, qsar_source_folder=None, normalized=True, onehot=True):
        super(QSARDataset, self).__init__(
            normalized=normalized, onehot=onehot, kind='cls')

        data = []
        labels = []

        attributes = [
            'SpMax_L',
            'J_Dz',
            'nHM',
            'F01',
            'F04',
            'NssssC',
            'nCb',
            'C',
            'nCp',
            'nO',
            'F03',
            'SdssC',
            'from',
            'LOC',
            'SM6_L',
            'F03',
            'Me',
            'Mi',
            'nN',
            'nArNO2',
            'nCRX3',
            'SpPosA_B',
            'nCIR',
            'B01',
            'B03',
            'N',
            'SpMax_A',
            'Psi_i_1d',
            'B04',
            'SdO',
            'TI2_L',
            'nCrt',
            'C',
            'F02',
            'nHDon',
            'SpMax_B',
            'Psi_i_A',
            'nN',
            'SM6_B',
            'nArCOOR',
            'nX',
            'class'
        ]

        with open(path.join(qsar_source_folder, "biodeg.csv"), "rb") as csvfile:
            file_wrapper = io.TextIOWrapper(csvfile, newline='')
            reader = csv.DictReader(file_wrapper,
                                    delimiter=';',
                                    fieldnames=attributes
                                    )
            rows = list(reader)

            min_max = check_min_max(rows)

            for row in rows:
                new_data = []
                for attr in attributes[:-1]:
                    tmp = float(row[attr])
                    if normalized:
                        tmp = (
                            tmp - min_max[attr][0]) / (
                            min_max[attr][1] - min_max[attr][0]
                        )
                    new_data.append(tmp)
                data.append(np.array(new_data))
                if onehot:
                    if row['class'] == 'RB':
                        labels.append(np.array([0, 1]))
                    else:
                        labels.append(np.array([1, 0]))
                else:
                    labels.append(1 if row['class'] == 'RB' else 0)

        data = np.array(data)
        labels = np.array(labels)

        self.insert(Resource(data, labels))

def check_min_max(rows):
    """Check min and max in a csv table."""
    min_max = {}
    for row in rows:
        for key, value in row.items():
            try:
                tmp = float(value)
                if key not in min_max:
                    min_max[key] = [0, 0]
                if tmp > min_max[key][1]:
                    min_max[key][1] = tmp
                elif tmp < min_max[key][0]:
                    min_max[key][0] = tmp
            except ValueError:
                pass
    return min_max

class GasSensorArrayDriftDataset(Dataset):

    """Gas Sensor Array Drif dataset."""

    def __init__(self, gass_source_file=None, normalized=True, onehot=True):
        super(GasSensorArrayDriftDataset, self).__init__(
            normalized=normalized, onehot=onehot, kind='cls')

        assert gass_source_file, "You have to specify the GasSensorArrayDrift zip file"

        data = []
        labels = []

        mins = np.zeros(128)
        maxs = np.zeros(128)

        with zipfile.ZipFile(gass_source_file, 'r') as gas_zip:

            for batch_file in gas_zip.namelist():
                if batch_file.find(".dat") != -1:
                    cur_batch = gas_zip.read(batch_file).decode("utf-8")
                    for line in cur_batch.split("\n"):
                        tmp = line.strip().split(" ")
                        if len(tmp) > 1:
                            class_ = int(tmp[0]) - 1
                            cur_data = [float(feature.split(":")[1])
                                        for feature in tmp[1:]]
                            class_ = int(tmp[0]) - 1
                            cur_data = [float(feature.split(":")[1])
                                        for feature in tmp[1:]]
                            ##
                            # Min max
                            for idx, val in enumerate(cur_data):
                                if val < mins[idx]:
                                    mins[idx] = val
                                if val > maxs[idx]:
                                    maxs[idx] = val
                            if len(cur_data) != 128:
                                raise Exception(
                                    "Error on extract gas features")
                            data.append(np.array(cur_data))
                            if onehot:
                                labels.append(np.zeros(6))
                                labels[-1][class_] = 1
                            else:
                                labels.append(class_)

                    if normalized:
                        for record in data:
                            for idx, val in enumerate(record):
                                record[idx] = (val - mins[idx]) / \
                                    (maxs[idx] - mins[idx])

                    self.insert(Resource(np.array(data), np.array(labels)))

                    data = []
                    labels = []

class MNISTDataset(Dataset):

    """MNIST dataset."""

    def __init__(self, mnist_source_folder=None, normalized=True, onehot=True):
        super(MNISTDataset, self).__init__(
            normalized=normalized, onehot=onehot, kind='cls')

        assert mnist_source_folder, "You have to specify in which folder are the mnist files"

        train_images = load_mnist_images(
            path.join(mnist_source_folder, 'train-images-idx3-ubyte.gz'), normalized)
        train_labels = load_mnist_labels(
            path.join(mnist_source_folder, 'train-labels-idx1-ubyte.gz'), onehot)
        test_images = load_mnist_images(
            path.join(mnist_source_folder, 't10k-images-idx3-ubyte.gz'), normalized)
        test_labels = load_mnist_labels(
            path.join(mnist_source_folder, 't10k-labels-idx1-ubyte.gz'), onehot)

        self.insert(Resource(train_images, train_labels))
        self.insert(Resource(test_images, test_labels), 'test')

class NBitParity(Dataset):

    """n-bit parity dataset."""

    def __init__(self, nbit=3, nsample_training=5000, nsample_test=1000,  normalized=True, onehot=True):
        super(NBitParity, self).__init__(normalized=normalized, onehot=onehot, kind='cls')

        def tofuzzyclass(i):
             clazz_min = int(np.min(i))
             clazz_max = int(np.max(i))
             clazz = clazz_max - clazz_min + 1
             size = np.size(i)
             out = np.zeros((size,clazz))
             for k in range(0,size):
                 out[k,int(i[k]) - clazz_min]=1.0
             return out

        training_x_set = (np.random.rand(nsample_training,nbit) > 0.5).astype(float)
        training_y_set = (np.sum(training_x_set, axis=1) % 2 != 0).astype(float)

        test_x_set = (np.random.rand(nsample_test,nbit) > 0.5).astype(float)
        test_y_set = (np.sum(test_x_set, axis=1) % 2 != 0).astype(float)

        self.insert(Resource(training_x_set, tofuzzyclass(training_y_set)))
        self.insert(Resource(test_x_set, tofuzzyclass(test_y_set)), 'test')

class FashionMNISTDataset(Dataset):

    """Fashion-MNIST dataset."""

    def __init__(self, fashion_mnist_source_folder=None, normalized=True, onehot=True):
        super(FashionMNISTDataset, self).__init__(
            normalized=normalized, onehot=onehot, kind='cls')

        assert fashion_mnist_source_folder, "You have to specify in which folder are the fashion-mnist files"

        train_images = load_mnist_images(
            path.join(fashion_mnist_source_folder, 'train-images-idx3-ubyte.gz'), normalized)
        train_labels = load_mnist_labels(
            path.join(fashion_mnist_source_folder, 'train-labels-idx1-ubyte.gz'), onehot)
        test_images = load_mnist_images(
            path.join(fashion_mnist_source_folder, 't10k-images-idx3-ubyte.gz'), normalized)
        test_labels = load_mnist_labels(
            path.join(fashion_mnist_source_folder, 't10k-labels-idx1-ubyte.gz'), onehot)

        self.insert(Resource(train_images, train_labels))
        self.insert(Resource(test_images, test_labels), 'test')

def load_mnist_images(file_name, normalized):
    """Load MNIST images."""
    with gzip.open(file_name, 'rb') as mnist_file:
        _, num_images, rows, cols = struct.unpack(
            ">IIII", mnist_file.read(16))
        buf = mnist_file.read(rows * cols * num_images)
        data = np.frombuffer(buf, dtype=np.uint8)
        if normalized:
            data = np.divide(data, 255.)
        data = data.reshape(num_images, rows * cols)
        return data

def load_mnist_labels(file_name, onehot, num_classes=10):
    """Load MNIST image labels."""
    with gzip.open(file_name, 'rb') as mnist_file:
        _, num_labels = struct.unpack(
            ">II", mnist_file.read(8))
        buf = mnist_file.read(num_labels)
        labels = np.frombuffer(buf, dtype=np.uint8)
        if onehot:
            data = np.zeros((num_labels, num_classes))
            data[np.arange(labels.size), labels] = 1
        else:
            data = labels
        return data

READER_LIST = {
    'MNISTDataset': MNISTDataset,
    'FashionMNISTDataset': FashionMNISTDataset,
    'GasSensorArrayDriftDataset': GasSensorArrayDriftDataset,
    'MAGICGammaTelescopeDataset': MAGICGammaTelescopeDataset,
    'QSARDataset': QSARDataset,
    'NBitParity': NBitParity
}


## Loading library

import numpy as np
import pandas as pd
import tqdm

from cdlib import algorithms as al, ensemble as en, evaluation as ev, viz
import networkx as nx

## Defining the deporal descounting function

def temporal_dis(df,dis_cond):

    TempDis_df = df.copy()

    ### Hyperbolic discounting
    if (dis_cond == 'hypb_dis'):
        TempDis_df['Ans_Cnt_Wt'] = 1/(1+TempDis_df['Months_Post'])
        TempDis_df['Ans_Scr_Wt'] = TempDis_df['A_Scr']/(1+TempDis_df['Months_Post'])

    ### exponential discounting
    elif (dis_cond == 'exp_dis'):
        TempDis_df['Ans_Cnt_Wt'] = 1/(np.exp(TempDis_df['Months_Post']))
        TempDis_df['Ans_Scr_Wt'] = TempDis_df['A_Scr']/(np.exp(TempDis_df['Months_Post']))

    return TempDis_df

def month_diff(a, b):
    return 12 * (a.dt.year - b.dt.year) + (a.dt.month - b.dt.month)

## defing the training dataset
def train_dat(year1,year2,post_df):
    AAAI_df = post_df[(post_df['CreationDate'] > year1) & (post_df['CreationDate'] < year2)].reset_index(drop=True)

    arr_slice = AAAI_df[['OwnerUserId']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    AAAI_df['Ttl_Posts'] = counts[unqtags]
    AAAI_df['A_Scr'] = AAAI_df.Score.astype(str).astype(int)

    AAAI_df = AAAI_df[AAAI_df['Ttl_Posts'] > 4].reset_index(drop=True)

    return AAAI_df

#### Defining TAG graph
def Tag_Graph(TagPair_df,Date1,Date2,t_sup):

    df = TagPair_df.loc[(TagPair_df['CreationDate'] >  Date1) &
                         (TagPair_df['CreationDate'] <  Date2)]

    df = df.reset_index(drop=True)

    arr_slice = df[['Tag-Pair']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    df['Tag-Pair_Qcnt'] = counts[unqtags]

    ##### Keeping the TAG-pairs where number of questions are above a threshold
    df = df.loc[(df['Tag-Pair_Qcnt'] >= t_sup)]
    df = df.reset_index(drop=True)

    Ttl_num_ques = df.Id.nunique()
    Sample_num_ques = df.Id.nunique()

    graph_df = df[['Uni_Tag_x','Uni_Tag_y','Tag-Pair_Qcnt']].drop_duplicates(keep='first')
    graph_df.columns = ['source', 'target','weight']
    g = nx.Graph()
    #defining the weighted network
    g.add_weighted_edges_from(graph_df[['source', 'target', 'weight']].values)

    return g,df

def comm_map(g,coms):

    degree_cen = nx.degree_centrality(g)
    btw_cen = nx.betweenness_centrality(g)
    degr  = pd.DataFrame(nx.degree(g))

    # mapping the Tags with identified topics
    com_map=pd.DataFrame(coms.to_node_community_map()).T
    com_map = com_map.rename_axis('User_Id').reset_index()
    com_map.columns = ['Tag', 'Comm']

    degr.columns = ['Tag', 'Deg']
    d_cen = pd.DataFrame(degree_cen.items(), columns=['Tag', 'DegCentrality'])
    b_cen = pd.DataFrame(btw_cen.items(), columns=['Tag', 'BtwCentrality'])

    com_map = pd.merge(com_map,degr, how = 'left',left_on = 'Tag',right_on = 'Tag')
    com_map = pd.merge(com_map,d_cen, how = 'left',left_on = 'Tag',right_on = 'Tag')
    com_map = pd.merge(com_map,b_cen, how = 'left',left_on = 'Tag',right_on = 'Tag')

    com_map = com_map.sort_values(['Comm', 'Deg','BtwCentrality'], ascending=[True, False, False])

    df_com = com_map.copy()

    arr_slice = df_com[['Comm']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    df_com["Ttl_Tags"] = counts[unqtags]

    # getting the final Topics
    df_com["Sr_No"] = df_com.groupby("Comm")["Deg"].rank(method="first", ascending=False)
    df_com = df_com.loc[df_com['Sr_No'] <2].reset_index()

    return com_map,df_com

def commTAG_df(grph_df,train_dat,com_map,com_df):

    graph_df_x = grph_df[['Id','Uni_Tag_x']].drop_duplicates()
    graph_df_x.columns = ['Q_Id','Tags']
    graph_df_y = grph_df[['Id','Uni_Tag_y']].drop_duplicates()
    graph_df_y.columns = ['Q_Id','Tags']

    TAG_Qdf = graph_df_x.append(graph_df_y).drop_duplicates().reset_index(drop=True)

    arr_slice = TAG_Qdf[['Q_Id']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    TAG_Qdf['N_Tags'] = counts[unqtags]

    comm_TAG_df = pd.merge(TAG_Qdf,com_map.iloc[:,[0,1]],how = 'left', left_on='Tags',right_on = 'Tag')
    comm_TAG_df = comm_TAG_df.drop('Tag', 1)
    comm_TAG_df = pd.merge(comm_TAG_df,com_df.iloc[:,[1,2]],how = 'left', left_on='Comm',right_on = 'Comm')

    comm_Q_df = comm_TAG_df.filter(items=['Q_Id','N_Tags','Tag'])
    comm_Q_df.columns = ['Q_Id','N_Tags','Tags']
    commTag_Q_df = comm_Q_df.value_counts(['Q_Id','N_Tags','Tags']).reset_index(name='N_CTag')

    TAG_Adf = train_dat[(train_dat['PostTypeId'] == '2') & (train_dat['A_Scr'] > 0)].reset_index(drop=True)
    TAG_Adf = TAG_Adf.filter(items=['Id', 'CreationDate','A_Scr','OwnerUserId','ParentId'])

    TAG_QnAdf = pd.merge(TAG_Adf,commTag_Q_df,how = 'left',left_on='ParentId',right_on='Q_Id')
    TAG_QnAdf = TAG_QnAdf[-TAG_QnAdf.Q_Id.isnull()]

    TAG_QnAdf['wt_Cnt'] = TAG_QnAdf['N_CTag']/TAG_QnAdf['N_Tags']
    TAG_QnAdf['wt_A_Scr'] = TAG_QnAdf['A_Scr']*TAG_QnAdf['wt_Cnt']

    return TAG_QnAdf

### Returns User activity - temporal discounted on each topic

def comm_Udf(Tag_df,present_dt,temp_dis_type):

    TAG_Udf = Tag_df.copy()
    TAG_Udf['U_Tag'] = TAG_Udf['OwnerUserId'] + TAG_Udf['Tags']

    arr_slice = TAG_Udf[['U_Tag']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    TAG_Udf['Ttl_Ans'] = counts[unqtags]
    TAG_Udf['wt_Ttl_Ans'] = TAG_Udf['wt_Cnt'].groupby(TAG_Udf['U_Tag']).transform('sum')

    TAG_Udf['ref_date'] = pd.Timestamp(present_dt)
    TAG_Udf['Months_Post'] = month_diff(TAG_Udf['ref_date'], TAG_Udf['CreationDate'])
    TAG_Udf['Days_Ans'] = (TAG_Udf['ref_date'] -  TAG_Udf['CreationDate']).dt.days#.astype('timedelta64[h]')

    TAG_U_TempD_df = temporal_dis(TAG_Udf,temp_dis_type)

    TAG_U_TempD_df['AC_Wt'] = TAG_U_TempD_df['Ans_Cnt_Wt'].groupby(TAG_U_TempD_df['U_Tag']).transform('sum')
    TAG_U_TempD_df['wt_Ans_Cnt_Wt'] = TAG_U_TempD_df['wt_Cnt']/(1+TAG_U_TempD_df['Months_Post'])

    ## Temporal weighted score for each user on the Topic
    TAG_U_TempD_df['wt_AC_Wt'] = TAG_U_TempD_df['wt_Ans_Cnt_Wt'].groupby(TAG_U_TempD_df['U_Tag']).transform('sum')

    TAG_df = TAG_U_TempD_df.filter(items=['OwnerUserId','Tags','Ttl_Ans','AC_Wt',
                                  'wt_Ttl_Ans','wt_AC_Wt']).drop_duplicates()

    return TAG_df

from surprise import SVD
from surprise import NMF
from surprise import accuracy
from surprise import Dataset
from surprise.model_selection import train_test_split
from surprise import Reader#, evaluate#, print_perf

def NMF_Model(train_df,criteria,nfactors):

    TAG_df = train_df.filter(items=['OwnerUserId','Tags',criteria])
    TAG_df.columns = ['OwnerUserId','Tags','Rating']
    x, y = TAG_df.Rating.min(), TAG_df.Rating.max()

    # A reader is still needed but only the rating_scale param is requiered.
    reader = Reader(rating_scale=(x, y))

    # The columns must correspond to user id, item id and ratings (in that order).
    data = Dataset.load_from_df(TAG_df[['OwnerUserId','Tags','Rating']], reader)

    algo = NMF(n_factors =nfactors, reg_pu=0.01,reg_qi=0.01)

    # retrain on the whole set
    trainset = data.build_full_trainset()
    algo.fit(trainset)

    TAG_df['AU_Id'] = TAG_df['OwnerUserId'].apply(lambda x: trainset.to_inner_uid(x))
    TAG_df['Tag_Id'] = TAG_df['Tags'].apply(lambda x: trainset.to_inner_iid(x))

    prediction = pd.DataFrame(np.dot(algo.pu, algo.qi.T))

    return TAG_df,trainset,prediction

def comm_testDf(start,end,train_df,com_map,df_com,post_dat):

    test_df = post_dat[(post_dat['CreationDate'] > start) & (post_dat['CreationDate'] < end)].reset_index(drop=True)

    testQdf = test_df[test_df['PostTypeId'] == '1'].reset_index(drop=True)
    testQdf = testQdf[testQdf['AcceptedAnswerId'].notnull()]
    testQdf = testQdf.filter(items=['Id','Tags','AcceptedAnswerId'])

    testQdf['N_Tags'] = testQdf.Tags.str.count("<")
    testQdf.Tags = testQdf.Tags.apply(lambda x: x.replace('<',''))
    testQdf = testQdf.drop('Tags', axis=1).join(testQdf['Tags'].str.split('>', expand=True).stack().reset_index(level=1, drop=True).rename('Tags'))
    testQdf = testQdf.loc[testQdf['Tags'] != '']

    comm_test_df = pd.merge(testQdf,com_map.iloc[:,[0,1]],how = 'left', left_on='Tags',right_on = 'Tag')
    comm_test_df = comm_test_df.drop('Tag', 1)
    comm_test_df = pd.merge(comm_test_df,df_com.iloc[:,[1,2]],how = 'left', left_on='Comm',right_on = 'Comm')
    comm_test_df = comm_test_df[comm_test_df['Tag'].notnull()]
    comm_test_df = comm_test_df[comm_test_df.Tags.isin(train_df.Tags)]
    comm_test_df = comm_test_df.value_counts(['Id','AcceptedAnswerId','Tag','N_Tags']).reset_index(name='N_CTag')
    comm_test_df.columns = ['Id','AcceptedAnswerId','Tags','N_Tags','N_CTag']
    comm_test_df['CTag_wt'] = comm_test_df['N_CTag']/comm_test_df['N_Tags']

    testAdf = test_df[test_df['PostTypeId'] == '2'].reset_index(drop=True)
    testAAdf = testAdf[testAdf['Id'].isin(testQdf.AcceptedAnswerId)]
    testAAdf = testAAdf[testAAdf['OwnerUserId'] != '']
    testAAdf = testAAdf[testAAdf['OwnerUserId'].isin(train_df.OwnerUserId)]
    testAAdf = testAAdf.filter(items=['Id','OwnerUserId','ParentId'])
    testAAdf.columns = ['A_Id','OwnerUserId','ParentId']

    testAAdf_U = pd.merge(comm_test_df,testAAdf,how='left',left_on='AcceptedAnswerId',right_on='A_Id')
    testAAdf_U = testAAdf_U[testAAdf_U.A_Id.notnull()]

    return testAAdf_U

#####

def commMF_eval_MatrixApp(test_df,train_df,trainset,prediction):

    tag_df = test_df.copy()
    tag_df['Tags_Id'] = tag_df['Tags'].apply(lambda x: trainset.to_inner_iid(x))
    tag_df['AU_Id'] = tag_df['OwnerUserId'].apply(lambda x: trainset.to_inner_uid(x))
    tag_df['AU_Ind'] = 1

    pivt_Usr_tag_df = tag_df[['Id','AU_Id','AU_Ind']]

    all_Users = pd.DataFrame(prediction.index.values)
    all_Users['Id'] = -9
    all_Users['AU_Ind'] = 0
    all_Users.columns = ['AU_Id','Id','AU_Ind']

    pivt_allUsr_tag_df = pivt_Usr_tag_df.append(all_Users)

    Usr_Ind = pd.pivot_table(pivt_allUsr_tag_df,values = 'AU_Ind', index=['Id'], columns = 'AU_Id',fill_value=0)
    Usr_Ind = Usr_Ind.iloc[1:,:]

    pivt_Topic_df = tag_df[['Id','Tags_Id','CTag_wt']]

    all_Topics = pd.DataFrame(prediction.columns.values)
    all_Topics['Id'] = -9
    all_Topics['CTag_wt'] = 0
    all_Topics.columns = ['Tags_Id','Id','CTag_wt']

    pivt_allTopic_df = pivt_Topic_df.append(all_Topics)

    Q_Topic_mat = pd.pivot_table(pivt_allTopic_df, values = 'CTag_wt', index=['Id'], columns = 'Tags_Id').fillna(0)
    Q_Topic_mat = Q_Topic_mat.iloc[1:,:]
    Q_U_mat = Q_Topic_mat.dot(prediction.T)
    Q_U_rank_mat = Q_U_mat.rank(axis = 1, ascending = False).astype(int)

    Q_U_rank = Q_U_rank_mat.mul(Usr_Ind)
    Q_U_rank["Rank"] = Q_U_rank.sum(axis=1)

    Q_U_rank['wt_RR'] = 1/Q_U_rank['Rank']
    Q_U_rank['wt_Prec_at_5'] = np.where(Q_U_rank['Rank'] < 6, 1, 0) #### Change for Precision measure at any n
    Q_U_rank['wt_Prec_at_10'] = np.where(Q_U_rank['Rank'] < 11, 1, 0) #### Change for Precision measure at any n

    wt_MRR = round(Q_U_rank['wt_RR'].mean(),4)
    wt_Prec = round(Q_U_rank['wt_Prec_at_5'].mean(),4)
    wt_HIT = round(Q_U_rank['wt_Prec_at_10'].mean(),4)

    return wt_MRR, wt_Prec, wt_HIT

#### MAIN

t = input("Enter the CQA folder downloaded from SE archieve (example-askubuntu.com)")

print("StackExchange Community:",t)

### Getting the post history data
post_dat = pd.read_feather('PreprocessedData/{}_Posts.ft'.format(t))
post_dat['CreationDate'] =  pd.to_datetime(post_dat['CreationDate'], format='%Y-%m-%dT%H:%M:%S.%f')
post_dat['Post_Year'] = post_dat['CreationDate'].dt.year

### Getting the TAG graph data
TagCnt_Pair = pd.read_feather('PreprocessedData/{}_TagCnt_Pair.ft'.format(t))
TagCnt_Pair['CreationDate'] =  pd.to_datetime(TagCnt_Pair['CreationDate'], format='%Y-%m-%dT%H:%M:%S.%f')

##### GRAPH TIME PERIOD
### enter the graph start date
G_date1 = '2015-01-01'

start_date = input("Enter start date (example-'2015-01-01')")
end_date = input("Enter end date (example-'2019-05-01')")

q_dat = post_dat[-post_dat.AcceptedAnswerId.isna()].reset_index(drop=True)
q_dat = q_dat[(q_dat['CreationDate'] > start_date) & (q_dat['CreationDate'] < end_date)].reset_index(drop=True)
q_dat = q_dat[['Id','CreationDate','OwnerUserId','Tags','AcceptedAnswerId']]
q_dat.columns= ['Q_Id','Q_Date','Q_UserId','Tags','AcceptedAnswerId']
### date split - Train, Test data
date_split = q_dat.Q_Date.quantile(0.8)
print('date split - Train, Test data:',date_split)

### enter the temporal discouting type
temp_dis_typ = 'hypb_dis'

train_df = train_dat(start_date,date_split,post_dat)

##### Minimum edge weight for defining TAG graph
t_sup = 5

g,graph_df = Tag_Graph(TagCnt_Pair,G_date1,date_split,t_sup)
coms = al.louvain(g)
CD_algo = 'louvain'
com_map_dat,dat_com = comm_map(g,coms)

# TOPIC MODEL DATA
comm_model_df = commTAG_df(graph_df,train_df,com_map_dat,dat_com)
comm_model_Udf = comm_Udf(comm_model_df,date_split,temp_dis_typ)

TopCommMF_n_users = comm_model_Udf.OwnerUserId.nunique()
TopCommMF_n_tags = comm_model_Udf.Tags.nunique()
TopCommMF_n_ratings = comm_model_Udf.shape[0]

nfac = 10 ## NUMBER OF LATENT FACTORS for Matrix Factorisation

#### CRITERIA :
################ 'wt_Ttl_Ans', 'wt_AC_Wt'

temporal_ind = input("Input 1 for temporal discounting and 0 for ranking without temporal discounts")
if temporal_ind == 1:
    crt = 'wt_AC_Wt'
else:
    crt = 'wt_Ttl_Ans'

##### Matrix factorisation
comm_Model_Tag_df,comm_trainset,comm_prediction = NMF_Model(comm_model_Udf,crt,nfac)

### Test data and evaluation
comm_eval_df = comm_testDf(date_split,end_date,comm_Model_Tag_df,com_map_dat,dat_com,post_dat)
Topic_MRR, Topic_Prec5, Topic_Prec10 = commMF_eval_MatrixApp(comm_eval_df,comm_Model_Tag_df,comm_trainset,comm_prediction)

print('MRR:',Topic_MRR,
     'Prec@5:',Topic_Prec5,
     'Prec@10:',Topic_Prec10)


from flask import Blueprint, request, make_response, current_app
from werkzeug.exceptions import BadRequest

from src.transactions.models import Transaction
from src.transactions.parser import parse_storage

import json

def create_blueprint():
    bp = Blueprint('transactions', __name__, url_prefix='/transactions')
    return bp

mod_transactions = create_blueprint()

@mod_transactions.route('/load', methods=['POST'])
def load():
    def get_transactions():
        with open('./transactions.json', 'r') as file:
            return json.loads(file.read())

    try:
        jdata = request.get_json()
        transactions = jdata['transactions']
    except (KeyError, BadRequest) as e:
        # we load the default data if the transactions were not provided.
        transactions = get_transactions()

    current_app.storage = parse_storage(transactions)

    return make_response('OK'), 200

@mod_transactions.route('/get_sequence', methods=['POST'])
def get_sequence():
    jdata = request.get_json()
    try:
        jtransaction = jdata['transaction']
    except KeyError:
        raise BadRequest("The Transaction was not provided")

    try:
        storage = current_app.storage
    except AttributeError:
        raise BadRequest("The Storage was not loaded")

    transaction = Transaction(**jtransaction)
    sequence = storage.get_sequence(transaction)

    response = make_response(str(sequence))
    response.mimetype = 'application/json'
    return response, 200


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# %pip install pandas
# %pip install gingerit
# %pip install regex
# %pip install openpyxl
#pip install Flask

import pandas as pd
import regex as re
from gingerit.gingerit import GingerIt

def remove_emoticons_hashtags_tags(text):
    # Remove emoticons
    text = re.sub(r'[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F700-\U0001F77F\U0001F800-\U0001F8FF\U0001F900-\U0001F9FF\U0001FA00-\U0001FA6F\U0001FA70-\U0001FAFF\U00002702-\U000027B0\U000024C2-\U0001F251]+', '', text)

    # Remove hashtags and @tags
#     text = re.sub(r'[@#]\w+', '', text)
# The pattern [#@]\S+\s will find words that start with @ or # and continue until a space is found in the given text.
#  It will capture the entire word along with any special characters until the space is reached.
    text +=" "
    text = re.sub(r'[#@]\S+\s', '', text)

    # Remove anything inside parentheses
    text = re.sub(r'\([^)]*\)', '', text)

    # Remove any trailing punctuations
    text = text.replace("@","").replace("#","")

    return text.strip()

def check_correction_type(correction):
    excluded_correction_types = ['Accept space','Accept comma addition','None']
    for correction in ginger_cap['corrections']:
        print("\t ------------------",correction['definition'])
        if correction['definition'] in excluded_correction_types:
            return False
    return True

def fix_my_cap(text):
    parser = GingerIt()
    return parser.parse(remove_emoticons_hashtags_tags(text))

if __name__ == "__main__":
    text= """Gamers or hustlers. Corporate warriors or students. Whatever be their trip, what trips them up is a laptop that just cannot keep pace. #NewWork for @reliance_digital as part of “The Hunt for India’s Slowest Laptop” campaign by Team 21N.

Credits:
Brand: @reliance_digital
Client: Manoj Jain, Nisha Kallianpur, Anamika Tiwari, Esha Nath
Production House: @firsttakefilms
Direction: @sagnik6285
Creative Team: @bingeljell, @inde.stagram, @_kashishkakkar_
Account Management: @neerajkarkera , @divya_jain_dj, @aarsha_rathi
Founder & CEO: @the.malayali"""

    result = fix_my_cap(text)
    print(result)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import
import re

from django import forms
from django.core import validators
from django.db import IntegrityError
from django.forms import ModelForm

from crispy_forms.bootstrap import StrictButton
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Div, Field

from .models import Quote

class QuoteForm(ModelForm):
    """
    Form for sending a new message
    """
    message = forms.CharField(label='Desahógate aquí',
                              max_length=140,
                              initial='Me caga ')

    class Meta:
        model = Quote
        fields = ['message',]

    def __init__(self, *args, **kwargs):
        super(QuoteForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_tag = False
        self.helper.help_text_inline = True
        self.helper.html5_required = True
        self.helper.layout = Layout(
            Field('message', style="width: 680px;"),
            StrictButton('Enviar', type="submit", name="submit-btn", css_class="btn-primary")
        )

    def clean_message(self):
        """
        Checks for a valid message
        """
        quote = self.cleaned_data['message']
        if not quote:
            raise forms.ValidationError("Me caga que intentes enviar un mensaje vacío.")
        # check for quotes without the primary words
        phrase = re.compile(r'me\s+caga', re.I)
        if not phrase.match(quote):
            raise forms.ValidationError("Mensaje inválido. Sólo dinos qué te caga.")
        # check for short or empty quote
        if quote.lower() == 'me caga' or len(quote) <= 7:
            raise forms.ValidationError("Tu mensaje es demasiado corto.")
        # check for long quotes
        if len(quote) > 140:
            raise forms.ValidationError("Tu mensaje esta demasiado largo. Dinos qué te caga en 140 caracteres.")
        # check for urls
        if re.search(r'(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?', quote):
            raise forms.ValidationError("A mi me caga que pongas URLs en tu mensaje.")
        # check for emails
        if re.search(r'([a-z0-9_\.-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})', quote):
            raise forms.ValidationError('Me caga que pongas emails en tu mensaje.')
        return self.cleaned_data['message']


""""
pip install -r requirements.txt
to run file
set FLASK_APP=app.py
flask run
"""

from flask import Flask, render_template, request
from PIL import Image

import pytesseract

UPLOAD_FOLDER = '/static/uploads/'

# allow files of a specific type
ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg'])

def read_img(img):
    # pytesseract.pytesseract.tesseract_cmd = '/app/.apt/usr/bin/tesseract'
    # /app/.apt/usr/bin/tesseract
    pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
    text = pytesseract.image_to_string(Image.open(img))
    return(text)

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

app = Flask(__name__, template_folder='templates')

@app.route('/', methods=['GET', 'POST'])
def main():
    if request.method == 'GET':
        return render_template("index.html")
    if request.method == 'POST':
        if 'file' not in request.files:
            return render_template('index.html', msg='1')
        file = request.files['file']
        # if no file is selected
        if file.filename == '':
            return render_template('index.html', msg='2')

        if file and allowed_file(file.filename):

            # call the OCR function on it
            extracted_text = read_img(file)

            # extract the text and display it
            return render_template('index.html',
                                   msg='true',
                                   extracted_text=extracted_text)

if __name__ == '__main__':
    app.run(debug=True)


from bs4 import BeautifulSoup
import requests
from selenium import webdriver
import time

def get_list_of_songs(url):
    options = webdriver.ChromeOptions()
    driver = webdriver.Chrome(options=options)
    driver.get(url)
    time.sleep(5) # wait a second for <div id="root"> to be fully loaded
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    driver.quit()
    music_links = soup.findAll("div", {"class": "col1"})
    song_titles = []
    for link in music_links:
        song_title = link.findChild("music-link")['title']
        song_titles.append(song_title)
    return song_titles

# playlist_url = "https://music.amazon.com/playlists/B07WV4DNDR"
# get_list_of_songs(playlist_url)


import random

names_string=input("Please type friends names seperated by comma: ")
names=names_string.split(", ")

#we can also use this way to understand deeply how it was created
#number_names=len(names)
#random_choice=random.randint(0,number_names-1)
#who_will_pay=names[random_choice]

who_will_pay=random.choice(names)
print(who_will_pay + " is gonna pay the meal today")


import math

import torch
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
#from  attention_is_all_you_need_pytorch_master.transformer.Modules import ScaledDotProductAttention
#from attention_is_all_you_need_pytorch_master.transformer.SubLayers import MultiHeadAttention
torch.set_printoptions(sci_mode=False)

class AttFeatTrans(nn.Module):
    def __init__(self, config, name):
        super(AttFeatTrans,self).__init__()
        self.config = config
        self.name = name
        self.head = config.num_head
        # some setting
        self.in_feat_dim = config.in_feat_dim
        self.feat_dim = config.feat_dim  # don't know where it is manifested since then
        self.query = nn.Linear(self.in_feat_dim, self.in_feat_dim, bias=config.qk_have_bias)
        self.key = nn.Linear(self.in_feat_dim, self.in_feat_dim, bias=config.qk_have_bias)
        self.value = nn.Linear(self.in_feat_dim, self.in_feat_dim, bias = config.v_has_bias)
        #self.out_trans = MultiHeadAttention(self.head, self.in_feat_dim * self.head, self.in_feat_dim, self.in_feat_dim, dropout=0.1) #some dim in it
        self.att_dropout = nn.Dropout(config.attention_probs_dropout_prob)
        self.feat_dim_onehead = self.in_feat_dim // self.head

        self.intermediate = config.act_fun #gelu
        self.tie_qk_scheme = config.tie_qk_scheme  #not use it first
        print("{} in_feat_dim: {}, feat_dim: {}, qk_have_bias: {}".format(
        self.name, self.in_feat_dim, self.feat_dim, config.qk_have_bias))

    def tie_qk(self, tie_qk_scheme=None):
        # override config.tie_qk_scheme
        if tie_qk_scheme is not None:
            self.tie_qk_scheme = tie_qk_scheme

        print("Initialize QK scheme: {}".format(self.tie_qk_scheme))
        if self.tie_qk_scheme == 'shared':
            self.key.weight = self.query.weight
            if self.key.bias is not None:
                self.key.bias = self.query.bias

        elif self.tie_qk_scheme == 'loose':
            self.key.weight.data.copy_(self.query.weight)
            if self.key.bias is not None:
                self.key.bias.data.copy_(self.query.bias)

    def transpose_for_scores(self, x):
        x_new_shape = x.size()[:-1] + (self.head, -1)
        x = x.view(*x_new_shape)
        return x.permute(0, 2, 1, 3)

    def forward(self, in_query, in_key=None, pos_biases=None): #[50,256,768] [50,1,768]
        if in_key is None:
            in_key = in_query

        mixed_query_feat = self.query(in_query) #[50,256,768]
        mixed_key_feat = self.key(in_key)       #[50,1,768]
        v_first_feat = self.value(in_key)       #[50,1,768]#q,v will same?  infact the dim of qv not same
        '''
        n_head = self.head
        B = in_query.shape[0]
        D = in_query.shape[2]
        q = mixed_query_feat.view(B, n_head , -1 , D)
        k = mixed_key_feat.view(B, n_head, -1, D)
        v = mixed_value_feat.view(B, n_head, -1, D)
        '''
        q = self.transpose_for_scores(mixed_query_feat) #[50 2 256 384]
        k = self.transpose_for_scores(mixed_key_feat)   #[50 2 1 384]
        att_score = torch.matmul(q,k.transpose(-1, -2))
        #output = self.out_trans(mixed_query_feat,mixed_key_feat,mixed_value_feat)
        att_score = att_score / math.sqrt(q.shape[3])  #[50 2 256 1]
        attention_probs = F.softmax(att_score, dim=-1)
        attention_probs = self.att_dropout(attention_probs) #[50 2 256 1]

        # about v(f)
        v_first_feat = v_first_feat.permute(0, 2, 1)  #[50 786 1]
        shape_4d = (v_first_feat.shape[0], self.head, self.feat_dim_onehead, v_first_feat.shape[2]) # [50 2 384 1]
        v_first_feat_4d = v_first_feat.view(shape_4d).permute([0, 1, 3, 2]) #[50 2 1 384]
        v_first_feat_fusion = torch.matmul(attention_probs, v_first_feat_4d) #[50, 2, 256, 384]
        #v_first_feat_fusion_3d = v_first_feat_fusion.permute([0, 1, 3, 2]).reshape(v_first_feat.shape)
        v_first_feat_fusion_3d = v_first_feat_fusion.permute([0, 1, 3, 2]).reshape(v_first_feat.shape[0],v_first_feat.shape[1], -1) # 50 768 256
        v_first_feat = v_first_feat_fusion_3d.permute([0,2,1]) #50 256 768

        v_mid_feat = self.intermediate(v_first_feat) #gelu #50 256 768
        v_last_feat = v_mid_feat + v_first_feat #res #50 256 768
        v_trans_feat = v_last_feat  #50 256 768
        trans_feat = v_trans_feat.squeeze(1) #50 256 768
        return trans_feat

class InitWeights(nn.Module):
    def __init__(self, config, *inputs, **kwargs):
        super(InitWeights, self).__init__()
        self.config = config
        self.act_fun = F.gelu

    def init_weights(self, module):
            if isinstance(module, (nn.Linear, nn.Embedding)):
                if (np.array(module.weight.shape) < self.config.min_feat_dim).all():
                    print("Skip init of Linear weight %s" % (list(module.weight.shape)))
                else:
                    base_initializer_range = self.config.base_initializer_range
                    module.weight.data.normal_(mean=0.0, std=base_initializer_range)
                # Slightly different from the TF version which uses truncated_normal
                # for initialization cf https://github.com/pytorch/pytorch/pull/5617
            if isinstance(module, nn.Linear) and module.bias is not None:
                module.bias.data.zero_()

    def tie_qk(self, module):
            if isinstance(module, AttFeatTrans) and module.tie_qk_scheme != 'none':
                module.tie_qk()
    '''
        def add_identity_bias(self, module):
            if isinstance(module, AttFeatTrans) or isinstance(module, ExpandedFeatTrans):
                module.add_identity_bias()
     '''


import torch.nn as nn

class CompletionNetwork(nn.Module):

    def __init__(self):
        super(CompletionNetwork, self).__init__()
        # input shape: (None, 4, height, width), with mask
        # output (input_channel, output_channel)
        self.conv1 = nn.Conv2d(4, 64, kernel_size=5, stride=1, padding=2)
        self.bn1 = nn.BatchNorm2d(64)
        self.act1 = nn.ReLU()

        self.conv2 = nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1)
        self.bn2 = nn.BatchNorm2d(128)
        self.act2 = nn.ReLU()

        self.conv3 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
        self.bn3 = nn.BatchNorm2d(128)
        self.act3 = nn.ReLU()

        self.conv4 = nn.Conv2d(128, 256, kernel_size=3, stride=2, padding=1)
        self.bn4 = nn.BatchNorm2d(256)
        self.act4 = nn.ReLU()

        self.conv5 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.bn5 = nn.BatchNorm2d(256)
        self.act5 = nn.ReLU()

        self.conv6 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.bn6 = nn.BatchNorm2d(256)
        self.act6 = nn.ReLU()

        self.conv7 = nn.Conv2d(256, 256, kernel_size=3, stride=1, dilation=2, padding=2)
        self.bn7 = nn.BatchNorm2d(256)
        self.act7 = nn.ReLU()

        self.conv8 = nn.Conv2d(256, 256, kernel_size=3, stride=1, dilation=4, padding=4)
        self.bn8 = nn.BatchNorm2d(256)
        self.act8 = nn.ReLU()

        self.conv9 = nn.Conv2d(256, 256, kernel_size=3, stride=1, dilation=8, padding=8)
        self.bn9 = nn.BatchNorm2d(256)
        self.act9 = nn.ReLU()

        self.conv10 = nn.Conv2d(256, 256, kernel_size=3, stride=1, dilation=16, padding=16)
        self.bn10 = nn.BatchNorm2d(256)
        self.act10 = nn.ReLU()

        self.conv11 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.bn11 = nn.BatchNorm2d(256)
        self.act11 = nn.ReLU()

        self.conv12 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.bn12 = nn.BatchNorm2d(256)
        self.act12 = nn.ReLU()

        # deconv
        self.deconv13 = nn.ConvTranspose2d(256, 128, kernel_size=4, stride=2, padding=1)
        self.bn13 = nn.BatchNorm2d(128)
        self.act13 = nn.ReLU()

        self.conv14 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
        self.bn14 = nn.BatchNorm2d(128)
        self.act14 = nn.ReLU()

        self.deconv15 = nn.ConvTranspose2d(128, 64, kernel_size=4, stride=2, padding=1)
        self.bn15 = nn.BatchNorm2d(64)
        self.act15 = nn.ReLU()

        self.conv16 = nn.Conv2d(64, 32, kernel_size=3, stride=1, padding=1)
        self.bn16 = nn.BatchNorm2d(32)
        self.act16 = nn.ReLU()

        # output layer (None, 3, height. width)
        self.conv17 = nn.Conv2d(32, 3, kernel_size=3, stride=1, padding=1)
        self.act17 = nn.Sigmoid()

    def forward(self, x):
        x = self.bn1(self.act1(self.conv1(x)))
        x = self.bn2(self.act2(self.conv2(x)))
        x = self.bn3(self.act3(self.conv3(x)))
        x = self.bn4(self.act4(self.conv4(x)))
        x = self.bn5(self.act5(self.conv5(x)))
        x = self.bn6(self.act6(self.conv6(x)))
        x = self.bn7(self.act7(self.conv7(x)))
        x = self.bn8(self.act8(self.conv8(x)))
        x = self.bn9(self.act9(self.conv9(x)))
        x = self.bn10(self.act10(self.conv10(x)))
        x = self.bn11(self.act11(self.conv11(x)))
        x = self.bn12(self.act12(self.conv12(x)))
        x = self.bn13(self.act13(self.deconv13(x)))
        x = self.bn14(self.act14(self.conv14(x)))
        x = self.bn15(self.act15(self.deconv15(x)))
        x = self.bn16(self.act16(self.conv16(x)))
        x = self.act17(self.conv17(x))
        return x

if __name__ == '__main__':
    net = CompletionNetwork()
    print(net)


import sys

import os
from django.shortcuts import render, redirect
from .models import Image, Theme
from .forms import ImageForm, ThemeForm

# Create your views here.
def index(request):
    return render(request, 'settings_index.html', {})

def themes_index(request):
    themes = Theme.objects.all().order_by('name')
    return render(request, 'themes_index.html', {'themes': themes})

def themes_add(request):
    if not request.method == "POST":
        form = ThemeForm()
        return render(request, 'themes_add.html', {'form': form})
    else:
        form = ThemeForm(request.POST)
        if form.is_valid():
            i = Theme()
            i.name = form.cleaned_data['name']
            i.description = form.cleaned_data['description']
            i.save()
            return redirect('/settings/themes')
        else:
            return render(request, 'themes_add.html', {'form': form})

def themes_delete(request, id):
    theme = Theme.objects.get(pk=id)
    theme.delete()
    return redirect('/settings/themes')

def images_index(request):
    images = Image.objects.all().order_by('theme')
    return render(request, 'images_index.html', {'images': images})

def images_add(request):
    if not request.method == "POST":
        form = ImageForm()
        return render(request, 'images_add.html', {'form': form})
    else:
        form = ImageForm(request.POST, request.FILES)
        if form.is_valid():
            i = Image()
            i.image = form.cleaned_data['image']
            i.theme = form.cleaned_data['theme']
            i.save()
            return redirect('/settings/images')
        else:
            return render(request, 'images_add.html', {'form': form})

def images_delete(request, id):
    image = Image.objects.get(pk=id)
    if os.path.isfile(image.image.path):
        os.remove(image.image.path)
    image.delete()
    return redirect('/settings/images')


# -*- coding: utf-8 -*-
"""
Created on Fri May 22 11:16:18 2020

@author: Mack Pro
"""

import random

import matplotlib.pyplot as plt
import numpy as npy

import volmdlr as volmdlr
import volmdlr.primitives2D as primitives2D
import volmdlr.primitives3D as primitives3D

######## SWEEP TEST RANDOM

nb_point1, nb_point2 = 6, 5

radius_circle1, radius_circle2 = 0.008, 0.01
c1 = volmdlr.Circle2D(volmdlr.Point2D((0,0)), radius_circle1)
contour1 = volmdlr.Contour2D([c1])

c2 = volmdlr.Circle2D(volmdlr.Point2D((0,0)), radius_circle2)
contour2 = volmdlr.Contour2D([c2])

mini, maxi = -1, 1
pts1 = []
for k in range (nb_point1):
    a1, a2, a3 = random.randint(mini, maxi), random.randint(mini, maxi), random.randint(mini, maxi)
    c1, c2, c3 = random.randrange(0,100,1), random.randrange(0,100,1), random.randrange(0,100,1)
    pts1.append(volmdlr.Point3D((a1*c1/100, a2*c2/100, a3*c3/100)))

radius1 = {1: 0.03, 2: 0.01, 3: 0.07, 4: 0.01}#, 5: 0.07, 6: 0.02, 7: 0.03, 8: 0.04}
rl1 = primitives3D.OpenedRoundedLineSegments3D(pts1, radius1, adapt_radius=True, name='wire1')
sweep1 = primitives3D.Sweep(contour1, rl1, name = 'pipe1')

pts2 = []
for k in range (nb_point2):
    a1, a2, a3 = random.randint(mini, maxi), random.randint(mini, maxi), random.randint(mini, maxi)
    c1, c2, c3 = random.randrange(0,100,1), random.randrange(0,100,1), random.randrange(0,100,1)
    pts2.append(volmdlr.Point3D((a1*c1/100, a2*c2/100, a3*c3/100)))

radius2 = {1: 0.01, 2: 0.05, 3: 0.06}#, 4: 0.02, 5: 0.01, 6: 0.03}
rl2 = primitives3D.OpenedRoundedLineSegments3D(pts2, radius2, adapt_radius=True, name='wire2')
sweep2 = primitives3D.Sweep(contour2, rl2, name = 'pipe2')

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
for prim1 in rl1.primitives :
    prim1.MPLPlot(ax=ax)
for prim2 in rl2.primitives :
    prim2.MPLPlot(ax=ax)

minimum_distance = rl1.minimum_distance(rl2)
print('minimum_distance', minimum_distance)

try:
    distance_sweep = sweep1.distance_to_shell(sweep2)
    print('distance_sweep', distance_sweep.distance)

    pt1, pt2 = distance_sweep.points

    dmin, p1 ,p2 = sweep1.faces[0].minimum_distance(sweep2.faces[0], return_points=True)
    face_min1, face_min2 = sweep1.faces[0], sweep2.faces[0]
    # print('dmin', dmin)
    for face1 in sweep1.faces :
        for face2 in sweep2.faces :
            dtest, ptest1, ptest2 = face1.minimum_distance(face2, return_points=True)
            # print('dtest', dtest)
            if dtest < dmin :
                p1, p2 = ptest1, ptest2
                dmin = dtest
                face_min1, face_min2 = face1, face2

    print('distancep1p2', (p1-p2).Norm())

    pt1.MPLPlot(ax=ax, color='g')
    pt2.MPLPlot(ax=ax, color='g')
    p1.MPLPlot(ax=ax, color='r')
    p2.MPLPlot(ax=ax, color='r')

    # model=volmdlr.VolumeModel([sweep1, sweep2])

    # model.babylonjs()

    vol = volmdlr.VolumeModel([sweep1, sweep2, p1, p2])
    vol.babylonjs_from_script()

    vol = volmdlr.VolumeModel([sweep1, sweep2, pt1, pt2])
    vol.babylonjs_from_script()
except AttributeError:
    print('Nothing was tested,  CylindricalFace3D object has no attribute linesegment_intersection')


from random import randint

k1=0
k2=0

flag=True

# Populate board with random digits
board = []
for i in range(8):
    board.append([])
    for j in range(8):
        board[i].append(randint(0,9))
        print(board[i][j], end=" ")
    print("\n")

def displayBoard(board):
    print("---------------")
    for row in board:
        for col in row:
            print(f"{col}", end=" ")
        print("\n")
    print("---------------")

def traverseBoard(i,j, board):

    global k1
    global k2
    global flag

    if i >= 8 or j >= 8:
        if flag:
            a = k1
            k1 = k2
            k2 = a

            if flag:
                flag = False
            else:
                flag = True
            k1 += 1
        else:
            a = k1
            k1 = k2
            k2 = a
            if flag:
                flag = False
            else:
                flag = True
        print()
        return False

    print(board[i][j], end=" ")

    if (traverseBoard(i+1,j+1,board)):
        return True

    if (traverseBoard(k1, k2, board)):
        return True
    return True

traverseBoard(0,0,board)

# displayBoard(board)


import glob
import os
import apa
import pybio
import sys
import json
import shutil
import regex
import gzip
from queue import *
from threading import *
from multiprocessing import Process # http://stackoverflow.com/questions/4496680/python-threads-all-executing-on-a-single-core

def map_experiment(lib_id, exp_id, map_id = 1, force=False, mapper="star", cpu=1, minlen=0.66, append=""):
    lib = apa.annotation.libs[lib_id]

    exp_id = int(exp_id)
    exp_data = apa.annotation.libs[lib_id].experiments[exp_id]
    map_folder = apa.path.map_folder(lib_id, exp_id, map_id = map_id, append=append)

    if os.path.exists(map_folder):
        if map_folder.find(lib_id)>0 and map_folder.find("m%s" % map_id)>0 and len(lib_id)>6: # security
            shutil.rmtree(map_folder)
    try:
        os.makedirs(map_folder)
    except:
        pass
    if os.path.exists(map_folder) and force==False:
        print("{lib_id}_e{exp_id} : MAP : skip (already mapped) or currently mapping".format(lib_id=lib_id, exp_id=exp_id))
        return

    if lib.seq_type=="single":
        fastq_file = apa.path.map_fastq_file(lib_id, exp_id, append=append)
        print("{lib_id}_e{exp_id} : MAP : {map_folder}".format(lib_id=lib_id, exp_id=exp_id, map_folder=map_folder))
        if mapper=="star":
            pybio.map.star(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu, minlen=minlen)
        if mapper=="sege":
            pybio.map.sege(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)
        if mapper=="bowtie":
            pybio.map.bowtie(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)
        if mapper=="bowtie2":
            pybio.map.bowtie2(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)
        if mapper=="nano":
            pybio.map.nano(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)

    if lib.seq_type=="paired":
        fastq_file1 = apa.path.map_fastq_file(lib_id, exp_id, append="_R1")
        fastq_file2 = apa.path.map_fastq_file(lib_id, exp_id, append="_R2")
        if mapper=="star":
            pybio.map.star_pair(exp_data["map_to"], fastq_file1, fastq_file2, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu, minlen=minlen)

def preprocess_lexfwd(lib_id):
    threads = []
    for exp_id, exp_data in apa.annotation.libs[lib_id].experiments.items():
        t = Process(target=preprocess_lexfwd_thread, args=[exp_id, lib_id])
        threads.append(t)

    while len(threads)>0:
        to_process = min(apa.config.cores, len(threads))
        for i in range(to_process):
            threads[i].start()
        for i in range(to_process):
            threads[i].join()
        threads[:to_process] = []

def preprocess_lexfwd_thread(exp_id, lib_id):
    fastq_file_raw = apa.path.map_fastq_file_raw(lib_id, exp_id)
    fastq_file = apa.path.map_fastq_file(lib_id, exp_id)
    if not os.path.exists(fastq_file_raw):
        return
    processed, written, cut = 0, 0, 0
    fout = gzip.open(fastq_file, "wb")
    f = pybio.data.Fastq(fastq_file_raw)
    prog = regex.compile(r'AAA(?:AAAAAAA){s<=2}') # first AAA, then 7A with possible 2 mismatches
    while f.read():
        processed += 1
        i = prog.search(f.sequence)
        if i!=None:
            seq = f.sequence[:i.start(0)]
            cut += 1
        else:
            seq = f.sequence
        qual = f.quality[:len(seq)]
        if len(seq)>12:
            seq = seq[12:]
            qual = qual[12:]
        if len(seq)>10:
            fout.write("%s\n%s\n+\n%s\n" % (f.id, seq, qual))
            written += 1
        if processed%100000==0:
            sys.stdout.write("e%s, processed=%.2fM, cut=%s, kept=%s\n" % (exp_id, processed/1000000.0, "%.2f%%" % (float(cut)/processed*100.0), "%.2f%%" % (float(written)/processed*100.0)))
    fout.close()
    fname = os.path.join(apa.path.data_folder, lib_id, "e%s" % exp_id, "%s_e%s_preprocess.tab" % (lib_id, exp_id))
    f = open(fname, "wt")
    f.write("e%s\nprocessed=%.2fM\ncut=%s\nkept=%s" % (exp_id, processed/1000000.0, "%.2f%%" % (float(cut)/processed*100.0), "%.2f%%" % (float(written)/processed*100.0)))
    f.close()
    return

def stats_to_tab(lib_id, map_id=1, append=""):
    fname_json = os.path.join(apa.path.lib_folder(lib_id), "%s_m%s%s.stats.json" % (lib_id, map_id, append))
    data = json.loads(open(fname_json).readline())
    fname = os.path.join(apa.path.lib_folder(lib_id), "%s_m%s%s.stats.tab" % (lib_id, map_id, append))
    print("writting statistics: {fname}".format(fname=fname))
    f = open(fname, "wt")
    header = [x for x in apa.annotation.libs[lib_id].experiments[list(apa.annotation.libs[lib_id].experiments.keys())[0]].keys() if x not in ["exp_id"]]
    header = ["exp_id"] + header + ["#reads [M]", "#mapped [M]" , "mapped [%]"]
    f.write("\t".join(header) + "\n")
    for exp_id, exp_data in apa.annotation.libs[lib_id].experiments.items():
        row = [exp_id]
        for x in header[1:-3]:
            row.append(exp_data[x])
        num_reads = data.get(str(exp_id), {}).get("num_reads", 0)
        map_reads = data.get(str(exp_id), {}).get("map_reads", 0)
        row = row + ["%.2f" % (num_reads/1e6), "%.2f" % (map_reads/1e6), "%.2f" % (map_reads*100.0/max(1, num_reads))]
        f.write("\t".join(str(x) for x in row) + "\n")
    f.close()

def stats(lib_id, map_id=1, append=""):
    for exp_id, exp_data in apa.annotation.libs[lib_id].experiments.items():
        stats_experiment(lib_id, exp_id, map_id=map_id)
    return

def stats_experiment(lib_id, exp_id, map_id=1, append=""):
    lib = apa.annotation.libs[lib_id]
    fname_json = os.path.join(apa.path.lib_folder(lib_id), "%s_m%s%s.stats.json" % (lib_id, map_id, append))
    map_folder = apa.path.map_folder(lib_id, exp_id, map_id=map_id, append=append)
    bam_file = os.path.join(map_folder, "%s_e%s_m%s%s.bam" % (lib_id, exp_id, map_id, append))
    if lib.seq_type=="single":
        fastq_files = [apa.path.map_fastq_file(lib_id, exp_id, append=append)]
    if lib.seq_type=="paired":
        fastq_files = [apa.path.map_fastq_file(lib_id, exp_id, append="_R1"), apa.path.map_fastq_file(lib_id, exp_id, append="_R2")]
    if os.path.exists(fname_json):
        data = json.loads(open(fname_json).readline())
    else:
        data = {}
    print("processing statistics: {lib_id}_e{exp_id}".format(lib_id=lib_id, exp_id=exp_id))
    if not os.path.exists(bam_file):
        return
    num_reads = 0
    for fastq_file in fastq_files:
        if not os.path.exists(fastq_file):
            continue
        output, error = pybio.utils.Cmd("zcat {fastq_file} | wc -l".format(fastq_file=fastq_file)).run()
        temp_reads = output.split("\n")[0] # get last line of output
        temp_reads = int(temp_reads)/4
        num_reads += temp_reads
    output, error = pybio.utils.Cmd("samtools view -c {bam_file}".format(bam_file=bam_file)).run()
    map_reads = output.split("\n")[0] # get last line of output
    map_reads = int(map_reads)
    data[exp_id] = {"num_reads":num_reads, "map_reads":map_reads}
    f = open(fname_json, "wt")
    f.write(json.dumps(data))
    f.close()
    stats_to_tab(lib_id)
    return

# make transcript expression table (salmon)
def salmon(lib_id):
    library = apa.annotation.libs[lib_id]
    script_fname = os.path.join(apa.path.data_folder, lib_id, "salmon", "%s_salmon.sh" % lib_id)
    table_fname = os.path.join(apa.path.data_folder, lib_id, "%s_salmon.tab" % lib_id)
    salmon_folder = os.path.join(apa.path.data_folder, lib_id, "salmon")
    try:
        os.makedirs(salmon_folder)
    except:
        pass
    max_exp = len(library.experiments)
    f = open(script_fname, "wt")
    map_to = library.experiments[next(iter(library.experiments))]["map_to"]
    version = pybio.genomes.get_latest_version(map_to)
    genome_folder = os.path.join(pybio.path.genomes_folder, "%s.transcripts.%s.salmon" % (map_to, version))
    f.write("#!/bin/bash\n")
    for exp_id in library.experiments.keys():
        fastq_file = apa.path.map_fastq_file(lib_id, exp_id)
        output_folder = os.path.join(apa.path.data_folder, lib_id, "salmon", "e%s" % exp_id)
        f.write("salmon quant -i %s -l A -r <(gunzip -c %s) --validateMappings -o %s\n" % (genome_folder, fastq_file, output_folder))
    f.close()

    os.system("chmod +x %s" % script_fname)
    os.system(script_fname)

    # find name mapping between transcripts and genes
    transcript_gene = {}

    t_files = glob.glob(os.path.join(apa.path.pybio_folder, "genomes", "%s.transcripts.*/*.fa.gz" % library.genome))
    t_fname = t_files[0]
    f = pybio.data.Fasta(t_fname)

    print("reading transcript->gene data: {t_fname}".format(t_fname=t_fname))
    while f.read():
        id = f.id.split(" ")
        t_id = id[0]
        gene_id = id[3].split(":")[1]
        gene_name = ""
        if len(id)>6:
            gene_name = id[6].split(":")[1]
        transcript_gene[t_id] = [gene_id, gene_name]

    # combine results for individual experiments into the master quant table
    master_quant_fname = os.path.join(apa.path.data_folder, lib_id, "salmon", "%s_salmon.tab" % lib_id)
    fout = open(master_quant_fname, "wt")
    row = ["transcript_id", "gene_id", "gene_name"]
    for exp_id in library.experiments.keys():
        row.append("e%s_TPM" % exp_id)
    fout.write("\t".join(row) + "\n")
    data = {}
    for exp_id in library.experiments.keys():
        data[exp_id] = {}
        quant_fname = os.path.join(apa.path.data_folder, lib_id, "salmon", "e%s" % exp_id, "quant.sf")
        f = open(quant_fname)
        r = f.readline()
        r = f.readline()
        while r:
            r = r.replace("\n", "").replace("\r", "").split("\t")
            transcript_id = r[0]
            tpm = r[3]
            data[exp_id][transcript_id] = tpm
            r = f.readline()
        f.close()
    tids = set()
    for exp_id in library.experiments.keys():
        tids = tids.union(data[exp_id].keys())
    tids = list(tids)
    for tid in tids:
        gene_id, gene_name = transcript_gene[tid]
        row = [tid, gene_id, gene_name]
        for exp_id in library.experiments.keys():
            row.append(data[exp_id].get(tid, 0))
        fout.write("\t".join(row) + "\n")
    f.close()


def function():
    return None

input = '......^.^^.....^^^^^^^^^...^.^..^^.^^^..^.^..^.^^^.^^^^..^^.^.^.....^^^^^..^..^^^..^^.^.^..^^..^^^..'
#input = '..^^.'

def is_save(index, inp):
    l = inp[index - 1] if index > 0 else '.'
    c = inp[index]
    r = inp[index + 1] if index < len(inp) - 1 else '.'

    if l == '.' and c == '.' and r == '.':
        return '.'

    if l == '.' and c == '.' and r == '^':
        return '^'

    if l == '^' and c == '.' and r == '.':
        return '^'

    if l == '^' and c == '^' and r == '.':
        return '^'

    if l == '.' and c == '^' and r == '^':
        return '^'

    return '.'

def main():
    out = input + '\n'
    line = input
    for x in range(399999):
        k = ''
        for c in range(len(line)):
            k += is_save(c, line)
        out += (k + '\n')
        line = k

    #print(out)

    #print([x for x in out if out == '.'])
    print(out.count('.'))

if __name__ == '__main__':
    main()


from models.dragon import dragon as Dragon_model
from schemas.dragon import Dragon
class dragonService():

    def __init__(self,db) -> None:
        self.db = db

    def get_dragon(self):
        result = self.db.query(Dragon_model).all()
        return result

    def create_dragon(self,dragon: Dragon):
        new_dragon = Dragon_model(**dragon.dict())
        self.db.add(new_dragon)
        self.db.commit()
        return
    def update_dragon(self, id: int, dragon:Dragon):
        result =   self.db.query(Dragon_model).filter(Dragon_model.id ==id).first()
        result.name = dragon.name
        result.color = dragon.color
        result.kingdom = dragon.kingdom
        result.flame = dragon.flame
        self.db.commit()
        return
    def delete_dragon(self, id: int):
            self.db.query(Dragon_model).filter(Dragon_model.id ==id).delete()

            self.db.commit()
            return


import io
import os
import requests
import urllib.request
from bs4 import BeautifulSoup
import random
import string
from google.cloud import vision
from os import listdir
from imageHTML import downloadImg, generateName, uploadFile

# Imports the Google Cloud client library
from google.cloud import storage

# Collects data from images and organizes it
def cVision(safe, labels):
    #safe search check
    adultContent.add(safe.adult)
    medicalContent.add(safe.medical)
    spoofedContent.add(safe.spoof)
    violentContent.add(safe.violence)
    racyContent.add(safe.racy)

class Statistic(object):

    vLikely = 5
    likely = 4
    possible = 3
    unlikely = 2
    vUnlikely = 1
    unknown = 0

    def __init__ (self, cat):
        self.cat = cat
        self.data = [0,0,0,0,0,0]
        self.total = 0;
        self.possible = 0;

    def add(self, input):
        self.data[input] += 1
        self.total += 1

# Instantiates storage and vision clients
storage_client = storage.Client()
client = vision.ImageAnnotatorClient()

# Stats to be collected
adultContent = Statistic("adult")
medicalContent = Statistic("medical");
spoofedContent = Statistic("spoofed");
violentContent = Statistic("violent");
racyContent = Statistic("racy")
allContent = [adultContent,medicalContent,spoofedContent,violentContent,racyContent]

def main(url):
    # Generate name for new bucket to be made in the cloud
    newBucketName = generateName(6)

    # Make the bucket, will be deleted
    bucket = storage_client.create_bucket(newBucketName)

    # Receives a url and returns it as text
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'html.parser')

    num = 0
    for image in soup.find_all('img'):
        image_url = image['src']
        #print(image_url)
        if(image_url != "" and image_url.find("http") != -1):
            if(image_url.split('.')[-1] != 'svg'):
                downloadImg(image['src'], "images", newBucketName, num)
                num += 1

    # List of items in cloud bucket
    bucketList = bucket.list_blobs()

    # Checks each image in bucket using cloud vision
    for x in bucketList:
        print('checking ' + x.name)
        content = x.download_as_bytes()
        image = vision.Image(content=content)
        labelresponse = client.label_detection(image=image)
        ssresponse = client.safe_search_detection(image=image)

        checkSafe = ssresponse.safe_search_annotation
        checkLabels = labelresponse.label_annotations

        cVision(checkSafe, checkLabels)

        x.delete()

    bucket = storage_client.get_bucket(newBucketName)
    bucket.delete()

# Data output functions

# Number of images 'possibly,' 'likely,' and 'very likely' belonging to a specified sensitive category
def calcPossible():
    for i in allContent:
        i.possible = i.data[5] + i.data[4] + i.data[3]

# List numbers of 'very likely' images there are in each category
def vLikely():
    out = [adultContent.data[5],medicalContent.data[5],spoofedContent.data[5],violentContent.data[5],racyContent.data[5]]
    return out

# List numbers of 'possible' images there are in each category
def possible():
    out = [adultContent.data[3],medicalContent.data[3],spoofedContent.data[3],violentContent.data[3],racyContent.data[3]]
    return out

# List numbers of 'likely' images there are in each category
def likely():
    out = [adultContent.data[4],medicalContent.data[4],spoofedContent.data[4],violentContent.data[4],racyContent.data[4]]
    return out

# List percent (<1) of images belonging to each category per category
def percentLikely():
    calcPossible()
    return [ adultContent.possible / (adultContent.total - adultContent.data[0]),
    medicalContent.possible / (medicalContent.total - medicalContent.data[0]),
    spoofedContent.possible / (spoofedContent.total - spoofedContent.data[0]),
    violentContent.possible / (violentContent.total - violentContent.data[0]),
    racyContent.possible / (racyContent.total - racyContent.data[0])]

# Returns total amount of images processed
def total():
    return adultContent.total


import os
import forca
import velha

def cls():
    os.system('cls' if os.name == 'nt' else 'clear')

def menu():
    print('Vamos jogar ?')
    print('1 - jogo da velha')
    print('2 - jogo da forca')
    print('3 - para sair')

    valido = False
    while not valido:
        op = int(input('Sua escolha: '))
        if op in [1, 2, 3]:
            return op

def main():

    continua = True

    while continua:
        opcao = menu()

        if opcao == 1:
            velha.main()
        elif opcao == 2:
            forca.main()
        elif opcao == 3 :
            continua = False
        cls()

if __name__ == '__main__':
    main()


import logging

import module_a
from utils import MyLogger

logger = MyLogger(name=__name__)
logger.setLevel(logging.DEBUG)
logger.debug("This is debug from main")
logger.info("This is info from main")

def divide_func(x,y):
    try:
        result = x / y
    except ZeroDivisionError:
        logger.exception(f"Cannot divide x={x} with y={y}")
    else:
        return result
divide_func(10,0)


#Kacper Słoniec

#Opis algorytmu:
#Obliczamy czas budowy wszystkich możliwych autostrad. Czasy te sortujemy. A następnie szukamy grafu spójnego z minimalną różnicą czasów (metodą brute force).

from collections import deque
from zad8testy import runtests

def time(A, B):
    length = ((A[0]-B[0])**2 + (A[1]-B[1])**2)**0.5
    if int(length) == length: return length
    else: return int(length) + 1

def spojne(n, Tab, start, stop):
    G = [[] for _ in range(n)]
    for i in range(start, stop+1):
        G[Tab[i][1]].append(Tab[i][2])
        G[Tab[i][2]].append(Tab[i][1])

    Q = deque()
    visited = 0
    V = [False for _ in range(len(G))]

    V[0] = True
    visited += 1
    Q.append(0)

    while Q:
        u = Q.popleft()
        for v in G[u]:
            if V[v] == False:
                V[v] = True
                visited += 1
                Q.append(v)

    if visited == n:
        return True
    else:
        return False

def highway(A):
    n = len(A)

    Times = []
    for i in range(n-1):
        for j in range(i+1, n):
            Times.append((time(A[i], A[j]), i, j))

    Times.sort(key = lambda tup: tup[0])
    r = len(Times)

    minimum = float('inf')
    for start in range(r):
        for end in range(start+n-2, r):
            if spojne(n, Times, start, end):
                minimum = min(minimum, Times[end][0] - Times[start][0])

    return minimum

#A =[(10,10),(15,25),(20,20),(30,40)]
#print(highway(A))
# zmien all_tests na True zeby uruchomic wszystkie testy
runtests( highway, all_tests = True )


from math import log
def solution(n, k):
    answer, x, result = "", int(log(n,k)), 0
    for i in range(x+1):
        answer += str(n//(k**(x-i)))
        n = n%(k**(x-i))
    answer = sorted(answer.split("0"))
    while answer:
        tmp = answer.pop()
        try:
            tmp = int(tmp)
        except:
            continue
        if prime(tmp):
            result += 1
    return result

def prime(number):
    if number==1:
        return False
    for i in range(2, int(number**(0.5))+1):
        if number%i==0:
            return False
    return True

프로그래머스/lv2/92335. k진수에서 소수 개수 구하기/k진수에서 소수 개수 구하기.py

k진수에서 소수 개수 구하기.py


from src.models.inverseModel1 import *
from src.Labyrinth import *
from src.models.helper import *
import os

if __name__=='__main__':
    COUNT_ITERATIONS=100
    COUNT_TIMESTEPS=3

    lab= Labyrinth.standardVersion()
    lab.position=Vector2f(1.75,0.75)
    configuration={
        "cell_type":"LSTMCell",
        "num_hidden_units": 16,
        "size_output":2,
        "size_input":4,
        "use_biases":True,
        "use_peepholes":True,
        "tag":"Labyrinth_200_0.2"
    }

    #outputs=[[-0.0]*configuration["size_output"] for i in range(COUNT_TIMESTEPS)]
    outputs=[2.8,1.8]
    path=checkpointDirectory=os.path.dirname(__file__)+"/../../data/checkpoints/"+createConfigurationString(configuration)+".chkpt"

    iModel=inverseModel(configuration)
    iModel.create_self_feeding(COUNT_TIMESTEPS)
    iModel.create_all_timesteps_optimizer(clip_min=-0.2,clip_max=0.2)
    iModel.restore(path)

    begin=time.time()
    for i in range(200):
        print(iModel.infer_self_feeding([outputs for i in range(COUNT_TIMESTEPS)],COUNT_ITERATIONS)[0])
    end=time.time()
    print(end-begin)


def is_prime(n):
	i = 2
	prime = True

	if n < 2:
		prime = False
	while i < n:
		if n % i == 0:
			prime = False
			break
		else:
			i += 1
	return 	prime

n = input("Enter a number: ")
n = int(n)

n1 = n - 2
n2 = n + 2

if is_prime(n) and (not is_prime(n1)) and (not is_prime(n2)):
    print(str(n) + " is prime")
    print("But " + str(n1) + " and " + str(n2) + " are not.")
elif is_prime(n):
    if is_prime(n1):
        print("Twin brother with: " + str(n))
        print(n1, n)
    if is_prime(n2):
        print("Twin brother with: " + str(n))
        print(n, n2)
else:
    print(str(n) + " is not prime.")


from urllib.request import urlopen
from datetime import datetime
from monitor.models import client, leftClient

# ===========================================================================
# Check the availability of existing clients
# ===========================================================================
def check_client_availability():
	all_clients = client.objects.all()
	for cl in all_clients:
		check_url = "http://%s:8717/latest" % cl.ip
		print(check_url)
		print("[Client-INFO]Checking Client " + cl.name + " availability!")
		try:
			rsp = urlopen(check_url)
			cl.last_visit = datetime.now()
			cl.save()
		except:
			print("[Client-INFO]Client " + cl.name + " leaves the system!")
			num_results = leftClient.objects.filter(ip=cl.ip).count()
			if num_results > 0:
				cur_obj = leftClient.objects.filter(ip=cl.ip)[0]
				cur_obj.left_ts = datetime.now()
				cur_obj.save()
			else:
				left_cl = leftClient(name=cl.name, ip=cl.ip)
				left_cl.save()
			pass


import common as aux
import codecs
import os

dbPath = "data.db"
dbHeaderPath = "data.h"

class Record:

    def __init__(self, listOfValues, dataInBytes):
        if (not dataInBytes):
            self.cod                                 = listOfValues[0]
            self.sgEntidadeEnsino                    = listOfValues[1]
            self.nmEntidadeEnsino                    = listOfValues[2]
            self.cdCursoPpg                          = listOfValues[3]
            self.nmCurso                             = listOfValues[4]
        else:
            listOfValues = listOfValues.decode("utf-8")
            self.cod                                 = listOfValues[0:4]
            self.sgEntidadeEnsino                    = listOfValues[4:24]
            self.nmEntidadeEnsino                    = listOfValues[24:110]
            self.cdCursoPpg                          = listOfValues[110:123]
            self.nmCurso                             = listOfValues[123:233]

        self.sizeInBytes = len(str(self))

    def __str__(self):
        return self.cod + self.sgEntidadeEnsino + self.nmEntidadeEnsino + self.cdCursoPpg + self.nmCurso

    def Clear(self):
        self.cod                 = '\x00' * 4
        self.sgEntidadeEnsino    = '\x00' * 20
        self.nmEntidadeEnsino    = '\x00' * 86
        self.cdCursoPpg          = '\x00' * 13
        self.nmCurso             = '\x00' * 110

        self.sizeInBytes = len(str(self))
class Block:

    def __init__(self, recordBytes):
        self.recordList = []
        for b in range(0, len(recordBytes), (aux.recordSize -1)):
            self.recordList += [Record(recordBytes[b : b + (aux.recordSize -1)], True)]

        self.firstEmptyRecordIndex = self.__FirstEmptyRecordIndex()

    def SizeInBytes(self):
        sizeInBytes = 0
        for record in self.recordList:
            sizeInBytes += record.sizeInBytes

        return sizeInBytes

    def __FirstEmptyRecordIndex(self):
        for i in range(len(self.recordList)):
            try:
                if (self.recordList[i].cod.index('\x00') >= 0):
                    return i
            except:
                pass
        return -1

    def __str__(self):
        str_block = ""
        for record in self.recordList:
            str_block += str(record)

        return str_block
class Bucket:
    def __init__(self, hashFile, startOffset):
        self.blocksList = []
        for i in range(startOffset, startOffset + aux.bucketSize * aux.blockSize * (aux.recordSize -1) - 1, aux.blockSize * (aux.recordSize -1)):
            self.blocksList += [Block(fetchBlockBytes(hashFile, i))]
        self.firstBlockWithEmptyRecordIndex = self.__FirstBlockWithEmptyRecordIndex()

    def __FirstBlockWithEmptyRecordIndex(self):
        for i in range(len(self.blocksList)):
            if (self.blocksList[i].firstEmptyRecordIndex != -1):
                return i

        return -1

def calculateHashKey(key):
    return int(key)

def calculateHashAddress(hashKey):
    return hashKey % aux.numberOfBuckets

def fetchBlockBytes(hashFile, startOffset):
    hashFile.seek(startOffset)
    return hashFile.read((aux.blockSize * (aux.recordSize -1)))

def createHashBD(csvFilePath):

    valuesToLoad = aux.padRecords(aux.readFromFile(csvFilePath))

    if os.path.exists(dbPath):
        os.remove(dbPath)

    with open(dbPath, 'wb') as hashFile:
        hashFile.seek((aux.bucketSize * aux.numberOfBuckets * aux.blockSize * (aux.recordSize -1)) - 1)
        hashFile.write(b'\0')
    recordCounter = 0
    aux.makeHEAD(dbHeaderPath, "Hash", 0)

    for row in valuesToLoad:
        recordCounter += 1
        record = Record(row, False)
        hashInsertRecordSingle(record)
    aux.updateHEAD(dbHeaderPath, "Hash", recordCounter)

def hashInsertRecordSingle(record):
    freeBlockIndex = -1
    freeSpaceIndex = -1

    hashKey     = calculateHashKey(record.cod)
    hashAddress = calculateHashAddress(hashKey)

    startingOffset = hashAddress * aux.bucketSize * aux.blockSize * (aux.recordSize - 1)

    with open(dbPath, 'r+b') as hashFile:
        while freeBlockIndex == -1:
            currentBucket = Bucket(hashFile, startingOffset)
            freeBlockIndex = currentBucket.firstBlockWithEmptyRecordIndex

            if (freeBlockIndex == -1):
                startingOffset += aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
                pass
            else:
                currentBlock = currentBucket.blocksList[freeBlockIndex]
                freeSpaceIndex = currentBlock.firstEmptyRecordIndex
                currentBlock.recordList[freeSpaceIndex] = record

        hashFile.seek(startingOffset + (freeBlockIndex * aux.blockSize * (aux.recordSize - 1)))
        hashFile.write(str(currentBlock).encode("utf-8"))

def hashDeleteRecordById(searchKeys):
    for searchKey in searchKeys:
        freeBlockIndex = -1
        blocksVisitedCount = 0

        hashKey     = calculateHashKey(searchKey)
        hashAddress = calculateHashAddress(hashKey)

        startingOffset = hashAddress * aux.bucketSize * aux.blockSize * (aux.recordSize - 1)

        with open(dbPath, 'r+b') as hashFile:
            while freeBlockIndex == -1:
                currentBucket = Bucket(hashFile, startingOffset)
                freeBlockIndex = currentBucket.firstBlockWithEmptyRecordIndex
                foundRecord = False

                for i in range(len(currentBucket.blocksList)):
                    block = currentBucket.blocksList[i]
                    blocksVisitedCount += 1
                    for record in block.recordList:
                        if (record.cod == searchKey):
                            record.Clear()
                            foundRecord = True
                            hashFile.seek(startingOffset + (i * aux.blockSize * (aux.recordSize - 1)))
                            hashFile.write(str(block).encode("utf-8"))
                            print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                            print("Record deleted")
                            if (freeBlockIndex == -1):
                                freeBlockIndex = 0
                            break

                    if (foundRecord):
                        break

                if (not foundRecord):
                    if (freeBlockIndex == -1):
                        startingOffset += aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
                        pass
                    else:
                        print("Record {} not found".format(searchKey))
                        print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                        pass

def hashSelectId(searchKeys):
    recordList = []
    for searchKey in searchKeys:
        freeBlockIndex = -1
        blocksVisitedCount = 0

        hashKey     = calculateHashKey(searchKey)
        hashAddress = calculateHashAddress(hashKey)

        startingOffset = hashAddress * aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
        print("\nRunning query: ")
        print("\nSELECT * FROM TABLE WHERE COD in " + str(searchKeys) + ";\n\n")

        with open(dbPath, 'r+b') as hashFile:
            while freeBlockIndex == -1:
                currentBucket = Bucket(hashFile, startingOffset)
                freeBlockIndex = currentBucket.firstBlockWithEmptyRecordIndex
                foundRecord = False

                for block in currentBucket.blocksList:
                    blocksVisitedCount += 1
                    for record in block.recordList:
                        if (record.cod == searchKey):
                            recordList += [record]
                            foundRecord = True
                            print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                            if (freeBlockIndex == -1):
                                freeBlockIndex = 0
                            break

                    if (foundRecord):
                        print(record)
                        break

                if (not foundRecord):
                    if (freeBlockIndex == -1):
                        startingOffset += aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
                        pass
                    else:
                        print("Record {} not found".format(searchKey))
                        print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                        pass

    return recordList

def hashSelectColumns(colName, value, singleRecordSelection = False, valueIsArray = False, secondColName = "", secondValue = "", idResult = []):
    blocksScanned = 0
    recordFound = False
    endOfFile = False
    values = ""
    if valueIsArray:
        for val in value:
            values+= val + ", "
        values = values[:len(values)-2]

    if colName not in aux.colHeadersList:
        print("Error: Column name not found in relation.")
        return
    columnIndex = aux.colHeadersList.index(colName)
    secondValuePresent = False

    secondColumnIndex = -1
    if secondColName != "" and secondValue != "":
        if secondColName not in aux.colHeadersList:
            print("Error: Second column name not found in relation")
            return
        secondColumnIndex = aux.colHeadersList.index(secondColName)
        secondValuePresent = True

    print("\nRunning query: ")
    if singleRecordSelection:
        if valueIsArray:
            print("\nSELECT * FROM TABLE WHERE " + colName + " IN (" + values + ") LIMIT 1;\n\n")
        else:
            if secondValuePresent:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + " AND " + secondColName + "=" + secondValue + " LIMIT 1;\n\n")
            else:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + " LIMIT 1;\n\n")
    else:
        if valueIsArray:
            print("\nSELECT * FROM TABLE WHERE " + colName + " IN (" + values + ");\n\n")
        else:
            if secondValuePresent:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + " AND " + secondColName + "=" + secondValue + ";\n\n")
            else:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + ";\n\n")

    customRecord= 0
    results = []
    while not (recordFound or endOfFile):
        currentBlock = aux.fetchBlock(dbPath, customRecord)
        if currentBlock == []:
            endOfFile = True
            break
        blocksScanned +=1
        for i in range(len(currentBlock)):
            if (not valueIsArray and ((not secondValuePresent and currentBlock[i][columnIndex] == value) or (secondValuePresent and currentBlock[i][columnIndex]==value and currentBlock[i][secondColumnIndex]==secondValue) ) ) or (valueIsArray and currentBlock[i][columnIndex] in value):
                results += [currentBlock[i]]
                idResult += [currentBlock[i][0]]
                if singleRecordSelection:
                    recordFound = True
                    break
        customRecord +=aux.blockSize

    if results == []:
        if valueIsArray:
            print("Não foi encontrado registro com "+colName+ " in (" + values +")")
        else:
            print("Não foi encontrado registro com valor " +colName+ " = " + value)

    else:
        print("Results found: \n")
        for result in results:
            print(result)

    print("\n")
    print("End of search.")
    print("Number of blocks fetched: " + str(blocksScanned))
    return idResult

def hashSelectRecord(input1, input2 = "SINGLE", singleRecordSelection = False, valueIsArray = False, secondColName = "", secondValue = "", betweenTwoValues = False):
    if(str(input2) == "SINGLE"):
        if(betweenTwoValues == False):
            hashSelectId(input1)
        else:
            for i in range(int(input1[0]), int(input1[1])):
                values = []
                values.append(str(i))
                hashSelectId(values)
    else:
        hashSelectColumns(input1, input2, singleRecordSelection, valueIsArray, secondColName, secondValue)

def hashInsertRecord(records):
    records = aux.padRecords(records)
    for i in range(len(records)):
        valueToInsert = Record(records[i], False)
        hashInsertRecordSingle(valueToInsert)

def hashDeleteRecord(input1, input2 = "SINGLE"):
    if(input2 == "SINGLE"):
        hashDeleteRecordById(input1)
    else:
        listToDelete = hashSelectColumns(input1, input2)
        hashDeleteRecord(listToDelete)