import sys
import decimal
import codecs
import re
import time
import collections
#nltk.download("stopwords")
from nltk.corpus import stopwords
import pymorphy2
import itertools
import datetime
import numpy as np
import matplotlib.pyplot as plt

russian_stopwords = stopwords.words("russian")
stoptags = ['NUMR', 'NPRO', 'PRED', 'CONJ', 'PRCL', 'INTJ', 'None', 'PREP', 'GRND', 'LATN', 'PNCT']
start = time.time()

main_data = r'C:\Users\Boris\Desktop\data.txt'
tvits = r"E:\data\first_parsed.txt"

morphalizer = pymorphy2.MorphAnalyzer()

error_log = open(r"E:\data\error_log.txt",'w',encoding='utf-8')

class Computing:

    def __init__(self):
        self.twit_count = 0  # Количество твитов
        self.estimation_table = {}  # таблица со словами и их оценкой (-1,0,1)
        self.frequency_table = {}  # частота встрачемости слов (слов - частота)
        self.words_in_twits = {}    # таблица хранящая твит как ключ и тональные слова как значение

        self.twit_tonality_rule1 = {}  # хранит твит и его тональность согласно 1 правилу
        self.twit_tonality_rule2 = {}  # хранит твит и его тональность согласно 2 правилу
        self.twit_tonality_rule3 = {}  # хранит твит и его тональность соласно 3 правилу
        self.twit_tonality_rule4 = {}  # хранит твит и его тональность соласно 4 правилу

        self.adjectives_store = {}  # хранит прилагательные и их частоту встречаемости
        self.adjective_marks = {}  # хранит прилагательные и его оценку

        self.mark_counts_1rule = []  # хранит кол во твитов по 1 правилу 1-pos 2 neg 3 neu
        self.mark_counts_2rule = []  # хранит кол во твитов по 2 правилу 1-pos 2 neg 3 neu
        self.mark_counts_3rule = []  # хранит кол во твитов по 3 правилу 1-pos 2 neg 3 neu
        self.mark_counts_4rule = []  # хранит кол во твитов по 4 правилу 1-pos 2 neg 3 neu

        self.top_adj_pos = {}  # Хранит топ положительных прил
        self.top_adj_neg = {}  # Хранит топ отриц прил
        self.twits_with_dates = {}  # твит и час

        # Хранят данные по распределениям классов во времени в зависимости от выбора функции оценивания
        self.pos_distribution1 = []
        self.neg_distribution1 = []
        self.neu_distribution1 = []
        self._time_distribution1 = []
        self.current_twits_sum1 = []

        self.pos_distribution2 = []
        self.neg_distribution2 = []
        self.neu_distribution2 = []
        self._time_distribution2 = []
        self.current_twits_sum2 = []

        self.pos_distribution3 = []
        self.neg_distribution3 = []
        self.neu_distribution3 = []
        self._time_distribution3 = []
        self.current_twits_sum3 = []

        self.pos_distribution4 = []
        self.neg_distribution4 = []
        self.neu_distribution4 = []
        self._time_distribution4 = []
        self.current_twits_sum4 = []

    def cleaner(self):
        with codecs.open(r"E:\data\data.txt", 'r', encoding="utf-8") as readable:
            with open(r"E:\data\first_parsed_tvits.txt", 'w', encoding="utf-8") as writable:
                for string in readable:

                    string = re.sub("#[ ]*[A-Za-zA-Яа-яё0-9]*",'',string) # tags
                    string = re.sub("(?:pic.twitter.com/|https://|http://|.twitter.com/|.ru|.com|.org)[^ \n]*", '', string)  # references
                    string = re.sub("@[ ]*[^ \n]*", '', string)  # @user
                    string = re.sub("[!.,@$%^&*()\-_=+\"№;?/`:<>{}\[\]']", '', string)
                    if not string.isspace():
                        data = string.split()[1]
                        data = datetime.datetime(1, 1, 1, int(data[0:2]), int(data[2:4]))
                    string = re.sub("\d", '', string)
                    if not string.isspace():
                        zir = re.sub('\n','',string)
                        zir = re.sub('\ufeff','',string)
                        self.twits_with_dates[data] = zir
                        writable.write(string)
        second_parsed_lemmatized = open(r"E:\data\second_parsed_lemmatized.txt", 'w', encoding="utf-8")

        with codecs.open(r"E:\data\first_parsed_tvits.txt", 'r', encoding="utf-8") as twiter:

            for line in twiter:
                self.twit_count += 1
                for string in line.split():
                    if (string not in russian_stopwords) or (4 > len(string) > 15):
                        if string not in self.frequency_table:
                            new_word = morphalizer.parse(string)[0]
                            TAG = str(new_word.tag.POS)
                            if TAG in stoptags:
                                #print("break")
                                break
                            new_word = new_word.normal_form.lower()
                            if (new_word not in russian_stopwords) and (15 > len(new_word) > 4):  # удаляем слова из stopwords а так же все слова длиной короче 4 букв
                                if new_word not in self.frequency_table:
                                    self.frequency_table[new_word] = 1  # если элемент встретился 1 раз, добавляем его в таблицу
                                    second_parsed_lemmatized.write(new_word + '\n')
                                    #print("Записано новое слово")

                                else:
                                    self.frequency_table[new_word] += 1
                                    #print("Значение слова увеличилось на 1")
                            else:
                                continue

            second_parsed_lemmatized.close()
            print("Функция cleaner работает", time.time() - start, "секунд\n")
            return 0

    def frequency(self):
        start_time = time.time()
        frequency = open(r"E:\data\frequency.txt", 'w', encoding="utf-8")
        for i in sorted(self.frequency_table, key=self.frequency_table.get):
            frequency.write(str(self.frequency_table[i]) + ' - ' + str(i)+' - ' + str(round((((self.frequency_table[i]) / self.twit_count) * 100),5))+'%' + '\n')
        frequency.close()
        print("Функция frequency работает", time.time() - start_time, "секунд\n")
        return 0

    def declarator(self):
        start_time = time.time()
        parsed_twits = []

        estimator = open(r"E:\data\estimation.txt", 'r', encoding='utf-8')
        self.estimation_table = {line.split()[0]: line.split()[1] for line in estimator}
        estimator.close()

        twits = open(r"E:\data\first_parsed_tvits.txt", 'r', encoding="utf-8")
        for twit in twits:
            twit = re.sub('\n', '', twit)
            twit = re.sub('\ufeff', '', twit)
            parsed_twits.append(twit)
        twits.close()

        '''Создаем список слов из файла с лемматизированными словами'''
        info = open(r"E:\data\second_parsed_lemmatized.txt", 'r', encoding='utf-8')
        words_lemmatized = []
        for word in info:
            words_lemmatized.append(re.sub('\n','',word))
        info.close()

        '''Создаем словарь {key - Твит, value - список слов}'''
        for twit in parsed_twits:
            for word in twit.split():
                check_word = morphalizer.parse(word)[0].normal_form.lower()
                if check_word in words_lemmatized:
                    if self.words_in_twits.get(twit):
                        # append the new number to the existing array at this slot
                        self.words_in_twits[twit].append(check_word)
                    else:
                        # create a new array in this slot
                        self.words_in_twits[twit] = [check_word]
        self.twit_count = len(self.words_in_twits)

        print("Функция declartor работает", time.time() - start_time, "секунд\n")
        return 0

    def rules_maker(self):
        start_time = time.time()

        def first_rule():
            for twit in self.words_in_twits:
                twit_mark = 0
                try:
                    for word in self.words_in_twits.get(twit):
                        twit_mark += int(self.estimation_table.get(word))
                    self.twit_tonality_rule1[twit] = twit_mark
                except TypeError:
                    error_log.writelines("Ошибка в твите: {} \nПрисвоено значение 0".format(twit))
                    self.twit_tonality_rule1[twit] = 0

            neg = 0
            pos = 0
            neu = 0
            for twit in self.twit_tonality_rule1:
                if -1 <= self.twit_tonality_rule1[twit] <= 0:
                    neu += 1
                elif self.twit_tonality_rule1[twit] < -1:
                    neg += 1
                elif self.twit_tonality_rule1[twit] > 0:
                    pos += 1
                else:
                    neu += 1
            self.mark_counts_1rule.append(pos)
            self.mark_counts_1rule.append(neg)
            self.mark_counts_1rule.append(neu)

            positive = "Positive - {0} - {1}%\n".format(pos, round(pos/self.twit_count*100, 3))
            negative = "Negative - {0} - {1}%\n".format(neg, round(neg/self.twit_count*100, 3))
            neutral = "Neutral - {0} - {1}%\n".format(neu, round(neu/self.twit_count*100, 3))

            with open(r"E:\data\classification.txt", 'w', encoding="utf-8") as classification:
                classification.write("Classification by mark summary\n")
                classification.write(str(positive))
                classification.write(str(negative))
                classification.write(str(neutral))
                classification.write("______________________________________________________")

        def second_rule():
            for twit in self.words_in_twits:
                sum_words = len(twit.split())
                sum_pos = 0
                sum_neg = 0
                sum_neu = 0
                try:
                    word_list = self.words_in_twits.get(twit)
                    for word in word_list:
                        word_estimation = int(self.estimation_table.get(word))
                        if word_estimation == 0:
                            sum_neu += 1
                        elif word_estimation == -1:
                            sum_neg += 1
                        elif word_estimation == 1:
                            sum_pos += 1
                        else:
                            sum_neu += 1

                    pos_percentage = sum_pos / sum_words * 100
                    neg_percentage = sum_neg / sum_words * 100
                    neu_percentage = sum_neu / sum_words * 100

                    if max(pos_percentage, neg_percentage, neu_percentage) == pos_percentage:
                        self.twit_tonality_rule2[twit] = 1
                    elif max(pos_percentage, neg_percentage, neu_percentage) == neg_percentage:
                        self.twit_tonality_rule2[twit] = -1
                    elif max(pos_percentage, neg_percentage, neu_percentage) == neu_percentage:
                        self.twit_tonality_rule2[twit] = 0
                except:
                    error_log.writelines("Ошибка в твите: {} \nПрисвоено значение 0".format(twit))
                    self.twit_tonality_rule2[twit] = 0

            neg = 0
            pos = 0
            neu = 0
            for twit in self.twit_tonality_rule2:
                if self.twit_tonality_rule2[twit] == 0:
                    neu += 1
                elif self.twit_tonality_rule2[twit] == -1:
                    neg += 1
                elif self.twit_tonality_rule2[twit] == 1:
                    pos += 1
                else:
                    neu += 1

            self.mark_counts_2rule.append(pos)
            self.mark_counts_2rule.append(neg)
            self.mark_counts_2rule.append(neu)

            positive = "Positive - {0} - {1}%\n".format(pos, round(pos / self.twit_count * 100, 4))
            negative = "Negative - {0} - {1}%\n".format(neg, round(neg / self.twit_count * 100, 4))
            neutral = "Neutral - {0} - {1}%\n".format(neu, round(neu / self.twit_count * 100, 4))

            with open(r"E:\data\classification.txt", 'a', encoding="utf-8") as classification:
                classification.write("\n")
                classification.write("Classification by word types\n")
                classification.write(str(positive))
                classification.write(str(negative))
                classification.write(str(neutral))
                classification.write("______________________________________________________\n")
           # print("\nClassification by word types\n")
           # print(positive)
           # print(negative)
            #print(neutral)

        def third_rule():

            for twit in self.words_in_twits:
                adj_weight = 0
                adv_weight = 0
                noun_weight = 0
                verb_weight = 0
                word_list = self.words_in_twits.get(twit)
                for word in word_list:
                    word = morphalizer.parse(word)[0]
                    TAG = str(word.tag.POS)
                    word = word.normal_form.lower()
                    try:
                        if TAG == "ADJF" or TAG =="ADJS":
                            par = int(self.estimation_table.get(word))
                            adj_weight += par

                        elif TAG == "ADVB":
                            par = int(self.estimation_table.get(word))
                            adv_weight += par

                        elif TAG =="NOUN":
                            par = int(self.estimation_table.get(word))
                            noun_weight += par

                        elif TAG == "VERB" or TAG == "INFN":
                            par = int(self.estimation_table.get(word))
                            verb_weight += par

                    except BaseException as exception:
                        error_log.write(str("\nОшибка в применении 3 правила: "))
                        error_log.write(str(exception))
                        continue

                twit_mark = 0.1*noun_weight + 0.3 * adv_weight + 0.4*adj_weight + 0.2*verb_weight
                self.twit_tonality_rule3[twit] = twit_mark
            neg = 0
            pos = 0
            neu = 0
            for twit in self.twit_tonality_rule3:
                if self.twit_tonality_rule3.get(twit) > 0:
                    pos += 1
                elif self.twit_tonality_rule3.get(twit) < 0:
                    neg += 1
                else:
                    neu += 1

            self.mark_counts_3rule.append(pos)
            self.mark_counts_3rule.append(neg)
            self.mark_counts_3rule.append(neu)
            positive = "Positive - {0} - {1}%\n".format(pos, round(pos / self.twit_count * 100, 3))
            negative = "Negative - {0} - {1}%\n".format(neg, round(neg / self.twit_count * 100, 3))
            neutral = "Neutral - {0} - {1}%\n".format(neu, round(neu / self.twit_count * 100, 3))

            with open(r"E:\data\classification.txt", 'a', encoding="utf-8") as classification:
                classification.write("\nClassification by speech part diversification\n")
                classification.write(str(positive))
                classification.write(str(negative))
                classification.write(str(neutral))
                classification.write("______________________________________________________\n")

        def forth_rule():
            for twit in self.words_in_twits:
                pos_weight = 0
                neg_weight = 0
                neu_weight = 0
                try:
                    for word in self.words_in_twits.get(twit):
                        par = int(self.estimation_table.get(word))
                        if par == 1:
                            pos_weight += 1

                        elif par == -1:
                            neg_weight += 1

                        elif par == 0:
                            neu_weight += 1

                    twit_mark = 0.4*pos_weight + 0.4*neg_weight + 0.2*neu_weight

                    self.twit_tonality_rule4[twit] = twit_mark
                except TypeError:
                    self.twit_tonality_rule4[twit] = 0

            neg = 0
            pos = 0
            neu = 0
            for twit in self.twit_tonality_rule4:
                if 2 <= self.twit_tonality_rule4[twit] <= 6:
                    neu += 1
                elif self.twit_tonality_rule4[twit] < 2:
                    neg += 1
                elif self.twit_tonality_rule4[twit] > 6:
                    pos += 1
                else:
                    neu += 1

            self.mark_counts_4rule.append(pos)
            self.mark_counts_4rule.append(neg)
            self.mark_counts_4rule.append(neu)

            positive = "Positive - {0} - {1}%\n".format(pos, round(pos/self.twit_count*100, 3))
            negative = "Negative - {0} - {1}%\n".format(neg, round(neg/self.twit_count*100, 3))
            neutral = "Neutral - {0} - {1}%\n".format(neu, round(neu/self.twit_count*100, 3))

            with open(r"E:\data\classification.txt", 'a', encoding="utf-8") as classification:
                classification.write("Classification by average mark summary\n")
                classification.write(str(positive))
                classification.write(str(negative))
                classification.write(str(neutral))
                classification.write("______________________________________________________")

        first_rule()
        second_rule()
        third_rule()
        forth_rule()
        print("\nФункция rules_maker работает ", time.time() - start_time, 'секунд.\n\r')

    def adjectiver(self):
        start_time = time.time()
        for twit in self.words_in_twits:
                for word in self.words_in_twits.get(twit):
                        word = morphalizer.parse(word)[0]
                        TAG = str(word.tag.POS)
                        if TAG == "ADJF" or TAG == "ADJS":
                            word = word.normal_form.lower()
                            if word not in self.adjectives_store:
                                self.adjectives_store[word] = 1
                            else:
                                self.adjectives_store[word] += 1

        with open(r"E:\data\adjectivefrequency.txt", 'w', encoding='utf-8') as f:
            for note in sorted(self.adjectives_store.items()):
                f.write(str(note))
                f.write('\n')

        adj_pos = {}
        adj_neg = {}
        ordered = dict(sorted(self.adjectives_store.items(), key=lambda x: x[1], reverse=True))  # СКОРОСТЬ!!!!!

        for adj in self.adjectives_store:
            mark = self.estimation_table.get(adj)
            if mark == str(1):
                adj_pos[adj] = self.estimation_table.get(adj)
            elif mark == str(-1):
                adj_neg[adj] = self.estimation_table.get(adj)
        j = 0

        for adj in ordered:
            if j != 5:
                if adj in adj_pos:
                    self.top_adj_pos[adj] = self.adjectives_store.get(adj)
                    j += 1
                else:
                    continue
            else:
                break

        j = 0
        for adj in ordered:
            if j != 5:
                if adj in adj_neg:
                    self.top_adj_neg[adj] = self.adjectives_store.get(adj)
                    j += 1
                else:
                    continue
            else:
                break

        with open(r"E:\data\adjectives.txt", 'w', encoding='utf-8') as adjective_file:

            adjective_file.write("TOP-5 POSITIVE\n")
            for i in self.top_adj_pos:
                adjective_file.write(str(i + ' ' + str(self.top_adj_pos.get(i)) + ' - ' +
                                     str(round(((self.top_adj_pos.get(i)/self.twit_count)*100), 3)) + '%' + '\n'))
            adjective_file.write("\n___________________________________________\n")

            adjective_file.write("\nTOP-5 NEGATIVE\n")

            for i in self.top_adj_neg:
                adjective_file.write(str(i + ' ' + str(self.top_adj_neg.get(i)) + ' - ' +
                                     str(round((self.top_adj_neg.get(i)/self.twit_count*100),3)) + '%' + '\n'))
            adjective_file.write("\n___________________________________________\n")

        print("\nФункция adjectiver работает ", time.time()-start_time," секунд.\n")  #

    def time_distribution(self):

        starter = time.time()
        new_twits_dates = {}
        writable = open(r"E:\data\hours.txt", 'w', encoding="utf-8")
        for i in self.twits_with_dates.keys():
            string = re.sub('\n', '', self.twits_with_dates.get(i))
            new_twits_dates[i] = string
        def rool1():
            time_gap = sorted(self.twits_with_dates)[0] + datetime.timedelta(minutes=30)
            start_time = sorted(self.twits_with_dates)[0]

            start_time_r = str(start_time)[11:]
            writable.write("\n______First rule estimation _______\n")
            while time_gap <= sorted(self.twits_with_dates)[len(sorted(self.twits_with_dates))-1]:
                pos = 0
                neg = 0
                neu = 0
                for i in sorted(new_twits_dates.keys()):
                    if i <= time_gap:
                        twit = new_twits_dates.get(i)
                        if twit in self.twit_tonality_rule1:
                            key = self.twit_tonality_rule1.get(twit)
                            if key:
                                if key > 0:
                                    pos += 1
                                elif key < -1:
                                    neg += 1
                                else:
                                    neu += 1
                            else:
                                continue
                        else:
                            continue
                count = pos+neg+neu
                positive = round((pos/count*100), 2)
                negative = round((neg/count*100), 2)
                neutral = round((neu/count*100), 2)
                time_to_write = str(time_gap)[11:20]

                to_write = str("\n{0} - {1}:{2}        {3} / {4} / {5}\n").format(start_time_r,time_to_write,count,positive,negative,neutral)

                writable.write(to_write)
                self.pos_distribution1.append(pos)
                self.neg_distribution1.append(neg)
                self.neu_distribution1.append(neu)
                self._time_distribution1.append(time_gap)
                self.current_twits_sum1.append(count)
                time_gap += datetime.timedelta(minutes=10)

            """______________Второе правило_________________"""
        def rool2():
            time_gap = sorted(self.twits_with_dates)[0] + datetime.timedelta(minutes=30)
            start_time = sorted(self.twits_with_dates)[0]
            start_time_r = str(start_time)[11:]
            writable.write("\n______Second rule estimation _______\n")
            while time_gap <= sorted(self.twits_with_dates)[len(sorted(self.twits_with_dates)) - 1]:
                pos = 0
                neg = 0
                neu = 0
                for i in sorted(new_twits_dates.keys()):

                    if i <= time_gap:
                        twit = new_twits_dates.get(i)
                        if twit in self.twit_tonality_rule2:
                            key = self.twit_tonality_rule2.get(twit)
                            if key:
                                if key == 1:
                                    pos += 1
                                elif key == -1:
                                    neg += 1
                                else:
                                    neu += 1
                            else:
                                continue
                        else:
                            continue
                count = pos + neg + neu
                positive = round((pos / count * 100), 2)
                negative = round((neg / count * 100), 2)
                neutral = round((neu / count * 100), 2)
                time_to_write = str(time_gap)[11:20]

                to_write = str("\n{0} - {1}:{2}        {3} / {4} / {5}\n").format(start_time_r, time_to_write,
                                                                                count, positive, negative,
                                                                                neutral)

                writable.write(to_write)
                self.pos_distribution2.append(pos)
                self.neg_distribution2.append(neg)
                self.neu_distribution2.append(neu)
                self._time_distribution2.append(time_gap)
                self.current_twits_sum2.append(count)
                time_gap += datetime.timedelta(minutes=10)

            """______________Третье правило_________________"""
        def rool3():
            time_gap = sorted(self.twits_with_dates)[0] + datetime.timedelta(minutes=30)
            start_time = sorted(self.twits_with_dates)[0]
            start_time_r = str(start_time)[11:]
            writable.write("\n______Third rule estimation _______\n")
            while time_gap <= sorted(self.twits_with_dates)[len(sorted(self.twits_with_dates)) - 1]:
                pos = 0
                neg = 0
                neu = 0
                for i in sorted(new_twits_dates.keys()):
                    if i <= time_gap:
                        twit = new_twits_dates.get(i)
                        if twit in self.twit_tonality_rule2:
                            key = self.twit_tonality_rule2.get(twit)
                            if key:
                                if key > 0:
                                    pos += 1
                                elif key < 0:
                                    neg += 1
                                else:
                                    neu += 1
                            else:
                                continue
                        else:
                            continue
                count = pos + neg + neu
                positive = round((pos / count * 100), 2)
                negative = round((neg / count * 100), 2)
                neutral = round((neu / count * 100), 2)
                time_to_write = str(time_gap)[11:20]

                to_write = str("\n{0} - {1}:{2}        {3} / {4} / {5}\n").format(start_time_r, time_to_write,
                                                                                count, positive, negative,
                                                                                neutral)

                writable.write(to_write)
                self.pos_distribution3.append(pos)
                self.neg_distribution3.append(neg)
                self.neu_distribution3.append(neu)
                self._time_distribution3.append(time_gap)
                self.current_twits_sum3.append(count)

                time_gap += datetime.timedelta(minutes=10)
        def rool4():
            time_gap = sorted(self.twits_with_dates)[0] + datetime.timedelta(minutes=30)
            start_time = sorted(self.twits_with_dates)[0]
            start_time_r = str(start_time)[11:]
            writable.write("\n______Forth rule estimation _______\n")
            while time_gap <= sorted(self.twits_with_dates)[len(sorted(self.twits_with_dates)) - 1]:
                pos = 0
                neg = 0
                neu = 0
                for i in sorted(new_twits_dates.keys()):
                    if i <= time_gap:
                        twit = new_twits_dates.get(i)
                        if twit in self.twit_tonality_rule2:
                            key = self.twit_tonality_rule2.get(twit)
                            if key:
                                if key >= 6:
                                    pos += 1
                                elif key <= 2:
                                    neg += 1
                                else:
                                    neu += 1
                            else:
                                continue
                        else:
                            continue
                count = pos + neg + neu
                positive = round((pos / count * 100), 2)
                negative = round((neg / count * 100), 2)
                neutral = round((neu / count * 100), 2)
                time_to_write = str(time_gap)[11:20]

                to_write = str("\n{0} - {1}:{2}        {3} / {4} / {5}\n").format(start_time_r, time_to_write,
                                                                                count, positive, negative,
                                                                                neutral)

                writable.write(to_write)
                self.pos_distribution4.append(pos)
                self.neg_distribution4.append(neg)
                self.neu_distribution4.append(neu)
                self._time_distribution4.append(time_gap)
                self.current_twits_sum4.append(count)

                time_gap += datetime.timedelta(minutes=10)
        rool1()
        rool2()
        rool3()
        rool4()
        print("\nФункция time_distribution  работает ", time.time() - starter, 'секунд.\n\r')

    def bar_charts(self):
        start_time = time.time()
        plt.rcdefaults()

        objects = ('Good', 'Bad', 'Neutral')

        x = np.arange(len(objects))

        height = [self.mark_counts_1rule[0], self.mark_counts_1rule[1], self.mark_counts_1rule[2]]

        plt.bar(x, height, align='center', alpha=0.5)
        plt.xticks(x, objects)
        plt.ylabel('Count')
        plt.title('Classification by sum')
        plt.savefig(r'E:\data\1 bar chart.png')

        height = [self.mark_counts_2rule[0], self.mark_counts_2rule[1], self.mark_counts_2rule[2]]

        plt.bar(x, height, align='center', alpha=0.5)
        plt.xticks(x, objects)
        plt.ylabel('Count')
        plt.title('Classification by dominance')
        plt.savefig(r'E:\data\2 bar chart.png')

        height = [self.mark_counts_3rule[0], self.mark_counts_3rule[1], self.mark_counts_3rule[2]]

        plt.bar(x, height, align='center', alpha=0.5)
        plt.xticks(x, objects)
        plt.ylabel('Count')
        plt.title('Classification by speech part diversification')
        plt.savefig(r'E:\data\3 bar chart.png')

        height = [self.mark_counts_4rule[0], self.mark_counts_4rule[1], self.mark_counts_4rule[2]]

        plt.bar(x, height, align='center', alpha=0.5)
        plt.xticks(x, objects)
        plt.ylabel('Count')
        plt.title('Classification by average mark')
        plt.savefig(r'E:\data\4 bar chart.png')
        plt.show()

        x = ('1st', '2nd', '3d', '4th', '5th')
        x_pos = np.arange(len(x))
        # Толщина столбца
        width = 0.2
        # Значения по Oy
        y_positive = [x for x in self.top_adj_pos.values()]
        y_negative = [x for x in self.top_adj_neg.values()]
        # Графики
        plt.bar(x_pos, y_positive, width=width, label='Positive', color='y')
        plt.bar(x_pos + width, y_negative, width=width, label='Negative', color='g')
        plt.ylabel('Amount')
        plt.xticks(x_pos + width / 2, x)
        plt.title('Top-5 Positive/Negative Adjectives')
        plt.legend()
        plt.savefig(r'E:\data\Adjectives bar chart.png')

        # Создаем области
        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
        # Значения по Ox
        x = [key for key in self._time_distribution1]
        x_pos = np.arange(len(x))
        # Первый график
        ax1.plot(x_pos, self.pos_distribution1, 'g.-', linewidth=1, markersize=7, label='N_pos')
        ax1.plot(x_pos, self.neg_distribution1, 'b.--', linewidth=1, markersize=7, label='N_neg')
        ax1.plot(x_pos, self.neu_distribution1, 'r.-.', linewidth=1, markersize=7, label='N_neu')
        ax1.set_xticks([])
        ax1.grid(True)
        ax1.legend()
        ax1.set_ylabel('Fraction')
        # столбцы
        ax2.stem(x_pos, self.current_twits_sum1, basefmt=" ")
        # координатн ось
        ax2.set_xticks(x_pos)
        ax2.set_xticklabels(x)
        ax2.grid(True)
        ax2.set_ylabel('Number of tweets')
        fig.suptitle('Distribution of tweets classes in time rule 1', fontsize=16, y=1.0)
        fig.savefig(r'E:\data\Time distribution1.png')

        """2 правило """
        # Создаем области
        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
        # Значения по Ox
        x = [key for key in self._time_distribution2]
        x_pos = np.arange(len(x))
        # Первый график
        ax1.plot(x_pos, self.pos_distribution2, 'g.-', linewidth=1, markersize=7, label='N_pos')
        ax1.plot(x_pos, self.neg_distribution2, 'b.--', linewidth=1, markersize=7, label='N_neg')
        ax1.plot(x_pos, self.neu_distribution2, 'r.-.', linewidth=1, markersize=7, label='N_neu')
        ax1.set_xticks([])
        ax1.grid(True)
        ax1.legend()
        ax1.set_ylabel('Fraction')
        # столбцы
        ax2.stem(x_pos, self.current_twits_sum2, basefmt=" ")
        # координатн ось
        ax2.set_xticks(x_pos)
        ax2.set_xticklabels(x)
        ax2.grid(True)
        ax2.set_ylabel('Number of tweets')
        fig.suptitle('Distribution of tweets classes in time rule 2', fontsize=16, y=1.0)
        fig.savefig(r'E:\data\Time distribution2.png')

        """3 правило """
        # Создаем области
        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
        # Значения по Ox
        x = [key for key in self._time_distribution3]
        x_pos = np.arange(len(x))
        # Первый график
        ax1.plot(x_pos, self.pos_distribution3, 'g.-', linewidth=1, markersize=7, label='N_pos')
        ax1.plot(x_pos, self.neg_distribution3, 'b.--', linewidth=1, markersize=7, label='N_neg')
        ax1.plot(x_pos, self.neu_distribution3, 'r.-.', linewidth=1, markersize=7, label='N_neu')
        ax1.set_xticks([])
        ax1.grid(True)
        ax1.legend()
        ax1.set_ylabel('Fraction')
        # столбцы
        ax2.stem(x_pos, self.current_twits_sum3, basefmt=" ")
        # координатн ось
        ax2.set_xticks(x_pos)
        ax2.set_xticklabels(x)
        ax2.grid(True)
        ax2.set_ylabel('Number of tweets')
        fig.suptitle('Distribution of tweets classes in time rule 3', fontsize=16, y=1.0)
        fig.savefig(r'E:\data\Time distribution3.png')

        """4 правило """
        # Создаем области
        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
        # Значения по Ox
        x = [key for key in self._time_distribution4]
        x_pos = np.arange(len(x))
        # Первый график
        ax1.plot(x_pos, self.pos_distribution4, 'g.-', linewidth=1, markersize=7, label='N_pos')
        ax1.plot(x_pos, self.neg_distribution4, 'b.--', linewidth=1, markersize=7, label='N_neg')
        ax1.plot(x_pos, self.neu_distribution4, 'r.-.', linewidth=1, markersize=7, label='N_neu')
        ax1.set_xticks([])
        ax1.grid(True)
        ax1.legend()
        ax1.set_ylabel('Fraction')
        # столбцы
        ax2.stem(x_pos, self.current_twits_sum4, basefmt=" ")
        # координатн ось
        ax2.set_xticks(x_pos)
        ax2.set_xticklabels(x)
        ax2.grid(True)
        ax2.set_ylabel('Number of tweets')
        fig.suptitle('Distribution of tweets classes in time rule 4', fontsize=16, y=1.0)
        fig.savefig(r'E:\data\Time distribution4.png')
        print("\nГрафики строятся  за ", time.time() - start_time," секунд.\n")

if __name__ == "__main__":

    start_program = time.time()

    parser = Computing()

    parser.cleaner()

    parser.frequency()

    parser.declarator()

    parser.rules_maker()

    parser.adjectiver()

    parser.time_distribution()

    parser.bar_charts()

    error_log.close()

    print("\nПрограмма работает", time.time() - start_program, "секунд\n")


from django.urls import path
from .views import TestEvaluationView,MyTestView,ViewTestResult,Myoptions, MyQuestions,MyLessonTests,MySubjectTests,MyUnitTests

urlpatterns = [
    path('evaluate/<int:test_id>/', TestEvaluationView.as_view(), name='evaluate_test'),
    path('getSubjTest/<int:subject_id>/', MySubjectTests.as_view(), name='subject_tests'),
    path('getLessonTest/<int:lesson_id>/', MyLessonTests.as_view(), name='lesson_tests'),
    path('getUnitTest/<int:unit_id>/', MyUnitTests.as_view(), name='unit_tests'),
    path('mytest/', MyTestView.as_view(), name='mytest'),
    path('viewtest/<int:test_id>/', ViewTestResult.as_view(), name='viewtest'),
    path('questions/', MyQuestions.as_view(), name='questions'),
    #path('options/', MyOptions.as_view(), name='options'),
]


def main():
    n = int(input())
    d = {}
    for i in range(n):
        s = input()
        let_s = tuple(sorted(list(s)))
        if let_s in d:
            d[let_s].append(s)
        else:
            d[let_s] = [s]

    for let_s in d:
        print(*d[let_s])

if __name__ == '__main__':
    main()


import simulation_class as smc
import argparse
import os

parser = argparse.ArgumentParser(description='Script for modifying, compiling lt and xyz template files and copying the LAMMPS input script output into a standalone simulation folder')
parser.add_argument('-N','--total_monomers',dest='total_monomers',type=int,default=3000, help="Total number of monomers in the system")
parser.add_argument('-fA','--fractionA',dest='fA',type=float,default=0.5, help="Fraction of monomers of species A")
parser.add_argument('--epsAA',dest='eAA',type=float,default=1.0,help="Attraction between monomers AA in kbT")
parser.add_argument('--epsBB',dest='eBB',type=float,default=1.0,help="Attraction between monomers BB in kbT")
parser.add_argument('--epsAB',dest='eAB',type=float,default=1.0,help="Attraction between monomers AB in kbT")
parser.add_argument('--angle',dest='angle',type=int,default=20,help="Angle strength of polymer chain in kbT")
parser.add_argument('-p',dest='p',type=float,default=0.9,help="Monomer conversion where simulation will end")
parser.add_argument('-f','--folder',dest='dest_folder',help='Destination folder for simulation')
parser.add_argument('--servername',dest='servername',default=None,help='If sending simulation to a remote server this is the IP/hostname of the server.')
parser.add_argument('--send_to_cluster',dest='send_to_cluster',action='store_true',help='If True send the project to the destination folder on the remote server specified by dest_folder')
parser.add_argument('--xyz',dest='xyz_folder',help='Path to folder containing xyz file for initial configuration')
parser.add_argument('--lt',dest='lt_folder',help='Path to folder containing lt files for simulation')
parser.add_argument('--slurm',dest='slurm',action='store_true',help='If true use slurm queueing system else run job from console')
parser.add_argument('--suffix',dest='suffix',type=str,default='',help="Suffix to add onto the end of the preformatted simulation folder name.")

args = parser.parse_args()

epsilons = (args.eAA,args.eBB,args.eAB)

project_path = os.path.abspath(os.path.expanduser(args.dest_folder))

sim = smc.Simulation(total_monomers=args.total_monomers,
                     monomer_attractions=epsilons,
                     p = args.p,monomer_A_fraction=args.fA,
                     angle_strength=args.angle,
                     lt_dir = os.path.expanduser(args.lt_folder),
                     xyz_dir = os.path.expanduser(args.xyz_folder),
                     send_to_cluster=args.send_to_cluster,
                     servername=args.servername)

sim.compile_simulation()
if not args.send_to_cluster:
    sim.move_simulation_files(project_path,slurm=args.slurm)
    sim.start_simulation(slurm  = args.slurm,singularity=os.path.expanduser("~/copoly_bondreact_kspace.sif"))
else:
    sim.move_simulation_files_remote(project_path,slurm=args.slurm,suffix=args.suffix)
    sim.start_simulation_remote()


#!usr/lib/python2.7
# Auther    : Haonan Li <haonanl5@student.unimelb.edu.au>
# Porpuse   : Circular Binary Segmentation algorithm

import getopt
import sys
import numpy as np
import math
import matplotlib.pyplot as plt

# input the bins
def input_bin(bin_file):
    bin_f = open(bin_file,'r')
    # read table title
    bin_f.readline()
    # bins and position index
    bins = []
    index = []
    for line in bin_f.readlines():
        line = line.strip().split()
        # average copy number
        acn = float(line[3]) / (int(line[2])-int(line[1]))
        index.append((int(line[1]),int(line[2])))
        bins.append(acn)
    ch = line[0]
    bin_f.close()
    return bins,index,ch

# visulization log-ratio with highlight cnv
def plot_cnv(X, cnvi):
    x = np.array(range(len(X)))
    # y1 = np.zeros((len(X)))
    # plt.plot(x,y1)
    # plot all log-ratio
    plt.scatter(x,X,s=10)
    cnvx = []
    cnvy = []
    for seg in cnvi:
        cnvx += [i for i in range(seg[0],seg[1])]
        cnvy += list(X[seg[0]:seg[1]])
    # plot cnvs
    plt.scatter(cnvx,cnvy,s=10,marker='*',color='r')
    plt.show()

# output the segments with abs average log-ratio >= 0.1
def output(X,D,index,out_file,ch):
    out_f = open(out_file,'w')
    # change point index
    cpi = np.where(D >= 1)
    cpi = cpi[0]
    # change points
    cps = [index[i][0] for i in cpi]
    # save CNV index and CNVs
    cnvi = []
    cnv = []
    for i in range(len(cps)-1):
        T = X[cpi[i]+1:cpi[i+1]]
        if len(T)>0:
            avg = np.average(T)
        else:
            avg = 0.0
        if abs(avg) >= 0.1:
            cnvi.append((cpi[i]+1,cpi[i+1]))
            cnv.append((cps[i],cps[i+1],avg))
    # output cnv
    for seg in cnv:
        out_f.write('%s\t%d\t%d\t%f\n'%(ch,seg[0],seg[1],seg[2]))
    out_f.close()
    # plot
    # plot_cnv(X,cnvi)

# recursive cbs
def cbs(X,I,D,t):

    # too near
    n = len(X)
    if n < 2:
        return D

    # S : cumulative sum of first n bins
    S = np.zeros((n))
    for i in range(n):
        S[i] = X[0:i+1].sum()

    # Z :
    Z = np.zeros((n,n))
    for i in range(0,n-1):
        for j in range(i+1,n):
            coef = 1 / math.sqrt(1.0/(j-i) + 1.0/(n-j+i+1))
            factor = (S[j]-S[i])/(j-i) - (S[n-1]-S[j]+S[i])/(n-j+i)
            Z[i,j] = abs(coef * factor)

    # find max value and position
    maximum = np.max(Z)
    if maximum < t:
        return D
    else:
        # find change points recursively
        pos = np.where(Z == maximum)
        pos1 = min(pos[0][0],pos[1][0])+1
        pos2 = max(pos[0][0],pos[1][0])+1

        # boundary detecta and single extremum ignore
        if pos2 - pos1 <= 1 or pos2 - pos1 >= n-1:
            return D
        if pos1 > 1:
            D[I[pos1]] += 1
        if pos2 < n-1:
            D[I[pos2]] += 1

        X1 = X[pos1:pos2]
        X2 = np.append(X[0:pos1],X[pos2:n])

        I1 = I[pos1:pos2]
        I2 = np.append(I[0:pos1],I[pos2:n])

        D1 = cbs(X1,I1,D,t)
        D2 = cbs(X2,I2,D,t)

        return D1 + D2

# init the log-ratio data
def cbs_init(bin_file, threshold, out_file):

    # CBS initialization
    bins,index,ch = input_bin(bin_file)

    # speed up with numpy
    bins = np.array(bins)
    n = len(bins)

    # X : log ratios of each bin
    X = np.log2(bins / np.median(bins[:n/3]))
    X[X > 2] = 0
    X[X < -5] = 0

    # I : index of X
    I = np.arange(0,n)

    # D : segmentation index
    D = np.zeros((n))
    D[0] = 1
    D[n-1] = 1

    # call cbs
    cbs(X,I,D,threshold)

    # output
    output(X,D,index,out_file,ch)

# Usage of the tool
def usage():
   print ("usage:python phaser.py [options] ... [-b bins | -z threshold] ...")
   print ("Options and arguments:")
   print ("-h     :Help")
   print ("-b     :Input bins.")
   print ("-z     :Z threshold")
   print ("-o     :Output file")

def main(argv):
    try:
        opts, args = getopt.getopt(argv[1:], "hb:z:o:", \
        ["bin=", "threshold=", "output="])
    except getopt.GetoptError:
        usage()
        sys.exit(2)
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-b', '--bin='):
            bin_file = arg
        elif opt in ('-z', '--threshold='):
            threshold = float(arg)
        elif opt in ('-o', '--output='):
            out_file = arg

    # Make sure the parameters were defined
    if not('bin_file' in locals().keys()) or \
       not('threshold' in locals().keys()) or \
       not('out_file' in locals().keys()):
        usage()
        sys.exit()

    # main process
    cbs_init(bin_file, threshold, out_file)

if __name__ == "__main__":
    main(sys.argv)


import os
from flask import Flask
from flask_cors import CORS
from flask_jwt_extended import (
    JWTManager, jwt_required, create_access_token,
    get_jwt_identity
)
from flask_pymongo import PyMongo
from celery import Celery

def make_celery(app):
    celery = Celery(
        app.import_name,
        backend=app.config['CELERY_RESULT_BACKEND'],
        broker=app.config['CELERY_BROKER_URL']
    )
    celery.conf.update(app.config)

    class ContextTask(celery.Task):
        def __call__(self, *args, **kwargs):
            with app.app_context():
                return self.run(*args, **kwargs)

    celery.Task = ContextTask
    return celery

app = Flask(__name__)
app.config.from_object(os.environ['APP_SETTINGS'])
application = app
mongo = PyMongo(app)
CORS(app)
jwt = JWTManager(app)
celery = make_celery(app)

from src import api


from keras.models import Model
from keras.models import Sequential
from keras.models import load_model
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Input, Dense, Dropout, Flatten, Activation, Reshape
from keras.layers.convolutional import Conv2D, ZeroPadding2D
from keras.layers.pooling import MaxPooling2D, AveragePooling2D
from keras.optimizers import SGD, Adam, Adadelta
from keras.utils import multi_gpu_model
from keras.layers.normalization import BatchNormalization
import tensorflow as tf
import sys
def build_model():

    input_img = Input(shape=(48, 48, 1))

    block1 = Conv2D(64, (5, 5), padding='valid', activation='relu')(input_img)
    block1 = ZeroPadding2D(padding=(2, 2), data_format='channels_last')(block1)
    block1 = MaxPooling2D(pool_size=(5, 5), strides=(2, 2))(block1)
    block1 = ZeroPadding2D(padding=(1, 1), data_format='channels_last')(block1)

    block2 = Conv2D(64, (3, 3), activation='relu')(block1)
    block2 = ZeroPadding2D(padding=(1, 1), data_format='channels_last')(block2)

    block3 = Conv2D(64, (3, 3), activation='relu')(block2)
    block3 = AveragePooling2D(pool_size=(3, 3), strides=(2, 2))(block3)
    block3 = ZeroPadding2D(padding=(1, 1), data_format='channels_last')(block3)

    block4 = Conv2D(128, (3, 3), activation='relu')(block3)
    block4 = ZeroPadding2D(padding=(1, 1), data_format='channels_last')(block4)

    block5 = Conv2D(128, (3, 3), activation='relu')(block4)
    block5 = ZeroPadding2D(padding=(1, 1), data_format='channels_last')(block5)
    block5 = AveragePooling2D(pool_size=(3, 3), strides=(2, 2))(block5)
    block5 = Flatten()(block5)

    fc1 = Dense(1024, activation='relu')(block5)
    fc1 = Dropout(0.5)(fc1)

    fc2 = Dense(1024, activation='relu')(fc1)
    fc2 = Dropout(0.5)(fc2)

    predict = Dense(7)(fc2)
    predict = Activation('softmax')(predict)
    model = Model(inputs=input_img, outputs=predict)

    # opt = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
    # opt = Adam(lr=1e-3)
    opt = Adadelta(lr=0.1, rho=0.95, epsilon=1e-08)
    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
    model.summary()
    return model

import os
import numpy as np
import argparse
import time
import pickle
import csv
from math import log, floor
import matplotlib.pyplot as plt
os.system('echo $CUDA_VISIBLE_DEVICES')
PATIENCE = 5 # The parameter is used for early stopping

def _shuffle(X, Y):
    randomize = np.arange(len(X))
    np.random.shuffle(randomize)
    return (X[randomize], Y[randomize])
def split_valid_set(X_all, Y_all, percentage):
    all_data_size = len(X_all)
    print('len(X_all)=',len(X_all))
    valid_data_size = int(floor(all_data_size * percentage))

    X_all, Y_all = _shuffle(X_all, Y_all)

    X_valid, Y_valid = X_all[0:valid_data_size], Y_all[0:valid_data_size]
    X_train, Y_train = X_all[valid_data_size:], Y_all[valid_data_size:]

    return X_train, Y_train, X_valid, Y_valid

def load_pickle(file):
    f=open(file,'rb')
    data=np.load(f)

    return (data)

def datatonpy(csvFile,k):
    '''
    trp=open('train_pixels', 'w')
    trl=open('train_labels', 'w')
    vap=open('valid_pixels', 'w')
    val=open('valid_labels', 'w')
    '''
    f=open(csvFile, 'r')
    s=np.loadtxt(f,delimiter=',',skiprows=1,dtype='str')
    l=np.array(s[:,0])
    print (l)
    p=np.zeros((l.shape[0],48*48))
    for i in range(l.shape[0]):
        p[i]=np.array(s[i][1].split(' '))
    print(l.shape[0])
    print(p[0])

#        p=np.array(s[:][1])
 #       l=np.array(s[:][0])

  #  (tp,tl,vp,vl)=split_valid_set(p, l, 0.1)
  #  np.save('train_pixels.pkl',tp)
  #  pickle.dump(tp, trp)
    if k==0:
        np.save('train_pixels',p)
        np.save('train_labels',l)
   # np.save('valid_pixels',vp)
   # np.save('valid_labels',vl)
    return (l,p)
    #    print(f.shape)
    '''
        for i, line in enumerate(f):
            data = line.split(',')
            label = data[0]
            pixel = data[1]
            l=np.array(label)
			p=np.array(pixel.split(' ')
    '''

def main():

    epoch=1
    batch=64
    pretrain=True
    save_every=1
    model_name='modelok'

# ====  load_data  ====
   # datatonpy('train.csv',0)
#=======================

#    To begin with, you should first read your csv training file and
#    cut them into training set and validation set.
#   Such as:

#    In addition, we maintain it in array structure and save it in pickle

    # start training
    train(batch, epoch, pretrain, save_every,

          model_name)
def train(batch_size, num_epoch, pretrain, save_every, model_name=None):

    sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
    '''
    "1 Epoch" means you have been looked all of the training data once already.
    Batch size B means you look B instances at once when updating your parameter.
    Thus, given 320 instances, batch size 32, you need 10 iterations in 1 epoch.
    '''

    if pretrain == False:
        np.save('valid_l',val_labels)
        np.save('valid_p',val_pixels/255.0)
        np.save('train_p',train_pixels)
        np.save('train_l',train_labels)

        model=Sequential()

        model.add(Conv2D(64, (3, 3), padding='same',
                         input_shape=(48,48,1),
                         activation='relu'))
        model.add(Conv2D(64, (3, 3), activation='relu'))
        model.add(MaxPooling2D(pool_size=(3, 3)))
        model.add(Dropout(0.2))

        model.add(Conv2D(128, (3, 3), padding='same',
                         activation='relu'))
        model.add(Conv2D(128, (3, 3), activation='relu'))
        model.add(MaxPooling2D(pool_size=(2, 2)))
        model.add(Dropout(0.2))

        model.add(Conv2D(128, (3, 3), padding='same',
                         activation='relu'))
        model.add(Conv2D(128, (3, 3), activation='relu'))
        model.add(AveragePooling2D(pool_size=(2, 2)))
        model.add(Dropout(0.2))

        model.add(Flatten())
 #       model.add(Dense(30, activation='relu'))
        model.add(Dense(512, activation='relu'))
        model.add(Dropout(0.7))

        model.add(Dense(7, activation='softmax'))
        #opt = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=False)
        opt = Adam(lr=5e-4)
        #opt = Adadelta(lr=0.1, rho=0.95, epsilon=1e-08)
        model.compile(loss='categorical_crossentropy',optimizer=opt,metrics=['accuracy'])
        history=model.fit(train_pixels,train_labels,batch_size=64,epochs=20,validation_data=(val_pixels/255,val_labels))

#        model.fit(train_pixels,train_labels,batch_size=32,epochs=3,validation_data=(val_pixels/255,val_labels))
    else:
        model = load_model('modelok')

    epochss=3

    # compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)

  # model.fit_generator(datagen.flow(train_pixels, train_labels, batch_size=32),
     #               steps_per_epoch=3000, epochs=8,validation_data=(val_pixels/255,val_labels))

# fits the model on batches with real-time data augmentation:
 # parallel_model = multi_gpu_model(model, gpus=8)
   # for i in range(epochss):

    model.save('modelok')

    (no,test)=datatonpy(sys.argv[1],1)
    test=np.reshape(test,(-1,48,48,1))
    np.save('testnp',test)
    test=np.load('testnp.npy')
    y=model.predict(test/255.0)
    yl=np.zeros((len(y),2),dtype=int)
    for i in range(len(y)):
        max=0
        pos=0
        for j in range(7):
            if y[i][j]>max:
                max=y[i][j]
                pos=j

        yl[i][0]=i
        yl[i][1]=pos

  #  out=open('predict.csv','w')
   # out.write('id,value\n')
    np.savetxt(sys.argv[2],yl,fmt='%d',delimiter=',',header='id,label',comments='')

if __name__=='__main__':
    main()


def solution(n, results):
    answer = 0
    win = [[] for _ in range(n)]
    lost = [[] for _ in range(n)]
    for res in results:
        p, c = res[0]-1, res[1]-1
        win[p].append(c)
        lost[c].append(p)

    for i in range(n):
        for p in win[i]:
            for c in win[p]:
                if not c in win[i]:
                    win[i].append(c)
        for c in lost[i]:
            for p in lost[c]:
                if not p in lost[i]:
                    lost[i].append(p)

    for i in range(n):
        count = len(win[i]) + len(lost[i])
        if count == n-1:
            answer += 1
    return answer


from decimal import Decimal
from xml.dom import minidom
import datetime
from dimagi.utils.parsing import json_format_datetime

def serialize(value):
    """
    Serializes a value so it can properly be parsed into XML
    """
    if isinstance(value, datetime.datetime):
        return json_format_datetime(value)
    elif isinstance(value, datetime.date):
        return value.isoformat()
    elif isinstance(value, datetime.time):
        return value.strftime('%H:%M:%S')
    elif isinstance(value, (int, Decimal, float)):
        return str(value)
    else:
        return value if value is not None else ""

def indent_xml(xml_string):
    """
    Takes an xml string and returns an indented and properly tabbed version of the string
    """
    if isinstance(xml_string, str):
        xml_string = xml_string.encode('utf-8')
    return minidom.parseString(xml_string).toprettyxml()


from flask import Flask, render_template, request
import rf_send

app = Flask(__name__)

@app.route('/', methods=["GET", "POST"])
def index():
    if request.method == "POST":
        if request.form.get("lighton"):
            rf_send.control_light("on")
        elif request.form.get("lightoff"):
            rf_send.control_light("off")

    return render_template('index.html')

if __name__ == "__main__":
    app.run(debug=True, host='0.0.0.0')


import requests,os, re
import shutil

nile="7.0W"
astra='19.2E'
thor='0.8W'
hispa='30.0W'
badr='26.0E'
hotbird='13.0E'
cookies = {
    '_ga': 'GA1.2.1469267885.1651189139',
    '_gid': 'GA1.2.1013217857.1651189139',
    '_gat_gtag_UA_167951831_1': '1',
}

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    # 'Accept-Encoding': 'gzip, deflate, br',
    'Referer': 'https://picon.cz/download-picons/',
    'Connection': 'keep-alive',
    # Requests sorts cookies= alphabetically
    # 'Cookie': '_ga=GA1.2.1469267885.1651189139; _gid=GA1.2.1013217857.1651189139; _gat_gtag_UA_167951831_1=1',
    'Upgrade-Insecure-Requests': '1',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
}

r1 = requests.get('https://picon.cz/download-picons/picontransparent-400x170/', cookies=cookies, headers=headers)

#sats=["7.0W",'19.2E','0.8W','30.0W','26.0E','13.0E']
sats=["7.0W"]
links=[]
for i in sats:
    a1=r1.content.find(i)
    a2=r1.content[:a1].rfind('href="')
    a3=r1.content[a2:a1]
    a4=re.findall(r'href="(.*?)"',a3)[0]
    reg="("+i+".*?)\t"
    links.append([a4,re.findall(reg,r1.content)[0]])

if os.path.exists(r"/tmp/ere"):
    shutil.rmtree(r"/tmp/ere", ignore_errors=True)
    os.makedirs(r"/tmp/ere")
else:
    os.makedirs(r"/tmp/ere")
for i in links:
    r = requests.get(i[0], cookies=cookies, headers=headers)

    with open(r'/tmp/ere/' +i[1], 'wb') as out:
        out.write(r.content)
    print('File '+i[1]+' is created')


from torch.autograd.grad_mode import no_grad
from .base import base
import argparse
import torch
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d, Axes3D
import sys
import numpy as np

from derivs import *

# generates point in the interior and on the boundary of a unit ball in d dimensions.
# It also puts all of them together

def generate_points(dimension, npoints_interior, npoints_boundary):

    #(Muller method)
    # It returns an array

    x_array = []
    ## Points in the interior
    for i in range(npoints_interior):
        u = np.random.normal(0,1,dimension)
        r = np.random.rand()**(1.0/dimension)
        w = r*u/np.sqrt(np.sum(u**2))
        x_array.append(w)

    ## Points on the boundary

    for i in range(npoints_boundary):
        u = np.random.normal(0,1,dimension)
        w = u/np.sqrt(np.sum(u**2))
        x_array.append(w)

    return x_array

# defines the network for the PDE problem
class network_PDE(torch.nn.Module):
    def __init__(self):
        # call constructor from superclass
        super().__init__()

        self.depth = 3

        self.width = 200

        # here define the layers
        # first layer
        self.linear_input = torch.nn.Linear(2, self.width)
        # middle layers
        self.linear_layers = torch.nn.ModuleList([torch.nn.Linear(self.width, self.width) for i in range(self.depth)])
        # final layer
        self.linear_output = torch.nn.Linear(self.width, 1)

        # here we initialize the network
        torch.nn.init.xavier_normal_(self.linear_input.weight, gain=torch.sqrt(torch.tensor(1.)))

        for i in range(self.depth):
            torch.nn.init.xavier_normal_(self.linear_layers[i].weight, gain=torch.sqrt(torch.tensor(1.)))

        torch.nn.init.xavier_normal_(self.linear_output.weight, gain=torch.sqrt(torch.tensor(1.)))

    def forward(self, x):
        # This is a simple residual NN.
        # The difference with a fully connected is the x+ ... in each intermediate layer
        # notice that by design all the intermediate layers have the same width

        x = torch.sigmoid(self.linear_input(x))

        for layer in (self.linear_layers):
            x = x + torch.sigmoid(layer(x))

        x = self.linear_output(x)

        return x

class PDE_PoissonD(base):
    def __init__(self):
        super().__init__()

    def parser(self):
        parser = argparse.ArgumentParser(
            description="Using NNs to solve Poisson equations in arbitrary dimensions", add_help=False
        )
        parser.add_argument(
            "--iterations",
            default=1000,
            type=int,
            metavar="iterations_per_epoch",
            help="Number of iterations for each epoch.",
        )

        parser.add_argument(
            "--progress",
            default="true",
            type=str,
            metavar="progress",
            help="Show the progress bar, default true.",
        )

        parser.add_argument(
            "--feature",
            default=0,
            type=int,
            metavar="feature",
            help="The feature number to add to the loss. Defaults to 0 (no feature).",
        )

        return parser

    # Here the different features to add to the loss

    def feature_1(self):
        # just \theta^2 (L2)
        barrier_smooth = torch.tensor(0., device=self.device)
        for param in self.net.parameters():
            barrier_smooth += 2e-4*torch.sum(torch.pow(param,2))
        return barrier_smooth

    def feature_42(self):
        # This is an "empty" feature.
        barrier_smooth = torch.tensor(0., device=self.device)
        return barrier_smooth

    def initialize(self, args, files):
        super().initialize(args, files)

        # If "True" generates new points in the domain at each epoch (adds noise)
        self.update_points = False

        self.dtype = torch.float32

        self.iterations_per_epoch = args.iterations
        self.criterion = torch.nn.MSELoss()

        self.epochs = args.epochs
        self.progress = args.progress

        self.criterion_test = torch.nn.L1Loss()

        self.stats["loss train"] = list()
        self.stats["solution"] = list()

        self.stats["losses check"] = list()
        self.stats["parameters"] = list()
        self.stats["inputs"] = list()

        ## The equation is Delta u +coeff*u^2 = inhom
        ## we impose also Dirichlet boundary conditions

        self.dimension = 2
        self.non_linearity = 1.0
        self.analytic_sol = lambda x: torch.sin(20*torch.norm(x,dim = 1)**2)**2
        self.inhom_term = lambda x: 0.125*( 3-4*(1+6400*torch.norm(x,dim = 1)**2)*torch.cos(40*(torch.norm(x,dim = 1)**2))+torch.cos(80*(torch.norm(x,dim = 1)**2))-640*torch.sin(40*(torch.norm(x,dim = 1)**2)))
        self.Dirichlet_value = 0.833469

        # network
        self.net = network_PDE().to(self.device)

        self.npoints_interior = int(1e4)
        self.npoints_boundary = int(1e3)

        # The weight factor of the boundary conditions in the loss
        self.bc_weight = 100000.0

        #If this is 1 it saves the full model at each epoch, otherwise only each self.saving.epochs epochs
        # The features, loss, and plots are always saved after each epoch
        self.saving_epochs = 1000

        ## Remember to add the feature also here!
        if args.feature == 1:
            self.feature = self.feature_1

        elif args.feature == 42:
            self.feature = self.feature_42

        else:
            if (self.progress == 'true'): print("Feature not defined, or no feature specified. Working without features")
            self.feature = lambda : torch.tensor(0., device=self.device)

        #Points
        x_array = generate_points(self.dimension, self.npoints_interior, self.npoints_boundary)

        self.x = torch.tensor(x_array, device=self.device, requires_grad=True, dtype = self.dtype)

        ### Check the analytic solution
        points = torch.tensor(x_array, device='cpu', requires_grad=True, dtype = self.dtype)
        f1 = self.analytic_sol(points)
        f1p = diff_single(f1, points, device = 'cpu')
        f1pp = diff_many(f1p, points, device = 'cpu')

        lap = torch.zeros_like(f1pp[0][0], device='cpu')
        for i in range(self.dimension): lap-= f1pp[i][i]
        diff_eq1 =lap + self.non_linearity*(f1**2) - self.inhom_term(points)

        if (self.progress == 'true'): print("L1 error of the analytic solution (should be small): ", torch.norm(diff_eq1).item())
        if (self.update_points == False): self.stats["inputs"].append(self.x.tolist())

    def train(self, epoch, optimizer):
        super().train(epoch, optimizer)

        for i in range(self.iterations_per_epoch):

            y = self.net(self.x)
            loss = self.get_loss(y)

            # kill if there are nans
            if (torch.isnan(loss).any().item() == True): sys.exit()

            #bp()
            optimizer.zero_grad()
            loss.backward()

            if  self.progress == "true":
                if i == 0: print("Starting loss: ", loss.item())

            def closure():
                return loss

            optimizer.step(closure)
            if  self.progress == "true":
                self.progress_bar.next(
                    i, self.iterations_per_epoch, "Loss: %.3f" % (loss.item())
                )

        if (self.update_points == True):
            x_array = generate_points(self.dimension, self.npoints_interior, self.npoints_boundary)
            self.x = torch.tensor(x_array, device=self.device, requires_grad=True, dtype = self.dtype)

        self.stats["loss train"].append(loss.item())
        self.stats["solution"].append(y.tolist())

        if (epoch%self.saving_epochs == 0) or (epoch == self.start_epoch+self.epochs-1):
            new_list = list(self.net.parameters())
            list_pars = []
            for item in new_list:
                list_pars.append(item.tolist())

            self.stats["parameters"].append(list_pars)

    def test(self, epoch):
        super().test(epoch)

        y = self.net(self.x)
        losses = self.get_loss_test(y)
        if  self.progress == "true": print(losses)

        #if (epoch%self.saving_epochs == 0):
        state = {
            "net": self.net.state_dict(),
            "epoch": epoch,
        }

        torch.save(state, self.files["checkpoint"])

        self.stats["losses check"].append(losses)

         # saving plot loss
        plt.figure(1)

        # set the title
        plt.suptitle("Inhomogeneous Poisson")
        plt.suptitle(" ".join(sys.argv[1:]))

        ii = 0
        for loss in list(zip(*self.stats["losses check"])):

            if (ii == 0): lab ="log10(L1 error eq 1)"
            if (ii == 1): lab ="log10(L1 error bc 1)"
            if (ii == 2): lab ="log10(max abs error eq 1)"

            plt.plot(
                self.stats["epoch"],
                np.log10(loss),
                "--.",
                #color="tab:red",
                label = lab,
            )
            plt.grid()
            ii += 1

        plt.setp(plt.gca().spines.values(), color="#bfbfbf")
        lgd = plt.gca().legend()
        lgd.set_frame_on(True)

        # save the image
        plt.savefig(self.files["plot_check_losses"])
        plt.clf()
        plt.close()

    def save(self, epoch):
        super().save(epoch)

        # saving plot loss
        plt.figure(1)

        # set the title
        plt.suptitle("Solving higher dimensional PDE")
        plt.suptitle(" ".join(sys.argv[1:]))

        plt.plot(
            self.stats["epoch"],
            np.log10(self.stats["loss train"]),
            "--.",
            color="tab:red",
            label="log10(loss)",
        )
        plt.grid()

        plt.setp(plt.gca().spines.values(), color="#bfbfbf")
        lgd = plt.gca().legend()
        lgd.set_frame_on(True)

        # save the image
        plt.savefig(self.files["plot"])
        plt.clf()
        plt.close()

        #save the first function

        starting_boundary =self.npoints_interior

        #plot a radial slice
        ax = plt.axes()

        points_plot = []
        elems = np.linspace(-1, 1, 1000)
        xs = np.array(elems)

        # create the radial slice
        for elem in elems:
            temp = np.zeros(self.dimension-1)
            temp = np.insert(temp,0,elem)
            points_plot.append(temp)

        xs2 = torch.tensor(points_plot, device = self.device, dtype = self.dtype, requires_grad = True)

        with torch.no_grad():
            pred = np.array(self.net(xs2).tolist())
            analytic = np.array(self.analytic_sol(xs2).tolist())

        ax.plot(xs,
            pred,
            ".",
            color="tab:orange",
            label="NN")
        ax.plot(xs,
            analytic,
            ".",
            color="tab:blue",
            label="analytic")
        #bp()

        plt.grid()

        plt.setp(plt.gca().spines.values(), color="#bfbfbf")
        lgd = plt.gca().legend()
        lgd.set_frame_on(True)

        # save the image
        plt.savefig(self.files["plot_function"])
        plt.clf()
        plt.close()

        if  self.progress == "true": print("Image saved as: %s" % self.files["plot"])

    def get_loss(self, y):
        f1 = y.transpose(0,1)[0]

        f1p = diff_single(f1, self.x, self.device)
        f1pp = diff_many(f1p, self.x, self.device)

        starting_boundary =self.npoints_interior

        lap = torch.zeros_like(f1pp[0][0], device=self.device)
        for i in range(self.dimension): lap-= f1pp[i][i]
        diff_eq1 =lap + self.non_linearity*(f1**2) - self.inhom_term(self.x)

        eqn1 =  self.criterion(diff_eq1, torch.zeros_like(diff_eq1))+self.feature()

        if self.bc_weight == 0: return eqn1
        else:
            bc1 =  self.criterion(f1[starting_boundary:], torch.full_like(f1[starting_boundary:],self.Dirichlet_value))
            return eqn1+self.bc_weight*bc1

    def get_loss_test(self, y):

        f1 = y.transpose(0,1)[0]

        f1p = diff_single(f1, self.x, self.device)
        f1pp = diff_many(f1p, self.x, self.device)

        starting_boundary =self.npoints_interior

        bc1 =  self.criterion_test(f1[starting_boundary:], torch.full_like(f1[starting_boundary:],self.Dirichlet_value))

        lap = torch.zeros_like(f1pp[0][0], device=self.device)
        for i in range(self.dimension): lap-= f1pp[i][i]
        diff_eq1 =lap + self.non_linearity*(f1**2) - self.inhom_term(self.x)

        eqn1 =  self.criterion_test(diff_eq1, torch.zeros_like(diff_eq1))

        max1 = (torch.max(torch.abs(diff_eq1)))

        return (eqn1.item(), bc1.item(), max1.item())


import os
from django.http import FileResponse
import pandas as pd
import numpy as np
from rest_framework.views import APIView, Response, status
from django.core.paginator import Paginator
import re

from .models import Algorithm, Result
from file.models import File
from .lib.run import train
from utils.db import redis_conn

class GetAlgorithmView(APIView):
    """查询全部模型及算法
    """
    def post(self, request):
        pages = Paginator(Algorithm.objects.order_by("name"), int(request.data.get("size")))
        algos = pages.get_page(request.data.get("currentPage")).object_list
        res = {
            "status": 200,
            "message": "查询成功",
            "content": {
                "columns": [
                    {
                        "name": "模型名称",
                        "value": "modelName"
                    },
                    {
                        "name": "模型描述",
                        "value": "modelDescription",
                        "tooltip": True
                    },
                    {
                        "name": "数据集",
                        "value": "dataset"
                    },
                    {
                        "name": "算法名称",
                        "value": "neuralNetwork"
                    },
                    {
                        "name": "预测目标",
                        "value": "target"
                    },
                    {
                        "name": "训练状态",
                        "value": "status",
                    },
                    {
                        "name": "训练进度",
                        "value": "progress",
                    }
                    ],
                "data": [],
                "componentTitle": "",
                "pageTotal": pages.count
            }
        }

        for algo in algos:
            a = {
                "id": algo.pk,
                "modelName": algo.name,
                "modelDescription": algo.description,
                "dataset": algo.dataset.name,
                "neuralNetwork": algo.neuralNetwork,
                "target": algo.target,
                "status": algo.status
            }
            if redis_conn.get(algo.pk) is not None:
                progress = redis_conn.get(algo.pk).decode()
                a['progress'] = progress
            else:
                a['progress'] = 0

            res["content"]["data"].append(a)
        return Response(res, status=status.HTTP_200_OK)

class AlgorithmView(APIView):
    def post(self, request):
        '''新增模型和算法
        '''
        # 未对算法/target等进行进一步验证，日后可进行完善
        dataset = File.objects.get(pk=request.data.get("dataset"))
        if dataset is not None:
            algo = Algorithm(
                description = request.data.get("description"),
                name = request.data.get("name"),
                neuralNetwork = request.data.get("neuralNetwork"),
                layers = request.data.get("layers"),
                learningRate = request.data.get("learningRate"),
                neurons = request.data.get("neurons"),
                epoch = request.data.get("rounds"),
                batchSize = request.data.get("batchSize"),
                optimization = request.data.get("optimization"),
                selected = request.data.get("selected"),
                target = request.data.get("target"),
                verificationRate = request.data.get("verificationRate"),
                dataset = dataset,
            )
            try:
                algo.save()
                res = {
                    "status": 200,
                    "content": "上传成功"
                }
                return Response(res, status=status.HTTP_200_OK)
            except:
                res = {
                    "status": 500,
                    "content": "模型名称重复"
                }
                return Response(res, status=status.HTTP_400_BAD_REQUEST)
        else:
            res = {
                "status": 500,
                "content": "数据集不存在"
            }
            return Response(res, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request):
        """删除模型及算法
        """
        algo = Algorithm.objects.get(pk=request.data.get("id"))
        result = Result.objects.get(algo=algo)
        if algo is not None and result is not None:
            print(algo)
            # 删除神经网络模型
            img_path = [result.difference, result.loss]
            for p in img_path:
                os.remove(p[6:])
            model_name = algo.name + '.pth'
            os.remove('./torch_models/'+model_name)
            # 删除数据库内算法记录
            algo.delete()
            res = {
                "status": 200,
                "content": "删除成功"
            }
            return Response(res, status=status.HTTP_200_OK)
        else:
            res = {
                "status": 500,
                "content": "数据集不存在"
            }
            return Response(res, status=status.HTTP_400_BAD_REQUEST)

def analyze_csv(file_path):
    '''分析CSV文件
    '''
    # 读取 CSV 文件
    df = pd.read_csv(file_path)
    global current_file_row, current_file_column
    current_file_row, current_file_column = df.shape
    # 存放所有列的分析结果
    all_columns_analysis = []
    # 从数据框中随机选择一个实例值
    random_row = df.sample(1).iloc[0]
    count_i = 1
    for column in df.columns:
        # 列的分析结果
        column_analysis = {"columnName": column}
        column_analysis["id"] = str(count_i)
        # 数据类型
        dtype = df[column].dtype
        if dtype == "object":
            column_analysis["dataType"] = "string"
            column_analysis["min"] = "-"
            column_analysis["max"] = "-"
        else:
            column_analysis["dataType"] = str(dtype)
            column_analysis["min"] = str(np.float64(df[column].min()))
            column_analysis["max"] = str(np.float64(df[column].max()))
        # 为每一列添加一个实例值
        column_analysis["exampleData"] = str(random_row[column])
        count_i += 1
        # 将列的分析结果添加到总列表中
        all_columns_analysis.append(column_analysis)
    return np.array(all_columns_analysis)

class TrainingView(APIView):
    def post(self, request):
        algo: Algorithm = Algorithm.objects.get(pk=request.data.get("id"))
        if algo is not None:
            # 搭配celery在后台进行训练
            train.delay(algo.pk)
            res = {
                "status": 200,
                "message": "提交成功",
            }
            return Response(res, status=status.HTTP_200_OK)
        else:
            res = {
                "status": 500,
                "content": "算法不存在"
            }
            return Response(res, status=status.HTTP_400_BAD_REQUEST)

class ResultView(APIView):
    def get(self, request, image):
        print("--", os.getcwd())
        image_path = os.path.join('result', image)

        print(image_path)
        if os.path.exists(image_path):
            response = FileResponse(open(image_path, 'rb'), content_type='image/png')
            response['Content-Disposition'] = f'inline; filename={image}'
            return response
        else:
            return Response(status=status.HTTP_404_NOT_FOUND)
    def post(self, request):
        algo: Algorithm = Algorithm.objects.get(pk=request.data.get("id"))
        if algo is not None:
            result = Result.objects.get(algo=algo)
            if result is not None:
                res = {
                    "status": 200,
                    "message": "查询成功",
                    "content": {
                        "mse": str(result.mse),
                        "rmse": str(result.rmse),
                        "mae": str(result.mae),
                        "difference": result.difference,
                        "loss": result.loss,
                    },
                }
                return Response(res, status=status.HTTP_200_OK)
            else:
                res = {
                    "status": 500,
                    "message": "训练未完成"
                }
                return Response(res, status=status.HTTP_400_BAD_REQUEST)
        else:
            res = {
                "status": 500,
                "message": "算法不存在"
            }
            return Response(res, status=status.HTTP_400_BAD_REQUEST)


from typing import List, Tuple, Dict, Any, Type

from .objects import Route, Middleware, Listener

__all__ = (
    'ResourceMeta',
    "Resource",
)

class ResourceMeta(type):
    """
    A metaclass for resources.
    """
    __routes__: Dict[str, Route]
    __listeners__: List[Listener]
    __middlewares__: List[Middleware]
    __url_prefix__: str

    def __new__(cls, cls_name: str, bases: Tuple[Type[Any]], attrs: Dict[str, Any], **kwargs):
        attrs['__resource_name__'] = kwargs.get('name', cls_name)

        routes: Dict[str, Route] = {}
        listeners = []
        middlewares = []

        url_prefix = kwargs.get('url_prefix', '')
        self = super().__new__(cls, cls_name, bases, attrs, **kwargs)

        for base in reversed(self.__mro__):
            for elem, value in base.__dict__.items():
                if isinstance(value, Route):
                    path = url_prefix + value.path
                    routes[path] = value

                elif isinstance(value, Middleware):
                    middlewares.append(value)

                elif isinstance(value, Listener):
                    listeners.append(value)

        self.__url_prefix__ = url_prefix
        self.__routes__ = routes
        self.__listeners__ = listeners
        self.__middlewares__ = middlewares

        return self

class Resource(metaclass=ResourceMeta):
    """
    A resource to be subclassed and used.

    Example
    -------
    .. code-block:: python3

        import subway

        app = subway.Application()

        class MyResource(subway.Resource):

            @subway.route('/hello/{name}', 'GET')
            async def hello(self, request: subway.Request, name: str):
                return f'Hello, {name}!'

        app.add_resource(MyResource())

    """
    __routes__: Dict[str, Route]
    __listeners__: List[Listener]
    __middlewares__: List[Middleware]
    __url_prefix__: str
    __resource_name__: str

    def __repr__(self) -> str:
        return '<Resource name={0.name!r}>'.format(self)

    @property
    def name(self) -> str:
        """
        The name of the resource.
        """
        return self.__resource_name__

    @name.setter
    def name(self, value):
        if not isinstance(value, str):
            raise TypeError("name must be a string")

        self.__resource_name__ = value

    @property
    def routes(self) -> List[Route]:
        """
        The routes of the resource.
        """
        return list(self.__routes__.values())

    @property
    def listeners(self) -> List[Listener]:
        """
        The listeners of the resource.
        """
        return self.__listeners__

    @property
    def middlewares(self) -> List[Middleware]:
        """
        The middlewares of the resource.
        """
        return self.__middlewares__


from django.shortcuts import render, redirect
from .models import Article
from .forms import ArticleForm

def index(request):
    articles = Article.objects.all()
    context = {
        'articles': articles,
    }
    return render(request, 'article/index.html', context)

def create(request):
    if request.method == 'POST':
        form = ArticleForm(request.POST)
        if form.is_valid():
            article = form.save()
            return redirect('article:detail', article.pk)
    else:
        form = ArticleForm()
    context = {
        'form': form,
    }
    return render(request, 'article/create.html', context)

def detail(request, article_pk):
    article = Article.objects.get(pk=article_pk)
    context = {
        'article': article,
    }
    return render(request, 'article/detail.html', context)

def delete(request, article_pk):
    article = Article.objects.get(pk=article_pk)
    article.delete()
    return redirect('article:index')

def update(request, article_pk):
    article = Article.objects.get(pk=article_pk)
    if request.method == 'POST':
        form = ArticleForm(request.POST)
        if form.is_valid():
            article = form.saver()
            return redirect('article:detail', article.pk)
    else:
        form = ArticleForm(instance=article)
    context = {
        'article': article,
        'form': form,
    }
    return render(request, 'article/create.html', context)


from torch import nn
from torch import cat

class MelCNN(nn.Module):
    def __init__(self):
        super(MelCNN, self).__init__()

        self.conv1 = self.conv_block(1, 16)
        self.conv2 = self.conv_block(16, 32)
        self.conv3 = self.conv_block(32, 64)
        self.conv4 = self.conv_block(64, 64)

        self.flatten = nn.Flatten()
        self.linear = nn.Sequential(nn.Linear(252 * 64, 128),
                                    nn.ReLU(),
                                    nn.Linear(128, 1),
                                    nn.Sigmoid())

    @staticmethod
    def conv_block(n_in, n_out):
        return nn.Sequential(nn.Conv2d(n_in, n_out, (3, 3)),
                             nn.ReLU(),
                             nn.AvgPool2d(2))

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.conv4(x)
        x = self.flatten(x)
        return self.linear(x)

class EncoderBlock(nn.Module):
    def __init__(self, f_in, f_out):
        super(EncoderBlock, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(f_in, f_out, (3, 3), padding=(1, 1)),
            nn.ReLU(),
            # nn.Dropout2d(drop_p),
            nn.Conv2d(f_out, f_out, (3, 3), padding=(1, 1)),
            nn.ReLU(),
            # nn.Dropout2d(drop_p)
        )

        self.pool = nn.AvgPool2d(2)

    def forward(self, x):
        x = self.conv(x)
        return x, self.pool(x)

class DecoderBlock(nn.Module):
    def __init__(self, f_in, f_out):
        super(DecoderBlock, self).__init__()
        self.upconv = nn.Sequential(
            nn.Upsample(scale_factor=2),
            nn.Conv2d(f_in, f_out, (3, 3), padding=(1, 1)),
        )

        self.conv = nn.Sequential(
            nn.Conv2d(f_out * 2, f_out, (3, 3), padding=(1, 1)), nn.ReLU(),
            # nn.Dropout2d(drop_p),
            nn.Conv2d(f_out, f_out, (3, 3), padding=(1, 1)), nn.ReLU(),
            # nn.Dropout2d(drop_p)
        )

    def forward(self, x, e):
        x = self.upconv(x)
        return self.conv(cat((e, x), dim=1))

class Mel2MelCNN(nn.Module):
    def __init__(self):
        super(Mel2MelCNN, self).__init__()

        self.enc1 = EncoderBlock(1, 8)    # [80 1376] -> [40 688]
        self.enc2 = EncoderBlock(8, 16)   # [40 688]  -> [20 344]
        self.enc3 = EncoderBlock(16, 32)  # [20 344]  -> [10 172]
        self.enc4 = EncoderBlock(32, 64)  # [10 172]  -> [5   86]

        self.bneck = nn.Sequential(
            nn.Conv2d(64, 128, (3, 3), padding=(1, 1)), nn.ReLU(),
            # nn.Dropout2d(drop_p),
            nn.Conv2d(128, 128, (3, 3), padding=(1, 1)), nn.ReLU(),
            # nn.Dropout2d(drop_p),
        )

        self.dec1 = DecoderBlock(128, 64)  # [5   86] -> [10  172]
        self.dec2 = DecoderBlock(64, 32)   # [10 172] -> [20  344]
        self.dec3 = DecoderBlock(32, 16)   # [20 344] -> [40  688]
        self.dec4 = DecoderBlock(16, 8)    # [40 688] -> [80 1376]

        self.outconv = nn.Sequential(
            nn.Conv2d(8, 1, (1, 1)), nn.Sigmoid(),
        )

    def forward(self, x):
        x_in = x

        e1, x = self.enc1(x)
        e2, x = self.enc2(x)
        e3, x = self.enc3(x)
        e4, x = self.enc4(x)

        x = self.bneck(x)

        x = self.dec1(x, e4)
        x = self.dec2(x, e3)
        x = self.dec3(x, e2)
        x = self.dec4(x, e1)

        return x_in * self.outconv(x)


# unknown_words.py
# Find English words not in dictionary or exceptions
#
# 2022-07-04 PV

from collections import Counter
# from typing import Counter
from common_fs import get_all_files
from casefix_En import set_casefix, process_exceptions_En
import os

source = r'W:\Livres\A_Trier'
doit = True

unknowns: Counter[str] = Counter()

# dme est l'ensemble des mots anglais
with open(r'words\words.en.txt', 'r', encoding='UTF-8') as f:
    sme = set(mot for mot in f.read().splitlines())
with open(r'words\extra.en.txt', 'r', encoding='UTF-8') as f:
    sme |= set(mot for mot in f.read().splitlines())

def process_word(w: str, first: bool):
    if len(w) <= 1:
        return w
    w0 = w

    while w and str.isdigit(w[0]):
        w = w[1:]
    while w and str.isdigit(w[-1]):
        w = w[:-1]
    if len(w) == 0:   # Only digits
        return w0

    w = w0

    if w in sme or w in set_casefix or ('A' <= w[0] <= 'Z' and w not in set_casefix and w[0].lower()+w[1:] in sme) or w == 'js':
        return w0
    unknowns.update([w])
    # if w == 'CS':
    #     breakpoint()
    return '«'+w0+'»'

def process_segment(s):
    w = ''
    res = ''
    firstword = True
    for c in s:
        if c in [' ', '+', '.', ',', '-']:     # Simple quote is not considered a separator, maybe should only if not sandwich between letters
            # typically {'+', '#', '.', ',', ' ', '-'}
            if w != '':
                res += process_word(w, firstword)
                w = ''
                firstword = c != ' '
            res += c
        else:
            w += c
    if w != '':
        res += process_word(w, firstword)
    # Exceptions
    res = process_exceptions_En(res)
    return res

def process_name(name: str):
    ts = name.split(' - ')
    s1 = ts[0]

    #bp = ''
    try:
        pp = s1.index('(')
        #bp = s1[pp:]
        s1 = s1[:pp]
    except Exception:
        pass
    process_segment(s1)

nt = 0
for filefp in get_all_files(source):
    folder, file = os.path.split(filefp)
    bname, ext = os.path.splitext(file)
    nt += 1
    process_name(bname)

print()
print(nt, 'file(s) processed')
# print(unknowns.most_common(20))

with open(r'c:\temp\newwords.txt', 'w', encoding='utf-8') as out:
    for w, c in unknowns.most_common():
        if c == 1:
            break
        print(w, c)
        out.write(w+'\n')


import scrapy
from scrapy import Request
from okooo.items import OkoooItem

class QuotesSpider(scrapy.Spider):
    name = "okooo"
    start_urls = [
        'http://www.okooo.com/danchang/',
    ]

    default_headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, sdch, br',
        'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Host': 'www.douban.com',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
    }

#    def start_requests(self):

#        for url in self.start_urls:
#            yield Request(url=url, headers=self.default_headers, callback=self.parse)

    def parse(self, response):
        for tr in response.css('tr.alltrObj'):
            item = OkoooItem()
            #item['matchId'] = tr.css('::attr(matchid)').extract_first()
            td1 = tr.css('td.td1')
            index = td1.css('i::text').extract_first()
            item['matchTitle'] = td1.css('a.ls::text').extract_first()
            td2 = tr.css('td.td2')
            item['matchtime'] = td2.css('span.BuyTime::attr(endtime)').extract_first()
            td3 = tr.css('td.td3')
            host = td3.css('a.sbg')
            draw = td3.css('a.pbg')
            guest = td3.css('a.fbg')
            item['host'] = host.css('span.homenameobj::attr(title)').extract_first()
            leageInfo = host.css('p.float_left i::text').extract()
            if leageInfo.len <= 1:
                item['hostLeage'] = item['matchTitle']
                item['hostLeageRank'] = leageInfo[0]
            else:
                item['hostLeage'] = leageInfo[0]
                item['hostLeageRank'] = leageInfo[1]
            item['hostRangfen'] = host.css('span.handicapobj::text').extract_first()
            item['hostSP'] = host.css('em.pltxt::text').extract_first()
            item['drawSP'] = draw.css('em::text').extract_first()
            item['guest'] = guest.css('span.awaynameobj::text').extract_first()
            item['guestSP'] = guest.css('em.pltxt::text').extract_first()
            guestInfo = guest.css('p.float_right i::text').extract()
            if guestInfo.len <= 1:
                item['guestLeage'] = item['matchTitle']
                item['guestLeageRank'] = guestInfo[0]
            else:
                item['guestLeage'] = guestInfo[0]
                item['guestLeageRank'] = guestInfo[1]
            td8 = tr.css('td.td8')
            link = tr.css('td.td8 a::attr(href)').extract()
            tem['oddsLink'] = link[0]
            tem['historyLink'] = link[1]
            td9 = tr.css('td.td9')
            item['winOdds'] = td9.css('span.noborder0::text').extract_first()
            item['drawOdds'] = td9.css('span.noborder1::text').extract_first()
            item['loseOdds'] = td9.css('span.noborder2::text').extract_first()
            yield item


from tkinter import *

root = Tk()

# bg -> background, fg -> foreground
one = Label(root, text = "One", bg = "red", fg = "white")
one.pack()

two = Label(root, text = "Two", bg = "green", fg = "black")
two.pack(fill = X) # Will stretch in x-direction

three = Label(root, text = "Three", bg = "blue", fg = "black")
three.pack(side = LEFT, fill = Y) # Will stretch in x-direction

root.mainloop()


"""Verb data controller.

Controlls I/O of verb data between the model and the actual data.
"""

from tkinter import messagebox
import typing as t
from model import Verb
import pandas as pd
from ast import literal_eval

class VerbDataController:
    """Verb data controller class.

    Controlls I/O of verb data between the model and the actual data.
    """

    data_path: str
    verbs: t.List[Verb]
    #is_modified: bool

    def __init__(self, data_path: str = "data/verbs.csv"):
        """Create a verb data controller.

        Controlls I/O of verb data between the model and the actual data.

        Parameters:
            data_path: Path to the verb data CSV.
        """
        self.data_path = data_path
        self.verbs = []
        self.load()

    def save(self):
        """Save the verb data to the CSV file."""
        if messagebox.askokcancel(
                message=f'Are you sure you want to overwrite the data?',
                icon='warning', title='Save'):
            df = pd.DataFrame.from_records([v.__dict__ for v in self.verbs])
            df.to_csv(self.data_path)

    def load(self):
        """Load the verb data from the CSV file."""
        self.verbs = []
        try:
            df = pd.read_csv(self.data_path, index_col=0, keep_default_na=False,
                converters={'nvn_syllables': literal_eval})
            for index, row in df.iterrows():
                verb_dict = {name: value for name, value in zip(df.columns, row)}
                verb = Verb(**verb_dict)
                self.verbs.append(verb)
        except Exception as e:
            print(f"Unable to load file '{self.data_path}', aborting: {e}")


import math
import csv
import xlrd
import numpy

def calculate_standard_deviation(arr_values):
    return numpy.std(arr_values)

location = ("1650107370-2--Sample----with-criteria.xlsx")
wb = xlrd.open_workbook(location)
s1 = wb.sheet_by_index(0)
s1.cell_value(0, 0)  # initializing cell from the cell position

print(" No. of rows: ", s1.nrows)
print(" No. of columns: ", s1.ncols)
stock_data = list(s1.get_rows())[2:]
len_stock_data = len(stock_data)
forward_column_map = {"1-Day Return": -1, "10-Day Return": 10, "25-Day Return": 25}
new_column_map_results = {"criteria-90": [None] * 10}
first, last = 0, 0
for key in forward_column_map:
    forward_index = forward_column_map[key]
    if key not in new_column_map_results:
        new_column_map_results[key] = []
    for index in range(0, len_stock_data):
        try:
            stat_value = stock_data[index][4].value
            last_value = (stock_data[index + forward_index][4]).value
            if ((index + forward_index) < 0) or ((index + forward_index) > len_stock_data):
                result = -1
            else:
                if forward_index > 0:
                    result = ((last_value - stat_value) / (stat_value * 1.0)) * 100.0
                else:
                    result = ((stat_value - last_value) / (last_value * 1.0)) * 100.0
            new_column_map_results[key].append(result)
        except:
            # noinspection PyTypeChecker
            new_column_map_results[key].append(-1)

last_new_column_map = {"V-10": 10, "V-90": 90}
for key in last_new_column_map:
    last = last_new_column_map[key]
    if key not in new_column_map_results:
        new_column_map_results[key] = [0] * last
    for index in range(last, len_stock_data):
        last = last + 1
        start_index = last - last_new_column_map[key]
        values = new_column_map_results["1-Day Return"][start_index:last]
        std_dev = calculate_standard_deviation([i for i in values if i != -1])
        new_column_map_results[key].append(std_dev * math.sqrt(365))
hvg = []
for index in range(0, len_stock_data):
    try:
        hvg.append(new_column_map_results["V-10"][index] / new_column_map_results["V-90"][index])
    except:
        hvg.append(-1)

criteria_map = {"criteria-10": 10}

for key in criteria_map:
    last = criteria_map[key]
    if key not in new_column_map_results:
        new_column_map_results[key] = [None] * 90
    for index in range(90, len_stock_data):
        last = index
        start_index = last - criteria_map[key]
        values = hvg[start_index + 1:last + 1]
        max_value = max([i for i in values if i != math.inf])
        max_value_index = [index for index in range(0, len(values)) if max_value == values[index]][-1]
        main_index = (last - 10) + max_value_index + 1
        close_value = stock_data[len_stock_data - 1 if main_index == len_stock_data else main_index][4].value
        divide_value = ((stock_data[len_stock_data - 1 if last == len_stock_data else last][
                             4].value * 1.0) / close_value) / (hvg[index] / max_value)
        new_column_map_results[key].append(divide_value * 100.0)

for index in range(10, len_stock_data):
    last = index
    start_index = last - 10
    values_2 = new_column_map_results["V-10"][start_index + 1:last + 1]
    # noinspection PyTypeChecker
    max_value_2 = max([i for i in values_2 if i != math.inf])
    max_value_index_2 = [index for index in range(0, len(values_2)) if max_value_2 == values_2[index]][-1]
    main_index_2 = (last - 10) + max_value_index_2 + 1
    close_value_2 = stock_data[len_stock_data - 1 if main_index_2 == len_stock_data else main_index_2][4].value
    divide_value_2 = ((stock_data[len_stock_data - 1 if last == len_stock_data else last][
                           4].value * 1.0) / close_value_2) / (new_column_map_results["V-10"][index] / max_value_2)
    new_column_map_results["criteria-90"].append(divide_value_2 * 100.0)

t1, t2, n = 200, 200, 3
check_mapping = {str(n) + "days": n}
c1_check_array, check_one_passed = [], False
c1, c2, cons_check_days_count = False, False, 0

for index in range(0, len_stock_data):
    c1, c2 = False, False
    # noinspection PyTypeChecker
    if new_column_map_results["criteria-90"][index] is not None and float(
            new_column_map_results["criteria-90"][index]) > t1:
        c1 = True
    elif new_column_map_results["criteria-10"][index] is not None and float(
            new_column_map_results["criteria-10"][index]) > t2:
        c2 = True
    if c1 or c2:
        cons_check_days_count += 1
        c1_check_array.append(index)
    else:
        if check_one_passed:
            break
        cons_check_days_count = 0
        c1_check_array = []
    if cons_check_days_count >= n:
        check_one_passed = True

if not check_one_passed:
    print("Check one failed")
    exit()
print(new_column_map_results)