from django.shortcuts import render, render_to_response
import django.apps
from django.http import HttpResponse, HttpResponseRedirect
from django.views.generic import ListView, DetailView, DeleteView, UpdateView, CreateView
from django.utils import timezone
from django.views.decorators.http import require_http_methods
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse_lazy

from myadmin.models import *

# VERY IMPORTANT
# HERE IS THE LIST OF MODELS "REGISTERED" IN ADMIN PANEL
MODELS = ["Model1", "Person", "Book", "Fruit"]

def get_model(model_name):
    tmp = django.apps.apps.get_models()
    return [obj for obj in tmp if obj.__name__ == model_name][0]

@login_required(login_url='admin/')
def myadmin_home(request):
    """ Home page for app"""
    model_list = []
    for obj in MyAdmin.objects.all():
        model_list.append(obj.name)
    return render(request, 'myadmin/myadmin_home.html',
        context = {'model_list': model_list})

# @require_http_methods(['GET', 'POST'])
@login_required(login_url='admin/')
def myadmin_add(request):
    """
    This method adds selected model to model list
    """
    success_url = reverse_lazy('myadmin-home')
    model_list = django.apps.apps.get_models()
    chosen_model = [obj.__name__ for obj in model_list if obj.__name__ in request.POST.values()]
    if MyAdmin.objects.filter(name = chosen_model[0]).exists():
        return HttpResponse("Model {} already exists in database".format(chosen_model[0]))
    elif chosen_model:
        a = MyAdmin()
        a.name = chosen_model[0]
        a.save()

    return HttpResponseRedirect(success_url)

def myadmin_delete(request, model_name):
    """
    Remove model from the list
    """
    success_url = reverse_lazy('myadmin-home')
    model = MyAdmin.objects.get(name = model_name)
    print(request.method, request.POST)
    if request.method == "POST" and "delete" in request.POST:
        model.delete()
        return HttpResponseRedirect(success_url)
    else:
        return HttpResponse("Not implemented")

@login_required(login_url='admin/')
def myadmin_all(request):
    """
    List all models added to db of MyAdmin
    """
    model_list = django.apps.apps.get_models()
    model_list = [obj.__name__ for obj in model_list if obj.__name__ in MODELS]
    return render(request, 'myadmin/myadmin_list.html',
        context = {'model_list': model_list})

# PARTICUAL OBJECT RELATED VIEWS
@login_required(login_url='admin/')
def myadmin_object_list(request, model_name):
    """
    Lists objects of selected model
    """
    view = ListView
    view.model = get_model(model_name)
    view.template_name = "myadmin/object_list.html"

    def get_context_data(self, **kwargs):
        context = super(view, self).get_context_data(**kwargs)
        context['model_name'] = model_name
        return context

    view.get_context_data = get_context_data
    return view.as_view()(request)

@login_required(login_url='admin/')
def myadmin_detail(request, model_name, pk):
    """
    Object detail
    """
    from collections import OrderedDict
    detail = DetailView
    detail.model = get_model(model_name)
    detail.template_name = "myadmin/object_detail.html"
    obj = detail.model.objects.get(id = pk)
    def get_model_fields(model):
        f_name = [f.name for f in detail.model._meta.get_fields()]
        a = []
        for f in f_name:
            a.append(getattr(obj, f))

        # very naive, but it works
        my_list = zip(f_name, a)
        return my_list

    detail.model.get_model_fields = get_model_fields

    def get_context_data(self, **kwargs):
        context = super(detail, self).get_context_data(**kwargs)
        context['model_name'] = model_name
        return context

    detail.get_context_data = get_context_data
    return detail.as_view()(request, pk=pk)

def myadmin_object_delete(request, model_name, pk):
    """
    Delete object from particular database
    """
    deletator = DeleteView
    deletator.model = get_model(model_name)
    deletator.success_url = reverse_lazy('myadmin-object-list',
        kwargs={'model_name': model_name},)
    def get_template_names(instance):
        return ['myadmin/delete_confirm.html']

    deletator.get_template_names = get_template_names
    return deletator.as_view()(request, pk=pk)

@login_required(login_url='admin/')
def myadmin_object_update(request, model_name, pk):
    updator = UpdateView
    updator.model = get_model(model_name)
    updator.success_url = reverse_lazy('myadmin-object-list',
        kwargs={'model_name': model_name})

    updator.fields = '__all__'
    def get_template_names(instance):
        return ['myadmin/object_update.html']

    updator.get_template_names = get_template_names
    return updator.as_view()(request, pk=pk)

@login_required(login_url='admin/')
def myadmin_object_create(request, model_name):
    """
    Adds new entry to a database
    """
    creator = CreateView

    creator.model = get_model(model_name)
    creator.fields = '__all__'
    creator.success_url = reverse_lazy('myadmin-object-list', kwargs={'model_name': model_name})

    def get_template_names(instance):
        return ['myadmin/object_create.html']

    creator.get_template_names = get_template_names

    return creator.as_view()(request)


from django.shortcuts import render

from .core import Page_video

# Create your views here.
def index(request):
    if request.method == 'POST':
        try:
            search_text = request.POST["Search_box"]
            videos=Page_video.search_videos(search_text)
            if(len(videos) == 1):
                col_size = 6
                offset_size = 3
            else:
                col_size = int(12/len(videos))
                offset_size = 0
        except ZeroDivisionError as e:
            if settings.DEBUG:
                return render_template("index.html",error_msg="Nenhum video encontrado")

        return render(request, "app/index.html", {"videos" : videos, "col_size" : col_size,
                                                         "offset_size" : offset_size})

    return render(request,"app/index.html")


"""QQP dataset."""
import torch
from .data_utils import clean_text
from .abstract_dataset import GLUEAbstractDataset

LABELS = [0, 1]

class QQPDataset(GLUEAbstractDataset):

    def __init__(self, name, datapaths, tokenizer, max_seq_length, data_as_tuple=False,
                 test_label=0):
        self.test_label = test_label
        super().__init__('QQP', name, datapaths,
                         tokenizer, max_seq_length, data_as_tuple)

    def process_samples_from_single_path(self, filename):
        """"Implement abstract method."""
        print(' > Processing {} ...'.format(filename))

        samples = []
        total = 0
        first = True
        is_test = False
        with open(filename, 'r') as f:
            for line in f:
                row = line.strip().split('\t')
                if first:
                    first = False
                    if len(row) == 3:
                        is_test = True
                        print('   reading {}, {}, and {} columns and '
                                     'setting labels to {}'.format(
                                         row[0].strip(), row[1].strip(),
                                         row[2].strip(), self.test_label))
                    else:
                        assert len(row) == 6
                        print('    reading {}, {}, {}, and {} columns'
                                     ' ...'.format(
                                         row[0].strip(), row[3].strip(),
                                         row[4].strip(), row[5].strip()))
                    continue

                if is_test:
                    assert len(row) == 3, 'expected length 3: {}'.format(row)
                    uid = int(row[0].strip())
                    text_a = clean_text(row[1].strip())
                    text_b = clean_text(row[2].strip())
                    label = self.test_label
                    assert len(text_a) > 0
                    assert len(text_b) > 0
                else:
                    if len(row) == 6:
                        uid = int(row[0].strip())
                        text_a = clean_text(row[3].strip())
                        text_b = clean_text(row[4].strip())
                        label = int(row[5].strip())
                    else:
                        print('***WARNING*** index error, ' 'skipping: {}'.format(row))
                        continue
                    if len(text_a) == 0:
                        print('***WARNING*** zero length a, ' 'skipping: {}'.format(row))
                        continue
                    if len(text_b) == 0:
                        print('***WARNING*** zero length b, ' 'skipping: {}'.format(row))
                        continue
                assert label in LABELS
                assert uid >= 0
                sample = {'uid': uid,
                          'text_a': text_a,
                          'text_b': text_b,
                          'label': label}
                total += 1
                samples.append(sample)

                if total % 50000 == 0:
                    print('  > processed {} so far ...'.format(total))

        print(' >> processed {} samples.'.format(len(samples)))
        return samples

def get_glue_qqp_train_data_loader(args, tokenizer, num_workers=0):
    train_dataset = QQPDataset('training', args.train_data, tokenizer, args.seq_length)
    train_sampler = torch.utils.data.RandomSampler(train_dataset)
    train_data_loader = torch.utils.data.DataLoader(train_dataset,
                                                    batch_size=args.batch_size,
                                                    sampler=train_sampler,
                                                    shuffle=False,
                                                    num_workers=num_workers,
                                                    drop_last=True,
                                                    pin_memory=True,
                                                    collate_fn=None)
    return train_data_loader

def get_glue_qqp_test_data_loader(args, tokenizer, num_workers=0):
    test_dataset = QQPDataset('testing', args.test_data, tokenizer, args.seq_length)
    test_sampler = torch.utils.data.RandomSampler(test_dataset)
    test_data_loader = torch.utils.data.DataLoader(test_dataset,
                                                    batch_size=args.batch_size,
                                                    sampler=test_sampler,
                                                    shuffle=False,
                                                    num_workers=num_workers,
                                                    drop_last=True,
                                                    pin_memory=True,
                                                    collate_fn=None)
    return test_data_loader


import math
import os
import pickle
import re
import string
import time

from data.models import Data
from nltk import sent_tokenize, stem, word_tokenize
from nltk.corpus import stopwords
from nltk.probability import FreqDist
from nltk.stem import PorterStemmer, SnowballStemmer
from sklearn.feature_extraction.text import TfidfVectorizer

INPUT_FILES = "./input/Cranfield"

# Clear text

def remove_special_character(text):
    processed_text = text.lower()
    processed_text = processed_text.replace("’", "'")
    processed_text = processed_text.replace("“", '"')
    processed_text = processed_text.replace("”", '"')

    non_words = re.compile(r"[^A-Za-z']+")
    processed_text = re.sub(non_words, ' ', processed_text)

    return processed_text

def remove_stopwords(text):  # step 1
    stop_words = set(stopwords.words('english'))
    # xóa stopwords
    words = [
        w for w in text.split(" ")
        if w not in stop_words
    ]
    return ' '.join(words)

def remove_punctuation(text):  # step 2
    words = [
        char for char in text.split(" ")
        if char not in string.punctuation
    ]
    return " ".join(words)

def remove_stem(text):  # step 3
    porter = PorterStemmer()
    token_words = word_tokenize(text)
    words = [
        porter.stem(word) for word in token_words
    ]
    return " ".join(words)

def clear_text(text):
    # processing
    text = text.lower()
    text = remove_stopwords(text)
    text = remove_special_character(text)
    text = remove_punctuation(text)
    text = remove_stem(text)

    return text

# Read data

def get_text_from_file(filename):
    with open(filename, encoding='cp1252', mode='r') as f:
        text = f.read()
    f.close()
    return text

def read_data():
    data = []
    arr_file = []
    for doc_file in os.listdir(INPUT_FILES):
        filename = os.path.join(INPUT_FILES, doc_file)
        text = get_text_from_file(filename)
        if len(text) == 0:
            continue
        arr_file.append(doc_file.split(".")[0])
        data.append(clear_text(text))
    return data, arr_file

def read_data_from_database():
    data = []
    arr_file = []
    allFiles = Data.objects.all()
    for file in allFiles:
        text = file.text
        words = get_words_from_text(text)
        if len(text) == 0:
            continue
        arr_file.append(file.id)
        data.append(clear_text(text))
    return data, arr_file

# build index, tf_idf arrays

def build_tf_idf_sklearn(data):
    tfidfVectorizer = TfidfVectorizer(
        analyzer='word', ngram_range=(1, 6), min_df=0.01, sublinear_tf=True,
        use_idf=True, smooth_idf=True
    )

    tfidf_matrix = tfidfVectorizer.fit_transform(data)
    feature_names = tfidfVectorizer.get_feature_names()

    # init
    tfidf_scores = dict()
    for index in feature_names:
        tfidf_scores[index] = []
    for index in range(len(data)):
        feature_index = tfidf_matrix[index, :].nonzero()[1]
        for x in feature_index:
            tfidf_scores[feature_names[x]].append(
                [index, tfidf_matrix[index, x]])
    return tfidfVectorizer, tfidf_scores

def open_queries():
    result = dict()
    for i in open("./input/query.txt").readlines():
        t = i.split('\t')
        result[t[0]] = t[1]
    return result

def get_data_ground_truth():
    path = os.path.join('./input', 'RES')
    data = dict()

    for file in os.listdir(path):
        filename = os.path.join(path, file)
        text = get_text_from_file(filename)
        text = text.rstrip('\n')
        cutLine = text.split('\n')
        for index, line in enumerate(cutLine):
            # cutTab[1] chua can quan tam toi do chua can dung
            cutTab = line.split('\t')
            cutSpace = cutTab[0].split(" ")
            if cutSpace[0] not in data.keys():
                data[cutSpace[0]] = [cutSpace[1]]
            else:
                data[cutSpace[0]].append(cutSpace[1])
    return data

def get_relevant_ranking_for_query(query, tfidfVectorizer, tfidf_scores, feature_names, arr_file):
    # clear query
    query = clear_text(query)
    query = " ".join(
        [
            word for word in query.split(" ")
            if word in feature_names
        ]
    )

    # compute tf_idf
    tfidf_matrix = tfidfVectorizer.fit_transform([query])
    feature_index = tfidf_matrix[0, :].nonzero()[1]

    # get vocal in query
    feature_names_query = tfidfVectorizer.get_feature_names()

    # check word in vocal
    query_tfidf_scores = dict()

    for x in feature_index:
        if feature_names_query[x] in feature_names:
            if feature_names_query[x] not in query_tfidf_scores.keys():
                query_tfidf_scores[feature_names_query[x]] = [
                    tfidf_matrix[0, x]]
            else:
                query_tfidf_scores[feature_names_query[x]].append(
                    tfidf_matrix[0, x])

    # find q length
    q_length = 0

    relevant_between_words = dict()
    # compute relevant query and data_train
    relevant_between_words = {
        word: [
            [
                item[0],
                item[1] * query_tfidf_scores[word][0]
            ] for item in tfidf_scores[word]
        ] for word in query_tfidf_scores.keys()
    }

    for key, value in query_tfidf_scores.items():
        q_length += math.pow(value[0], 2)
    q_length = math.sqrt(q_length)

    q_score = dict()
    for _, value in relevant_between_words.items():
        for i in value:
            if i[0] not in q_score.keys():
                q_score[i[0]] = i[1]
            else:
                q_score[i[0]] += i[1]
    for key in q_score.keys():
        q_score[key] = q_score[key] / (q_length + 0.01)

    # rank
    q = sorted(q_score.items(), key=lambda item: item[1], reverse=True)

    x_retrieved = []
    for i in q:
        x_retrieved.append(arr_file[i[0]])
    return x_retrieved

def get_Average_Precision(x_retrieved, relevant_docs):
    # find R_Precision value
    validation_result = {'R': [], 'P': []}
    c = 0
    for i in range(len(relevant_docs)):
        if x_retrieved[i] in relevant_docs:
            c += 1
        validation_result['R'].append((c / len(relevant_docs)))
        validation_result['P'].append((c / (i + 1)))
    return sum(validation_result['P']) / len(validation_result['P'])

def train():
    try:
        pkl_file = open(os.path.join('.\input', 'data',
                                     'train_in_database.pickle'), 'rb')
        arr_file = pickle.load(pkl_file)
        tfidfVectorizer = pickle.load(pkl_file)
        tfidf_scores = pickle.load(pkl_file)
        pkl_file.close()
    except:
        data, arr_file = read_data_from_database()
        tfidfVectorizer, tfidf_scores = build_tf_idf_sklearn(data)
        with open(os.path.join('.\input', 'data', 'train_in_database.pickle'), mode='wb') as f:
            pickle.dump(arr_file, f)
            pickle.dump(tfidfVectorizer, f)
            pickle.dump(tfidf_scores, f)
        f.close()
    return arr_file, tfidfVectorizer, tfidf_scores

def search_in_database(query, arr_file, tfidfVectorizer, tfidf_scores):
    feature_names = tfidfVectorizer.get_feature_names()
    return get_relevant_ranking_for_query(
        query, tfidfVectorizer, tfidf_scores, feature_names, arr_file
    )

def main():
    try:
        pkl_file = open(os.path.join('.\input', 'data', 'train.pickle'), 'rb')
        arr_file = pickle.load(pkl_file)
        tfidfVectorizer = pickle.load(pkl_file)
        tfidf_scores = pickle.load(pkl_file)
        pkl_file.close()
    except:
        data, arr_file = read_data()
        tfidfVectorizer, tfidf_scores = build_tf_idf_sklearn(data)
        with open(os.path.join('.\input', 'data', 'train.pickle'), mode='wb') as f:
            pickle.dump(arr_file, f)
            pickle.dump(tfidfVectorizer, f)
            pickle.dump(tfidf_scores, f)
        f.close()

    feature_names = tfidfVectorizer.get_feature_names()
    queries = open_queries()
    list_of_x_retrieved = dict()
    for key, query in queries.items():
        list_of_x_retrieved[key] = get_relevant_ranking_for_query(
            query, tfidfVectorizer, tfidf_scores, feature_names, arr_file
        )

     # Bắt đầu đánh giá mô hình.
    data_ground_truth = get_data_ground_truth()
    Average_precision_of_all_x_retrieved = \
        {
            key: get_Average_Precision(value, data_ground_truth[key])
            for key, value in list_of_x_retrieved.items()
        }
    MAP = 0
    for key, value in Average_precision_of_all_x_retrieved.items():
        MAP += value
    MAP = MAP / len(Average_precision_of_all_x_retrieved)
    print("MAP:", MAP)

if __name__ == '__main__':
    t0 = time.clock()
    main()
    t1 = time.clock() - t0
    print("Time elapsed: ", t1)


from django.shortcuts import render,redirect
from .form import CategoryCreateForm,DoctorCreateForm
from .models import Category,Doctor

def home(request):
	cats = Category.objects.all()
	dict = {
		"title":"Find A Doctor",
		"cats":cats
	}
	return render(request,"doctor/home.html",dict)

def catcreate(request):
	if request.method == "POST":
		form = CategoryCreateForm(request.POST)
		form.is_valid()
		form.save()
		return redirect('dhome-page')
	else:
		form = CategoryCreateForm

	cats = Category.objects.all()
	dict = {
		"title":"Create Category",
		"form":form,
		"cats":cats
	}
	return render(request,"doctor/catcreate.html",dict)

def createDoctor(request):
	if request.method == "POST":
		form = DoctorCreateForm(request.POST)
		if form.is_valid():
			form.save()
			return redirect('doctor-show-page')
	else:
		form = DoctorCreateForm()

	dict ={
		"title":"Doctor Create Page",
		"form":form
	}

	return render(request,"doctor/doctor_create.html",dict)

def showDoctors(request):
	docts = Doctor.objects.all()
	dict = {
		"title":"Show All Doctors",
		"docts":docts
	}
	return render(request,"doctor/show_all_doctors.html",dict)

def filterDoctors(request,pk):
	docts = Doctor.objects.filter(category=pk)
	print(docts)

	dict = {
		"title":"Find A Doctor",
		"docts":docts
	}

	return render(request,"doctor/show_all_doctors.html",dict)

# def filterDoctors(request):
# 	docts = Doctor.objects.all()
# 	dict = {
# 		"title":"Show All Doctors",
# 		"docts":docts
# 	}
# 	return render(request,"doctor/show_all_doctors.html",dict)

def editDoctor(request,id):
	doctors = Doctor.objects.filter(id=id)
	return render(request,"doctor/show_all_doctors1.html",{"doctors":doctors})

def updateDoctor(request,id):
	doctor1 = Doctor.objects.get(pk=id)
	form = DoctorCreateForm(request.POST,instance=doctor1)

	if form.is_valid():
		form.save()
		return redirect('doctor-show-page')
	else:
		form = DoctorCreateForm(instance=doctor1)

	content = {
		"doctor1":doctor1,
		"form":form
	}
	return render(request,"doctor/doctor_create.html", content)

def deleteDoctor(request,id):
	doctor2 = Doctor.objects.get(pk=id)
	if request.method =="POST":
		doctor2.delete()
		return redirect('doctor-show-page')

	return render(request,"doctor/show_all_doctors.html",{"doctor2":doctor2})

def findDoctor(request):
	dict = {
		"title":"Find A Doctor"
	}

	return render(request,"doctor/findDoctor.html",dict)

# def find1Doctor(request):

# 	names = Doctor.objects.all()
# 	if 'name' in request.GET:
# 		names = Doctor.objects.filter(name__icontains=request.GET['name'])

# 	return render(request,"doctor/show_find_doctors.html",{"names":names})


"""
James found a love letter that his friend Harry has
written to his girlfriend. James is a prankster, so he
decides to meddle with the letter. He changes all the
words in the letter into palindromes.
To do this, he follows two rules:
    1.He can only reduce the value of a letter by 1, i.e.
    he can change d to c, but he cannot change c to d or d
    to b.
    2.The letter a may not be reduced any further.
Each reduction in the value of any letter is counted as a
single operation. Find the minimum number of operations
required to convert a given string into a palindrome.
Example
    s=cde
    The following two operations are performed: cde → cdd
    → cdc. Return 2.
Function Description
    Complete the theLoveLetterMystery function in the
    editor below.
    theLoveLetterMystery has the following parameter(s):
        string s: the text of the letter
Returns
    int: the minimum number of operations
Input Format
    The first line contains an integer q, the number of
    queries.
    The next q lines will each contain a string s.
Constraints
    1<=q<=10
    1<=| s |<=10^4
    All strings are composed of lower case English letters,
    ascii[a-z], with no spaces.
Sample Input
    STDIN   Function
    -----   --------
    4       q = 4
    abc     query 1 = 'abc'
    abcba
    abcd
    cba
Sample Output
    2
    0
    4
    2
Explanation
    1.For the first query, abc → abb → aba.
    2.For the second query, abcba is already a palindromic
    string.
    3.For the third query, abcd → abcc → abcb → abca
    → abba.
    4.For the fourth query, cba → bba → aba.
"""

#!/bin/python3

import math
import os
import random
import re
import sys

#
# Complete the 'theLoveLetterMystery' function below.
#
# The function is expected to return an INTEGER.
# The function accepts STRING s as parameter.
#

def theLoveLetterMystery(s):
    # Write your code here
    count = 0
    j= len(s)-1
    for i in range(0, len(s)//2):
        if(s[i] != s[j]):
            while(s[i] != s[j]):
                if(ord(s[i]) > ord(s[j])):
                    s = s[:i] + chr(ord(s[i])-1) + s[i+1:]
                    count += 1
                else:
                    s = s[:j] + chr(ord(s[j])-1) + s[j+1:]
                    count += 1
        j-=1
    return count
if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    q = int(input().strip())

    for q_itr in range(q):
        s = input()

        result = theLoveLetterMystery(s)

        fptr.write(str(result) + '\n')

    fptr.close()


#*************************************************
# Stage 6 - Feature Extraction
#*************************************************

import os
import sys
import subprocess
import cv2
import csv

import stages

stage = stages.stage6

openface_FaceLandmarkVidMulti_path = stage.dependencies['openface_FaceLandmarkVidMulti_path']

def PV(arr):
    for x in arr:
        print(x);

def removeTrailingBackslash(path):
    if (path[-1] == '/'):
        path = path[:-1];

    return path;

def get_csv_data(csvFile):
    with open(csvFile, 'r') as fin:
        reader = csv.reader(fin)
        data = list(reader)

    return data

def do_feature_extraction(input_folder, output_folder):
    """
    Execute feature extraction program using OpenFace
    """
    program_location = stage.dependencies['feature_extraction_script_path']
    c1 = 'python \'' + program_location + '\' \'' + openface_FaceLandmarkVidMulti_path + '\' \'' + input_folder + '\' \'' + output_folder + '\'';

    # execute command

    print(c1)
    subprocess.call(c1, shell=True)

input_folder = removeTrailingBackslash(stage.inputLocation)
output_folder = removeTrailingBackslash(stage.outputLocation)

print('Begin Stage ', stage.number, ' : ', stage.name)

assert os.path.exists(input_folder), "Stage input folder : " + input_folder + " not found."
assert os.path.exists(output_folder), "Stage output folder : " + output_folder + " not found."

# process files
do_feature_extraction(input_folder, output_folder);

print('End Stage ', stage.number, ' : ', stage.name)


"""Lineages module."""

from typing import Union, TypeVar, Optional, Sequence

import pandas as pd

from cellrank import logging as logg
from cellrank.ul._docs import d
from cellrank.tl._utils import TestMethod
from cellrank.tl.kernels import PrecomputedKernel
from cellrank.tl._constants import AbsProbKey, TermStatesKey, TerminalStatesPlot
from cellrank.tl.estimators import GPCCA
from cellrank.tl.estimators._constants import P
from cellrank.tl.kernels._precomputed_kernel import DummyKernel

AnnData = TypeVar("AnnData")

@d.dedent
def lineages(
    adata: AnnData,
    backward: bool = False,
    copy: bool = False,
    return_estimator: bool = False,
    **kwargs,
) -> Optional[AnnData]:
    """
    Compute probabilistic lineage assignment using RNA velocity.

    For each cell `i` in :math:`{1, ..., N}` and %(initial_or_terminal)s state `j` in :math:`{1, ..., M}`,
    the probability is computed that cell `i` is either going to %(terminal)s state `j` (``backward=False``)
    or is coming from %(initial)s state `j` (``backward=True``).

    This function computes the absorption probabilities of a Markov chain towards the %(initial_or_terminal) states
    uncovered by :func:`cellrank.tl.initial_states` or :func:`cellrank.tl.terminal_states` using a highly efficient
    implementation that scales to large cell numbers.

    It's also possible to calculate mean and variance of the time until absorption for all or just a subset
    of the %(initial_or_terminal)s states. This can be seen as a pseudotemporal measure, either towards any terminal
    population of the state change trajectory, or towards specific ones.

    Parameters
    ----------
    %(adata)s
    %(backward)s
    copy
        Whether to update the existing ``adata`` object or to return a copy.
    return_estimator
        Whether to return the estimator. Only available when ``copy=False``.
    kwargs
        Keyword arguments for :meth:`cellrank.tl.estimators.BaseEstimator.compute_absorption_probabilities`.

    Returns
    -------
    :class:`anndata.AnnData`, :class:`cellrank.tl.estimators.BaseEstimator` or :obj:`None`
        Depending on ``copy`` and ``return_estimator``, either updates the existing ``adata`` object,
        returns its copy or returns the estimator.
    """

    if backward:
        lin_key = AbsProbKey.BACKWARD
        fs_key = TermStatesKey.BACKWARD
        fs_key_pretty = TerminalStatesPlot.BACKWARD
    else:
        lin_key = AbsProbKey.FORWARD
        fs_key = TermStatesKey.FORWARD
        fs_key_pretty = TerminalStatesPlot.FORWARD

    try:
        pk = PrecomputedKernel(adata=adata, backward=backward)
    except KeyError as e:
        raise RuntimeError(
            f"Compute transition matrix first as `cellrank.tl.transition_matrix(..., backward={backward})`."
        ) from e

    start = logg.info(f"Computing lineage probabilities towards {fs_key_pretty.s}")
    mc = GPCCA(
        pk, read_from_adata=True, inplace=not copy
    )  # GPCCA is more general than CFLARE, in terms of what is saves
    if mc._get(P.TERM) is None:
        raise RuntimeError(
            f"Compute the states first as `cellrank.tl.{fs_key.s}(..., backward={backward})`."
        )

    # compute the absorption probabilities
    mc.compute_absorption_probabilities(**kwargs)

    logg.info(f"Adding lineages to `adata.obsm[{lin_key.s!r}]`\n    Finish", time=start)

    return mc.adata if copy else mc if return_estimator else None

@d.dedent
def lineage_drivers(
    adata: AnnData,
    backward: bool = False,
    lineages: Optional[Union[Sequence, str]] = None,
    method: str = TestMethod.FISCHER.s,
    cluster_key: Optional[str] = None,
    clusters: Optional[Union[Sequence, str]] = None,
    layer: str = "X",
    use_raw: bool = False,
    confidence_level: float = 0.95,
    n_perms: int = 1000,
    seed: Optional[int] = None,
    return_drivers: bool = True,
    **kwargs,
) -> Optional[pd.DataFrame]:
    """
    %(lineage_drivers.full_desc)s

    Parameters
    ----------
    %(adata)s
    %(backward)s
    %(lineage_drivers.parameters)s

    Returns
    -------
    %(lineage_drivers.returns)s

    References
    ----------
    %(lineage_drivers.references)s
    """  # noqa: D400

    # create dummy kernel and estimator
    pk = DummyKernel(adata, backward=backward)
    g = GPCCA(pk, read_from_adata=True, write_to_adata=False)
    if g._get(P.ABS_PROBS) is None:
        raise RuntimeError(
            f"Compute absorption probabilities first as `cellrank.tl.lineages(..., backward={backward})`."
        )

    # call the underlying function to compute and store the lineage drivers
    return g.compute_lineage_drivers(
        method=method,
        lineages=lineages,
        cluster_key=cluster_key,
        clusters=clusters,
        layer=layer,
        use_raw=use_raw,
        confidence_level=confidence_level,
        n_perms=n_perms,
        seed=seed,
        return_drivers=return_drivers,
        **kwargs,
    )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

##########################################################
# Author: Yichen Huang (Eugene)
# GitHub: https://github.com/yichen0831/opencc-python
# January, 2016 - Original
# January 2017 - Update to run under Python 2 by Hopkins1
##########################################################

import sys
import os
import io

DICT_DIRECTORY = '.'

MER_INPUTS = [
    'TWPhrasesIT.txt',
    'TWPhrasesName.txt',
    'TWPhrasesOther.txt'
]

MER_OUTPUT = 'TWPhrases.txt'

def merge(mer_inputs=MER_INPUTS, mer_output=MER_OUTPUT):
    """
    merge the phrase files into one file
    :param mer_inputs: the phrase files
    :param mer_output: the output file
    :return: None
    """
    dirname = os.path.dirname(__file__)
    output_file = os.path.join(dirname, DICT_DIRECTORY, mer_output)
    lines = []
    for in_file in MER_INPUTS:
        input_file = os.path.join(dirname, DICT_DIRECTORY, in_file)
        with io.open(input_file, 'r', encoding='utf-8') as f:
            for line in f:
                lines.append(line)

    with io.open(output_file, 'w', encoding='utf-8') as f:
        for line in lines:
            f.write(line)

if __name__ == '__main__':
    merge()


import json
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Iterator
from uuid import UUID

from shell_commands.commands import CommandRepository, CommandData
from shell_commands.history import HistoryRepository, HistoryEntry
from shell_commands.packages import PackageRepository
from shell_commands.utils import RunCommandOutput, serialize_datetime, deserialize_datetime

class JSONRepository(CommandRepository, PackageRepository, HistoryRepository):
    SCHEMA_VERSION = 3

    def __init__(self, dbfile: Path):
        super(JSONRepository, self).__init__()
        self.dbfile = dbfile
        if not self.dbfile.exists():
            self._write({
                'schemaVersion': self.SCHEMA_VERSION,
                'commands': {},
                'packages': [],
                'history': [],
            })

    def _load(self) -> dict:
        with self.dbfile.open('r') as fp:
            data = json.load(fp)
        assert 'schemaVersion' in data
        if data['schemaVersion'] != self.SCHEMA_VERSION:
            if data['schemaVersion'] == 1:
                # Upgrade from schema 1 to 2
                data['packages'] = []
                data['schemaVersion'] = 2
            if data['schemaVersion'] == 2:
                data['history'] = []
                data['schemaVersion'] = 3
            assert data['schemaVersion'] == self.SCHEMA_VERSION
            self._write(data)
        return data

    def _write(self, data: dict):
        with self.dbfile.open('w') as fp:
            json.dump(data, fp, indent=2)

    def save_command(self, name: str, user: str, cwd: str, command: List[str]):
        data = self._load()
        assert name not in data['commands'], 'Command with this name already exists'
        data['commands'][name] = {
            'user': user,
            'cwd': cwd,
            'command': command,
        }
        self._write(data)

    def get_command(self, name: str) -> CommandData:
        data = self._load()
        assert name in data['commands'], 'Command with this name does not exist'
        user = data['commands'][name]['user']
        cwd = data['commands'][name]['cwd']
        command = data['commands'][name]['command']
        return CommandData(name=name, user=user, cwd=cwd, command=command)

    def get_all_commands(self) -> List[CommandData]:
        return [self.get_command(name) for name in self._load()['commands'].keys()]

    def delete_command(self, name: str):
        data = self._load()
        assert name in data['commands'], 'Command with this name does not exist'
        del data['commands'][name]
        self._write(data)

    def require_package(self, name: str):
        data = self._load()
        assert name not in data['packages'], 'Package is already required so cannot require again'
        data['packages'].append(name)
        self._write(data)

    def get_required_packages(self) -> List[str]:
        data = self._load()
        return data['packages']

    def unrequire_package(self, name: str):
        data = self._load()
        assert name in data['packages'], 'Package is not required so cannot unrequire'
        data['packages'].remove(name)
        self._write(data)

    def write_to_history(self, entry: HistoryEntry):
        data = self._load()
        data['history'].append({
            'id': entry.id.hex,
            'timestamp': serialize_datetime(entry.timestamp),
            'command_name': entry.command_name,
            'command': entry.command,
            'cwd': entry.cwd,
            'user': entry.user,
            'return_code': entry.return_code,
        })
        self._write(data)

    def iterate_history(self) -> Iterator[HistoryEntry]:
        for entry in self._load()['history']:
            yield HistoryEntry(
                id=UUID(hex=entry['id']),
                timestamp=deserialize_datetime(entry['timestamp']),
                command_name=entry['command_name'],
                command=entry['command'],
                cwd=entry['cwd'],
                user=entry['user'],
                return_code=entry['return_code'],
            )


import random
import string

columna1 = []
columna2 = []
columna3 = []
columna4 = []

ingresa = int(input("Ingrese la cantidad de lineas: "))
for i in range(ingresa):
    columna1.append(random.randint(0,5000))
    columna2.append(random.choice(string.ascii_letters))
    columna3.append(random.randint(0,5000))
    columna4.append(random.randint(0,5000))

f = open("paraPoblar.csv",'w')
for i in range(len(columna1)):
    titulo="Columna1 \tColumna2 \tColumna3 \tColumna4 "
    f.write('{},{},{},{}\n'.format(columna1[i], columna2[i], columna3[i], columna4[i]))
f.close()


# -*- coding: utf-8 -*-
"""
Created on Mon Aug  9 16:36:21 2021

@author: 20210595
"""

# -*- coding: utf-8 -*-
"""
Created on Sun Jul 25 00:58:52 2021

@author: 20210595
"""

# from . import space_autogenerated
# positionsClassifiers = space_autogenerated.positionsClassifiers
# positionsPreprocessingTechniques = space_autogenerated.positionsPreprocessingTechniques
# positions = space_autogenerated.positions
# float_int_positions = space_autogenerated.float_int_positions
from space_autogenerated import positionsPreprocessingTechniques, positionsModels, positions, float_int_positions

#Pipeline
from sklearn.pipeline import make_pipeline, Pipeline

#pre-processing techniques

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Normalizer

# statsmodels
from statsmodels.tsa.ar_model import AutoReg
from statsmodels.tsa.arima.model import ARIMA
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.holtwinters import ExponentialSmoothing
from statsmodels.tsa.holtwinters import Holt
from statsmodels.tsa.holtwinters import SimpleExpSmoothing
from statsmodels.tsa.statespace.exponential_smoothing import ExponentialSmoothing
from statsmodels.tsa.exponential_smoothing.ets import ETSModel
from statsmodels.tsa.statespace.dynamic_factor import DynamicFactor
from statsmodels.tsa.statespace.dynamic_factor_mq import DynamicFactorMQ
from statsmodels.tsa.vector_ar.var_model import VAR
from statsmodels.tsa.statespace.varmax import VARMAX
from statsmodels.tsa.vector_ar.svar_model import SVAR
from statsmodels.tsa.vector_ar.vecm import VECM
from statsmodels.tsa.statespace.structural import UnobservedComponents

#example_int_to_string = int_to_string(3, rbf = 0, kernel2 = 1)

class ChoosePipeline(object):
    def __init__(self, positionsModels = positionsModels, positions = positions, position_int_float = float_int_positions, positionsPreprocessingTechniques= positionsPreprocessingTechniques):
        self.position_int_float = position_int_float
        self.positions = positions
        self.positionsPreprocessingTechniques = positionsPreprocessingTechniques
        self.positionsModels = positionsModels

    def __call__(self, x, serie):

        #NewSerie = values.reshape((len(values), 1))
        _positions = self.positions
        build_vec = self._build_vector(x)
        list_index_techniques_to_use = []
        for i in self.positionsPreprocessingTechniques:
            if x[i] > 15:
                list_index_techniques_to_use.append(i)

        lista_index_final_prepross = []
        for i in range(len(positionsPreprocessingTechniques)):
            for j in list_index_techniques_to_use:
                if positionsPreprocessingTechniques[i]==j:
                    lista_index_final_prepross.append(i)

        preprocessPipeline = self._preprossessing_techniques(lista_index_final_prepross, build_vec)
        #return preprocessPipeline

        valueIndicesModels = [x[i] for i in self.positionsModels]
        max_value = max(valueIndicesModels)
        max_index = valueIndicesModels.index(max_value)
        indexModel = self.positionsModels[max_index]

        #print(positionModelNew)
        # valueIndicesModels = [x[i] for i in self.positionsModels]
        # max_value = max(valueIndicesModels)
        # max_index = valueIndicesModels.index(max_value)
        # indexModel = self.positionsModels[max_index]
        # #The last index is the classifier
        # list_index_techniques_to_use.append(indexModel)
        # lista_index_final = []
        # for i in range(len(_positions)):
        #     for j in list_index_techniques_to_use:
        #         if positions[i][1]==j:
        #             lista_index_final.append(i)
        try:
            PipelineTSFinal = self._techniques(indexModel, build_vec, preprocessPipeline, serie=serie)
        except:
            PipelineTSFinal = 'invalid'
        # PipelineToUse = self._techniques(indexModel, build_vec, preprocessPipeline, serie=serie)
        return preprocessPipeline, PipelineTSFinal
        # return PipelineTSFinal

    def _build_vector(self, x):
        _vector_int_float = x.copy()
        _vector_pos_copy = self.position_int_float.copy()
        for i in range(len(_vector_int_float)):
            if _vector_pos_copy[i][1] == 'int':
                _vector_int_float[i] = int(round(_vector_int_float[i]))
        return _vector_int_float

    def _preprossessing_techniques(self, lista_index_final_prepross, build_vec):
        x=build_vec
        estimators = [('SimpleImputer', SimpleImputer(strategy = self._int_to_string(round(x[1]), mean=0, median=1, most_frequent=2),
                                                      fill_value=x[2]))
                      ]
        new_estimators = [estimators[i] for i in lista_index_final_prepross]
        preprossPipeline = Pipeline(new_estimators)
        return preprossPipeline

    def _techniques(self, indexModel, build_vec, preprocessPipeline, serie):
        x=build_vec
        if indexModel == self.positions[len(self.positionsPreprocessingTechniques)][1]:
            pipelineModelTS = Pipeline([('AutoReg', AutoReg(preprocessPipeline.fit(serie).transform(serie),
                                          lags=round(x[4]),
                                          trend=self._int_to_string(round(x[5]), n=0, c=1, t=2, ct=3)
                                          ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+1][1]:
            pipelineModelTS = Pipeline([('ARIMA', ARIMA(preprocessPipeline.fit(serie).transform(serie),
                                      order=(round(x[7]),round(x[8]),round(x[9])),
                                      trend=self._int_to_string(round(x[10]), n=0, c=1, t=2, ct=3),
                                      enforce_stationarity=self._int_to_bool(round(x[11])),
                                      enforce_invertibility=self._int_to_bool(round(x[12])),
                                      concentrate_scale=self._int_to_bool(round(x[13]))
                                      ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+2][1]:
            pipelineModelTS = Pipeline([('SARIMAX', SARIMAX(preprocessPipeline.fit(serie).transform(serie),
                                          order=(round(x[15]),round(x[16]),round(x[17])),
                                          trend=self._int_to_string(round(x[18]), n=0, c=1, t=2, ct=3),
                                          enforce_stationarity=self._int_to_bool(round(x[19])),
                                          enforce_invertibility=self._int_to_bool(round(x[20]))
                                          ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+3][1]:
            pipelineModelTS = Pipeline([('ExponentialSmoothing', ExponentialSmoothing(preprocessPipeline.fit(serie).transform(serie),
                                                                  trend=self._int_to_string(round(x[22]), add=0, mul=1, additive=2, multiplicative=3),
                                                                  damped_trend=self._int_to_bool(round(x[23])),
                                                                  seasonal=self._int_to_string(round(x[24]), add=0, mul=1, additive=2, multiplicative=3),
                                                                  initialization_method=self._int_to_string(round(x[25]), estimated=0, heuristic=1, known=2),
                                                                  initial_level=x[26],
                                                                  initial_trend=x[27]
                                                                  ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+4][1]:
            pipelineModelTS = Pipeline([('Holt', Holt(preprocessPipeline.fit(serie).transform(serie),
                                    exponential=self._int_to_bool(round(x[29])),
                                    damped_trend=self._int_to_bool(round(x[30])),
                                    initialization_method=self._int_to_string(round(x[31]), estimated=0, heuristic=1, known=2),
                                    initial_level=x[32],
                                    initial_trend=x[33]
                                    ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+5][1]:
            pipelineModelTS = Pipeline([('SimpleExpSmoothing', SimpleExpSmoothing(preprocessPipeline.fit(serie).transform(serie),
                                                                initialization_method=self._int_to_string(round(x[35]), estimated=0, heuristic=1, known=2),
                                                                initial_level=x[36]
                                                                ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+6][1]:
            pipelineModelTS = Pipeline([('ETSModel', ETSModel(preprocessPipeline.fit(serie).transform(serie),
                                            error=self._int_to_string(round(x[38]), add=0, mul=1),
                                            trend=self._int_to_string(round(x[39]), add=0, mul=1),
                                            damped_trend=self._int_to_bool(round(x[40])),
                                            seasonal=self._int_to_string(round(x[41]), add=0, mul=1),
                                            seasonal_periods=round(x[42]),
                                            initialization_method=self._int_to_string(round(x[43]), estimated=0, heuristic=1, known=2),
                                            initial_level=x[44],
                                            initial_trend=x[45]
                                            ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+7][1]:
            pipelineModelTS = Pipeline([('DynamicFactor', DynamicFactor(preprocessPipeline.fit(serie).transform(serie),
                                                      k_factors=round(x[47]),
                                                      factor_order=round(x[48]),
                                                      error_cov_type=self._int_to_string(round(x[49]), scalar=0, diagonal=1, unstructured=2)
                                                      ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+8][1]:
            pipelineModelTS = Pipeline([('DynamicFactorMQ', DynamicFactorMQ(preprocessPipeline.fit(serie).transform(serie),
                                                          k_endog_monthly =round(x[51]),
                                                          factors=round(x[52]),
                                                          factor_orders=round(x[53]),
                                                          factor_multiplicities=round(x[54]),
                                                          idiosyncratic_ar1=self._int_to_bool(round(x[55])),
                                                          standardize=self._int_to_bool(round(x[56])),
                                                          init_t0=self._int_to_bool(round(x[57]))
                                                          ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+9][1]:
            pipelineModelTS = Pipeline([('VAR', VAR(preprocessPipeline.fit(serie).transform(serie)))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+10][1]:
            pipelineModelTS = Pipeline([('VARMAX', VARMAX(preprocessPipeline.fit(serie).transform(serie),
                                        order=(round(x[60]),round(x[61])),
                                        trend=self._int_to_string(round(x[62]), n=0, c=1, t=2, ct=3),
                                        error_cov_type=self._int_to_string(round(x[63]), diagonal=0, unstructured=1),
                                        measurement_error=self._int_to_bool(round(x[64])),
                                        enforce_stationarity=self._int_to_bool(round(x[65])),
                                        enforce_invertibility=self._int_to_bool(round(x[66]))
                                        ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+11][1]:
            pipelineModelTS = Pipeline([('SVAR', SVAR(preprocessPipeline.fit(serie).transform(serie),
                                    svar_typestr = self._int_to_string(round(x[68]), A=0, B=1, AB=2),
                                    ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+12][1]:
            pipelineModelTS = Pipeline([('VECM', VECM(preprocessPipeline.fit(serie).transform(serie),
                                    k_ar_diff=round(x[70]),
                                    coint_rank=round(x[71]),
                                    deterministic=self._int_to_string(round(x[72]), nc=0, co=1, ci=2, lo=3, li=4)
                                    ))])
        elif indexModel == self.positions[len(self.positionsPreprocessingTechniques)+13][1]:
            pipelineModelTS = Pipeline([('UnobservedComponents', UnobservedComponents(preprocessPipeline.fit(serie).transform(serie),
                                                                    level=self._int_to_bool(round(x[74])),
                                                                    trend=self._int_to_bool(round(x[75])),
                                                                    seasonal=round(x[76]),
                                                                    cycle=self._int_to_bool(round(x[77])),
                                                                    autoregressive=round(x[78]),
                                                                    irregular=self._int_to_bool(round(x[79])),
                                                                    stochastic_level=self._int_to_bool(round(x[80])),
                                                                    stochastic_trend=self._int_to_bool(round(x[81])),
                                                                    stochastic_seasonal=self._int_to_bool(round(x[82])),
                                                                    stochastic_cycle=self._int_to_bool(round(x[83])),
                                                                    damped_cycle=self._int_to_bool(round(x[84])),
                                                                    use_exact_diffuse=self._int_to_bool(round(x[85]))
                                                                    ))])
        return pipelineModelTS

    def _int_to_string(self, value, **kwargs):
        for element in kwargs:
            if kwargs[element] == value:
                return element

    def _int_to_bool(self, value):
            if value == 1:
                return True
            else:
                return False

# from numpy import genfromtxt
# examples = genfromtxt('prueba.csv', delimiter=',')
# from pandas import read_csv
# import pandas as pd
# series = read_csv('ibm-common-stock-closing-prices.csv', header=0, index_col=0)
# NewSerie = series.values
# prueba = ChoosePipeline()
# # x_va = prueba(examples[6], NewSerie)

# nueva = list()
# for i in examples:
#     x_va = prueba(i, NewSerie)
#     nueva.append(x_va)

if __name__=='__main__':
    from numpy import genfromtxt
    examples = genfromtxt('prueba.csv', delimiter=',')
    from pandas import read_csv
    import pandas as pd
    series = read_csv('ibm-common-stock-closing-prices.csv', header=0, index_col=0)
    NewSerie = series.values
    prueba = ChoosePipeline()
    #x_va = prueba(examples[1], NewSerie)

    nueva = list()
    for i in examples:
        x_va = prueba(i, NewSerie)
        nueva.append(x_va)

# from sklearn import datasets

# # import some data to play with
# iris = datasets.load_iris()
# X = iris.data  # we only take the first two features.
# y = iris.target

# def _build_vector(x):
#     _vector_int_float = x.copy()
#     _vector_pos_copy = float_int_positions.copy()
#     for i in range(len(_vector_int_float)):
#         if _vector_pos_copy[i][1] == 'int':
#             _vector_int_float[i] = int(round(_vector_int_float[i]))
#     return _vector_int_float

# x = examples[1]
# build_vec = _build_vector(x)
# list_index_techniques_to_use = []
# for i in positionsPreprocessingTechniques:
#     if round(x[i]) == 1:
#         list_index_techniques_to_use.append(i)
# valueIndicesClassifiers = [x[i] for i in positionsClassifiers]
# max_value = max(valueIndicesClassifiers)
# max_index = valueIndicesClassifiers.index(max_value)
# indexClassifier = positionsClassifiers[max_index]
# #The last index is the classifier
# list_index_techniques_to_use.append(indexClassifier)

# auxiliar = []
# for i in range(len(positions)):
#     for j in list_index_techniques_to_use:
#         if positions[i][1]==j:
#             auxiliar.append(i)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 22 10:03:38 2022

@author: ixl
"""

import numpy as np
import matplotlib.pyplot as plt
import glob

single_list = glob.glob('/Users/ixl/Desktop/Cones/Single/*')
many_list = glob.glob('/Users/ixl/Desktop/Cones/Many/*')

singles = []
labels = []
many = []
for sin in single_list:
    if 'xy' in sin:
        singles.append(np.loadtxt(sin,delimiter=' ',usecols=(0,1)))
        labels.append(int(sin.split('/')[6].strip('mm_xy.txt')))
'''
for man in many_list:
    many.append(np.loadtxt(man,delimiter=' '))
'''
plt.figure()
for i in range(len(singles)):
    plt.hist(singles[i][:,0],label=labels[i],histtype='barstacked',bins=50)
plt.legend()
'''
plt.plot(singles[0][:,0])
plt.plot(many[0][:,1])
'''
plt.show()


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
import os
import math
import yaml
import csv
import cv2
import shutil
import numpy as np
import tensorflow as tf
from tensorflow.keras import datasets,layers,models,losses,optimizers,Model
from params import *

def inception(inputs,filters):
    a = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    b = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    b = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(b)
    #c = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    #c = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(c)
    #c = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(c)
    #d = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(inputs)
    #d = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(d)
    #d = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(d)
    #d = layers.Conv2D(filters,(3,3),activation='relu',padding='same')(d)
    #e = layers.MaxPooling2D((3,3),strides=(1,1),padding='same')(inputs)
    #e = layers.Conv2D(filters,(1,1),activation='relu',padding='same')(e)
    #return layers.Concatenate(axis=3)([a,b,c,d,e])
    #return layers.Concatenate(axis=3)([a,b,c,e])
    return layers.Concatenate(axis=3)([a,b])

def reduction(inputs,f):
    a = layers.Conv2D(f,(3,3),strides=(2,2),activation='relu')(inputs)
    return a

def create(cutout_size,depth,filters,rate):
    if depth:
        inputs = layers.Input(shape=(cutout_size,cutout_size,4))
    else:
        inputs = layers.Input(shape=(cutout_size,cutout_size,3))
    a = inception(inputs,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = reduction(a,filters)
    a = inception(a,filters)
    a = layers.Flatten()(a)
    outputs = layers.Dense(5)(a)
    model = Model(inputs=inputs,outputs=outputs)
    model.compile(
        optimizer=optimizers.Nadam(learning_rate=rate),
        loss='mean_absolute_error',
        metrics=['acc'])
    model.summary()
    return model

def infer(model,inputs):
    return model.predict(tf.expand_dims(inputs,0),steps=1)[0]

def load_dataset(depth,path,csv_name):
    inputs = []
    outputs = []
    instances = []
    with open(csv_name,newline='') as file:
        for row in csv.reader(file):
            r = 0
            name = row[r]
            r += 1
            center = (float(row[r]),float(row[r + 1]))
            r += 2
            screen = (float(row[r]),float(row[r + 1]))
            r += 2
            ndc = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            head_pos = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            head_dir = (float(row[r]),float(row[r + 1]))
            r += 2
            light_dir = (float(row[r]),float(row[r + 1]))
            r += 2
            light_color = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            ambient_color = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            skin_color = (float(row[r]),float(row[r + 1]),float(row[r + 2]))
            r += 3
            if depth:
                image = np.multiply(cv2.imread(path + name,cv2.IMREAD_UNCHANGED).astype(np.float32),1.0 / 255.0)
            else:
                image = np.multiply(cv2.imread(path + name).astype(np.float32),1.0 / 255.0)
            inputs.append(image)
            dx = screen[0] - center[0]
            dy = screen[1] - center[1]
            output = [dx,dy,ndc[2],head_dir[0],head_dir[1]]
            outputs.append(output)
            instances.append((name,center,screen,ndc,head_pos,head_dir,light_dir,light_color,ambient_color,skin_color))
    return (np.array(inputs),np.array(outputs),instances)

def train(model,dataset,epochs,batch_size):
    n = int(0.2 * dataset[0].shape[0])
    x_train = dataset[0][n:]
    y_train = dataset[1][n:]
    x_val = dataset[0][:n]
    y_val = dataset[1][:n]
    history = model.fit(
        x=x_train,
        y=y_train,
        validation_data=[x_val,y_val],
        epochs=epochs,
        batch_size=batch_size,
        verbose=2,
    )

def test(model,dataset,frame_width,frame_height,cutout_size):
    errors = []
    error_names = ['screen.x','screen.y','ndc.x','ndc.y','ndc.z','head_pos.x','head_pos.y','head_pos.z','head_dir.y','head_dir.p','skin.r','skin.g','skin.b']

    mxx = 2.7990382
    myy = 3.7320509
    mzz = -1.0020020
    mzw = -1.0
    mwz = -0.2002002
    mww = 0.0

    for i in range(0,len(dataset[0])):
        print('    {} / {}'.format(i,len(dataset[0]) - 1))
        inference = infer(model,dataset[0][i])
        dx = inference[0]
        dy = inference[1]
        ndcz = inference[2]
        heady = inference[3]
        headp = inference[4]
        screenx = dataset[2][i][1][0] + dx
        screeny = dataset[2][i][1][1] + dy
        escreenx = dataset[2][i][2][0] - screenx
        escreeny = dataset[2][i][2][1] - screeny
        ndcx = 2.0 * screenx / frame_width - 1.0
        ndcy = 1.0 - 2.0 * screeny / frame_height
        endcx = dataset[2][i][3][0] - ndcx
        endcy = dataset[2][i][3][1] - ndcy
        endcz = dataset[2][i][3][2] - ndcz
        z = (mzw - mww * ndcz) / (mwz * ndcz - mzz)
        homw = mwz * z + mww
        x = homw * ndcx / mxx
        y = homw * ndcy / myy
        ex = dataset[2][i][4][0] - x
        ey = dataset[2][i][4][1] - y
        ez = dataset[2][i][4][2] - z
        eheady = dataset[2][i][5][0] - heady
        eheadp = dataset[2][i][5][1] - headp
        errors.append((escreenx,escreeny,endcx,endcy,endcz,ex,ey,ez,eheady,eheadp))
    total = len(errors)
    n = len(errors[0])
    avg = []
    for i in range(0,n):
        avg.append(0.0)
    for error in errors:
        for i in range(0,n):
            avg[i] += error[i]
    for i in range(0,n):
        avg[i] /= total
    dif = []
    for i in range(0,n):
        dif.append(0.0)
    for error in errors:
        for i in range(0,n):
            d = error[i] - avg[i]
            dif[i] += d * d
    almost_total = total - 1.0
    for i in range(0,n):
        dif[i] /= almost_total
    stddev = []
    for i in range(0,n):
        stddev.append(math.sqrt(dif[i]))
    print('statistics:')
    for i in range(0,n):
        print('    {:>16}: {:>6.3f} +/- {:>6.3f}'.format(error_names[i],avg[i],stddev[i]))

if __name__ == '__main__':
    params = Params('./params.yaml')

    weights_name = './pose.h5'

    path1data = './data1/'
    csv1data = './data1/files.csv'
    path1test = './test1/'
    csv1test = './test1/files.csv'

    if len(sys.argv) < 2:
        print('usage:')
        print('')
        print('    python3 pose.py <command>')
        print('')
        print('where command is:')
        print('    train       - start training')
        print('    train more  - improve training')
        print('    test        - numerically test the network')
        exit(-1)

    if sys.argv[1] == 'train':
        print('creating model...')
        model = create(params.cutout_size,params.depth,params.pose_filters,params.pose_rate)
        print('loading data1 dataset...')
        dataset = load_dataset(params.depth,path1data,csv1data)
        if (len(sys.argv) > 2) and (sys.argv[2] == 'more'):
            print('loading old weights...')
            model.load_weights(weights_name)
        print('training...')
        train(model,dataset,params.pose_epochs,params.pose_batch_size)
        print('saving new weights...')
        model.save_weights(weights_name)

    elif sys.argv[1] == 'test':
        print('creating model...')
        model = create(params.cutout_size,params.depth,params.pose_filters,params.pose_rate)
        print('loading weights...')
        model.load_weights(weights_name)
        print('loading test0 dataset...')
        dataset = load_dataset(params.depth,path1test,csv1test)
        print('measuring statistics...')
        test(model,dataset,params.frame_width,params.frame_height,params.cutout_size)


# 参考地址：https://github.com/PaddlePaddle/PGL/blob/main/examples/gcn/train.py
# 模型的保存与加载：https://www.paddlepaddle.org.cn/documentation/docs/zh/tutorial/quick_start/save_model/save_model.html
import paddle
import argparse
from preprocess import load
from GCN import GCN
from GAT import GAT
from GCNII import GCNII
from pgl.utils.logger import log
from paddle.optimizer import Adam
def train(node_index, node_label, gnn_model, graph, criterion, optim):
    gnn_model.train()
    pred = gnn_model(graph, graph.node_feat["feat"])
    pred = paddle.gather(pred, node_index)
    loss = criterion(pred, node_label)
    loss.backward()
    acc = paddle.metric.accuracy(input=pred, label=node_label, k=1)
    optim.step()
    optim.clear_grad()
    return loss, acc
@paddle.no_grad()
def eval(node_index, node_label, gnn_model, graph, criterion):
    gnn_model.eval()
    pred = gnn_model(graph, graph.node_feat["feat"])
    pred = paddle.gather(pred, node_index)
    loss = criterion(pred, node_label)
    acc = paddle.metric.accuracy(input=pred, label=node_label, k=1)
    return loss, acc
def main(args):
    # 初始化数组
    dur = []
    best_test = []
    cal_val_acc = []
    cal_test_acc = []
    cal_val_loss = []
    cal_test_loss = []
    # step1. 数据加载
    dataset = load()
    # dataset.graph.tensor()
    graph = dataset.graph
    train_index = dataset.train_index
    train_label = dataset.train_label
    val_index = dataset.valid_index
    val_label = dataset.valid_label
    test_index = dataset.test_index
    # step2. 损失函数配置
    criterion = paddle.nn.loss.CrossEntropyLoss()
    # step3. 模型配置
    if args.model_name == 'GCN':
        gnn_model = GCN(input_size=graph.node_feat["feat"].shape[1],
                                num_class=dataset.num_classes,
                                num_layers=args.num_layers,
                                dropout=0.5,
                                hidden_size=128)
    if args.model_name == 'GAT':
        gnn_model = GAT(input_size=graph.node_feat["feat"].shape[1],
                        num_class=dataset.num_classes,
                        num_layers=args.num_layers,
                        feat_drop=0.6,
                        attn_drop=0.6,
                        num_heads=8,
                        hidden_size=64)
    if args.model_name == 'GCNII':
        gnn_model = GCNII(input_size=graph.node_feat["feat"].shape[1],
                        num_class=dataset.num_classes,
                        num_layers=args.num_layers,
                        )
    # step4. 优化方法配置
    optim = Adam(
            learning_rate=args.lr,
            parameters=gnn_model.parameters(),
            weight_decay=0.0005)
    # step4. 模型训练
    gnn_model.train()
    for epoch in range(args.epoch):
        train_loss, train_acc = train(train_index, train_label, gnn_model,
                                          graph, criterion, optim)

        val_loss, val_acc = eval(val_index, val_label, gnn_model, graph,
                                    criterion)
        cal_val_acc.append(val_acc.numpy())
        cal_val_loss.append(val_loss.numpy())
        layer_state_dict = gnn_model.state_dict()
        paddle.save(layer_state_dict,"work/GCN.pdparams")
        # paddle.save(gnn_model.weight,"work/GCN.weight.pdtensor")
        log.info(f"epoch{epoch}:Model:val Accuracy:{val_acc.numpy()}   Loss:{val_loss.numpy()}")
        if val_acc.numpy()>0.71:
            print(val_acc.numpy())
            break
    # log.info("Runs %s: Model: GCN Best Test Accuracy: %f" %
    #              (run, cal_test_acc[np.argmin(cal_val_loss)]))
    # best_test.append(cal_test_acc[np.argmin(cal_val_loss)])
    # log.info("Average Speed %s sec/ epoch" % (np.mean(dur)))
    # log.info("Dataset: %s Best Test Accuracy: %f ( stddev: %f )" %
    #          (args.dataset, np.mean(best_test), np.std(best_test)))

if __name__=="__main__":
    parser = argparse.ArgumentParser(
        description='Benchmarking Citation Network')
    parser.add_argument("--model_name", type=str,default='GCN', help="model_name")
    parser.add_argument("--num_layers",type=int,default=1,help="layers number")
    parser.add_argument("--lr",type=float,default=0.01,help="learning_rate")
    parser.add_argument("--epoch", type=int, default=200, help="Epoch")
    parser.add_argument("--runs", type=int, default=10, help="runs")
    parser.add_argument(
        "--feature_pre_normalize",
        type=bool,
        default=True,
        help="pre_normalize feature")
    args = parser.parse_args()
    log.info(args)
    main(args = args)


import argparse
import collections
import operator
import os
import subprocess
from typing import Any, Dict

from sigma.backends.loki import LogQLBackend
from sigma.collection import SigmaCollection
from sigma.pipelines.sysmon import sysmon_pipeline
from sigma.pipelines.loki import loki_grafana_logfmt, loki_promtail_sysmon
from sigma.rule import SigmaDetection, SigmaError

parser = argparse.ArgumentParser(
    description="A script to help test pySigma backends using Sigma signature files",
    epilog="For any issues or requests for support, see the GitHub page",
)
parser.add_argument(
    "signature_path",
    help="A path to either a single Sigma signature YAML file or a directory containing "
    "one or more signatures (incl. sub-folders)",
)
parser.add_argument(
    "-a",
    "--add-line-filters",
    action="store_true",
    help="Attempt to add a single line filter to queries that otherwise lack any, to help improve "
    "the overall performance of the query when being run on Loki.",
)
parser.add_argument(
    "-c",
    "--counts",
    action="store_true",
    help="Produce counts of the numbers of signatures processed and counts of successes/fails, "
    "along with error counts",
)
parser.add_argument(
    "-p",
    "--print",
    action="store_true",
    help="Print the query(s) that were generated for the backend, and validation stdout if "
    "applicable",
)
parser.add_argument(
    "-s",
    "--summarize",
    action="store_true",
    help="Produce a summary of the processed signature's log source information",
)
parser.add_argument(
    "-t",
    "--tests",
    type=str,
    help="A path to either a single test log file or a directory containing one or more test log "
    "files, used during the validation of the generated rule(s). If a path to a directory is "
    "provided, this script will look for .log files with the same directory structure of the "
    "signature_path (i.e., if there is a rules/bad.yml file within signature_path, the script "
    "will look for a rules/bad.log file within the path specified in tests)",
)
parser.add_argument(
    "-u",
    "--unique",
    action="store_true",
    help="Print the unique types and messages of errors that occur during the process",
)
parser.add_argument(
    "-v",
    "--validate",
    action="store_true",
    help="Validate the generated rule(s) through the backend engine, where one or more lines of "
    "stdout denotes a successful validation",
)
parser.add_argument(
    "-w",
    "--windows",
    action="store_true",
    help="Use Windows sysmon pipelines",
)

args = parser.parse_args()

rule_path = args.signature_path

pipeline = (
    sysmon_pipeline() + loki_promtail_sysmon()
    if args.windows
    else loki_grafana_logfmt()
)

backend = LogQLBackend(
    processing_pipeline=pipeline,
    add_line_filters=args.add_line_filters,
)

counters: Dict[str, Any] = {
    "parse_error": 0,
    "convert_error": 0,
    "validate_error": 0,
    "total_sigs": 0,
    "total_queries": 0,
    "total_files": 0,
    "total_test_logs": 0,
    "convert_success": 0,
    "validate_success": 0,
    "fields": {},
    "error_types": {},
    "error_messages": {},
    "validate_stdout": {},
    "validate_stderr": {},
    "categories": {},
    "products": {},
    "services": {},
}

def validate_with_backend(query, test_file=subprocess.DEVNULL):
    if test_file is None:
        test_file = subprocess.DEVNULL
    result = subprocess.run(
        ["logcli", "--stdin", "query", query], stdin=test_file, capture_output=True
    )
    stdout = result.stdout.decode()
    stderr = result.stderr.decode()
    valid = False
    # If we have an input file, the query is valid if logcli produces one or more lines
    # of output. Otherwise, check logcli's return code to ensure the query is
    # syntactically valid
    if test_file is not subprocess.DEVNULL:
        valid = stdout.count(os.linesep) > 0
    else:
        valid = result.returncode == 0
    return (result.returncode, stdout, stderr, valid)

def find_all_detection_items(detection, acc):
    for value in detection.detection_items:
        if isinstance(value, SigmaDetection):
            return find_all_detection_items(value, acc)
        else:
            return acc + [value]

def process_file(file_path, test_file, args, counters):
    with open(file_path) as rule_file:
        sigma_rules = None
        yaml = rule_file.read()
        counters["total_files"] += 1
        try:
            sigma_rules = SigmaCollection.from_yaml(yaml)
            counters["total_sigs"] += len(sigma_rules)
            if args.summarize:
                for rule in sigma_rules:
                    fields = (
                        list(
                            item.field
                            for detection in rule.detection.detections.values()
                            for item in find_all_detection_items(detection, [])
                            if item.field is not None
                        )
                        + rule.fields
                    )
                    for field in fields:
                        counters["fields"][field] = counters["fields"].get(field, 0) + 1
                    cat = rule.logsource.category
                    prod = rule.logsource.product
                    serv = rule.logsource.service
                    if prod:
                        counters["products"][prod] = (
                            counters["products"].get(prod, 0) + 1
                        )
                    if serv:
                        counters["services"][serv] = (
                            counters["services"].get(serv, 0) + 1
                        )
                    if cat:
                        counters["categories"][cat] = (
                            counters["categories"].get(cat, 0) + 1
                        )
        except SigmaError as err:
            counters["parse_error"] += 1
            if args.unique:
                error_type = type(err).__name__
                counters["error_types"][error_type] = (
                    counters["error_types"].get(error_type, 0) + 1
                )
                counters["error_messages"][str(err).strip()] = (
                    counters["error_messages"].get(str(err).strip(), 0) + 1
                )
            return
        try:
            loki_rules = backend.convert(sigma_rules)
            counters["convert_success"] += len(sigma_rules)
            counters["total_queries"] += len(loki_rules)
            if args.validate or args.print:
                for loki_query in loki_rules:
                    if args.print:
                        print(loki_query)
                    if args.validate:
                        (returncode, stdout, stderr, valid) = validate_with_backend(
                            loki_query, test_file
                        )
                        if returncode != 0:
                            counters["validate_error"] += 1
                        elif valid:
                            counters["validate_success"] += 1
                        if args.print and len(stdout) > 0:
                            print(stdout.strip())
                        if args.unique and len(stderr) > 0:
                            counters["validate_stderr"][stderr.strip()] = (
                                counters["validate_stderr"].get(stderr.strip(), 0) + 1
                            )
        except SigmaError as err:
            counters["convert_error"] += 1
            if args.unique:
                error_type = type(err).__name__
                counters["error_types"][error_type] = (
                    counters["error_types"].get(error_type, 0) + 1
                )
                counters["error_messages"][str(err)] = (
                    counters["error_messages"].get(str(err), 0) + 1
                )

def print_counts(dct):
    if len(dct) == 0:
        print("\tNo results")
        return
    for k, v in collections.OrderedDict(
        sorted(dct.items(), key=operator.itemgetter(1), reverse=True)
    ).items():
        print(f"\t{k}: {v}")

def get_log_file(rule_file_path):
    (root, _) = os.path.splitext(os.path.basename(rule_file_path))
    return root + ".log"

test_file = None
test_dir = None
if args.tests:
    if os.path.isfile(args.tests):
        test_file = open(args.tests)
        counters["total_test_logs"] += 1
    elif os.path.isdir(args.tests):
        test_dir = args.tests
    else:
        print(f"Could not find test file/directory: {args.tests}")
        exit(1)

if os.path.isfile(rule_path):
    if test_dir:
        test_file_path = os.path.join(test_dir, get_log_file(rule_path))
        if os.path.isfile(test_file_path):
            test_file = open(test_file_path)
            counters["total_test_logs"] += 1
    process_file(rule_path, test_file, args, counters)
elif os.path.isdir(rule_path):
    for dirpath, dirnames, filenames in os.walk(rule_path):
        for filename in filenames:
            rule_file_path = os.path.join(dirpath, filename)
            if test_dir:
                if test_file:
                    test_file.close()
                test_file_path = os.path.join(
                    test_dir,
                    os.path.relpath(dirpath, start=rule_path),
                    get_log_file(filename),
                )
                if os.path.isfile(test_file_path):
                    test_file = open(test_file_path)
                    counters["total_test_logs"] += 1
                else:
                    test_file = None
            elif test_file:
                test_file.seek(0)  # reset the stream position each time
            process_file(rule_file_path, test_file, args, counters)
else:
    print(f"Could not find rule file/directory: {rule_path}")
    exit(1)

if args.counts:
    percent_conv = counters["convert_success"] / counters["total_sigs"] * 100
    print(
        f"Successfully converted {counters['convert_success']} out of "
        f"{counters['total_sigs']} ({percent_conv:.2f}%) signatures"
    )
    if args.validate:
        percent_valid = counters["validate_success"] / counters["total_queries"] * 100
        print(
            f"Successfully validated {counters['validate_success']} out of "
            f"{counters['total_queries']} ({percent_valid:.2f}%) queries"
        )
    print(f"YAML parse errors: {counters['parse_error']}")
    print(f"Conversion errors: {counters['convert_error']}")
    if args.validate:
        print(f"Validation errors: {counters['validate_error']}")
        if args.tests:
            print(f"Test log files used: {counters['total_test_logs']}")

if args.unique:
    print("Error counts:")
    print_counts(counters["error_types"])
    print("Error messages:")
    print_counts(counters["error_messages"])
    if args.validate:
        print("Validation stderr:")
        print_counts(counters["validate_stderr"])

if args.summarize:
    print("Fields:")
    print_counts(counters["fields"])
    print("Products:")
    print_counts(counters["products"])
    print("Services:")
    print_counts(counters["services"])
    print("Categories:")
    print_counts(counters["categories"])

if test_file:
    test_file.close()


from models import Note
from django.contrib.auth.models import User
from random import randrange
from sys import stdout
from django.utils.timezone import now
from django.utils.encoding import smart_unicode

def gen_random_string_en(dicts_array, size):
    string = ""
    dicts_size = len(dicts_array)

    while size > len(string):
        string += dicts_array[randrange(dicts_size)] + " "

    return string

def gen_random_string_zh(dicts_array, size):
    string = ""
    dicts_size = len(dicts_array)

    while size > len(string):
        string += smart_unicode(dicts_array[randrange(dicts_size)])
        if 0 == (len(string) % randrange(2, 5)):
            string += " "

    return string

def gen(count, dicts, get_string_function):
    user = User.objects.all()[0]
    date_now = now()
    total = count

    while 0 < count:
        stdout.write("%d\r" % (total - count))
        stdout.flush()
        count -= 1
        note = Note()
        note.user = user
        note.pub_date = date_now
        note.title = get_string_function(dicts, randrange(20, 100))
        note.body = get_string_function(dicts, randrange(500, 1000))
        note.save()

def gen_en(count):
    f = open("dict.txt", "r")
    dicts = [line.strip() for line in f]
    f.close()

    gen(count, dicts, gen_random_string_en)

def gen_zh(count):
    f = open("zh_dict.txt", "r")
    dicts = [line.strip() for line in f]
    f.close()

    gen(count, dicts, gen_random_string_zh)


# GENELLER PYTHON NOTLARI
#-----------------------------------------------------------------
# NUMBERS AND CHARACTERS INFORMATION (NUMARA VE KARAKTER BILGILERI)
#-----------------------------------------------------------------
# Sayilar integer ve float olarak ikiye ayrilir. int= tam sayilari;
# float= ondalikli yani kesirlileri gosterir.
# Stringler(Karakter Dizinleri)= Cift tirnak veya tek tirnak kullanilir.
# "type()" fonksiyonu bize parantez icine girilen her seyin turunu gosterir.
# "print()" fonksiyonu bizim ekrana yazdir fonksiyonumuzdur.

print("Merhabalar")

type(9)

type(12.6)

type("R")

type("Merhabalar")

#-----------------------------------------------------------------

# STRINGS CLOSER LOOK (STRINGLERE YAKINDAN BAKMAK)
# " " kullandigimizda python str olarak algiliyor

"123"
type("123")

#-----------------------------------------------------------------
# iki str ifadesini bir araya getirmek

"a" + "-b" # ( buradaki + isareti toplama anlaminda degildir. )
"a" " b" # veya bu sekilde de olabilir.

# str ifadesini cogaltmak

"a "*7

#------------------------------------------------------------------
# UZUNLUK (ELEMAN SAYISI) BILGISINE ERISMEK = len() FONSIYONU
# Metodlar veri veya belili yapilar uzerine uygulanan cesitli fonsiyonlardir.

pyt_org = "python_ogreniyorum"

# onceki w hala degisken tablosunda duruyor onu silmek icin sag tiklayip remove
# veya asagiya yazacagim gibi kod silme komutu da oluyor
# del w silme islemi bittikten sonra diyez isareti ile yorum satiri yapiyoruz.

a = 10
b = 215

a*b

len(pyt_org)
len("python_ogreniyorum")

#------------------------------------------------------------------
# BUYUK-KUCUK HARF DONUSUMLERI = upper() &lower()  FONSIYONU

pyt_org = "python_ogreniyorum"

pyt_org.upper() # buyuk harflerle

pyt_org.lower() # kucuk harflerle

pyt_org.islower()
B = pyt_org.upper()

pyt_org.isupper()

B.isupper()

B.islower()

#------------------------------------------------------------------
# KARAKTER DEGISTIRME = replace() FONSIYONU
# harf degistirmeye yarar. Ornegin e'leri a yapalim.

pyt_org.replace("e", "a")

pyt_org.replace("a", "e")

#------------------------------------------------------------------
# KARAKTER KIRPMA = strip() FONSIYONU
# istenmeyen karakterleri kirpmak icin kullanilir
# Orengin asagidakinin kenarlarindaki cizgileri yok edelim.

pyt_org = " _python_ogreniyorum_ "

pyt_org.strip("_")

#------------------------------------------------------------------
#SUBSTRINGLER
gel_yaz = "gelecegi_yazanlar"

gel_yaz[1]

gel_yaz[15]

gel_yaz[0:5] # 0'dan basla 5'e kadar anlaminda

gel_yaz[3:11]

#------------------------------------------------------------------
#DEGISKENLER

A = 99
b =  "ali_uzaya_git"
C = A/3

A/C

A*C

A*5

#------------------------------------------------------------------
# TYPE DONUSUMLERI

toplama_bir = input()
toplama_iki = input()

toplama_bir + toplama_iki #Burda ikisi de str deger oldugu icin yan yana yazdi yani birlestirdi.

int(toplama_bir) + int(toplama_iki)

10.0    # float degeri int donusturme;

type(10.0)
int(10.0)

12      # int degeri float'a donusturme;

type(12)
float(12)

#------------------------------------------------------------------
#-print() Fonksiyonu = Ekrana yazdirmaya yariyor

print("gelecegi" , "yazanlar")

print("gelecegi" , "yazanlar" , sep= "_") # sep fonk birlestirmeye yariyor.


import json
from unified.core.actions import Actions
from documents.google_docs import util
from documents.google_docs.entities.google_docs_document import GoogledocsDocument

class GoogledocsAction(Actions):

    def create_document_from_text(self, context, payload):
        '''Create a  file with text.'''

        access_token = util.get_access_token(context["headers"])
        url = "https://docs.googleapis.com/v1/documents"
        file = GoogledocsDocument(**payload)
        create_file = {
            "title": file.name
        }
        response = util.rest("post", url, access_token, create_file)

        if response.ok:
            file_id = json.loads(response.text)['documentId']
            # updating content in docs
            update_content_response = self.create_content_in_file(file_id, file.content, access_token)

            if update_content_response.ok:
                # move file to given folder id
                if file.folder_id:
                    move_file = self.move_file(access_token, file_id, file.folder_id)
                update_content_obj = json.loads(update_content_response.text)
                update_content_obj['id'] = file_id

                return json.loads(update_content_obj), update_content_response.status_code

        return json.loads(response.text), response.status_code

    def create_content_in_file(self, file_id, file_content, access_token):
        '''create a file content'''

        url = f"https://docs.googleapis.com/v1/documents/{file_id}:batchUpdate"
        request_body = {
            "requests": [{"insertText": {
                "text": file_content,
                "endOfSegmentLocation": {"segmentId": ""}}}]}
        return util.rest("POST", url, access_token, request_body)

    def move_file(self, access_token, file_id, folder_id):
        '''Move a file from one folder to another.'''

        url = f"https://www.googleapis.com/drive/v2/files/{file_id}/parents"
        request_body = {
            "id": folder_id
        }
        response = util.rest("POST", url, access_token, request_body)

        return response

    def create_document_from_template(self, context, payload):
        '''Creates a new doc based on an existing one'''

        access_token = util.get_access_token(context["headers"])
        file = GoogledocsDocument(**payload)
        url = f"https://www.googleapis.com/drive/v2/files/{file.template_id}/copy"
        file_body = {
            "title": file.name
        }

        response = util.rest("POST", url, access_token, file_body)

        if response.ok:
            templete = json.loads(response.text)
            file_id = templete.get('id')
            # Moving file to given folder
            if file.folder_id:
                move_file_response = self.move_file(access_token, file_id, file.folder_id)
                if not move_file_response.ok:
                    return json.loads(move_file_response.text), move_file_response.status_code

            return json.loads(response.text), response.status_code

    def append_text_to_document(self, context, payload):
        '''Appends text to an existing document.'''

        file = GoogledocsDocument(**payload)
        access_token = util.get_access_token(context["headers"])
        data = self.create_content_in_file(file.document_id, file.text_to_append, access_token)
        return json.loads(data.text), data.status_code


from __future__ import unicode_literals

from django.utils.encoding import python_2_unicode_compatible
from jsonfield import JSONField

from multigtfs.models.base import models, Base

@python_2_unicode_compatible
class FareRule(Base):
    """Associate a Fare with a Route and/or Zones"""
    fare = models.ForeignKey('Fare', on_delete=models.CASCADE)
    route = models.ForeignKey(
        'Route', null=True, blank=True, on_delete=models.SET_NULL,
        help_text="Fare class is valid for this route.")
    origin = models.ForeignKey(
        'Zone', null=True, blank=True, on_delete=models.SET_NULL,
        related_name='fare_origins',
        help_text="Fare class is valid for travel originating in this zone.")
    destination = models.ForeignKey(
        'Zone', null=True, blank=True, on_delete=models.SET_NULL,
        related_name='fare_destinations',
        help_text="Fare class is valid for travel ending in this zone.")
    contains = models.ForeignKey(
        'Zone', null=True, blank=True, on_delete=models.SET_NULL,
        related_name='fare_contains',
        help_text="Fare class is valid for travel withing this zone.")
    extra_data = JSONField(default={}, blank=True, null=True)

    def __str__(self):
        u = "%d-%s" % (self.fare.feed.id, self.fare.fare_id)
        if self.route:
            u += '-%s' % self.route.route_id
        return u

    class Meta:
        db_table = 'fare_rules'
        app_label = 'multigtfs'

    # For Base import/export
    _column_map = (
        ('fare_id', 'fare__fare_id'),
        ('route_id', 'route__route_id'),
        ('origin_id', 'origin__zone_id'),
        ('destination_id', 'destination__zone_id'),
        ('contains_id', 'contains__zone_id')
    )
    _filename = 'fare_rules.txt'
    _rel_to_feed = 'fare__feed'
    _sort_order = ('route__route_id', 'fare__fare_id')
    _unique_fields = (
        'fare_id', 'route_id', 'origin_id', 'destination_id', 'contains_id')


from pathlib import Path

from django.core.exceptions import SuspiciousFileOperation
from django.template import Origin, TemplateDoesNotExist
from django.template.loaders.filesystem import Loader as FilesystemLoader
from django.utils._os import safe_join

from . import themes

class Loader(FilesystemLoader):
    def get_template_sources(self, template_name):
        template_path = template_name.split(";")
        if (
            len(template_path) == 3
            and template_path[0] == "themes"
            and template_path[1] in themes
        ):
            theme_info = themes[template_path[1]]
            try:
                name = safe_join(theme_info["path"] / "templates", template_path[2])
            except SuspiciousFileOperation:
                return

            yield Origin(
                name=name,
                template_name=template_name,
                loader=self,
            )


import pandas as pd
import numpy as np
import numpy.random as rd
import sklearn
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

def pca_trafo(pose_feats,comp):
    n_comp = comp
    pca_trafo = PCA(n_components=n_comp)
    data = pose_feats

    pca_data = pca_trafo.fit_transform(data)
    pca_inv_data = pca_trafo.inverse_transform(pca_data)

    """plt.figure(figsize = (10,6.5));
    plt.semilogy(pca_trafo.explained_variance_ratio_, '--o');
    plt.xlabel('principal component', fontsize = 20);
    plt.ylabel('explained variance', fontsize = 20);
    plt.tick_params(axis='both', which='major', labelsize=10);
    plt.tick_params(axis='both', which='minor', labelsize=10);
    plt.xlim([0, 66]);"""

    z_scaler = StandardScaler()
    z_data = z_scaler.fit_transform(data)
    pca_trafo2 = PCA().fit(z_data);

    fig, ax1 = plt.subplots(figsize = (10,6.5))
    ax1.set_xlabel('principal component', fontsize = 10);

    plt.legend(loc=(0.01, 0.075) ,fontsize = 10);

    ax2 = ax1.twinx()
    ax2.semilogy(pca_trafo.explained_variance_ratio_.cumsum(), '--go', label = 'cumulative explained variance ratio');
    for tl in ax2.get_yticklabels():
        tl.set_color('g')

    ax2.tick_params(axis='both', which='minor', labelsize=10);
    ax2.tick_params(axis='both', which='minor', labelsize=10);
    plt.xlim([0, 66]);
    plt.legend(loc=(0.01, 0),fontsize = 18);

    fig = plt.figure(figsize=(12, 10))
    sns.heatmap(pca_trafo.inverse_transform(np.eye(n_comp)), cmap="hot", cbar=False)
    plt.ylabel('principal component', fontsize=20);
    plt.xlabel('original feature index', fontsize=20);
    plt.tick_params(axis='both', which='major', labelsize=10);
    plt.tick_params(axis='both', which='minor', labelsize=10);

    fig = plt.figure(figsize=(12, 10))
    plt.plot(pca_inv_data.mean(axis=0), '--o', label = 'mean')
    plt.plot(np.square(pca_inv_data.std(axis=0)), '--o', label = 'variance')
    plt.legend(loc='lower right')
    plt.ylabel('feature contribution', fontsize=20);
    plt.xlabel('feature index', fontsize=20);
    plt.tick_params(axis='both', which='major', labelsize=10);
    plt.tick_params(axis='both', which='minor', labelsize=10);
    plt.xlim([0, 66])
    plt.legend(loc='lower left', fontsize=10)
    plt.show()
    return fig


from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('evaluation', '0033_remove_likert_and_grade_answer'),
    ]

    operations = [
        migrations.AddField(
            model_name='course',
            name='gets_no_grade_documents',
            field=models.BooleanField(default=False, verbose_name='gets no grade documents'),
        ),
    ]


from unified.core.triggers import Triggers
from accounting_invoicing.zoho_books.entities.zoho_books_customer import  ZohobooksCustomer
from accounting_invoicing.zoho_books.entities.zoho_books_item import ZohobooksItem
from accounting_invoicing.zoho_books.entities.zoho_books_sales_invoice import ZohobooksSalesinvoice
from accounting_invoicing.zoho_books.entities.zoho_books_estimate import ZohobooksEstimate
from accounting_invoicing.zoho_books.entities.zoho_books_expense import ZohobooksExpense
import json

class ZohobooksTriggers(Triggers):

    def new_customer(self, context, payload):
        ''' triggers when new customer created'''
        customer=payload["JSONString"]["contact"]
        customer_obj = ZohobooksCustomer(
                                    contact_name = customer["contact_name"],
                                    company_name = customer["company_name"],
                                    website = customer["website"],
                                    payment_terms= customer["payment_terms"],
                                    notes= customer["notes"],
                                    billing_address=customer["billing_address"],
                                    billing_address_city= customer["billing_address"]["city"],
                                    billing_address_state= customer["billing_address"]["state"],
                                    billing_address_zip= customer["billing_address"]["zip"],
                                    billing_address_country= customer["billing_address"]["country"],
                                    billing_address_fax= customer["billing_address"]["fax"],
                                    shipping_address=customer["shipping_address"],
                                    shipping_address_city= customer["shipping_address"]["city"],
                                    shipping_address_state= customer["shipping_address"]["state"],
                                    shipping_address_zip= customer["shipping_address"]["zip"],
                                    shipping_address_country= customer["shipping_address"]["country"],
                                    shipping_address_fax= customer["shipping_address"]["fax"],
                                    first_name=customer["first_name"],
                                    last_name= customer["last_name"],
                                    email= customer["email"],
                                    phone= customer["phone"],
                                    mobile= customer["mobile"]
                                    )
        return customer_obj.__dict__

    def new_estimate(self, context, payload):
        ''' triggers when new estimate created'''
        estimate=payload["JSONString"]["estimate"]
        data=estimate["line_items"][0]
        estimate_obj = ZohobooksEstimate(
                                        customer_id= estimate["customer_id"],
                                        start_date= estimate["date"],
                                        end_date= estimate["expiry_date"],
                                        exchange_rate= estimate["exchange_rate"],
                                        discount= data["discount"],
                                        item_id= data["item_id"],
                                        rate= data["rate"],
                                        quantity= data["quantity"],
                                        estimate_notes= estimate["notes"],
                                        estimate_terms= estimate["terms"]
                                        )
        return estimate_obj.__dict__

    def new_item(self, context, payload):
        ''' triggers when new item created'''
        item=payload["JSONString"]["item"]
        item_obj = ZohobooksItem(
                                 item_name= item["name"],
                                 description= item["description"],
                                 rate= item["rate"],
                                 tax_percentage= item["tax_percentage"]
                                )
        return item_obj.__dict__

    def new_sales_invoice(self, context, payload):
        ''' triggers when new invoice created'''
        print(payload.keys())
        invoice=payload['JSONString']["invoice"]
        data=invoice["line_items"][0]

        invoice_obj = ZohobooksSalesinvoice(
                                            customer_id= invoice["customer_id"],
                                            exchange_rate= invoice["exchange_rate"],
                                            item_id= data["line_item_id"],
                                            description= data["description"],
                                            rate= data["rate"],
                                            quantity= data["quantity"],
                                            discount= data["discount"],
                                            adjustment= invoice["adjustment_description"],
                                            start_date= invoice["date"],

                                          )

        return invoice_obj.__dict__

    def new_expense(self, context, payload):
        ''' triggers when new expence created'''
        expense=payload["JSONString"]["expense"]
        expense_obj = ZohobooksExpense(
                                       account_id= expense["account_id"],
                                       date= expense["date"],
                                       amount= expense["amount"]
                                      )
        return expense_obj.__dict__