"""4-class IEMOCAP dataset."""

import ast
import collections
import os

from glob import glob

import numpy as np
import soundfile as sf
import tensorflow as tf
import tensorflow_datasets.public_api as tfds

# Markdown description  that will appear on the catalog page.
_DESCRIPTION = """
The Interactive Emotional Dyadic Motion Capture (IEMOCAP) database is an acted, multimodal and multispeaker database, recently collected at SAIL lab at USC.

It contains approximately 12 hours of audiovisual data, including video, speech, motion capture of face, text transcriptions.

It consists of dyadic sessions where actors perform improvisations or scripted scenarios, specifically selected to elicit emotional expressions.

In previous research, IEMOCAP has often been evaluated as a four-class problem (Anger/Happiness/Sadness/Neutral). The resulting subset is called IEM4.

Use cases of IEM4:
* https://isca-speech.org/archive/Interspeech_2020/pdfs/3007.pdf
* https://doi.org/10.1016/j.bspc.2020.101894
* https://doi.org/10.1109/ICASSP.2016.7472790
"""

# BibTeX citation
_CITATION = """
@article{busso2008iemocap,
  title={IEMOCAP : Interactive emotional dyadic motion capture database},
  author={Busso, Carlos and Bulut, Murtaza and Lee, Chi-Chun and Kazemzadeh, Abe and Mower, Emily and Kim, Samuel and Chang, Jeannette N and Lee, Sungbok and Narayanan, Shrikanth S},
  journal={Language Resources and Evaluation},
  volume={42},
  number={4},
  pages={335--359},
  year={2008},
  publisher={Springer}
}
"""

_HOMEPAGE = 'https://sail.usc.edu/iemocap/index.html'

_LABEL_MAP = {
    'ang': 'anger',
    'hap': 'happiness',
    'exc': 'happiness',
    'sad': 'sadness',
    'neu': 'neutral',
}

_SAMPLE_RATE = 16000

def parse_name(name, from_i, to_i, mapping=None):
    """Source: https://audeering.github.io/audformat/emodb-example.html"""
    key = name[from_i:to_i]
    return mapping[key] if mapping else key

def _compute_split_boundaries(split_probs, n_items):
    """Computes boundary indices for each of the splits in split_probs.
    Args:
      split_probs: List of (split_name, prob), e.g. [('train', 0.6), ('dev', 0.2),
        ('test', 0.2)]
      n_items: Number of items we want to split.
    Returns:
      The item indices of boundaries between different splits. For the above
      example and n_items=100, these will be
      [('train', 0, 60), ('dev', 60, 80), ('test', 80, 100)].
    """
    if len(split_probs) > n_items:
        raise ValueError('Not enough items for the splits. There are {splits} '
                         'splits while there are only {items} items'.format(splits=len(split_probs), items=n_items))
    total_probs = sum(p for name, p in split_probs)
    if abs(1 - total_probs) > 1E-8:
        raise ValueError('Probs should sum up to 1. probs={}'.format(split_probs))
    split_boundaries = []
    sum_p = 0.0
    for name, p in split_probs:
        prev = sum_p
        sum_p += p
        split_boundaries.append((name, int(prev * n_items), int(sum_p * n_items)))

    # Guard against rounding errors.
    split_boundaries[-1] = (split_boundaries[-1][0], split_boundaries[-1][1],
                            n_items)

    return split_boundaries

def _get_inter_splits_by_group(items_and_groups, split_probs, split_number):
    """Split items to train/dev/test, so all items in group go into same split.
    Each group contains all the samples from the same speaker ID. The samples are
    splitted so that all each speaker belongs to exactly one split.
    Args:
      items_and_groups: Sequence of (item_id, group_id) pairs.
      split_probs: List of (split_name, prob), e.g. [('train', 0.6), ('dev', 0.2),
        ('test', 0.2)]
      split_number: Generated splits should change with split_number.
    Returns:
      Dictionary that looks like {split name -> set(ids)}.
    """
    groups = sorted(set(group_id for item_id, group_id in items_and_groups))
    rng = np.random.RandomState(split_number)
    rng.shuffle(groups)

    split_boundaries = _compute_split_boundaries(split_probs, len(groups))

    group_id_to_split = {}
    for split_name, i_start, i_end in split_boundaries:
        for i in range(i_start, i_end):
            group_id_to_split[groups[i]] = split_name

    split_to_ids = collections.defaultdict(set)
    for item_id, group_id in items_and_groups:
        split = group_id_to_split[group_id]
        split_to_ids[split].add(item_id)

    return split_to_ids

def parse_txt_file(txt):
    with open(txt, 'r') as f:
        lines = f.readlines()

    lines = np.array([line.strip().split('\t') for line in lines if line[0] == '['])

    _, fnames, labels, VAD = np.split(lines, lines.shape[1], axis=1)

    VAD = np.array(list(map(ast.literal_eval, VAD.flatten())))  # str to list

    vals, acts, _ = np.split(VAD, 3, axis=1)  # valence-arousal (+ dominance)

    return fnames.flatten(), labels.flatten(), vals.flatten(), acts.flatten()

def filter_label(label):
    if label == 'exc':
        return 'hap'
    elif label in ['xxx', 'fru', 'sur', 'oth', 'fea', 'dis']:
        return None
    else:
        return label

def clip_audio(wrd_path, fs=_SAMPLE_RATE):
    """
    Clip start and end silence according to forced alignment.
    """
    with open(wrd_path, 'r') as f:
        lines = f.readlines()
        lines = np.array([line.strip().split() for line in lines[1:-1]])
        times, words = lines[:, :2].astype(np.int32), lines[:, -1]
        sil_id = (words == '<s>') + (words == '</s>') + (words == '<sil>')
        times, words = times[~sil_id], words[~sil_id]
        start, end = int(fs * times[0, 0] / 100), int(fs * times[-1, -1] / 100)
    return start, end

class Iem4(tfds.core.GeneratorBasedBuilder):
    """DatasetBuilder for the 4-class IEMOCAP (IEM4) dataset."""

    VERSION = tfds.core.Version('1.0.0')
    RELEASE_NOTES = {
        '1.0.0': 'Initial release.',
    }

    MANUAL_DOWNLOAD_INSTRUCTIONS = """\
    manual_dir should contain the file IEMOCAP_full_release.zip.
    """

    def _info(self) -> tfds.core.DatasetInfo:
        """Returns the dataset metadata."""
        # Specifies the tfds.core.DatasetInfo object
        return tfds.core.DatasetInfo(
            builder=self,
            description=_DESCRIPTION,
            features=tfds.features.FeaturesDict({
                'audio': tfds.features.Audio(file_format='wav', sample_rate=_SAMPLE_RATE),  # file_format='np.ndarray', dtype=tf.float32, sample_rate=_SAMPLE_RATE),
                'label': tfds.features.ClassLabel(names=set(_LABEL_MAP.values())),
                'arousal': tf.float32,
                'valence': tf.float32,
                'speaker_id': tf.string
            }),
            # If there's a common (input, target) tuple from the
            # features, specify them here. They'll be used if
            # `as_supervised=True` in `builder.as_dataset`.
            supervised_keys=('audio', 'label'),  # Set to `None` to disable
            homepage=_HOMEPAGE,
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager: tfds.download.DownloadManager):
        """Returns SplitGenerators."""
        self.metadata = {}

        # Downloads the data and defines the splits
        zip_path = os.path.join(dl_manager.manual_dir, 'IEMOCAP_full_release.zip')

        if not tf.io.gfile.exists(zip_path):
            raise AssertionError(
                'IEMOCAP requires manual download of the data. Please download '
                'the audio data and place it into: {}'.format(zip_path))

        extract_path = dl_manager.extract(zip_path)

        txt_paths = sorted(glob(f'{extract_path}/IEMOCAP_full_release/Session*/dialog/EmoEvaluation/*.txt'))
        wav_paths = sorted(glob(f'{extract_path}/IEMOCAP_full_release/Session*/sentences/wav/*'))
        align_paths = sorted(glob(f'{extract_path}/IEMOCAP_full_release/Session*/sentences/ForcedAlignment/*'))

        items_and_groups = []

        for t_path, w_path, a_path in zip(txt_paths, wav_paths, align_paths):
            fnames, labels, vals, acts = parse_txt_file(t_path)

            for i, fname in enumerate(fnames):
                filtered_label = filter_label(labels[i])

                if filtered_label is not None:
                    path = os.path.join(w_path, (fname + '.wav'))
                    audio, fs = sf.read(path)
                    speaker_id = parse_name(fname, from_i=3, to_i=6)
                    items_and_groups.append((path, speaker_id))

                    self.metadata[path] = {
                        'label': filtered_label,
                        'act': acts[i],
                        'val': vals[i]
                    }

        split_probs = [('train', 0.6), ('validation', 0.2), ('test', 0.2)]  # Like SAVEE (https://github.com/tensorflow/datasets/blob/master/tensorflow_datasets/audio/savee.py)

        splits = _get_inter_splits_by_group(items_and_groups, split_probs, 0)

        # Returns the Dict[split names, Iterator[Key, Example]]
        return [
            tfds.core.SplitGenerator(
                name=tfds.Split.TRAIN,
                gen_kwargs={'file_names': splits['train']},
            ),
            tfds.core.SplitGenerator(
                name=tfds.Split.VALIDATION,
                gen_kwargs={'file_names': splits['validation']},
            ),
            tfds.core.SplitGenerator(
                name=tfds.Split.TEST,
                gen_kwargs={'file_names': splits['test']},
            ),
        ]

    def _generate_examples(self, file_names):
        for fname in file_names:
            wavname = os.path.basename(fname)
            label = _LABEL_MAP[self.metadata[fname]['label']]
            arousal, valence = self.metadata[fname]['act'], self.metadata[fname]['val']
            speaker_id = parse_name(wavname, from_i=3, to_i=6)
            example = {'audio': fname, 'label': label, 'arousal': arousal, 'valence': valence, 'speaker_id': speaker_id}
            yield fname, example


from django.db import models
from django.db.models import Max

from djangocities.cities.models import City
from djangocities.user.models import CustomUser as User

class Site(models.Model):
    class Meta:
        verbose_name_plural = "sites"
        unique_together = ("city", "address")

    city = models.ForeignKey(City, on_delete=models.CASCADE)
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    address = models.PositiveIntegerField()
    description = models.CharField(max_length=500)

    def __str__(self):
        return f"{self.city}/{self.address}"

    def save(self, *args, **kwargs):
        if self.pk is None:
            current_address = Site.objects.filter(city=self.city).aggregate(
                Max("address")
            )["address__max"]
            if current_address:
                self.address = current_address + 1
            else:
                self.address = 1

        super(Site, self).save(*args, **kwargs)


from gurobipy import *
import math
from matplotlib import pyplot as plt
from gcn_flc.graphutils import load_adj, vis_graph
import networkx as nx
import numpy as np
import json

def distance(a,b):
    '''
    Euclidean distance between any two point
    Args:
        a,b numpy array with same dim (both 1*n vector)
    Return:
        distance between a and b
    '''
    dx = a[0] - b[0]
    dy = a[1] - b[1]
    return math.sqrt(dx*dx + dy*dy)

def graph_dis(G,c_indx,f_indx,numFacilities):
    '''
    Graph distance between any two point
    Args:
        G: graph we generated
        c_indx: the index of client
        f_indx: the index of facility
        numFacilities: the number of all the facilities
    Return:
        distance between a and b with dijkstra algorithm
    '''
    distance=nx.dijkstra_path_length(G, source = f_indx, target = c_indx + numFacilities)
    return distance

def gurobi_solver(data):
    """
    Exact solver for facility location
    Args:
    data.keys():
    clients: [N,2] list, e.g. [[0, 1.5],[2.5, 1.2]]
    facilities: [M,2] list, e.g. [[0,0],[0,1],[0,1],
                                 [1,0],[1,1],[1,2],
                                 [2,0],[2,1],[2,2]]
    charge: [M,2] list, e.g. [3,2,3,1,3,3,4,3,2]
    alpha: const, cost per mile
    Return:
    x: [M] binary array
    y: [N] scalar array
    d: [N,M] distance array
    """
    # Problem data
    clients = data['clients']
    facilities = data['facilities']
    charge = data['charge']
    alpha = data['alpha']

    numFacilities = len(facilities)
    numClients = len(clients)

    G = load_adj(data['graph_dict'], numFacilities + numClients)

    m = Model()
    #m.setParam( 'TuneOutput', 0 )
    m.setParam( 'OutputFlag', False )
    # Add variables
    x = {}
    y = {}
    d = {} # Distance matrix (not a variable)

    for j in range(numFacilities):
        x[j] = m.addVar(vtype=GRB.BINARY, name="x%d" % j)

    for i in range(numClients):
        for j in range(numFacilities):
            y[(i,j)] = m.addVar(lb=0, vtype=GRB.CONTINUOUS, name="t%d,%d" % (i,j))
            d[(i,j)] = graph_dis(G,i,j,numFacilities)

    m.update()

    # Add constraints
    for i in range(numClients):
        for j in range(numFacilities):
            m.addConstr(y[(i,j)] <= x[j])

    for i in range(numClients):
        m.addConstr(quicksum(y[(i,j)] for j in range(numFacilities)) == 1)

    m.setObjective( quicksum(charge[j]*x[j] + quicksum(alpha*d[(i,j)]*y[(i,j)]
                    for i in range(numClients)) for j in range(numFacilities)) )

    m.optimize()
    # return a stardard result list
    x_result = []
    for j in range(numFacilities):
        x_result.append(x[j].X)
    y_result = []
    for i in range(numClients):
        for j in range(numFacilities):
            if y[(i,j)].X == 1:
                y_result.append(j)
                continue
    d_results = np.zeros((numClients, numFacilities))
    for i in range(numClients):
        for j in range(numFacilities):
            d_results[i,j] = d[(i,j)]
    return x_result, y_result, d_results

def visualize(data, x, y, vis=True):
    clients = np.array(data['clients'])
    facilities = np.array(data['facilities'])
    charge = np.array(data['charge'])
    alpha = data['alpha']

    fig=plt.figure(figsize=(16, 8))
    fig.add_subplot(1, 2, 1)
    plt.scatter(clients[:,0],clients[:,1])
    color = np.vstack([1/(max(charge)/charge)] + \
                      [[0]*len(charge)] + [[0]*len(charge)]).T
    plt.scatter(facilities[:,0],facilities[:,1],c=color)
    for i in range(len(y)):
        plt.plot([clients[i,0],facilities[y[i],0]],
                 [clients[i,1],facilities[y[i],1]],
                 c=[y[i]/len(facilities),
                    y[i]/len(facilities),
                    y[i]/len(facilities)])

    clients = data['clients']
    facilities = data['facilities']

    numFacilities = len(facilities)
    numClients = len(clients)

    G = load_adj(data['graph_dict'], numFacilities + numClients)
    nodes = np.concatenate((facilities,clients),axis = 0)

    fig.add_subplot(1, 2, 2)
    vis_graph(nodes,G,numFacilities,numClients)

    if vis:
        plt.show()
    return fig

def savedata(data, cfg, name=None,data_dir = 'dataset/synthetic'):
    """
    Data will be saved in json format
    save the configuration and data both
    in the json file.

    Args:
        data: data generated with optimal solutions
        cfg: configuration loaded from file
    Return:
        file_name: absolute path for the data file
    """
    s_data = {'cfg':cfg, 'data':list(data)}
    if not name:
        now = datetime.datetime.now()
        name = 'dataset-%02d-%02d-%02d-%02d-%02d.json' % ( now.month, now.day, now.hour, now.minute, now.second)

    file_name = os.path.join(data_dir,name)
    with open(file_name, 'w') as fp:
        fp.write(json.dumps(s_data, indent=3))
    return file_name

def loaddata(file_name):
    """
    Data will be loaded from json format

    Args:
        file_name: absolute path for the data file
    Return:
        cfg: configuration of the generated data
        data: data generated with ground truth
    """
    assert(os.path.isfile(file_name))
    s_data = json.load(open(file_name, 'r'))
    return s_data['cfg'], s_data['data']

if __name__ == '__main__':
    '''
        test code for developing
    '''
    data = {
        'clients': [[0, 1.5],[2.5, 1.2]],
        'facilities': [[0,0],[0,1],[0,1],
                     [1,0],[1,1],[1,2],
                     [2,0],[2,1],[2,2]],
        'charge': [3,2,3,1,3,3,4,3,2],
        'alpha': 10
    }
    x, y, d = gurobi_solver(data)
    visualize(data, x, y)


import phonenumbers
from phonenumbers import timezone,geocoder,carrier

def phone(number):
    try:
        phone_number = phonenumbers.parse(number)
        country_phone = geocoder.description_for_number(phone_number, "en")
        sim_card = carrier.name_for_number(phone_number, "en")
        time_zone = timezone.time_zones_for_number(phone_number)
        valid_number = phonenumbers.is_valid_number(phone_number)
        possible_number = phonenumbers.is_possible_number(phone_number)
        validation_number = "is it a valid phone number?"
        possibility_number = "is it a possible number?"
        print(f"Country of origin: {country_phone}")
        print(f"SIM CARD: {sim_card}")
        print(f"Time Zone: {time_zone}")
        if valid_number == True:
            print(f"{validation_number} yes, this number is a valid phone number")
        else:
            print(f"{validation_number} no, this number is not a valid phone number")
        if possible_number == True:
            print(f"{possibility_number} yes, this number is a possible phone number")
        else:
            print(f"{possibility_number} no, this number is not a possible phone number")
    except Exception:
        print("Please input a right phone number, don't forget to input + and try this program once again")

number = input("Input your phone number (using +): \n")
phone(number)


# importing datetime module
import datetime
import time
from dateutil import parser
#
# date in yyyy/mm/dd format
#d1 = "2021-10-01 19:26:03+00:00" # avoid all recs afrer this date
#d1 = datetime.datetime(2018, 5, 3)
d2 = datetime.datetime(2018, 6, 1)
#
dtstr="Thu, 10 Dec 2021 19:26:03 +0000" #current
dtstr2="Thu, 28 Oct 2022 19:26:03 +0000"  # oct 28th 2021

#d3 = datetime.datetime(2018, 6, 1)

#parser.parse("Aug 28 1999 12:00AM")
d3=parser.parse(dtstr)
#
d1=parser.parse(dtstr2)  # constant date
#
print("parsed date d1 = ", d1)
print("parsed date d3 = ", d3)
#ioutput 2021-12-09 19:26:03+00:00
# Comparing the dates will return
# either True or False
print("d3 is greater than d1 : ", d3 > d1)
#print("d1 is less than d2 : ", d1 < d2)
#print("d1 is not equal to d2 : ", d1 != d2)


from tkinter import *
from topmenu import *
from H import *
from insert_dishe import *
from update_dish import *
from show_dish import *

class App(H):
    def __init__(self,master):
        self.master = master
        self.frames = {}
        self.h = H()
        self.setUpUi()

    def showMe(self):
        print("Hey now")

    def setUpUi(self):
        self.frames["insert_dishes"] = Frame(self.master)
        self.frames["update_dishes"] = Frame(self.master)
        self.frames["show_dishes"] = Frame(self.master)

        TopMenu(self.master,self.h,self.frames)

        InsertDishes(self.frames["insert_dishes"],self.h)
        UpdateDish(self.frames["update_dishes"], self.h)
        ShowDishes(self.frames["show_dishes"], self.h)

        for frame in self.frames.values():
            frame.grid(row=0,column=0,sticky="news")

        self.h.raiseFrame(self.frames["show_dishes"])

root = Tk()
App(root)
root.geometry("500x300+200+200")
root.mainloop()


class Solution:
    def findPeakElement(self, nums: List[int]) -> int:
        if len(nums) == 1:
            return 0
        def findPeak(nums, low, high):
            if low > high:
                return -1
            mid= low+(high-low)//2
            if mid==0 and nums[mid] > nums[mid+1]:
                return mid
            elif mid == len(nums)-1 and nums[mid] > nums[mid-1]:
                return mid
            elif nums[mid-1] < nums[mid] > nums[mid+1]:
                return mid
            left = findPeak(nums, low, mid-1)
            if left == -1:
                return findPeak(nums, mid+1, high)
            return left

        return findPeak(nums, 0 ,len(nums)-1)


import argparse
import time
import os

import tensorflow as tf
from tqdm import tqdm
import numpy as np

from common import middle_load, middle_save
from data_loader import DataLoader
from model import Transformer
from extract_feature import BertVector

parser = argparse.ArgumentParser(description='poiter network')
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--pre_epochs', type=int, default=10)
parser.add_argument('--cuda_device', type=str, default='0')
parser.add_argument('--batch_size', type=int, default=32)
parser.add_argument('--seed', type=int, default=1234)
parser.add_argument('--logdir', type=str, default='logdir')
parser.add_argument('--model_path', type=str, default='weights')
parser.add_argument('--data', type=str, default=f'./data/corpus')
parser.add_argument('--dropout', type=float, default=0.9)
parser.add_argument('--d_model', type=int, default=512)
parser.add_argument('--d_ff', type=int, default=2048)
parser.add_argument('--n_head', type=int, default=8)
parser.add_argument('--d_k', type=int, default=64)
parser.add_argument('--d_v', type=int, default=64)
parser.add_argument('--enc_stack_layers', type=int, default=2)
parser.add_argument('--dec_stack_layers', type=int, default=6)
parser.add_argument('--learning_rate', type=float, default=0.00005)
parser.add_argument('--dist_model', type=int, default=768)
parser.add_argument('--dist_rate', type=float, default=0.3)
parser.add_argument('--not_use_pretrain', action='store_true')
args = parser.parse_args()

tf.compat.v1.set_random_seed(args.seed)

args.turn_size = 4
os.environ["CUDA_VISIBLE_DEVICES"] = args.cuda_device

data = middle_load(args.data)
args.vocab_size = len(data["word2idx"])
args.max_context_len = data["max_len"]

args.use_pretrain = not args.not_use_pretrain

training_data = DataLoader(
    data["train"]["src_texts"],
    data["train"]["src_turn"],
    data["train"]["tgt_indexs"],
    data["train"]["tgt_texts"],
    data["train"]["eos_indexs"],
    data["train"]["src_context"],
    batch_size=args.batch_size)

validation_data = DataLoader(
    data["valid"]["src_texts"],
    data["valid"]["src_turn"],
    data["valid"]["tgt_indexs"],
    data["valid"]["tgt_texts"],
    data["valid"]["eos_indexs"],
    data["valid"]["src_context"],
    batch_size=args.batch_size)

model = Transformer(args)

saver = tf.train.Saver()
sv = tf.train.Supervisor(logdir=f"{args.logdir}_{args.cuda_device}",
                         saver=saver,
                         global_step=model.global_step,
                         summary_op=None)

config = tf.ConfigProto()
config.gpu_options.allow_growth = True

best_score = -1

bert = BertVector(max_seq_len=args.max_context_len)

with sv.managed_session(config=config) as sess:
    os.makedirs(f"{args.model_path}_{args.cuda_device}", exist_ok=True)
    save_data = {
        "word2idx": data["word2idx"],
        "max_context_len": data["max_len"],
        "turn_size": args.turn_size,
    }
    middle_save(save_data, f"{args.model_path}_{args.cuda_device}/corpus")

    if args.use_pretrain:
        print("pretrain...")
    for epoch in range(1, args.pre_epochs+1):
        print(
            f"pre-train epoch {epoch}/{args.pre_epochs} loss: {model.pre_train(training_data, sess, bert)/training_data.stop_step:.4f}")

    for epoch in range(1, args.epochs + 1):
        if sv.should_stop():
            break

        total_loss, total_correct, total_gold, rouge_scores = model.train(
            training_data, sess, bert)
        print(f"train epoch {epoch}/{args.epochs} loss: {total_loss/training_data.stop_step:.4f} correct: {total_correct} gold count: {total_gold} presicion: {total_correct/total_gold:.4f} rouge score: {rouge_scores/training_data.sents_size:.4f}")

        total_loss, total_correct, total_gold, rouge_scores = model.valid(
            validation_data, sess, bert)
        print(f"valid epoch {epoch}/{args.epochs} loss: {total_loss/validation_data.stop_step:.4f} correct: {total_correct} gold count: {total_gold} presicion: {total_correct/total_gold:.4f} rouge score: {rouge_scores/validation_data.sents_size:.4f}")

        score = rouge_scores/validation_data.sents_size
        if score > best_score:
            print(f"new best presicion score {score:.4f} and save model")
            best_score = score
            saver.save(sess, f"{args.model_path}_{args.cuda_device}/model")


import logging
import traceback

class SQLHandler(logging.Handler):
    # A very basic logger that commits a LogRecord to the SQL Db
    # https://docs.pylonsproject.org/projects/pyramid-cookbook/en/latest/logging/sqlalchemy_logger.html

    def emit(self, record):
        from .models import session_scope, Log
        trace = None
        exc = record.__dict__['exc_info']
        if exc:
            trace = traceback.format_exc()
        with session_scope() as s:
            log = Log(
                logger=record.__dict__['name'],
                level=record.__dict__['levelname'],
                trace=trace,
                msg=str(record.__dict__['msg']),)
            s.add(log)

def init_log(level=logging.INFO, sql=False):
    # create logger
    # Don't use the root logger, or else all other modules such as werkzeug
    # will use this one too.
    logger = logging.getLogger('crackomatic')
    logger.setLevel(level)

    # create formatter
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    )

    # create console handler and set level to debug
    ch = logging.StreamHandler()
    ch.setLevel(level)

    # add formatter to ch
    ch.setFormatter(formatter)

    # add ch to logger
    logger.addHandler(ch)

    # Shut up sqlalchemy
    logging.getLogger('sqlalchemy').setLevel(logging.WARN)
    logging.getLogger('sqlalchemy').addHandler(ch)

    if sql:
        # create SQL handler and set level to debug
        sh = SQLHandler()
        sh.setLevel(logging.INFO)
        sh.setFormatter(formatter)
        logger.addHandler(sh)


import math

class Node:
    def __init__(self, value = None):
        self.value = value
        self.leftChild = None
        self.rightChild = None

class binaryTree:
    root = None
    count = 0

    def insert(self, val):
        self.count += 1
        if self.root == None:
            self.root = Node(val)
        else:
            current = self.root
            while(True):
                if val < current.value:
                    if current.leftChild == None:
                        current.leftChild = Node(val)
                        return
                    current = current.leftChild
                else:
                    if current.rightChild == None:
                        current.rightChild = Node(val)
                        return
                    current = current.rightChild

    def find(self, key):
        current = self.root
        while(current != None):
            if key < current.value:
                current = current.leftChild
            elif key > current.value:
                current = current.rightChild
            else:
                return 1
        return 0

    def traversePreOrder(self):
        self.__traversePreOrder(self.root)

    def __traversePreOrder(self, root):
        if root == None:
            return
        print(root.value)
        self.__traversePreOrder(root.leftChild)
        self.__traversePreOrder(root.rightChild)

    def traverseInOrder(self):
        self.__traverseInOrder(self.root)

    def __traverseInOrder(self, root):
        if root == None:
            return
        self.__traverseInOrder(root.leftChild)
        print(root.value)
        self.__traverseInOrder(root.rightChild)

    def traversePostOrder(self):
        self.__traversePostOrder(self.root)

    def __traversePostOrder(self, root):
        if root == None:
            return
        self.__traversePostOrder(root.leftChild)
        self.__traversePostOrder(root.rightChild)
        print(root.value)

    def height(self):
        return self.__height(self.root)

    def __height(self, root):
        if root == None:
            return -1

        if root.leftChild == None and root.rightChild == None:
            return 0

        return 1 + max(self.__height(root.leftChild), self.__height(root.rightChild))

    def minBST(self):
        if self.root == None:
            raise Exception("Tree is empty")

        current = self.root
        while(current.leftChild != None):
            current = current.leftChild
        print(current.value)

    def minBT(self):
        print(self.__minBT(self.root))

    def __minBT(self, root):
        if root.leftChild == None or root.rightChild == None:
            return root.value

        left = self.__minBT(root.leftChild)
        right = self.__minBT(root.rightChild)

        return min(min(left, right), root.value)

    def equal(self, second):
        if second.root == None:
            print(0)
            return
        print(self.__equal(self.root, second.root))

    def __equal(self, first, second):
        if first == None and second == None:
            return 1
        if first != None and second != None:
            if first.value == second.value:
                tmp = self.__equal(first.leftChild, second.leftChild) and self.__equal(first.rightChild, second.rightChild)
                if tmp == None:
                    return 0
                else:
                    return tmp
            return 0

    def validate(self):
        print(self.__validate(self.root, -math.inf, math.inf))

    def __validate(self, root, min, max):
        if root == None:
            return 1

        if root.value < min or root.value > max:
            return 0

        return self.__validate(root.leftChild, -math.inf, root.value) and self.__validate(root.rightChild, root.value, math.inf)

    def nodesAtK(self, distance):
        if self.root == None:
            raise Exception("Tree is empty")

        self.__nodesAtK(self.root, distance)

    def __nodesAtK(self, root, distance):
        if root != None:
            if distance == 0:
                print(root.value)
                return

            self.__nodesAtK(root.leftChild, distance - 1)
            self.__nodesAtK(root.rightChild, distance - 1)

    def traverseLevelOrder(self):
        if self.root == None:
            raise Exception("Tree is empty")

        for i in range(self.height()+1):
            self.nodesAtK(i)

    def size(self):
        print(self.count)

    tmp = 0
    def countLeaf(self):
        if self.root != None:
            self.__countLeaf(self.root)
        print(self.tmp)

    def __countLeaf(self, root):
        if root == None:
            self.tmp += 1
            return
        self.__countLeaf(root.leftChild)
        self.__countLeaf(root.rightChild)

    def maxBT(self):
        print(self.__maxBT(self.root))

    def __maxBT(self, root):
        if root == None:
            return

        if root.leftChild == None or root.rightChild == None:
            return root.value

        left = self.__maxBT(root.leftChild)
        right = self.__maxBT(root.rightChild)

        return max(max(left, right), root.value)

    def contains(self, key):
        self.__contains(self.root, key)

    def __contains(self, root, key):
        if root == None:
            return

        if root.value == key:
            print("Found")
            return

        self.__contains(root.leftChild, key)
        self.__contains(root.rightChild, key)

    def areSibling(self, firstVal, secondVal):
        self.__areSibling(self.root, firstVal, secondVal)

    def __areSibling(self, root, firstVal, secondVal):
        if root == None:
            return

        if root.leftChild == None or root.rightChild == None:
            return

        if root.leftChild.value == firstVal and root.rightChild.value == secondVal:
            print("Yes")
            return

        self.__areSibling(root.leftChild, firstVal, secondVal)
        self.__areSibling(root.rightChild, firstVal, secondVal)

bt = binaryTree()

#INSERT
bt.insert(10)
bt.insert(5)
bt.insert(20)
bt.insert(1)
bt.insert(6)
bt.insert(15)
bt.insert(25)
bt.insert(8)

#FIND
# print(bt.find(20))

#TRAVERSE - PRE ORDER
# bt.traversePreOrder()

#TRAVERSE - IN ORDER
# bt.traverseInOrder()

#TRAVERSE - POST ORDER
# bt.traversePostOrder()

#HEIGHT
# print(bt.height())

#MIN IN A BINATY SEARCH TREE
# bt.minBST()

#MIN IN A BINATY TREE
# bt.minBT()

#EQUALITY CHECKING
# bt1 = binaryTree()
# bt1.insert(10)
# bt1.insert(5)
# bt1.insert(20)
# bt1.insert(15)
# bt1.insert(25)
# bt1.insert(1)
# bt1.insert(6)
# bt1.insert(8)

# bt.equal(bt1)

#VALIDATE
# bt.validate()

#PRINT NODES AT K FROM ROOT
# bt.nodesAtK(0)

#LEVEL ORDER TRAVERSAL
# bt.traverseLevelOrder()

#SIZE
# bt.size()

#COUNT LEAF NODES
# bt.countLeaf()

#MAXIMUM IN A BINARY TREE
# bt.maxBT()

#CONTAINS - BINARY TREE
# bt.contains(25)

#SIBLINGS
bt.areSibling(5, 20)


# Create flood inundation data from HEC-RAS
#
# Purpose:
# Create flood inundation rasters and supporting InFRM data from the
# preprocessed HEC-RAS geospatial 1D data.  This creates data per
# feature-id for the National Water Model
#
# Created by: Andy Carter, PE
# Created: 2021-08-12
# Last revised - 2021.10.24
#
# Uses the 'ras2fim' conda environment
# ************************************************************
import argparse
import datetime as dt
import multiprocessing as mp
import os
import sys
import traceback
from functools import partial
from multiprocessing import Pool

import geopandas as gpd
import pandas as pd

import shared_functions as sf
import shared_variables as sv
import worker_fim_rasters

# Global Variables
RLOG = sv.R2F_LOG

# -------------------------------------------------
# Print iterations progress
def fn_print_progress_bar(
    iteration, total, prefix="", suffix="", decimals=0, length=100, fill="█", printEnd="\r"
):
    """
    from: https://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console
    Call in a loop to create terminal progress bar
    Keyword arguments:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + "-" * (length - filledLength)
    print(f"\r{prefix} |{bar}| {percent}% {suffix}", end=printEnd)
    # Print New Line on Complete
    if iteration == total:
        print()

# -------------------------------------------------
def fn_create_fim_rasters(
    str_desired_huc8,
    str_input_folder,
    str_output_folder,
    str_projection_path,
    str_terrain_path,
    flt_interval,
    b_terrain_check_only,
    is_verbose=False,
):
    # TODO: Oct 25, 2023, continue with adding the "is_verbose" system
    start_dt = dt.datetime.utcnow()

    # Hard coded constants for this routine

    INT_XS_BUFFER = 2  # Number of XS to add upstream and downstream
    # of the segmented

    # Constant - Toggle the Creation of RAS Map products
    IS_CREATE_MAPS = True

    # Constant - number of flood depth profiles to run on the first pass
    INT_NUMBER_OF_STEPS = 75

    # Constant - Starting flow for the first pass of the HEC-RAS simulation
    INT_STARTING_FLOW = 1

    # Constant - Maximum flow multiplier
    # up-scales the maximum flow from input
    FLT_MAX_MULTIPLY = 1.2

    # Constant - buffer of dem around floodplain envelope
    FLT_BUFFER = 15

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    RLOG.lprint("")
    RLOG.lprint("+=================================================================+")
    RLOG.lprint("|                NWM RASTER LIBRARY FROM HEC-RAS                  |")
    RLOG.lprint("+-----------------------------------------------------------------+")

    STR_HUC8 = str_desired_huc8
    RLOG.lprint("  ---(w) HUC-8 WATERSHED: " + STR_HUC8)

    STR_INPUT_FOLDER = str_input_folder
    RLOG.lprint("  ---(i) INPUT PATH: " + STR_INPUT_FOLDER)

    STR_ROOT_OUTPUT_DIRECTORY = str_output_folder
    RLOG.lprint("  ---(o) OUTPUT PATH: " + STR_ROOT_OUTPUT_DIRECTORY)

    STR_PATH_TO_PROJECTION = str_projection_path
    RLOG.lprint("  ---(p) PROJECTION PATH: " + STR_PATH_TO_PROJECTION)

    STR_PATH_TO_TERRAIN = str_terrain_path
    RLOG.lprint("  ---(t) TERRAIN PATH: " + STR_PATH_TO_TERRAIN)

    # Path to the standard plan file text
    current_script_dir = os.path.dirname(__file__)
    STR_PLAN_MIDDLE_PATH = os.path.join(current_script_dir, "\PlanStandardText01.txt")
    STR_PLAN_FOOTER_PATH = os.path.join(current_script_dir, "\PlanStandardText02.txt")
    STR_PROJECT_FOOTER_PATH = os.path.join(current_script_dir, "\ProjectStandardText01.txt")

    FLT_INTERVAL = flt_interval
    RLOG.lprint("  ---[z]   Optional: Output Elevation Step: " + str(FLT_INTERVAL))
    RLOG.lprint("  ---[c]   Optional: Terrain Check Only: " + str(b_terrain_check_only))

    RLOG.lprint("===================================================================")

    # "" is just a filler (for an old redundant parameter) simply to keep the order of item unchanged.
    # TODO: Oct 25, 2023 - complete the is_verbose system
    tpl_input = (
        STR_HUC8,
        STR_INPUT_FOLDER,
        STR_ROOT_OUTPUT_DIRECTORY,
        STR_PATH_TO_PROJECTION,
        STR_PATH_TO_TERRAIN,
        STR_PLAN_MIDDLE_PATH,
        STR_PROJECT_FOOTER_PATH,
        FLT_INTERVAL,
        "",
        INT_XS_BUFFER,
        IS_CREATE_MAPS,
        INT_NUMBER_OF_STEPS,
        INT_STARTING_FLOW,
        FLT_MAX_MULTIPLY,
        FLT_BUFFER,
        STR_PLAN_FOOTER_PATH,
        b_terrain_check_only,
        is_verbose,
    )

    list_huc8 = []
    list_huc8.append(STR_HUC8)

    str_stream_csv = STR_INPUT_FOLDER + "\\" + str(list_huc8[0]) + "_stream_qc.csv"

    str_stream_nwm_ln_shp = STR_INPUT_FOLDER + "\\" + str(list_huc8[0]) + "_nwm_streams_ln.shp"
    str_huc12_area_shp = STR_INPUT_FOLDER + "\\" + str(list_huc8[0]) + "_huc_12_ar.shp"

    # read the two dataframes
    df_streams = gpd.read_file(str_stream_csv)
    gdf_streams = gpd.read_file(str_stream_nwm_ln_shp)

    # convert the df_stream 'feature_id' to int64
    df_streams = df_streams.astype({"feature_id": "int64"})

    # left join on feature_id
    df_streams_merge = pd.merge(df_streams, gdf_streams, on="feature_id")

    # limit the fields
    df_streams_merge_2 = df_streams_merge[
        ["feature_id", "reach", "us_xs", "ds_xs", "peak_flow", "ras_path_x", "huc12"]
    ]

    # rename the ras_path_x column to ras_path
    df_streams_merge_2 = df_streams_merge_2.rename(columns={"ras_path_x": "ras_path"})

    # add the settings tuple
    df_streams_merge_2["settings"] = ""
    df_streams_merge_2["settings"] = df_streams_merge_2["settings"].astype(object)

    for index, row in df_streams_merge_2.iterrows():
        df_streams_merge_2.at[index, "settings"] = tpl_input

    RLOG.lprint("+-----------------------------------------------------------------+")
    RLOG.lprint("Start of processing in HEC-RAS")
    log_file_prefix = "fn_main_hecras"
    fn_main_hecras_partial = partial(
        worker_fim_rasters.fn_main_hecras, RLOG.LOG_DEFAULT_FOLDER, log_file_prefix
    )
    # create a pool of processors
    num_processors = mp.cpu_count() - 2
    with Pool(processes=num_processors) as executor:
        df_huc12 = gpd.read_file(str_huc12_area_shp)
        int_huc12_index = 0

        len_df_huc12 = len(df_huc12)
        str_prefix = r"Processing HUC12s (0 of " + str(len_df_huc12) + "):"
        fn_print_progress_bar(0, len_df_huc12, prefix=str_prefix, suffix="Complete", length=27)

        # Loop through each HUC-12
        for i in df_huc12.index:
            str_huc12 = str(df_huc12["HUC_12"][i])
            int_huc12_index += 1
            # print(str_huc12)
            str_prefix = r"Processing HUC12s (" + str(int_huc12_index) + " of " + str(len_df_huc12) + "):"
            fn_print_progress_bar(
                int_huc12_index, len_df_huc12, prefix=str_prefix, suffix="Complete", length=27
            )

            # Constant - Folder to write the HEC-RAS folders and files
            str_root_folder_to_create = STR_ROOT_OUTPUT_DIRECTORY + "\\HUC_" + str_huc12

            # Select all the 'feature_id' in a given huc12
            df_streams_huc12 = df_streams_merge_2.query("huc12 == @str_huc12")

            # Reset the query index
            df_streams_huc12 = df_streams_huc12.reset_index()

            # Create a folder for the HUC-12 area
            os.makedirs(str_root_folder_to_create, exist_ok=True)

            # amend the pandas dataframe
            df_streams_huc12_mod1 = df_streams_huc12[
                ["feature_id", "us_xs", "ds_xs", "peak_flow", "ras_path", "huc12", "settings"]
            ]

            # create a list of lists from the dataframe
            list_of_lists_df_streams = df_streams_huc12_mod1.values.tolist()

            try:
                if len(list_of_lists_df_streams) > 0:
                    executor.map(fn_main_hecras_partial, list_of_lists_df_streams)
            except Exception:
                # It has already been logged in fn_main_hecras, might get a dup error rec
                RLOG.critical(traceback.format_exc())
                executor.terminate()
                RLOG.critical("Pool terminated")
                sys.exit(1)

    # Now that multi-proc is done, lets merge all of the independent log file from each
    RLOG.merge_log_files(RLOG.LOG_FILE_PATH, log_file_prefix)

    tif_count = 0
    for root, dirs, files in os.walk(STR_ROOT_OUTPUT_DIRECTORY):
        for file in files:
            if file.endswith(".tif"):
                tif_count += 1

    RLOG.lprint("")
    RLOG.success("ALL AREAS COMPLETE")
    RLOG.success("Number of tif's generated: " + str(tif_count))

    dur_msg = sf.print_date_time_duration(start_dt, dt.datetime.utcnow())
    RLOG.lprint(dur_msg)

    RLOG.lprint("====================================================================")

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if __name__ == "__main__":
    # Sample

    # TODO: RESEARCH REQUIRED. Does the "-c" flag even work?

    # python create_fim_rasters.py -w 12030105
    #  -i c:\ras2fim_data\output_ras2fim\12030105_2276_231024\02_shapes_from_conflation
    #  -o c:\ras2fim_data\output_ras2fim\12030105_2276_231024\05_hecras_output
    #  -p c:\....\12030105_2276_231024\02_shapes_from_conflation\12030105_huc_12_ar.prj
    #  -t c:\ras2fim_data\output_ras2fim\12030105_2276_231024\04_hecras_terrain
    #  -s C:\Users\.....\Documents\NOAA-OWP\Projects\ras2fim\dev-logging\ras2fim\src
    #  -z 0.5  (leave off -c )

    parser = argparse.ArgumentParser(
        description="================ NWM RASTER LIBRARY FROM HEC-RAS =================="
    )

    parser.add_argument(
        "-w",
        dest="str_desired_huc8",
        help=r"REQUIRED: the desired huc-8 watershed: Example:  12090301",
        required=True,
        metavar="STRING",
        type=str,
    )

    parser.add_argument(
        "-i",
        dest="str_input_folder",
        help="REQUIRED: directory containing results of conflation (step 2):"
        r" Example: C:\ras2fim_12090301\02_shapes_from_conflation",
        required=True,
        metavar="DIR",
        type=str,
    )

    parser.add_argument(
        "-o",
        dest="str_output_folder",
        help=r"REQUIRED: path to write ras2fim output files: Example: C:\ras2fim_12090301\05_hecras_output",
        required=True,
        metavar="DIR",
        type=str,
    )

    parser.add_argument(
        "-p",
        dest="str_projection_path",
        help="REQUIRED: path the to the projection file: Example: "
        r"C:\ras2fim_12090301\02_shapes_from_conflation\12090301_huc_12_ar.prj",
        required=True,
        metavar="FILE",
        type=str,
    )

    parser.add_argument(
        "-t",
        dest="str_terrain_path",
        help=r"REQUIRED: path the to hdf5 terrain folder: Example: C:\ras2fim_12090301\04_hecras_terrain",
        required=True,
        metavar="FILE",
        type=str,
    )

    parser.add_argument(
        "-z",
        dest="flt_interval",
        help=r"  OPTIONAL: elevation interval of output grids: Example: 0.2 : Default: 0.5",
        required=False,
        default=0.5,
        metavar="FLOAT",
        type=float,
    )

    parser.add_argument(
        "-c",
        dest="b_terrain_check_only",
        help="OPTIONAL: check terrain only-skip HEC-RAS simulation and mapping: Default=False",
        required=False,
        default=False,
        action="store_true",
    )

    parser.add_argument(
        "-v",
        "--is_verbose",
        help="OPTIONAL: Adding this flag will give additional tracing output."
        "Default = False (no extra output)",
        required=False,
        default=False,
        action="store_true",
    )

    args = vars(parser.parse_args())
    # --------------------------------

    str_desired_huc8 = args["str_desired_huc8"]
    str_input_folder = args["str_input_folder"]
    str_output_folder = args["str_output_folder"]
    str_projection_path = args["str_projection_path"]
    str_terrain_path = args["str_terrain_path"]
    flt_interval = args["flt_interval"]
    b_terrain_check_only = args["b_terrain_check_only"]
    is_verbose = args["is_verbose"]

    log_file_folder = args["str_output_folder"]
    try:
        # Catch all exceptions through the script if it came
        # from command line.
        # Note.. this code block is only needed here if you are calling from command line.
        # Otherwise, the script calling one of the functions in here is assumed
        # to have setup the logger.

        # creates the log file name as the script name
        script_file_name = os.path.basename(__file__).split('.')[0]
        # Assumes RLOG has been added as a global var.
        RLOG.setup(os.path.join(log_file_folder, script_file_name + ".log"))

        # call main program
        fn_create_fim_rasters(
            str_desired_huc8,
            str_input_folder,
            str_output_folder,
            str_projection_path,
            str_terrain_path,
            flt_interval,
            b_terrain_check_only,
            is_verbose,
        )

    except Exception:
        RLOG.critical(traceback.format_exc())


"""
Boost the Monte Carlo noise simulation (specifically the staging time)
on general error with circuit layerwise slicing: new paradigm,
essentially the same as v1, but much simpler
"""

import time
import sys

sys.path.insert(0, "../")
import tensorcircuit as tc

tc.set_backend("jax")

n = 6  # 10
nlayer = 5  # 4

def precompute(c):
    s = c.state()
    return tc.Circuit(c._nqubits, inputs=s)

def f1(key, param, n, nlayer):
    if key is not None:
        tc.backend.set_random_state(key)
    c = tc.Circuit(n)
    for i in range(n):
        c.H(i)
    for j in range(nlayer):
        for i in range(n - 1):
            c.cnot(i, i + 1)
            c.apply_general_kraus(tc.channels.phasedampingchannel(0.15), i)
            c.apply_general_kraus(tc.channels.phasedampingchannel(0.15), i + 1)
        for i in range(n):
            c.rx(i, theta=param[j, i])
    return tc.backend.real(c.expectation((tc.gates.z(), [int(n / 2)])))

def f2(key, param, n, nlayer):
    c = tc.Circuit(n)
    for i in range(n):
        c.H(i)
    for j in range(nlayer):
        for i in range(n - 1):
            c.cnot(i, i + 1)
            c = precompute(c)
            c.apply_general_kraus(tc.channels.phasedampingchannel(0.15), i)
            c = precompute(c)
            c.apply_general_kraus(tc.channels.phasedampingchannel(0.15), i + 1)
        for i in range(n):
            c.rx(i, theta=param[j, i])
    return tc.backend.real(c.expectation((tc.gates.z(), [int(n / 2)])))

vagf1 = tc.backend.jit(tc.backend.value_and_grad(f1, argnums=1), static_argnums=(2, 3))
vagf2 = tc.backend.jit(tc.backend.value_and_grad(f2, argnums=1), static_argnums=(2, 3))

param = tc.backend.ones([nlayer, n])

def benchmark(f, tries=3):
    time0 = time.time()
    key = tc.backend.get_random_state(42)
    print(f(key, param, n, nlayer)[0])
    time1 = time.time()
    for _ in range(tries):
        print(f(key, param, n, nlayer)[0])
    time2 = time.time()
    print(
        "staging time: ",
        time1 - time0,
        "running time: ",
        (time2 - time1) / tries,
    )

print("without layerwise slicing jit")
benchmark(vagf1)
print("=============================")
print("with layerwise slicing jit")
benchmark(vagf2)

# mac16 intel cpu: (6*5, jax) 1015, 0.0035; 31.68, 0.00082


from django.urls import path
from . import views

app_name = "facultymgr"

urlpatterns = [
    path("upload_config", views.ConfigUpload.as_view(), name="create_api_eval"),
    path(
        "upload_config2", views.ConfigUploadCodeEval.as_view(), name="create_code_eval"
    ),
    path("create_eval", views.CreateEval.as_view(), name="create_eval"),
    path(
        "container_test_cases",
        views.ContainerTestCases.as_view(),
        name="container_test_cases",
    ),
    path("scale_test_cases", views.ScaleTestCases.as_view(), name="scale_test_cases"),
    path("view_submissions", views.SubmissionView.as_view(), name="view_submission"),
]


# LeetCode problem 654. Maximum Binary Tree (medium)
# Algorithm: Locate the index of the max value in the list. Use that location to determine: Max Value, Left Partition, Right Partition.  Recursively create tree using nodes from max value as root.val, root.left from left partion, and root.right from right partition.

# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution:
    def constructMaximumBinaryTree(self, nums: List[int]) -> TreeNode:
        if nums == [] or nums == None:
            return None
        i = nums.index(max(nums))
        maxVal = nums[i]
        left = nums[0:i]
        right = nums[i:len(nums)]
        root = TreeNode(maxVal)
        right.remove(maxVal)
        root.left = self.constructMaximumBinaryTree(left)
        root.right = self.constructMaximumBinaryTree(right)
        return root

LeetCode/654. Maximum Binary Tree (medium).py


import random
a=random.randint(1,100)
i=int(input())
while i!=a:
    if i<a:
        print(str(i)+' меньше, чем загаданное число')
    else:
        print(str(i)+' больше, чем загаданное число')
    i=int(input())
print('Угадали:)')


#安装相关依赖库 如果是windows系统，cmd命令框中输入pip安装，参考上述环境配置
#!pip install sklearn
#!pip install pandas
#---------------------------------------------------
#导入库
import pandas as pd
import json
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import cross_val_score

#----------------数据探索----------------
# #数据预处理
# #加载训练集
# train_df = pd.read_csv('./data/train.json', sep=',')
# #加载测试集
# test_df = pd.read_csv('./基于论文摘要的文本分类与查询性问答公开数据/test.csv', sep=',')

# #EDA数据探索性分析
# train_df.head()

# test_df.head()

#----------------特征工程----------------
#将Topic(Label)编码
with open('./data/train.json', mode='r', encoding='utf-8') as f:
    text = []
    label = []
    for line in f.readlines():
        json_info = json.loads(line)
        url = json_info['url']
        entity_length = 0
        if "entities" in json_info:
            entity_length = len(json_info['entities'])
        title_length = len(json_info['title'])
        content_length = len(json_info['content'])
        entity_str = ""
        baike_list = []
        for entity in json_info['entities']:
            entity_str += str(json_info['entities'][entity]['co-occurrence'])
            baike_list.extend([str(item['name'])+","+str(item['value']) for item in json_info['entities'][entity]["entity_baike_info"]])
            entity_str += "".join(baike_list)
        entity_str.replace("[","")
        entity_str.replace("]","")
        entity_str.replace(",","")
        entity_str.replace('\"',"")
        text.append(json_info['title'] + json_info['content'] +entity_str)
        label.append(int(json_info["label"]))
    f.close()
train_df = pd.DataFrame()
train_df["Text"] = pd.Series(text)
train_df["Label"] = pd.Series(label)
print(train_df.head())
with open('./data/test_B.json', mode='r', encoding='utf-8') as f:
    text = []
    urls = []
    for line in f.readlines():
        json_info = json.loads(line)
        url = json_info['url']
        urls.append(url)
        entity_length = 0
        if "entities" in json_info:
            entity_length = len(json_info['entities'])
        title_length = len(json_info['title'])
        content_length = len(json_info['content'])
        entity_str = ""
        baike_list = []
        for entity in json_info['entities']:
            entity_str += str(json_info['entities'][entity]['co-occurrence'])
            baike_list.extend([str(item['name'])+","+str(item['value']) for item in json_info['entities'][entity]["entity_baike_info"]])
            entity_str += "".join(baike_list)
        entity_str.replace("[","")
        entity_str.replace("]","")
        entity_str.replace(",","")
        entity_str.replace('\"',"")
        text.append(json_info['title'] + json_info['content'] + entity_str)
    f.close()
test_df = pd.DataFrame()
test_df["Text"] = pd.Series(text)
test_df["url"] = pd.Series(urls)
print(test_df.head())
# train_df['Topic(Label)'], lbl = pd.factorize(train_df['Topic(Label)'])

# #将论文的标题与摘要组合为 text 特征
# train_df['Title'] = train_df['Title'].apply(lambda x: x.strip())
# train_df['Abstract'] = train_df['Abstract'].fillna('').apply(lambda x: x.strip())
# train_df['text'] = train_df['Title'] + ' ' + train_df['Abstract']
# train_df['text'] = train_df['text'].str.lower()

# test_df['Title'] = test_df['Title'].apply(lambda x: x.strip())
# test_df['Abstract'] = test_df['Abstract'].fillna('').apply(lambda x: x.strip())
# test_df['text'] = test_df['Title'] + ' ' + test_df['Abstract']
# test_df['text'] = test_df['text'].str.lower()

#使用tfidf算法做文本特征提取
tfidf = TfidfVectorizer(max_features=2500)

#----------------模型训练----------------

train_tfidf = tfidf.fit_transform(train_df['Text'])
clf = SGDClassifier()
cross_val_score(clf, train_tfidf, train_df['Label'], cv=7)

test_tfidf = tfidf.transform(test_df['Text'])
clf = SGDClassifier()
clf.fit(train_tfidf, train_df['Label'])
test_df['Label'] = clf.predict(test_tfidf)

#----------------结果输出----------------
#test_df['Topic(Label)'] = test_df['Topic(Label)'].apply(lambda x: lbl[x])
cnt_0 = 0
cnt_1 = 0
s = ""
for i in range(test_df.shape[0]):
    item = test_df.iloc[i,:]
    dic = dict()
    dic["url"] = item['url']
    dic["label"] = int(item['Label'])
    if dic["label"] == 1:
        cnt_1 += 1
    else:
        cnt_0 += 1
    #                 if x_2[i][7] < 20:
    #                     if dic["label"] != 0:
    #                         dic["label"] = 0
    #                         cnt_0 += 1

    #                 elif x_2[i][7] > 150:
    #                     if dic["label"] != 1:
    #                         cnt_1 += 1
    #                         dic["label"] = 1
    s += json.dumps(dic) + "\n"
print(cnt_0,cnt_1)
with open(f"output/result_tf_idf.txt",mode='w') as f:
    f.write(s)
    f.close()


class UartUtil:

	UART_START_CHAR =       0x7E
	UART_ESCAPE_CHAR =      0x5C
	UART_ESCAPE_FLIP_MASK = 0x40

	@staticmethod
	def uartEscape(val):
		if isinstance(val, list):
			# Escape special chars:
			escapedMsg = []
			for c in val:
				if c == UartUtil.UART_ESCAPE_CHAR or c == UartUtil.UART_START_CHAR:
					escapedMsg.append(UartUtil.UART_ESCAPE_CHAR)
					c = UartUtil.uartEscape(c)
				escapedMsg.append(c)
			return escapedMsg
		else:
			return val ^ UartUtil.UART_ESCAPE_FLIP_MASK

	@staticmethod
	def uartUnescape(val):
		return val ^ UartUtil.UART_ESCAPE_FLIP_MASK

	# Copied implementation of nordic
	@staticmethod
	def crc16_ccitt(arr8):
		"""
		:param arr8:
		:return:
		"""
		crc = 0xFFFF
		for i in range(0, len(arr8)):
			crc = (crc >> 8 & 0xFF) | (crc << 8 & 0xFFFF)
			crc ^= arr8[i]
			crc ^= (crc & 0xFF) >> 4
			crc ^= (crc << 8 & 0xFFFF) << 4 & 0xFFFF
			crc ^= ((crc & 0xFF) << 4 & 0xFFFF) << 1 & 0xFFFF
		return crc


#!/usr/bin/env python3

"""Upload images to kartaview.

The images must have been sequenced before.

Upload images from a directory:

  kv_upload.py ~/Pictures/kartaview/*.jpg
  kv_upload.py -n -v ~/Pictures/kartaview/*.jpg

Upload images from a list of images (one complete filepath per line):

  kv_upload.py @picture_list.txt
  find ~/Pictures/kartaview/ -name "*.jpg" | kv_upload.py @/dev/stdin

"""

import collections
import concurrent.futures
import logging
import operator
from typing import List

from tqdm import tqdm

import kartaview_tools as kt
import kartaview_tools.api10 as api
from kartaview_tools import Geotags

SEQUENCE_THREADS = 2  # upload this many sequences in parallel
IMAGE_THREADS = 2  # upload this many images in parallel per sequence

class SequenceJob:
    """A job that uploads a sequence of images."""

    def __init__(self, args, sequence: List[Geotags]) -> None:
        """Initialize this."""
        self.args = args
        self.sequence = sequence
        self.uploaded = 0
        self.errors = 0

    def upload(self) -> "SequenceJob":  # noqa: C901
        """Upload a sequence of images."""
        self.uploaded = 0
        self.errors = 0

        sequence_id = self.sequence[0].get("sequence_id", 0)

        # create a new sequence on the server
        if sequence_id == 0:
            parameters = {}
            if "deviceName" in self.sequence[0]:
                parameters["deviceName"] = self.sequence[0]["deviceName"]
            sequence_id = api.create_sequence(self.args, parameters)
            logging.info(f"Created sequence {sequence_id}")

            # mark all images as belonging to this open sequence
            for gt in self.sequence:
                gt["sequence_id"] = sequence_id
                gt["sequence_status"] = "open"
                kt.write_sidecar_file(gt["filename"], gt)

        # upload images in sequence
        with concurrent.futures.ThreadPoolExecutor(
            max_workers=IMAGE_THREADS
        ) as executor:
            futures = []
            for gt in sorted(self.sequence, key=operator.itemgetter("sequence_index")):
                if gt.get("status_code", 0) == 200:
                    self.args.pbar.update()
                else:
                    futures.append(
                        executor.submit(api.upload_image, self.args, sequence_id, gt)
                    )
                    logging.debug(f"queued: {gt['filename']} to sequence {sequence_id}")
            logging.info(
                f"Uploading {len(futures)} images to sequence {sequence_id} ..."
            )
            for job in concurrent.futures.as_completed(futures):
                try:
                    gt = job.result()
                    kt.write_sidecar_file(gt["filename"], gt)
                    self.uploaded += 1
                    self.args.pbar.update()
                except kt.ImageUploadError as e:
                    logging.error(e)
                    self.errors += 1
                    # logging.exception(e)

        # cleanup
        if self.errors:
            logging.error(
                f"Uploaded {self.uploaded} images with {self.errors} errors to sequence {sequence_id}"
            )
        if not self.errors or self.args.force_close:
            api.close_sequence(self.args, sequence_id)
            # mark all images as belonging to this closed sequence
            for gt in self.sequence:
                gt["sequence_status"] = "closed"
                kt.write_sidecar_file(gt["filename"], gt)
            logging.info(f"Uploaded {self.uploaded} images to sequence {sequence_id}")
            logging.info(f"Sequence {sequence_id} closed")
        return self

def build_parser():
    """Build the commandline parser."""
    parser = kt.build_parser(__doc__)

    parser.add_argument(
        "images",
        metavar="FILENAME",
        type=str,
        nargs="+",
        help="the images to upload",
    )

    parser.add_argument(
        "-n",
        "--dry-run",
        action="store_true",
        help="dry run: do not upload any images",
    )

    parser.add_argument(
        "--force-close",
        action="store_true",
        help="close open sequence even with pending errors",
    )
    return parser

def main():
    """Run this."""
    args = build_parser().parse_args(namespace=kt.args)
    kt.init_logging(args.verbose)
    kt.read_config_file()

    queued = 0
    errors = 0
    sequences = collections.defaultdict(list)

    for filename_or_glob in args.images:
        for filename in kt.ffmpeg_glob(filename_or_glob):
            logging.debug("Found image: %s" % filename)
            gt = kt.read_sidecar_file(filename)

            if "tmp_sequence_id" not in gt:
                # this image has not been sequenced
                continue

            sequences[gt["tmp_sequence_id"]].append(gt)
            queued += 1

    with tqdm(
        total=queued,
        desc="Dry run" if args.dry_run else "Uploading",
        unit="image",
        smoothing=0,
        # disable=args.verbose == 0,
    ) as args.pbar:

        with concurrent.futures.ThreadPoolExecutor(
            max_workers=SEQUENCE_THREADS
        ) as executor:
            futures = [
                executor.submit(SequenceJob.upload, SequenceJob(args, sequence))
                for sequence in sequences.values()
            ]
            for future in concurrent.futures.as_completed(futures):
                try:
                    job = future.result()
                    errors += job.errors
                except kt.KartaviewError as e:
                    logging.exception(e)

    if errors:
        logging.error(f"There where {errors} errors reported.")
        logging.error("To resume uploading re-run the program with the same arguments.")
    else:
        logging.info("Done.")

if __name__ == "__main__":
    main()


from django.shortcuts import render, redirect
from homepage.models import Image, Follow
from django.contrib.auth.decorators import login_required
from .forms import NewProfileForm, NewImageForm
from homepage.models import Profile

# Create your views here.
@login_required(login_url='/accounts/login')
def edit_profile_page(request):
    current_user = request.user
    print('-'* 40)
    print(current_user.id)
    all_images = Image.objects.all()
    all_instagram_followers = Follow.objects.all()
    following = Follow.objects.filter(user_id=current_user.id)
    print('=>'*10, following.count())
    user_images = []
    for j in all_images:
        if j.uploaded_by==current_user:
            print(j.uploaded_by)
            user_images.append(j.image)
    print('+' * 30)
    print(user_images)
    form = NewProfileForm(request.POST, request.FILES)
    all_profile = Profile.objects.filter(user_id=current_user.id)
    print('--->' * 30)
    print(all_profile)
    user_name = current_user

    if Profile.objects.filter(user_id=current_user.id):
        profile_personal = Profile.objects.filter(user_id=current_user.id)
        print('--->' * 30)
        return render(request, 'profile/edit_user_profile_page.html', {'form':form, 'user_images':user_images, 'following':following.count(), 'user_name':user_name, 'profile_personal':profile_personal})

    return render(request, 'profile/edit_user_profile_page.html', {'form':form, 'user_images':user_images, 'following':following.count(), 'user_name':user_name})

@login_required(login_url='/accounts/login')
def profile_page(request):
    current_user = request.user
    print('-'* 40)
    print(current_user.id)
    all_images = Image.objects.all()
    all_instagram_followers = Follow.objects.all()
    following = Follow.objects.filter(user_id=current_user.id)
    print('=>'*10, following.count())
    user_images = []
    for j in all_images:
        if j.uploaded_by==current_user:
            print(j.uploaded_by)
            user_images.append(j.image)
    print('+' * 30)
    print(user_images)
    return render(request, 'profile/profile_page.html', {'user_images':user_images, 'following':following.count()})

def new_image(request):
    if request.method == 'POST':
        form = NewImageForm(request.POST, request.FILES)
        current_user = request.user
        image = form.save(commit=False)
        image.uploaded_by = current_user
        image.save()
        if form.is_valid():
            image = form.save(commit=False)
            image.uploaded_by = current_user
            image.save()
            image.save_image()

    return redirect(edit_profile_page)

@login_required(login_url='/accounts/login')
def new_profile(request):
    current_user = request.user
    if request.method == 'POST'  :
        first_name = request.POST.get('first_name')
        last_name = request.POST.get('last_name')
        phone = request.POST.get('phone')
        email = request.POST.get('email')
        bio = request.POST.get('bio')
        print('*' * 30)
        print(bio)
        print('*' * 30)
        print(first_name, last_name, phone, email)
        user_id = current_user.id
        if Profile.objects.filter(user_id=user_id):
            user = Profile.objects.filter(user_id=user_id)

            form = NewProfileForm(request.POST, request.FILES)
            if form.is_valid():
                form = form.save()

            print('*' * 30)
            # print(user)
            user.update(first_name=first_name, last_name=last_name,phone=phone, email=email, bio=bio)
        else:
            new_profile = Profile.objects.create(user_id=current_user.id, first_name=first_name, last_name=last_name,email=email,phone=phone, bio=bio)
    return redirect(edit_profile_page)


from discord.ext import commands
import discord
import requests as r

class Scam(commands.Cog):

  def __init__(self, client):
      self.client = client

  @commands.command(aliases=['s'])
  async def scam(self, ctx, *, coin):
    try:
      response = getLatestPrice(coin)
      await ctx.send(response)
    except:
        url = "https://media.tenor.com/images/1c77bac5a51111ade38d81de5250e563/tenor.gif"
        return url

def setup(client):
    client.add_cog(Scam(client))

def getLatestPrice(coin, currency='usdt'):
    try:
        url = f"https://x.wazirx.com/api/v2/tickers/{coin.lower()}{currency.lower()}"
        response = r.get(url).json()
        LastPrice = response['ticker']['last']
        ReturnText = f"{coin.upper()} - {LastPrice} {currency.upper()}"
        return ReturnText
    except:
        url = "https://media.tenor.com/images/1c77bac5a51111ade38d81de5250e563/tenor.gif"
        return url


def next(generator, respect_mod = False):
	divisor = 2147483647
	while True:
		next = (generator['last'] * generator['coeff']) % divisor
		generator['last'] = next
		if not respect_mod or next % generator['mod'] == 0:
			break
	return next

def check(number, picky = False):
	generators = [
		{
			'name': 'A',
			'last': 679,
			'coeff': 16807,
			'mod': 4,
		},
		{
			'name': 'B',
			'last': 771,
			'coeff': 48271,
			'mod': 8
		}
	]

	matches = 0
	for i in range(0, number):
		o = []
		for generator in generators:
			r = next(generator, picky)
			o.append(r & 0xFFFF)
		if all(x == o[0] for x in o):
			matches += 1
	return matches

if __name__ == "__main__":
	print("Part 1: {}".format(check(40000000)))
	print("Part 2: {}".format(check(5000000, True)))


import numpy as np
# import math
import matplotlib.pyplot as plt

def f(x_n, y_n):
    # y_this = -y_n + x_n + 2
    # Exercicio 10.2.1 - livro-py.pdf
    y_this = -0.5*y_n + x_n + 2
    # derivada da solução exata
    # y_this = 2 - 8 * np.exp(-x_n/2)
    return y_this

def euler(n_max, h, x, y):

    for n in range(0, n_max):
        y[n + 1] = y[n] + h * f(x[n], y[n])
        x[n + 1] = x[n] + h

    return x, y

def rk4(n_max, h, x, y):

    for n in range(0, n_max):
        k1 = h * f(x[n], y[n])
        k2 = h * f(x[n] + h / 2, y[n] + k1 / 2)
        k3 = h * f(x[n] + h / 2, y[n] + k2 / 2)
        k4 = h * f(x[n] + h, y[n] + k3)

        y[n + 1] = y[n] + (k1 + 2 * k2 + 2 * k3 + k4) / 6
        x[n + 1] = x[n] + h

    return x, y

def initializeParams(x_max_, h,  x0, y0):
    n_max = int(x_max_ / h)  # numero de passos
    x = [None] * (n_max + 1)
    x[0] = x0
    y = [None] * (n_max + 1)
    y[0] = y0
    return n_max, x, y

if __name__ == '__main__':
    X_max = 1.0       # max valor em x
    h = 0.5         # passo em x
    x0 = 0.0
    y0 = 8.0

    n_max, x, y = initializeParams(X_max, h, x0, y0)
    print("X max = {}\nPasso = {}\nPassos = {}".format(X_max, h, n_max))
    sol_exata = 2 + 8 * np.exp(-0.5)
    print("Usando solução exata: Para x=1 => y= {}".format(sol_exata))
    for i in range(0, n_max+1):
        y[i] = 2 * x[i] + 8 * np.exp(-x[i]/2)
        if(i < n_max):
            x[i+1] = x[i] + h

    print("Usando equação diferencial exata:\nx = {}\ny = {}".format(x, y))
    plt.text(x[n_max], y[n_max], str(y[n_max]))
    plt.plot(x, y, label="Exata = " + str(y[n_max]))

    x0 = 0.0
    n_max, x, y = initializeParams(X_max, h, x0, y0)
    x, y = euler(n_max, h, x, y)
    print("Usando Euler: \nx = {}\ny = {}".format(x, y))
    plt.plot(x, y, label="Euller = " + str(y[n_max]))

    n_max, x, y = initializeParams(X_max, h, x0, y0)
    x, y = rk4(n_max, h, x, y)
    print("Usando Runge-Kuta de 4a ordem: \nx = {}\ny = {}".format(x, y))
    # plt.plot(x, y, label="RK4   = " + str(y[n_max]))
    plt.legend()
    plt.show()


# Import modules
import os as os
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import numpy as np
import subprocess as sp
import pathlib as path
from sklearn.decomposition import PCA
import h5py
import time
from scipy.stats import pearsonr
import seaborn as sns
import cooler

# To generate .cool files from BEDPE count files using cooler cload pairs in command line:
# cooler cload pairs -c1 1 -p1 2 -c2 4 -p2 5 --field count=7:dtype=int --zero-based
# /Users/GBS/Master/Reference/hg19/chrom_hg19.sizes:1000000 /Users/GBS/Master/HiC-Data/testing_chrom_parallelization/output/temp_dir/bedpe/mcf10_1000000.bedpe matrix_mcf10.cool

class Directories:
    base_path = path.Path("/Users/GBS/Master/Figures")
    compartment_path = base_path / "comparment_plots"

    if not compartment_path.exists():
        compartment_path.mkdir(parents=True)

class Dirs:
    root_path = path.Path("/Users/GBS/Master/HiC-Data")
    base_path_figures = path.Path("/Users/GBS/Master/Figures")

    chrom_sizes_file = "/Users/GBS/Master/Reference/hg19/chrom_hg19.sizes"
    hg19_cytoband_file = "/Users/GBS/Master/Reference/hg19/cytoBand_hg19.txt"

    compartments_path = root_path / "compartments"
    bedpe_path_intra = root_path / "bedpe_files/bedpe_intra"
    bedpe_path_inter = root_path / "bedpe_files/bedpe_inter"
    cool_path_intra = root_path / "coolfiles/intra"
    cool_path_inter = root_path / "coolfiles/inter"

    heatmap_dir_intra = base_path_figures / "heatmaps/intra"
    distance_dir_intra = base_path_figures / "decay_distances"
    heatmap_dir_inter = base_path_figures / "heatmaps/inter"
    distance_dir_inter = base_path_figures / "decay_distances/inter"

def bedpe_to_cool(bedpe_file, chrom_sizes_file, bin_size, cool_file):
    """
    Convert a BEDPE file to a COOL file using the Cooler command-line tool.

    Parameters:
    bedpe_file (str): The path to the BEDPE file.
    chrom_sizes_file (str): The path to the chrom_sizes file.
    bin_size (int): The bin size in base pairs.
    cool_file (str): The path to the output COOL file.
    """
    command = [
        "cooler",
        "cload",
        "pairs",
        "-c1",
        "1",
        "-p1",
        "2",
        "-c2",
        "4",
        "-p2",
        "5",
        "--field",
        "count=7:dtype=int",
        "--zero-based",
        f"{str(chrom_sizes_file)}:{str(bin_size)}",
        str(bedpe_file),
        str(cool_file)
    ]

    sp.run(command, check=True)

def bedpe_to_cool_py(bedpe_file, chrom_file, resolution, output_file):

    # API is not working
    """
    Convert a BEDPE file to a COOL file using the Cooler Python API.

    Parameters:
    bedpe_file (str): The path to the BEDPE file.
    chrom_file (str): The path to the chromosome sizes file.
    resolution (int): The resolution in base pairs.
    output_file (str): The path to the output COOL file.
    """
    # Load the BEDPE file into a DataFrame
    pairs_df = pd.read_csv(bedpe_file, sep='\t', names=['chrom1', 'start1', 'end1', 'chrom2', 'start2', 'end2', 'count'])

    # Generate the bins
    chromsizes = cooler.util.read_chromsizes(chrom_file)
    bins = cooler.binnify(chromsizes, resolution)

    # Generate the pixels
    pixels = cooler.tools.digest(bins, pairs_df, cooler.util.get_binsize(resolution))

    # Create the COOL file
    cooler.create_cooler(cool_uri=f'{output_file}::{resolution}', bins=bins, pixels=pixels)

# bedpe_to_cool_py("/Users/GBS/Master/HiC-Data/bedpe_files/bedpe_intra/mcf7_1000000.bedpe", "/Users/GBS/Master/Reference/hg19/chrom_hg19.sizes", 1000000, "/Users/GBS/Master/HiC-Data/coolfiles/intra/mcf7_1000000.cool")

def plot_contact_decay(cool_path, log_bins=True, bin_num=100):
    """
    Generate contact decay plot for Hi-C data.

    Parameters:
    cool_path: string
        Path to the .cool file.
    log_bins: Boolean
        - If True, use logarithmic bins.
        - If False, use linear bins.
    bin_num: The number of bins to use

    Returns:
    Matplotlib figure
    """

    # Load the Cooler file
    c = cooler.Cooler(cool_path)

    # Get the genomic positions of the bins
    bin_positions = c.bins()[:]["start"].values

    # Get the pairs DataFrame
    pairs = c.pixels()[:]

    # Calculate the genomic distances for each interaction
    pairs["genomic_distance"] = np.abs(bin_positions[pairs["bin2_id"]] - bin_positions[pairs["bin1_id"]])

    # If log_bins is True, calculate log bin edges
    if log_bins:
        bin_edges = np.logspace(np.log10(pairs["genomic_distance"].min()), np.log10(pairs["genomic_distance"].max()), num=bin_num)
        pairs['bin'] = pd.cut(pairs["genomic_distance"], bins=bin_edges)
        pairs.dropna(subset=["bin"], inplace=True)  # Drop any pairs not falling into a bin
        pairs["bin_center"] = pairs["bin"].apply(lambda x: (x.right + x.left) / 2)

    # Otherwise, use linear bins
    else:
        pairs['bin_center'] = pd.cut(pairs['genomic_distance'], bins=bin_num, labels=False)

    # Compute average interaction frequency for each genomic distance bin
    average_interactions = pairs.groupby("bin_center")["count"].mean()

    # Plot
    plt.figure(figsize=(10, 7))
    plt.loglog(average_interactions.index.values, average_interactions.values)
    plt.title("Contact Decay")
    plt.xlabel("Genomic Distance (bp)")
    plt.ylabel("Average Interaction Frequency")
    plt.show()

# plot_contact_decay("/Users/GBS/matrix_mcf10.cool", log_bins=True, bin_num=100)

def balance_cooler_file(cool_file):
    """Load and balance a cooler file."""
    c = cooler.Cooler(str(cool_file))
    if "weight" in c.bins().columns:
        print(f"The cooler file {cool_file} is already balanced.")
        return c
    else:
        weights, _ = cooler.balance.balance_cooler(c)
        bins = c.bins()[:]
        bins['weight'] = weights
        pixels = c.pixels()[:]
        pixels = pixels[['bin1_id', 'bin2_id', 'count']]
        pixels = pixels.join(bins['weight'], on='bin1_id', how='left').rename(columns={'weight': 'weight1'})
        pixels = pixels.join(bins['weight'], on='bin2_id', how='left').rename(columns={'weight': 'weight2'})
        pixels['weight'] = pixels['weight1'] * pixels['weight2']
        return c, bins, pixels

def validate_and_save_cooler(bins, pixels, balanced_file):
    """Save a balanced cooler file and validate it."""
    cooler.create_cooler(cool_uri=str(balanced_file), bins=bins, pixels=pixels)
    time.sleep(5)
    if os.path.exists(balanced_file):
        if h5py.is_hdf5(str(balanced_file)):
            print("Balanced cooler file was successfully created and is a valid HDF5 file.")
        else:
            print("File exists but is not a valid HDF5 file.")
    else:
        print("File does not exist.")
    print(f"The cooler file is now balanced and saved to {balanced_file}.")

def create_cooler_file():
    bedpe_file = Dirs.bedpe_path_intra / "mcf7_1000000.bedpe"
    chrom_file = Dirs.chrom_sizes_file
    resolution = 1000000
    output_file = Dirs.cool_path_intra / "mcf7_1000000.cool"

    bedpe_to_cool_py(bedpe_file, chrom_file, resolution, output_file)

# create_cooler_file()

def balance_and_save_cooler(cool_file, balanced_file):
    """Balance and save a cooler file."""
    c, bins, pixels = balance_cooler_file(cool_file)
    validate_and_save_cooler(bins, pixels, balanced_file)

# balance_and_save_cooler("/Users/GBS/Master/HiC-Data/coolfiles/inter/mcf7_1000000.cool", "/Users/GBS/Master/HiC-Data/coolfiles/intra/mcf7_1000000_balanced.cool")

def process_chromosome(chrom, cooler_obj, bins_df):
    """
    Process a single chromosome: compute its correlation matrix,
    perform PCA, and assign compartments
    """

    # Fetch the contact matrix for the chromosome
    chrom_matrix = cooler_obj.matrix(balance=True, as_pixels=False, join=True).fetch(chrom)

    # Skip chromosome if it only contains one bin
    if chrom_matrix.shape[0] <= 1:
        print(f"Skipping chromosome {chrom} because it only contains one bin.")
        return None

    # Replace zero values with a small number (e.g., 1e-10)
    small_value = 1e-10
    chrom_matrix = np.where(chrom_matrix == 0, small_value, chrom_matrix)

    # Prepare the observed/expected matrix
    distance_matrix = np.abs(np.arange(chrom_matrix.shape[0])[:, None] - np.arange(chrom_matrix.shape[0]))
    expected = np.bincount(distance_matrix.flatten(), weights=chrom_matrix.flatten()) / np.bincount(distance_matrix.flatten())
    oe_matrix = chrom_matrix / expected[distance_matrix]

    # Compute the correlation matrix and replace NaN/infinite values (no NaNs/infs in oe_matrix)
    correlation_matrix = np.corrcoef(oe_matrix)
    correlation_matrix = np.nan_to_num(correlation_matrix)

    # Compute the first principal component
    pca = PCA(n_components=1)
    principal_components = pca.fit_transform(correlation_matrix)

    # Assign compartments based on the sign of PC1
    compartments = np.sign(principal_components)

    # Create and return a DataFrame for this chromosome
    chrom_bins = bins_df[bins_df['chrom'] == chrom]
    chrom_df = pd.DataFrame({
        "chrom": chrom,
        "start": chrom_bins["start"].values,
        "end": chrom_bins["end"].values,
        "compartment": ['A' if x > 0 else 'B' for x in compartments.flatten()],
        "eigenvector": principal_components.flatten()  # save the eigenvectors
    })

    return chrom_df

def calculate_distance_matrix(bin_starts):
    """
    Calculate a matrix where each cell contains the genomic distance between two bins.
    """
    bin_starts = bin_starts.reshape(-1, 1)  # convert to column vector
    return np.abs(bin_starts - bin_starts.T)

def call_compartments(balanced_file, output_file):
    """
    Call chromatin compartments using Hi-C data in a cooler file
    """

    # Load the balanced cooler file
    c_balanced = cooler.Cooler(str(balanced_file))

    # Fetch all the bins once
    all_bins = c_balanced.bins()[:]

    # Create a DataFrame to hold the results
    result = pd.DataFrame(columns=["chrom", "start", "end", "compartment", "eigenvector"])

    # Loop over each chromosome
    for chrom in c_balanced.chromnames:
        if chrom in ['chrM', 'chrX', 'chrY']:
            print(f"Skipping {chrom}.")
            continue
        chrom_bins = all_bins[all_bins['chrom'] == chrom]
        chrom_df = process_chromosome(chrom, c_balanced, chrom_bins)
        if chrom_df is not None:
            result = result.append(chrom_df)

    # Save the results to a BED file
    result.to_csv(output_file, sep='\t', header=False, index=False)

def inspect_cooler(file_path):
    c = cooler.Cooler(file_path)
    df = c.matrix(balance=False, as_pixels=True)[:]

    # print basic statistics
    print(df.describe())

    # print the variance
    print("\nVariance:\n", df.var())

    # print number of zero values
    print("\nNumber of zero values:\n", (df==0).sum())

    # print the number of NaN values
    print("\nNumber of NaN values:\n", df.isna().sum())

    # print the number of infinite values
    print("\nNumber of infinite values:\n", np.isinf(df).sum())

    # print the number of negative values
    print("\nNumber of negative values:\n", (df<0).sum())

    # print the number of positive values
    print("\nNumber of positive values:\n", (df>0).sum())

# inspect_cooler("/Users/GBS/Master/HiC-Data/coolfiles/intra/mcf10_1000000_balanced.cool")

def compartment_calling():
    balanced_cool_file = Dirs.cool_path_intra / "mcf10_1000000_balanced.cool"
    output_file = Dirs.compartments_path / "mcf10_1mb_pca1_compartments.bed"

    call_compartments(balanced_cool_file, output_file)

# compartment_calling()

def compute_pearson_and_plot(bed_file1, bed_file2, cell_line1, cell_line2):
    # Load BED files into pandas DataFrames
    df1 = pd.read_csv(bed_file1, sep='\t', header=None)
    df2 = pd.read_csv(bed_file2, sep='\t', header=None)

    # Extract the eigenvector values and compartment calls
    eigenvector1 = df1.iloc[:, 4]
    eigenvector2 = df2.iloc[:, 4]
    compartments1 = df1.iloc[:, 3]
    compartments2 = df2.iloc[:, 3]

    # Compute the Pearson correlation coefficient
    correlation = pearsonr(eigenvector1, eigenvector2)[0]

    # Create a new column in the DataFrame indicating whether the compartment call changed
    compartment_change = compartments1 != compartments2

    # Create a scatter plot with points colored based on the compartment call
    plt.figure(figsize=(8, 6))
    plt.scatter(eigenvector1[compartment_change], eigenvector2[compartment_change], color='red', alpha=0.5, label='Compartment change')
    plt.scatter(eigenvector1[~compartment_change], eigenvector2[~compartment_change], color='blue', alpha=0.5, label='No compartment change')
    plt.xlabel(f'Eigenvector - {cell_line1}')
    plt.ylabel(f'Eigenvector - {cell_line2}')
    plt.title(f'Correlation between {cell_line1} and {cell_line2}: r = {correlation:.2f}')
    plt.legend()
    plt.show()

    # Return the correlation
    return correlation

def compare_compartments():
    # Load the BED files
    bed_file1 = Dirs.compartments_path / "mcf7_1mb_compartments.bed"
    bed_file2 = Dirs.compartments_path / "mcf10_1mb_compartments.bed"

    # Compute the Pearson correlation coefficient
    correlation = compute_pearson_and_plot(bed_file1, bed_file2, 'MCF7', 'MCF10')

# compare_compartments()

def merge_bed_files(bed_file1, bed_file2):
    # Load BED files into pandas DataFrames
    df1 = pd.read_csv(bed_file1, sep='\t', header=None, names=['chrom', 'start', 'end', 'compartment', 'eigenvector'])
    df2 = pd.read_csv(bed_file2, sep='\t', header=None, names=['chrom', 'start', 'end', 'compartment', 'eigenvector'])

    # Merge the two dataframes on chrom, start and end
    merged_df = pd.merge(df1, df2, on=['chrom', 'start', 'end'], suffixes=('_1', '_2'))

    return merged_df

def plot_compartments_on_heatmap_sns(cool_path, compartments_bed_path, chrom=None, saveas=None):
    clr = cooler.Cooler(str(cool_path))
    if chrom == "all":
        matrix = clr.matrix()[:]
    else:
        matrix = clr.matrix(balance=True).fetch(chrom)

    diag_sums = [np.nansum(np.diagonal(matrix, offset=i)) for i in range(matrix.shape[0])]
    diag_counts = [np.sum(~np.isnan(np.diagonal(matrix, offset=i))) for i in range(matrix.shape[0])]
    exp_const = 1e-10
    expected = np.array(diag_sums) - np.array(diag_counts) + exp_const
    print(matrix)
    print(expected)
    observed_over_expected = matrix / expected[:, None]
    print(observed_over_expected)

    corr = np.corrcoef(observed_over_expected, rowvar=False)
    compartments = pd.read_csv(compartments_bed_path, sep="\t", header=None, names=["chrom", "start", "end", "compartment", "score"])

    # Convert the genomic coordinates to bin coordinates
    binsize = clr.binsize
    if chrom != "all":
        compartments = compartments[compartments["chrom"] == chrom]
        compartments["start"] = compartments["start"] // binsize
        compartments["end"] = compartments["end"] // binsize

    # Generate the plot
    # fig, (ax1, ax2) = plt.subplots(nrows=2, figsize=(10, 10), gridspec_kw={'height_ratios': [1, 4]}, sharex="all")
    fig, ax2 = plt.subplots(figsize=(10, 10))

    # Plot PCa1
    # ax1.plot(compartments["start"], compartments["score"], color='k')

    # Plot the contact map
    vmax = np.percentile(corr[~np.isnan(corr)], 99)
    vmin = np.percentile(corr[~np.isnan(corr)], 1)

    # Make colors not blend:
    colormap = ["blue", "blue", "black", "red", "red"]
    gradient = [0.0, 0.3, 0.5, 0.7, 1.0]
    cmap = mcolors.LinearSegmentedColormap.from_list("cmap", list(zip(gradient, colormap)))
    sns.heatmap(corr, cmap=cmap, ax=ax2)

    # Create ticks at every 25 Mb
    tick_frequency = 25
    ticks = np.arange(0, matrix.shape[0], tick_frequency * 10 ** 6 // binsize)

    # Create tick labels with 'Mb' added
    tick_labels = [f"{tick * binsize // 10 ** 6} Mb" for tick in ticks]

    ax2.set_xticks(ticks[1:])
    ax2.set_xticklabels(tick_labels[1:])
    ax2.set_yticks(ticks[1:])
    ax2.set_yticklabels(tick_labels[1:])

    ax2.set_title(f"Contact map for {chrom}")
    # ax1.set_title(f"Compartments for {chrom}")
    plt.tight_layout()

    if saveas:
        plt.savefig(saveas, dpi=1000, format="png")

    plt.show()

# plot_compartments_on_heatmap_sns(Dirs.cool_path_intra / "mcf7_1000000_balanced.cool", Dirs.compartments_path / "mcf10_250kb_compartments.bed", chrom="chr2")  # :0-95000000")  # , saveas=Dirs.heatmap_dir_intra / "mcf10_250kb_chr2_compartments_plot.png")

def plot_heatmap2(cool_file, chrom=None, saveas=None, log=False):
    # Load the Cooler file
    c = cooler.Cooler(str(cool_file))

    # Define the region to plot
    if chrom == "all":
        matrix = c.matrix(balance=False)[:]
    else:
        matrix = c.matrix(balance=False).fetch(chrom)

    # Copy matrix for processing
    matrix_copy = matrix.copy()

    if log:
        matrix_copy = np.log1p(matrix_copy)
        matrix_copy[matrix_copy == -np.inf] = 0

    # Calculate the limits for the color scale
    vmin, vmax = np.percentile(matrix_copy, [0, 100])

    # Define colormap
    cmap = mpl.cm.get_cmap("YlOrRd")
    cmap.set_bad("white")

    # Plot the heatmap
    plt.matshow(matrix_copy, cmap=cmap, norm=mcolors.PowerNorm(gamma=0.2, vmin=vmin, vmax=vmax))
    plt.title(f"Heatmap of {chrom}")

    # Get current axes
    ax = plt.gca()

    # Create ticks at every 25 Mb
    tick_frequency = 25
    ticks = np.arange(0, matrix.shape[0], tick_frequency * 10 ** 6 // c.binsize)

    # Create tick labels with 'Mb' added
    tick_labels = [f"{tick * c.binsize // 10 ** 6} Mb" for tick in ticks]

    ax.set_xticks(ticks[1:])
    ax.set_xticklabels(tick_labels[1:])
    ax.set_yticks(ticks[1:])
    ax.set_yticklabels(tick_labels[1:])
    ax.xaxis.tick_bottom()
    plt.xticks(rotation='vertical')
    ax.tick_params(axis='x', which='major', pad=10)
    ax.tick_params(axis='x', labelsize=7)
    ax.tick_params(axis='y', labelsize=7)

    ax.set_title(f"Contact map for chromosome 2")
    plt.tight_layout()

    if saveas:
        plt.savefig(saveas, dpi=1000, format="png", bbox_inches='tight')

    plt.show()

# plot_heatmap2(Dirs.cool_path_intra / "imr90_250000_balanced.cool", chrom="chr2", saveas=Dirs.heatmap_dir_intra / "imr90_example250kb_chr2.png")

if __name__ == "__main__":
    print("RUN")


import seaborn as sb
import numpy as np
import osmnx as ox
from geopy.distance import distance
from matplotlib import pyplot as plt

def evaluate_coverage_distance(df):
    sb.set_style("dark")
    f, axes = plt.subplots(2, 2, figsize=(12,8))
    axes[0][0].title.set_text(f"Coverage of [2011-2015]: {len(df)} / 5000 = {len(df)/5000*100:.2f}%")
    axes[0][1].title.set_text(f"Coverage of [2016-2020]: {len(df)} / 5000 = {len(df)/5000*100:.2f}%")
    sb.countplot(x="year_pre", data=df, ax=axes[0][0], palette=['#432371'])
    sb.countplot(x="year_post", data=df, ax=axes[0][1], palette=["#FAAE7B"])
    axes[0][0].set_xlabel('')
    axes[0][1].set_xlabel('')

    d1, i1 = zip(*get_closest_distances(df, 'pre'))
    d2, i2 = zip(*get_closest_distances(df, 'post'))
    sb.lineplot(x=range(len(d1)), y=d1, ax=axes[1][0])
    sb.lineplot(x=range(len(d2)), y=d2, ax=axes[1][1])
    axes[1][0].title.set_text(f"Top 50 closest distance of [2011-2015] panoramas")
    axes[1][1].title.set_text(f"Top 50 closest distance of [2016-2020] panoramas")
    return f

def get_closest_distances(df, suffix='pre', n=50):
    lat = df[f'lat_{suffix}'].values
    lon = df[f'lon_{suffix}'].values
    D = np.sqrt(np.square(lat[:,np.newaxis] - lat) + np.square(lon[:,np.newaxis] - lon))
    D = np.tril(D) + np.triu(np.ones_like(D))
    d = []
    for i in range(n):
        x, y = np.unravel_index(D.argmin(), D.shape)
        _d = distance((lat[x], lon[x]), (lat[y], lon[y])).m
        d.append((_d, x))
        D[x,:] = D[:,x] = 1
    return sorted(d)

def evaluate_spatial_distribution(df, city):
    sb.set_style("white")
    G = ox.graph_from_place(city, network_type='drive')
    try:
        G = ox.simplify_graph(G)
    except:
        G = G

    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24,12))
    ox.plot.plot_graph(G,
                       ax=ax1,
                       bgcolor='white',
                       node_color='#696969',
                       edge_color="#A9A9A9",
                       edge_linewidth=0.8,
                       node_size=0,
                       save=False,
                       show=False)
    ax1.scatter(df.lon_anchor, df.lat_anchor, s=3, c='red', alpha=0.5)
    ax1.scatter(df.lon_pre, df.lat_pre, s=3, c='blue', alpha=0.5)

    ox.plot.plot_graph(G,
                       ax=ax2,
                       bgcolor='white',
                       node_color='#696969',
                       edge_color="#A9A9A9",
                       edge_linewidth=0.8,
                       node_size=0,
                       save=False,
                       show=False)
    ax2.scatter(df.lon_anchor, df.lat_anchor, s=3, c='red', alpha=0.5)
    ax2.scatter(df.lon_post, df.lat_post, s=3, c='blue', alpha=0.5)
    plt.show()