from typing import Callable, List, Union

import numpy as np
from numpy import argsort, ceil, exp, mod, zeros
from numpy.random import geometric, rand, randint, randn

from ...search_space import SearchSpace
from ...solution import Solution
from ...utils import dynamic_penalty, handle_box_constraint

__author__ = "Hao Wang"

# TODO: improve efficiency, e.g. compile it with cython
class MIES:
    """Mixed-integer Evolution Strategy"""

    def __init__(
        self,
        search_space: SearchSpace,
        obj_func: Callable,
        eq_func: Callable = None,
        ineq_func: Callable = None,
        x0: Union[List, Solution] = None,
        ftarget: float = None,
        max_eval: float = np.inf,
        minimize: bool = True,
        elitism: bool = False,
        mu_: int = 4,
        lambda_: int = 10,
        sigma0: float = None,
        eta0: float = None,
        P0: float = None,
        verbose: bool = False,
        eval_type: str = "list",
    ):
        # TODO: constructor is too long...
        self.mu_ = mu_
        self.lambda_ = lambda_
        self.eval_count = 0
        self.iter_count = 0
        self.minimize = minimize
        self.obj_func = obj_func
        self.eq_func = eq_func
        self.ineq_func = ineq_func
        self.stop_dict = {}
        self.verbose = verbose
        self.max_eval = max_eval
        self.ftarget = ftarget
        self.elitism = elitism
        self._penalty_func = dynamic_penalty
        self._eval_type = eval_type

        self._space = search_space
        self.var_names = self._space.var_name
        self.param_type = self._space.var_type

        if self._eval_type == "list":
            self._to_pheno = lambda x: x
        elif self._eval_type == "dict":
            self._to_pheno = lambda x: x.to_dict(space=self._space)

        # index of each type of variables in the dataframe
        self.id_r = self._space.real_id  # index of continuous variable
        self.id_i = self._space.integer_id  # index of integer variable
        self.id_d = self._space.categorical_id  # index of categorical variable

        # the number of variables per each type
        self.N_r = len(self.id_r)
        self.N_i = len(self.id_i)
        self.N_d = len(self.id_d)
        self.dim = self.N_r + self.N_i + self.N_d

        # by default, we use individual step sizes for continuous and
        # integer variables and global strength for the nominal variables
        self.N_p = min(self.N_d, int(1))

        # total length of the solution vector
        self._len = self.dim + self.N_r + self.N_i + self.N_p

        # unpack interval bounds
        self.bounds_r = np.asarray([self._space.bounds[_] for _ in self.id_r])
        self.bounds_i = np.asarray([self._space.bounds[_] for _ in self.id_i])
        # NOTE: bounds might be ragged
        self.bounds_d = [self._space.bounds[_] for _ in self.id_d]
        self._check_bounds(self.bounds_r)
        self._check_bounds(self.bounds_i)

        # step default step-sizes/mutation strength
        par_name = []
        if sigma0 is None and self.N_r:
            sigma0 = 0.05 * (self.bounds_r[:, 1] - self.bounds_r[:, 0])
            par_name += ["sigma" + str(_) for _ in range(self.N_r)]
        if eta0 is None and self.N_i:
            eta0 = 0.05 * (self.bounds_i[:, 1] - self.bounds_i[:, 0])
            par_name += ["eta" + str(_) for _ in range(self.N_i)]
        if P0 is None and self.N_d:
            P0 = 1.0 / self.N_d
            par_name += ["P" + str(_) for _ in range(self.N_p)]

        # column indices: used for slicing
        self._id_var = np.arange(self.dim)
        self._id_sigma = np.arange(self.N_r) + len(self._id_var)
        self._id_eta = np.arange(self.N_i) + len(self._id_var) + len(self._id_sigma)
        self._id_p = np.arange(self.N_p) + len(self._id_var) + len(self._id_sigma) + len(self._id_eta)
        self._id_hyperpar = np.arange(self.dim, self._len)

        # initialize the populations
        if x0 is not None:  # given x0
            par = []
            if self.N_r:
                par += [sigma0]
            if self.N_i:
                par += [eta0]
            if self.N_p:
                par += [P0] * self.N_p

            self.pop = Solution(
                np.tile(np.r_[x0, par], (self.mu_, 1)),
                var_name=self.var_names + par_name,
                verbose=self.verbose,
            )
            fitness0 = self.evaluate(self.pop[0])
            self.fitness = np.repeat(fitness0, self.mu_)
            self.xopt = x0
            self.fopt = sum(fitness0)
        else:  # uniform sampling
            x = np.asarray(self._space.sample(self.mu_), dtype="object")

            par = []
            if self.N_r:
                par += [np.tile(sigma0, (self.mu_, 1))]
            if self.N_i:
                par += [np.tile(eta0, (self.mu_, 1))]
            if self.N_p:
                par += [np.tile([P0] * self.N_p, (self.mu_, 1))]

            par = np.concatenate(par, axis=1)
            x = np.c_[x, par].tolist()

            self.pop = Solution(x, var_name=self.var_names + par_name, verbose=self.verbose)
            self.fitness = self.evaluate(self.pop)
            self.fopt = min(self.fitness) if self.minimize else max(self.fitness)
            _ = np.nonzero(self.fopt == self.fitness)[0][0]
            self.xopt = self.pop[_, self._id_var]

        self.offspring = self.pop[0] * self.lambda_
        self.f_offspring = np.repeat(self.fitness[0], self.lambda_)
        self._set_hyperparameter()

        # stopping criteria
        self.tolfun = 1e-5
        self.nbin = int(3 + ceil(30.0 * self.dim / self.lambda_))
        self.histfunval = zeros(self.nbin)

    def _check_bounds(self, bounds):
        if len(bounds) == 0:
            return
        if any(bounds[:, 0] >= bounds[:, 1]):
            raise ValueError("lower bounds must be smaller than upper bounds")

    def _set_hyperparameter(self):
        # hyperparameters: mutation strength adaptation
        if self.N_r:
            self.tau_r = 1 / np.sqrt(2 * self.N_r)
            self.tau_p_r = 1 / np.sqrt(2 * np.sqrt(self.N_r))

        if self.N_i:
            self.tau_i = 1 / np.sqrt(2 * self.N_i)
            self.tau_p_i = 1 / np.sqrt(2 * np.sqrt(self.N_i))

        if self.N_d:
            self.tau_d = 1 / np.sqrt(2 * self.N_d)
            self.tau_p_d = 1 / np.sqrt(2 * np.sqrt(self.N_d))

    def recombine(self, id1, id2):
        p1 = self.pop[id1].copy()  # IMPORTANT: make a copy
        if id1 != id2:
            p2 = self.pop[id2]
            # intermediate recombination for the mutation strengths
            p1[self._id_hyperpar] = (np.array(p1[self._id_hyperpar]) + np.array(p2[self._id_hyperpar])) / 2

            # dominant recombination for solution parameters
            (_,) = np.nonzero(randn(self.dim) > 0.5)
            p1[_] = p2[_]
        return p1

    def select(self):
        pop = self.pop + self.offspring if self.elitism else self.offspring
        fitness = np.r_[self.fitness, self.f_offspring] if self.elitism else self.f_offspring
        rank = argsort(fitness)

        if not self.minimize:
            rank = rank[::-1]

        _ = rank[: self.mu_]
        self.pop = pop[_]
        self.fitness = fitness[_]

    def evaluate(self, pop, return_penalized=True):
        X = self._to_pheno(pop[:, self._id_var])
        if len(pop.shape) == 1:  # one solution
            X = [X]

        pop.fitness = np.array(list(map(self.obj_func, X))).ravel()
        self.eval_count += pop.N
        _penalized_fitness = (
            self._penalty_func(X, self.iter_count + 1, self.eq_func, self.ineq_func, minimize=self.minimize)
            + pop.fitness
        )
        return _penalized_fitness if return_penalized else pop.fitness

    def mutate(self, individual):
        if self.N_r:
            self._mutate_r(individual)
        if self.N_i:
            self._mutate_i(individual)
        if self.N_d:
            self._mutate_d(individual)
        return individual

    def _mutate_r(self, individual):
        sigma = np.asarray(individual[self._id_sigma], dtype="float")
        # mutate step-sizes
        if len(self._id_sigma) == 1:
            sigma = sigma * exp(self.tau_r * randn())
        else:
            sigma = sigma * exp(self.tau_r * randn() + self.tau_p_r * randn(self.N_r))

        # Gaussian mutation
        R = randn(self.N_r)
        x = np.asarray(individual[self.id_r], dtype="float")
        x_ = x + sigma * R

        # Interval Bounds Treatment
        x_ = handle_box_constraint(x_, self.bounds_r[:, 0], self.bounds_r[:, 1])
        # rounding if a coarser numerical precision is provided
        x_ = self._space[self._space.real_id].round(x_).ravel()
        # NOTE: experimental correction to the step-size when the box constraints are violated
        # the constraint handling method will (by chance) turn bad candidates (which are generated
        # by large step-sizes) to good ones, hence confusing the self-adaptation for step-sizes.
        if 1 < 2:
            individual[self._id_sigma] = np.abs((x_ - x) / R)
        else:
            individual[self._id_sigma] = sigma
        individual[self.id_r] = x_

    def _mutate_i(self, individual):
        eta = np.asarray(individual[self._id_eta].tolist(), dtype="float")
        x = np.asarray(individual[self.id_i], dtype="int")
        if len(self._id_eta) == 1:
            eta = eta * exp(self.tau_i * randn())
        else:
            eta = eta * exp(self.tau_i * randn() + self.tau_p_i * randn(self.N_i))
        eta[eta > 1] = 1

        p = 1 - (eta / self.N_i) / (1 + np.sqrt(1 + (eta / self.N_i) ** 2.0))
        x_ = x + geometric(p) - geometric(p)

        # Interval Bounds Treatment
        x_ = np.asarray(handle_box_constraint(x_, self.bounds_i[:, 0], self.bounds_i[:, 1]), dtype="int")
        individual[self.id_i] = x_
        individual[self._id_eta] = eta

    def _mutate_d(self, individual):
        P = np.asarray(individual[self._id_p], dtype="float")
        #  Unbiased mutation on the mutation probability
        P = 1.0 / (1.0 + (1.0 - P) / P * exp(-self.tau_d * randn()))
        individual[self._id_p] = handle_box_constraint(P, 1.0 / (3.0 * self.N_d), 0.5)

        (idx,) = np.nonzero(rand(self.N_d) < P)
        # TODO: this can be accelerated
        for i in idx:
            levels = self.bounds_d[i]
            individual[self.id_d[i]] = levels[randint(0, len(levels))]

    def stop(self):
        if self.eval_count > self.max_eval:
            self.stop_dict["max_eval"] = True

        if self.eval_count != 0 and self.iter_count != 0:
            fitness = self.f_offspring
            # sigma = np.atleast_2d([__[self._id_sigma] for __ in self.pop])
            # sigma_mean = np.mean(sigma, axis=0)

            # tolerance on fitness in history
            self.histfunval[int(mod(self.eval_count / self.lambda_ - 1, self.nbin))] = fitness[0]
            if (
                mod(self.eval_count / self.lambda_, self.nbin) == 0
                and (max(self.histfunval) - min(self.histfunval)) < self.tolfun
            ):
                self.stop_dict["tolfun"] = True

            # flat fitness within the population
            if fitness[0] == fitness[int(min(ceil(0.1 + self.lambda_ / 4.0), self.mu_ - 1))]:
                self.stop_dict["flatfitness"] = True

            # TODO: implement more stop criteria
            # if any(sigma_mean < 1e-10) or any(sigma_mean > 1e10):
            #     self.stop_dict['sigma'] = True

            # if cond(self.C) > 1e14:
            #     if is_stop_on_warning:
            #         self.stop_dict['conditioncov'] = True
            #     else:
            #         self.flg_warning = True

            # # TolUPX
            # if any(sigma*sqrt(diagC)) > self.tolupx:
            #     if is_stop_on_warning:
            #         self.stop_dict['TolUPX'] = True
            #     else:
            #         self.flg_warning = True

        return any(self.stop_dict.values())

    def _better(self, f1, f2):
        return f1 < f2 if self.minimize else f1 > f2

    # TODO: optimize -> run
    def optimize(self):
        while not self.stop():
            # TODO: vectorize this part
            for i in range(self.lambda_):
                p1, p2 = randint(0, self.mu_), randint(0, self.mu_)
                individual = self.recombine(p1, p2)
                self.offspring[i] = self.mutate(individual)

            # NOTE: `self.fitness` and `self.f_offspring` are penalized function values
            self.f_offspring[:] = self.evaluate(self.offspring)
            self.select()

            curr_best = self.pop[0]
            xopt_, fopt_ = curr_best[self._id_var], self.fitness[0]

            if self._better(fopt_, self.fopt):
                self.xopt, self.fopt = xopt_, fopt_
            self.iter_count += 1

            if self.verbose:
                print("iteration {}, fopt: {}".format(self.iter_count + 1, self.fopt))
                print(self.xopt)

        self.stop_dict["funcalls"] = self.eval_count
        return self.xopt.tolist(), self.fopt, self.stop_dict


import argparse
import os
import random
from pathlib import Path

from tqdm import tqdm

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Break a large data file into a specified number of shards"
    )
    parser.add_argument('input_file', type=str)
    parser.add_argument('output_dir', type=str)
    parser.add_argument('num_shards', type=int)
    parser.add_argument('--seed', type=int, default=1)
    parser.add_argument('--shuffle_first', action='store_true')
    args = parser.parse_args()

    print("Reading in lines")
    lines = [
        line.strip()
        for line in open(args.input_file, 'r') if line.strip() != ''
    ]
    num_lines = len(lines)

    if args.shuffle_first:
        print("Shuffling lines")
        random.seed(args.seed)
        random.shuffle(lines)

    quotient, remainder = divmod(num_lines, args.num_shards)
    lower_elements = [quotient for i in range(args.num_shards - remainder)]
    higher_elements = [quotient + 1 for i in range(remainder)]
    shard_lengths = lower_elements + higher_elements

    os.makedirs(args.output_dir, exist_ok=True)

    print("Writing shards")
    for shard_num, shard_length in tqdm(zip(range(args.num_shards), shard_lengths), desc="shard progress", unit="shard"):
        filestem = Path(args.input_file).stem
        filename = os.path.join(
            args.output_dir, f'{filestem}_shard{shard_num+1}.txt'
        )
        start_index = sum(shard_lengths[:shard_num])
        with open(filename, 'w') as fout:
            for line in lines[start_index:start_index+shard_length]:
                print(line, file=fout)


import os
import logging

import numpy as np
from traits.api import Undefined

from hyperspy.misc.array_tools import sarray2dict

_logger = logging.getLogger(__name__)

# Plugin characteristics
# ----------------------
format_name = 'MRC'
description = ''
full_support = False
# Recognised file extension
file_extensions = ['mrc', 'MRC', 'ALI', 'ali']
default_extension = 0
# Writing capabilities
writes = False
non_uniform_axis = False
# ----------------------

def get_std_dtype_list(endianess='<'):
    end = endianess
    dtype_list = \
        [
            ('NX', end + 'u4'),
            ('NY', end + 'u4'),
            ('NZ', end + 'u4'),
            ('MODE', end + 'u4'),
            ('NXSTART', end + 'u4'),
            ('NYSTART', end + 'u4'),
            ('NZSTART', end + 'u4'),
            ('MX', end + 'u4'),
            ('MY', end + 'u4'),
            ('MZ', end + 'u4'),
            ('Xlen', end + 'f4'),
            ('Ylen', end + 'f4'),
            ('Zlen', end + 'f4'),
            ('ALPHA', end + 'f4'),
            ('BETA', end + 'f4'),
            ('GAMMA', end + 'f4'),
            ('MAPC', end + 'u4'),
            ('MAPR', end + 'u4'),
            ('MAPS', end + 'u4'),
            ('AMIN', end + 'f4'),
            ('AMAX', end + 'f4'),
            ('AMEAN', end + 'f4'),
            ('ISPG', end + 'u2'),
            ('NSYMBT', end + 'u2'),
            ('NEXT', end + 'u4'),
            ('CREATID', end + 'u2'),
            ('EXTRA', (np.void, 30)),
            ('NINT', end + 'u2'),
            ('NREAL', end + 'u2'),
            ('EXTRA2', (np.void, 28)),
            ('IDTYPE', end + 'u2'),
            ('LENS', end + 'u2'),
            ('ND1', end + 'u2'),
            ('ND2', end + 'u2'),
            ('VD1', end + 'u2'),
            ('VD2', end + 'u2'),
            ('TILTANGLES', (np.float32, 6)),
            ('XORIGIN', end + 'f4'),
            ('YORIGIN', end + 'f4'),
            ('ZORIGIN', end + 'f4'),
            ('CMAP', (bytes, 4)),
            ('STAMP', (bytes, 4)),
            ('RMS', end + 'f4'),
            ('NLABL', end + 'u4'),
            ('LABELS', (bytes, 800)),
        ]

    return dtype_list

def get_fei_dtype_list(endianess='<'):
    end = endianess
    dtype_list = [
        ('a_tilt', end + 'f4'),  # Alpha tilt (deg)
        ('b_tilt', end + 'f4'),  # Beta tilt (deg)
        # Stage x position (Unit=m. But if value>1, unit=???m)
        ('x_stage', end + 'f4'),
        # Stage y position (Unit=m. But if value>1, unit=???m)
        ('y_stage', end + 'f4'),
        # Stage z position (Unit=m. But if value>1, unit=???m)
        ('z_stage', end + 'f4'),
        # Signal2D shift x (Unit=m. But if value>1, unit=???m)
        ('x_shift', end + 'f4'),
        # Signal2D shift y (Unit=m. But if value>1, unit=???m)
        ('y_shift', end + 'f4'),
        ('defocus', end + 'f4'),  # Defocus Unit=m. But if value>1, unit=???m)
        ('exp_time', end + 'f4'),  # Exposure time (s)
        ('mean_int', end + 'f4'),  # Mean value of image
        ('tilt_axis', end + 'f4'),  # Tilt axis (deg)
        ('pixel_size', end + 'f4'),  # Pixel size of image (m)
        ('magnification', end + 'f4'),  # Magnification used
        # Not used (filling up to 128 bytes)
        ('empty', (np.void, 128 - 13 * 4)),
    ]
    return dtype_list

def get_data_type(index, endianess='<'):
    end = endianess
    data_type = [
        end + 'u2',         # 0 = Signal2D     unsigned bytes
        end + 'i2',         # 1 = Signal2D     signed short integer (16 bits)
        end + 'f4',         # 2 = Signal2D     float
        (end + 'i2', 2),    # 3 = Complex   short*2
        end + 'c8',         # 4 = Complex   float*2
    ]
    return data_type[index]

def file_reader(filename, endianess='<', **kwds):
    metadata = {}
    f = open(filename, 'rb')
    std_header = np.fromfile(f, dtype=get_std_dtype_list(endianess),
                             count=1)
    fei_header = None
    if std_header['NEXT'] / 1024 == 128:
        _logger.info("%s seems to contain an extended FEI header", filename)
        fei_header = np.fromfile(f, dtype=get_fei_dtype_list(endianess),
                                 count=1024)
    if f.tell() == 1024 + std_header['NEXT']:
        _logger.debug("The FEI header was correctly loaded")
    else:
        _logger.warning("There was a problem reading the extended header")
        f.seek(1024 + std_header['NEXT'])
        fei_header = None
    NX, NY, NZ = std_header['NX'], std_header['NY'], std_header['NZ']
    mmap_mode = kwds.pop('mmap_mode', 'c')
    lazy = kwds.pop('lazy', False)
    if lazy:
        mmap_mode = 'r'
    data = np.memmap(f, mode=mmap_mode, offset=f.tell(),
                     dtype=get_data_type(std_header['MODE'][0], endianess)
                     ).squeeze().reshape((NX[0], NY[0], NZ[0]), order='F').T

    original_metadata = {'std_header': sarray2dict(std_header)}
    # Convert bytes to unicode
    for key in ["CMAP", "STAMP", "LABELS"]:
        original_metadata["std_header"][key] = \
            original_metadata["std_header"][key].decode()
    if fei_header is not None:
        fei_dict = sarray2dict(fei_header,)
        del fei_dict['empty']
        original_metadata['fei_header'] = fei_dict

    dim = len(data.shape)
    if fei_header is None:
        # The scale is in Amstrongs, we convert it to nm
        scales = [10 * float(std_header['Zlen'] / std_header['MZ'])
                  if float(std_header['MZ']) != 0 else 1,
                  10 * float(std_header['Ylen'] / std_header['MY'])
                  if float(std_header['MY']) != 0 else 1,
                  10 * float(std_header['Xlen'] / std_header['MX'])
                  if float(std_header['MX']) != 0 else 1, ]
        offsets = [10 * float(std_header['ZORIGIN']),
                   10 * float(std_header['YORIGIN']),
                   10 * float(std_header['XORIGIN']), ]

    else:
        # FEI does not use the standard header to store the scale
        # It does store the spatial scale in pixel_size, one per angle in
        # meters
        scales = [1, ] + [fei_header['pixel_size'][0] * 10 ** 9, ] * 2
        offsets = [0, ] * 3

    units = [Undefined, 'nm', 'nm']
    names = ['z', 'y', 'x']
    metadata = {'General': {'original_filename': os.path.split(filename)[1]},
                "Signal": {'signal_type': "",
                           'record_by': 'image', },
                }
    # create the axis objects for each axis
    axes = [
        {
            'size': data.shape[i],
            'index_in_array': i,
            'name': names[i + 3 - dim],
            'scale': scales[i + 3 - dim],
            'offset': offsets[i + 3 - dim],
            'units': units[i + 3 - dim], }
        for i in range(dim)]

    dictionary = {'data': data,
                  'axes': axes,
                  'metadata': metadata,
                  'original_metadata': original_metadata,
                  'mapping': mapping}

    return [dictionary, ]

mapping = {
    'fei_header.a_tilt':
    ("Acquisition_instrument.TEM.Stage.tilt_alpha", None),
    'fei_header.b_tilt':
    ("Acquisition_instrument.TEM.Stage.tilt_beta", None),
    'fei_header.x_stage':
    ("Acquisition_instrument.TEM.Stage.x", None),
    'fei_header.y_stage':
    ("Acquisition_instrument.TEM.Stage.y", None),
    'fei_header.z_stage':
    ("Acquisition_instrument.TEM.Stage.z", None),
    'fei_header.exp_time':
    ("Acquisition_instrument.TEM.Detector.Camera.exposure", None),
    'fei_header.magnification':
    ("Acquisition_instrument.TEM.magnification", None),
}


import random
from parrot import Parrot

def paraphrase_text(text, parrot, do_diverse=True, use_gpu=False,
            adequacy_threshold = 0.90, fluency_threshold = 0.90) :

    if not parrot:
        print("Warning: parrot not provided")
        return text

    texts = text.split('\n')
    ptexts = []
    for text in texts:
        if not text: continue
        para_phrases = parrot.augment(
            input_phrase=text,
            use_gpu=use_gpu,
            do_diverse=do_diverse,
            adequacy_threshold=adequacy_threshold,
            fluency_threshold=fluency_threshold)

        print(f">>This is the original text:{text}")
        if para_phrases is None: continue
        if len(para_phrases) < 1: para_phrases=[""]
        text=random.choice(para_phrases)[0]
        print(f">> And the paraphrased text:{text}")
        ptexts.append(text)
    text='\n'.join(ptexts)

    return text

if __name__ == "__main__":

    parrot = Parrot(model_tag="prithivida/parrot_paraphraser_on_T5")
    text = "I've always loved to take long walks in the woods."

    for i in range(100):
        text = paraphrase_text(text, parrot,
                adequacy_threshold=0.3,
                fluency_threshold=0.3)


import csv
import re
import json
import os
import matplotlib.pyplot as plt
import seaborn as sns
import sys
import math
from datetime import *

window_smoothing_width_days = 50 # stdev of gaussian to convolve over the data
window_size_days = 8 * window_smoothing_width_days # Window size to average # of messages over
num_top_people = 30 # How many of the top people to display
color_spread_skip = num_top_people // 5 # stride over color
enable_group_chats = False
anonymize = False

# for word count
def get_message_weight(text_contents):
    return len(text_contents.split(' '))

# for just per-message
# def get_message_weight(msg):
#     return 1

# how much to weigh a message based on distance
# use a gaussian for smoothing
def get_weight_for_time(message_day, graph_day):
    stdev = window_smoothing_width_days
    return gaussian_pdf(message_day, stdev, graph_day)

# average over window
# def get_weight_for_time(message_day, graph_day):
#     return 1.0 / window_size_days

def gaussian_pdf(mean, stdev, x):
    variance = stdev ** 2
    offset = abs(mean - x)
    return math.exp(-(offset ** 2)/(2 * variance)) / math.sqrt(2 * math.pi * variance)

def spread_list(items, k):
    res = []
    for i in range(0, k):
        res = res + items[i::k]
    return res

if anonymize:
    anon_names = []
    with open('anon_names.txt') as f:
        anon_names = [name.strip() for name in f.readlines()]

name_norm_filename = 'name_normalization.json'
name_normalization = {}
if os.path.exists(name_norm_filename) and os.path.isfile(name_norm_filename):
    with open(name_norm_filename) as f:
        name_normalization = json.load(f)

def normalize_name(name):
    if name in name_normalization:
        return name_normalization[name]
    else:
        return name

def graph_messages_window(all_messages, start_date, end_date):

    delta = end_date - start_date

    xs = [start_date + timedelta(days=i) for i in range(delta.days)]

    # List of tuples of the form (total number of messages, y-values corresponding to every day, their name)
    ys_and_labels = []

    for name in all_messages:
        total_messages = 0
        y_name = [0] * len(xs)

        message_counts = all_messages[name]

        for day in message_counts:
            message_delta = day - start_date
            for windowed_day in range(message_delta.days - int(window_size_days / 2),
                                      message_delta.days + int(window_size_days / 2) + 1):
                if windowed_day > 0 and windowed_day < len(y_name):
                    weight = get_weight_for_time(message_delta.days, windowed_day)
                    inc = message_counts[day] * weight
                    y_name[windowed_day] += inc

            if message_delta.days > 0 and message_delta.days < len(y_name):
                total_messages += message_counts[day]

        ys_and_labels.append((total_messages, y_name, name, total_messages))

    # Display label sorted by most messages
    ys_and_labels.sort(reverse=True)

    # Sort y-values into a set of y-values for each of the top n people and "Other"
    ys = [y_and_label[1] for y_and_label in ys_and_labels[:num_top_people]]
    other_ys = [0] * len(xs)
    total_other_messages = 0
    for y_and_label in ys_and_labels[num_top_people:]:
        y = y_and_label[1]
        for i, count in enumerate(y):
            other_ys[i] += count
        total_other_messages += y_and_label[0]

    top_ys_and_labels = ys_and_labels[:num_top_people]

    anon_mapping = { name: name for _, _, name, _ in top_ys_and_labels }
    if anonymize:
        anon_mapping = { name: anon_names[i] for i, (_, _, name, _) in enumerate(top_ys_and_labels) }

    labels = ["%s (%d)" % (anon_mapping[y_and_label[2]], int(y_and_label[3])) for y_and_label in top_ys_and_labels]
    labels.append("Other (%d)" % int(total_other_messages))

    # Make colors prettier
    pal = sns.color_palette("hls", num_top_people)
    # This is really hardcoded for 20 basically
    colors = spread_list(pal, color_spread_skip)
    other_color = (0.9, 0.9, 0.9) #Grey
    colors.append(other_color)

    plt.rc('xtick', labelsize=16)
    plt.rc('ytick', labelsize=16)
    plt.rc('legend', fontsize=8)
    plt.rc('axes', titlesize=24)
    plt.title("Message word count by person over time")
    plt.stackplot(xs, *ys, other_ys, colors=colors, labels=labels, baseline='wiggle')
    plt.legend(loc='upper right')
    plt.show()

def aggregate_messages(
        messages_by_day_by_person,
        all_messages):

    all_people = set.union(*[ps for _, ps, _ in all_messages])

    for person in all_people:
        if person in messages_by_day_by_person:
            messages_per_day = messages_by_day_by_person[person]
        else:
            messages_per_day = {}
            messages_by_day_by_person[person] = messages_per_day
        for message_date, creditors, weight in all_messages:
            if person in creditors:
                num_creditors = len(creditors)
                if message_date in messages_per_day:
                    messages_per_day[message_date] += 1.0 * weight / num_creditors
                else:
                    messages_per_day[message_date] =  1.0 * weight / num_creditors

# returns a list of (timestamp, participants, weight) tuples, where participants equally share message weight
def process_adium_conversation(convo):
    namespaces = { 'default': "http://purl.org/net/ulf/ns/0.4-02" }
    user_name = convo.get('account')
    messages = [x for x in convo.findall('default:message', namespaces = namespaces)]
    participants = set([normalize_name(p.get('sender')) for p in messages])
    if user_name in participants:
        participants.remove(user_name)
    if len(participants) == 0:
        return []
    if len(participants) != 1 and not enable_group_chats:
        return []

    def get_local_participants(msg):
        sender = normalize_name(msg.get('sender'))
        if sender == user_name:
            return participants
        else:
            return set([sender])

    return [(
        datetime.fromisoformat(msg.get('time')).date(),
        get_local_participants(msg),
        get_message_weight(ET.tostring(msg, method='text').decode())
    ) for msg in messages]

import xml.etree.ElementTree as ET

def get_all_adium_messages(
        data_directory,
        user_name,
        start_date):
    # all_messages: list of (time_seconds, creditors)
    all_messages = []
    for msg_file in [os.path.join(data_directory, c) for c in os.listdir(data_directory)]:
        if msg_file.endswith(".xml"):
            convo = ET.parse(msg_file).getroot()
            all_messages.extend(process_adium_conversation(convo))

    return all_messages

# returns a list of (timestamp, participants, weight) tuples, where participants equally share message weight
def process_facebook_conversation(convo, user_name):
    participants = set([normalize_name(p['name']) for p in convo['participants']])
    if user_name in participants:
        participants.remove(user_name)
    if len(participants) == 0:
        return []
    if len(participants) != 1 and not enable_group_chats:
        return []

    messages = convo['messages']
    # This is a dumb starting message that Facebook leads with sometimes, ignore it
    if 'content' in messages[-1] and messages[-1]['content'].startswith('Say hi to your'):
        messages = messages[:-1]

    def get_local_participants(msg):
        sender = normalize_name(msg['sender_name'])
        if sender == user_name:
            return participants
        else:
            return set([sender])

    return [(
        date.fromtimestamp(msg['timestamp_ms'] / 1000.0),
        get_local_participants(msg),
        get_message_weight(msg.get('content') or '')
    ) for msg in messages]

def get_all_facebook_messages(
        data_directory,
        user_name,
        start_date):
    inbox = os.path.join(data_directory, 'messages', 'inbox')
    all_messages = []
    for thread_dir in [os.path.join(inbox, c) for c in os.listdir(inbox)]:
        for msg_file in [os.path.join(thread_dir, c) for c in os.listdir(thread_dir)]:
            if msg_file.endswith(".json"):
                with open(msg_file) as f:
                    convo = json.load(f)
                    all_messages.extend(process_facebook_conversation(convo, user_name))

    return all_messages

# returns a list of (timestamp, participants, weight) tuples, where participants equally share message weight
def process_hangouts_conversation(convo, user_name):
    participant_id_to_name = {
        p['id']['chat_id']: normalize_name(p.get('fallback_name') or p['id']['chat_id'])
        for p in convo['conversation']['conversation']['participant_data']
    }
    participants = set([name for _, name in participant_id_to_name.items()])
    if user_name in participants:
        participants.remove(user_name)
    if len(participants) == 0:
        return []
    if len(participants) != 1 and not enable_group_chats:
        return []

    messages = []
    for event in convo['events']:
        if event['event_type'] == 'REGULAR_CHAT_MESSAGE':
            sender = participant_id_to_name[event['sender_id']['chat_id']]
            if sender == user_name:
                creditors = participants
            else:
                creditors = set([sender])
            message_content = event['chat_message']['message_content']
            if 'segment' in message_content:
                message_text =  " ".join([
                    segment['text']
                    for segment in message_content['segment']
                    if segment['type'] == 'TEXT'
                ])

                messages.append((
                    date.fromtimestamp(int(event['timestamp']) // 1000000),
                    creditors,
                    get_message_weight(message_text)
                ))

    return messages

def get_all_hangouts_messages(
        data_directory,
        user_name,
        start_date):
    hangouts_file = os.path.join(data_directory, 'Hangouts', 'Hangouts.json')
    # all_messages: list of (time_seconds, creditors)
    with open(hangouts_file) as f:
        conversations = json.load(f)['conversations']

    all_messages = []
    for convo in conversations:
        all_messages.extend(process_hangouts_conversation(convo, user_name))

    return all_messages

# returns a list of (timestamp, participants, weight) tuples, where participants equally share message weight
def process_gtalk_conversation(convo, user_name):
    def get_date(line):
        try:
            return date.fromisoformat(line[:line.find(' ')])
        except ValueError:
            return None
    def get_sender(line):
        return line[line.find('<') + 1 : line.find('>')]
    def get_message(line):
        return line[line.find('>') + 2:]

    participants = set([normalize_name(get_sender(m)) for m in convo if get_date(m) is not None])
    if user_name in participants:
        participants.remove(user_name)
    if len(participants) == 0:
        return []
    if len(participants) != 1 and not enable_group_chats:
        return []

    def get_local_participants(sender):
        sender = normalize_name(sender)
        if sender == user_name:
            return participants
        else:
            return set([sender])

    messages = []
    last_message = None
    for line in convo:
        dt = get_date(line)
        if dt is not None:
            if last_message is not None:
                last_message[2] = get_message_weight(last_message[2])
                messages.append(last_message)
            last_message = [
                dt,
                get_local_participants(get_sender(line)),
                get_message(line)
            ]
        else:
            assert last_message is not None
            last_message[2] += '\n' + line
    if last_message is not None:
        last_message[2] = get_message_weight(last_message[2])
        messages.append(last_message)
    return messages

def get_all_gtalk_messages(
        data_directory,
        user_name,
        start_date):
    all_messages = []
    for msg_file in [os.path.join(data_directory, c) for c in os.listdir(data_directory)]:
        with open(msg_file) as f:
            convo = [l for l in f.readlines()]
            all_messages.extend(process_gtalk_conversation(convo, user_name))

    return all_messages

def get_all_iphone_messages(
        data_directory,
        user_name,
        start_date):
    contact_mapping = {}
    with open(os.path.join(data_directory, 'contacts.csv')) as csvfile:
        contacts_reader = csv.reader(csvfile)
        for row in contacts_reader:
            normalized_number = re.sub('[- ()]', '', row[0])
            full_name = (row[1] + ' ' + row[2]).strip()
            # contact_mapping[normalized_number] = full_name
            if len(normalized_number) == 10 and normalized_number.isnumeric():
                contact_mapping['+1' + normalized_number] = full_name
            elif len(normalized_number) == 11 and normalized_number.isnumeric() and normalized_number.startswith('1'):
                contact_mapping['+' + normalized_number] = full_name
            elif len(normalized_number) == 12 and normalized_number[1:].isnumeric() and normalized_number.startswith('+1'):
                contact_mapping[normalized_number] = full_name
            elif '@' in normalized_number:
                contact_mapping[normalized_number] = full_name
            else:
                print(normalized_number)
                contact_mapping[normalized_number] = full_name

    def get_name(contact):
        return contact_mapping.get(contact) or contact

    all_messages = []
    with open(os.path.join(data_directory, 'messages.csv')) as csvfile:
        messages_reader = csv.reader(csvfile)
        for row in messages_reader:
            dt = date.fromisoformat(row[0][:row[0].find(' ')])
            interlocutor = normalize_name(get_name(row[1]))
            weight = get_message_weight(row[2])
            all_messages.append((dt, set([interlocutor]), weight))

    return all_messages

def main():
    start_date_str = sys.argv[1]
    start_date = datetime.strptime(start_date_str, '%Y-%m-%d').date()
    end_date_str = sys.argv[2]
    end_date = datetime.strptime(end_date_str, '%Y-%m-%d').date()
    user_name = sys.argv[3]
    messages_by_day_by_person = {}
    for arg in sys.argv[4:]:
        format_spec, data_dir = arg.split(':')
        data_dir = os.path.expanduser(data_dir)
        if format_spec == 'facebook':
            all_messages = get_all_facebook_messages(data_dir, user_name, start_date)
        elif format_spec == 'adium':
            all_messages = get_all_adium_messages(data_dir, user_name, start_date)
        elif format_spec == 'hangouts':
            all_messages = get_all_hangouts_messages(data_dir, user_name, start_date)
        elif format_spec == 'gtalk':
            all_messages = get_all_gtalk_messages(data_dir, user_name, start_date)
        elif format_spec == 'iphone':
            all_messages = get_all_iphone_messages(data_dir, user_name, start_date)
        aggregate_messages(messages_by_day_by_person, all_messages)

    graph_messages_window(messages_by_day_by_person, start_date, end_date)

if __name__ == "__main__":
    main()


"""
Loop Detection: Given a circular linked list, implement an algorithm that returns the node at the
beginning of the loop.
DEFINITION
Circular linked list: A (corrupt) linked list in which a node's next pointer points to an earlier node, so
as to make a loop in the linked list.
EXAMPLE
Input: A -> B -> C - > D -> E -> C [the same C as earlier)
Output: C

Source: Cracking the Coding Interview: 189 Programming Questions and Solutions
by Gayle Laakmann McDowell
"""

from lists.node import Node

def get_loop(head_of_list: Node) -> Node:
    """
    Time complexity: O(2N)
    Space complexity: O(1)

    :param head_of_list: First element of the list.
    :return: Instance of the first node of the loop. None if there is not a loop.
    """
    if head_of_list is None or head_of_list.next is None or head_of_list.next.next is None:
        return None

    slow_runner = head_of_list.next
    fast_runner = head_of_list.next.next

    while slow_runner is not fast_runner:

        slow_runner = slow_runner.next
        if fast_runner.next is not None:
            fast_runner = fast_runner.next.next
        else:
            return None

        if slow_runner is None or fast_runner is None:
            return None

    curr = head_of_list
    while curr is not slow_runner:
        curr = curr.next
        slow_runner = slow_runner.next

    return curr

def get_loop_(head_of_list: Node) -> Node:
    """
    Time complexity: O(N)
    Space complexity: O(N)

    :param head_of_list: The head of the list.
    :return: Instance of the first node of the loop. None if there is not a loop.
    """
    uniques = {}

    head_of_loop = None
    while head_of_list is not None:
        if head_of_list not in uniques:
            uniques[head_of_list] = True
        else:
            head_of_loop = head_of_list
            break

        head_of_list = head_of_list.next
    return head_of_loop


import threading
from flask import Flask
import os
import sys
import logging

app = Flask("keepalive")

@app.route('/', methods=['GET', 'POST', 'CONNECT', 'PUT', 'DELETE', 'PATCH', 'OPTIONS', 'TRACE', 'HEAD'])
def main():
    repl_owner = os.environ.get('REPL_OWNER')
    return f'''Hey there, {repl_owner}! To keep your Replit running continuously, you'll need to use an uptime monitoring service like UptimeRobot'''

log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
app.logger.disabled = True
cli = sys.modules['flask.cli']
cli.show_server_banner = lambda *x: None

def run_flask_app():
    app.run(host='0.0.0.0', port=3000, debug=False, use_reloader=False)

Welcomer = """\033[1;31m⚠ Looks like you are running this project on Replit\033[0m
\033[1;33mPlease note that the .env file cannot exist on Replit.
Instead, create environment variable DISCORD_TOKEN in the "Secrets" tab under "Tools". \033[0m
"""

def run_flask_in_thread():
    threading.Thread(target=run_flask_app).start()
    print(Welcomer)
    repl_owner_name = os.environ.get('REPL_OWNER')
    repl_project_name = os.environ.get('REPL_SLUG')
    print(f"\033[1;32m\n\nTo ensure your bot runs 24/7 on Replit, you can use services like Uptime Robot to ping the following URL:\033[0m \n\n https://{repl_project_name}.{repl_owner_name}.repl.co\n\n")

if __name__ == "__main__":
    run_flask_in_thread()


import pygame
from pygame.sprite import Sprite

class Alien(Sprite):
    """docstring for Alien."""

    def __init__(self, ai_settings, screen):
        super(Alien, self).__init__()
        self.screen = screen
        self.rect = screen
        self.ai_settings = ai_settings
        # load alien image and set attributes
        self.image = pygame.image.load('images/alien.bmp')
        self.rect = self.image.get_rect()
        # set every alien in screen left the up
        self.rect.x = self.rect.width
        self.rect.y = self.rect.height
        # store alien accurate
        self.x = float(self.rect.x)

    def blitme(self):
        self.screen.blit(self.image, self.rect)

    def update(self):
        """move right"""
        self.x += (self.ai_settings.alien_speed_factor * self.ai_settings.fleet_direction)
        self.rect.x = self.x

    def check_edges(self):
        screen_rect = self.screen.get_rect()
        if self.rect.right >= screen_rect.right:
            return True
        elif self.rect.left <= 0:
            return True


import os
import sys
import collections
import numpy as np

class graph(object):
    def __init__(self):
        self.n_nodes = 0
        self.n_edges = 0
        self.n_feature = 0
        self.graph_dict = collections.defaultdict(list)
        self.feature_embed = None

    def read_graph_files(self, file_path):
        assert os.exists(graph_path)
        assert not os.path.isdir(graph_path)

        fp = open(file_path, "r")
        for line in fp:
            tmp = line.strip("\n").split(' ')
            src, trg = int(tmp[0]), int(tmp[1])
            self.graph_dict[trg].append(src)
            self.n_edges += 1
        self.n_nodes = len(graph)
        print("{}\n{}\n{}".format(file_path, nodes, n_edges))

    def gen_graph_embedding(self, n_features):
        self.n_feature = n_features
        self.feature_embed = np.random.rand(self.nodes, self.n_feature)


#!/usr/bin/env python3
"""A simple script to move 2 files into ceph_storage/"""

# importing the needed standard libraries
import shutil   #shell utilities to move the files
import os       #access to low level os operations (changing program start directory)

def main():
    """called at runtime"""
    # force app to start in the student home directory
    os.chdir('/home/student/mycode/')

    # moving file from source to destination folder, will overwrite the file if already exists
    shutil.move('raynor.obj', 'ceph_storage/raynor.obj')

    # prompt for new file name
    xname = input('What is the new name for kerigan.obj? ')
    # "renaming" file by moving old file and giving it new name in destination
    shutil.move('ceph_storage/kerrigan.obj', 'ceph_storage/' + xname)

main()  # calling the main function


from ddfpuzzles.puzzsolver import PuzzSolver
import re

class NineBlocksPuzzle(PuzzSolver):
    '''.. _NineBlocksPuzzle:

    NineBlocksPuzzle inherited from Puzzle is an iterable which when iterated
    goes through all possible moves of the game in the current state.  The member
    'pos' is the current state of the game.

    Attributes:

    pos
       -- Starting position of the game.

    goal
       -- regular expression which any valid solution must match.

    b_equiv
       -- string translation dict for transforming an arbitrary 'pos' into
       a canonical form..

    block
       -- set of tuples of the form ('place', 'move') specifying disallowed
       moves of any holes in the given 'place' on the game grid.

    Usage:

       >>> from nine_block_puzzle import NineBlocksPuzzle
       >>> from pprint import pprint
       >>> pprint(NineBlocksPuzzle().solve())

    Notes:

    Puzzle implements a general solve() function which uses the iterator to
    search either Breadth-first or Depth-first to find a valid solution. To do so
    the inheriting child class (here NineBlocksPuzzle ) must implement an isgoal()
    method returning True iff the current state is a solution.

    See the module documentation for an overview.  This class starts from a
    game position:

    | 1122
    | 1133
    | 4500
    | 6788
    | 6799

    represented by the string

    >>> pos = '112211343456685779'

    The goal is a regular expression, and we'd expect:

    >>> re.match(goal,'22222222222222222221') is not None
    False
    >>> re.match(goal,'22222222222222221xx1') is not None
    True

    '''
    pos = '11221133450067886799'
    b_equivs = str.maketrans('38975','22264')
    goal = re.compile(r'.'*16+'1'+'.'*3)

    def __init__(self,*args,**kwargs):
        if 'start' in kwargs: self.pos = kwargs['start']
        if args :
            self.pos = args[0]
        assert len(self.pos)==20, f'Bad init(), need string of len=20, your\'s was {self.pos} len={len(self.pos)}'
        for x in self.pos:
            assert int(x)<=9 and int(x)>=0, f'Bad init(), game state must be a string of single digits, your\'s is pos = {self.pos}'

    def __repr__(self):
        ans = '\n'
        p = self.pos.replace('0', '.')
        for i in [0, 4, 8, 12, 16]:
            ans = ans + p[i:i+4] + '\n'
        return ans

    def isgoal(self):
        return self.goal.search(self.pos) != None

    def canonical(self):
        '''
        The member 'b_equivs' is a dictionary specifying a transform of
        letters for a substitution pattern.  So if

        >>> b_equivs = str.maketrans('45678','33229')

        then we are saying '4' and '5' are equivalent to '3',
        '6' and '7' are equivalent to '2', and '8' is equivalent to'9'
        This means we can eliminate all '4','5','6','7','8' from any state.
        So a canonical form of the general position,

        >>> x = '23230081111944556767'

        would be

        >>> canonical(x) -> '23230091111933332222'

        Basically a '3' is a generic vertical 2x1 block, and a '2' is a
        generic horizontal 1x2 block.
        '''
        return self.pos.translate(self.b_equivs)

    """The 'block' set defines moves that are not allowed.  The first of a tuple
    to be checked against 'block' should be a position in the 'pos' grid
    currently free, a '0' position, the only such positions blocked are around
    the circumference, so these are, going around clockwise:
    0,1,2,3,7,11,15,19,18,17,16,12,8,4.
    A move right|left along a row is a +1|-1 shift along the 'pos' string.
    A move up|down a column is a -4|+4 shift along the 'pos' string.
    In our grid the 0,1,2,3 row is top, so if a '0' is in any of these
    positions they cannot go up, so a -4 move is not allowed,
    sly. a '0' in any place along the 0,4,8,12 (leftmost)edge cannot move
    left, so a -1 move is disallowed for these.  All the other blocked moves
    are listed in the set 'block'.
    """
    block: set = { (0,-4), (1,-4), (2,-4), (3,-4),
                (16,4), (17,4), (18,4), (19,4),
                (0,-1), (4,-1), (8,-1), (12,-1), (16,-1),
                (3,1), (7,1), (11,1), (15,1), (19,1) }

    def __iter__(self):
        """
        Lists all possible moves from current state 'pos' returned as a new Puzzle.

        The algorithm here is to imagine moving the '0' labels (holes).  A hole has 4
        adjacent cells it can shift to unless it is at a corner or edge.

        After blocking disallowed moves for the pair of current holes '0', we retrieve
        the label 'piece' of the block at one of the possible adjacent places a '0' could
        move.  If this is also a '0' we ignore and go to the next adjacent place available
        for the hole. A possible move is added to the Puzzle iterable.
        """
        dsone = self.pos.find('0')
        dstwo = self.pos.find('0', dsone+1)
        for dest in [dsone, dstwo]:
            for adj in [-4, -1, 1, 4]:
                if (dest, adj) in self.block: continue
                piece = self.pos[dest+adj]
                if piece == '0': continue
                #print('In __iter__ evaluating at piece =',piece)
                newmove = self.pos.replace(piece, '0')
                for i in range(len(self.pos)):
                    if 0 <= i+adj < len(self.pos) and self.pos[i+adj]==piece:
                        newmove = newmove[:i] + piece + newmove[i+1:]
                if newmove.count('0') != 2: continue
                #print('   yielding newmove = %s' % newmove )
                yield NineBlocksPuzzle(newmove)

#-----------------------------------------------
if __name__ == '__main__':
    from pprint import pprint

    puzz = NineBlocksPuzzle()
    soln_steps = puzz.solve()
    print('Solution:')
    pprint(soln_steps)

#if __name__ == "__main__":
    #import doctest
    #doctest.testmod()


#libraries
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split #splits data
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from string import punctuation
from sklearn.pipeline import Pipeline #steps can be run in series
from sklearn.feature_extraction.text import CountVectorizer #converts text into number vectors
from sklearn.feature_extraction.text import TfidfTransformer #transforms count vectors into tfidf
from sklearn.svm import SVC #SVM model
import matplotlib.pyplot as plt

path = 'comedy_central.csv'
data = pd.read_csv(path)
joke_content = data.content.tolist() #actual joke
categories = data.flattened_categories.tolist()

#creates list of most relevant category label for each joke
categories_1 = []
for comma_categ in categories:
    categ_split = comma_categ.split(',') #splits into list of categories
    categories_1.append(categ_split[0]) #appends 1st (most relevant) category

#list of unique most relevant categories of all jokes
unique_categ = []
for categ in categories_1:
    if categ not in unique_categ:
        unique_categ.append(categ)

#cleans jokes of stopwords, punctuation, and irrelevant info
print('Cleaning jokes ...')
stop_words = set(stopwords.words('english')) #set of predefined english stop words
joke_content_cleaned = []
for sent in joke_content:
    words = word_tokenize(sent) #splits into words
    #print('joke content before being cleaned: ', words)
    words_cleaned = [] #joke, list of meaningful words
    for word in words:
        if word not in stop_words and word not in punctuation and word not in ['Q', 'A', '``']:
            words_cleaned.append(word)
    words_cleaned = ' '.join(words_cleaned)
    #print('joke content after being cleaned: ', words_cleaned)
    joke_content_cleaned.append(words_cleaned.strip())

# pipeline to transform words into tf-idf vectors, then trains SVM model
print('Creating pipeline ...')
model = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', SVC(kernel='rbf'))])

#splits jokes and categories into testing and training sets
print('Splitting into testing and training sets ...')
X_train, X_test, y_train, y_test = train_test_split(joke_content_cleaned, categories_1, test_size=0.1)

#trains model
print('Training model ...')
model.fit(X_train, y_train)
#prints accuracy score when tested against testing data and labels
print('Accuracy: ', model.score(X_test, y_test))


import os
import click
import requests
from pprint import pprint

# Local imports
from util import *

def get_hist_orders(base_url, apikey, secret, symbol, start_time, end_time, order_type, side,
                    method="order/hist/current"):
    ts = utc_timestamp()
    url = "{}/{}".format(base_url, method)
    headers = make_auth_headers(ts, method, apikey, secret)
    params = {"symbol": symbol, "startTime": start_time, "endTime": end_time, "orderType": order_type, "side": side}
    return requests.get(url, headers=headers, params=params)

def get_open_orders(base_url, apikey, secret, symbol, method="order/open"):
    ts = utc_timestamp()
    url = "{}/{}".format(base_url, method)
    headers = make_auth_headers(ts, method, apikey, secret)
    params = {"symbol": symbol}
    return requests.get(url, headers=headers, params=params)

def get_order_status(base_url, apikey, secret, order_id, method="order/status"):
    url = "{}/{}".format(base_url, method, order_id)
    print(url)
    ts = utc_timestamp()
    headers = make_auth_headers(ts, method, apikey, secret)
    return requests.get(url, headers=headers, params={"orderId": order_id})

@click.command()
@click.option("--account", type=click.Choice(['cash', 'margin']), default="cash")
@click.option("--symbol", type=str, default='BTC/USDT')
@click.option("--start_time", type=int, default=0)
@click.option("--end_time", type=int, default=utc_timestamp())
@click.option("--order_type", type=str, default=None)  # "market" or "limit"
@click.option("--side", type=click.Choice(['buy', 'sell']), default=None)
@click.option("--order_id", type=str, default=None)
@click.option("--config", type=str, default="config_local.json", help="path to the config file")
def run(account, symbol, start_time, end_time, order_type, side, config, order_id):

    ascdex_cfg = load_config(get_config_or_default(config))['ascendex']

    host = ascdex_cfg['https']
    group = ascdex_cfg['group']
    apikey = ascdex_cfg['apikey']
    secret = ascdex_cfg['secret']

    base_url = f"{host}/{group}/{ROUTE_PREFIX}/{account}"

    print("Get history orders")
    res = get_hist_orders(base_url, apikey, secret, symbol, start_time, end_time, order_type=order_type, side=side)
    pprint(parse_response(res))

    print("\n ****** \n")
    print("Get open orders")
    res = get_open_orders(base_url, apikey, secret, None)
    pprint(parse_response(res))

    if order_id is not None:
        print("\n ****** \n")
        print(f"Query status for order {order_id}")
        res = get_order_status(base_url, apikey, secret, order_id)
        pprint(parse_response(res))

if __name__ == "__main__":
    run()


import sys
import os
import argparse
import structlog

from . import core

logger = structlog.get_logger(logger_name=__name__)

def take():
  parser = argparse.ArgumentParser(description='update version records of nvchecker')
  core.add_common_arguments(parser)
  parser.add_argument('--all', action='store_true',
                      help='take all updates')
  parser.add_argument('--ignore-nonexistent', action='store_true',
                      help='ignore nonexistent names')
  parser.add_argument('names', metavar='NAME', nargs='*',
                      help='software name to be updated')
  args = parser.parse_args()
  if core.process_common_arguments(args):
    return

  s = core.Source(args.file)
  if not s.oldver or not s.newver:
    logger.critical(
      "doesn't have both 'oldver' and 'newver' set.", source=s,
    )
    sys.exit(2)

  oldvers = core.read_verfile(s.oldver)
  newvers = core.read_verfile(s.newver)

  if args.all:
    oldvers.update(newvers)
  else:
    for name in args.names:
      try:
        oldvers[name] = newvers[name]
      except KeyError:
        if args.ignore_nonexistent:
          logger.warning('nonexistent in newver, ignored', name=name)
          continue

        logger.critical(
          "doesn't exist in 'newver' set.", name=name,
        )
        sys.exit(2)

  try:
      os.rename(s.oldver, s.oldver + '~')
  except FileNotFoundError:
      pass
  core.write_verfile(s.oldver, oldvers)

def cmp():
  parser = argparse.ArgumentParser(description='compare version records of nvchecker')
  core.add_common_arguments(parser)
  args = parser.parse_args()
  if core.process_common_arguments(args):
    return

  s = core.Source(args.file)
  oldvers = core.read_verfile(s.oldver) if s.oldver else {}
  newvers = core.read_verfile(s.newver)
  for name, newver in sorted(newvers.items()):
    oldver = oldvers.get(name, None)
    if oldver != newver:
      print('%s %s -> %s' % (name, oldver, newver))


try:
    import os
except:
    print("ExceptionERROR: Missing fundamental packages (required: os).")

def chk_dir(directory):
    if not(os.path.exists(directory)):
            os.makedirs(directory)

def clean_dir(directory):
    # Delete everything reachable IN the directory named in 'directory',
    # assuming there are no symbolic links.
    # CAUTION:  This is dangerous!  For example, if directory == '/', it
    # could delete all your disk files.
    for root, dirs, files in os.walk(directory, topdown=False):
        for name in files:
            os.remove(os.path.join(root, name))
        for name in dirs:
            os.rmdir(os.path.join(root, name))

def dict_values2list(dictionary):
    # converts the values of a dicionary into a list
    l = []
    for e in dictionary.values():
        l.append(e)
    return l

def get_subdir_names(directory):
    subdir_list = [name for name in os.listdir(directory) if os.path.isdir(os.path.join(directory, name))]
    return subdir_list

def interpolate_linear(x1, x2, y1, y2, xi):
    # returns linear interpolation yi of xi between two points 1 and 2
    return float(y1) + ((float(xi) - float(x1)) / (float(x2) - float(x1)) * (float(y2) - float(y1)))

def is_number(e):
    try:
        float(e)
        return True
    except ValueError:
        return False

def read_file(full_path, col_separator):
    # full_path = STR of full path to text file
    # col_separator = STR, e.g., ',' or 'TAB'
    # returns nested list of data contained in a text file

    data = []
    f = open(full_path)
    f_lines = f.readlines()
    for line in f_lines:
        try:
            l = line.strip('\n').split(col_separator)  # remove newline symbol if exists
        except:
            try:
                l = line.split(col_separator)
            except:
                l = line
        l_entries = []
        for e in l:
            try:
                l_entries.append(float(e))
            except:
                l_entries.append(str(e))
        try:
            data.append(l_entries)
        except ValueError:
            print('Could not read file information from: ' + str(full_path))
    f.close()
    return data

def rm_dir(directory):
    # Deletes everything reachable from the directory named in 'directory', and the directory itself
    # assuming there are no symbolic links.
    # CAUTION:  This is dangerous!  For example, if directory == '/' deletes all disk files
    for root, dirs, files in os.walk(directory, topdown=False):
        for name in files:
            os.remove(os.path.join(root, name))
        for name in dirs:
            os.rmdir(os.path.join(root, name))
    os.rmdir(directory)

def rm_file(full_name):
    # fullname = str of directory + file name
    try:
        os.remove(full_name)
    except:
        pass

def str2frac(arg):
    arg = arg.split('/')
    return int(arg[0]) / int(arg[1])

def str2num(arg, sep):
    # function converts string of type 'X[sep]Y' to number
    # sep is either ',' or '.'
    # e.g. '2,30' is converted with SEP = ',' to 2.3
    _num = arg.split(sep)
    _a = int(_num[0])
    _b = int(_num[1])
    _num = _a + _b * 10 ** (-1 * len(str(_b)))
    try:
        return float(_num)
    except:
        return _num

def str2tuple(arg):
    try:
        arg = arg.split(',')
    except ValueError:
        print('ERROR: Bad assignment of separator.\nSeparator must be [,].')
    tup = (int(arg[0]), int(arg[1]))
    return tup

def tuple2num(arg):
    # function converts float number with ',' separator for digits to '.' separator
    # type(arg) = tuple with two entries, e.g. (2,40)
    # call: tuple2num((2,3))
    new = arg[0] + arg[1] * 10 ** (-1 * len(str(arg[1])))
    return new

def write_data(folder_dir, file_name, data):
    if not os.path.exists(folder_dir):
            os.mkdir(folder_dir)
    os.chdir(folder_dir)

    f = open(file_name, 'w')
    for i in data:
        line = str(i)+'\n'
        f.write(line)
    print('Data written to: \n' + folder_dir + '\\' + str(file_name))


import discord
from discord.ext import commands
from os import listdir
from os.path import isfile
from os.path import isfile, join
import traceback

cogs_dir = "cogs"
configs = "cogs/configs"

bot = commands.Bot(command_prefix='!')

@bot.event
async def on_ready():
	print('Logged in as')
	print(bot.user.name)
	print(bot.user.id)
	print('------')
	await bot.change_presence(status=discord.Status.online,activity=discord.Activity(type=discord.ActivityType.playing,name="{Providing Configs!"))

@bot.event
async def on_command_error(ctx,error):
	await ctx.send("Something happened "+str(error))

if __name__ == "__main__":
	for extension in [f.replace('.py', '') for f in listdir(cogs_dir) if isfile(join(cogs_dir, f))]:
		try:
			bot.load_extension(cogs_dir + "." + extension)
		except Exception as e:
			print(f'Failed to load extension {extension}.')
			traceback.print_exc()

bot.run('Token_Here')


__author__ = 'xyliao'
import numpy as np

def fig2data(fig):
    """Brief convert a Matplotlib figure to numpy.ndarray.

    Args:
        fig: a matplotlib figure

    Returns:
        return a 4D numpy.ndarray with RGBA channels.
    """
    fig.canvas.draw()

    w, h = fig.canvas.get_width_height()
    buf = np.frombuffer(fig.canvas.tostring_argb(), dtype=np.uint8)
    buf.shape = (w, h, 4)

    buf = np.roll(buf, 3, axis=2)
    return buf.reshape(h, w, 4)

def fig2img(fig):
    """Convert a Matplotlib figure to tensorbaord image, which is a 3D np.ndarray, shape is :math:`(height, width, 3)`.
    This is in RGB format and the range of its value is :math:`[0, 255]`.

    Args:
        fig: Matplotlib figure

    Returns:
        a tensorboard image with shape :math:`(height, width, 3)`, value is :math:`[0, 255]`.
    """
    from matplotlib import pyplot as plot
    ax = fig.get_figure()
    img_data = fig2data(ax).astype(np.uint8)
    plot.close()
    return img_data[:, :, :3]


# -*- coding: utf-8 -*-

import json
from shutil import which
from subprocess import check_output
import typing

from preview_generator import utils
from preview_generator.exception import BuilderDependencyNotFound
from preview_generator.exception import PreviewGeneratorException
from preview_generator.preview.generic_preview import PreviewBuilder

ffmpeg_installed = True
try:
    import ffmpeg
except ImportError:
    ffmpeg_installed = False

class NoVideoStream(PreviewGeneratorException):
    pass

class VideoPreviewBuilderFFMPEG(PreviewBuilder):
    page_nb = 10
    weight = 80

    @classmethod
    def get_label(cls) -> str:
        return "Video files - based on ffmpeg"

    @classmethod
    def check_dependencies(cls) -> None:
        if not ffmpeg_installed:
            raise BuilderDependencyNotFound("this builder requires ffmpeg to be available")

    @classmethod
    def dependencies_versions(cls) -> typing.Optional[str]:
        return "{} from {}".format(
            check_output(["ffmpeg", "-version"], universal_newlines=True).strip().split("\n")[0],
            which("ffmpeg"),
        )

    @classmethod
    def get_supported_mimetypes(cls) -> typing.List[str]:
        return [
            "application/x-videolan",
            "video/3gpp",
            "video/annodex",
            "video/dl",
            "video/dv",
            "video/fli",
            "video/gl",
            "video/mpeg",
            "video/mp2t",
            "video/mp4",
            "video/quicktime",
            "video/mp4v-es",
            "video/ogg",
            "video/parityfec",
            "video/pointer",
            "video/webm",
            "video/vnd.fvt",
            "video/vnd.motorola.video",
            "video/vnd.motorola.videop",
            "video/vnd.mpegurl",
            "video/vnd.mts",
            "video/vnd.nokia.interleaved-multimedia",
            "video/vnd.vivo",
            "video/x-flv",
            "video/x-la-asf",
            "video/x-mng",
            "video/x-ms-asf",
            "video/x-ms-wm",
            "video/x-ms-wmv",
            "video/x-ms-wmx",
            "video/x-ms-wvx",
            "video/x-msvideo",
            "video/x-sgi-movie",
            "video/x-matroska",
            "video/x-theora+ogg",
            "video/x-m4v",
        ]

    def get_dims_from_ffmpeg_probe(
        self, probe_result: typing.Dict[str, typing.Any]
    ) -> utils.ImgDims:
        """
        Extract the width and height of the video stream from probe dict and return it
        """
        video_stream_data = {}  # type:typing.Dict[str,typing.Any]
        for stream_data in probe_result["streams"]:
            if stream_data["codec_type"] == "video":
                video_stream_data = stream_data

        if not video_stream_data:
            raise NoVideoStream

        return utils.ImgDims(width=video_stream_data["width"], height=video_stream_data["height"])

    def _get_frame_time(self, page_id: int, page_nb: int, video_duration: float) -> float:
        """
        Compute time of frame #page_id
        The algorithm is:
        - first frame is at 2%
        - last frame is at 98%
        - time is plitted between 2% and 98% according to the number of requested frames
        - if 1 frame only, then return 2%

        :param page_id: id of the page, value must be between 0 and page_nb-1. If -1, return the first frame
        :param page_nb: total number of frames
        :param video_duration: duration of the full video (float)
        :return: the time of the requested frame
        """
        if page_nb > 1:
            # 100 - 4 is:
            # - we take first frame at 2% of full duration
            # - we take last frame at 2% of end of movie
            # - we split the 96% remaining for all frames
            delta_between_two_frames_in_percent = (100 - 4) / (page_nb - 1)
        else:
            delta_between_two_frames_in_percent = 0

        return video_duration * (2 + delta_between_two_frames_in_percent * page_id) / 100

    def _get_extraction_size(
        self, video_dims: utils.ImgDims, preview_dims: utils.ImgDims
    ) -> utils.ImgDims:
        """
        Compute extraction dimensions.

        The extract size in order to directly get the right height or width according
        to what is expected for preview

        :param video_dims: ImgDims object representing width and height of the video stream
        :param preview_dims: ImgDims object representing width and height of the preview to generate
        :return: ImgDims to use for ffmpeg video frame extraction
        """
        extract_size = utils.ImgDims(-1, -1)
        if video_dims.ratio() > preview_dims.ratio():
            extract_size.width = preview_dims.width
        else:
            extract_size.height = preview_dims.height

        return extract_size

    def build_jpeg_preview(
        self,
        file_path: str,
        preview_name: str,
        cache_path: str,
        page_id: int,
        extension: str = ".jpg",
        size: utils.ImgDims = None,
        mimetype: str = "",
    ) -> None:
        """
        generate the jpeg preview
        """
        if not size:
            size = self.default_size

        preview_path = "{path}{file_name}{extension}".format(
            file_name=preview_name, path=cache_path, extension=extension
        )

        video_probe_data = ffmpeg.probe(file_path)
        video_size = self.get_dims_from_ffmpeg_probe(video_probe_data)
        extraction_size = self._get_extraction_size(video_size, size)

        video_duration = float(video_probe_data["format"]["duration"])
        page_nb = self.get_page_number(file_path, preview_name, cache_path)
        frame_time = self._get_frame_time(page_id, page_nb, video_duration)

        (
            ffmpeg.input(file_path, ss=frame_time)
            .filter("scale", extraction_size.width, extraction_size.height)
            .output(preview_path, vframes=1)
            # INFO - G.M - 2020-07-03 we do allow overwrite to allow forcing the refresh of
            # the preview.
            .overwrite_output()
            .run()
        )

    def build_json_preview(
        self,
        file_path: str,
        preview_name: str,
        cache_path: str,
        page_id: int = 0,
        extension: str = ".json",
    ) -> None:
        """
        generate the json preview. Default implementation is based on ExifTool
        """
        metadata = ffmpeg.probe(file_path)

        with open(cache_path + preview_name + extension, "w") as jsonfile:
            json.dump(metadata, jsonfile)

    def set_page_nb(self, page_nb: int) -> int:
        """
        Allow to override default page_nb (10 by default)
        """
        self.page_nb = page_nb
        return self.page_nb

    def get_page_number(
        self,
        file_path: str,
        preview_name: str,
        cache_path: str,
        mimetype: typing.Optional[str] = None,
    ) -> int:
        return self.page_nb

    def has_jpeg_preview(self) -> bool:
        return True


from airflow import DAG
from datetime import datetime, timedelta
from airflow.operators.python_operator import PythonOperator
import random
from pprint import pprint
from icecream_sunday_dag_def import choose_icecream_flavor, choose_cone, choose_toppings, make_icecream_sundae, \
    default_args

icecream_sundae_linear_dag = DAG('ice_cream_sundae_linear', default_args=default_args, schedule_interval=None)

"""
Linear Ice Cream Sundae DAG Def
"""

def generate_choose_cone_op(dag, task_id):
    return PythonOperator(
        task_id='choose_cone_task_{}'.format(task_id),
        python_callable=choose_cone,
        provide_context=True,
        dag=dag
    )

def generate_choose_icecream_flavor_op(dag, task_id):
    return PythonOperator(
        task_id='choose_icecream_flavor_task_{}'.format(task_id),
        provide_context=True,
        dag=dag,
        python_callable=choose_icecream_flavor,
    )

def generate_choose_toppings_op(dag, task_id):
    return PythonOperator(
        task_id='choose_toppings_task_{}'.format(task_id),
        provide_context=True,
        dag=dag,
        python_callable=choose_toppings,
    )

def generate_make_icecream_sundae_op(dag, task_id):
    return PythonOperator(
        task_id='make_icecream_sundae_task_{}'.format(task_id),
        provide_context=True,
        dag=dag,
        python_callable=make_icecream_sundae,
    )

choose_cones_op_list = [generate_choose_cone_op(icecream_sundae_linear_dag, 1),
                        generate_choose_cone_op(icecream_sundae_linear_dag, 2)]
choose_flavors_op_list = [generate_choose_icecream_flavor_op(icecream_sundae_linear_dag, 1),
                          generate_choose_icecream_flavor_op(icecream_sundae_linear_dag, 2)]
choose_toppings_op_list = [generate_choose_toppings_op(icecream_sundae_linear_dag, 1),
                           generate_choose_toppings_op(icecream_sundae_linear_dag, 2)]

for index, choose_cone_OP in enumerate(choose_cones_op_list):
    choose_cone_OP.set_downstream(choose_flavors_op_list[index])
    choose_flavors_op_list[index].set_downstream(choose_toppings_op_list[index])


import pygame

window = pygame.display.set_mode((800, 600), pygame.RESIZABLE)
clock = pygame.time.Clock()

SIZE = 100

x = 350
y = 250
dx = 1
dy = 1
v = 0.1
a = 0.01

while True:
    # 1. INPUT
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            exit()
    # 2. UPDATE
    x += 2 * v * dx
    y += v * dy
    v += a
    if x <= 0 or x >= window.get_width() - SIZE:
        dx = -dx
    if y <= 0 or y >= window.get_height() - SIZE:
        dy = -dy
    if v < -10 or v > 10:
        a = -a
    # 3. OUTPUT
    window.fill((255, 255, 0))
    pygame.draw.rect(window, (0, 0, 255), (x, y, SIZE, SIZE))
    pygame.display.flip()
    clock.tick(60)


#!/usr/bin/python

import os
import errno
import grp
import time
from stat import *
import atexit

LOCKDIR = "/var/tmp/devicelocks"

def pid_exists(pid):
    if pid < 0:
        return False
    if pid == 0:
        raise ValueError('invalid PID 0')
    try:
        os.kill(pid, 0)
    except OSError as err:
        if err.errno == errno.ESRCH:
            # ESRCH == No such process
            return False
        elif err.errno == errno.EPERM:
            # EPERM clearly means there's a process to deny access to
            return True
        else:
            # According to "man 2 kill" possible error values are
            # (EINVAL, EPERM, ESRCH)
            raise
    else:
        return True

class DeviceIsLockedError(SystemError):
    pass

def cleanup_lock(lockname):
    if os.path.exists(lockname):
        os.unlink(lockname)

class DeviceLock:
    """Creates a reliable inter-process lock for exclusive
    access to a resource with the unique name DEVICENAME.
    When the DeviceLock object is deleted using del,
    or the process exists, the lock file is auomatically
    deleted. Stale lockfiles are detected by probing for
    the process ID (pid) of the creating process.
    """
    def __init__(self, devicename, usergroup=''):
        self.pidfile = None
        try:
            os.mkdir(LOCKDIR, S_IRWXU | S_IRWXG |  S_IROTH)
            if usergroup != '':
                # set group
                gid = grp.getgrnam(usergroup)[2]
                os.chown(LOCKDIR, -1, gid)

        except OSError  as e:
            if e.errno == errno.EEXIST:
                pass
            else:
                raise

        mypid = str(os.getpid())
        tmpname = "%s/%s_%s-%s.lock" % (LOCKDIR,devicename,mypid,time.time())
        with open(tmpname,"w") as f:
            f.writelines(mypid)

        lockname = "%s/%s.lock" % (LOCKDIR,devicename)
        # this is atomic and can only succeed if lockname
        # does not exist
        try:
            try:
                # give everyone read permission
                os.chmod(tmpname, S_IRUSR | S_IRGRP | S_IROTH)
                # link to temporary lock file
                # This is important because we need to keep
                # the lock while we check whether the last owner
                # of the lock is alive
                os.link(tmpname, lockname)
            except:
                # raise SystemError("Error: can't lock Device %s" % devicename)
                raise

        finally:
            # we don't need the temporary file name any more
            os.unlink(tmpname)

        pidfile = "%s/%s.pid" % (LOCKDIR,devicename)

        try:
            if os.path.exists(pidfile):
                oldpid = int(open(pidfile).readline())
                if pid_exists(oldpid):
                    raise DeviceIsLockedError("Device %r is locked with lock file %r by running process %i" % (devicename, pidfile, oldpid))
                else:
                    print("Info: replacing stale lock file for device %r of dead process %s" % (devicename, oldpid))

            # this is also atomic, and unlinks the old pidfile
            os.rename(lockname, pidfile)
            self.pidfile = pidfile

        finally:
            cleanup_lock(lockname)

        atexit.register(cleanup_lock, pidfile)

    def __del__(self):
        if not self.pidfile is None:
            cleanup_lock(self.pidfile)


from __future__ import print_function
import datetime
import os
import matplotlib;   matplotlib.use('TkAgg');
import numpy as np
import random
import os
import random
import operator

import wandb

from Trainer import Initializer as init

from Trainer import Router_utils as utils
from Trainer import Router_utils_plt as save_utils
from Trainer import GridGraphV2
from datetime import datetime
import importlib
np.random.seed(10701);  random.seed(10701);
def saveResults():
    return
def timestamp():
    return datetime.now().strftime("%B %d, %H:%M:%S")

def train_one_epoch(filename,  # benchmark_reduced/test_benchmark_i.gr
                    env,
                    algos_fn,algos_name,
                    hid_layer,
                    globali,
                    self_play_episode_num,
                    result_dir,
                    logger,
                    save_ckpt:bool,
                    load_ckpt:bool,
                    early_stop,
                    ckpt_folder,
                    emb_dim,
                    context_len,
                    rainbow_mode):
    #* print("---data preprocessing---")
    # # Getting Net Info
    grid_info = init.read(filename)
    gridParameters:dict = init.gridParameters(grid_info)

    sortedHalfWireLength = sorted(init.VisualGraph(gridParameters).bounding_length().items(),
                                key=operator.itemgetter(1),reverse=True) # Large2Small
    netSort = []
    for i in range(gridParameters['numNet']):   #20    #A1~A20  for each teset_benchmark.gr
        order = int(sortedHalfWireLength[i][0])
        netSort.append(order)           #arrange net by its wireLength
    #* print(f"---netsort {netSort} len {len(netSort)}---")
    twoPinEachNetClear:list = utils.gen_2pinListClear(gridParameters)

    twopinlist_nonet = utils.gen2pinlistNet(
                            utils.gen2pinListCombo(
                                    gridParameters,
                                    sortedHalfWireLength
                            ))
    assert np.sum(twoPinEachNetClear) == len(twopinlist_nonet)   #== 49,  20net has 49 pin connect, avg_connect/net ~=2.5
    #* print("---DRL Module from here---")
    os.makedirs(ckpt_folder,exist_ok=True);

    Env_graph = env(gridParameters,
                                max_step=100,   #?20
                                twopin_combo=twopinlist_nonet,
                                net_pair=twoPinEachNetClear)
    # Training DRL
    #!!!  core  DQN_implement.py
    success = 0
    #* print("----start training---")
    if algos_name in ['rainbow_dqn']:
        print(algos_name,">>>>>>>")
        agent = algos_fn.DQN_Agent( env=Env_graph, rainbow_mode=rainbow_mode, hid_layer=hid_layer,
        emb_dim=emb_dim, self_play_num =self_play_episode_num, context_len=context_len,)
        results, solutionTwoPin,posTwoPinNum,success = agent.train(
                            twoPinEachNetClear,
                            netSort,
                            ckpt_path=f"{ckpt_folder}{algos_name}.ckpt",
                            logger=logger,
                            save_ckpt=save_ckpt,
                            load_ckpt=load_ckpt,
                            early_stop=early_stop,
            )
    else:
        print(algos_name,">>>>>>>")
        agent = algos_fn.DQN_Agent( Env_graph, hid_layer,emb_dim,
                                   self_play_episode_num =self_play_episode_num,
                                   context_len=context_len)
        results, solutionTwoPin,posTwoPinNum,success = agent.train(
                            twoPinEachNetClear,
                            netSort,
                            ckpt_path=f"{ckpt_folder}{algos_name}.ckpt",
                            logger=logger,
                            save_ckpt=save_ckpt,
                            load_ckpt=load_ckpt,
                            early_stop=early_stop,
            )
    # print("======---saving results(Generate output file for DRL solver)---======")
    assert len(Env_graph.twopin_combo)==len(twopinlist_nonet)
    print(f"=====posTwoPinNum  {posTwoPinNum}/{len(Env_graph.twopin_combo)}======")
    if posTwoPinNum >= len(twopinlist_nonet):
        save_utils.save(result_dir,globali,agent.max_episodes,gridParameters,
            results['reward_plot_combo'], results['reward_plot_combo_pure'],
            results['solutionDRL'],sortedHalfWireLength, solutionTwoPin)
    else:
        print("DRL fails with existing max episodes! : (")
    return success

class Print_log:
    '''just a fake wandDB log (used when --enable_wandb=False)'''
    def __init__(self) -> None:
        pass
    def log(self,item):
        # print(item)
        pass
def main_fn(
        algos="dtqn",
        mode="train",  #train, eval
        hid_layer=1, emb_dim=64,
        context_len = 5,   #try other numbers 1~30,  I known 50 is bad and slow
        early_stop=False,
        result_dir = "solutionsDRL",
        save_ckpt=True,load_ckpt=True,
        self_play_episode_num = 150,
        enable_wandb=True, wandbName="",
        data_folder='train_data_/benchmark_reduced',
        run_benchmark_num = None,
        verbose = False,
        rainbow_mode = ['double','nstep']#['double','duel','noisy','per','cat','nstep']
    ):
    '''
    [double] is nice (20/20),
    In the current setting [duel, noisy] are bad,
    todo : save replay ckpt when training (this will influence the beta update of PER)
    #[double] at train and [double,per] at eval is still bad
    '''

    print(">>>>>>>>>>>>>>>\n",locals())
    algos_name = algos
    rain_dict = {}
    if 'rainbow' in algos:
        stardard_rainbow_list = ['double','duel','noisy','per','cat','nstep']
        for r_mode in stardard_rainbow_list:
            if r_mode in rainbow_mode:
                rain_dict[r_mode] = True
                algos_name+= f'_{r_mode}'
            else:
                rain_dict[r_mode] = False
        print('rainbow_mode--->',rain_dict)
        print('algos_name--->',algos_name)
    if enable_wandb:
        wandb.login()
        project_name = "Global_route"
        config={
                "algos":algos_name,
                "mode":mode,
                "layer":hid_layer,
                "emb_dim":emb_dim,
                "episode":self_play_episode_num,
                "context_len":context_len,
        }
        if verbose:
            group = wandbName+"_"+"_".join(
                [f"{key}={val}" for key, val in config.items()]
            ),
        else:
            group = f"{algos_name}_{mode}_depth111"
        wandb.init(
            project=project_name,
            name = timestamp(),
            group = group,
            config=config,
        )
        logger = wandb
    else:
        print("disable wandb")
        logger = Print_log()
    print(self_play_episode_num,result_dir)
    os.system(f'rm -r {result_dir}');    os.makedirs(result_dir);
    benchmark_reduced_path = data_folder
    src_benchmark_file = [li for li in os.listdir(benchmark_reduced_path) if "rewardData" not in li]
    #*  ex. test_benchmark_1.gr,  test_benchmark_2.gr .....
    success_count = 0

    env = GridGraphV2.GridGraph
    algos_fn = importlib.import_module(f'Trainer.algos.agent.{algos}')
    print(algos_fn.__name__,"...algos module name...")

    if run_benchmark_num == None:
        run_benchmark_num = len(src_benchmark_file)
    run_benchmark_num = min(run_benchmark_num,len(src_benchmark_file))
    for i in range(run_benchmark_num):
        #benchmark_reduced/test_benchmark_1.gr
        read_file_name = f"{benchmark_reduced_path}/test_benchmark_{i+1}.gr"
        print (f'\n********{i+1}/{run_benchmark_num}******Working on {read_file_name}****************')
        success = train_one_epoch(read_file_name,
                         env,
                         algos_fn,
                         algos_name=algos,
                         hid_layer=hid_layer,
                         globali=i,
                         self_play_episode_num=self_play_episode_num,
                         result_dir=result_dir,
                         logger=logger,
                         save_ckpt=save_ckpt,
                         ckpt_folder = "./model/",
                         early_stop=early_stop,
                         load_ckpt=load_ckpt,
                         emb_dim=emb_dim,
                         context_len=context_len,
                         rainbow_mode=rain_dict)
        success_count+=success
        logger.log({'success_count':success_count})
    return


# 알파벳 대소문자로 된 단어가 주어지면,
# 이 단어에서 가장 많이 사용된 알파벳이 무엇인지 알아내는 프로그램
# 대소문자 구분 X

a = input()
a = a.lower()

lst = [a[v] for v in range(len(a))]
lst = list(set(lst))

cnt = [a.count(i) for i in lst]
max_n = max(cnt)

if cnt.count(max_n) > 1:
      print('?')
else:
      idx = cnt.index(max_n)
      print(max(lst[idx].upper()))


import os, sys

if len(sys.argv) < 3:
	print("Not enough inputs specified! Must specify 2 input arguments!")
	exit(-1)

pipe_in = sys.argv[1]
pipe_out = sys.argv[2]

id_in = os.open(pipe_in, os.O_WRONLY)
id_ou = os.open(pipe_out, os.O_RDONLY)

print(f"The input pipe has id {id_in}, and the output pipe has id {id_ou}")

buf = os.read(id_ou,5000)

print(f"The first {len(buf)} bytes have been read as:\n{buf.decode()}")

os.close(id_ou)

os.write(id_in, b"Message recieved!\n\0")

os.close(id_in)


from django.shortcuts import render
from django.contrib import messages
import random
from django.shortcuts import render
from .models import Ticket
from django.http import FileResponse, HttpResponseNotFound
from io import BytesIO
import os
from docx import Document
from docx.shared import Pt, RGBColor
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
import os
import pythoncom
from win32com.client import Dispatch

def replace_text_in_word_file(input_file, output_file, old_text1, new_text1,old_text2,new_text2):
    doc = Document(input_file)

    for paragraph in doc.paragraphs:
        if old_text1 in paragraph.text:
            paragraph.text = paragraph.text.replace(old_text1, "")
            run = paragraph.add_run(new_text1)
            run.font.name = 'Arial'
            run.font.size = Pt(12)
            run.font.bold = True
            run.font.color.rgb = RGBColor(255, 255, 255)

        if old_text2 in paragraph.text:
            paragraph.text = paragraph.text.replace(old_text2, "")
            run = paragraph.add_run(new_text2)
            run.font.name = 'Arial'
            run.font.size = Pt(12)
            run.font.bold = True
            run.font.color.rgb = RGBColor(255, 255, 255)

    for table in doc.tables:
        for row in table.rows:
            for cell in row.cells:
                if old_text1 in cell.text:
                    cell.text = cell.text.replace(old_text1, "")
                    run = cell.paragraphs[0].add_run(new_text1)
                    run.font.name = 'Arial'
                    run.font.size = Pt(12)
                    run.font.bold = True
                    run.font.color.rgb = RGBColor(255, 255, 255)

                if old_text2 in cell.text:
                    cell.text = cell.text.replace(old_text2, "")
                    run = cell.paragraphs[0].add_run(new_text2)
                    run.font.name = 'Arial'
                    run.font.size = Pt(12)
                    run.font.bold = True
                    run.font.color.rgb = RGBColor(255, 255, 255)

    doc.save(output_file)
    convert_to_pdf(output_file)

def convert_to_pdf(docx_file):
    pythoncom.CoInitialize()
    word = Dispatch('Word.Application')
    docx_file = os.path.abspath(docx_file)  # Convert to absolute path
    doc = word.Documents.Open(docx_file)
    pdf_file = os.path.splitext(docx_file)[0] + '.pdf'
    pdf_file = os.path.abspath(pdf_file)  # Convert to absolute path
    doc.SaveAs(pdf_file, FileFormat=17)  # 17 is the PDF file format
    doc.Close()
    word.Quit()

def save(request):
    return render(request, 'index.html')

def process_form(request):
    if request.method == 'POST':
        name = request.POST.get('name')
        surname = request.POST.get('surname')
        random_number = request.POST.get('randomNumber')

        # Check if a user with the same name and surname already exists
        existing_users = Ticket.objects.filter(name=name, surname=surname)
        if existing_users.exists():
            error_message = "A user with the same name and surname already exists."
            context = {'error_message': error_message}
        else:
            # Save the new user or perform any other required actions
            user = Ticket(name=name, surname=surname, ticket_number=random_number)
            user.save()

            # Prepare context for successful submission
            context = {'name': name, 'surname': surname, 'random_number': random_number}

        input_file_path = 'input.docx'
        output_file_path = 'output.docx'

        old_text1 = 'Participant : Turyan Azizov'
        new_text1 = f'Participant : {name} {surname}'

        old_text2 = 'Ticket number : 123454321'
        new_text2 = f'Ticket number : {random_number}'

        replace_text_in_word_file(input_file_path, output_file_path, old_text1, new_text1,old_text2,new_text2)

        pdf_path = 'output.pdf'
        if os.path.exists(pdf_path):
            # Read the PDF content into a BytesIO buffer
            with open(pdf_path, 'rb') as pdf_file:
                pdf_content = pdf_file.read()

            # Create and return the FileResponse using the buffer
            buffer = BytesIO(pdf_content)
            response = FileResponse(buffer, content_type='application/pdf')
            response['Content-Disposition'] = 'attachment; filename="ticket.pdf"'
            return response
    return render(request, 'index.html')


# =============================================================================
# Fog Blocking Clustering
# =============================================================================
#
# Implementation of a blocking clusterer that starts by dispatching given
# items to one or more buckets before computing pairwise comparisons on them.
#
import dill
from collections import defaultdict
from multiprocessing import Pool
from fog.clustering.utils import make_similarity_function, clusters_from_pairs

# TODO: max_block_size to avoid ngrams with high DF
# TODO: worker using a VPTree
# TODO: custom inner algorithm
# TODO: fuzzy blocking variant
# TODO: possibility not to merge if sure cannot collide twice
# TODO: the worker_graph is not used right now, the code does not use it (make a clusters from graph also)

def block_worker(payload):
    """
    Worker function used compute pairwise distance/similarity over a whole
    block.

    """
    similarity, block, serialized, graph = payload

    if serialized:
        similarity = dill.loads(similarity)

    pairs = []
    n = len(block)

    for i in range(n):
        A = block[i]

        for j in range(i + 1, n):
            B = block[j]

            if graph is not None and A in graph and B in graph[A]:
                continue

            if similarity(A, B):
                pairs.append((A, B))

    return pairs

def blocking(data, block=None, blocks=None, similarity=None, distance=None,
             radius=None, min_size=2, max_size=float('inf'), processes=1,
             mode='connected_components'):
    """
    Function returning an iterator over found clusters using the blocking
    method.

    It works by dispatching given items into one or more buckets before
    computing pairwise comparisons on each bucket.

    Args:
        data (iterable): Arbitrary iterable containing data points to gather
            into clusters. Will be fully consumed.
        block (callable): A function returning an item's block.
        blocks (callable): A function returning an item's blocks.
        similarity (callable): If radius is specified, a function returning
            the similarity between two points. Else, a function returning
            whether two points should be deemed similar. Alternatively, one can
            specify `distance` instead.
        distance (callable): If radius is specified, a function returning
            the distance between two points. Else, a function returning
            whether two point should not be deemed similar. Alternatively, one
            can specify `similarity` instead.
        radius (number, optional): produced clusters' radius.
        min_size (number, optional): minimum number of items in a cluster for
            it to be considered viable. Defaults to 2.
        max_size (number, optional): maximum number of items in a cluster for
            it to be considered viable. Defaults to infinity.
        processes (number, optional): number of processes to use.
            Defaults to 1.
        mode (string, optional): 'fuzzy_clusters', 'connected_components'.
            Defaults to 'connected_components'.

    Yields:
        list: A viable cluster.

    """

    # Formatting similarity
    similarity = make_similarity_function(similarity=similarity, distance=distance, radius=radius)

    # Single block or blocks?
    if blocks is None:
        graph = defaultdict(list)
        worker_graph = None

        def add(x, y):
            x.append(y)
    else:
        graph = defaultdict(set)
        worker_graph = graph

        def add(x, y):
            x.add(y)

    # Grouping items into buckets
    buckets = defaultdict(list)

    for item in data:
        if blocks is None:
            buckets[block(item)].append(item)
        else:
            bs = set(blocks(item))

            for b in bs:
                buckets[b].append(item)

    # Actual clustering
    def clustering():
        if processes == 1:
            for bucket in buckets.values():
                if len(bucket) < 2:
                    continue

                if type(bucket) is not list:
                    bucket = list(bucket)

                yield from block_worker((similarity, bucket, False, worker_graph))
        else:

            pickled_similarity = dill.dumps(similarity)

            pool_iter = (
                (pickled_similarity, bucket if type(bucket) is list else list(bucket), True, None)
                for bucket
                in buckets.values()
                if len(bucket) > 1
            )

            with Pool(processes=processes) as pool:
                for pairs in pool.imap_unordered(block_worker, pool_iter):
                    yield from pairs

    yield from clusters_from_pairs(
        clustering(),
        fuzzy=blocks is not None,
        min_size=min_size,
        max_size=max_size,
        mode=mode
    )


#!/usr/bin/env python
# -*- coding: utf-8 -*-

INPUT_FILE = 'input.txt'

def main():
    with open('input.txt', 'r') as fp:
        modules = [int(line) for line in fp if line.strip()]
    print(modules, len(modules))
    tmp = []
    for mod in modules:
        tmp.append(mod // 3 - 2)
    print(tmp, len(tmp))
    print(sum(tmp))

if __name__ == '__main__':
    main()


#!/usr/bin/env python3

from .basemodel import BaseModel
from typing import Optional, Dict

class AutomateConnectionInfo(BaseModel):
    """Class definition for AutomateConnectionInfo."""

    def __init__(self, host: str, port: int, is_tls: bool):
        """Initialize AutomateConnectionInfo class instance."""
        self.host = host
        self.port = port
        self.is_tls = is_tls

    def __repr__(self) -> str:
        """Pretty class representation when printing."""
        if hasattr(self, "name"):
            return f"<AutomateConnectionInfo {self.name}>"
        elif hasattr(self, "id"):
            return f"<AutomateConnectionInfo {self.id}>"
        else:
            return f"<AutomateConnectionInfo>"

    @classmethod
    def from_graphql(cls, response: Dict):
        """Generate AutomateConnectionInfo instance from GraphQL query result."""
        instance = cls(
            host=response["host"],
            port=response["port"],
            is_tls=response["isTls"],
        )

        return instance


from fastapi import Request, FastAPI
import pandas as pd
import Models
import json
from fastapi.middleware.cors import CORSMiddleware

app = FastAPI()

origins = [
    "http://localhost",
    "http://localhost:3000",
    "http://localhost:8080",
    "http://localhost:8000",

]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

ML_model = Models.SvmModel()

@app.post("/predict")
def make_predictions(prediction_model: Models.DataModel):
    df = pd.DataFrame(prediction_model.dict(),
                      columns=prediction_model.dict().keys(), index=[0])
    result = ML_model.make_predictions(df)
    return result.tolist()

@app.post("/predictAll")
async def predictAll(request_body: Request):
    req_info = await request_body.json()
    json_string = json.dumps([ob for ob in req_info])
    df = pd.read_json(json_string)
    df.columns = Models.columns()

    result = ML_model.make_predictions(df)
    return result.tolist()

@app.get("/")
def read_root():
    return {"Hello": "World"}

# if __name__ == '__main__':
#    uvicorn.run(app, host='0.0.0.0', port=8000)