from common import saveData, loadData, padSpectrogram, getTrainingFiles, getTrainingData, trainClassifier

LOADEXISTINGFEATURES = False

trainingWav = getTrainingFiles(dotted=False, allpitches=False)
trainingWav.sort()

print("\n>> Extracting features\n")
X, additionalX, y = [], [], []
for n, _file in enumerate(trainingWav):
  if LOADEXISTINGFEATURES: continue
  print(f"File {n+1}/{len(trainingWav)} - {_file}")
  localX, localY = getTrainingData(_file, 2, True, "restShortTemp")
  for note in localX:
    X.append(note["spectrogram"])
    additionalX.append([note["pitch"], note["harmonic"], note["harmonicShort"]])
  y += localY
  print()

if LOADEXISTINGFEATURES:
  print(">> Loading existing features")
  X = loadData("models/rest_X.pkl")
  additionalX = loadData("models/rest_additionalX.pkl")
  y = loadData("models/rest_y.pkl")
else:
  print(">> Saving features")
  saveData(X, "models/rest_X.pkl")
  saveData(additionalX, "models/rest_additionalX.pkl")
  saveData(y, "models/rest_y.pkl")

print("\n>> Padding\n")
padded_X = []
for n, sp in enumerate(X):
  print(f"\033[1ANote {n+1}/{len(X)}")
  padded_X.append([additionalX[n]["pitch"], additionalX[n]["harmonic"]] + padSpectrogram(sp) + additionalX[n]["harmonicShort"])

print("\n>> Training")
classifier = trainClassifier(padded_X, y, True, 'rest_spectClassifierShort')

print("\n>> Model saved\n")


import numpy as np

class AMBTC(object):
    def __init__(self, image, m):
        self.image = image
        self.height, self.width = image.shape[:2]
        self.m = m
        self.encoding = self.decoding = None
        ## User Requirement defined 2 is symbolic of 2 bit
        self.n = 2

    def encode(self):
        if self.image.ndim == 3:
            self.encode_rgb()
        else:
            self.encode_gray()

    def decode(self):
        if self.image.ndim == 3:
            self.decode_rgb()
        else:
            self.decode_gray()

    def encode_rgb(self):
        self.height, self.width = self.image.shape[:2]
        enData = []
        for i in range(0, self.height, self.m):
            enData_line = []
            for j in range(0, self.width, self.m):
                enData_dim = []
                for k in range(0, 3):
                    block = self.image[i:i+self.m, j:j+self.m, k]
                    enData_dim.append(self.encoded_block(block))
                enData_line.append(enData_dim)
            enData.append(enData_line)
        self.encoding = enData

    def encode_gray(self):
        self.height, self.width = self.image.shape[:2]
        enData = []
        for i in range(0, self.height, self.m):
            enData_line = []
            for j in range(0, self.width, self.m):
                block = self.image[i:i+self.m, j:j+self.m]
                enData_line.append(self.encoded_block(block))
            enData.append(enData_line)
        self.encoding = enData

    def decode_rgb(self):
        inputData = np.array(self.encoding)
        deData = np.zeros((self.height, self.width, 3), dtype=np.uint8)
        for i in range (0, inputData.shape[0]):
            for j in range (0, inputData.shape[1]):
                for k in range(0, 3):
                    deData[i*self.m:i*self.m+self.m, j*self.m:j*self.m+self.m, k] = self.decoded_block(inputData[i, j, k, 0], inputData[i, j, k, 1], inputData[i, j, k, 2])

        self.decoding = deData

    def decode_gray(self):
        inputData = np.array(self.encoding)
        self.height, self.width = inputData[:, :, 2][0, :].shape[0]*self.m,inputData[:, :, 2][:, 0].shape[0]*self.m
        deData = np.zeros((self.height, self.width), dtype=np.uint8)
        for i in range (0, inputData.shape[0]):
            for j in range (0, inputData.shape[1]):
                deData[i*self.m:i*self.m+self.m, j*self.m:j*self.m+self.m] = self.decoded_block(inputData[i, j, 0], inputData[i, j, 1], inputData[i, j, 2])

        self.decoding = deData

    def encoded_block(self, block):
        Ii = np.average(block)
        Bi = np.array(block > Ii, dtype=np.bool)
        ai = np.around(np.mean((Bi*block)[Bi > 0])).astype(np.uint8)
        bi = np.around(np.mean((~Bi*block)[Bi <= 0])).astype(np.uint8)
        return ai if ai is np.nan else bi, bi, Bi

    def decoded_block(self, ai, bi, Bi):
        return ai*Bi + bi*(~Bi)

    def encode2nBitBlocks(self):
        self.height, self.width = self.image.shape[:2]
        blocks = np.zeros((self.height, self.width), dtype=np.uint8)

        for i in range(0, self.height, self.m):
            for j in range(0, self.width, self.m):
                for k in range(0, 3):
                    blocks[i:i+self.m, j:j+self.m, k] = self.encodeing2nBitBlock(self.image[i:i+self.m, j:j+self.m, k])
        self.encoding_blocks = blocks

    def encodeing2nBitBlock(self, block):
        encoded = self.encoded_block(block)
        block = np.zeros((self.m, self.m), dtype=np.uint8)
        block[0, 0:self.m] = self.split2nBitArray(encoded[0])
        block[1, 0:self.m] = self.split2nBitArray(encoded[1])
        block[2:self.m, 0:self.m] = self.flattenBinaryArray2nBitArray(encoded[2]).reshape(self.m-2, self.m)
        return block

    def split2nBitArray(self, num, bit_size=8):
        num_array = np.zeros(bit_size//self.n, dtype=np.uint8)
        for i in range(bit_size//self.n):
            # num_array[bit_size//nbit-i-1]=(num % 2**nbit)
            num_array[i]=(num % 2**self.n)
            num = num >> self.n
        return num_array

    def flattenBinaryArray2nBitArray(self, arr):
        fArr = arr.flatten()
        print(fArr)
        num_array = np.zeros(len(fArr)//self.n, dtype=np.uint8)
        for i in range (len(fArr)):
            num_array[i//self.n] = num_array[i//self.n]*2 + fArr[i]
        return num_array

Image Recovery/A Novel Tamper Detection and Image Recovery Technique for Color Image Demosaicking/libs/AMBTC.py


import neat
import tqdm
from typing import List, Type
from ..game_env import get_genv
from ..metascorekeeper import Metascorekeeper
from ..metascorekeeper.summer import Summer
from ..scenario import Scenario
from .. import discretizers
from typing import Callable

class Replayer:

    def __init__(self, scenario: Scenario,
                 metascorekeeper: Type[Metascorekeeper],
                 feature_vector: Callable[[dict], List[float]],
                 neat_settings_file: str,
                 discretizer: Type[discretizers.Independent] = None):
        self.scenario = scenario
        self.listeners = []
        self.metascorekeeper = metascorekeeper
        self.feature_vector = feature_vector
        self.neat_settings_file = neat_settings_file
        self.discretizer = discretizer

    def replay(self, genome: neat.DefaultGenome):
        # Create neat config
        config_filename = self.neat_settings_file

        # Setup Neat
        neat_config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
                                  neat.DefaultSpeciesSet, neat.DefaultStagnation,
                                  config_filename)

        self._eval_genome(genome, neat_config)

    def _eval_genome(self, genome: neat.DefaultGenome, config: neat.Config):
        """
        Evaluate a single genome
        :return: The trainer's stats, as a dictionary
        """

        env = get_genv()
        if self.discretizer is not None:
            env = self.discretizer(env)

        scenario = self.scenario

        scorekeeper = scenario.scorekeeper()

        env.load_state(str(scenario.save_state))
        _ = env.reset()
        net = neat.nn.recurrent.RecurrentNetwork.create(genome, config)

        # No buttons pressed in first frame
        next_action = [0] * config.genome_config.num_outputs
        scenario.scorekeeper.env = env

        frames_since_done = 0
        total_frames = 0
        stoppage_frames = 60 * 5

        while not scorekeeper.done or frames_since_done < stoppage_frames:

            if scorekeeper.done:
                frames_since_done += 1

            # Run the next step in the simulation
            step = env.step(next_action)

            # Save the latest state
            info = step[3]
            scorekeeper.info = info

            # Determine the next action so it can be fed into the scorekeeper
            next_action = net.activate(self.feature_vector(info))
            scorekeeper.buttons_pressed = env.action_labels(next_action)

            scorekeeper.tick()

            for listener in self.listeners:
                listener(*step, {'scorekeeper': scorekeeper})
            total_frames += 1

        genome.fitness = scorekeeper.score

        msk = Summer()
        msk.add("Passthru", scorekeeper)
        return scorekeeper.stats, scorekeeper


def linearsearch(arr,element):
    '''
    Using Linear Search to find the element in list.
    Method: iterative

    Parameter
    ---------
    arr: (list)
    A sorted list on which we are going to apply binary search

    element: (any)
    The element which we are searching for in the list

    Returns
    -------
    (int) Index of element if found else None
    '''
    for i in range(len(arr)):
        if arr[i] == element:
            return i
    return None

if __name__ == "__main__":
    l = [1, 2, 54, 56, 63, 64, 75, 84, 86, 543, 987, 2624, 2665]
    print(linearsearch(l,2))


# Definition for an interval.
# class Interval:
#     def __init__(self, s=0, e=0):
#         self.start = s
#         self.end = e

class Solution:
    def merge(self, intervals):
        """
        :type intervals: List[Interval]
        :rtype: List[Interval]
        """
        ret = []
        if not intervals:
            return ret
        intervals.sort(key = lambda x : x.start)
        now =intervals[0]
        for inter in intervals[1:]:
            if inter.start <= now.end:
                now.start = min(inter.start,now.start)
                now.end = max(inter.end,now.end)
            else:
                ret.append(now)
                now = inter
        ret.append(now)
        return ret


from typing import Callable, Tuple

import numpy as np

def posterior_factory(y: np.ndarray, sigma_y: float, sigma_theta: float) -> Tuple[Callable]:
    """The banana distribution is a distribution that exhibits a characteristic
    banana-shaped ridge that resembles the posterior that can emerge from
    models that are not identifiable. The distribution is the posterior of the
    following generative model.

        y ~ Normal(theta[0] + theta[1]**2, sigma_sq_y)
        theta[i] ~ Normal(0, sigma_sq_theta)

    Args:
        y: Observations of the banana model.
        sigma_y: Standard deviation of the observations.
        sigma_theta: Standard deviation of prior over linear coefficients.

    Returns:
        log_posterior: Function to compute the log-posterior.
        grad_log_posterior: Function to compute the gradient of the log-posterior.
        hess_log_posterior: Function to compute the Hessian of the log-posterior.
        metric: Function to compute the Fisher information metric.
        grad_metric: Function to compute the gradient of the Fisher information
            metric.
        hess_metric: Function to compute the Hessian of the Fisher information
            metric.

    """
    sigma_sq_y = np.square(sigma_y)
    sigma_sq_theta = np.square(sigma_theta)

    def log_posterior(theta: np.ndarray) -> float:
        """The banana-shaped distribution posterior.

        Args:
            theta: Linear coefficients.

        Returns:
            out: The log-posterior of the banana-shaped distribution.

        """
        p = theta[0] + np.square(theta[1])
        ll = -0.5 / sigma_sq_y * np.square(y - p).sum()
        lp = -0.5 / sigma_sq_theta * np.square(theta).sum()
        return ll + lp

    def grad_log_posterior(theta: np.ndarray) -> np.ndarray:
        """Gradient of the banana-shaped distribution with respect to the linear
        coefficients.

        Args:
            theta: Linear coefficients.

        Returns:
            out: The gradient of the log-posterior of the banana-shaped
                distribution with respect to the linear coefficients.

        """
        p = theta[0] + np.square(theta[1])
        d = np.sum(y - p)
        ga = d / sigma_sq_y - theta[0] / sigma_sq_theta
        gb = 2.0*d / sigma_sq_y * theta[1] - theta[1] / sigma_sq_theta
        return np.hstack((ga, gb))

    def hess_log_posterior(theta: np.ndarray) -> np.ndarray:
        """Hessian of the banana-shaped distribution with respect to the linear
        coefficients.

        Args:
            theta: Linear coefficients.

        Returns:
            out: The Hessian of the log-posterior of the banana-shaped
                distribution with respect to the linear coefficients.

        """
        n = y.size
        p = theta[0] + np.square(theta[1])
        s = 2.0*n*theta[1] / sigma_sq_y
        H = -metric(theta)
        U = np.array([
            [0.0, 0.0],
            [0.0, 2.0 * (y - p).sum() / sigma_sq_y]
        ])
        H += U
        return H

    def metric(theta: np.ndarray) -> np.ndarray:
        """The Fisher information is the negative expected outer product of the
        gradient of the posterior.

        Args:
            theta: Linear coefficients.

        Returns:
            G: The Fisher information metric of the banana-shaped distribution.

        """
        n = y.size
        s = 2.0*n*theta[1] / sigma_sq_y
        G = np.array([[n / sigma_sq_y + 1.0 / sigma_sq_theta, s],
                      [s, 4.0*n*np.square(theta[1]) / sigma_sq_y + 1.0 / sigma_sq_theta]])
        return G

    def grad_metric(theta: np.ndarray) -> np.ndarray:
        """The gradient of the Fisher information metric with respect to the linear
        coefficients.

        Args:
            theta: Linear coefficients.

        Returns:
            dG: The gradient of the Fisher information metric with respect to the
                linear coefficients.

        """
        n = y.size
        dG = np.array([
            [[0.0, 0.0], [0.0, 2.0*n / sigma_sq_y]],
            [[0.0, 2.0*n / sigma_sq_y], [0.0, 8.0*n*theta[1] / sigma_sq_y]]
        ])
        return dG

    def hess_metric(theta: np.ndarray) -> np.ndarray:
        """The Hessian of the Fisher information metric with respect to the linear
        coefficients.

        Args:
            theta: Linear coefficients.

        Returns:
            ddG: The Hessian of the Fisher information metric with respect to the
                linear coefficients.

        """
        l = 8*y.size / sigma_sq_y
        ddG = np.array([
            0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
            0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, l
        ]).reshape((2, 2, 2, 2))
        return ddG

    def grad_log_posterior_and_metric_and_grad_metric(theta):
        glp = grad_log_posterior(theta)
        G = metric(theta)
        dG = grad_metric(theta)
        return glp, G, dG

    return (
        log_posterior, grad_log_posterior, hess_log_posterior,
        metric, grad_metric, hess_metric,
        grad_log_posterior_and_metric_and_grad_metric)

def generate_data(t: float, sigma_y: float, sigma_theta: float, num_obs: int) -> np.ndarray:
    """Generate data from the banana-shaped posterior distribution.

    Args:
        t: Free-parameter determining the thetas.
        sigma_y: Noise standard deviation.
        sigma_theta: Prior standard deviation over the thetas.
        num_obs: Number of observations to generate.

    Returns:
        theta: Linear coefficients of the banana-shaped distribution.
        y: Observations from the unidentifiable model.

    """
    theta = np.array([t, np.sqrt(1.0 - t)])
    y = theta[0] + np.square(theta[1]) + sigma_y * np.random.normal(size=(num_obs, ))
    return theta, y


# Note: The interpretation of a piece-wise linear model cannot be attacked using 2nd order gradients!!!
import numpy as np
import os
import sys
import torch
from tqdm import tqdm
import matplotlib.pyplot as plt
from explainer_gradient import ExplainerGradient
from attacker_explain import AttackerOfExplain
from utils import evaluation, clip_image_perturb, topk_intersection, rank_correlation

def attack_explanation(model, loaders, args, DEVICE, log_dir, model_dir, flag):
    train_loader, valid_loader, test_loader = loaders
    model.to(DEVICE)
    if os.path.exists(model_dir):
        if flag == 'org':
            filename = os.path.join(model_dir, args.dataset + '_params.pt')
        else:
            filename = os.path.join(model_dir, args.dataset + '_params_advintp.pt')
        if torch.cuda.is_available():
            model.load_state_dict(torch.load(filename))
        else:
            model.load_state_dict(torch.load(filename, map_location='cpu'))
        acc_train = evaluation(model, train_loader, DEVICE)
        acc_valid = evaluation(model, valid_loader, DEVICE)
        acc_test = evaluation(model, test_loader, DEVICE)
        if flag == 'org':
            print('Original model loaded. Train acc: %0.2f, Valid acc: %0.2f, Test acc: %0.2f' %
                  (acc_train, acc_valid, acc_test))
        else:
            print('New model loaded. Train acc: %0.2f, Valid acc: %0.2f, Test acc: %0.2f' %
                  (acc_train, acc_valid, acc_test))
    else:
        sys.exit("Pre-trained model does not exist! Terminate.")

    explainer = None
    if args.intp == 'grad':
        explainer = ExplainerGradient(model)
    intp_attacker = AttackerOfExplain(model)

    metric1 = 0.
    metric2 = 0.
    n_iters = 0
    num_insts = 0
    for i, (images, labels) in tqdm(enumerate(test_loader)):
        images = images.to(DEVICE).requires_grad_()
        labels = labels.to(DEVICE)

        # attack
        images_adv = images + 0.0
        for t in range(args.iter_attack):
            perturb = intp_attacker.attack(images_adv, labels, DEVICE,
                                           attack_type=args.attack_type) * args.perturb_step
            images_adv, diff = clip_image_perturb(images_adv, images, perturb, model, args.epsilon, DEVICE)
            # if t == 50:    # check perturbation magnitude
            #     print(t)
            #     print(torch.max(torch.flatten(diff, start_dim=1), dim=1)[0])

        # evaluate attack
        intp_org = explainer.generate_sensitivemap(images, labels, DEVICE)
        intp_new = explainer.generate_sensitivemap(images_adv, labels, DEVICE)
        metric1 += topk_intersection(intp_org, intp_new, args.K_intersection)
        metric2 += rank_correlation(intp_org, intp_new)

        n_iters += 1
        num_insts += images.shape[0]
        if num_insts >= args.intp_attack_size:
            break
    metric1 /= n_iters
    metric2 /= n_iters
    if flag == 'org':
        print('Original model Top-k intersection: %0.3f; Rank correlation: %0.3f' %
          (metric1, metric2))
    else:
        print('New model Top-k intersection: %0.3f; Rank correlation: %0.3f' %
              (metric1, metric2))

    return metric1, metric2

def attack_explanation_random(model, loader, args, DEVICE, log_dir, model_dir):
    model.to(DEVICE)
    filename = os.path.join(model_dir, 'params.pt')
    if torch.cuda.is_available():
        model.load_state_dict(torch.load(filename))
    else:
        model.load_state_dict(torch.load(filename, map_location='cpu'))

    # iterating into the data
    dataiter = iter(loader)
    images, labels = dataiter.next()

    ID = 4
    img = images[ID:ID+4].to(DEVICE).requires_grad_()  # .requires_grad_()
    label = labels[ID:ID+4].to(DEVICE)
    print(img[0,0,0])
    print(label)

    explainer = None
    if args.intp == 'grad':
        explainer = ExplainerGradient(model)

    intp_attacker = AttackerOfExplain(model)
    img_adv = img + 0.0
    for t in range(args.iter_attack):
        perturb = intp_attacker.attack(img_adv, label, DEVICE) * args.perturb_step
        img_adv, _ = clip_image_perturb(img_adv, img, perturb, model, args.epsilon, DEVICE)
        intp = explainer.generate_sensitivemap(img_adv, label, DEVICE)
        # print(intp_attacker.attack_objective_targeted(intp))

    # Visualization
    fig = plt.figure(figsize=(8, 8))
    # show clean image
    fig.add_subplot(2, 2, 1)
    img_show = np.squeeze(np.transpose(img.detach().cpu().numpy()[0], (1, 2, 0)))
    plt.imshow(img_show)

    # show clean explanation
    intp = explainer.generate_sensitivemap(img, label, DEVICE)
    intp_arr = intp.data.cpu().numpy()[0]         # show the first sample
    intp_arr = np.transpose(intp_arr, (1, 2, 0))  # shape of the image should be height * width * channels
    intp_arr = intp_arr.clip(min=0)
    fig.add_subplot(2, 2, 2)
    plt.imshow(np.abs(np.squeeze(intp_arr)))  # (28, 28, 1) -> (28, 28)

    # show perturbed image
    fig.add_subplot(2, 2, 3)
    img_adv_show = np.squeeze(np.transpose(img_adv.detach().cpu().numpy()[0], (1, 2, 0)))
    plt.imshow(img_adv_show)

    # show perturbed explanation
    intp = explainer.generate_sensitivemap(img_adv, label, DEVICE)
    intp_arr = intp.data.cpu().numpy()[0]
    intp_arr = np.transpose(intp_arr, (1, 2, 0))  # shape of the image should be height * width * channels
    intp_arr = intp_arr.clip(min=0)
    print(-intp[:, :, 10:15, 20:25].sum())
    fig.add_subplot(2, 2, 4)
    plt.imshow(np.abs(np.squeeze(intp_arr)))  # (28, 28, 1) -> (28, 28)

    plt.show()


"""Train lib function."""
import datetime
import functools
import math
import re
import time
from contextlib import nullcontext

# pylint: disable=fixme,import-error,import-outside-toplevel,invalid-name,no-name-in-module,wrong-import-order
import numpy as np
import torch
import torch.distributed as dist
import torch.sagemaker as tsm
import torch.utils.data
import transformers
from accelerate import init_empty_weights
from checkpoints import (
    _CHECKPOINT_DIR_REGEX,
    _DEFAULT_STATE_DICT_TYPE,
    CheckpointingMethod,
    get_coordinator_rank,
    is_action_rank,
    load_checkpoint,
    save_checkpoint,
)
from data.pipelines import GPTDataPipeline, create_data_pipeline
from fsdp_utils import get_backward_fetch_policy, get_sharding_strategy, get_transformer_layer
from logging_utils import (
    create_args_table,
    get_logger,
    log_and_write_eval_metrics,
    log_train_metrics,
    show_env_vars,
    write_metrics_train_step,
)
from memory_tracker import memory_status, memory_status_cpu
from packaging import version as pversion
from torch import optim
from torch.distributed.elastic.multiprocessing.errors import record
from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
from torch.distributed.fsdp import MixedPrecision
from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy, transformer_auto_wrap_policy
from torch.sagemaker import transform
from torch.sagemaker.delayed_param import DelayedParamIniter
from torch.sagemaker.distributed.fsdp.fully_sharded_data_parallel import (
    register_tp_processing_pre_comm_hook,
)
from torch.sagemaker.grad_norm import clip_grad_norm_
from torch.sagemaker.utils import utils as tsm_utils  # pylint: disable=no-name-in-module
from train_utils import (
    apply_activation_checkpoint,
    compute_num_params,
    compute_tflops,
    create_model,
    get_learning_rate_scheduler,
    get_model_config,
    get_param_groups_by_weight_decay,
    patch_neox_rope,
)
from transformers import set_seed

logger = get_logger()

def finetune_with_pretrained_weights_check(args):
    # returns True for start of finetuning only
    return args.hf_pretrained_model_name_or_dir is not None and args.resume_from_checkpoint is None

def finetune_check(args):
    # returns True for start of finetuning as well as resuming
    return args.hf_pretrained_model_name_or_dir is not None

def eval_model(model, data_pipeline, num_batches):
    """Eval step."""
    model = model.eval()
    n_batches = 0
    loss = 0.0

    with torch.no_grad():
        for batch_idx, input_data in enumerate(data_pipeline.val_dataloader):
            input_ids, mask = data_pipeline.get_val_batch(input_data)

            if batch_idx >= num_batches:
                break

            loss += model(input_ids=input_ids, attention_mask=None, labels=input_ids)["loss"]
            n_batches += 1

    if n_batches > 0:
        detached_loss = loss.detach()
        torch.distributed.all_reduce(detached_loss)
        loss = detached_loss.item() / dist.get_world_size()
        loss /= n_batches
        ppl = math.exp(loss)
    else:
        loss = -1.0
        ppl = -1.0

    return loss, ppl

def reduce_loss(loss):
    loss_detached = loss.detach()
    dist.all_reduce(loss_detached)
    loss_scalar = loss_detached.item() / dist.get_world_size()
    return loss_scalar

def train_step(
    args, batch_idx, nvtx_warmup_iters, data_pipeline, input_data, model, optimizer, lr_scheduler
):
    if batch_idx >= nvtx_warmup_iters:
        torch.cuda.nvtx.range_push(f"iteration{batch_idx}")

    input_ids, mask, labels = data_pipeline.get_batch(input_data)

    if batch_idx == 0:
        # checking only on batch 0 to reduce checks during runtime
        assert (
            input_ids.shape[1] == args.max_context_width
        ), f"Input data passed {input_ids.shape} does not respect max_context_width set. Note that this is not strictly necessary, but added to prevent mistakes. If you intend to do this, please remove this check."
        assert (
            input_ids.shape[1] <= args.max_context_width
        ), "Input data passed is larger than max_context_width for model. You need to change max_context_width so model can expect larger sequences"

    optimizer.zero_grad(set_to_none=True)

    torch.cuda.synchronize()
    step_start = time.time()

    if batch_idx >= nvtx_warmup_iters:
        torch.cuda.nvtx.range_push("forward")

    # uses default causal mask
    loss = model(input_ids=input_ids, attention_mask=None, labels=labels)["loss"]

    if batch_idx >= nvtx_warmup_iters:
        # for forward
        torch.cuda.nvtx.range_pop()

    if args.enable_memory_profiling > 0 and batch_idx < 5:
        memory_status_cpu("After_forward_cpu")
        memory_status(msg="After_forward")

    if batch_idx >= nvtx_warmup_iters:
        torch.cuda.nvtx.range_push("backward")

    loss.backward()

    if batch_idx >= nvtx_warmup_iters:
        # for backward
        torch.cuda.nvtx.range_pop()

    if args.enable_memory_profiling > 0 and batch_idx < 5:
        memory_status_cpu("After_train_step_cpu")
        memory_status(msg="After_train_step")

    if batch_idx >= nvtx_warmup_iters:
        torch.cuda.nvtx.range_push("opt_step")

    grad_norm = clip_grad_norm_(model, args.grad_clip)
    optimizer.step()
    lr_scheduler.step()

    if batch_idx >= nvtx_warmup_iters:
        # for opt step
        torch.cuda.nvtx.range_pop()

    if args.clean_cache > 0:
        # empty the cache to avoid OOM
        torch.cuda.empty_cache()

    if batch_idx >= nvtx_warmup_iters:
        # for step
        torch.cuda.nvtx.range_pop()

    torch.cuda.synchronize()
    step_time = time.time() - step_start

    if args.enable_memory_profiling > 0 and batch_idx < 5:
        memory_status(msg="After_opt_step")

    batch_num_sequences = input_ids.shape[0]
    batch_seqlen = input_ids.shape[1]
    return loss, step_time, batch_num_sequences, batch_seqlen, grad_norm

# pylint: disable=no-member,too-many-arguments,too-many-branches,too-many-locals,too-many-statements
def train(
    model,
    optimizer,
    lr_scheduler,
    writers,
    model_config,
    start_epoch,
    start_train_path_index,
    resume_from_sequence_number,
    num_params,
    total_steps,
    args,
    global_rank,
    world_size,
    checkpointing_pg_metadata,
):
    """Train."""
    if args.enable_memory_profiling > 0:
        memory_status_cpu(msg="before train step")

    model.train()
    dp_rank = global_rank
    dp_size = world_size

    if args.tensor_parallel_degree > 1:
        dp_rank //= args.tensor_parallel_degree
        dp_size //= args.tensor_parallel_degree

    if global_rank == 0:
        logger.info("Creating train dataloader")

    throughputs = []
    # Set the same seed for computation
    set_seed(args.seed)

    data_pipeline = create_data_pipeline(
        args, start_train_path_index, resume_from_sequence_number, dp_rank, dp_size
    )
    cur_seq_index = resume_from_sequence_number
    epoch = start_epoch
    while total_steps < args.max_steps:
        nvtx_warmup_iters = 3
        if global_rank == 0:
            logger.info(f"Starting training with epoch {epoch}")

        # additional loop around is for GPTDataset as there can be multiple dataloaders
        if isinstance(data_pipeline, GPTDataPipeline):
            # with new path if incremented at the end of this for loop
            data_pipeline.create_train_dataset()

        for batch_idx, input_data in enumerate(data_pipeline.train_dataloader):
            if total_steps >= args.max_steps:
                break

            if args.profile_nsys > 0 and batch_idx == nvtx_warmup_iters:
                torch.cuda.cudart().cudaProfilerStart()

            loss, step_time, batch_num_sequences, batch_seqlen, grad_norm = train_step(
                args,
                batch_idx,
                nvtx_warmup_iters,
                data_pipeline,
                input_data,
                model,
                optimizer,
                lr_scheduler,
            )
            total_steps += 1
            cur_seq_index += batch_num_sequences
            sample_processed = batch_num_sequences * dp_size
            throughput = sample_processed / step_time
            throughputs.append(throughput)

            tflops_per_gpu = compute_tflops(throughput, num_params, world_size, batch_seqlen)

            if not total_steps % args.logging_freq and args.log_reduced_training_loss > 0:
                loss_scalar = reduce_loss(loss)
            else:
                loss_scalar = loss.item()

            current_lr = lr_scheduler.get_lr()
            display_step = total_steps - 1
            if global_rank == 0:
                write_metrics_train_step(
                    writers,
                    display_step,
                    loss_scalar,
                    throughput,
                    tflops_per_gpu,
                    current_lr,
                    grad_norm,
                )
                if not total_steps % args.logging_freq:
                    log_train_metrics(
                        args,
                        total_steps,
                        display_step,
                        loss_scalar,
                        throughput,
                        tflops_per_gpu,
                        current_lr,
                        grad_norm,
                        throughputs,
                        num_params,
                        dp_size,
                        batch_seqlen,
                    )

            # evaluate on validation
            if args.validation_freq and not total_steps % args.validation_freq:
                cur_state = np.random.get_state()
                torch.cuda.empty_cache()
                val_loss, val_ppl = eval_model(model, data_pipeline, args.validation_batches)
                if global_rank == 0:
                    log_and_write_eval_metrics(writers, display_step, val_loss, val_ppl)
                model = model.train()
                if args.preserve_np_state > 0:
                    np.random.set_state(cur_state)

            # checkpoint
            if not total_steps % args.checkpoint_freq[0]:

                if isinstance(data_pipeline, GPTDataPipeline):
                    save_train_path_index = data_pipeline.cur_train_path
                else:
                    save_train_path_index = 0
                save_train_seq_index = cur_seq_index
                # technically we have processed save_train_seq_index sequences in this file
                # and so index to start from is save_train_seq_index
                user_content = {
                    "cli_args": args.__dict__,
                    "model_config": model_config,
                    "num_params": num_params,
                    "total_steps": total_steps,
                    "epoch": epoch,
                    "start_train_path_index": save_train_path_index,
                    "resume_from_sequence_number": save_train_seq_index,
                }

                subdir = f"{args.model_type}-{total_steps}steps"
                if global_rank == 0 and not re.match(_CHECKPOINT_DIR_REGEX, subdir):
                    raise ValueError(
                        f"Please double check hard-coded checkpoint subdir pattern: `{subdir}` "
                        f"not matching `{_CHECKPOINT_DIR_REGEX}`."
                    )

                if args.enable_memory_profiling > 0:
                    msg = f"({_DEFAULT_STATE_DICT_TYPE})"
                    memory_status(msg=f"Before ckpt @{display_step} {msg}")
                save_checkpoint(
                    model,
                    optimizer,
                    lr_scheduler,
                    user_content,
                    get_sharding_strategy(args.sharding_strategy),
                    args.checkpoint_dir[0],
                    subdir,
                    args.num_kept_checkpoints[0],
                    checkpointing_pg_metadata,
                    tensor_parallel_degree=int(args.tensor_parallel_degree),
                    checkpoint_type=args.checkpoint_type,
                )
                if args.enable_memory_profiling > 0:
                    msg = f"({_DEFAULT_STATE_DICT_TYPE})"
                    memory_status(msg=f"After ckpt  @{display_step} {msg}")

        if isinstance(data_pipeline, GPTDataPipeline):
            incremented_in_epoch = data_pipeline.increment_path_in_epoch()
            if not incremented_in_epoch:
                # path index set to 0
                epoch += 1
        else:
            epoch += 1
    # Using median throughput across all steps, could be more robust.
    return total_steps, np.median(throughputs) if throughputs else 0

@record
def main(args):
    """Main function to train GPT."""
    # Sanity check for args.
    # - Checkpoints.
    # TODO(sliuxl): Supporting one single checkpoint dir now, and multiple dirs support is missing.
    ckpt_lens = (
        len(args.checkpoint_dir),
        len(args.checkpoint_freq),
        len(args.num_kept_checkpoints),
    )
    if len(set(ckpt_lens)) != 1:
        raise ValueError(f"Len mismtach for checkpoint dir, freq vs num to keep:  {ckpt_lens}.")

    if args.distributed_backend == "smddp":
        import smdistributed.dataparallel.torch.torch_smddp  # pylint: disable=unused-import

    dist.init_process_group(args.distributed_backend, timeout=datetime.timedelta(seconds=7200))
    global_rank = dist.get_rank()
    device = global_rank % torch.cuda.device_count()
    world_size = dist.get_world_size()

    if args.tensorboard_dir and global_rank == 0:
        from torch.utils.tensorboard import SummaryWriter

        logger.info("Writing metrics for tensorboard to %s.", args.tensorboard_dir)
        writers = tuple(SummaryWriter(log_dir=tb_dir) for tb_dir in args.tensorboard_dir)
        table_str = create_args_table(args.__dict__)
        for writer in writers:
            writer.add_text("Arguments", table_str)
    else:
        writers = ()

    smp_config_dict = {
        "activation_loading_horizon": args.activation_loading_horizon,
        "sm_activation_offloading": args.offload_activations > 0,
    }
    if args.shard_degree is not None:
        smp_config_dict["hybrid_shard_degree"] = args.shard_degree

    smp_config_dict["tensor_parallel_degree"] = args.tensor_parallel_degree
    if args.tensor_parallel_degree > 1 and args.use_smp_implementation < 1:
        args.use_smp_implementation = 1
        if global_rank == 0:
            logger.info(
                f"Tensor parallelism (TP) is enabled as tensor_parallel_degree is set to {args.tensor_parallel_degree} (>1). Switching use_smp_implementation to 1 so we can use SMP optimized implementation for TP."
            )
    if args.use_smp_implementation:
        # For our Mem usage fix to TE, this needs to be True
        args.use_orig_params = 1

    if args.use_synthetic_data and args.validation_freq is not None:
        # Overriding validation freq to None as synthetic data
        args.validation_freq = None

    tsm.init(smp_config_dict)
    show_env_vars(0)

    if global_rank == 0:
        for index, (key, value) in enumerate(sorted(args.__dict__.items()), 1):
            logger.info("Arguments [%03d/%03d] %-30s: %s", index, len(args.__dict__), key, value)
        logger.info("Transformers version: %s", transformers.__version__)
        logger.info("World size = %d: # nodes = %d.", world_size, world_size / 8)

        gbs = (
            world_size
            * args.max_context_width
            * args.train_batch_size
            / args.tensor_parallel_degree
        )
        logger.info("Global batch size in tokens: %10d (%5.2fM).", gbs, gbs / 1e6)

    set_seed(args.seed)

    if args.enable_memory_profiling > 0:
        memory_status_cpu(msg="before model creation")

    if args.bf16:
        dtype = torch.bfloat16
    else:
        dtype = torch.get_default_dtype()

    if finetune_check(args):
        from transformers import AutoConfig

        # Using config for finetune mode, else uses args to create model
        model_config = AutoConfig.from_pretrained(args.hf_pretrained_model_name_or_dir)
    else:
        model_config = get_model_config(args)

    if pversion.parse(transformers.__version__) >= pversion.parse("4.34.0"):
        model_config._flash_attn_2_enabled = True

    delayed_param_initer = None
    with tsm_utils.timeit(True, "Model creation", global_rank):
        if args.delayed_param:
            if finetune_with_pretrained_weights_check(args) and dist.get_rank() == 0:
                # create model with pretrained weights on one rank even if we want to use
                # delayed param, param init on other ranks will still be delayed
                model = create_model(
                    args,
                    model_config=model_config,
                    dtype=dtype,
                    pretrained_model_weights=args.hf_pretrained_model_name_or_dir
                    if finetune_with_pretrained_weights_check(args)
                    else None,
                )
                num_params = compute_num_params(model)
            else:
                with init_empty_weights():
                    model = create_model(
                        args,
                        model_config=model_config,
                        dtype=dtype,
                    )
                num_params = compute_num_params(model)
            if finetune_check(args):
                dist.barrier()
        else:
            model = create_model(
                args,
                model_config=model_config,
                dtype=dtype,
                pretrained_model_weights=args.hf_pretrained_model_name_or_dir
                if finetune_with_pretrained_weights_check(args) and dist.get_rank() == 0
                else None,
            )
            num_params = compute_num_params(model)

        if args.use_smp_implementation:
            load_state_dict_from_rank0 = (
                True if finetune_with_pretrained_weights_check(args) else False
            )
            model = transform(model, load_state_dict_from_rank0=load_state_dict_from_rank0)

        if args.delayed_param:
            # param init fn for delayed param creation
            if finetune_check(args):
                if dist.get_rank() != 0:
                    delayed_param_initer = DelayedParamIniter(model)
            else:
                delayed_param_initer = DelayedParamIniter(model)

    assert set(x.dtype for x in model.parameters()) == set(
        [torch.float32]
    ), "Model parameters should be in fp32 for FSDP mixed precision"

    start = time.time()

    if global_rank == 0:
        logger.info(
            "Created model with total parameters: %d (%.2f B)", num_params, num_params * 1e-9
        )

    transformer_layer = get_transformer_layer(args.model_type, args.use_smp_implementation)

    if args.auto_wrap_policy == "transformer_auto_wrap_policy":
        gpt_auto_wrap_policy = functools.partial(
            transformer_auto_wrap_policy,
            transformer_layer_cls={
                transformer_layer,
            },
        )
    elif args.auto_wrap_policy == "size_based_auto_wrap_policy":
        gpt_auto_wrap_policy = functools.partial(
            size_based_auto_wrap_policy,
        )

    torch.cuda.set_device(device)
    if args.bf16:
        # buffer set to fp32 as some models in HF such as llama hard code buffers to fp32
        # to be similar with that we set this to fp32
        buffer_dtype = torch.float32 if args.use_smp_implementation else dtype
        mixed_precision_policy = MixedPrecision(
            param_dtype=dtype, reduce_dtype=dtype, buffer_dtype=buffer_dtype
        )
    else:
        mixed_precision_policy = None

    if args.enable_memory_profiling > 0:
        memory_status_cpu(msg="before fsdp wrapper")

    sharding_strategy = get_sharding_strategy(args.sharding_strategy)

    with (
        delayed_param_initer.validate_params_and_buffers_inited()
        if (delayed_param_initer and not finetune_with_pretrained_weights_check(args))
        else nullcontext(),
        tsm_utils.timeit(True, "FSDP constructor", global_rank),
    ):
        model = FSDP(  # pylint: disable=unexpected-keyword-arg
            model,
            auto_wrap_policy=gpt_auto_wrap_policy,
            mixed_precision=mixed_precision_policy,
            sharding_strategy=sharding_strategy,
            backward_prefetch=get_backward_fetch_policy(args.backward_fetch_policy),
            forward_prefetch=args.forward_prefetch,
            limit_all_gathers=args.limit_all_gathers,
            device_id=torch.cuda.current_device(),
            use_orig_params=args.use_orig_params > 0,
            param_init_fn=delayed_param_initer.get_param_init_fn()
            if delayed_param_initer
            else None,
            post_param_init_fn=delayed_param_initer.get_post_param_init_fn()
            if delayed_param_initer
            else None,
            sync_module_states=True if finetune_with_pretrained_weights_check(args) else False,
        )
    # Barrier is a workaround to reduce extra memory usage with SMDDP backend
    # after the broadcast that happens when we use sync_module_states
    # This can be removed once the SMDDP issue is fixed
    dist.barrier()
    memory_status(msg="After FSDP")

    if global_rank == 0:
        logger.info("Wrapped model with FSDP")

    if args.enable_memory_profiling > 0:
        memory_status(msg="after fsdp wrapper")

    if args.activation_checkpointing > 0:
        apply_activation_checkpoint(args, model=model)

    if args.offload_activations > 0:
        from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import offload_wrapper

        model = offload_wrapper(model)

        # Patch RoPE for GPT NEoX where they are created on Host to move them to Device
        if args.model_type == "gpt_neox" and args.patch_neox_rope > 0:
            patch_neox_rope(model)

    param_groups = get_param_groups_by_weight_decay(model)

    optimizer = optim.AdamW(
        param_groups, betas=(args.beta1, args.beta2), lr=args.lr, weight_decay=args.weight_decay
    )

    if global_rank == 0:
        logger.info("Created optimizer")

    lr_scheduler = get_learning_rate_scheduler(optimizer, args)

    checkpointing_pg_metadata = (
        model.process_group,
        get_coordinator_rank(model.process_group),
        is_action_rank(global_rank),
    )

    if args.resume_from_checkpoint:
        (
            model,
            optimizer,
            lr_scheduler,
            epoch,
            total_steps,
            start_train_path_index,
            resume_from_sequence_number,
        ) = load_checkpoint(
            args,
            model,
            optimizer,
            lr_scheduler,
            args.resume_from_checkpoint,
            sharding_strategy,
            checkpointing_pg_metadata,
            tensor_parallel_degree=int(args.tensor_parallel_degree),
            checkpoint_type=args.checkpoint_type,
        )

    else:
        total_steps = 0
        epoch = 0
        start_train_path_index = 0
        resume_from_sequence_number = 0

    start = time.time()
    # total_steps, throughput, loss
    total_steps, _ = train(
        model,
        optimizer,
        lr_scheduler,
        writers,
        model_config,
        epoch,
        start_train_path_index,
        resume_from_sequence_number,
        num_params,
        total_steps,
        args,
        global_rank,
        world_size,
        checkpointing_pg_metadata,
    )
    time_to_train = time.time() - start

    dist.barrier()

    if args.save_final_model:
        save_checkpoint(
            model,
            None,
            None,
            {"model_config": model_config},
            None,
            args.model_dir if args.model_dir is not None else args.checkpoint_dir[0],
            "" if args.model_dir is not None else "model",
            1,
            None,
            int(args.tensor_parallel_degree),
            checkpoint_type=CheckpointingMethod.FULL,
        )

    if global_rank == 0:
        logger.info(
            "FSDP training finished successfully %fs (%fmin).", time_to_train, time_to_train / 60.0
        )
    dist.destroy_process_group()


import pandas as pd
import json
from os import system

filename = "database.json"

def judul():
    print('=====================================')
    print('|    PROGRAM INPUT DATA MAHASISWA   |')
    print('=====================================')

def dosen():
    system('cls')
    print('=====================================')
    print('|               Login               |')
    print('=====================================')
    print('Masukkan kode Login')
    print('\n')
    kode = input('Masuk : ')
    if kode == 'admin' or kode == 'ADMIN':
        menu_dosen()
    else:
        salah = input('Kode salah')
        dosen()

def menu_dosen():
    system('cls')
    print('=====================================')
    print('Input Data Nilai Mahasiswa'.center(40))
    print('=====================================')
    print('| 1. Tambah Data Mahasiswa          |')
    print('| 2. Lihat Data Mahasiswa           |')
    print('| 3. Ubah Data Mahasiswa            |')
    print('| 4. Hapus Data Mahasiswa           |')
    print('| 5. Selesai                        |')
    print('=====================================')
    pilih2 = input('Pilih Menu : ')
    if pilih2 == '1':
        tambah()
    elif pilih2 == '2':
        lihat()
    elif pilih2 == '3':
        ubah()
    elif pilih2 == '4':
        hapus()
    elif pilih2 == '5':
        selesai()
    else:
        tidak = input('Menu Tidak Ada ')
        system('cls')
        menu_dosen()

def tambah():
    system('cls')
    judul()
    print('Tambah Data Mahasiswa'.center(40))
    print('=====================================')
    with open(filename, "r") as f:
        temp = json.load(f)
    item_data = {}
    item_data["Nim"] = input('Masukkan Nim: ')
    item_data["Nama"] = input('Masukkan nama: ')
    item_data["Jurusan"] = input('Masukkan Jurusan: ')
    item_data["Kelas"] = input('Masukkan Kelas: ')
    item_data["Uts"] = input('Masukkan nilai Uts: ')
    item_data["Uas"] = input('Masukkan nilai Uas: ')
    temp.append(item_data)
    with open (filename, "w") as f:
        json.dump(temp, f, indent=4)
    print('INI ADALAH DAFTAR HASIL TAMBAH DATA MAHASISWA')
    df = pd.read_json("database.json")
    print(df)
    pertanyaan = input('Apakah masih ingin menambah data mahasiswa :[Y/N] ')
    while pertanyaan != 'N':
        tambah()
    kembali = input('Tekan enter untuk kembali [enter]')
    menu_dosen()

def view():
    system('cls')
    judul()
    df = pd.read_json("database.json")
    print(df)

def ubah():
    system('cls')
    print('Ubah Data Mahasiswa'.center(40))
    print('=====================================')
    judul()
    view()
    new_data = []
    with open(filename, "r") as f:
        temp = json.load(f)
        data_length = len(temp)-1
    print('*tekan [Enter] untuk kembali*')
    ubah_data = input(f'Pilih nomor yang ingin diubah 0-{data_length}: ')
    if ubah_data == '':
        menu_dosen()
    i= 0
    for entry in temp:
        if i == int(ubah_data):
            print('''
            nama - ubah nama
            nim - ubah nim
            jurusan - ubah jurusan
            kelas - ubah kode kelas
            uts - ubah nilai uts
            uas - ubah nilai uas
            all - ubah semua data\n''')
            pertanyaan = input('Apa yang ingin anda rubah? : ')
            nama = entry["Nama"]
            nim = entry["Nim"]
            jurusan = entry["Jurusan"]
            kelas = entry["Kelas"]
            uts = entry["Uts"]
            uas = entry["Uas"]

            if pertanyaan == 'nama':
                nama = input('Masukkan nama baru : ')
            elif pertanyaan == 'nim':
                nim = input('Masukkan Nim baru : ')
            elif pertanyaan == 'jurusan':
                jurusan = input('Masukkan jurusan baru : ')
            elif pertanyaan == 'kelas':
                kelas = input('Masukkan kode kelas baru : ')
            elif pertanyaan == 'uts':
                uts = input('Masukkan nilai uts baru : ')
            elif pertanyaan == 'uas':
                uas = input('Masukkan nilai uas baru : ')
            elif pertanyaan == 'all':
                nim = input('Masukkan Nim baru : ')
                nama = input('Masukkan nama baru : ')
                jurusan = input('Masukkan jurusan baru : ')
                kelas = input('Masukkan kode kelas baru : ')
                uts = input('Masukkan nilai uts baru : ')
                uas = input('Masukkan nilai uas baru : ')
            else:
                print('Masukkan sesuai keyword yang tersedia\n')
                a = input('Tekan enter untuk melanjutkan [ENTER]')

                ubah()

            new_data.append({ "Nim":nim,"Nama": nama, "Jurusan": jurusan, "Kelas": kelas,"Uts": uts, "Uas": uas})
            i = i+1
        else:
            new_data.append(entry)
            i = i+1
    with open (filename, "w") as f:
        json.dump(new_data, f, indent=4)
    system('cls')
    print('INI ADALAH DAFTAR HASIL UPDATE DATA MAHASISWA')
    df = pd.read_json("database.json")
    print(df)
    pertanyaan = input('Apakah ingin mengubah data lagi?[Y/N]')
    while pertanyaan != 'N':
        ubah()
    a = input('Tekan enter untuk kembali [ENTER]')
    menu_dosen()

def hapus():
    system('cls')
    judul()
    print('Penghapusan Data Mahasiswa'.center(40))
    print('=====================================')
    view()
    new_data = []
    with open(filename, "r") as f:
        temp = json.load(f)
        data_length = len(temp)-1
    hapus_data = input('Masukkan nomor untuk hapus: ')
    y = 0
    for entry in temp:
        if y == int(hapus_data):
            pass
            y=y+1
        else:
            new_data.append(entry)
            y=y+1
    with open (filename, "w") as f:
        json.dump(new_data, f, indent=4)
        print('INI ADALAH DAFTAR HASIL HAPUS DATA MAHASISWA')
    df = pd.read_json("database.json")
    print(df)
    a = input('Tekan enter untuk kembali [ENTER]')
    menu_dosen()

def lihat():
    system('cls')
    print('DATA MAHASISWA'.center(40))
    print('================================================================================')
    df = pd.read_json("database.json")
    print(df)

    a = input('Tekan enter untuk kembali [ENTER]')
    menu_dosen()

def selesai():
    exit

dosen()


# Import path method from pathlib
from pathlib import Path
# Import csv module
import csv
# Import module for regex
import re

# Instantiate file path object to Cash_on_Hand.csv file under csv_reports folder
file_path = Path.cwd()/"csv_reports"/"Cash on Hand.csv"
# Instantiate file patch object to summary_report.txt
summary_path = Path.cwd()/"summary_report.txt"

# Creating a function called cash_on_hand_function
def cash_on_hand_function(forex):
   """
   - This function highlights the days where cash on hand was lower than the previous day and also shows the value difference as a deficit
   """
   # Open file and read the file
   # Newline parameter prevents extra line in csv file
   # File is a variable assigned to file after opening and reading
   with file_path.open(mode = "r", encoding = "UTF-8", newline = "") as file:

        # To skip the header
        next(file)
        # Assigning value of 0 to previous_day and value_difference
        previous_day = 0
        value_difference = 0

        # Created nested loop to access all values in the file
        for line in file.readlines():

            line = re.findall(pattern = r"[0-9][0-9]+", string = line)
            # Subtract values from the current day with previous day to find the difference of cash in hand
            value_difference = float(line[1]) - previous_day
            # Cash on hand for previous day
            previous_day = float(line[1])
            # If statement runs code when difference is a negative value

            if value_difference < 0:

                # Used abs() to always return a positive number
                # Value_difference is multiplied by the forex for current price
                value_difference = round(abs(value_difference) * forex, 1)

                # To open summary_path and append data to file
                with summary_path.open(mode = "a", encoding = "UTF-8", newline = "" ) as file:

                    # .writelines() to write data into summary_path
                    file.writelines("\n[CASH DEFICIT]"+" DAY: "+str(round(float(line[0]),1))+", AMOUNT: SGD"+str(value_difference))

            # Use elif to only execute the code the difference is 0 or more
            elif value_difference == 0:
                # Open summary_path and append
                with summary_path.open(mode = "a", encoding="UTF-8", newline = "") as file:
                    # Use .writelines() to write the data into summary_path
                    file.writelines("\n[CASH SURPLUS] CASH ON HAND ON EACH DAY IS HIGHER THAN THE PREVIOUS DAY")

        # Close files
        file.close()


def voc_as_mask(label, class_id):
    # copied from https://github.com/facebookresearch/TorchRay/blob/6a198ee61d229360a3def590410378d2ed6f1f06/torchray/benchmark/datasets.py#L98
    """Convert a VOC detection label to a mask.

    Return a boolean mask selecting the region contained in the bounding boxes
    of :attr:`class_id`.

    Args:
        label (dict): an image label in the VOC detection format.
        class_id (int): ID of the requested class.

    Returns:
        :class:`torch.Tensor`: 2D boolean tensor.
    """
    weight, height = voc_as_image_size(label)
    mask = torch.zeros((height, weight), dtype=torch.uint8)
    objs = label['annotation']['object']
    if not isinstance(objs, list):
        objs = [objs]
    for obj in objs:
        this_class_id = _VOC_CLASS_TO_INDEX[obj['name']]
        if this_class_id != class_id:
            continue
        bbox = obj['bndbox']
        ymin = int(bbox['ymin']) - 1
        ymax = int(bbox['ymax']) - 1
        xmin = int(bbox['xmin']) - 1
        xmax = int(bbox['xmax']) - 1
        mask[ymin:ymax + 1, xmin:xmax + 1] = 1

    if version.parse(torch.__version__) >= version.parse("1.2.0"):
        mask = mask.to(torch.bool)
    return mask
def evaluate(self, mask, point):
    # copied from https://github.com/facebookresearch/TorchRay/blob/6a198ee61d229360a3def590410378d2ed6f1f06/torchray/benchmark/pointing_game.py#LL58C11-L58C11
    r"""Evaluate a point prediction.

    The function tests whether the prediction :attr:`point` is within a
    certain tolerance of the object ground-truth region :attr:`mask`
    expressed as a boolean occupancy map.

    Use the :func:`reset` method to clear all counters.

    Args:
        mask (:class:`torch.Tensor`): :math:`\{0,1\}^{H\times W}`.
        point (tuple of ints): predicted point :math:`(u,v)`.

    Returns:
        int: +1 if the point hits the object; otherwise -1.
    """
    # Get an acceptance region around the point. There is a hit whenever
    # the acceptance region collides with the class mask.
    v, u = torch.meshgrid((
        (torch.arange(mask.shape[0],
                        dtype=torch.float32) - point[1])**2,
        (torch.arange(mask.shape[1],
                        dtype=torch.float32) - point[0])**2,
    ))
    accept = (v + u) < self.tolerance**2

    # Test for a hit with the corresponding class.
    hit = (mask & accept).view(-1).any()

    return +1 if hit else -1


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

#configurar gráficos
font_title = {'family': 'DejaVu Sans',
        'color':  'white',
        'weight': 'bold',
        'size': 26,
        }
plt.style.use('dark_background')

#importar dataframe

df = pd.read_json("03092020_2kcomments.json")

#EDA

#lowercases

df = df.applymap(lambda x: x.lower())

import string

#separar em lista e retirar pontuação
retirarPontos = lambda x: x.lower().replace(",","").replace(".","").replace(")","").replace("(","").replace("!","").replace("?","").replace("-"," ").replace("_"," ").replace('"','').replace("'","").split()

df = df.applymap(retirarPontos).applymap(lambda x: " ".join(x))

#contar palavras

df["tamanho"] = df[0].apply(lambda x: len(x))

#wordcloud

from wordcloud import WordCloud

wordcloud = WordCloud(width=3000,height=2000,background_color="black",stopwords=stopwords).generate(str(df[0].values).replace("'",""))

#plotar o wordcloud

#fig = plt.figure(figsize = (40, 30), facecolor = 'k', edgecolor = 'k')
#plt.imshow(wordcloud, interpolation = 'bilinear')
#plt.axis('off')
#plt.tight_layout(pad=0)
#fig.savefig("wordcloud.png")

#verificar distribuição de frequências e correlação das palavras para cada texto
#definir stopwords

from nltk.corpus import stopwords
stopwords = set(stopwords.words("portuguese"))

stopRemove = lambda x: [palavra for palavra in x.split(" ") if palavra not in stopwords]

df["sem stopwords"] = df[0].apply(stopRemove)
#df["sem stopwords"] = df["sem stopwords"].apply(lambda x: " ".join(x))

#Criar distribuição de frequencias, juntar todas strings em uma só string

stringTotal = []
for i, x in df.iterrows():
    stringTotal = stringTotal + x["sem stopwords"]

#stringTotal = " ".join(stringTotal)

from nltk.probability import FreqDist
from nltk.probability import ProbDistI

Freq = FreqDist(stringTotal)
#plt.title("Distribuição de Frequências de Ocorrência das palavras")

fig2 = plt.figure(figsize = (40,30))
Freq.plot(30,cumulative=False,color="red")
fig2.savefig("teste.png")

#Tamanho das frases -

tamanho = df[0]

tamanho = tamanho.apply(lambda x: len(x.split(" ")))
tamanho.plot(cmap="viridis")

#analise

#polaridade

from textblob import TextBlob

#Analisar sentimento
df["polaridade"] = df["sem stopwords"].apply(lambda x: TextBlob(" ".join(x)).sentiment.polarity)

#criar serie para polaridade neutra, positiva, negativa

df["status"]=df["polaridade"].apply(lambda x: "neutro" if type(x)==np.float and x==0 else x)
df["status"]=df["status"].apply(lambda x: "negativo" if type(x)==np.float and x<0 else x)
df["status"]=df["status"].apply(lambda x: "positivo" if type(x)==np.float and x>0 else x)

#analisar wordcloud de cada sentiment

wordcloudPositivo = WordCloud(width=3000,height=2000,background_color="black",stopwords=stopwords).generate(str(df[df["status"]=="positivo"][0].values).replace("'",""))
""" fig1 = plt.figure(figsize = (40, 30), facecolor = 'k', edgecolor = 'k')
plt.imshow(wordcloudPositivo, interpolation = 'bilinear')
plt.axis('off')
plt.title("Wordcloud - positivo")
plt.tight_layout(pad=0)
fig1.savefig("positivoWC.png")

wordcloudNeutro = WordCloud(width=3000,height=2000,background_color="black",stopwords=stopwords).generate(str(df[df["status"]=="neutro"][0].values).replace("'",""))
fig2 = plt.figure(figsize = (40, 30), facecolor = 'k', edgecolor = 'k')
plt.imshow(wordcloudNeutro, interpolation = 'bilinear')
plt.axis('off')
plt.title("Wordcloud - positivo")
plt.tight_layout(pad=0)
fig2.savefig("neutroWC.png")

wordcloudNegativo = WordCloud(width=3000,height=2000,background_color="black",stopwords=stopwords).generate(str(df[df["status"]=="negativo"][0].values).replace("'",""))
fig3 = plt.figure(figsize = (40, 30), facecolor = 'k', edgecolor = 'k')
plt.imshow(wordcloudNegativo, interpolation = 'bilinear')
plt.axis('off')
plt.title("Wordcloud - positivo")
plt.tight_layout(pad=0)
fig3.savefig("negativoWC.png") """

#Remover palavras que se repetem nos três diagramas

stringTotalPositivo, stringTotalNeutro, stringTotalNegativo = [],[],[]

#analisar tamanho do texto X polaridade

def inList(list,element):
    try:
        list.index(element)
    except ValueError:
        return False
    else:
        return True

#verificar se há relação entre o tamanho das frases e o sentimento

sns.pairplot(df,hue="status",palette="husl")
plt.savefig("ParPlotSemFiltrar.png")

#muitos resultados neutros, descobri que o TextBlob não funciona muito bem para português
#traduzir para inglês
#from googletrans import Translator

#translator = Translator(service_urls=["translate.google.com"])
#translator.translate("string",dest="lan")
#translator.detect("text in here")


from yolov3 import utils
from config import *

# jeżeli programy mają działać oddzielnie to wywołaj odpowiednie metody
# Działanie oddzielne YOLO zapisuje rekordy do pliku json a następnie SLAM odczytuje dane z pliku
# i odpowiednio porównuje ze swoimi wynikami. format pliku json:
# {frame : nr [{obj : name ; x_min : value; y_min : value; x_max : value; y_max : value]},

# działanie jednoczesne
# zwracany jest obiek YOLO do Programu SLAM i rozpoznawanie obiektów jednoczeńnie.

def YoloWork(save_to_file):
    if save_to_file:
        print("zapisuje dane do pliku json")
        # zapisuje rekordy do pliku json
    else:
        return utils.detect_image()


import nltk

with open('text.txt', 'r') as f:
    sample = f.read()

sentences = nltk.sent_tokenize(sample)
tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
# if set to True, named entity is only tagged as NE, otherwise the classifier add labels onto every NE
# All Tags are in https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html
chunked_sentences = nltk.ne_chunk_sents(tagged_sentences, binary=True)

def extract_entity_names(t):
    entity_names = []

    if hasattr(t, 'label') and t.label():
        if t.label()=='NE':
            entity_names.append(' '.join([child[0] for child in t]))
        else:
            for child in t:
                entity_names.extend(extract_entity_names(child))

    return entity_names

entity_names = []
for tree in chunked_sentences:
    entity_names.extend(extract_entity_names(tree))

print(sample)
print(set(entity_names))


# 0-1背包问题
"""
给你一个可装载重量为W的背包和N个物品，每个物品有重量和价值两个属性。其中第i个物品的重量为wt[i]，价值为val[i]，现在让你用这个背包装物品，最多能装的价值是多少？

举个简单的例子，输入如下：

N = 3, W = 4
wt = [2, 1, 3]
val = [4, 2, 3]

输出：6
"""

def solve(weights, values, n, w):
    dp = [[0 for _ in range(w+1)] for _ in range(n+1)]
    for i in range(1, n+1):
        for j in range(1, w+1):
            if j - weights[i-1] < 0:
                # 当前背包容量装不下，只能选择不装入背包
                dp[i][j] = dp[i-1][j]
            else:
                # 装入或者不装入背包，择优
                dp[i][j] = max(dp[i-1][j-weights[i-1]] + values[i-1],
                               dp[i-1][j])
    return dp[-1][-1]

if __name__ == '__main__':
    weights = [2, 1, 3]
    values = [4, 2, 3]
    print(solve(weights, values, 3, 4))


from time import time
from collections import defaultdict
from collections import deque
from collections import namedtuple
from os.path import basename
from os.path import splitext
from logging import getLogger

from . import pathmaker
from .archive import ArchiveFile
from .archive import FileNotFoundError
from .exception import BadArchiveFile
from .exception import UnsupportedArchiveFile

logger = getLogger(__name__)

# XXX the i prefix here means archive internal, not for mapper.
FileEntry = namedtuple(
    'FileEntry', ['archive_path', 'ifilename', 'ifile_size'])

class DefaultMapper(object):
    """
    Mapper that tracks the nested structure within archive files.
    """

    def __init__(self, path=None, pathmaker_name='default', _pathmaker=None,
            overwrite=False, include_arcname=False, splitext_arcname=False):
        """
        Initialize the mapping, optionally with a path to an archive
        file.

        Mapping dict keys are names of file or directory, values are
        either a tuple that represent a file, or a dict to represent a
        directory.
        """

        self.include_arcname = include_arcname
        self.splitext_arcname = splitext_arcname
        self.overwrite = overwrite
        if _pathmaker:
            self.pathmaker = _pathmaker
        else:
            self.pathmaker = getattr(pathmaker, pathmaker_name)()

        # The actual filesystem mapping
        self.mapping = {}
        # a mapping with keys of generated paths against source archive.
        # its keys includes a map of directory to its source archive.
        self.reverse_mapping = defaultdict(deque)
        # Tracked file added timestamps (see _load_infolist)
        self.archives = {}
        # A flattened mapping of archive to its list of internal entries
        # including directory entries.
        self.archive_ifilenames = {}

        if path:
            self.load_archive(path)

    def mkdir(self, path_fragments):
        """
        Creates the dir entries identified by path if not already exists
        and return the complete directory.
        """

        # set current to root node
        current = self.mapping

        for c, frag in enumerate(path_fragments):
            if frag in current:
                current = current[frag]
                if not isinstance(current, dict):
                    raise ValueError(
                        'cannot create directory `%(filename)s` at '
                        '`%(path)s/`: file entry exists.' % {
                            'filename': frag,
                            'path': '/'.join(path_fragments[:c]),
                        }
                    )
            else:
                # create directory dict entry and set current.
                current[frag] = current = {}

        return current

    def traverse(self, path):
        """
        Traverse to path, or return the entry identified by path.
        """

        path_fragments = path and path.split('/') or []
        return self._traverse(path_fragments)

    def _traverse(self, path_fragments):
        current = self.mapping

        for frag in path_fragments:
            if not isinstance(current, dict) or frag not in current:
                # No such frag in dir.
                return None
            current = current[frag]

        return current

    def _load_infolist(self, archive_path, infolist):
        self.archives[archive_path] = time()
        archive_name = (
            splitext(basename(archive_path))[0]
            if self.splitext_arcname else
            basename(archive_path)
        ) + '/'
        self.archive_ifilenames[archive_path] = i_filenames = []

        for info in infolist:
            if self.include_arcname:
                raw_filename = archive_name + info.filename
            else:
                raw_filename = info.filename
            frags, filename = self.pathmaker(raw_filename)

            # the internal filename.
            ifilename = '/'.join([] + frags + [filename])

            fentry = FileEntry(archive_path, info.filename, info.file_size)

            # These may appear redundant compared to the ones below, but
            # do note these are always added despite conflict that will
            # appear at the filesystem presentation level.  This is used
            # for locating current zip files for regeneration if an
            # archive was removed from this mapping.
            self.reverse_mapping[ifilename].append(fentry)
            i_filenames.append(ifilename)

            try:
                target = self.mkdir(frags)
            except ValueError as e:
                # using info.filename rather than raw_filename because
                # the message is provided by the exception generated by
                # self.mkdir.
                logger.warning(
                    '`%s` could not be created: %s', info.filename, e.args[0])
                continue

            if not filename:
                # was a directory entry
                continue

            if filename in target:
                if not self.overwrite:
                    logger.info('`%s` already exists; ignoring', info.filename)
                    continue
            target[filename] = fentry

    def _unload_functions(self):
        """
        Return a tuple of needed functions for unloading an archive out
        of the mappings.
        """

        if self.overwrite:
            # check the rightmost (newest) item, pop from right
            return -1, deque.pop
        else:
            # check the leftmost (oldest) item, pop from left
            return 0, deque.popleft

    def _unload_infolist(self, archive_path):
        # pop this out right away to mark this as to be pruned off.
        ifilenames = self.archive_ifilenames.pop(archive_path)
        index, pop = self._unload_functions()
        all_frags = set()
        for ifilename in ifilenames:
            # lookup via the reverse mapping to see that this ifilename
            # is the active check that the current active
            fentries = self.reverse_mapping[ifilename]

            if fentries[index].archive_path != archive_path:
                # leave the reverse mapping in place.
                continue

            original_filename = fentries[index].ifilename

            archive_ifilenames = self.archive_ifilenames
            # We have a match, time to clean up to the latest fresh and
            # active entry that other previous calls to this function
            # have left behind.
            fentry_replacement = None
            try:
                while fentries[index].archive_path not in archive_ifilenames:
                    pop(fentries)
                else:
                    # Successfully left the loop, the fileentry is now
                    # the replacement.
                    fentry_replacement = fentries[index]
            except IndexError:
                # This no longer exists in any active archive.
                self.reverse_mapping.pop(ifilename)

            # We have an ifilename created in _load_infolist, invert
            # operation to derive filename and frags.
            raw = ifilename.split('/')
            filename = raw[-1]
            frags = raw[:-1]

            if frags:
                all_frags.add(tuple(frags))

            info = self._traverse(frags)
            if not info is None:
                # The file's directory may not have been added to
                # self.mapping, if its creation may have been blocked
                # by another file.
                if filename in info:
                    # Only remove, for the same reason as above.
                    if (original_filename.endswith('/') ==
                            isinstance(info[filename], dict)):
                        # Only if the types are the same type should
                        # this entry be removed (file for file, dir for
                        # dir).
                        info.pop(filename)

                    if fentry_replacement:
                        # Restore the fileentry with the replacement
                        # from above.
                        info[filename] = fentry_replacement

                # XXX filename could be empty, does it mean the
                # directory can be removed if empty?

            # if ifilename_paths is not empty, it may be possible to
            # restore that entry with the newer (or previous) "version"
            # from the archive, if it also resolves to a file, but this
            # will need more thought to do, given that files and dirs
            # are two different types.

        # finally, purge all empty directories.  Yes this includes
        # directories that may not be wholly owned by this archive.
        for frags in sorted(all_frags, key=lambda x: len(x)):
            target = self._traverse(frags[:-1])
            if target.get(frags[-1]) == {}:
                target.pop(frags[-1])

        # discard the date associated with this archive path too.
        self.archives.pop(archive_path)

    def load_archive(self, archive_path):
        """
        Load an archive file identified by archive_path into the
        mapping.
        """

        try:
            with ArchiveFile(archive_path) as af:
                self._load_infolist(archive_path, af.infolist())
            logger.info('loaded `%s`', archive_path)
            return True
        except BadArchiveFile:
            logger.warning(
                '`%s` appears to be an invalid archive file', archive_path)
        except UnsupportedArchiveFile as e:
            logger.warning('`%s` %s', archive_path, str(e))
        except FileNotFoundError:
            logger.warning(
                '`%s` does not exist.', archive_path)
        except:
            logger.exception('Exception')
        return False

    def unload_archive(self, archive_path):
        self._unload_infolist(archive_path)
        logger.info('unloaded `%s`', archive_path)

    def open(self, path):
        info = self.traverse(path)
        if info is None:
            return
        archive_path, filename, _ = info
        # it is possible to return those values, but given that the
        # underlying files can change, or that new stack comes in, it's
        # best not to directly expose this.
        try:
            with ArchiveFile(archive_path) as zf:
                # zf does remain open because of this.
                return (id(info), zf.open(filename))
        except BadArchiveFile:  # pragma: no cover
            logger.warning(
                '`%s` became an invalid archive file', archive_path)
        except FileNotFoundError:  # pragma: no cover
            # shoud unload the archive?
            logger.warning(
                '`%s` no longer exists.', archive_path)
        except:  # pragma: no cover
            logger.exception('Exception')
        return False

    def readfile(self, path):
        """
        Return the complete file with information contained in path.
        """

        # This is an alternative implementation that returns the entire
        # contents.  Should include similar exception handling as above,
        # but that pattern should really be generalized.

        info = self.traverse(path)

        archive_filename, filename, _ = info
        with ArchiveFile(archive_filename) as af:
            with af.open(filename) as f:
                return f.read()

    def readdir(self, path):
        """
        Return a listing of all files in a directory
        """

        info = self.traverse(path)
        if not isinstance(info, dict):
            return []
        return list(info.keys())


import yaml
import pytest
from selenium import webdriver
from selenium.webdriver.chrome.service import Service

with open('config.yaml') as f:
    testdata = yaml.safe_load(f)

@pytest.fixture()
def browser():
    service = Service(testdata['driver_path'])
    options = webdriver.ChromeOptions()
    driver = webdriver.Chrome(service=service, options=options)
    driver.implicitly_wait(3)
    driver.maximize_window()
    yield driver
    driver.quit()


import re
import sys
from os.path import join, dirname
from setuptools import setup, find_packages
readme = open('README.rst').read()

# reading package version (without reloading it)
with open(join(dirname(__file__), 'neviseh', '__init__.py')) as v_file:
    package_version = re.compile(r".*__version__ = '(.*?)'", re.S).match(v_file.read()).group(1)

dependencies = []
if sys.version_info <= (3, 5):
    dependencies.append('typing')

setup(
    name='neviseh',
    description='Simple text processing tools in persian',
    long_description=readme,
    version=package_version,
    author='Mahdi Ghane.g',
    license='MIT',
    keywords='text-processing persian iran localization',
    url='https://github.com/meyt/neviseh',
    install_requires=dependencies,
    zip_safe=False,
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: POSIX :: Linux',
        'Operating System :: Unix',
        'Natural Language :: Persian',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Topic :: Software Development :: Libraries',
        'Topic :: Text Processing'
    ],
    packages=find_packages()
)


#     2
#  3     4
# 5 6   7 8

import linkedList
import binaryTree

g = binaryTree.Graph()
listMap = {}

nodeH = binaryTree.Node("H")
nodeG = binaryTree.Node("G")
nodeF = binaryTree.Node("F")
nodeE = binaryTree.Node("E")
nodeD = binaryTree.Node("D")
nodeC = binaryTree.Node("C")
nodeB = binaryTree.Node("B")
nodeA = binaryTree.Node("A")

nodeA.addChildLeft(nodeB)
nodeA.addChildRight(nodeC)
nodeB.addChildLeft(nodeD)
nodeD.addChildRight(nodeE)
nodeE.addChildLeft(nodeF)
nodeF.addChildLeft(nodeG)
nodeF.addChildRight(nodeH)

g.addNode(nodeA)
g.addNode(nodeB)
g.addNode(nodeC)
g.addNode(nodeD)
g.addNode(nodeE)
g.addNode(nodeF)
g.addNode(nodeG)
g.addNode(nodeH)

def listOfDepths(depthLevel,root):
    if root.children == [None,None]:
        return
    else:
        for child in root.children:
            if child is not None:
                linkedListDepth = listMap.get(depthLevel,linkedList.LinkedList())
                linkedListDepth.append_node(linkedListDepth,child.value)
                listMap[depthLevel] = linkedListDepth
                listOfDepths(depthLevel+1,child)

linkedListDepth = listMap.get(0,linkedList.LinkedList())
linkedListDepth.append_node(linkedListDepth,nodeA.value)
listMap[0] = linkedListDepth
listOfDepths(1,nodeA)

for depth in listMap:
    print(depth)
    linkedListDepth = listMap[depth]
    print(linkedListDepth)
    print()


# -*- coding: utf-8 -*-
"""

@author: DELINTE Nicolas
"""

from setuptools import setup

import utracto

# read the contents of your README file
from pathlib import Path
this_directory = Path(__file__).parent
long_description = (this_directory / "README.md").read_text()

setup(
    name='utracto',
    version=utracto.__version__,
    description='Implementation of UTracto',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/DelinteNicolas/UTracto',
    author='Nicolas Delinte',
    author_email='nicolas.delinte@uclouvain.be',
    license='GNU General Public License v3.0',
    packages=['utracto'],
    install_requires=['dipy',
                      'nibabel',
                      'numpy',
                      'tqdm',
			    'pilab-binama',
			    'unravel-python',
                      ],

    classifiers=['Development Status :: 4 - Beta',
                 'Intended Audience :: Science/Research',
                 'Natural Language :: English',
                 'Programming Language :: Python'],
)


from events import EventClass
from PySide6.QtWidgets import QFileDialog
from UI.Templates import customFileDialog
import os

class MenuHandler(EventClass):
    def __init__(self, window):
        self.handler = window.handler
        self.window = window
        self.connectActions()

    def connectActions(self):
        handler, window = (self.handler, self.window)
        mb = window.menuBar()

        # FILE
        File = mb.addMenu("&File")
        File.addAction("Save", self.onSave, "Ctrl+s")
        File.addAction("Load", self.onLoad, "Ctrl+l")

        File.addAction("Load Dataset", self.onDatasetLoad, "Ctrl+d")
        File.addAction("Load Model", self.onModelLoad, "Ctrl+e")

        File.addAction("Load Zero Model", self.loadZeroModel, "Ctrl+0")
        File.addAction("Load Prediction", self.loadPrepredictedModel)

        # File.addAction("Preferences", self.onPreferences)
        # File.addAction("Exit", self.onExit)

        # LOUPE
        Loupe = mb.addMenu("&Loupe")
        Loupe.addAction("New", self.newLoupe, "Ctrl+n")

    def onSave(self):
        (path, _) = QFileDialog.getSaveFileName(self.handler.window)
        if path is None or path.strip() == "":
            return

        self.handler.env.newTask(
            self.handler.env.save,
            args=(path,),
            visual=True,
            name=f"Saving at {os.path.basename(path)}",
            threaded=True,
        )

    def onLoad(self):
        path = QFileDialog.getExistingDirectory(self.handler.window)
        if path is None or path.strip() == "":
            return

        self.handler.env.newTask(
            self.handler.env.load,
            args=(path,),
            visual=True,
            name=f"Loading {os.path.basename(path)}",
            threaded=True,
        )

    def onPreferences(self):
        pass

    def onExit(self):
        self.eventPush("QUIT_EVENT")

    def onDatasetLoad(self):
        env = self.handler.env
        fileTypes = list(env.datasetTypes.keys())
        extensions = [
            env.datasetTypes[x].datasetFileExtension for x in fileTypes
        ]
        path, typ = customFileDialog(
            self.handler.window, fileTypes=fileTypes, extensions=extensions
        )

        env.taskLoadDataset(path, typ)

    def onModelLoad(self):
        env = self.handler.env
        fileTypes = list(env.modelTypes.keys())
        extensions = [env.modelTypes[x].modelFileExtension for x in fileTypes]
        path, typ = customFileDialog(
            self.handler.window, fileTypes=fileTypes, extensions=extensions
        )

        env.taskLoadModel(path, typ)

    def loadPrepredictedModel(self):
        env = self.handler.env
        names = [x.getName() for x in env.getAllDatasets(excludeSubs=True)]
        keys = [x.fingerprint for x in env.getAllDatasets(excludeSubs=True)]
        extensions = ["*.npz"] * len(names)

        path, typ = customFileDialog(
            self.handler.window, fileTypes=names, extensions=extensions
        )
        idx = names.index(typ)
        env.loadPrepredictedDataset(path, keys[idx])

    def newLoupe(self):
        self.handler.newLoupe()

    def loadZeroModel(self):
        env = self.handler.env
        env.loadZeroModel()


#!/usr/bin/env python3
# -*- author:xuanGe -*-
# -*- coding:utf-8 -*-
# -*- date:2020-2-13 -*-
import sys
import time
import asyncio
import libpcap as pcap
import ctypes as ct
from config import device
from log import info, success, debug, warning, error

async def collector(q=None):
    # open network interfaces
    errbuf = ct.create_string_buffer(pcap.PCAP_ERRBUF_SIZE + 1)
    pd = pcap.open_live(device, 65535, 1, 1000, errbuf)
    if not pd:
        error(f"c: pcap.open_live failed: {errbuf.value.decode('utf-8')}")
        debug(pd)
    if pcap.setnonblock(pd, 1, errbuf) == -1:
        error(f"c: pcap.setnonblock failed: {errbuf.value.decode('utf-8')}")
    # capture data paket
    try:
        while True:
            status = pcap.dispatch(pd, -1, handle_data, ct.cast(id(q), ct.POINTER(ct.c_ubyte)))
            await asyncio.sleep(0)
    except Exception as e:
        if str(e) != '':
            error(f"c: while filed: {e}")

@pcap.pcap_handler
def handle_data(q_id_pointer, hdr, pkt):
    q = ct.cast(q_id_pointer, ct.py_object).value
    debug(f"c: caplen: {hdr.contents.caplen}")
    debug(f"c: len: {hdr.contents.len}")
    data = ct.string_at(pkt, hdr.contents.caplen)
    #debug(f'data:{data}')
    if q:
        q.put_nowait(data)
        debug(f'c: q.put done: {q.qsize()}')

# start
if __name__ == "__main__":
    try:
        asyncio.run(collector(asyncio.Queue()))
    except KeyboardInterrupt:
        info('exit]')


# coding: utf-8

# HTTP 共享配置
HTTP_WEB_USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 " \
                      "(KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"

# 头条 配置
TOUTIAO_BASE_URL = "https://www.toutiao.com"
TOUTIAO_FEEDS_URL_PATH = "/api/pc/feed/"
TOUTIAO_SCPIDER_INTERVAL = 100                                          # 每两次爬取之间的停留时间间隔 单位 毫秒
TOUTIAO_SCPIDER_COUNT = 100                                             # 每次爬取新闻条数
TOUTIAO_SIGNATURE_JAVASCRIPT_FILE_NAME = "src/common/new_sign.js"       # 头条_signature 字段生成的javascript代码文件
TOUTIAO_SIGNATURE_JAVASCRIPT_FUNCTION_NAME = "get_signature"            # 头条_signature 字段生成的javascript 函数入口名称
TOUTIAO_AS_CP_JAVASCRIPT_FILE_NAME = "src/common/new_sign.js"           # 头条 AS CP 字段生成的javascript 代码文件
TOUTIAO_AS_CP_JAVASCRIPT_FUNCTION_NAME = "get_as_cp"                    # 头条 AS CP 字段生成的javascript 函数入口名称

# 头条APP 配置
TOUTIAO_APP_FEED_URL_V2 = "http://is.snssdk.com/api/news/feed/v2/"      # 头条app v2版本feeds流接口
TOUTIAO_APP_FEED_CATEGORY = \
    "https://is.snssdk.com/article/category/get_subscribed/" \
    "v4/?iid=45032656046&aid=13&version_code=693"                       # 头条app 获取feeds流 分类接口
TOUTIAO_APP_FEED_DETAIL = \
    "http://a3.pstatp.com/article/content/21/1/" \
    "{group_id}/{group_source}/1/0/?"                                   # 头条app feeds流 详情
TOUTIAO_APP_DEFAULT_CATEGORY = \
    ['关注', '__all__', 'news_hot', 'news_local', 'nineteenth',
     'video', '组图', 'news_entertainment', 'question_and_answer',
     'news_tech', 'news_car', 'news_finance', 'news_military',
     'live', 'news_sports', 'news_world', 'news_health',
     'news_house', '冬奥会', 'traditional_culture', 'news_history',
     'news_regimen', 'funny', 'news_food', 'emotion', 'movie',
     'news_baby', 'weitoutiao', 'fangxingou', 'NBA', '漫画']              # 头条app feeds流默认分类


#Ethan Bennett
import random

#Change the mydictionary into a dictionary data structure

def hangman(Computer, Graphic, Length):
   """
   I DON'T HAVE TO DO GRAPHICS
   This function starts a game where the computer is choosing the word
   if Computer is True and you choose the word if Computer is False.
   The words with be of length length
   Graphics is either True or False and indicates whether the man is drawn.
   It returns True if the computer wins and False if the computer Looses.
   """
   if Computer == True:
      tries = 0
      word = '_'*Length
      print("Hello, let's play hangman!\nI'm choosing a word... thinking...\nOk.")
      MakeMyDictionary('american-english.txt')
      ComputerGuess(computerWordPick(Length),tries,Length,word)
   if Computer == False:
      tries = 0
      word = '_'*Length
      print('You are going to pick this time!\nOnce you have a word that is',Length,'letters long, hit any key')
      input('...')
      MakeMyDictionary('american-english.txt')
      alphabet = 'abcdefghijklmnopqrstuvwxyz'
      humanWordPick(tries,Length, word,alphabet)
   if Graphic == True:
      turtleDrawing()

def MakeMyDictionary(filename):
   '''This function cleans up and makes a list of strings with all the possible words for the computer to choose from'''
   f = open(filename, encoding='utf-8')
   mydictionary = []
   for word in f:
      word = word.strip()
      if word.endswith("'s"):
         word = word[:-2]
      else:
         words = []
      mydictionary.append(word)
   f.close()
   return mydictionary

def computerWordPick(Length):
   """ This fuction has the computer pick the word for the game"""
   for word in MakeMyDictionary('american-english.txt'):
      pick = random.choice(MakeMyDictionary('american-english.txt'))
      pickLength = len(pick)
      if pickLength != Length:
         pick
      else:
         return pick

def ComputerGuess(pick,tries,Length,word):
   '''Computer asks for a guess from the human and confirm or deny its presence in the word'''
   print('here is my word so far')
   print(word)
   print(pick)
   Guess = input('What is your guess?')
   if '_' in word:
      if Guess in pick:
         print('yes!',Guess,'is in my word')
         for letters in pick:
            if Guess == letters:
               l = pick.index(letters)
               word = list(word)
               word[l] = Guess
               word = ''.join(word)
               x = repeatedLetter1(Guess,word)
               print(word)
               ComputerGuess(pick,tries,Length,word)
      else:
         print(Guess,'is not in my word. Try again')
         ComputerGuess(pick,tries,Length,word)
   else:
      print('Wow!, you win!')

def repeatedLetter1(Guess,word):

   word = list(word)
   word[l] = Guess
   word = ''.join(word)
   print(word)
   repeatedLetter(Guess,word)
   return word

def humanWordPick(tries, Length, word,alphabet):
   '''Problem: will ask about letters multiple times'''
   if '_' in word:
      Guess = random.choice(alphabet)
      alphabet = alphabet.replace(Guess,'')
      if tries <= 20:
         print('Is', Guess,'in your word? (Y/N)')
         if input() == 'Y':
            l = input('What letter of the word is it? (0,1,2,3...)')
            l = int(l)
            word = list(word)
            word[l] = Guess
            word = ''.join(word)
            print(word)
            x = repeatedLetter(Guess,word)
            print(x)
            print('Good!')
            humanWordPick(tries,Length, word,alphabet)
         else:
            tries += 1
            print('Ok,',tries,'/20')
            humanWordPick(tries,Length,word,alphabet)
      else:
         print("'You win! I couldn't guess your word!")
   else:
      print('I win!')

def repeatedLetter(Guess,word):
   if input('Does it occur again? (Y/N)') == 'Y':
      l1 = input('What letter of the word is it? (0,1,2,3...)')
      l1 = int(l1)
      word = list(word)
      word[l1] = Guess
      word = ''.join(word)
      print(word)
      repeatedLetter(Guess,word)
      return word


import csv
import numpy as np

def get_csv():
    with open('./words.csv', newline='', eoncding="utf8") as f:
        reader = csv.reader(f)
        data = list(reader)
    data = [d[0] for d in data]
    return data[1:]

def get_txt():
    with open("./words.txt", newline='', encoding="utf8") as f:
        reader = csv.reader(f)
        data = list(reader)
    data = [d[0] for d in data]
    return data

def shuffle(words):
    words_copy = np.array(words)
    np.random.shuffle(words_copy)
    return words_copy


"""
748. 最短补全词
给你一个字符串 licensePlate 和一个字符串数组 words ，请你找出并返回 words 中的 最短补全词 。
补全词 是一个包含 licensePlate 中所有的字母的单词。在所有补全词中，最短的那个就是 最短补全词 。
在匹配 licensePlate 中的字母时：
忽略 licensePlate 中的 数字和空格 。
不区分大小写。
如果某个字母在 licensePlate 中出现不止一次，那么该字母在补全词中的出现次数应当一致或者更多。
例如：licensePlate = "aBc 12c"，那么它的补全词应当包含字母 'a'、'b' （忽略大写）和两个 'c' 。可能的 补全词 有 "abccdef"、"caaacab" 以及 "cbca" 。
请你找出并返回 words 中的 最短补全词 。题目数据保证一定存在一个最短补全词。当有多个单词都符合最短补全词的匹配条件时取 words 中 最靠前的 那个。
"""
import collections

def shortestCompletingWord(licensePlate: str, words: [str]) -> str:
    def counter(string):
        temp_dict = collections.defaultdict(int)
        for char in string:
            if char.isalpha():
                temp_dict[char.lower()] += 1
        return temp_dict

    def is_valid(dictionary):
        for char in license_dict:
            if license_dict[char] > dictionary[char]:
                return False
        return True

    result_len, result = float('INF'), ''
    license_dict = counter(licensePlate)
    for word in words:
        word_dict = counter(word)
        if is_valid(word_dict):
            if len(word) < result_len:
                result_len = len(word)
                result = word
    return result

shortestCompletingWord("Ah71752", ["suggest","letter","of","husband","easy","education","drug","prevent","writer","old"])


# https://adventofcode.com/2021/day/1

def _parse_input(input_filename: str) -> list:
    lines = []
    with open(input_filename, "r") as file_in:
        for line in file_in:
            lines.append(int(line))
    return lines

def one(measurements: list) -> int:
    increases = 0
    previous_measurement = 1e9

    for measurement in measurements:
        if measurement > previous_measurement:
            increases = increases + 1
        previous_measurement = measurement

    return increases

def two(measurements: list) -> int:
    sliding_sums = []

    for index, _ in enumerate(measurements):
        try:
            measurements[index+2]
        except IndexError:
            break
        else:
            sliding_sums.append(sum(measurements[index:index+3]))

    return one(sliding_sums)

if __name__ == "__main__":
    measurements = _parse_input("01_input.txt")
    for part in [one, two]:
        print(f"Part {part.__name__}: {part(measurements)}")


from flask import session

from todo_app.data.Item import Item

import os, requests

def get_items():
    """
    Fetches all saved items from the session.

    Returns:
        list: The list of saved items.
    """

    BOARD_ID = os.getenv('CORNDEL_BOARD_ID')
    API = os.getenv('TRELLO_API')
    BOARD = os.getenv('TRELLO_BOARD').replace('{id}', BOARD_ID)

    # https://api.trello.com/1/board/{board_id}/cards
    request_url = API+BOARD+'?key='+os.getenv('TRELLO_API_KEY')+'&token='+os.getenv('TRELLO_TOKEN')
    trello_response = requests.get(request_url)

    return build_card_items(trello_response.json())

def build_card_items(trello_response):
    card_items = []

    for trello_list in trello_response:
        card_items.append(Item.from_trello_card(trello_list))

    return card_items

def get_item(id):
    """
    Fetches the saved item with the specified ID.

    Args:
        id: The ID of the item.

    Returns:
        item: The saved item, or None if no items match the specified ID.
    """
    items = get_items()
    return next((item for item in items if item.id == id), None)

def add_item(title, list_id):
    """
    Adds a new item with the specified title to the session.

    Args:
        title: The title of the card.
        list_id: The list to add the card to

    Returns:
        item: The saved item.
    """
    # Add the item to the list
    API = os.getenv('TRELLO_API')
    CARD = os.getenv('TRELLO_CARDS')
    request_url = API+CARD+'?key='+os.getenv('TRELLO_API_KEY')+'&token='+os.getenv('TRELLO_TOKEN')

    trello_response = requests.post(request_url, data={'name': title, 'idList': list_id})

    return trello_response

def save_item(card_id, list_id):
    """
    Updates an existing item in the session. If no existing item matches the ID of the specified item, nothing is saved.

    Args:
        card_id: The card_id to move to the new list.
        list_id: The list id to move the card to
    """

    API = os.getenv('TRELLO_API')
    CARD = os.getenv('TRELLO_CARD').replace('{id}', card_id)

    # https://api.trello.com/1/cards/{card_id}?key=....
    request_url = API+CARD+'?key='+os.getenv('TRELLO_API_KEY')+'&token='+os.getenv('TRELLO_TOKEN')

    trello_response = requests.put(request_url, data={'idList': list_id})

    return trello_response