import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics.pairwise import cosine_similarity

from main import user_item_rating
import modelEvaluator

class PopularityRecommender:
    MODEL_NAME = 'Popularity'

    def __init__(self, interactions_test_indexed_df, popularity_df, items_df=None):
        """
            This class is a Popularity Recommendation Model. It recommends based on the global popularity of items

            Args:
                interactions_test_indexed_df: the test sub-set of interactions_full_indexed_df
                popularity_df: the global item popularity dataframe
                items_df: a complete dataframe of the articles
        """

        self.interactions_test_indexed_df = interactions_test_indexed_df
        self.popularity_df = popularity_df
        self.items_df = items_df

    def get_model_name(self):
        """
            Returns:
                model_name: the name of the model
        """

        return self.MODEL_NAME

    def recommend_items(self, user_id, items_to_ignore=None, topn=10, verbose=False):
        """
            Args:
                user_id: the id of the user that the model is recommending items to
                items_to_ignore: the items that should be ignored by the recommender
                topn: defaults to 10. Defines amount of items to be recommended
                verbose: defaults to False
            Returns:
                recommendations_df: the topn recommended items
        """

        # Recommend the more popular items that the user hasn't seen yet.
        if items_to_ignore is None:
            items_to_ignore = []
        recommendations_df = self.popularity_df[~self.popularity_df['contentId'].isin(items_to_ignore)] \
            .sort_values('Rate', ascending=False) \
            .head(topn)

        if verbose:
            if self.items_df is None:
                raise Exception('"items_df" is required in verbose mode')

            recommendations_df = recommendations_df.merge(self.items_df, how='left',
                                                          left_on='contentId',
                                                          right_on='contentId')[
                ['Rate', 'contentId', 'title', 'url', 'lang']]

        return recommendations_df

def inspect_interactions(articles_df, interactions_train_indexed_df, interactions_test_indexed_df, person_id,
                         test_set=True):
    """
        Args:
            articles_df: a full dataframe of the articles
            interactions_train_indexed_df: the train sub-set of interactions_full_indexed_df
            interactions_test_indexed_df: the test sub-set of interactions_full_indexed_df
            person_id: the user's id
            test_set: defaults to True. Decides whether to use the train set or the test set for inspection
        Returns:
            interactions_df: the inspected interactions dataframe
    """
    if test_set:
        interactions_df = interactions_test_indexed_df
    else:
        interactions_df = interactions_train_indexed_df
    return interactions_df.loc[person_id].merge(articles_df, how='left',
                                                left_on='contentId',
                                                right_on='contentId') \
        .sort_values('Rate', ascending=False)[['Rate',
                                               'contentId',
                                               'title', 'url', 'lang']]

def inspect_interactions_user(user_id, interactions_df, users_enough_interactions, items_enough_rated):
    """
        Args:
            user_id: the user's id
            interactions_df: a complete dataframe of user interactions with the articles
            users_enough_interactions: a list of users that have interacted with at minimum three (3) articles
            items_enough_rated: a list of articles that have at least two (2) different users interacting with them
        Returns:
            item_scores_df: an item scores dataframe
    """
    # the amount of most similar users chosen for recommendation
    top = 5

    user_item_df, user_item_matrix = user_item_rating(interactions_df, users_enough_interactions, items_enough_rated)

    # The Sample User's content scores
    user_index = np.where(users_enough_interactions == user_id)[0][0]

    similar_users = []
    for i in range(len(users_enough_interactions)):
        if i != user_index:
            similar_users.append(
                [i, cosine_similarity([user_item_matrix[:, i]], [user_item_matrix[:, user_index]])[0][0]])

    similar_users = np.array(similar_users)
    ind = np.argsort(similar_users[:, -1])
    similar_users = similar_users[ind]

    # sorted in asc order, therefore most similar users are put in the end of the array
    similar_users = similar_users[-top:]

    # simply initialising the items array
    items = np.zeros(len(items_enough_rated))

    # for each item there will be a summed score based on the ratings of each similar user
    for similar_user in similar_users:
        s_u_index = int(similar_user[0])
        for i in range(len(items_enough_rated)):
            s = user_item_matrix[:, s_u_index]
            items[i] += s[i]

    item_scores_df = pd.DataFrame()
    item_scores_df['contentId'] = items_enough_rated
    item_scores_df['Rate'] = items

    item_scores_df = item_scores_df.sort_values(by='Rate',
                                                ascending=False)

    return item_scores_df

def item_collaborative_filtering(articles_enough_df, articles_df, interactions_full_df):
    """
        Performs item-based collaborative filtering. This model only takes global item ratings into account and thus is
        not personalised for the user

        Args:
            articles_enough_df: a list of articles that have at least two (2) different users interacting with them
            articles_df: a complete dataframe of the articles
            interactions_full_df: a complete dataframe of all user ratings for the articles

        Returns:
            global_metrics: the metrics of the evaluation
    """

    # The User Sample's ID
    user_id = -1479311724257856983

    print('\nPreparing data for Item-based Collaborative Filtering model...\n')

    interactions_train_df, interactions_test_df = train_test_split(interactions_full_df,
                                                                   stratify=interactions_full_df['personId'],
                                                                   test_size=0.20,
                                                                   random_state=42)
    # Indexing by personId to speed up the searches during evaluation
    interactions_full_indexed_df = interactions_full_df.set_index('personId')
    interactions_train_indexed_df = interactions_train_df.set_index('personId')
    interactions_test_indexed_df = interactions_test_df.set_index('personId')

    items = inspect_interactions(articles_df, interactions_train_indexed_df, interactions_test_indexed_df,
                                 user_id, test_set=True)

    item_popularity_df = interactions_full_indexed_df.groupby('contentId')['Rate'].sum().sort_values(
        ascending=False).reset_index()
    model_evaluator = modelEvaluator.ModelEvaluator(interactions_full_indexed_df, interactions_train_indexed_df,
                                                    interactions_test_indexed_df, articles_df, item_popularity_df)

    print("\n Here there are the ten most relevant items basing on the sum of ratings:\n")
    print(item_popularity_df.head(10))
    popularity_model = PopularityRecommender(interactions_test_indexed_df, item_popularity_df, articles_df)

    print('\nEvaluating Popularity recommendation model...\n')
    global_metrics, pop_detailed_results_df = model_evaluator.evaluate_model(popularity_model, items)

    print('\nGlobal metrics:\n%s' % global_metrics)
    print(pop_detailed_results_df.head(10))

    print(
        "\nFirst 20 relevant items for the user ID -1479311724257856983 \n(just a sample one, if you want to change "
        "user just call the function with an other ID): \n")

    print(items.head(20))

    return global_metrics

def user_collaborative_filtering(articles_df, interactions_full_df, interactions_df):
    """
        Performs user-based collaborative filtering. This model takes user-item ratings into account and is personalised

        Args:
            articles_df: a complete dataframe of the articles
            interactions_full_df: a complete dataframe of all user ratings for the articles
            interactions_df: a complete dataframe of all user interactions with the articles

        Returns:
            global_metrics: the metrics of the evaluation
    """
    # The User Sample's ID
    user_id = -1479311724257856983

    print('\nPreparing data for User-based Collaborative Filtering model...\n')

    interactions_train_df, interactions_test_df = train_test_split(interactions_full_df,
                                                                   stratify=interactions_full_df['personId'],
                                                                   test_size=0.20,
                                                                   random_state=42)
    # Indexing by personId to speed up the searches during evaluation
    interactions_full_indexed_df = interactions_full_df.set_index('personId')
    interactions_train_indexed_df = interactions_train_df.set_index('personId')
    interactions_test_indexed_df = interactions_test_df.set_index('personId')

    users_enough_interactions_train = interactions_train_df['personId'].unique()
    users_enough_interactions_test = interactions_test_df['personId'].unique()
    items_enough_rated_train = interactions_train_df['contentId'].unique()
    items_enough_rated_test = interactions_test_df['contentId'].unique()

    item_scores_df = inspect_interactions_user(user_id, interactions_df, users_enough_interactions_test,
                                               items_enough_rated_test)

    print(
        "\nFirst 10 most relevant items for the user ID -1479311724257856983 \n(just a sample one, if you want to "
        "change user just call the function with an other ID): \n")

    print(item_scores_df.head(10))

    model_evaluator = modelEvaluator.ModelEvaluator(interactions_full_indexed_df, interactions_train_indexed_df,
                                                    interactions_test_indexed_df, articles_df, item_scores_df)

    popularity_model = PopularityRecommender(interactions_test_indexed_df, item_scores_df, articles_df)

    print('\nEvaluating Recommendation model...\n')
    global_metrics, pop_detailed_results_df = model_evaluator.evaluate_model(popularity_model, item_scores_df)

    print('\nGlobal metrics:\n%s' % global_metrics)
    print(pop_detailed_results_df.head(10))

    return global_metrics


#!/usr/bin/env python3

import argparse
import logging
import sys

import slack

logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)

def idb_excepthook(type, value, tb):
    """Call an interactive debugger in post-mortem mode

    If you do "sys.excepthook = idb_excepthook", then an interactive debugger
    will be spawned at an unhandled exception
    """
    if hasattr(sys, 'ps1') or not sys.stderr.isatty():
        sys.__excepthook__(type, value, tb)
    else:
        import pdb, traceback
        traceback.print_exception(type, value, tb)
        print
        pdb.pm()

@slack.RTMClient.run_on(event='message')
def handle_messages(**payload):
    """
    Does a lot. Fucking decorators.
    1. if the message has react-able strings, react to the message appropriately
    2. if the message starts with @botname and has a command i've planned for, do thing.
    """
    data = payload['data']
    channel_id = data['channel']
    thread_ts = data['ts']

    LOGGER.debug(payload)

    if not data['text'].startswith("MOONRITUAL"):
        return
    response = "ðŸ§ž"

    webclient = payload['web_client']
    webclient.chat_postMessage(
        channel=channel_id,
        text=response,
        timestamp=thread_ts
    )

def main(*args, **kwargs):
    parser = argparse.ArgumentParser(
        description="The Dicussion Archiver")
    parser.add_argument(
        "--debug", "-d", action='store_true',
        help="Include debugging output")
    parser.add_argument("--verbose", "-v", action="store_true", help="Include verbose logs")
    parser.add_argument(
        "--api-token", "-t", required=True, help="Slack API token")
    parsed = parser.parse_args()
    if parsed.verbose or parsed.debug:
        LOGGER.setLevel(logging.DEBUG)
    if parsed.debug:
        sys.excepthook = idb_excepthook

    rtm_client = slack.RTMClient(token=parsed.api_token)
    rtm_client.start()

if __name__ == '__main__':
    sys.exit(main(*sys.argv))


from rest_framework import status
from django.test import TestCase
from rest_framework.test import APIClient
from ..serializers import ClimateChangeInfo, ClimateChangeInfoSerializer
from ..views import *
import random

# initialize the APIClient app
client = APIClient()

class GetClimateChangeDataTest(TestCase):
    """ Test module for GET Weather API """

    def test_get_valid_single_response(self):
        limit = random.randint(1, 10)
        offset = random.randint(1, 10)
        Region = ['UK', 'England', 'Wales', 'Scotland', 'Northern_Ireland', 'England_and_Wales', 'England_N', 'England_S', 'Scotland_N', 'Scotland_E', 'Scotland_W', 'England_E_and_NE', 'England_NW_and_N_Wales', 'Midlands', 'East_Anglia', 'England_SW_and_S_Wales', 'England_SE_and_Central_S']
        Parameter = ['Tmax', 'Tmin', 'Tmean', 'Sunshine', 'Rainfall', 'Raindays1mm', 'AirFrost']
        region = random.choice(Region)
        parameter = random.choice(Parameter)
        response = client.get('/api/get-weather-data/', data = {'limit' : limit, 'offset' : offset, 'region': region, 'parameter': parameter}, format='json')
        response_new = parse_weather_data(region, parameter)
        data_points = response_new['data_points']
        climate_change_info_list = []
        for data in data_points:
            climate_change_object = ClimateChangeInfo(
                year = data[0],
                jan = data[1],
                feb = data[2],
                mar = data[3],
                apr = data[4],
                may = data[5],
                jun = data[6],
                jul = data[7],
                aug = data[8],
                sep = data[9],
                oct = data[10],
                nov = data[11],
                dec = data[12],
                win = data[13],
                spr = data[14],
                sum = data[15],
                aut = data[16],
                ann = data[17]
            )
            climate_change_info_list.append(climate_change_object)
        # the many param informs the serializer that it will be serializing more than a single climate change object.
        serializer = ClimateChangeInfoSerializer(climate_change_info_list, many=True)
        self.assertEqual(response.data['results'], serializer.data[offset:offset+limit])
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_get_invalid_single_response(self):
        limit = random.randint(1, 10)
        offset = random.randint(1, 10)
        response = client.get('/api/get-weather-data/', data = {'limit' : limit, 'offset' : offset, 'region': 'India', 'parameter': 'Rainfall'}, format='json')
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_invalid_request_method(self):
        response = client.post('/api/get-weather-data/', data = {'limit' : 10, 'offset' : 10, 'region': 'England', 'parameter': 'Rainfall'}, format='json')
        self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)


class Solution:
    def search(self, nums: List[int], target: int) -> int:
        # implement the binary search iterative algorithm
        #
        # binary search requires unique values, sorted input array
        # Runtime: O(logN) b/c the binary search eliminates half of the
        # search space with each additional loop.

        low = 0
        high = len(nums) - 1
        mid = (low + high) // 2     # double slash for explicit int division aka floor division

        while low <= high:
            if target == nums[mid]:
                return mid
            elif target < nums[mid]:
                # re-assign high to mid - 1 (effectively removes the half of the array from index (mid+1) to end of array from search space)
                high = mid - 1
                # re-calculate the new mid index value
                mid = (low + high) // 2
            else:
                # re-assign low to mid + 1 (effectively removes the half of the array from index 0 to index (mid-1))
                low = mid + 1
                mid = (low + high) // 2
        # if you didn't return from the while loop, then no match found.
        return -1


#!/usr/bin/env python3

"""Neural Network

Compiler: Python 3.6.5
OS: macOS 10.13.6
"""

import numpy as np
import time

# network architecture
INPUT_LAYER_SIZE = 354
HIDDEN1_LAYER_SIZE = 9
HIDDEN2_LAYER_SIZE = 8
OUTPUT_LAYER_SIZE = 3

# number of epochs before early stopping if there
# is no decrease in validation set error
PATIENCE = 35

def binarize(labels):
  """Converts the labels into their binary representation

  The labels consist of 8 classes (1, 2, ..., 8) and can be mapped
  to binaries (000, 001, ..., 111) to represent the 3 neurons in the
  output layer:

  Class 1 => 000
  Class 2 => 001
  ...
  Class 8 => 111

  Parameters:
    labels: The array of labels for each training example

  Returns:
    The array of binary vectors
  """

  y = []

  for label in labels:
    l = int(label) - 1
    vec = list(np.binary_repr(l, width=3))
    vec = [int(b) for b in vec]
    y.append(vec)

  return y

def decimalize(labels):
  """Converts the binary vectors back to their corresponding labels

  000 => Class 1
  001 => Class 2
  ...
  111 => Class 8

  Parameters:
    labels: The array of binarized labels

  Returns:
    The array of labels
  """

  label_values = [int(''.join(i), 2) + 1 for i in labels.astype(str)]
  return label_values

def randomize(size):
  """Returns samples from a uniform distribution with range -0.1 to 0.1

  Parameters:
    size: The number of samples to generate. If size is (m, n), then m * n samples are generated

  Returns:
    The array of generated samples
  """

  return np.random.uniform(-0.1, 0.1, size)

def init_weights():
  """Initializes the weights of the neurons of each layer in the network

  Returns:
    The initial weights grouped per layer
  """

  h1 = randomize((HIDDEN1_LAYER_SIZE, INPUT_LAYER_SIZE))
  h2 = randomize((HIDDEN2_LAYER_SIZE, HIDDEN1_LAYER_SIZE))
  out = randomize((OUTPUT_LAYER_SIZE, HIDDEN2_LAYER_SIZE))

  return { 'h1': h1, 'h2': h2, 'out': out }

def init_biases():
  """Initializes the biases of the neurons of each layer in the network

  Returns:
    The initial biases grouped per layer
  """

  h1 = randomize(HIDDEN1_LAYER_SIZE)
  h2 = randomize(HIDDEN2_LAYER_SIZE)
  out = randomize(OUTPUT_LAYER_SIZE)

  return { 'h1': h1, 'h2': h2, 'out': out }

def weighted_sum(x, w, b):
  """Returns the weighted sums of the inputs

  Parameters:
    x: The array of inputs
    w: The array of weights of the inputs
    b: The array of biases

  Returns:
    The array of weighted sums
  """

  return np.dot(w, x) + b

def sigmoid(v):
  """Returns the result of the sigmoid activation function

  Parameters:
    v: The array of weighted sums

  Returns:
    The array of sigmoid outputs
  """

  return 1.0 / (1 + np.exp(-v))

def feed_forward(x, weights, biases):
  """Performs a forward pass from the input layer to the output layer

  Parameters:
    x: The array of inputs
    weights: The array of weights grouped per layer
    biases: The array of biases grouped per layer

  Returns:
    The outputs of the neurons for each layer
  """

  v_h1 = weighted_sum(x, weights['h1'], biases['h1'])
  y_h1 = sigmoid(v_h1)

  v_h2 = weighted_sum(y_h1, weights['h2'], biases['h2'])
  y_h2 = sigmoid(v_h2)

  v_out = weighted_sum(y_h2, weights['out'], biases['out'])
  y_out = sigmoid(v_out)

  return { 'h1': y_h1, 'h2': y_h2, 'out': y_out }

def error(y, d):
  """Returns the error in the output layer

  Parameters:
    y: the response from the output layer
    d: the desired or expected output

  Returns:
    The error in the output layer
  """

  return d - y

def delta(*args):
  """Returns the local gradient for one layer

  If the parameters are the outputs and the error, the local gradients in the
  output layer will be computed. If the parameters are the outputs, the local
  gradients from the previous neurons and the weights, the local gradients in
  the hidden layer will be computed.

  Returns:
    The array of local gradients for the layer
  """

  if len(args) == 3:
    y, prev_delta, weights = args
    delta = y * (1 - y) * np.dot(prev_delta, weights)
  elif len(args) == 2:
    y, error = args
    delta = error * y * (1 - y)
  return delta

def reshape_delta(delta):
  """Reshapes the local gradient array into 2d to be used for calculating weights

  Parameters:
    delta: The array of local gradients to be reshaped

  Returns:
    The 2d array of local gradients
  """

  return np.reshape(delta, (len(delta), -1))

def compute_local_gradients(y, error, weights):
  """Computes the local gradients for all layers

  Parameters:
    y: The array of outputs per layer
    error: The error of each neuron in the output layer
    weights: The array of weights per layer

  Returns:
    dictionary
      The array of local gradients for each layer
  """

  delta_out = delta(y['out'], error)
  delta_h2 = delta(y['h2'], delta_out, weights['out'])
  delta_h1 = delta(y['h1'], delta_h2, weights['h2'])

  return { 'out': delta_out, 'h2': delta_h2, 'h1': delta_h1 }

def update_weights(weights, deltas, y, x, eta):
  """Updates the weights

  Parameters:
    weights: The array of current weights
    deltas: The array of local gradients per layer
    y: The array of outputs per layer
    x: The array of inputs
    eta: The learning rate

  Returns:
    The array of updated weights grouped per layer
  """

  out = weights['out'] + (eta * reshape_delta(deltas['out']) * y['h2'])
  h2 = weights['h2'] + (eta * reshape_delta(deltas['h2']) * y['h1'])
  h1 = weights['h1'] + (eta * reshape_delta(deltas['h1']) * x)

  return { 'out': out, 'h2': h2, 'h1': h1 }

def update_biases(biases, deltas, eta):
  """Updates the weights

  Parameters:
    weights: The array of current weights
    deltas: The array of local gradients per layer
    eta: The learning rate

  Returns:
    The array of updated biases grouped per layer
  """

  out = biases['out'] + (eta * deltas['out'])
  h2 = biases['h2'] + (eta * deltas['h2'])
  h1 = biases['h1'] + (eta * deltas['h1'])

  return { 'out': out, 'h2': h2, 'h1': h1 }

def cost(errors):
  """Computes the average error for each training epoch

  Parameters:
    errors: The errors from all training examples

  Returns:
    The average error
  """

  total_errors = [np.sum(e * e) * 0.5 for e in errors]
  return np.sum(total_errors) / len(total_errors)

def shuffle_indices(n):
  """Generates an array of numbers from 0 to n then shuffles it

  This is used for shuffling the data set before training.

  Parameters:
    n: The end point of the array of numbers

  Returns:
    Shuffled array of numbers
  """

  p = np.arange(n)
  np.random.shuffle(p)
  return p

def train(training_set, validation_set, eta=0.1, epochs=10000):
  """Trains the network

  Parameters:
    training_set: The training set and labels
    validation_set: The validation set and labels
    eta: The learning rate
    epochs: The number of iterations

  Returns:
    The final weights and biases
  """

  training_data, training_labels = training_set
  validation_data, validation_labels = validation_set

  training_size = len(training_labels)
  validation_size = len(validation_labels)

  # initialize weights & biases
  weights = init_weights()
  biases = init_biases()

  # patience counter
  j = 0

  # storage for the best parameters so far
  best_params = {
    'training_error': 1.0,
    'validation_error': 1.0,
    'epoch': 0,
    'weights': weights,
    'biases': biases
  }

  start = time.time()

  for epoch in range(0, epochs):
    training_errors = []
    validation_errors = []

    p = shuffle_indices(training_size)
    q = shuffle_indices(validation_size)

    # training phase
    for n in range(0, training_size):
      i = p[n]
      x = training_data[i]
      d = training_labels[i]
      y = feed_forward(x, weights, biases)

      # back propagate
      err = error(y['out'], d)
      deltas = compute_local_gradients(y, err, weights)
      weights = update_weights(weights, deltas, y, x, eta)
      biases = update_biases(biases, deltas, eta)
      training_errors.append(err)

    # validation phase
    for n in range(0, validation_size):
      i = q[n]
      x = validation_data[i]
      d = validation_labels[i]
      y = feed_forward(x, weights, biases)
      err = error(y['out'], d)
      validation_errors.append(err)

    training_error = cost(training_errors)
    validation_error = cost(validation_errors)

    print('Iteration: {} Training Error: {} Validation Error: {}'.format(epoch + 1, training_error, validation_error))

    # save the model parameters if there is a decrease in validation error
    if validation_error < best_params['validation_error']:
      best_params['training_error'] = training_error
      best_params['validation_error'] = validation_error
      best_params['epoch'] = epoch
      best_params['weights'] = weights
      best_params['biases'] = biases
      j = 0
    else:
      j = j + 1

    # early stopping
    if j > PATIENCE:
      break

  end = time.time()

  print('\nTraining time: {} seconds'.format(end - start))

  print('\nTotal epochs: {}'.format(epoch + 1))
  print('Training error at termination: {}'.format(training_error))
  print('Validation error at termination: {}'.format(validation_error))

  print('\nChosen iteration: {}'.format(best_params['epoch'] + 1))
  print('Training error at chosen iteration: {}'.format(best_params['training_error']))
  print('Validation error at chosen iteration: {}'.format(best_params['validation_error']))

  return best_params['weights'], best_params['biases']

def test(test_data, weights, biases):
  """Tests the network on the test set

  Parameters:
    test_data: The test data set
    weights: The weights of the network per layer
    biases: The biases of the network per layer

  Returns:
    The array of prediction results
  """

  results = []

  for x in test_data:
    y = feed_forward(x, weights, biases)
    labels = np.around(y['out']).astype(int)
    results.append(labels)

  return decimalize(np.array(results))

def start():
  training_set_file = 'training_set.csv'
  training_labels_file = 'training_labels.csv'

  validation_set_file = 'validation_set.csv'
  validation_labels_file = 'validation_labels.csv'

  print('Fetching training data from {}...'.format(training_set_file))
  training_data = np.genfromtxt(training_set_file, delimiter=',')

  print('Fetching training labels from {}...'.format(training_labels_file))
  training_labels = binarize(np.genfromtxt(training_labels_file, delimiter=','))

  print('Fetching validation data from {}...'.format(validation_set_file))
  validation_data = np.genfromtxt(validation_set_file, delimiter=',')

  print('Fetching validation labels from {}...'.format(validation_labels_file))
  validation_labels = binarize(np.genfromtxt(validation_labels_file, delimiter=','))

  # train network
  weights, biases = train((training_data, training_labels), (validation_data, validation_labels))

  test_set_file = 'test_set.csv'

  print('Fetching test data from {}...'.format(test_set_file))
  test_data = np.genfromtxt(test_set_file, delimiter=',')

  # test model on the test set
  results = test(test_data, weights, biases)

  # save predictions to file
  predicted_file = 'predicted_ann4.csv'

  print('Saving predictions to {}...'.format(predicted_file))
  np.savetxt(predicted_file, results, delimiter=',', fmt='%i')
  print('Test results saved to {}'.format(predicted_file))

start()


number_cats = int(input())
small_cats_count = 0
big_cats_count = 0
huge_cats_count = 0
total_grams_eaten = 0

for cat in range(number_cats):
    eaten_food = float(input())

    if 100 <= eaten_food < 200:
        small_cats_count += 1
    elif 200 <= eaten_food < 300:
        big_cats_count += 1
    elif 300 <= eaten_food < 400:
        huge_cats_count += 1

    total_grams_eaten += eaten_food

price_food_per_day = (total_grams_eaten / 1000) * 12.45

print(f"Group 1: {small_cats_count} cats.")
print(f"Group 2: {big_cats_count} cats.")
print(f"Group 3: {huge_cats_count} cats.")
print(f"Price for food per day: {price_food_per_day:.2f} lv.")


import bpy
import os

from pluginUtils.log import printLog
import pluginUtils.gltf as gltf

from .utils import *

def getUsedMaterials():
    """
    Gathers and returns all unfiltered, valid Blender materials.
    """

    materials = []

    for bl_mat in bpy.data.materials:
        materials.append(bl_mat)

    return materials

def getImageIndex(exportSettings, uri):
    """
    Return the image index in the glTF array.
    """

    if exportSettings['uri_cache'] is None:
        return -1

    if uri in exportSettings['uri_cache']['uri']:
        return exportSettings['uri_cache']['uri'].index(uri)

    return -1

def getTextureIndexByTexture(exportSettings, glTF, bl_texture):
    """
    Return the texture index in the glTF array by a given texture. Safer than
    "getTextureIndex" in case of different textures with the same image or linked textures with
    the same name but with different images.
    """

    if (exportSettings['uri_cache'] is None or glTF.get('textures') is None
            or bl_texture is None):
        return -1

    bl_image = getTexImage(bl_texture)
    if bl_image is None or bl_image.filepath is None:
        return -1

    uri = getImageExportedURI(exportSettings, bl_image)
    image_uri = exportSettings['uri_cache']['uri']
    tex_name = getTextureName(bl_texture)

    index = 0
    for texture in glTF['textures']:
        source = getTextureSource(texture)
        if source is not None and 'name' in texture:
            current_image_uri = image_uri[source]
            if current_image_uri == uri and texture['name'] == tex_name:
                return index

        index += 1

    return -1

def getTextureSource(tex):
    """texture source can also be in extension"""

    if 'source' in tex:
        return tex['source']
    else:
        ext = gltf.getAssetExtension(tex, 'KHR_texture_basisu')
        if ext:
            return ext['source']
        else:
            return None

def getTextureIndexNode(exportSettings, glTF, name, shaderNode):
    """
    Return the texture index in the glTF array.
    """

    if shaderNode is None:
        return -1

    if not isinstance(shaderNode, (bpy.types.ShaderNodeBsdfPrincipled,
                                   bpy.types.ShaderNodeMixShader,
                                   bpy.types.ShaderNodeGroup)):
        return -1

    if shaderNode.inputs.get(name) is None:
        return -1

    if len(shaderNode.inputs[name].links) == 0:
        return -1

    fromNode = shaderNode.inputs[name].links[0].from_node

    if isinstance(fromNode, bpy.types.ShaderNodeNormalMap):
        if len(fromNode.inputs['Color'].links) > 0:
            fromNode = fromNode.inputs['Color'].links[0].from_node
        else:
            return -1

    # COMPAT: Blender < 3.3, checking for type
    if hasattr(bpy.types, 'ShaderNodeSeparateColor') and isinstance(fromNode, bpy.types.ShaderNodeSeparateColor):
        if len(fromNode.inputs['Color'].links) > 0:
            fromNode = fromNode.inputs['Color'].links[0].from_node
        else:
            return -1

    # COMPAT: Blender < 3.3
    if isinstance(fromNode, bpy.types.ShaderNodeSeparateRGB):
        if len(fromNode.inputs['Image'].links) > 0:
            fromNode = fromNode.inputs['Image'].links[0].from_node
        else:
            return -1

    # color factor
    if isinstance(fromNode, bpy.types.ShaderNodeMixRGB) and fromNode.blend_type == 'MULTIPLY':
        if len(fromNode.inputs['Color1'].links) > 0:
            fromNode = fromNode.inputs['Color1'].links[0].from_node
        elif len(fromNode.inputs['Color2'].links) > 0:
            fromNode = fromNode.inputs['Color2'].links[0].from_node
        else:
            return -1

    if not isinstance(fromNode, bpy.types.ShaderNodeTexImage):
        return -1

    if getTexImage(fromNode) is None or getTexImage(fromNode).size[0] == 0 or getTexImage(fromNode).size[1] == 0:
        return -1

    return getTextureIndexByTexture(exportSettings, glTF, fromNode)

def getTexcoordIndex(glTF, name, shaderNode):
    """
    Return the texture coordinate index, if assigend and used.
    """

    if shaderNode is None:
        return 0

    if not isinstance(shaderNode, (bpy.types.ShaderNodeBsdfPrincipled,
                                   bpy.types.ShaderNodeMixShader,
                                   bpy.types.ShaderNodeGroup)):
        return 0

    if shaderNode.inputs.get(name) is None:
        return 0

    if len(shaderNode.inputs[name].links) == 0:
        return 0

    fromNode = shaderNode.inputs[name].links[0].from_node

    if isinstance(fromNode, bpy.types.ShaderNodeNormalMap):
        fromNode = fromNode.inputs['Color'].links[0].from_node

    # COMPAT: Blender < 3.3, checking for type
    if hasattr(bpy.types, 'ShaderNodeSeparateColor') and isinstance(fromNode, bpy.types.ShaderNodeSeparateColor):
        fromNode = fromNode.inputs['Color'].links[0].from_node

    # COMPAT: Blender < 3.3
    if isinstance(fromNode, bpy.types.ShaderNodeSeparateRGB):
        fromNode = fromNode.inputs['Image'].links[0].from_node

    if isinstance(fromNode, bpy.types.ShaderNodeMixRGB) and fromNode.blend_type == 'MULTIPLY':
        if len(fromNode.inputs['Color1'].links) > 0:
            fromNode = fromNode.inputs['Color1'].links[0].from_node
        elif len(fromNode.inputs['Color2'].links) > 0:
            fromNode = fromNode.inputs['Color2'].links[0].from_node

    if not isinstance(fromNode, bpy.types.ShaderNodeTexImage):
        return 0

    if len(fromNode.inputs['Vector'].links) == 0:
        return 0

    inputNode = fromNode.inputs['Vector'].links[0].from_node

    if not isinstance(inputNode, bpy.types.ShaderNodeUVMap):
        return 0

    if inputNode.uv_map == '':
        return 0

    # try to gather map index.
    for bl_mesh in bpy.data.meshes:
        texCoordIndex = bl_mesh.uv_layers.find(inputNode.uv_map)
        if texCoordIndex >= 0:
            return texCoordIndex

    return 0

def getMaterialType(bl_mat):
    """
    get blender material type: PBR, EEVEE, BASIC
    """

    if not bl_mat.use_nodes or bl_mat.node_tree == None:
        return 'BASIC'

    if bl_mat.v3d.gltf_compat:
        return 'PBR'

    return 'EEVEE'

def getSkinIndex(glTF, name, index_offset):
    """
    Return the skin index in the glTF array.
    """

    if glTF.get('skins') is None:
        return -1

    skeleton = gltf.getNodeIndex(glTF, name)

    index = 0
    for skin in glTF['skins']:
        if skin['skeleton'] == skeleton:
            return index + index_offset

        index += 1

    return -1

def getCameraIndex(glTF, name):
    """
    Return the camera index in the glTF array.
    """

    if glTF.get('cameras') is None:
        return -1

    index = 0
    for camera in glTF['cameras']:
        if camera['name'] == name:
            return index

        index += 1

    return -1

def getCurveIndex(glTF, name):
    """
    Return the curve index in the glTF array.
    """

    v3dExt = gltf.getAssetExtension(glTF, 'S8S_v3d_curves')

    if v3dExt == None:
        return -1

    if v3dExt.get('curves') == None:
        return -1

    curves = v3dExt['curves']

    index = 0
    for curve in curves:
        if curve['name'] == name:
            return index

        index += 1

    return -1

def getNodeGraphIndex(glTF, name):
    """
    Return the node graph index in the glTF array.
    """

    v3dExt = gltf.getAssetExtension(glTF, 'S8S_v3d_materials')

    if v3dExt == None:
        return -1

    if v3dExt.get('nodeGraphs') == None:
        return -1

    index = 0
    for graph in v3dExt['nodeGraphs']:
        if graph['name'] == name:
            return index

        index += 1

    return -1

def getImageExportedURI(exportSettings, bl_image):
    """
    Return exported URI for a blender image.
    """
    name, ext = os.path.splitext(bpy.path.basename(bl_image.filepath))

    name = name if name != '' else 'v3d_exported_image_' + bl_image.name.lower().replace(' ', '_')

    if imgNeedsCompression(bl_image, exportSettings):
        if bl_image.file_format == 'HDR':
            ext = '.hdr.xz'
        else:
            ext = '.ktx2'
    elif (bl_image.file_format == 'JPEG'
            or bl_image.file_format == 'WEBP'
            or bl_image.file_format == 'BMP'
            or bl_image.file_format == 'HDR'
            or bl_image.file_format == 'PNG'):
        if ext == '':
            ext = '.' + bl_image.file_format.lower()
    else:
        ext = '.png'

    unique_uri = name + ext
    uri_cache = exportSettings['uri_cache']

    i = 0
    while unique_uri in uri_cache['uri']:
        index = uri_cache['uri'].index(unique_uri)
        if uri_cache['bl_datablocks'][index] == bl_image:
            break

        i += 1
        unique_uri = name + '_' + integerToBlSuffix(i) + ext

    return unique_uri

def getImageExportedMimeType(bl_image, exportSettings):

    if imgNeedsCompression(bl_image, exportSettings):
        if bl_image.file_format == 'HDR':
            return 'application/x-xz'
        else:
            return 'image/ktx2'
    elif bl_image.file_format == 'JPEG':
        return 'image/jpeg'
    elif bl_image.file_format == 'WEBP':
        return 'image/webp'
    elif bl_image.file_format == 'BMP':
        return 'image/bmp'
    elif bl_image.file_format == 'HDR':
        return 'image/vnd.radiance'
    else:
        return 'image/png'

def getNameInBrackets(data_path):
    """
    Return Blender node on a given Blender data path.
    """

    if data_path is None:
        return None

    index = data_path.find("[\"")
    if (index == -1):
        return None

    node_name = data_path[(index + 2):]

    index = node_name.find("\"")
    if (index == -1):
        return None

    return node_name[:(index)]

def getAnimParamDim(fcurves, node_name):
    dim = 0

    for fcurve in fcurves:
        if getNameInBrackets(fcurve.data_path) == node_name:
            dim = max(dim, fcurve.array_index+1)

    return dim

def getAnimParam(data_path):
    """
    return animated param in data path:
    nodes['name'].outputs[0].default_value -> default_value
    """

    index = data_path.rfind('.')

    if index == -1:
        return data_path

    return data_path[(index + 1):]

def getScalar(default_value, init_value = 0.0):
    """
    Return scalar with a given default/fallback value.
    """

    return_value = init_value

    if default_value is None:
        return return_value

    return_value = default_value

    return return_value

def getVec3(default_value, init_value = [0.0, 0.0, 0.0]):
    """
    Return vec3 with a given default/fallback value.
    """

    return_value = init_value.copy()

    if default_value is None or len(default_value) < 3:
        return return_value

    index = 0
    for number in default_value:
        return_value[index] = number

        index += 1
        if index == 3:
            return return_value

    return return_value

def getVec4(default_value, init_value = [0.0, 0.0, 0.0, 1.0]):
    """
    Return vec4 with a given default/fallback value.
    """

    return_value = init_value.copy()

    if default_value is None or len(default_value) < 4:
        return return_value

    index = 0
    for number in default_value:
        return_value[index] = number

        index += 1
        if index == 4:
            return return_value

    return return_value

def getIndex(list, name):
    """
    Return index of a glTF element by a given name.
    """

    if list is None or name is None:
        return -1

    index = 0
    for element in list:
        if element.get('name') is None:
            continue

        if element['name'] == name:
            return index

        index += 1

    return -1

def getByName(list, name):
    """
    Return element by a given name.
    """

    if list is None or name is None:
        return None

    for element in list:
        if element.get('name') is None:
            continue

        if element['name'] == name:
            return element

    return None

def getOrCreateDefaultMatIndex(glTF):
    def_idx = gltf.getMaterialIndex(glTF, DEFAULT_MAT_NAME)

    if def_idx == -1:
        if 'materials' not in glTF:
            glTF['materials'] = []

        glTF['materials'].append(createDefaultMaterial())

        def_idx = len(glTF['materials']) - 1

    return def_idx

def createDefaultMaterial():
    return {
        "extensions" : {
            "S8S_v3d_materials" : {
                "nodeGraph" : {
                    "edges" : [
                        {
                            "fromNode" : 1,
                            "fromOutput" : 0,
                            "toInput" : 0,
                            "toNode" : 0
                        }
                    ],
                    "nodes" : [
                        {
                            "inputs" : [
                                [ 0, 0, 0, 0 ],
                                [ 0, 0, 0, 0 ],
                                [ 0, 0, 0 ]
                            ],
                            "is_active_output" : True,
                            "name" : "Material Output",
                            "outputs" : [],
                            "type" : "OUTPUT_MATERIAL_BL"
                        },
                        {
                            "inputs" : [
                                [ 0.800000011920929, 0.800000011920929, 0.800000011920929, 1.0 ],
                                0.0,
                                [ 0.0, 0.0, 0.0 ]
                            ],
                            "is_active_output" : False,
                            "name" : "Diffuse BSDF",
                            "outputs" : [
                                [ 0, 0, 0, 0 ]
                            ],
                            "type" : "BSDF_DIFFUSE_BL"
                        }
                    ]
                },
                "useCastShadows" : True,
                "useShadows" : True
            }
        },
        "name" : DEFAULT_MAT_NAME
    }

def getFontPath(bl_font):

    path = bl_font.filepath
    abspath = bpy.path.abspath(bl_font.filepath)

    # handle missing font as well
    if path == '<builtin>' or not os.path.isfile(abspath):
        return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fonts', 'bfont.woff')
    else:
        return abspath

def getFontExportedURI(bl_font):

    path = bl_font.filepath
    abspath = bpy.path.abspath(bl_font.filepath)

    # handle missing font as well
    if path == '<builtin>' or not os.path.isfile(abspath):
        return 'bfont.woff'
    else:
        return bpy.path.basename(path)

def getFontExportedMimeType(bl_font):

    path = bl_font.filepath
    abspath = bpy.path.abspath(bl_font.filepath)

    # handle missing font as well
    if path == '<builtin>' or not os.path.isfile(abspath):
        return 'font/woff'
    else:
        return 'font/ttf'


from neuralnetwork import NeuralNetwork
from readdata import read_data
from functions import *

dataset = read_data()
train, test = train_test_split(dataset, 0.7)

#print(len(train))
#print(len(test))

inputs_num = len(dataset[0]) - 1
hidden_layer = 1
class_labels = get_class_labels(dataset)
outputs_num = len(class_labels)

l_rate = 0.05
epoch = 10000

model = NeuralNetwork(inputs_num, hidden_layer, outputs_num)
#model = NeuralNetwork(inputs_num, 2, outputs_num, 10, 10)
training(l_rate, epoch, train, test, model.network, class_labels)

#model.show_weights()


from text_to_num import alpha2digit
import json
from tensorflow.keras.preprocessing.sequence import pad_sequences
from pickle import load
import numpy as np
from tensorflow.keras.models import load_model
model = load_model('nonamemodel.h5')
tokenizer = load(open('nonametokenizer.pkl', 'rb'))

#find 3 highest scorers
def get_high_scores(box):
    hi = [0,0,0]
    best=[0,0,0]
    for k in box["PTS"].keys():
        val = box["PTS"][k]
        if val.isnumeric():
            val = int(val)
            for i in range(3):
                if val>=hi[i]:
                    hi.insert(i,val)
                    best.insert(i,k)
                    break
        hi = hi[:3]
        best = best[:3]
    return best

# fill out dictionary with stats of highest scorers
def get_stats(best,box):
    tags_dict = {"FIRST_NAME":[],"SECOND_NAME":[],"PTS":[],"AST":[],"REB":[]}
    for tag in tags_dict:
        for player in best:
            tags_dict[tag].append(box[tag][player])
    return tags_dict

with open('../test.json','r') as fr: #read in test data
        d = fr.read()
        full_dict = json.loads(d)

for game in full_dict: #for game in test
    #generate sentence
    seq_length = 15
    result = list()
    out_word = ''
    in_text = []
    count = 0
    while not (out_word=="." or out_word=="*end*") and count<101:  #while not at end of sentence
            # encode the text as integer
            encodedp = tokenizer.texts_to_sequences([in_text])[0]
            words = encodedp
            # truncate sequences to a fixed length
            words_encoded = pad_sequences([words], maxlen=seq_length, truncating='pre',value=1,padding='post')
            # predict probabilities for each word
            pred = model.predict(words_encoded)
            yhat = np.random.choice(len(pred[0]), p=pred[0]) #pick by distribution
            #yhat = np.argmax(pred, axis=-1) #if you want max
            # map predicted word index to word
            out_word = ''
            for word, index in tokenizer.word_index.items():
                if index == yhat:
                    out_word = word
                    break
            # append to input
            in_text.append(out_word)
            result.append(out_word)
            count+=1

    #Get high scorers
    h = get_high_scores(game["box_score"])

    #get stats of high scorers
    stats_dict = get_stats(h,game["box_score"])

    tags = {"*firsttname*":"FIRST_NAME","*lastname*":"SECOND_NAME","*ptsval*":"PTS","*astval*":"AST","*rebval*":"REB"}
    tags_used = {"*firsttname*":0,"*lastname*":0,"*ptsval*":0,"*astval*":0,"*rebval*":0}

    #replace keys
    for i,tok in enumerate(result):
        if tok in tags.keys():
            result[i]=stats_dict[tags[tok]][tags_used[tok]]
            tags_used[tok]= (tags_used[tok]+1)%3 #use next player next time

    with open('results.txt','a') as fw:
        fw.write(' '.join(result)+ " \r\n") #write to file


from mininet.topo import Topo

class MyTopo( Topo ):
    "Simple topology example"

    def __init__( self ):
        "Create custom topo. "
        #Initialize topology
        Topo.__init__( self )

        s1 = self.addSwitch('s1', failmode='standalone', stp=True)
        s2 = self.addSwitch('s2', failmode='standalone', stp=True)
        s3 = self.addSwitch('s3', failmode='standalone', stp=True)
        s4 = self.addSwitch('s4', failmode='standalone', stp=True)
        """ current working
        h1 = self.addHost('h1', mac='40:0E:1A:00:00:00')
        h2 = self.addHost('h2', mac='C0:A5:2C:00:00:00')

        hex
        h1 : 1A 0E 4
        h2 : 2C A5 C
        """

        h1 = self.addHost('h1', mac='40:0E:1A:00:00:00')
        h2 = self.addHost('h2', mac='C0:A5:2C:00:00:00')

        self.addLink(s1, h1, port1=1, port2=0)
        self.addLink(s4, h2, port1=4, port2=0)

        self.addLink(s1, s2, port1=2, port2=1)
        self.addLink(s1, s3, port1=3, port2=1)
        self.addLink(s1, s4, port1=4, port2=2)

        self.addLink(s2, s4, port1=2, port2=3)

        self.addLink(s3, s4, port1=2, port2=1)

topos = { 'mytopo': (lambda:MyTopo()) }


import pandas, xlsxwriter, progressbar, datetime

def export(storage):
    if storage != []:
        now = datetime.datetime.now()
        filename = "bikescrape-%d-%d-%d-%d%d%d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)
        writer = pandas.ExcelWriter('%s.xlsx' % filename, engine='xlsxwriter')
        export = pandas.DataFrame(data=storage, columns=['Listing', 'Price', 'Link', 'Region'])
        export.to_excel(writer, sheet_name='Listings', index=False)
        worksheet = writer.sheets['Listings']
        worksheet.set_default_row(30)
        worksheet.set_column(0, 0, 45)
        worksheet.set_column(1, 1, 8)
        worksheet.set_column(2, 2, 80)
        worksheet.set_column(3, 3, 25)
        writer.book.formats[0].set_text_wrap(True)
        print("Exporting to %s.xlsx" % filename)
        writer.save()
        return ("%s.xlsx" %filename)
    else:
        print("Nothing to export :(")
        return None


import logging

from ..akamai_utils.client import AkamaiApiClient

from nodestream.pipeline.extractors import Extractor

class AkamaiWAFExtractor(Extractor):
    def __init__(self, **akamai_client_kwargs) -> None:
        self.client = AkamaiApiClient(**akamai_client_kwargs)
        self.logger = logging.getLogger(self.__class__.__name__)

    async def extract_records(self):
        ag_map = {
            'POLICY':'Web Policy Violation',
            'WAT':'Web Attack Tool',
            'TOOL': 'Web Attack Tool',
            'PROTOCOL':'Web Protocol Attack',
            'SQL':'SQL Injection',
            'XSS':'Cross Site Scripting',
            'CMD':'Command Injection',
            'CMDI': 'Command Injection',
            'LFI':'Local File Inclusion',
            'RFI':'Remote File Inclusion',
            'PLATFORM':'Web Platform Attack',
            'OUTBOUND':'Total Outbound'
        }

        for config in self.client.list_appsec_configs():
            try:
                if 'productionVersion' in config.keys():
                    export = self.client.export_appsec_config(config_id = config['id'], config_version = config['productionVersion'])
                    # Construct output dict
                    output_config = {
                        'configId': config['id'],
                        'configName': config['name'],
                        'productionVersion': config['productionVersion'],
                        'policies': []
                    }
                    # Iterate through policies and add custom dict to output
                    for policy in export['securityPolicies']:
                        output_policy = {
                            'policyId': policy['id'],
                            'policyName': policy['name'],
                            'attackGroupActions': []
                        }
                        if 'webApplicationFirewall' in policy.keys():
                            for action in policy['webApplicationFirewall']['attackGroupActions']:
                                output_policy['attackGroupActions'].append({
                                        'group': action['group'],
                                        'groupName': ag_map[action['group']],
                                        'action': action['action']
                                    }
                                )
                            output_config['policies'].append(output_policy)

                    yield output_config
                else:
                    self.logger.warning(f"No production version exists for config {config['name']}: {err}")
            except Exception as err:
                self.logger.error(f"Failed to export appsec configuration {config['name']}: {err}")


'''
    进行数据预处理，产生三种数据：
    imu数据：关节点顺序为右手左手、右脚左脚、头、根，acc（18）+ori（54），已经经过标准化。(加速度没有除以缩放因子)
    joint数据：按SMPL顺序的23个节点坐标，已经经过根节点标准化。
    pose：SMPL姿态参数真值，24个关节数据。
'''

import os
os.environ["CUDA_VISIBLE_DEVICES"] = '0'    # debug专用
import pickle
import torch
import numpy as np
from tqdm import tqdm
import glob

import articulate as art
from config import *
import data.utils.bvh as bvh
import data.utils.quat as quat
import config as conf

vi_mask = torch.tensor([1961, 5424, 1176, 4662, 411, 3021])
# vi_mask = torch.tensor([3021, 1176, 4662, 411, 1961, 5424])
ji_mask = torch.tensor([18, 19, 4, 5, 15, 0])   # 右手左手、右脚左脚、头、根
# ji_mask = torch.tensor([0, 4, 5, 15, 18, 19])
body_model = art.ParametricModel(paths.smpl_file)

def _syn_acc(v, smooth_n=2): # 4
    r"""
    Synthesize accelerations from vertex positions.
    """
    mid = smooth_n // 2
    acc = torch.stack([(v[i] + v[i + 2] - 2 * v[i + 1]) * 3600 for i in range(0, v.shape[0] - 2)])
    acc = torch.cat((torch.zeros_like(acc[:1]), acc, torch.zeros_like(acc[:1])))
    if mid != 0:
        acc[smooth_n:-smooth_n] = torch.stack(
            [(v[i] + v[i + smooth_n * 2] - 2 * v[i + smooth_n]) * 3600 / smooth_n ** 2
             for i in range(0, v.shape[0] - smooth_n * 2)])
    return acc

def _syn_vel(joints, root_rot):
    r"""
    Synthesize velocity from joints positions.

    >> joints: joints global position, can remake shape to [batch_size, 24, 3]
    >> root_rot: ground truth root rotation, can remake shape to [batch_size, 3, 3]
    """
    joints = joints.view(-1, 24, 3)
    root_rot = root_rot.view(-1, 3, 3)
    vel = torch.stack([(joints[i] - joints[i-1]) * 60 for i in range(1, joints.shape[0])])
    vel = torch.cat((torch.zeros_like(vel[:1]), vel))
    vel = vel.view(-1, 24, 3) / vel_scale
    # vel = root_rot.transpose(1, 2).matmul(vel.transpose(1, 2))
    # print(vel)
    return vel

def process_amass():
    data_pose, data_trans, data_beta, length = [], [], [], []
    for ds_name in amass_data:
    # for ds_name in amass_data_test_tmp:
        print('\rReading', ds_name)
        for npz_fname in tqdm(glob.glob(os.path.join(paths.raw_amass_dir, ds_name, ds_name, '*/*_poses.npz'))):
            try: cdata = np.load(npz_fname)
            except: continue

            framerate = int(cdata['mocap_framerate'])
            if framerate == 120: step = 2
            elif framerate == 60 or framerate == 59: step = 1
            else: continue

            data_pose.extend(cdata['poses'][::step].astype(np.float32))
            data_trans.extend(cdata['trans'][::step].astype(np.float32))
            data_beta.append(cdata['betas'][:10])
            length.append(cdata['poses'][::step].shape[0])

    assert len(data_pose) != 0, 'AMASS dataset not found. Check config.py or comment the function process_amass()'
    length = torch.tensor(length, dtype=torch.int)
    shape = torch.tensor(np.asarray(data_beta, np.float32))
    tran = torch.tensor(np.asarray(data_trans, np.float32))
    pose = torch.tensor(np.asarray(data_pose, np.float32)).view(-1, 52, 3)
    pose[:, 23] = pose[:, 37]     # right hand
    pose = pose[:, :24].clone()   # only use body

    # align AMASS global fame with DIP
    amass_rot = torch.tensor([[[1, 0, 0], [0, 0, 1], [0, -1, 0.]]])
    tran = amass_rot.matmul(tran.unsqueeze(-1)).view_as(tran)
    pose[:, 0] = art.math.rotation_matrix_to_axis_angle(
        amass_rot.matmul(art.math.axis_angle_to_rotation_matrix(pose[:, 0])))

    print('Synthesizing IMU accelerations and orientations')
    b = 0
    out_pose = []
    out_imus = []
    out_gt_pose = []
    out_joint = []
    out_shape = []
    for i, l in tqdm(list(enumerate(length))):
        if l <= 12: b += l; print('\tdiscard one sequence with length', l); continue
        p = art.math.axis_angle_to_rotation_matrix(pose[b:b + l]).view(-1, 24, 3, 3)
        grot, joint, vert = body_model.forward_kinematics(p, shape[i], tran[b:b + l], calc_mesh=True)

        # out_pose.append(pose[b:b + l].clone())  # N, 24, 3
        a_out_pose = pose[b:b + l].clone()  # 24,3
        a_out_pose_ = art.math.axis_angle_to_quaternion(a_out_pose).view(-1,24,4) # 24,4
        a_out_pose_ = a_out_pose_[:,joint_set.graphA]
        out_pose.append(a_out_pose_.detach().numpy())

        a_pose_gt = art.math.axis_angle_to_rotation_matrix(a_out_pose).view(-1, 24, 3, 3)
        out_gt_pose.append(a_pose_gt.detach().numpy())

        a_joint_all = joint[:, :24]
        p_all = torch.cat((a_joint_all[:,0:1], a_joint_all[:,1:]-a_joint_all[:,0:1]), dim=1)
        out_p_all = p_all[:, conf.joint_set.full, :]
        out_joint.append(out_p_all.detach().numpy())

        glb_acc = _syn_acc(vert[:, vi_mask])   # t,6,3
        glb_ori = grot[:, ji_mask]  # t,6,3,3

        # acc_tmp = torch.cat((a_out_vacc[:, :1], a_out_vacc[:, 1:] - a_out_vacc[:, :1]), dim=1).bmm(a_out_vrot[:, 0]) / conf.acc_scale
        # ori_tmp = torch.cat((a_out_vrot[:, :1], a_out_vrot[:, :1].transpose(2, 3).matmul(a_out_vrot[:, 1:])), dim=1)
        # ori_tmp = quat.from_xform(ori_tmp.detach().numpy()) # 6,4
        # acc_tmp = acc_tmp.detach().numpy()
        # a_out_imu = np.concatenate((acc_tmp, ori_tmp), axis=-1)   # N,6,7

        acc_tmp = torch.cat((glb_acc[:, :5] - glb_acc[:, 5:], glb_acc[:, 5:]), dim=1).bmm(glb_ori[:, -1])# / acc_scale
        ori_tmp = torch.cat((glb_ori[:, 5:].transpose(2, 3).matmul(glb_ori[:, :5]), glb_ori[:, 5:]), dim=1)
        a_out_imu = torch.cat((acc_tmp.flatten(1), ori_tmp.flatten(1)), dim=1)

        out_imus.append(a_out_imu)

        a_shape = shape[i].repeat(a_pose_gt.shape[0],1)
        out_shape.append(a_shape)

        b += l

    print('Saving')
    os.makedirs(paths.amass_dir, exist_ok=True)
    # np.save(os.path.join(paths.amass_dir, 'Smpl_amass_test_motion.npy'), out_pose)
    # np.save(os.path.join('data/dataset_work/GGIP_used/amass', 'Smpl_amass_joints23.npy'), out_joint)
    # np.save(os.path.join('data/dataset_work/GGIP_used/amass', 'Smpl_amass_imus.npy'), out_imus)
    np.save(os.path.join('data/dataset_work/GGIP_used/amass', 'Smpl_amass_motion_SMPL24.npy'), [out_gt_pose, out_shape])
    # np.save(os.path.join('data/dataset_work/GGIP_used/amass', 'Smpl_amass_shape.npy'), out_shape)

def process_dipimu():
    imu_mask = [7, 8, 11, 12, 0, 2]
    # imu_mask = [2, 11, 12, 0, 7, 8] # 根、左脚右脚、头、左手右手
    train_split = ['s_01','s_02','s_03','s_04','s_05','s_06','s_07','s_08']
    # train_split = ['s_09','s_10']
    accs, oris, poses, trans = [], [], [], []

    # for subject_name in test_split:
    for subject_name in train_split:
        for motion_name in os.listdir(os.path.join(paths.raw_dipimu_dir, subject_name)):
            path = os.path.join(paths.raw_dipimu_dir, subject_name, motion_name)
            data = pickle.load(open(path, 'rb'), encoding='latin1')
            acc = torch.from_numpy(data['imu_acc'][:, imu_mask]).float()
            ori = torch.from_numpy(data['imu_ori'][:, imu_mask]).float()
            pose = torch.from_numpy(data['gt']).float()

            # fill nan with nearest neighbors
            for _ in range(4):
                acc[1:].masked_scatter_(torch.isnan(acc[1:]), acc[:-1][torch.isnan(acc[1:])])
                ori[1:].masked_scatter_(torch.isnan(ori[1:]), ori[:-1][torch.isnan(ori[1:])])
                acc[:-1].masked_scatter_(torch.isnan(acc[:-1]), acc[1:][torch.isnan(acc[:-1])])
                ori[:-1].masked_scatter_(torch.isnan(ori[:-1]), ori[1:][torch.isnan(ori[:-1])])

            acc, ori, pose = acc[6:-6], ori[6:-6], pose[6:-6]
            if torch.isnan(acc).sum() == 0 and torch.isnan(ori).sum() == 0 and torch.isnan(pose).sum() == 0:
                accs.append(acc.clone())
                oris.append(ori.clone())
                poses.append(pose.clone())
                trans.append(torch.zeros(pose.shape[0], 3))  # dip-imu does not contain translations
            else:
                print('DIP-IMU: %s/%s has too much nan! Discard!' % (subject_name, motion_name))

    length = len(accs)

    out_pose = []
    out_imus = []
    out_gt_pose = []
    out_joint = []
    out_shape = []
    for i in tqdm(range(length)):
        p = art.math.axis_angle_to_rotation_matrix(poses[i]).view(-1, 24, 3, 3)
        grot, joint = body_model.forward_kinematics(p)
        a_joint_all = joint[:, :24]
        p_all = torch.cat((a_joint_all[:,0:1], a_joint_all[:,1:]-a_joint_all[:,0:1]), dim=1)
        out_p_all = p_all[:, conf.joint_set.full, :]
        out_joint.append(out_p_all.detach().numpy())

        a_out_pose = poses[i].clone().view(-1,24,3)  # 24,3
        a_out_pose_ = art.math.axis_angle_to_quaternion(a_out_pose).view(-1,24,4) # 24,4
        a_out_pose_ = a_out_pose_[:,joint_set.graphA]
        out_pose.append(a_out_pose_.detach().numpy())

        a_pose_gt = art.math.axis_angle_to_rotation_matrix(a_out_pose).view(-1, 24, 3, 3)
        out_gt_pose.append(a_pose_gt.detach().numpy())

        glb_acc = accs[i].clone().view(-1,6,3)   # 6,3
        glb_ori = oris[i].clone().view(-1,6,3,3)  # 6,3
        # acc_tmp = torch.cat((a_out_vacc[:, :1], a_out_vacc[:, 1:] - a_out_vacc[:, :1]), dim=1).bmm(a_out_vrot[:, 0]) / conf.acc_scale
        # ori_tmp = torch.cat((a_out_vrot[:, :1], a_out_vrot[:, :1].transpose(2, 3).matmul(a_out_vrot[:, 1:])), dim=1)
        # ori_tmp = quat.from_xform(ori_tmp.detach().numpy()) # 6,4
        # acc_tmp = acc_tmp.detach().numpy()
        # a_out_imu = np.concatenate((acc_tmp, ori_tmp), axis=-1)   # N,6,7
        acc_tmp = torch.cat((glb_acc[:, :5] - glb_acc[:, 5:], glb_acc[:, 5:]), dim=1).bmm(glb_ori[:, -1])# / acc_scale
        ori_tmp = torch.cat((glb_ori[:, 5:].transpose(2, 3).matmul(glb_ori[:, :5]), glb_ori[:, 5:]), dim=1)
        a_out_imu = torch.cat((acc_tmp.flatten(1), ori_tmp.flatten(1)), dim=1)

        out_imus.append(a_out_imu)

        a_shape = torch.zeros(a_pose_gt.shape[0], 10)
        out_shape.append(a_shape)

    print('Saving')
    os.makedirs(paths.amass_dir, exist_ok=True)
    # np.save(os.path.join(paths.amass_dir, 'Smpl_dipTrain_motion.npy'), out_pose)
    # np.save(os.path.join('data/dataset_work/GGIP_used/dip', 'Smpl_dipTrain_imus.npy'), out_imus)
    np.save(os.path.join('data/dataset_work/GGIP_used/dip', 'Smpl_dipTrain_motion_SMPL24.npy'), [out_gt_pose,out_shape])
    # np.save(os.path.join('data/dataset_work/GGIP_used/dip', 'Smpl_dipTrain_joints23.npy'), out_joint)

def process_totalcapture():
    inches_to_meters = 0.0254
    file_name = 'gt_skel_gbl_pos.txt'
    imuOrder = [2, 3, 0, 1, 4, 5]
    # imuOrder = [5,0,1,4,2,3]

    accs, oris, poses, trans = [], [], [], []
    # for file in sorted(os.listdir(paths.raw_totalcapture_dip_dir)):
    for file_folder in sorted(os.listdir(paths.raw_totalcapture_dip_dir)):
        for file in sorted(os.listdir(os.path.join(paths.raw_totalcapture_dip_dir, file_folder))):
            data = pickle.load(open(os.path.join(paths.raw_totalcapture_dip_dir, file_folder, file), 'rb'), encoding='latin1')
            ori = torch.from_numpy(data['ori']).float()[:, torch.tensor(imuOrder)]
            acc = torch.from_numpy(data['acc']).float()[:, torch.tensor(imuOrder)]
            pose = torch.from_numpy(data['gt']).float().view(-1, 24, 3)

            # acc/ori and gt pose do not match in the dataset
            if acc.shape[0] < pose.shape[0]:
                pose = pose[:acc.shape[0]]
            elif acc.shape[0] > pose.shape[0]:
                acc = acc[:pose.shape[0]]
                ori = ori[:pose.shape[0]]

            assert acc.shape[0] == ori.shape[0] and ori.shape[0] == pose.shape[0]
            accs.append(acc)    # N, 6, 3
            oris.append(ori)    # N, 6, 3, 3
            poses.append(pose)  # N, 24, 3

    for subject_name in ['S1', 'S2', 'S3', 'S4', 'S5']:
        for motion_name in sorted(os.listdir(os.path.join(paths.raw_totalcapture_official_dir, subject_name))):
            if subject_name == 'S5' and motion_name == 'acting3':
                continue   # no SMPL poses
            f = open(os.path.join(paths.raw_totalcapture_official_dir, subject_name, motion_name, file_name))
            line = f.readline().split('\t')
            index = torch.tensor([line.index(_) for _ in ['LeftFoot', 'RightFoot', 'Spine']])
            pos = []
            while line:
                line = f.readline()
                pos.append(torch.tensor([[float(_) for _ in p.split(' ')] for p in line.split('\t')[:-1]]))
            pos = torch.stack(pos[:-1])[:, index] * inches_to_meters
            pos[:, :, 0].neg_()
            pos[:, :, 2].neg_()
            trans.append(pos[:, 2] - pos[:1, 2])   # N, 3

    # match trans with poses
    for i in range(len(accs)):
        if accs[i].shape[0] < trans[i].shape[0]:
            trans[i] = trans[i][:accs[i].shape[0]]
        assert trans[i].shape[0] == accs[i].shape[0]

    # remove acceleration bias
    for iacc, pose, tran in zip(accs, poses, trans):
        pose = art.math.axis_angle_to_rotation_matrix(pose).view(-1, 24, 3, 3)
        _, _, vert = body_model.forward_kinematics(pose, tran=tran, calc_mesh=True)
        vacc = _syn_acc(vert[:, vi_mask])
        for imu_id in range(6):
            for i in range(3):
                d = -iacc[:, imu_id, i].mean() + vacc[:, imu_id, i].mean()
                iacc[:, imu_id, i] += d

    # 提取用于val训练结果的tc数据
    length = len(accs)
    out_pose, out_imus = [], []
    out_gt_pose = []
    for i in tqdm(range(length)):
        # out_pose.append(poses[i].clone())  # N, 24, 3
        a_out_pose = poses[i].clone().view(-1,24,3)  # 24,3
        a_out_pose_ = art.math.axis_angle_to_quaternion(a_out_pose).view(-1,24,4) # 24,4
        a_out_pose_ = a_out_pose_[:,joint_set.graphA]
        out_pose.append(a_out_pose_.detach().numpy())

        a_pose_gt = art.math.axis_angle_to_rotation_matrix(a_out_pose).view(-1, 24, 3, 3)
        out_gt_pose.append(a_pose_gt.detach().numpy())

        # out_vacc.append(accs[i].clone())  # N, 6, 3
        # out_vrot.append(oris[i].clone())  # N, 6, 3, 3
        glb_acc = accs[i].clone().view(-1,6,3)   # 6,3
        glb_ori = oris[i].clone().view(-1,6,3,3)  # 6,3
        # acc_tmp = torch.cat((a_out_vacc[:, :1], a_out_vacc[:, 1:] - a_out_vacc[:, :1]), dim=1).bmm(a_out_vrot[:, 0]) / conf.acc_scale
        # ori_tmp = torch.cat((a_out_vrot[:, :1], a_out_vrot[:, :1].transpose(2, 3).matmul(a_out_vrot[:, 1:])), dim=1)
        # ori_tmp = quat.from_xform(ori_tmp.detach().numpy()) # 6,4
        # acc_tmp = acc_tmp.detach().numpy()
        # a_out_imu = np.concatenate((acc_tmp, ori_tmp), axis=-1)   # N,6,7
        acc_tmp = torch.cat((glb_acc[:, :5] - glb_acc[:, 5:], glb_acc[:, 5:]), dim=1).bmm(glb_ori[:, -1])# / acc_scale
        ori_tmp = torch.cat((glb_ori[:, 5:].transpose(2, 3).matmul(glb_ori[:, :5]), glb_ori[:, 5:]), dim=1)
        a_out_imu = torch.cat((acc_tmp.flatten(1), ori_tmp.flatten(1)), dim=1)

        out_imus.append(a_out_imu)
    print('Saving')
    os.makedirs(paths.amass_dir, exist_ok=True)
    # np.save(os.path.join('data/dataset_work/GGIP_used/tc', 'Smpl_tc_motion_test.npy'), out_pose)
    np.save(os.path.join('data/dataset_work/GGIP_used/tc', 'Smpl_tc_imus_test.npy'), out_imus)
    np.save(os.path.join('data/dataset_work/GGIP_used/tc', 'Smpl_tc_motion_SMPL24_test.npy'), out_gt_pose)

if __name__ == '__main__':
    process_amass()
    # process_dipimu()
    # process_totalcapture()


#!/usr/bin/env python3

import matplotlib
matplotlib.use('Agg')

from matplotlib import pyplot as plt
from matplotlib.ticker import MultipleLocator, FuncFormatter

import argparse
import itertools
import numpy as np
import os
import pylab
import seaborn as sns
import sys

import base

from colorpy import ciexyz
from colorpy import colormodels

def main(inputs, output, colors):

    highlight, bi = base.init()
    if len(inputs) <= 1:
        _, ax = base.create_figure(6, 3)
    else:
        _, ax = base.create_figure(6, 2)

    xlim = (300, 700) if len(inputs) > 1 else None
    ymax = np.NINF

    for fname, highlight in itertools.zip_longest(inputs, colors):
        x = np.loadtxt(fname)
        Y = x[:, 1]
        x = x[:, 0]
        ymax = max(Y.max(), ymax)

        if xlim:
            i = np.argwhere(np.logical_and(xlim[0] <= x, x <= xlim[1]))
            Y = Y[i].flatten()
            x = x[i].flatten()

        a = np.trapz(Y)
        mean = np.dot(x, Y / a)

        if highlight is None:
            xyz = ciexyz.xyz_from_wavelength(mean)
            highlight = colormodels.irgb_string_from_xyz(xyz)

        if xlim:
            delta = x[1] - x[0]
            x = np.hstack(([xlim[0], x[0] - delta], x,
                           [x[-1] + delta, xlim[1]]))
            Y = np.hstack(([0, 0], Y, [0, 0]))

        plt.plot(x, Y, color=highlight)

    if xlim:
        plt.xlim(xlim)

    plt.xlabel("Wavelength (nm)")
    plt.ylim(None, (ymax *2 //1 +1) /2)
    plt.ylabel("Voltage (V)", labelpad=9)

    base.finalize(None, bi)

    #pylab.setp(ax.xaxis.get_majorticklabels(), rotation=45, ha='right')
    plt.tight_layout()

    base.save_figure(output)
    return 0

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("data", metavar="TSV", action="store", type=str, nargs="+",
                        help="the data to plot.")
    parser.add_argument("output", metavar="OUT", action="store", type=str,
                        help="the file to plot to. This also specifies the output type.")
    parser.add_argument("--colors", action="store", nargs="+", default=[],
                        help="the colors to be used for the LED plots.")

    args = parser.parse_args()
    sys.exit(main(args.data, args.output, args.colors))


class Normalize:
    def __init__(self, operand):
        self.binInput = operand.binary
        self.binExponent = operand.exponent

    def normalize(self):
        binary = self.getBinInput()
        exponent = self.getBinExponent()

        newBinary = list(binary)
        radixLoc = 0
        oneLoc = 0

        # look for the decimal point
        for i, char in enumerate(binary):
            if char == '.':
                radixLoc = i
                break

        # look for first 1
        for i, char in enumerate(binary):
            if char == '1':
                oneLoc = i
                break

        if binary[0] == '-':  # if value is negative
            del newBinary[radixLoc]
            if oneLoc > 0:
                if radixLoc < oneLoc:
                    del newBinary[1:oneLoc - 1]  # trim zeroes at the start
                else:
                    del newBinary[1:oneLoc]  # trim zeroes at the start
            newBinary.insert(2, '.')  # add radix point after first 1

            # modify exponent value
            if radixLoc < oneLoc:
                exponent = exponent - (oneLoc - radixLoc)
            elif oneLoc < radixLoc:
                exponent = exponent + ((radixLoc - 1) - oneLoc)
        else:
            # generate new string
            del newBinary[radixLoc]  # delete radix point
            if oneLoc > 0:
                if radixLoc < oneLoc:
                    del newBinary[:oneLoc - 1]  # trim zeroes at the start
                else:
                    del newBinary[:oneLoc]  # trim zeroes at the start

            newBinary.insert(1, '.')  # add radix point after first 1

            # modify exponent value
            if radixLoc < oneLoc:
                exponent = exponent - (oneLoc - radixLoc)
            elif oneLoc < radixLoc:
                exponent = exponent + ((radixLoc - 1) - oneLoc)

        self.setBinInput(''.join(newBinary))
        self.setBinExponent(exponent)

    def getBinInput(self):
        return self.binInput

    def getBinExponent(self):
        return self.binExponent

    def setBinInput(self, binInput):
        self.binInput = binInput

    def setBinExponent(self, binExponent):
        self.binExponent = binExponent


import random
import matplotlib.pyplot as plt

N = 10
P = 10

fitness_list = []  # All 3 used for plotting
fittest_list = []
average = []

class Individual(object):
    def __init__(self):
        self.gene = [0] * N
        self.fitness = 0

def initialise_pop():

    pop = []
    for i in range(0, P):
        temp = []
        fitness = 0
        for x in range(0, N):
            r = random.randint(0, 1)
            temp.append(r)  # Create binary strings chromosomes
            if r == 1:
                fitness += 1      # Give fitness value

        ind = Individual()
        ind.gene = temp.copy()
        ind.fitness = fitness
        pop.append(ind)   # Create a population
    return pop

population = initialise_pop()    # Initial population

print("Initial population")
for obj in population:
    print(obj.gene, obj.fitness)

def plot_value(pop):
    total_fitness = 0
    fittest = 0

    for i in range(0, len(pop)-1):
        ind = pop[i]                      # Get total fitness of the population
        total_fitness += ind.fitness

        if ind.fitness > fittest:           # Get the fittest individual
            fittest = ind.fitness

    av = total_fitness / len(pop)        # Get the average fitness of the population
    average.append(av)
    fitness_list.append(total_fitness)
    fittest_list.append(fittest)

    print("____________________________________________")
    print("Total fitness of the population: ", total_fitness)
    print(" ")
    print("The average fitness of the population: ", av)
    print(" ")
    print("Fittest individual: ", fittest)
    print("____________________________________________")

plot_value(population)     # Initial population values

def selection(pop):
    offs = []

    for n in range(0, len(pop)):
        p1 = random.randint(0, len(pop)-1)
        off1 = pop[p1]

        p2 = random.randint(0, len(pop)-1)
        off2 = pop[p2]

        if off1.fitness > off2.fitness:
            offs.append(off1)  # Selecting the better individuals from the tournament
        else:
            offs.append(off2)

        print(" ")
        print("Selected individuals for tournament: ")
        print(n, off1.gene, off1.fitness)  # Print competing individuals
        print(n+1, off2.gene, off2.fitness)
    print(" ")
    print("Selected parents: ")  # Print the chosen individuals (temporary offsprings)
    for ind in offs:
        print(ind.gene, " Fitness: ", ind.fitness)
        print("____________________________________________")

    return offs

# offspring = selection()

def crossover(pop):
    offs = []

    p = 0
    while p < len(pop):

        parent1 = pop[p]
        p += 1

        parent2 = pop[p]
        p += 1

        n = random.randint(0, N - 1)  # n is the point where cross over will occur
        i = 0
        child1 = []
        child2 = []
        while i < n:
            child1.append(parent1.gene[i])  # Up until n, the child receives the same genes as its parent
            child2.append(parent2.gene[i])
            i += 1

        child1.append(parent2.gene[n])  # The gene under n position is the first gene on each tail (to be swapped)
        child2.append(parent1.gene[n])

        i = n
        while i < N - 1:
            i += 1
            child2.append(parent1.gene[i])  # Tails being swapped (from parent1 to child2 and etc.)
            child1.append(parent2.gene[i])

        ind = Individual()
        ind.gene = child2        # Populate new generation
        ind.fitness = 0
        offs.append(ind)

        ind2 = Individual()
        ind2.gene = child1
        ind.fitness = 0
        offs.append(ind2)

        print(" ")
        print("Individuals to crossover: ")
        print(" ")
        print(parent1.gene, parent1.fitness)
        print(parent2.gene, parent2.fitness)
        print("Crossing point = ", n + 1)
        print("____________________________________________")
        print(" ")
        print("After crossover: ")
        print(" ")
        print(child1)
        print(child2)
        print("____________________________________________")

    return offs

# recombined = crossover()

def mutation(pop):
    mut_rate = 0.04
    new_pop = []

    print("")
    print("Mutation: ")
    print(" ")
    for p in range(0, len(pop)):
        ind = pop[p]
        for i in range(0, N):
            mut_prob = random.randint(0, 100)
            if mut_prob < (100 * mut_rate):  # MUTATION
                print("Mutation occurred on gene ", i + 1)
                if ind.gene[i] == 1:
                    ind.gene[i] = 0
                else:
                    ind.gene[i] = 1
                print(ind.gene)

        new_pop.append(ind)

    return new_pop

# new_generation = mutation()

def fitness_value(pop):

    for ind in pop:
        gene = ind.gene
        fitness = 0            # Give fitness value to a chosen population
        for i in range(0, N):
            if gene[i] == 1:
                fitness += 1
        ind.fitness = fitness

def best_solution(pop):

    fittest = 0
    for ind in pop:
        if ind.fitness > fittest:
            fittest = ind.fitness
            # best = ind.gene
            best = ind

    print("")
    print("Best solution: ", best.gene)

    return best

def worst_solution(pop):

    worst_ind = pop[0]
    for ind in pop:
        if worst_ind.fitness > ind.fitness:
            worst_ind.gene = ind.gene
            worst_ind.fitness = ind.fitness

    print("")
    print("Worst solution", worst_ind.gene, worst_ind.fitness)

    return worst_ind

def main():
    global population

    best = best_solution(population)

    offspring = selection(population)

    print(" Offspring ")
    for obj in offspring:
        print(obj.gene)

    worst = worst_solution(offspring)

    offspring.remove(worst)
    offspring.append(best)

    print(" NEW offspring: ")
    for obj in offspring:
        print(obj.gene)

    recombined = crossover(offspring)

    new_generation = mutation(recombined)

    fitness_value(new_generation)
    plot_value(new_generation)

    print("NEW GENERATION")
    for obj in new_generation:
        print(obj.gene, obj.fitness)

    population = new_generation.copy()

for m in range(0, 3):
    main()

plt.plot(average)  # Blue line
plt.plot(fittest_list)  # Orange line
plt.show()


# %%
# !pip install --upgrade pip
# !pip install sentencepiece
# !pip install datasets
# !pip install transformers

# %% [markdown]
# # Fine-tuning XLM-T
#
# This notebook describes a simple case of finetuning. You can finetune either the `XLM-T` language model, or XLM-T sentiment, which has already been fine-tuned on sentiment analysis data, in 8 languages (this could be useful to do sentiment transfer learning on new languages).,
#
# This notebook was modified from https://huggingface.co/transformers/custom_datasets.html

# %%
import transformers
from transformers import AutoTokenizer
from transformers import AutoModelForSequenceClassification, Trainer, TrainingArguments
import torch
import sys, os, json
import logging
import numpy as np
import pandas as pd
from sklearn.metrics import classification_report
from datetime import datetime
from pathlib import Path
from shutil import rmtree
import datasets
from datasets import load_metric

# TODO: Lage individuelle savefolders for best model?
# Shared parameters

BATCH_SIZE = 32
NUM_LABELS = 3
SAVE_DIR = '/fp/projects01/ec30/egilron/tweet-afri/submit' #afro-xlmr-mini'
# MODEL = os.path.join(SAVE_DIR, "final")#"cardiffnlp/twitter-xlm-roberta-base" # use this to finetune the language model
# MODEL = "Davlan/afro-xlmr-mini" #"sentence-transformers/paraphrase-mpnet-base-v2" #"cardiffnlp/twitter-xlm-roberta-base-sentiment" # use this to finetune the sentiment classifier
MAX_TRAINING_EXAMPLES = -1 # set this to -1 if you want to use the whole training set

TIMESTAMP = datetime.now().strftime("%m%d%H%M")

LANG_GROUPS = {'Volta-Congo': ['ts', 'twi', 'kr', 'yo', 'ig', 'sw'],
 'Afro-Asiatic-Semitic': ['tg', 'am', 'dz', 'ma'],
 'Creole': ['pcm', 'pt'],
 "multilingual": ['ts', 'twi', 'kr', 'yo', 'ig', 'sw', 'tg', 'am', 'dz', 'ma','pcm', 'pt' ]}
LANG_GROUPS = { "multilingual": ['ts', 'twi', 'kr', 'yo', 'ig', 'sw', 'tg', 'am', 'dz', 'ma','pcm', 'pt', 'ha' ]} # Catch up what failed
LANG_GROUPS = { "multilingual": ['ha']} # I had forgotten to add ha which has no group for training
## Logger
logger = logging.getLogger(__name__)
logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
    datefmt="%m/%d/%Y %H:%M:%S",
    level=logging.INFO,
    handlers=[logging.StreamHandler(sys.stdout)],
)
logs_folder = "logs"
ts = TIMESTAMP
filehandler = logging.FileHandler(os.path.join(logs_folder, f"{ts}.log"), "a")
logger.addHandler(filehandler)

log_level = "info"
transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()

class MyDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

# %% Preprocessing
def ds_dict(train_path="", dev_path="", pred_path=""):
  label2id = {'negative': 0, 'neutral': 1, 'positive': 2}
  train_df = pd.read_csv(train_path, sep="\t")
  val_df = pd.read_csv(dev_path, sep="\t")
  dataset_dict = {}
  dataset_dict["train"] = {}
  dataset_dict["train"]["text"] = train_df["tweet"].to_list()
  dataset_dict["train"]["labels"] = [label2id[l] for l in train_df["label"]]
  dataset_dict["val"] = {}
  dataset_dict["val"]["text"] = val_df["tweet"].to_list()
  dataset_dict["val"]["labels"] = [label2id[l] for l in val_df["label"]]
  if pred_path == "":
    dataset_dict["test"] = dataset_dict["val"].copy()
  else:
    pred_df = pd.read_csv(pred_path, sep="\t")
    dataset_dict["test"]["text"] = pred_df["tweet"].to_list()

  assert len(dataset_dict["val"]["text"]) == len(dataset_dict["val"]["labels"])
  return dataset_dict

def df_dicts_concat(train_paths = [], dev_paths = []):
  """Concatenate a number of language data into one ds_dict"""
  assert len(train_paths) == len(dev_paths)
  dataset_dict = {"train":{"text":[], "labels":[]},
                  "val":{"text":[], "labels":[]}}
  for tp, dp in zip(train_paths, dev_paths):
    dsd = ds_dict(tp, dp)
    dataset_dict["train"]["text"] += dsd["train"]["text"]
    dataset_dict["train"]["labels"] += dsd["train"]["labels"]
    dataset_dict["val"]["text"] += dsd["val"]["text"]
    dataset_dict["val"]["labels"] += dsd["val"]["labels"]
    dataset_dict["test"] = dataset_dict["val"].copy()

  assert len(dataset_dict["train"]["text"]) == len(dataset_dict["train"]["labels"])
  assert len(dataset_dict["val"]["text"]) == len(dataset_dict["val"]["labels"])
  return dataset_dict

def trainme(dataset_dict,model="cardiffnlp/twitter-xlm-roberta-base-sentiment",  tokenizer = None, finetune = True,  training_args = None):
  if not tokenizer:
    tokenizer = model
  tokenizer = AutoTokenizer.from_pretrained(tokenizer, use_fast=True,  model_max_length=512) # max_length=512,,truncation=True, padding=True
  train_encodings = tokenizer(dataset_dict['train']['text'], truncation=True, padding=True)
  val_encodings = tokenizer(dataset_dict['val']['text'], truncation=True, padding=True)
  test_encodings = tokenizer(dataset_dict['test']['text'], truncation=True, padding=True)
  train_dataset = MyDataset(train_encodings, dataset_dict['train']['labels'])
  val_dataset = MyDataset(val_encodings, dataset_dict['val']['labels'])
  test_dataset = MyDataset(test_encodings, dataset_dict['test']['labels'])
  model = AutoModelForSequenceClassification.from_pretrained(model, num_labels=NUM_LABELS,   max_length=512)
  metric = load_metric( 'f1') # , 'precision', 'recall', process_id=1

  if not training_args:
    training_args = TrainingArguments(
      output_dir=SAVE_DIR,                   # output directory
      num_train_epochs=14,                  # total number of training epochs
      per_device_train_batch_size=BATCH_SIZE,   # batch size per device during training
      per_device_eval_batch_size=BATCH_SIZE,    # batch size for evaluation
      learning_rate=2e-5,
      warmup_steps=100,                         # number of warmup steps for learning rate scheduler
      weight_decay=0.01,                        # strength of weight decay
      logging_dir='./logs',                     # directory for storing logs #logging_steps=5000,                         # when to print log
      load_best_model_at_end=True,              # load or not best model at the end
      save_strategy="epoch", #"epoch",                      # please select one of ['no', 'steps', 'epoch']
      evaluation_strategy="epoch", #  #eval_steps = 10, #eval_accumulation_steps = 1,
      save_total_limit=1,
      disable_tqdm=True,
      log_level = log_level,
      logging_strategy = "epoch",
      metric_for_best_model = "eval_f1"

    )

  def compute_metrics(eval_pred):
      predictions, labels = eval_pred
      predictions = np.argmax(predictions, axis=1)
      return metric.compute(predictions=predictions, references=labels, average="macro")

  trainer = Trainer(
      model=model,                              # the instantiated ðŸ¤— Transformers model to be trained
      args=training_args,                       # training arguments, defined above
      train_dataset=train_dataset,              # training dataset
      eval_dataset=val_dataset,                  # evaluation dataset
      compute_metrics=compute_metrics
  )
  if finetune:
    train_results = trainer.train()
  else:
    print("Deprecated, returning untrained model")

  return trainer

def train_individual(model= "cardiffnlp/twitter-xlm-roberta-base-sentiment",
                     train_folder = Path("datasets/train_final/"),
                     eval_folder = Path("datasets/dev_final/"),
                     dev_template = "_dev_gold_label.tsv" ,
                     training_args = None

                     ):

  results_path = f"results/{TIMESTAMP}_tr-individual_test_individual.json"
  results = []
  zeroshots = []
  train_eval_paths = []
  tsv_data = [{"lang":dp.stem.split("_")[0], "train_tsv":dp, "eval_tsv":None }for dp in train_folder.iterdir() if dp.suffix == ".tsv"]
  for data in tsv_data:
    eval_candidate = Path(eval_folder, data["lang"]+dev_template)
    if eval_candidate.is_file():
      data["eval_tsv"]= eval_candidate
    else:
      print("Language", data["lang"], "does not have a dev data file at", str(eval_candidate))

  for lang_data in tsv_data:
    language = lang_data["lang"]
    print("Starting individual training:", language)
    dataset_dict = ds_dict(train_path=lang_data["train_tsv"], dev_path=lang_data["eval_tsv"])
    trainer = trainme( dataset_dict, finetune = True, training_args = training_args)
    lang_f1 = evaulate_one(trainer=trainer, dataset_dict=dataset_dict)
    results.append({"timestamp": TIMESTAMP,"train":language , "test":language, "f1":lang_f1 })

    with open (results_path, "w") as wf:
      json.dump(results, wf)
    logger.warning(f"{language} Train_individual:{results}")
  return results

def evaulate_one(trainer= None,
        tokenizer =  AutoTokenizer.from_pretrained("cardiffnlp/twitter-xlm-roberta-base-sentiment"),
        dataset_dict = None #WIth test data (Ususally equals dev data)
        ):
  """Evaluate a language on a model, based on the eval_path sent
  return the f1
  """
  train_encodings = tokenizer(dataset_dict['train']['text'], truncation=True, padding=True)
  val_encodings = tokenizer(dataset_dict['val']['text'], truncation=True, padding=True)
  test_encodings = tokenizer(dataset_dict['test']['text'], truncation=True, padding=True)
  train_dataset = MyDataset(train_encodings, dataset_dict['train']['labels'])
  val_dataset = MyDataset(val_encodings, dataset_dict['val']['labels'])
  test_dataset = MyDataset(test_encodings, dataset_dict['test']['labels'])

  eval_result = trainer.evaluate(eval_dataset = test_dataset)
      #       {'eval_loss': 1.4181029796600342,
      #  'eval_f1': 0.5824834753269498,
      #  'eval_runtime': 63.585,
      #  'eval_samples_per_second': 6.102,
      #  'eval_steps_per_second': 0.771}
  return eval_result["eval_f1"]

def train_lg_groups(model= "cardiffnlp/twitter-xlm-roberta-base-sentiment",
                     train_folder = Path("datasets/group_final"),
                     eval_folder = Path("datasets/dev_final/"),
                     dev_template = "_dev_gold_label.tsv" ,
                     training_args = None
                     ):
  """ The language group belonging has been found in Ethnologue / Wikipedia and we now train the language group together, and evaluate on each of the languages in it
In stead of using this, create a main that uses train_individual for both folders.
New 16. feb: We have now prepared the concatenated datasets before getting here.
  """
  results_path = f"results/{TIMESTAMP}_tr-group_test_individual.json"
  groups = LANG_GROUPS
  # groups = {  'Creole': ['pcm', 'pt'] } #
  results = []

  for lg_group, eval_langs in groups.items():
    train_path = Path(train_folder, "{}_train.tsv".format(lg_group))
    dev_path = Path(train_folder, "{}_dev.tsv".format(lg_group))
    # Train with group train and dev data
    dataset_dict = ds_dict(train_path=train_path, dev_path=dev_path)
    trainer = trainme( dataset_dict, finetune = True)
    results.append({"timestamp": TIMESTAMP,"train":lg_group ,
          "test":lg_group, "f1": trainer.evaluate()["eval_f1"]})

    for eval_lang in  eval_langs: # Evaluate individual languages
      print(f"\nEvaluating {eval_lang} on a model trained on {lg_group}:")
      eval_path = Path(eval_folder, eval_lang+dev_template)
      dataset_dict = ds_dict(train_path=eval_path, dev_path=eval_path)
      lang_f1 = evaulate_one(trainer=trainer, dataset_dict=dataset_dict)
      results.append({"timestamp": TIMESTAMP,"train":lg_group , "test":eval_lang, "f1":lang_f1 })

      with open (results_path, "w") as wf:
        json.dump(results, wf)
      logger.warning(f"{lg_group} Test:{eval_lang} F1:{lang_f1}")
  return results

def final_train_A():
  # model = "cardiffnlp/twitter-xlm-roberta-base-sentiment"
  # train_langs = [l.stem.split("_")[0] for l in Path("afrisent-semeval-2023/SubtaskA/train").iterdir()]
  train_langs = {'dz':7, 'am':5, 'yo':6, 'twi':4, 'pcm':6, 'pt':7, 'ma':7, 'ha':4, 'ig':6, 'ts':5, 'kr':7}
  for lang in train_langs:
    if not lang in Path(SAVE_DIR).iterdir(): # Has been successfully completed
      print("Finetuning", lang)
      t_split = f"afrisent-semeval-2023/SubtaskA/train/{lang}_train.tsv"
      d_split = f"afrisent-semeval-2023/SubtaskA/dev_gold/{lang}_dev_gold_label.tsv"

      assert Path(t_split).is_file() and Path(d_split).is_file()
      dataset_dict = df_dicts_concat([t_split, d_split] , [d_split, d_split] ) # Need same length
      trainer = trainme( dataset_dict, finetune = True, epochs = train_langs[lang])
      trainer.save_model(Path(SAVE_DIR, lang)) # save best model

def final_train_B():
  t_split = "afrisent-semeval-2023/SubtaskB/multilingual_train.tsv"
  d_split = "afrisent-semeval-2023/SubtaskB/dev_gold/multilingual_dev_gold_label.tsv"
  lang = "multilingual"
  assert Path(t_split).is_file() and Path(d_split).is_file()
  dataset_dict = df_dicts_concat([t_split, d_split] , [d_split, d_split] ) # Need same length
  trainer = trainme( dataset_dict, finetune = True, epochs = 7)
  trainer.save_model(Path(SAVE_DIR, lang)) # save best model

if __name__ == "__main__":
    """16. feb: send the two folders from here"""
    if len(sys.argv) == 2:
        functions = {"train_lg_groups": train_lg_groups,
        "train_individual": train_individual,
        "train_lg_groups": train_lg_groups}
        print(sys.argv[0], "running function",sys.argv[1] )
        print(functions[sys.argv[1]]()) # Only default values (set in function) by now
    else:
        # "local run"
        training_args = TrainingArguments(
            output_dir=SAVE_DIR,                   # output directory
            num_train_epochs=14,                  # total number of training epochs
            per_device_train_batch_size=BATCH_SIZE,   # batch size per device during training
            per_device_eval_batch_size=BATCH_SIZE,    # batch size for evaluation
            learning_rate=2e-5,
            warmup_steps=100,                         # number of warmup steps for learning rate scheduler
            weight_decay=0.01,                        # strength of weight decay
            logging_dir='./logs',                     # directory for storing logs #logging_steps=5000,                         # when to print log
            load_best_model_at_end=True,              # load or not best model at the end
            save_strategy="epoch",                      # please select one of ['no', 'steps', 'epoch']
            evaluation_strategy="epoch", #  #eval_steps = 10, #eval_accumulation_steps = 1,
            save_total_limit=1,
            disable_tqdm=True,
            log_level = log_level,
            logging_strategy = "epoch",
            metric_for_best_model = "eval_f1"
        )
        results = train_individual(model= "cardiffnlp/twitter-xlm-roberta-base-sentiment",
                            train_folder = Path("datasets/train_final/"),
                            eval_folder = Path("datasets/dev_final/"),
                            dev_template = "_dev_gold_label.tsv" ,
                            training_args = training_args
                            )
        print(results)