from __future__ import absolute_import
from __future__ import division

from __future__ import print_function

import collections
import itertools
import math
import sys
from typing import Any, Dict, Iterable, List, Optional, Text

import apache_beam as beam
import numpy as np
import pyarrow as pa
import six
from tensorflow_data_validation import constants
from tensorflow_data_validation import types
from tensorflow_data_validation.arrow import arrow_util
from tensorflow_data_validation.statistics.generators import stats_generator
from tensorflow_data_validation.utils import quantiles_util
from tensorflow_data_validation.utils import schema_util
from tensorflow_data_validation.utils import stats_util
from tfx_bsl.arrow import array_util

from tensorflow_metadata.proto.v0 import schema_pb2
from tensorflow_metadata.proto.v0 import statistics_pb2

class _PresenceAndValencyStats(object):
  """Contains stats on presence and valency of a feature."""
  __slots__ = [
      'num_non_missing', 'min_num_values', 'max_num_values', 'total_num_values',
      'weighted_total_num_values', 'weighted_num_non_missing'
  ]

  def __init__(self):
    # The number of examples with at least one value for this feature.
    self.num_non_missing = 0
    # The minimum number of values in a single example for this feature.
    self.min_num_values = sys.maxsize
    # The maximum number of values in a single example for this feature.
    self.max_num_values = 0
    # The total number of values for this feature.
    self.total_num_values = 0
    # The sum of weights of all the values for this feature.
    self.weighted_total_num_values = 0
    # The sum of weights of all the examples with at least one value for this
    # feature.
    self.weighted_num_non_missing = 0

  def merge_with(self, other: '_PresenceAndValencyStats') -> None:
    self.num_non_missing += other.num_non_missing
    self.min_num_values = min(self.min_num_values, other.min_num_values)
    self.max_num_values = max(self.max_num_values, other.max_num_values)
    self.total_num_values += other.total_num_values
    self.weighted_num_non_missing += other.weighted_num_non_missing
    self.weighted_total_num_values += other.weighted_total_num_values

  def update(self, feature_array: pa.Array, presence_mask: np.ndarray,
             num_values: np.ndarray, num_values_not_none: np.ndarray,
             weights: Optional[np.ndarray]) -> None:
    """Updates the stats with a feature array."""
    self.num_non_missing += len(feature_array) - feature_array.null_count

    self.max_num_values = np.maximum.reduce(
        num_values_not_none, initial=self.max_num_values)
    self.min_num_values = np.minimum.reduce(num_values_not_none,
                                            initial=self.min_num_values)
    self.total_num_values += np.sum(num_values_not_none)

    if weights is not None:
      if weights.size != num_values.size:
        raise ValueError('Weight feature must not be missing.')
      self.weighted_total_num_values += np.sum(num_values * weights)
      self.weighted_num_non_missing += np.sum(weights[presence_mask])

class _PartialCommonStats(object):
  """Holds partial common statistics for a single feature."""

  __slots__ = [
      'type', 'has_weights', 'presence_and_valency_stats',
      'num_values_summaries'
  ]

  def __init__(self, has_weights: bool):
    # Type of the feature.
    self.type = None  # type: Optional[List[_PresenceAndValencyStats]]
    # This will be a List[_PresenceAndValencyStats] once `update()` is called.
    # presence_and_valency_stats[i] contains the stats at nest level i.
    # for example: a feature of type list<list<int>> will have
    # presence_and_valency_stats of length 2. presence_and_valency_stats[0]
    # contains the stats about the outer list.
    self.presence_and_valency_stats = None  # type: Optional[List[Any]]
    # Similar to num_values_summaries, this will be a List[QuantileSummaryType]
    # once `update()` is called.
    self.num_values_summaries = None
    self.has_weights = has_weights

  def merge_with(
      self, feature_path: types.FeaturePath, other: '_PartialCommonStats'
      ) -> None:
    """Merges two partial common statistics and return the merged statistics.

    Note that this DOES NOT merge self.num_values_summaries. See
    `merge_num_values_summaries()`.

    Args:
      feature_path: path of the feature that `self` is associated with.
      other: a _PartialCommonStats to merge with.
    """

    assert self.has_weights == other.has_weights
    if self.presence_and_valency_stats is None:
      self.presence_and_valency_stats = other.presence_and_valency_stats
    elif other.presence_and_valency_stats is not None:
      this_nest_level = len(self.presence_and_valency_stats)
      other_nest_level = len(other.presence_and_valency_stats)
      if this_nest_level != other_nest_level:
        raise ValueError(
            'Unable to merge common stats with different nest levels for '
            'feature {}: {} vs {}'.format(
                feature_path, this_nest_level, other_nest_level))
      for self_stats, other_stats in zip(self.presence_and_valency_stats,
                                         other.presence_and_valency_stats):
        self_stats.merge_with(other_stats)

    # Set the type of the merged common stats.
    # Case 1: Both the types are None. We set the merged type to be None.
    # Case 2: One of two types is None. We set the merged type to be the type
    # which is not None. For example, if left.type=FLOAT and right.type=None,
    # we set the merged type to be FLOAT.
    # Case 3: Both the types are same (and not None), we set the merged type to
    # be the same type.
    if self.type is None:
      self.type = other.type

  def merge_num_values_summaries(
      self, feature_path: types.FeaturePath,
      num_values_quantiles_combiner: quantiles_util.QuantilesCombiner,
      all_summaries_lists: List[Optional[List[Any]]]) -> None:
    """Merges num_values_summaries.

    This function merges the summaries in `all_summaries_list` into
    one num_values_summaries, and replaces self.num_values_summaries with it.

    Args:
      feature_path: path of the feature that `self` is associated with.
      num_values_quantiles_combiner: a QuantilesCombiner to combine the
        summaries.
      all_summaries_lists: a list of (list of quantile summaries or None).
        all elements that are lists must have a length that equals to
        len(self.num_values_summaries) (i.e. they are num values summaries of
        the same feature (of the same nest level)).
    """
    all_summary_lists = [s for s in all_summaries_lists if s is not None]
    if not all_summary_lists:
      self.num_values_summaries = None
      return

    num_summaries = len(all_summary_lists[0])
    if not all([len(l) == num_summaries for l in all_summary_lists]):
      raise ValueError('unable to merge num_values_summaries of incompatible '
                       'shapes for feature {}'.format(feature_path))
    assert (self.num_values_summaries is None or
            len(self.num_values_summaries) == num_summaries)

    merged_summaries = []
    for i in range(num_summaries):
      merged_summaries.append(
          num_values_quantiles_combiner.merge_accumulators(
              [l[i] for l in all_summary_lists]))
    self.num_values_summaries = merged_summaries

  def update(self,
             feature_path: types.FeaturePath,
             feature_array: pa.Array,
             feature_type: types.FeatureNameStatisticsType,
             num_values_quantiles_combiner: quantiles_util.QuantilesCombiner,
             weights: Optional[np.ndarray] = None) -> None:
    """Update the partial common statistics using the input value."""
    if self.type is None:
      self.type = feature_type
    elif feature_type is not None and self.type != feature_type:
      raise TypeError('Cannot determine the type of feature %s. '
                      'Found values of types %s and %s.' %
                      (feature_path, self.type, feature_type))

    nest_level = arrow_util.get_nest_level(feature_array.type)
    if self.presence_and_valency_stats is None:
      assert self.num_values_summaries is None
      self.presence_and_valency_stats = [
          _PresenceAndValencyStats() for _ in range(nest_level)
      ]
      self.num_values_summaries = [
          num_values_quantiles_combiner.create_accumulator()
          for _ in range(nest_level)
      ]
    elif nest_level != len(self.presence_and_valency_stats):
      raise ValueError('Inconsistent nestedness in feature {}: {} vs {}'.format(
          feature_path, nest_level, len(self.presence_and_valency_stats)))

    # And there's nothing we can collect in this case.
    if not feature_array:
      return

    level = 0
    while arrow_util.is_list_like(feature_array.type):
      presence_mask = ~np.asarray(
          array_util.GetArrayNullBitmapAsByteArray(feature_array)).view(np.bool)
      num_values = np.asarray(
          array_util.ListLengthsFromListArray(feature_array))
      num_values_not_none = num_values[presence_mask]
      self.presence_and_valency_stats[level].update(feature_array,
                                                    presence_mask, num_values,
                                                    num_values_not_none,
                                                    weights)
      self.num_values_summaries[level] = (
          num_values_quantiles_combiner.add_input(
              self.num_values_summaries[level], [num_values_not_none]))
      flattened = feature_array.flatten()
      if weights is not None:
        parent_indices = array_util.GetFlattenedArrayParentIndices(
            feature_array).to_numpy()
        weights = weights[parent_indices]
      feature_array = flattened
      level += 1

class _PartialNumericStats(object):
  """Holds partial numeric statistics for a single feature."""

  __slots__ = ['sum', 'sum_of_squares', 'num_zeros', 'num_nan', 'min', 'max',
               'finite_min', 'finite_max', 'quantiles_summary', 'has_weights',
               'weighted_sum', 'weighted_sum_of_squares',
               'weighted_total_num_values', 'weighted_quantiles_summary']

  def __init__(self, has_weights: bool):
    # Explicitly make the sum and the sum of squares to be float in order to
    # avoid numpy overflow warnings.
    # The sum of all the values for this feature.
    self.sum = 0.0
    # The sum of squares of all the values for this feature.
    self.sum_of_squares = 0.0
    # The number of values for this feature that equal 0.
    self.num_zeros = 0
    # The number of NaN values for this feature. This is computed only for
    # FLOAT features.
    self.num_nan = 0
    # The minimum value among all the values for this feature.
    self.min = float('inf')
    # The maximum value among all the values for this feature.
    self.max = float('-inf')
    # The minimum value among all the finite values for this feature.
    self.finite_min = float('inf')
    # The maximum value among all the finite values for this feature.
    self.finite_max = float('-inf')
    # Summary of the quantiles for the values in this feature.
    self.quantiles_summary = None

    self.has_weights = has_weights
    # Keep track of partial weighted numeric stats.
    if has_weights:
      # The weighted sum of all the values for this feature.
      self.weighted_sum = 0.0
      # The weighted sum of squares of all the values for this feature.
      self.weighted_sum_of_squares = 0.0
      # The sum of weights of all the values for this feature (
      # excluding the weights for NaN values)
      self.weighted_total_num_values = 0.0
      # Summary of the weighted quantiles for the values in this feature.
      self.weighted_quantiles_summary = None

  def __iadd__(self, other: '_PartialNumericStats') -> '_PartialNumericStats':
    """Merge two partial numeric statistics and return the merged statistics."""
    self.sum += other.sum
    self.sum_of_squares += other.sum_of_squares
    self.num_zeros += other.num_zeros
    self.num_nan += other.num_nan
    self.min = min(self.min, other.min)
    self.max = max(self.max, other.max)
    self.finite_min = min(self.finite_min, other.finite_min)
    self.finite_max = max(self.finite_max, other.finite_max)

    assert self.has_weights == other.has_weights
    if self.has_weights:
      self.weighted_sum += other.weighted_sum
      self.weighted_sum_of_squares += other.weighted_sum_of_squares
      self.weighted_total_num_values += other.weighted_total_num_values
    return self

  def update(
      self,
      feature_array: pa.Array,
      values_quantiles_combiner: Any,
      weights: Optional[np.ndarray] = None) -> None:
    """Update the partial numeric statistics using the input value."""

    # np.max / np.min below cannot handle empty arrays. And there's nothing
    # we can collect in this case.
    if not feature_array:
      return

    flattened_value_array, value_parent_indices = arrow_util.flatten_nested(
        feature_array, weights is not None)
    # Note: to_numpy will fail if flattened_value_array is empty.
    if not flattened_value_array:
      return
    values = np.asarray(flattened_value_array)
    nan_mask = np.isnan(values)
    self.num_nan += np.sum(nan_mask)
    non_nan_mask = ~nan_mask
    values_no_nan = values[non_nan_mask]

    # We do this check to avoid failing in np.min/max with empty array.
    if values_no_nan.size == 0:
      return
    # This is to avoid integer overflow when computing sum or sum of squares.
    values_no_nan_as_double = values_no_nan.astype(np.float64)
    self.sum += np.sum(values_no_nan_as_double)
    self.sum_of_squares += np.sum(
        values_no_nan_as_double* values_no_nan_as_double)
    # Use np.minimum.reduce(values_no_nan, initial=self.min) once we upgrade
    # to numpy 1.16
    curr_min = np.min(values_no_nan)
    curr_max = np.max(values_no_nan)
    self.min = min(self.min, curr_min)
    self.max = max(self.max, curr_max)
    if curr_min == float('-inf') or curr_max == float('inf'):
      finite_values = values_no_nan[np.isfinite(values_no_nan)]
      if finite_values.size > 0:
        self.finite_min = min(self.finite_min, np.min(finite_values))
        self.finite_max = max(self.finite_max, np.max(finite_values))

    self.num_zeros += values_no_nan.size - np.count_nonzero(values_no_nan)
    self.quantiles_summary = values_quantiles_combiner.add_input(
        self.quantiles_summary, [values_no_nan, np.ones_like(values_no_nan)])
    if weights is not None:
      flat_weights = weights[value_parent_indices]
      flat_weights_no_nan = flat_weights[non_nan_mask]
      weighted_values = flat_weights_no_nan * values_no_nan
      self.weighted_sum += np.sum(weighted_values)
      self.weighted_sum_of_squares += np.sum(weighted_values * values_no_nan)
      self.weighted_quantiles_summary = values_quantiles_combiner.add_input(
          self.weighted_quantiles_summary, [values_no_nan, flat_weights_no_nan])
      self.weighted_total_num_values += np.sum(flat_weights_no_nan)

class _PartialStringStats(object):
  """Holds partial string statistics for a single feature."""

  __slots__ = ['total_bytes_length']

  def __init__(self):
    # The total length of all the values for this feature.
    self.total_bytes_length = 0

  def __iadd__(self, other: '_PartialStringStats') -> '_PartialStringStats':
    """Merge two partial string statistics and return the merged statistics."""
    self.total_bytes_length += other.total_bytes_length
    return self

  def update(self, feature_array: pa.Array) -> None:
    """Update the partial string statistics using the input value."""
    if pa.types.is_null(feature_array.type):
      return
    # Iterate through the value array and update the partial stats.
    flattened_values_array, _ = arrow_util.flatten_nested(feature_array)
    if arrow_util.is_binary_like(flattened_values_array.type):
      # GetBinaryArrayTotalByteSize returns a Python long (to be compatible
      # with Python3). To make sure we do cheaper integer arithemetics in
      # Python2, we first convert it to int.
      self.total_bytes_length += int(array_util.GetBinaryArrayTotalByteSize(
          flattened_values_array))
    elif flattened_values_array:
      # We can only do flattened_values_array.to_numpy() when it's not empty.
      # This could be computed faster by taking log10 of the integer.
      def _len_after_conv(s):
        return len(str(s))
      self.total_bytes_length += np.sum(
          np.vectorize(_len_after_conv,
                       otypes=[np.int32])(np.asarray(flattened_values_array)))

class _PartialBytesStats(object):
  """Holds partial bytes statistics for a single feature."""

  __slots__ = ['total_num_bytes', 'min_num_bytes', 'max_num_bytes']

  def __init__(self):
    # The total number of bytes of all the values for this feature.
    self.total_num_bytes = 0
    # The minimum number of bytes among all the values for this feature.
    self.min_num_bytes = sys.maxsize
    # The maximum number of bytes among all the values for this feature.
    self.max_num_bytes = -sys.maxsize

  def __iadd__(self, other: '_PartialBytesStats') -> '_PartialBytesStats':
    """Merge two partial bytes statistics and return the merged statistics."""
    self.total_num_bytes += other.total_num_bytes
    self.min_num_bytes = min(self.min_num_bytes, other.min_num_bytes)
    self.max_num_bytes = max(self.max_num_bytes, other.max_num_bytes)
    return self

  def update(self, feature_array: pa.Array) -> None:
    """Update the partial bytes statistics using the input value."""
    if pa.types.is_null(feature_array.type):
      return
    # Iterate through the value array and update the partial stats.'
    flattened_values_array, _ = arrow_util.flatten_nested(feature_array)
    if (pa.types.is_floating(flattened_values_array.type) or
        pa.types.is_integer(flattened_values_array.type)):
      raise ValueError('Bytes stats cannot be computed on INT/FLOAT features.')
    if flattened_values_array:
      num_bytes = array_util.GetElementLengths(
          flattened_values_array).to_numpy()
      self.min_num_bytes = min(self.min_num_bytes, np.min(num_bytes))
      self.max_num_bytes = max(self.max_num_bytes, np.max(num_bytes))
      self.total_num_bytes += np.sum(num_bytes)

class _PartialBasicStats(object):
  """Holds partial statistics for a single feature."""

  __slots__ = ['common_stats', 'numeric_stats', 'string_stats', 'bytes_stats']

  def __init__(self, has_weights: bool):
    self.common_stats = _PartialCommonStats(has_weights=has_weights)
    self.numeric_stats = _PartialNumericStats(has_weights=has_weights)
    self.string_stats = _PartialStringStats()
    self.bytes_stats = _PartialBytesStats()

def _make_presence_and_valency_stats_protos(
    parent_presence_and_valency: Optional[_PresenceAndValencyStats],
    presence_and_valency: List[_PresenceAndValencyStats]
    ) -> List[statistics_pb2.PresenceAndValencyStatistics]:
  """Converts presence and valency stats to corresponding protos."""
  result = []
  # The top-level non-missing is computed by
  # num_examples - top_level.num_non_missing (outside BasicStatsGenerator as
  # num_examples cannot be computed here). For all other levels,
  # it's previous_level.total_num_values - this_level.num_non_missing.
  for prev_s, s in zip(
      itertools.chain([parent_presence_and_valency], presence_and_valency),
      presence_and_valency):
    proto = statistics_pb2.PresenceAndValencyStatistics()
    if prev_s is not None:
      proto.num_missing = (prev_s.total_num_values - s.num_non_missing)
    proto.num_non_missing = s.num_non_missing
    if s.num_non_missing > 0:
      proto.min_num_values = s.min_num_values
      proto.max_num_values = s.max_num_values
      proto.tot_num_values = s.total_num_values
    result.append(proto)
  return result

def _make_weighted_presence_and_valency_stats_protos(
    parent_presence_and_valency: Optional[_PresenceAndValencyStats],
    presence_and_valency: List[_PresenceAndValencyStats]
    ) -> List[statistics_pb2.WeightedCommonStatistics]:
  """Converts weighted presence and valency stats to corresponding protos."""
  result = []
  # The top-level non-missing is computed by
  # weighted_num_examples - top_level.weighted_num_non_missing (outside
  # BasicStatsGenerator as num_examples cannot be computed here).
  # For all other levels,
  # it's (previous_level.weighted_total_num_values -
  # this_level.weighted_num_non_missing).
  for prev_s, s in zip(
      itertools.chain([parent_presence_and_valency], presence_and_valency),
      presence_and_valency):
    proto = statistics_pb2.WeightedCommonStatistics()
    if prev_s is not None:
      proto.num_missing = (
          prev_s.weighted_total_num_values - s.weighted_num_non_missing)
    proto.num_non_missing = s.weighted_num_non_missing
    proto.tot_num_values = s.weighted_total_num_values
    if s.weighted_num_non_missing > 0:
      proto.avg_num_values = (
          s.weighted_total_num_values / s.weighted_num_non_missing)
    result.append(proto)
  return result

def _make_common_stats_proto(
    common_stats: _PartialCommonStats,
    parent_common_stats: Optional[_PartialCommonStats],
    q_combiner: quantiles_util.QuantilesCombiner,
    num_values_histogram_buckets: int,
    has_weights: bool
) -> statistics_pb2.CommonStatistics:
  """Convert the partial common stats into a CommonStatistics proto."""
  result = statistics_pb2.CommonStatistics()
  parent_presence_and_valency = None
  if parent_common_stats is not None:
    parent_presence_and_valency = (
        _PresenceAndValencyStats()
        if parent_common_stats.presence_and_valency_stats is None else
        parent_common_stats.presence_and_valency_stats[-1])

  presence_and_valency_stats = common_stats.presence_and_valency_stats
  # the CommonStatistics already contains the presence and valency
  # for a 1-nested feature.
  if (presence_and_valency_stats is not None and
      len(presence_and_valency_stats) > 1):
    result.presence_and_valency_stats.extend(
        _make_presence_and_valency_stats_protos(
            parent_presence_and_valency,
            common_stats.presence_and_valency_stats))
    if has_weights:
      result.weighted_presence_and_valency_stats.extend(
          _make_weighted_presence_and_valency_stats_protos(
              parent_presence_and_valency,
              common_stats.presence_and_valency_stats))

  top_level_presence_and_valency = (
      _PresenceAndValencyStats()
      if common_stats.presence_and_valency_stats is None else
      common_stats.presence_and_valency_stats[0])
  result.num_non_missing = top_level_presence_and_valency.num_non_missing

  if parent_presence_and_valency is not None:
    result.num_missing = (
        parent_presence_and_valency.total_num_values -
        top_level_presence_and_valency.num_non_missing)
  result.tot_num_values = top_level_presence_and_valency.total_num_values

  # TODO(b/79685042): Need to decide on what is the expected values for
  # statistics like min_num_values, max_num_values, avg_num_values, when
  # all the values for the feature are missing.
  if top_level_presence_and_valency.num_non_missing > 0:
    result.min_num_values = top_level_presence_and_valency.min_num_values
    result.max_num_values = top_level_presence_and_valency.max_num_values
    result.avg_num_values = (
        top_level_presence_and_valency.total_num_values /
        top_level_presence_and_valency.num_non_missing)

    if common_stats.num_values_summaries is not None:
      # Add num_values_histogram to the common stats proto.
      num_values_quantiles = q_combiner.extract_output(
          common_stats.num_values_summaries[0])
      histogram = quantiles_util.generate_quantiles_histogram(
          num_values_quantiles, top_level_presence_and_valency.num_non_missing,
          num_values_histogram_buckets)
      result.num_values_histogram.CopyFrom(histogram)

  # Add weighted common stats to the proto.
  if has_weights:
    weighted_common_stats_proto = statistics_pb2.WeightedCommonStatistics(
        num_non_missing=top_level_presence_and_valency.weighted_num_non_missing,
        tot_num_values=top_level_presence_and_valency.weighted_total_num_values)
    if parent_presence_and_valency is not None:
      weighted_common_stats_proto.num_missing = (
          parent_presence_and_valency.weighted_total_num_values -
          top_level_presence_and_valency.weighted_num_non_missing)

    if top_level_presence_and_valency.weighted_num_non_missing > 0:
      weighted_common_stats_proto.avg_num_values = (
          top_level_presence_and_valency.weighted_total_num_values /
          top_level_presence_and_valency.weighted_num_non_missing)

    result.weighted_common_stats.CopyFrom(
        weighted_common_stats_proto)
  return result

def _make_numeric_stats_proto(
    numeric_stats: _PartialNumericStats,
    total_num_values: int,
    quantiles_combiner: quantiles_util.QuantilesCombiner,
    num_histogram_buckets: int,
    num_quantiles_histogram_buckets: int,
    has_weights: bool
    ) -> statistics_pb2.NumericStatistics:
  """Convert the partial numeric statistics into NumericStatistics proto."""
  result = statistics_pb2.NumericStatistics()

  if numeric_stats.num_nan > 0:
    total_num_values -= numeric_stats.num_nan

  if total_num_values == 0:
    # If we only have nan values, we only set num_nan.
    if numeric_stats.num_nan > 0:
      result.histograms.add(type=statistics_pb2.Histogram.STANDARD).num_nan = (
          numeric_stats.num_nan)
      result.histograms.add(type=statistics_pb2.Histogram.QUANTILES).num_nan = (
          numeric_stats.num_nan)
    return result

  mean = numeric_stats.sum / total_num_values
  variance = max(
      0, (numeric_stats.sum_of_squares / total_num_values) -
      mean * mean)
  result.mean = float(mean)
  result.std_dev = math.sqrt(variance)
  result.num_zeros = numeric_stats.num_zeros
  result.min = float(numeric_stats.min)
  result.max = float(numeric_stats.max)

  # Extract the quantiles from the summary.
  quantiles = quantiles_combiner.extract_output(
      numeric_stats.quantiles_summary)

  # Find the median from the quantiles and update the numeric stats proto.
  result.median = float(quantiles_util.find_median(quantiles))

  # Construct the equi-width histogram from the quantiles and add it to the
  # numeric stats proto.
  std_histogram = quantiles_util.generate_equi_width_histogram(
      quantiles, numeric_stats.finite_min, numeric_stats.finite_max,
      total_num_values, num_histogram_buckets)
  std_histogram.num_nan = numeric_stats.num_nan
  new_std_histogram = result.histograms.add()
  new_std_histogram.CopyFrom(std_histogram)

  # Construct the quantiles histogram from the quantiles and add it to the
  # numeric stats proto.
  q_histogram = quantiles_util.generate_quantiles_histogram(
      quantiles, total_num_values, num_quantiles_histogram_buckets)
  q_histogram.num_nan = numeric_stats.num_nan
  new_q_histogram = result.histograms.add()
  new_q_histogram.CopyFrom(q_histogram)

  # Add weighted numeric stats to the proto.
  if has_weights:
    weighted_numeric_stats_proto = statistics_pb2.WeightedNumericStatistics()

    if numeric_stats.weighted_total_num_values == 0:
      weighted_mean = 0
      weighted_variance = 0
    else:
      weighted_mean = (numeric_stats.weighted_sum /
                       numeric_stats.weighted_total_num_values)
      weighted_variance = max(0, (numeric_stats.weighted_sum_of_squares /
                                  numeric_stats.weighted_total_num_values)
                              - weighted_mean**2)
    weighted_numeric_stats_proto.mean = weighted_mean
    weighted_numeric_stats_proto.std_dev = math.sqrt(weighted_variance)

    # Extract the weighted quantiles from the summary.
    weighted_quantiles = quantiles_combiner.extract_output(
        numeric_stats.weighted_quantiles_summary)

    # Find the weighted median from the quantiles and update the proto.
    weighted_numeric_stats_proto.median = float(
        quantiles_util.find_median(weighted_quantiles))

    # Construct the weighted equi-width histogram from the quantiles and
    # add it to the numeric stats proto.
    weighted_std_histogram = quantiles_util.generate_equi_width_histogram(
        weighted_quantiles, numeric_stats.finite_min, numeric_stats.finite_max,
        numeric_stats.weighted_total_num_values, num_histogram_buckets)
    weighted_std_histogram.num_nan = numeric_stats.num_nan
    weighted_numeric_stats_proto.histograms.extend([weighted_std_histogram])

    # Construct the weighted quantiles histogram from the quantiles and
    # add it to the numeric stats proto.
    weighted_q_histogram = quantiles_util.generate_quantiles_histogram(
        weighted_quantiles, numeric_stats.weighted_total_num_values,
        num_quantiles_histogram_buckets)
    weighted_q_histogram.num_nan = numeric_stats.num_nan
    weighted_numeric_stats_proto.histograms.extend([weighted_q_histogram])

    result.weighted_numeric_stats.CopyFrom(
        weighted_numeric_stats_proto)
  return result

def _make_string_stats_proto(string_stats: _PartialStringStats,
                             total_num_values: int
                            ) -> statistics_pb2.StringStatistics:
  """Convert the partial string statistics into StringStatistics proto."""
  result = statistics_pb2.StringStatistics()
  if total_num_values > 0:
    result.avg_length = string_stats.total_bytes_length / total_num_values
  return result

def _make_bytes_stats_proto(bytes_stats: _PartialBytesStats,
                            total_num_values: int
                           ) -> statistics_pb2.BytesStatistics:
  """Convert the partial bytes statistics into BytesStatistics proto."""
  result = statistics_pb2.BytesStatistics()
  if total_num_values > 0:
    result.avg_num_bytes = bytes_stats.total_num_bytes / total_num_values
    result.min_num_bytes = bytes_stats.min_num_bytes
    result.max_num_bytes = bytes_stats.max_num_bytes
  return result

def _make_num_values_custom_stats_proto(
    common_stats: _PartialCommonStats,
    q_combiner: quantiles_util.QuantilesCombiner,
    num_histogram_buckets: int,
    ) -> List[statistics_pb2.CustomStatistic]:
  """Returns a list of CustomStatistic protos that contains histograms.

  Those histograms captures the distribution of number of values at each
  nest level.

  It will only create histograms for nest levels greater than 1. Because
  the histogram of nest level 1 is already in
  CommonStatistics.num_values_histogram.

  Args:
    common_stats: a _PartialCommonStats.
    q_combiner: the QuantilesCombiners to extract the quantiles from summaries.
    num_histogram_buckets: number of buckets in the histogram.
  Returns:
    a (potentially empty) list of statistics_pb2.CustomStatistic.
  """
  result = []
  if common_stats.type is None:
    return result
  num_values_summaries = common_stats.num_values_summaries
  presence_and_valency_stats = common_stats.presence_and_valency_stats
  if num_values_summaries is None or presence_and_valency_stats is None:
    return result

  assert presence_and_valency_stats is not None
  # The top level histogram is included in CommonStats -- skip.
  for level, summary, parent_presence_and_valency in zip(
      itertools.count(2), num_values_summaries[1:], presence_and_valency_stats):
    num_values_quantiles = q_combiner.extract_output(
        summary)
    histogram = quantiles_util.generate_quantiles_histogram(
        num_values_quantiles, parent_presence_and_valency.num_non_missing,
        num_histogram_buckets)
    proto = statistics_pb2.CustomStatistic()
    proto.name = 'level_{}_value_list_length'.format(level)
    proto.histogram.CopyFrom(histogram)
    result.append(proto)
  return result

def _make_feature_stats_proto(
    feature_path: types.FeaturePath,
    basic_stats: _PartialBasicStats,
    parent_basic_stats: Optional[_PartialBasicStats],
    num_values_q_combiner: quantiles_util.QuantilesCombiner,
    values_q_combiner: quantiles_util.QuantilesCombiner,
    num_values_histogram_buckets: int,
    num_histogram_buckets: int,
    num_quantiles_histogram_buckets: int,
    is_bytes: bool, is_categorical: bool, has_weights: bool
) -> statistics_pb2.FeatureNameStatistics:
  """Convert the partial basic stats into a FeatureNameStatistics proto.

  Args:
    feature_path: The path of the feature.
    basic_stats: The partial basic stats associated with the feature.
    parent_basic_stats: The partial basic stats of the parent of the feature.
    num_values_q_combiner: The quantiles combiner used to construct the
        quantiles histogram for the number of values in the feature.
    values_q_combiner: The quantiles combiner used to construct the
        histogram for the values in the feature.
    num_values_histogram_buckets: Number of buckets in the quantiles
        histogram for the number of values per feature.
    num_histogram_buckets: Number of buckets in a standard
        NumericStatistics.histogram with equal-width buckets.
    num_quantiles_histogram_buckets: Number of buckets in a
        quantiles NumericStatistics.histogram.
    is_bytes: A boolean indicating whether the feature is bytes.
    is_categorical: A boolean indicating whether the feature is categorical.
    has_weights: A boolean indicating whether a weight feature is specified.

  Returns:
    A statistics_pb2.FeatureNameStatistics proto.
  """
  # Create a new FeatureNameStatistics proto.
  result = statistics_pb2.FeatureNameStatistics()
  result.path.CopyFrom(feature_path.to_proto())
  # Set the feature type.
  # If we have a categorical feature, we preserve the type to be the original
  # INT type. Currently we don't set the type if we cannot infer it, which
  # happens when all the values are missing. We need to add an UNKNOWN type
  # to the stats proto to handle this case.
  if is_categorical:
    result.type = statistics_pb2.FeatureNameStatistics.INT
  elif is_bytes:
    result.type = statistics_pb2.FeatureNameStatistics.BYTES
  elif basic_stats.common_stats.type is None:
    # If a feature is completely missing, we assume the type to be STRING.
    result.type = statistics_pb2.FeatureNameStatistics.STRING
  else:
    result.type = basic_stats.common_stats.type

  # Construct common statistics proto.
  common_stats_proto = _make_common_stats_proto(
      basic_stats.common_stats,
      parent_basic_stats.common_stats
      if parent_basic_stats is not None else None,
      num_values_q_combiner,
      num_values_histogram_buckets, has_weights)

  # this is the total number of values at the leaf level.
  total_num_values = (
      0 if basic_stats.common_stats.presence_and_valency_stats is None else
      basic_stats.common_stats.presence_and_valency_stats[-1].total_num_values)

  # Copy the common stats into appropriate numeric/string stats.
  # If the type is not set, we currently wrap the common stats
  # within numeric stats.
  if is_bytes:
    # Construct bytes statistics proto.
    bytes_stats_proto = _make_bytes_stats_proto(
        basic_stats.bytes_stats, common_stats_proto.tot_num_values)
    # Add the common stats into bytes stats.
    bytes_stats_proto.common_stats.CopyFrom(common_stats_proto)
    result.bytes_stats.CopyFrom(bytes_stats_proto)
  if (is_categorical or
      result.type == statistics_pb2.FeatureNameStatistics.STRING):
    # Construct string statistics proto.
    string_stats_proto = _make_string_stats_proto(basic_stats.string_stats,
                                                  total_num_values)
    # Add the common stats into string stats.
    string_stats_proto.common_stats.CopyFrom(common_stats_proto)
    result.string_stats.CopyFrom(string_stats_proto)
  elif result.type == statistics_pb2.FeatureNameStatistics.STRUCT:
    result.struct_stats.common_stats.CopyFrom(common_stats_proto)
  elif result.type in (statistics_pb2.FeatureNameStatistics.INT,
                       statistics_pb2.FeatureNameStatistics.FLOAT):
    # Construct numeric statistics proto.
    numeric_stats_proto = _make_numeric_stats_proto(
        basic_stats.numeric_stats, total_num_values, values_q_combiner,
        num_histogram_buckets, num_quantiles_histogram_buckets, has_weights)
    # Add the common stats into numeric stats.
    numeric_stats_proto.common_stats.CopyFrom(common_stats_proto)
    result.num_stats.CopyFrom(numeric_stats_proto)

  result.custom_stats.extend(_make_num_values_custom_stats_proto(
      basic_stats.common_stats,
      num_values_q_combiner,
      num_values_histogram_buckets))
  return result

# Named tuple containing TFDV metrics.
_TFDVMetrics = collections.namedtuple(
    '_TFDVMetrics', ['num_non_missing', 'min_value_count',
                     'max_value_count', 'total_num_values'])
_TFDVMetrics.__new__.__defaults__ = (0, sys.maxsize, 0, 0)

def _update_tfdv_telemetry(
    accumulator: Dict[types.FeaturePath, _PartialBasicStats]) -> None:
  """Update TFDV Beam metrics."""
  # Aggregate type specific metrics.
  metrics = {
      statistics_pb2.FeatureNameStatistics.INT: _TFDVMetrics(),
      statistics_pb2.FeatureNameStatistics.FLOAT: _TFDVMetrics(),
      statistics_pb2.FeatureNameStatistics.STRING: _TFDVMetrics(),
      statistics_pb2.FeatureNameStatistics.STRUCT: _TFDVMetrics(),
  }

  for basic_stats in six.itervalues(accumulator):
    common_stats = basic_stats.common_stats
    if common_stats.type is None:
      continue
    # Take the leaf level stats.
    presence_and_valency = (
        _PresenceAndValencyStats()
        if common_stats.presence_and_valency_stats is None else
        common_stats.presence_and_valency_stats[-1])
    # Update type specific metrics.
    type_metrics = metrics[common_stats.type]
    num_non_missing = (type_metrics.num_non_missing +
                       presence_and_valency.num_non_missing)
    min_value_count = min(type_metrics.min_value_count,
                          presence_and_valency.min_num_values)
    max_value_count = max(type_metrics.max_value_count,
                          presence_and_valency.max_num_values)
    total_num_values = (type_metrics.total_num_values +
                        presence_and_valency.total_num_values)
    metrics[common_stats.type] = _TFDVMetrics(num_non_missing, min_value_count,
                                              max_value_count, total_num_values)

  # Update Beam counters.
  counter = beam.metrics.Metrics.counter
  for feature_type in metrics:
    type_str = statistics_pb2.FeatureNameStatistics.Type.Name(
        feature_type).lower()
    type_metrics = metrics[feature_type]
    counter(
        constants.METRICS_NAMESPACE,
        'num_' + type_str + '_feature_values').inc(
            int(type_metrics.num_non_missing))
    if type_metrics.num_non_missing > 0:
      counter(
          constants.METRICS_NAMESPACE,
          type_str + '_feature_values_min_count').inc(
              int(type_metrics.min_value_count))
      counter(
          constants.METRICS_NAMESPACE,
          type_str + '_feature_values_max_count').inc(
              int(type_metrics.max_value_count))
      counter(
          constants.METRICS_NAMESPACE,
          type_str + '_feature_values_mean_count').inc(
              int(type_metrics.total_num_values / type_metrics.num_non_missing))

# Currently we construct the equi-width histogram by using the
# quantiles. Specifically, we compute a large number of quantiles (say, N),
# and then compute the density for each bucket by aggregating the densities
# of the smaller quantile intervals that fall within the bucket. We set N to
# be _NUM_QUANTILES_FACTOR_FOR_STD_HISTOGRAM * num_histogram_buckets,
# where num_histogram_buckets is the required number of buckets in the
# histogram.
_NUM_QUANTILES_FACTOR_FOR_STD_HISTOGRAM = 100

# TODO(b/79685042): Currently the stats generator operates on the
# Dict representation of input (mapping from feature name to a batch of
# values). But we process each feature independently. We should
# consider making the stats generator to operate per feature.
class BasicStatsGenerator(stats_generator.CombinerStatsGenerator):
  """A combiner statistics generator that computes basic statistics.

  It computes common statistics for all the features, numeric statistics for
  numeric features and string statistics for string/categorical features.
  """

  def __init__(
      self,  # pylint: disable=useless-super-delegation
      name: Text = 'BasicStatsGenerator',
      schema: Optional[schema_pb2.Schema] = None,
      weight_feature: Optional[types.FeatureName] = None,
      num_values_histogram_buckets: Optional[int] = 10,
      num_histogram_buckets: Optional[int] = 10,
      num_quantiles_histogram_buckets: Optional[int] = 10,
      epsilon: Optional[float] = 0.01) -> None:
    """Initializes basic statistics generator.

    Args:
      name: An optional unique name associated with the statistics generator.
      schema: An optional schema for the dataset.
      weight_feature: An optional feature name whose numeric value represents
          the weight of an example.
      num_values_histogram_buckets: An optional number of buckets in a quantiles
          histogram for the number of values per Feature, which is stored in
          CommonStatistics.num_values_histogram.
      num_histogram_buckets: An optional number of buckets in a standard
          NumericStatistics.histogram with equal-width buckets.
      num_quantiles_histogram_buckets: An optional number of buckets in a
          quantiles NumericStatistics.histogram.
      epsilon: An optional error tolerance for the computation of quantiles,
          typically a small fraction close to zero (e.g. 0.01). Higher values
          of epsilon increase the quantile approximation, and hence result in
          more unequal buckets, but could improve performance, and resource
          consumption.
    """
    super(BasicStatsGenerator, self).__init__(name, schema)

    self._bytes_features = set(
        schema_util.get_bytes_features(schema) if schema else [])
    self._categorical_features = set(
        schema_util.get_categorical_numeric_features(schema) if schema else [])
    self._weight_feature = weight_feature
    self._num_values_histogram_buckets = num_values_histogram_buckets
    # Initialize quantiles combiner for histogram over number of values.
    self._num_values_quantiles_combiner = quantiles_util.QuantilesCombiner(
        self._num_values_histogram_buckets, epsilon)

    self._num_histogram_buckets = num_histogram_buckets
    self._num_quantiles_histogram_buckets = num_quantiles_histogram_buckets
    num_buckets = max(
        self._num_quantiles_histogram_buckets,
        _NUM_QUANTILES_FACTOR_FOR_STD_HISTOGRAM * self._num_histogram_buckets)
    assert num_buckets % self._num_quantiles_histogram_buckets == 0
    # Initialize quantiles combiner for histogram over feature values.
    self._values_quantiles_combiner = quantiles_util.QuantilesCombiner(
        num_buckets, epsilon, has_weights=True)

  # Create an accumulator, which maps feature name to the partial stats
  # associated with the feature.
  def create_accumulator(self) -> Dict[types.FeaturePath, _PartialBasicStats]:
    return {}

  # Incorporates the input (a Python dict whose keys are feature names and
  # values are lists representing a batch of examples) into the accumulator.
  def add_input(
      self, accumulator: Dict[types.FeaturePath, _PartialBasicStats],
      examples: pa.RecordBatch
      ) -> Dict[types.FeaturePath, _PartialBasicStats]:
    for feature_path, feature_array, weights in arrow_util.enumerate_arrays(
        examples,
        weight_column=self._weight_feature,
        enumerate_leaves_only=False):
      stats_for_feature = accumulator.get(feature_path)
      if stats_for_feature is None:
        stats_for_feature = _PartialBasicStats(self._weight_feature is not None)
        # Store empty summary for each of the quantiles computations.
        stats_for_feature.numeric_stats.quantiles_summary = (
            self._values_quantiles_combiner.create_accumulator())
        stats_for_feature.numeric_stats.weighted_quantiles_summary = (
            self._values_quantiles_combiner.create_accumulator())
        accumulator[feature_path] = stats_for_feature

      feature_type = stats_util.get_feature_type_from_arrow_type(
          feature_path, feature_array.type)
      stats_for_feature.common_stats.update(feature_path,
                                            feature_array, feature_type,
                                            self._num_values_quantiles_combiner,
                                            weights)
      if feature_path in self._bytes_features:
        stats_for_feature.bytes_stats.update(feature_array)
      elif (feature_path in self._categorical_features or
            feature_type == statistics_pb2.FeatureNameStatistics.STRING):
        stats_for_feature.string_stats.update(feature_array)
      elif feature_type in (statistics_pb2.FeatureNameStatistics.INT,
                            statistics_pb2.FeatureNameStatistics.FLOAT):
        stats_for_feature.numeric_stats.update(feature_array,
                                               self._values_quantiles_combiner,
                                               weights)

    return accumulator

  # Merge together a list of basic common statistics.
  def merge_accumulators(
      self, accumulators: Iterable[Dict[types.FeaturePath, _PartialBasicStats]]
  ) -> Dict[types.FeaturePath, _PartialBasicStats]:
    result = {}
    num_values_summaries_per_feature = collections.defaultdict(list)
    values_summary_per_feature = collections.defaultdict(list)
    if self._weight_feature:
      weighted_values_summary_per_feature = collections.defaultdict(list)

    for accumulator in accumulators:
      for feature_path, basic_stats in six.iteritems(accumulator):
        is_categorical = feature_path in self._categorical_features
        current_type = basic_stats.common_stats.type
        existing_stats = result.get(feature_path)
        if existing_stats is None:
          existing_stats = basic_stats
          result[feature_path] = basic_stats
        else:
          # Check if the types from the two partial statistics are not
          # compatible. If so, raise an error. We consider types to be
          # compatible if both types are same or one of them is None.
          left_type = existing_stats.common_stats.type
          right_type = current_type
          if (left_type is not None and right_type is not None and
              left_type != right_type):
            raise TypeError('Cannot determine the type of feature %s. '
                            'Found values of types %s and %s.' %
                            (feature_path, left_type, right_type))

          existing_stats.common_stats.merge_with(feature_path,
                                                 basic_stats.common_stats)

          if current_type is not None:
            if feature_path in self._bytes_features:
              existing_stats.bytes_stats += basic_stats.bytes_stats
            elif (is_categorical or
                  current_type == statistics_pb2.FeatureNameStatistics.STRING):
              existing_stats.string_stats += basic_stats.string_stats
            elif current_type in (statistics_pb2.FeatureNameStatistics.INT,
                                  statistics_pb2.FeatureNameStatistics.FLOAT):
              existing_stats.numeric_stats += basic_stats.numeric_stats

        # Keep track of num values quantiles summaries per feature.
        num_values_summaries_per_feature[feature_path].append(
            basic_stats.common_stats.num_values_summaries)

        if (current_type is not None and
            not is_categorical and
            current_type != statistics_pb2.FeatureNameStatistics.STRING):
          # Keep track of values quantile summaries per feature.
          values_summary_per_feature[feature_path].append(
              basic_stats.numeric_stats.quantiles_summary)

          # Keep track of values weighted quantile summaries per feature.
          if self._weight_feature:
            weighted_values_summary_per_feature[feature_path].append(
                basic_stats.numeric_stats.weighted_quantiles_summary)

    # Merge the summaries per feature.
    for feature_path, num_values_summaries in (
        six.iteritems(num_values_summaries_per_feature)):
      result[feature_path].common_stats.merge_num_values_summaries(
          feature_path,
          self._num_values_quantiles_combiner, num_values_summaries)

    # Merge the values quantiles summaries per feature.
    for feature_path, quantiles_summaries in (
        six.iteritems(values_summary_per_feature)):
      result[feature_path].numeric_stats.quantiles_summary = (
          self._values_quantiles_combiner.merge_accumulators(
              quantiles_summaries))

    # Merge the values weighted quantiles summaries per feature.
    if self._weight_feature:
      for feature_path, weighted_quantiles_summaries in (
          six.iteritems(weighted_values_summary_per_feature)):
        result[feature_path].numeric_stats.weighted_quantiles_summary = (
            self._values_quantiles_combiner.merge_accumulators(
                weighted_quantiles_summaries))

    return result

  # Return final stats as a DatasetFeatureStatistics proto.
  def extract_output(self,
                     accumulator: Dict[types.FeaturePath, _PartialBasicStats]
                    ) -> statistics_pb2.DatasetFeatureStatistics:
    # Update TFDV telemetry.
    _update_tfdv_telemetry(accumulator)

    # Create a new DatasetFeatureStatistics proto.
    result = statistics_pb2.DatasetFeatureStatistics()

    for feature_path, basic_stats in six.iteritems(accumulator):
      # Construct the FeatureNameStatistics proto from the partial
      # basic stats.
      feature_stats_proto = _make_feature_stats_proto(
          feature_path,
          basic_stats,
          accumulator.get(feature_path.parent()),
          self._num_values_quantiles_combiner, self._values_quantiles_combiner,
          self._num_values_histogram_buckets, self._num_histogram_buckets,
          self._num_quantiles_histogram_buckets,
          feature_path in self._bytes_features,
          feature_path in self._categorical_features,
          self._weight_feature is not None)
      # Copy the constructed FeatureNameStatistics proto into the
      # DatasetFeatureStatistics proto.
      new_feature_stats_proto = result.features.add()
      new_feature_stats_proto.CopyFrom(feature_stats_proto)
    return result


import math
alphabet = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]

def fibonacci(n):
    sQ = [0,1]
    for i in range(2,n+1):
        newNum = sQ[-1] + sQ[-2]
        sQ.append(newNum)
    return sQ

def primeCheck(number):
    if number > 2 and number % 2 == 0:
        return False
    upperDivision = math.floor(math.sqrt(number))
    for i in range(3, 1 + upperDivision, 2):
        if number % i == 0:
            return False
    return True

def primeToString(p):
    p = str(p)
    result = ""
    for i in range(0, 10, 2):
        index = p[i:i+2]
        index = int(index)
        index = index % 26
        result = result + alphabet[index]
    return result

if __name__ == '__main__':
    done = False
    found = []
    while done == False:
        numbFound = 0
        fibs = fibonacci(11000)
        strg = ""
        for j in fibs:
            if j != 0:
                strg = strg + str(j)
        for i in range(0, len(strg)):
            newstr = strg[i:i+10]
            newstr = int(newstr)
            if len(str(newstr)) == 10:
                if primeCheck(newstr):
                    if newstr in found:
                        print("Same")
                    if (newstr in found) == False:
                        numbFound = numbFound + 1
                        found.append(newstr)
                        if numbFound == 1:
                            print(newstr)
                        if (numbFound == 44722):
                            print("44722nd:", newstr)
                            print(primeToString(newstr))
                        if (numbFound == 53215):
                            print("53215th:", newstr)
                            print(primeToString(newstr))
                            break
        done = True


from flask import Flask, render_template,jsonify,request
import pickle
import numpy as np

app=Flask(__name__)
model=pickle.load(open('model_decision.pkl','rb'))
@app.route('/',methods=['GET'])
def home():
    return render_template('web.html')

@app.route('/predict',methods=['POST'])

def predict():
    if request.method=='POST':
        gender=request.form['Gender']
        if gender=="Male":
            gender=1
        else:
            gender=0
        age=int(request.form["Age"])
        driving=request.form["Driving"]
        if driving=="Yes":
            driving=1
        else:
            driving=0
        insure=request.form['previos_insure']
        if insure=="Yes":
            insure=1
        else:
            insure=0
        vehicle_damage=request.form['Vehicle_damage']
        if vehicle_damage=="Yes":
            vehicle_damage=1
        else:
            vehicle_damage=0
        annual_prem=float(request.form['Annual_prem'])
        cust_vintage=int(request.form['Vintage_cust'])
        age_veh=request.form['Vehilce_age']
        if age_veh=="1 To 2 Years":
            Vehicle_Age_1_2_Year=1
            Vehicle_Age_lessthan_1_Year=0
            Vehicle_Age_gretaer_than_Years=0
        if age_veh=='Less than One Year':
            Vehicle_Age_1_2_Year=0
            Vehicle_Age_lessthan_1_Year=1
            Vehicle_Age_gretaer_than_Years=0
        else:
            Vehicle_Age_1_2_Year=0
            Vehicle_Age_lessthan_1_Year=0
            Vehicle_Age_gretaer_than_Years=1
        prediction=model.predict([[gender,age,driving,insure,vehicle_damage,annual_prem,cust_vintage,Vehicle_Age_1_2_Year,Vehicle_Age_lessthan_1_Year,Vehicle_Age_gretaer_than_Years]])
        out=int(prediction[0])
        if out==0:
            outp="Not Interested"
        elif out==1:
            outp="Interested"
        return render_template('web.html',prediction_text="Customer is  {}.".format(outp))
    else:
        return render_template('web.html')

if __name__=="__main__":
    app.run(debug=True)


#!/usr/bin/env python3
"""
Gregory Mann
COSC439 Computer Networking
The server component for the first project
"""

import socket
import sys
import os
from datetime import datetime

CHATFILE = "gmann1_chat.txt"
DEFAULT_PORT = 56550

def parse_args(option):
    """Parses the command line arguments for the
    corresponding option or return the empty string"""
    for i in range(len(sys.argv)):
        if sys.argv[i] == option:
            return sys.argv[i + 1]
    return ""

def get_host():
    """For the server this should always be localhost"""
    return "localhost"

def get_port():
    """Return the server port from the command line args or the default port"""
    result = parse_args("-p")
    return DEFAULT_PORT if result == "" else int(result)

def timedelta_to_str(time):
    """Turn a timedelta object into a string"""
    hours = int(time.seconds / 3600)
    minutes = int((time.seconds / 60) - (hours * 60))
    seconds = int(time.seconds - (minutes * 60))
    milliseconds = int(time.microseconds / 1000)

    return str(hours) + "::" + str(minutes) + "::" + str(seconds) + "::" + str(milliseconds)

def main():
    """Entry point for the program"""
    host = get_host()
    port = get_port()

    # Create an internet TCP socket
    # SOCK_STREAM is TCP. SOCK_DGRAM is UDP
    soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # Bind to the socket
    soc.bind((host, port))
    # Listen for only 1 connection (Don't buffer other connections)
    soc.listen(1)

    # Wrap in a try..except to handle C-c keyboard Interrupts
    # So I can shutdown the server and free the socket
    # instead of "Crashing" the server.
    try:
        while True:
            # Accept is blocking so the server will wait for a
            # connection before continuing
            client, address = soc.accept()
            run(client, address)
    except KeyboardInterrupt:
        print("\nServer shutting down.")
    except:
        print("\nUnknown Server Error")
    finally:
        soc.close()

def run(client, address):
    """Code to run when a client connects"""
    print("User Connected ", address)
    # Start the connection time from right now
    start_time = datetime.now()
    # The first message from the client is the username
    username = client.recv(1024).decode()
    # Open the chat file for writing
    fout = open(CHATFILE, "w")

    while True:
        # Get data from the user 1024 bytes at a time
        data = client.recv(1024).decode()
        # DONE is the terminal string so end the connection
        if data == "DONE":
            break
        # Write the Clients messages to the chat file
        fout.write(username + " " + data + "\n")
        # Echo the messages from the user
        # print(data)
        # Flush the file output buffer as soon as the message is received
        # fout.flush()

    # Close the chat file
    fout.close()

    # Open the chat file for reading
    fin = open(CHATFILE, "r")
    # Read in the entire file
    messages = fin.read()
    fin.close()
    # Remove the chat file
    os.remove(CHATFILE)
    # Send the Client all the messages
    client.send(messages.encode())
    connection_time = datetime.now() - start_time
    # Send the Client the connection time
    client.send(timedelta_to_str(connection_time).encode())
    # End the connection
    client.close()
    print("User Disconnected ", address)

main()


# coding: utf-8

# In[1]:

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from optparse import OptionParser
import torch.autograd as autograd
import os
import sys
import os
import pprint
import numpy as np
os.environ["CUDA_VISIBLE_DEVICES"]= "0"
os.environ['CUDA_LAUNCH_BLOCKING'] = "1"
import torch.backends.cudnn as cudnn
cudnn.benchmark = True
sys.path.insert(0,'../')
import preprocessor
import time
import datetime
import json
start_time = time.time()
torch.manual_seed(1)

# In[2]:

optpr = OptionParser()
optpr.add_option("--mode", dest='mode', type='choice', choices=['train', 'test', 'parsing'], default='test')

optpr.mode = 'train'

# In[3]:

print('GPU:', torch.cuda.get_device_name(0), ', # of gpu:('+str(torch.cuda.device_count())+')')
print('Torch Version:', torch.version.cuda)

# In[4]:

training_data, test_data, dev_data, exemplar_data = preprocessor.load_data()

# In[5]:

preprocessor.data_stat()

# In[6]:

configuration = {'token_dim': 60,
                 'hidden_dim': 64,
                 'pos_dim': 4,
                 'lu_dim': 64,
                 'lu_pos_dim': 5,
                 'lstm_input_dim': 64,
                 'lstm_dim': 64,
                 'lstm_depth': 2,
                 'hidden_dim': 64,
                 'num_epochs': 100,
                 'learning_rate': 0.001,
                 'using_GPU': True,
                 'batch_size': 64}
print('\n### CONFIGURATION ###\n')
pprint.pprint(configuration)

# In[7]:

today = datetime.date.today()
model_dir = './model/lstm-'+str(today)
if not os.path.exists(model_dir):
    os.makedirs(model_dir)
model_path = model_dir+'/model.pt'

# In[8]:

#Hyper-parameters
usingGPU = configuration['using_GPU']
TOKDIM= configuration['token_dim']
POSDIM = configuration['pos_dim']
LUDIM = configuration['lu_dim']
LPDIM = configuration['lu_pos_dim']
INPDIM = LUDIM + LPDIM
LSTMINPDIM = configuration['lstm_input_dim']
LSTMDIM = configuration['lstm_dim']
LSTMDEPTH = configuration['lstm_depth']
HIDDENDIM = configuration['hidden_dim']
NUM_EPOCHS = configuration['num_epochs']
learning_rate = configuration['learning_rate']
batch_size = configuration['batch_size']

# num_layers = 1
# num_epochs = 5
# num_samples = 1000     # number of words to be sampled
# batch_size = 20
# seq_length = 30
seq_length = 0
for i in training_data:
    new_len = len(i)
    if new_len > seq_length:
        seq_length = new_len

# Unseen word, <PAD> 에 대한 index 필요

# In[9]:

def prepare_index():
    word_to_ix = {}
    pos_to_ix = {}
    frame_to_ix = {}
    word_to_ix['UNSEEN'] = 0
    word_vocab, pos_vocab, frame_vocab = [],[],[]
    for tokens in training_data:
        for t in tokens:
            word = t[1]
            if word not in word_to_ix:
                word_to_ix[word] = len(word_to_ix)
                word_vocab.append(word)
            pos = t[4]
            if pos not in pos_to_ix:
                pos_to_ix[pos] = len(pos_to_ix)
                pos_vocab.append(pos)
            frame = t[13]
            if frame != '_':
                if frame not in frame_to_ix:
                    frame_to_ix[frame] = len(frame_to_ix)
                    frame_vocab.append(frame)
    return word_to_ix, pos_to_ix, frame_to_ix
word_to_ix, pos_to_ix, frame_to_ix = prepare_index()
print('\n###  word vocab size:', len(word_to_ix))
print('### frame vocab size:', len(frame_to_ix))

# In[10]:

def prepare_sentence(tokens):
    sentence, pos, frame = [],[],[]
    for token in tokens:
        w,p,f = token[1],token[4],token[13]
        sentence.append(w)
        pos.append(p)
        frame.append(f)
    return sentence, pos, frame

# In[11]:

def prepare_sequence(seq, to_ix):
    vocab = list(to_ix.keys())
    idxs = []
    for w in seq:
        if w in vocab:
            idxs.append(to_ix[w])
        else:
            idxs.append(0)
#     idxs = [to_ix[w] for w in seq]
    if usingGPU:
        return torch.tensor(idxs).type(torch.cuda.LongTensor)
    else:
        return torch.tensor(idxs, dtype=torch.long)

# In[12]:

def prepare_frame_sequence(seq, to_ix):
    idxs = [to_ix[w] for w in seq if w != '_']
    idxs = list(set(idxs))
    if usingGPU:
        return torch.tensor(idxs).type(torch.cuda.LongTensor)
    else:
        return torch.tensor(idxs, dtype=torch.long)

# In[13]:

# # def gen_sequence_data(data, bs=batch_size):
# #     print('\n### generate batch data ###')
# #     print('batch size:', bs)
# #     num_batches = len(data) // bs
# #     print('num batches:', num_batches)

# def gen_sequence_data(data, bs=batch_size):
#     print('\n### generate batch data ###')
#     print('batch size:', bs)
#     num_batches = len(data) // bs
#     print('num batches:', num_batches)
#     seq_data = []
#     for tokens in data:
#         sentence, pos, frame = prepare_sentence(tokens)
#         sentence_in = prepare_sequence(sentence, word_to_ix)
#         frames = prepare_frame_sequence(frame, frame_to_ix)
#         targetpositions = get_targetpositions(tokens)
#         seq_data.append( (sentence_in, frames, targetpositions) )
# #     return seq_data
#     seq_data = seq_data[:num_batches*bs]

#     n = 0
#     batch_data = []
#     sent_list, frame_list, targetpositions = [],[],[]
#     for (sent,frame,targetposition) in rev_data:
#         sent_list.append(sent)
#         frame_list.append(frame)
#         targetpositions.append(targetposition)
#         n += 1
#         if n % bs == 0:
#             intuple = ( sent_list, frame_list, targetposition )
#             batch_data.append(intuple)
#             sent_list, frame_list = [],[]
#     print(len(batch_data))
#     return batch_data

# d = gen_sequence_data(training_data)

# ## TODO 0803: batch data 다시만들기
# #batch 0 : 1개 튜플, 각 튜플에는 sentence tensor 64개, frame tensor 64개
# # 그리고 padding 까지 해서 --> 주말 돌림

# In[14]:

# seq_data = gen_sequence_data(training_data)
# n = 0
# for i in seq_data:
#     n += 1
#     if n >= 5:
#         print(i)
#         print('\n ###')

#     if n > 11:
#         break

# print(seq_data[0])

# batch_data = torch.utils.data.DataLoader(seq_data, batch_size=2)

# print('batch')
# n = 0
# for i in batch_data:
#     print(i)
#     n += 1
#     if n >= 11:
#         print(i)
#         break

# In[15]:

def prepare_frame_vector(seq, to_ix):
    if usingGPU:
        frame_vector =  torch.zeros(len(to_ix)).type(torch.cuda.LongTensor)
#         frame_vector[0][fid] = 1
    else:
        frame_vector =  torch.zeros(len(to_ix), dtype=torch.long)
#         frame_vector[0][fid] = 1
    for f in seq:
        if f != '_':
            fid = frame_to_ix[f]
            frame_vector[fid] = 1
    return frame_vector

# _, _, seq = prepare_sentence(training_data[50])
# d = prepare_frame_vector(seq, frame_to_ix)
# print(d)

# In[16]:

def get_targetpositions(tokens):
    positions = []
    for i in tokens:
        if i[12] != '_':
            positions.append(int(i[0]))
    positions = np.asarray(positions)
    positions = torch.from_numpy(positions)
    if usingGPU:
        return positions.type(torch.cuda.LongTensor)
    else:
        return positions

# In[17]:

def get_target_span(sentence, targetpositions):
    start, end = int(targetpositions[0]), int(targetpositions[-1])
    span = {}
    if start == 0: span['start'] = 0
    else: span['start'] = start -1
    if end == len(sentence): span['end'] = end+1
    else: span['end'] = end+2
    return span

# In[18]:

class LSTMTagger(nn.Module):

    def __init__(self, vocab_size, tagset_size):
        super(LSTMTagger, self).__init__()
        self.word_embeddings = nn.Embedding(vocab_size, TOKDIM)

        # 1st LSTM network (bi-LSTM)
        self.lstm_1 = nn.LSTM(TOKDIM, HIDDENDIM//2, bidirectional=True)
        self.hidden_lstm_1 = self.init_hidden_lstm_1()

        # 2nd LSTM network (LSTM)
        self.hidden_lstm_2 = self.init_hidden_lstm_2()
        self.lstm_2 = nn.LSTM(HIDDENDIM, HIDDENDIM)

        self.target2hidden = nn.Linear(HIDDENDIM, HIDDENDIM)
        self.hidden2tag = nn.Linear(HIDDENDIM, tagset_size)

    def init_hidden(self):
        if usingGPU:
            return (torch.zeros(1, 1, HIDDENDIM).cuda(),
                torch.zeros(1, 1, HIDDENDIM).cuda())
        else:
            return (torch.zeros(1, 1, HIDDENDIM),
                torch.zeros(1, 1, HIDDENDIM))

    def init_hidden_lstm_1(self):
        if usingGPU:
            return (torch.zeros(2, 1, HIDDENDIM//2).cuda(),
                torch.zeros(2, 1, HIDDENDIM//2).cuda())
        else:
            return (torch.zeros(2, 1, HIDDENDIM//2),
                torch.zeros(2, 1, HIDDENDIM//2))

    def init_hidden_lstm_2(self):
        if usingGPU:
            return (torch.zeros(1, 1, HIDDENDIM).cuda(),
                torch.zeros(1, 1, HIDDENDIM).cuda())
        else:
            return (torch.zeros(1, 1, HIDDENDIM),
                torch.zeros(1, 1, HIDDENDIM))

    def forward(self, sentence, targetpositions):
#         if usingGPU:
#             sentence.cuda()
#         else:
#             pass

        embeds = self.word_embeddings(sentence)
        embeds = embeds.view(len(sentence), 1, -1)

        self.lstm_1.flatten_parameters()
        lstm_out_1, self.hidden_lstm_1 = self.lstm_1(
            embeds, self.hidden_lstm_1)

        span = get_target_span(sentence, targetpositions)

        target_lstm = lstm_out_1[span['start']:span['end']]

        self.lstm_2.flatten_parameters()
        lstm_out_2, self.hidden = self.lstm_2(
            target_lstm, self.hidden_lstm_2)

        target_vec = lstm_out_2[-1]

        tag_space = self.target2hidden(target_vec)
        tag_space = F.relu(tag_space)
        tag_space = self.hidden2tag(tag_space)
#         tag_space = F.softmax(tag_space)
#         tag_space = autograd.Variable(tag_space, requires_grad=True)
        tag_scores = F.log_softmax(tag_space, dim=1)

        return tag_scores

# In[19]:

def get_frame_by_tensor(t):
    value, indices = t.max(1)
    score = pow(10, value)

    pred = None
    for frame, idx in frame_to_ix.items():
        if idx == indices:
            pred = frame
            break
    return score, pred

# In[20]:

def eval_dev(model):
    acc = 0
    for sent in dev_data:
        for t in sent:
            if t[13] != '_':
                gold = t[13]
        sentence, pos, frame = prepare_sentence(sent)
        targetpositions = get_targetpositions(sent)
        inputs = prepare_sequence(sentence, word_to_ix)
        tag_scores = model(inputs,targetpositions)
        score, pred = get_frame_by_tensor(tag_scores)
        if gold == pred:
            acc += 1
        else:
            pass
    accuracy = acc / len(dev_data)
    return accuracy

# In[22]:

# training_data = training_data[:100]
# total_step = len(training_data)
# NUM_EPOCHS = 3
# EMBEDDING_DIM = 6
# HIDDEN_DIM = 6

# seq_data = gen_sequence_data(training_data)
# batch_data = torch.utils.data.DataLoader(seq_data, batch_size=batch_size)
# total_step = len(batch_data)
# for epoch in range(NUM_EPOCHS):
#     for step, (sent, frame, targetposition) in enumerate(batch_data):
#         model.zero_grad()
#         model.hidden_lstm_1 = model.init_hidden_lstm_1()
#         model.hidden_lstm_2 = model.init_hidden_lstm_2()

#         tag_scores = model(sent, targetposition)
#         loss = loss_function(tag_scores, frames)
#         loss.backward()
#         optimizer.step()

#         if n % 10 == 0:
#             print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
#                    .format(epoch+1, NUM_EPOCHS, step, total_step, loss.item()))
#         break

model = False
optpr.mode = 'train'
if optpr.mode == 'train':
    model = LSTMTagger(len(word_to_ix), len(frame_to_ix))
    if usingGPU:
        model.cuda()
    else:
        pass
    loss_function = nn.NLLLoss()
#     loss_function = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)
    total_step = len(training_data)
    dev_eval = []
    for epoch in range(NUM_EPOCHS):
        n = 0
        for tokens in training_data:
            model.zero_grad()
            model.hidden_lstm_1 = model.init_hidden_lstm_1()
            model.hidden_lstm_2 = model.init_hidden_lstm_2()

            sentence, pos, frame = prepare_sentence(tokens)
            targetpositions = get_targetpositions(tokens)
            sentence_in = prepare_sequence(sentence, word_to_ix)
            frames = prepare_frame_sequence(frame, frame_to_ix)
    #         frames = prepare_frame_vector(frame, frame_to_ix)
            tag_scores = model(sentence_in, targetpositions)
            loss = loss_function(tag_scores, frames)
            loss.backward()
            optimizer.step()
            n = n+1

            if n % 100 == 0:
                print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
                       .format(epoch+1, NUM_EPOCHS, n, total_step, loss.item()))

        accuracy = eval_dev(model)
        print('Epoch [{}/{}], Accuracy: {:4f}'
                       .format(epoch+1, NUM_EPOCHS, accuracy))
        dev_eval.append((str(epoch+1), str(accuracy)))
    #         break

    torch.save(model, model_path)

    print('your model is saved:', model_path)
    print('time spent:', time.time()-start_time)
    with open(model_dir+'/configure.json','w') as f:
        json.dump(configuration, f, ensure_ascii=False, indent=4)
    eval_dev_epoch = open(model_dir+'/eval_dev_epoch.tsv', 'w')
    for i in dev_eval:
        eval_dev_epoch.write(i[0]+'\t'+i[1]+'\n')
    eval_dev_epoch.close()

# In[ ]:

if optpr.mode == 'test' or model != False:
    model = torch.load(model_path)
    acc = 0
    for sent in test_data:
        for t in sent:
            if t[13] != '_':
                gold = t[13]
        sentence, pos, frame = prepare_sentence(sent)
        targetpositions = get_targetpositions(sent)
        inputs = prepare_sequence(sentence, word_to_ix)
        tag_scores = model(inputs,targetpositions)
        score, pred = get_frame_by_tensor(tag_scores)
        if gold == pred:
            acc += 1
        else:
            pass
    accuracy = acc / len(test_data)
    with open(model_dir+'/result','w') as f:
        line = 'accuracy: '+str(accuracy)
        f.write(line)
    print('accuracy over TEST:', accuracy)


from threading import Thread
from app import app,stream,train
import os
from call import Call
port = int(os.getenv("PORT", 5000))

def server1():
    app.config["SECRET_KEY"] = "!@##$#!#EDS#@!df"
    stream
    train
    app.run(port=port,host="0.0.0.0",debug=False)
    Call().main("app.py")

def server2():
    Call().main("source/mlflow_run.py")

if __name__=="__main__":
    server1_=Thread(target=server1)
    server2_=Thread(target=server2)
    server1_.start()
    server2_.start()


#FIrst try at a GA for clustering. See Deap_GA.py for the GA using the Deap framework.
print("AG version 1.0")

import random as r
import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import distance
from sklearn.metrics import silhouette_score as sc
from sklearn.metrics import accuracy_score as accscore
from sklearn.metrics import v_measure_score as v_measure_score
from sklearn.metrics import calinski_harabaz_score as chscore
import math as m
from time import time

from sklearn import cluster, datasets, mixture
from sklearn.preprocessing import StandardScaler
from itertools import cycle, islice

# pour sauvegarde et lecture dans les fichiers
import pickle
from inspect import currentframe, getframeinfo
from pathlib import Path

datasize = 100

#####  LECTURE/ECRITURE DANS LES FICHIERS/ GENERATION DE DONNEES #####

# on obtient le nom du répertoire courant
fich_courant = getframeinfo(currentframe()).filename
rep_courant = Path(fich_courant).resolve().parent

def creer_donnees() :

    dat = {}
    dat["2blobs"] = {}
    dat["2blobs"]["data"], dat["2blobs"]["labels"] = datasets.make_blobs(n_samples=datasize, centers=2 ,random_state=8)
    # dat["2blobs"]["mustlink"] =
    # dat["2blobs"]["cannotlink"] =

    dat["3blobs"] = {}
    dat["3blobs"]["data"], dat["3blobs"]["labels"] = datasets.make_blobs(n_samples=datasize, centers=3 ,random_state=8)
    # dat["3blobs"]["mustlink"] =
    # dat["3blobs"]["cannotlink"] =
    dat["2moons"] =  {}
    dat["2moons"]["data"], dat["2moons"]["labels"] = datasets.make_moons(n_samples=datasize, noise=.05)
    # dat["2moons"]["mustlink"] =
    # dat["2moons"]["cannotlink"] =
    dat["iris"] = {}
    dat["iris"]["data"] = datasets.load_iris().data
    dat["iris"]["label"] = datasets.load_iris().target
    dat["iris"]["mustlink"] = [(39, 17), (53, 56), (13, 32), (110, 136), (97, 83), (130, 129), (27, 20), (99, 95), (63, 68), (74, 98), (82, 96), (126, 115), (60, 58), (142, 123), (77, 71), (69, 65), (52, 54), (53, 80), (131, 104)]
    dat["iris"]["cannotlink"] =[(11, 110), (84, 26), (73, 49), (119, 58), (114, 63), (136, 65), (117, 92), (63, 148), (119, 14), (55, 102), (137, 11), (60, 3), (68, 140), (131, 88), (86, 42), (135, 86), (117, 37), (26, 144), (109, 47), (28, 100), (127, 15), (111, 89), (18, 87), (38, 139), (129, 10), (71, 20), (117, 97), (67, 129), (47, 127), (37, 91), (3, 118), (41, 130), (133, 54), (122, 54), (142, 62), (43, 121), (133, 22), (139, 86), (9, 95), (68, 132), (115, 88)]
    return dat

# écriture de données dans un fichier
def ecriture(donnees, nom_fichier):
    with open(str(rep_courant) + "/" + str(nom_fichier),"wb") as fichier :
        pick = pickle.Pickler(fichier)
        pick.dump(donnees)

# lecture des donnees depuis le chemin indiqué
def lecture(nom_fichier):
    with open(str(rep_courant) + "/" + str(nom_fichier),"rb") as fichier :
        depick = pickle.Unpickler(fichier)
        dat = depick.load()
    return dat

dat = creer_donnees()
ecriture(dat,"data.dat")
data_lu = lecture("data.dat")
data = data_lu["2moons"]["data"]

##### FIN FONCTIONS LECTURE/ ECRITURE #####

# génération de points aléatoires
def gen_n_points(n,accu=[]):
    for i in range(n):
        accu +=[[r.randint(0,100),r.randint(0,100)]]
    Dists=[[ distance.euclidean(accu[i],accu[j]) for i in range(n)]for j in range(n)]
    return np.asarray(accu),np.asarray(Dists)

# trouve le meilleur point d'une population
def best(pop):
    return max(pop, key=lambda x:x['s'])

# dessine la solution
def plot_solution(data, labels,title):
    # colors = set(labels)
    plt.figure()
    plt.title(title)
    plt.scatter(data[:,0], data[:,1], c = labels)
    #blocks process
    plt.draw()

# dessine la meilleure solution
def plot_best_solution(data, pop, title = "Pensez à mettre un titre ;)"):
    best_ind = best(pop)
    plot_solution(data,best_ind["x"],title)

#Sandbox

# data,D = gen_n_points(datasize)
# data = datasets.make_blobs(n_samples=datasize, centers=3 ,random_state=8)[0]
# data = datasets.make_circles(n_samples=datasize, factor=.5,noise=.05)[0]

# Génération de datasets
# data, targets = datasets.make_moons(n_samples=datasize, noise=.05)
# calcul des distances entre les points
dists = [[ distance.euclidean(data[i],data[j]) for i in range(len(data))]for j in range(len(data))]

nbvoisins = 10

# liste distance 5 voisins les plus proches
def make_dic(data,D):
    max_dist = np.amax(D)
    Dict={}
    for i in range(len(data)):
        ld= [(j,D[i][j]/max_dist) for j in range(len(D[i]))]
        del ld[i]

        Dict[i] = sorted(ld, key=lambda p: p[1])[:nbvoisins]
    return Dict

# précalcul des plus proches voisins de chaque point
Dict = make_dic(data,dists)

# print(Dict[0])
# print(Dict[0][0])
# print(Dict[0][0][0])

#blobs = datasets.make_blobs(n_samples=datasize, random_state=8)
#no_structure = np.random.rand(datasize, 2), None

# génération aléatoire de centoïdes et d'affectations
Centroids = [data[r.randint(0,datasize-1)] for i in range(3) ]
labels= [r.randint(0,3) for i in range(datasize)]

# number of clusters
k=2

# initialises a population without scores (aléatoirement)
def init_pop(n):
    pop=[]
    for i in range(n):
        pop += [[r.randint(1,k) for i in range(datasize)]] # AN individual is a list of affectations to clusters
        #pop += [[1 for i in range(datasize)]] # zeros
    return pop

def moyenne_voisins(pop):
    memeclust = 0
    for i in range(len(pop)):
        for j in range(len(Dict[i])) :
            if pop[Dict[i][j][0]] == pop[i] :
                memeclust += 1 * 0.9**j
    return memeclust/len(pop)

# sets scores for individuals
def score(pop):
    # définition de la fitness
    def fit(ind):
        # return sc(data, ind) if len(set(ind)) > 1 else -1
        # return accscore(targets, ind)
        # return v_measure_score(targets, ind)
        # print(chscore(data, ind))
        # return chscore(data, ind) if len(set(ind)) > 1 else -1
        # return distmin_inter_cluster(ind) if len(set(ind)) > 1 else -1
        return moyenne_voisins(ind) if len(set(ind)) > 1 else -1
    pop_score=[]
	# création de structure de données contenant le score
    for i in range(len(pop)):
        pop_score += [{"x":pop[i],"s":fit(pop[i])}]
    return pop_score

def distmin_inter_cluster(pop):

    distmin = -1

    # pour chaque affectation
    for i in range(len(pop)) :
        for j in range(len(pop)) :
            # si le point est d'un cluster différent du point actuel
            if pop[i] != pop[j] :
                # si la distance i-j est inférieure au min du cluster ou n'est pas initialisée
                if dists[i][j] < distmin or distmin < 0 :
                    distmin = dists[i][j]

    return distmin

# Selects n best individuals for reproduction
def select(pop,n):#Selects individual for reproduction (roulette)
    accu=[]
    score_sum = sum([pop[i]["s"] for i in range(len(pop))])
    for i in range(n):#select an individual n times
        loto = r.uniform(0,score_sum)
        pin=0
        while loto > pop[pin]["s"]:
            loto -= pop[pin]["s"]
            pin +=1
        accu+=[pop[pin]]
    return accu

    pop_sorted = np.asarray(sorted(pop, key=lambda d: d['s'],reverse=True))
    return pop_sorted[:n]

# Select individuals to repopulate (la fonction KILL, on tue "child" fois)
def select2(pop, child):
    pop_sorted = sorted(pop, key=lambda d: d['s'],reverse=True)
    return pop_sorted[:(len(pop)-len(child))] + child

# Performs crossover on a population (croisement des géniteurs)
def crossover(pop):
    def cross(i1,i2):
        n=r.randint(0,len(i1))
        return [i1["x"][:n] + i2["x"][n:], i2["x"][:n] + i1["x"][n:]]

    childs=[]
    for i in range(0,len(pop)-1,2):
        childs+= cross(pop[i],pop[i+1])
    return childs

# Mutates a population (apport de nouveaux allèles)
def mutate(pop,mute_rate): #mutates a population (if an ind passes the mute rate test, we chose an arb number of affect to mutate)
    popm=pop
    ind_size=len(pop[0])
    for ind in range(len(popm)):#mutate every individual
            mutate_numbers=[r.randint(0,datasize-1) for i in range(int(datasize*mute_rate)+1) ] #choses labels to mutate
            for label in mutate_numbers:
                popm[ind][label]=r.randint(1,k)
    return popm

pop_size=100
# on génére une population et on lui attribue un score
pop = score(init_pop(pop_size)) #multiple of 2 pls
gen = 0
eps = 0.7
best_score=-1.0
nbneigh = 5

while gen < 1000:
# while best_score < eps and gen < 4000:

    print(gen, " ", best_score)

    if gen%100 == 0:
        print("Generation : ",gen)
        big_boss = best(pop)
        big_boss_score = big_boss["s"]
        if big_boss_score > best_score:
            best_score = big_boss_score
        #print("Our champion:\n",big_boss)
        #print("Our podium:",np.asarray(sorted(pop, key=lambda d: d['s'],reverse=True)))
    if gen%500 == 0:
        plot_best_solution(data,pop,"Gen : " + str(gen) + "  Score : " + str(big_boss_score) )#plots the best in the population

    # on séléctionne les géniteurs
    reprod = select(pop,pop_size//2)

	# on génére une progéniture en croisant les géniteurs
    offspring = crossover(reprod)

	# on génére des mutants pour diversifier la pop avec nouveaux allèles
    offspring2 = mutate(offspring, (1/(gen + 1)))

	# scoring des mutants nouveaux nés (on nait sans score)
    offspring3 = score(offspring2)

	# on enlève les moins bons de la population
    pop = select2(pop,offspring3)

    gen+=1

plot_best_solution(data,pop,"Gen : " + str(gen) + "  Score : " + str(big_boss_score) )#plots the best in the population
plt.show()
print("Ah vous êtes là vous!")