import csv
import json
import logging

import cloudstorage
from src.clients import gcs
from src.csvmatchreplace import transform
from src.pipelines import pipeline
from src.pipelines import shardstage

class CsvMatchReplace(shardstage.ShardStage):
  """Match and replace strings in csv files (Also remove columns)."""

  @staticmethod
  def GetHelp():
    return """**CsvMatchReplace** will process a csv file.

It can remove columns, convert datatypes and parse datestrings.
It can also run a search/replace on column values using regular expressions.

The stage config should look like this:
```python
{
  "type": "CsvMatchReplace",
  "fieldDelimiter": ",",
  "columns": [{
    "wanted": true,
    "type": "STRING",
    "name": "col_1",
    "transformations": [{
      "match": "a",
      "replace": "pp"
    }, ...],
  }, ...],
  "skipLeadingRows": 0,
  "start": first_byte,
  "length": number_of_bytes,
  "shardSize": number_of_bytes,
  "sinks": ["gs://bucket_name/results", "gs://bucket_name/badrows"]
}
```

* The second sink is optional and will contain all the bad (unprocessed)
rows from the input.
* start and length are also optional and used when this task is sharded.
* If shardSize is specified this stage will be split up into jobs
that are that big and then the results composited together.
"""

  CHUNK_SIZE_1MB = 1 << 20
  CHUNK_SIZE_2MB = 1 << 21
  CHUNK_SIZE_4MB = 1 << 22
  SHARD_CHUNK_SIZE = CHUNK_SIZE_4MB

  def run(self, config):
    """Transform data according to some search/replace patterns from config.

    Args:
      config: Specifies what to replace and which columns are wanted.
    Yields:
      possibly yields some sharded stages.
    """

    # quick check to skip the leading rows
    skip_leading_rows = config.get('skipLeadingRows', 0)
    start = config.get('start', 0)
    source_url = config['sources'][0]

    if 'length' not in config:
      config['length'] = gcs.Gcs().StatObject(
          url=source_url)['size'] - start

    if skip_leading_rows > 0 and start == 0:
      # We're skipping these rows by using the start parameter.
      config['skipLeadingRows'] = 0
      bytes_to_skip = FindStartAfterSkippingRows(skip_leading_rows,
                                                 source_url)
      # We start one byte back since we find the start of the next
      # line when we start processing (to avoid jumping in at the
      # start of a line.
      config['start'] = max(0, bytes_to_skip - 1)
      if 'length' in config:
        config['length'] -= bytes_to_skip

    if 'shardSize' not in config:
      config['shardSize'] = self.SHARD_CHUNK_SIZE

    (shards, compositors) = self.ShardStage(config)
    if shards and compositors:
      with pipeline.After(*[(yield shard) for shard in shards]):
        _ = [(yield compositor) for compositor in compositors]
    else:
      # TODO(user) handle this task dying halfway through and resuming.
      # TODO(user) handle some way to update progress (memcache!?)

      # TODO(user) if the input file/blob is over 10M split it into
      # chunks and run over those and then merge it. Should be easy with
      # Pipelines.

      logging.info('Transformer start\n%s',
                   json.dumps(config, indent=4, separators=(',', ': ')))

      logging.info('CsvMatchReplace called for = %s', source_url)

      sink_url = config['sinks'][0]
      if len(config['sinks']) > 1:
        badrows_url = config['sinks'][1]
      else:
        badrows_url = None

      finished = ReadTransformWrite(config, source_url, sink_url, badrows_url)
      if not finished:
        logging.error('Unable to CsvMatchReplace')
        return

  def Lint(self, linter):
    """Stage-specific configuration linting."""
    linter.FieldCheck('fieldDelimiter', required=True)
    linter.FieldCheck('columns', field_type=list, required=True)

def FindStartAfterSkippingRows(skip_leading_rows, source_url):
  source_filename = gcs.Gcs.UrlToBucketAndNamePath(source_url)
  with cloudstorage.open(source_filename) as source_file:
    for _ in range(skip_leading_rows):
      source_file.readline()
    return source_file.tell()

def ReadTransformWrite(config, source_url, sink_url, badrows_url=None):
  """Transformation from one GCS file into another.

  Args:
    config: the transform config section from table.AsDataPipelineJsonDict.
    source_url: The blob_key of the csv file to transform.
    sink_url: The gs://bucket/name url to write the output transformations to.
    badrows_url: (optional) The gs://bucket/name url to write the badrows to.
  Returns:
    True only if the function successfully runs to completion.
  """

  delimiter = str(config['fieldDelimiter'])

  source_filename = gcs.Gcs.UrlToBucketAndNamePath(source_url)
  sink_filename = gcs.Gcs.UrlToBucketAndNamePath(sink_url)

  logging.info('CsvMatchReplace %r -> %r', source_filename, sink_filename)

  start = config.get('start', 0)
  length = config.get('length', -1)

  with cloudstorage.open(source_filename) as source_file:
    with cloudstorage.open(sink_filename, 'w') as sink_file:
      if start > 0:
        source_file.seek(start)
        source_file.readline()
      csv_writer = csv.writer(sink_file)
      csv_reader = csv.reader(source_file, delimiter=delimiter)
      if length > 0:
        finished_func = lambda: source_file.tell() > start + length
      else:
        finished_func = None

      if badrows_url:
        badrows_filename = gcs.Gcs.UrlToBucketAndNamePath(badrows_url)
        with cloudstorage.open(badrows_filename, 'w') as badrows_file:
          (row_count, bad_row_count) = ReadTransformWriteRows(
              config, csv_reader, csv_writer, finished_func, badrows_file)
      else:
        (row_count, bad_row_count) = ReadTransformWriteRows(
            config, csv_reader, csv_writer, finished_func)

    logging.info('CsvMatchReplace complete. %d rows, %d bad.',
                 row_count, bad_row_count)
    return True

def ReadTransformWriteRows(config, csv_reader, csv_writer,
                           finished_func=None,
                           badrows_file=None):
  """Transform each from from the csv_reader into the csv_reader."""
  row_count = 0
  bad_row_count = 0
  for row in csv_reader:
    (transformed_row, bad_cols) = transform.TransformRow(row, config)
    if not bad_cols:
      csv_writer.writerow(transformed_row)
    else:
      bad_row_count += 1
      logging.debug('Row %d\'s bad cols = %r', row_count, bad_cols)
      if badrows_file:
        transform.WriteErrors(badrows_file,
                              transform.CellsToCsvString(row), bad_cols)
    row_count += 1
    if finished_func and finished_func():
      break
  return (row_count, bad_row_count)


"""Classification of asteroids following DeMeo+ 2009."""
from functools import lru_cache
from urllib.request import urlretrieve

import numpy as np
import pandas as pd

from classy import cache
from classy import config
from classy import core
from classy.log import logger
from classy import preprocessing
from classy import tools

CLASSES = [
    "B",
    "Cg",
    "Cgh",
    "C",
    "Ch",
    "Cb",
    "D",
    "T",
    "Xc",
    "Xk",
    "Xe",
    "X",
    "K",
    "L",
    "O",
    "Q",
    "Sq",
    "S",
    "Sr",
    "Sa",
    "Sv",
    "R",
    "A",
    "V",
]

def is_classifiable(spec):
    """Check if spectrum can be classified based on the wavelength range.

    Parameters
    ----------
    taxonomy : str
        The taxonomic scheme to check.

    Returns
    -------
    bool
        True if the spectrum can be classified, else False.
    """
    if spec.wave.min() > WAVE.min() or spec.wave.max() < WAVE.max():
        # Check if the extrapolation would be sufficient
        if preprocessing._within_extrapolation_limit(
            spec.wave.min(), spec.wave.max(), WAVE.min(), WAVE.max()
        ):
            return True

        logger.warning(
            f"[{spec.source + '/' if hasattr(spec, 'source') else ''}{spec.name}]: Insufficient wavelength range for DeMeo taxonomy ({spec.wave.min()} - {spec.wave.max()})."
        )
        return False
    return True

# ------
# Functions for preprocessing
def preprocess(spec):
    """Preprocess a spectrum for classification following DeMeo+ 2009.

    Parameters
    ----------
    spec : classy.Spectrum
        The spectrum to preprocess.

    Returns
    -------
    classy.Spectrum
        The preprocessed spectrum.

    Notes
    -----
    Preprocessing steps include slope removal, renormalization, and resampling.
    """

    # Remove slope and renormalize
    spec.normalize(at=0.55)
    spec.remove_slope(translate_to=0.55)

    # Resample to DeMeo+ 2009 wavelength grid
    spec.resample(WAVE)
    spec.detect_features()

# ------
# Functions for classification
def classify(spec):
    """Classify a spectrum in the system of DeMeo+ 2009.

    Parameters
    ----------
    spec : classy.Spectrum
        The spectrum to classify.

    Returns
    ----------
    classy.Spectrum
        The spectrum which added classification attributes: ['class_demeo', 'scores_demeo'].
    """

    # Check if it can be classified in this scheme
    # Check if it has been preprocessed for this scheme

    # Extract the reflectance and demean following DeMeo+ 2009
    refl = np.concatenate([spec.refl[:2], spec.refl[3:]])
    refl -= DATA_MEAN.T

    # Compute scores
    spec.scores_demeo = EIGENVECTORS @ refl.T

    # And compute the class
    class_ = decision_tree(spec)
    add_classification_results(
        spec, results={"class_demeo": class_, "scores_demeo": spec.scores_demeo}
    )

def decision_tree(spec):
    """Implements the class decision tree given in Table B in Appendix B of DeMeo+ 2009."""

    # Align with DeMeo's notation but dropping the '
    pc1, pc2, pc3, pc4 = spec.scores_demeo[:4]
    slope = spec.slope[0]

    # Lines
    alpha = lambda pc2: -3 * pc2 - 0.28  # = pc1
    beta = lambda pc2: -3 * pc2 + 0.35  # = pc1
    gamma = lambda pc2: -3 * pc2 + 1.0  # = pc1
    delta = lambda pc2: -3 * pc2 + 1.5  # = pc1
    epsilon = lambda pc2: 1 / 3 * pc2 + 0.55  # = pc1
    zeta = lambda pc2: 1 / 3 * pc2 - 0.10  # = pc1
    eta = lambda pc2: 1 / 3 * pc2 - 0.40  # = pc1
    theta = lambda pc2: -3 * pc2 + 0.7  # = pc1

    # Vis-IR step 1
    if (pc1 < -0.3) and (pc3 >= 0.2) and (slope >= 0.4):
        if 0.55 <= slope < 1.5:
            return "A"
        elif 0.4 <= slope < 0.55:
            return "Sa"
        else:
            logger.warning("DeMeo class is indeterminate after VisIR step 1")
            spec.class_demeo = ""

    # Vis-IR step 2
    if pc1 > alpha(pc2):  # lies above alpha line
        if pc1 >= gamma(pc2):
            if slope >= 0.25:
                return "Vw"
            else:
                return "V"

        if pc1 <= eta(pc2) and pc1 >= theta(pc2) and pc1 < delta(pc2):
            return "O"
        if pc1 <= eta(pc2) and pc1 >= alpha(pc2) and pc1 < theta(pc2):
            if slope >= 0.25:
                return "Qw"
            else:
                return "Q"
        if pc1 >= eta(pc2) and pc1 >= gamma(pc2) and pc1 < delta(pc2):
            return "R"
        return demeo_s_complex(spec)

    # Vis-IR step 3
    if 0.38 <= slope < 1.5 and -0.44 < pc1 < 0.4:
        corr_A, corr_D = _compute_template_correlation(spec, ["A", "D"])

        if corr_A > corr_D:
            return "A"
        return "D"

    if 0.25 < slope < 0.38 and -0.28 < pc2 < -0.2 and -0.2 < pc3 < -0.12:
        return "T"

    if 0.07 < pc1 < 1.0 and -0.5 < pc2 < -0.15:
        if spec.e.is_present:
            return "Xe"
        return "L"

    if -0.075 < pc3 < 0.14 and -0.2 <= pc2 < 0.1 and -0.8 < pc1 < 0.1:
        if spec.e.is_present:
            return "Xe"
        return "K"

    return demeo_c_and_x_complexes(spec)

def demeo_s_complex(spec):
    # Align with DeMeo's notation but dropping the '
    pc1, pc2, pc3, pc4 = spec.scores_demeo[:4]
    slope = spec.slope[0]

    # Lines
    alpha = lambda pc2: -3 * pc2 - 0.28  # = pc1
    beta = lambda pc2: -3 * pc2 + 0.35  # = pc1
    gamma = lambda pc2: -3 * pc2 + 1.0  # = pc1
    delta = lambda pc2: -3 * pc2 + 1.5  # = pc1
    epsilon = lambda pc2: 1 / 3 * pc2 + 0.55  # = pc1
    zeta = lambda pc2: 1 / 3 * pc2 - 0.10  # = pc1
    eta = lambda pc2: 1 / 3 * pc2 - 0.40  # = pc1
    theta = lambda pc2: -3 * pc2 + 0.7  # = pc1

    if pc1 < beta(pc2) and pc1 > zeta(pc2):
        if slope >= 0.25:
            return "Sw"
        else:
            return "S"
    if pc1 >= alpha(pc2) and pc1 < beta(pc2) and pc1 > eta(pc2) and pc1 <= zeta(pc2):
        if slope >= 0.25:
            return "Sqw"
        else:
            return "Sq"
    if pc1 >= beta(pc2) and pc1 < gamma(pc2) and pc1 > eta(pc2) and pc1 <= epsilon(pc2):
        if slope >= 0.25:
            return "Srw"
        else:
            return "Sr"
    if pc1 >= beta(pc2) and pc1 > epsilon(pc2) and pc1 < gamma(pc2):
        if slope >= 0.25:
            return "Svw"
        else:
            return "Sv"
    return "S"

def demeo_c_and_x_complexes(spec):
    # Align with DeMeo's notation but dropping the '
    pc1, pc2, pc3, pc4, pc5 = spec.scores_demeo[:5]
    slope = spec.slope[0]

    # Lines
    alpha = lambda pc2: -3 * pc2 - 0.28  # = pc1
    beta = lambda pc2: -3 * pc2 + 0.35  # = pc1
    gamma = lambda pc2: -3 * pc2 + 1.0  # = pc1
    delta = lambda pc2: -3 * pc2 + 1.5  # = pc1
    epsilon = lambda pc2: 1 / 3 * pc2 + 0.55  # = pc1
    zeta = lambda pc2: 1 / 3 * pc2 - 0.10  # = pc1
    eta = lambda pc2: 1 / 3 * pc2 - 0.40  # = pc1
    theta = lambda pc2: -3 * pc2 + 0.7  # = pc1

    if -0.2 < slope < 0 and -1.2 < pc1 < 0 and pc4 < 0:
        return "B"
    if 0.2 < slope < 0.38:
        if spec.k.is_present:
            return "Xk"
        elif spec.e.is_present:
            return "C"
        else:
            corr_C, corr_X = _compute_template_correlation(spec, ["C", "X"])

            if corr_C > corr_X:
                return "C"
            return "X"

    if 0.01 < pc4 < 0.14 and -0.75 < pc1 < -0.27 and spec.refl[0] < 0.92:
        if spec.h.is_present:
            return "Cgh"
        return "Xk"

    if 0.01 < pc4 < 0.14 and -0.75 < pc1 < -0.27:
        # Ch is a smaller class -> return Ch if h is present,
        # else retrun Xk, even if no k feature is present
        if spec.h.is_present:
            return "Ch"
        elif spec.k.is_present:
            return "Xk"
        return "X"
    if -0.04 < pc4 < 0.02 and -0.07 < pc5 < -0.04:
        return "Cb"
    if -0.85 < pc1 < -0.45 and -0.06 < pc5 < 0.02:
        if spec.h.is_present:
            return "Ch"
        elif spec.k.is_present:
            return "Xk"
        return "C"
    if 0.02 <= pc5 < 0.1 and -0.6 < pc1 < -0.16:
        if spec.h.is_present:
            return "Cgh"
        elif spec.k.is_present:
            return "Xk"
        return "Cg"
    if -0.45 <= pc1 < 0.1 and -0.06 < pc5 < 0.05:
        if spec.h.is_present:
            return "Ch"
        elif spec.e.is_present:
            return "Xe"
        elif spec.k.is_present:
            return "Xk"
        return "Xc"
    if -0.1 <= pc1 < 0.3 and -0.5 < pc2 < -0.2:
        if spec.e.is_present:
            return "Xe"
        return "L"
    logger.warning(
        "DeMeo class is indeterminate C/X-complex member after VisIR resolution"
    )
    return ""

def add_classification_results(spec, results=None):
    if results is None:
        spec.class_demeo = ""
        spec.scores_demeo = [np.nan] * 7
        return

    for key, val in results.items():
        setattr(spec, key, val)

# ------
# Functions for plotting
# Functions for classification
@lru_cache(maxsize=None)
def load_classification():
    """Load the DeMeo+ 2009 classification results like PC scores and classes from file.

    Returns
    -------
    pd.DataFrame
        The classification results of the 371 SMASS spectra.
    """

    PATH_DATA = config.PATH_CACHE / "demeo2009/scores.csv"

    if not PATH_DATA.is_file():
        tools._retrieve_from_github(host="demeo2009", which="scores", path=PATH_DATA)

    return pd.read_csv(PATH_DATA, dtype={"number": "Int64"})

@lru_cache(maxsize=None)
def load_templates():
    """Load the spectral templates of the DeMeo+ classes.

    Returns
    -------
    dict
        Dictionary with classes (str) as key and templates as values (classy.Spectrum).
    """

    PATH_DATA = config.PATH_CACHE / "demeo2009/templates.csv"

    if not PATH_DATA.is_file():
        tools._retrieve_from_github(host="demeo2009", which="templates", path=PATH_DATA)

    data = pd.read_csv(PATH_DATA)
    data = data.replace(-0.999, np.nan)

    templates = {}

    for class_ in CLASSES:
        template = core.Spectrum(
            wave=data["wave"],
            refl=data[f"{class_}_Mean"],
            refl_err=data[f"{class_}_Sigma"],
            class_=class_,
            source=f"DeMeo+ 2009 - Class {class_}",
            _source="DeMeo+ 2009",
            id_=f"Template Class {class_}",
        )
        templates[class_] = template
    return templates

def _compute_template_correlation(spec, classes):
    """Compute the correlation coefficients between a spectrum and class templates.

    Parameters
    ----------
    spec : classy.Spectrum
        The spectrum to compare to the templates.
    classes : list of str
        The classes to compare to the spectrum.

    Returns
    -------
    list of float
        The correlation coefficients in the same order as the passed classes.
    """
    templates = load_templates()
    coeffs = []

    for class_ in classes:
        temp = templates[class_]
        temp.remove_slope(translate_to=0.55)
        coeffs.append(np.corrcoef(spec.refl, temp.refl)[0][1])
    return coeffs

def plot_pc_space(ax, spectra):
    """Plot the distribution of classified spectra and the SMASS spectra in the DeMeo PC space.

    Parameters
    ----------
    ax : matplotlib.axes.Axes
        The matplotlib axis instance to plot to.
    spectra : classy.Spectra or list of classy.Spectrum
        One or more spectra which were previously classified in the DeMeo system.

    Returns
    -------
    matplotlib.axes.Axes
        The matplotlib axis with the plotted classification results.
    """

    opts_text = dict(va="center", ha="center", clip_on=True)

    # ------
    # Add the distribution of the ECAS asteroids
    demeo = load_classification()

    for _, ast in demeo.iterrows():
        # Dummy to ensure proper plot limits
        ax.scatter(ast.PC1, ast.PC2, alpha=0)

        # Add asteroid position in PC space represented by its number
        ax.text(ast.PC1, ast.PC2, str(ast.number), color="lightgray", **opts_text)

    # ------
    # Add the mean positions of the main classes
    for class_, pcs in demeo.groupby("class_"):
        # Only add the core classe
        if len(class_) > 1:
            continue

        pc0 = np.mean(pcs.PC1)
        pc1 = np.mean(pcs.PC2)

        # Small offsets for readability
        if class_ == "E":
            pc0 += 0.05
        elif class_ == "P":
            pc0 += 0.09

        # Add class indicator
        ax.text(pc0, pc1, class_, size=14, color="black", **opts_text)

    # ------
    # Add classified spectra
    for spec in spectra:
        if not spec.class_demeo:
            logger.debug(f"[{spec.name}]: Not classified in DeMeo+ 2009 system.")
            continue

        ax.scatter(
            spec.scores_demeo[0],
            spec.scores_demeo[1],
            marker="d",
            c=spec._color,
            s=40,
            label=f"{spec.source + ': ' if hasattr(spec, 'source') else ''}{spec.class_demeo}",
            zorder=100,
        )

    # ------
    # Final additions et voila
    ax.axvline(0, ls=":", c="gray")
    ax.axhline(0, ls=":", c="gray")
    ax.legend()

    return ax

# ------
# Defintions

# Central wavelengths of the VisNIR spectra
WAVE = np.arange(0.45, 2.5, 0.05)

# fmt: off
CLASSES = ["A", "B", "Cg", "Cgh", "C", "Cb", "D", "K", "L",
           "Q", "S", "Sa", "Sq", "Sr", "Sv", "V", "T",
           "X", "Xe", "Xk",
]
# fmt: on

# fmt: off
# Data mean of DeMeo+ 09 reflectance spectra
DATA_MEAN = np.array(
    [
        0.8840578, 0.94579985, 1.04016798, 1.07630094, 1.10387232, 1.10729138, 1.07101476, 1.02252107,
        0.99167561, 0.98766575, 1.00292349, 1.02223844, 1.04660108, 1.07201578, 1.08967345, 1.10014259,
        1.11101667, 1.12359452, 1.13128556, 1.13642896, 1.13467689, 1.12810013, 1.11471935, 1.09802574,
        1.07842635, 1.06127665, 1.04536074, 1.03360292, 1.02395605, 1.01587389, 1.01034821, 1.00915786,
        1.01078308, 1.01245031, 1.01298133, 1.01314109, 1.01236654, 1.01140562, 1.01090655, 1.00955344,
    ]
)
# fmt: on

# fmt: off
EIGENVECTORS = np.array(
    [
        [-0.0766, -0.0391, 0.0438, 0.0876, 0.1256, 0.1466, 0.1271, 0.0888, 0.0680, 0.0857, 0.1371,
          0.1921, 0.2322, 0.2566, 0.2704, 0.2787, 0.2849, 0.2852, 0.2782, 0.2641, 0.2427, 0.2154,
          0.1841, 0.1531, 0.1247, 0.1002, 0.0804, 0.0665, 0.0570, 0.0513, 0.0502, 0.0538, 0.0607,
          0.0690, 0.0778, 0.0859, 0.0934, 0.0997, 0.1050, 0.1090,
        ],
        [ -0.0643, -0.0279, 0.0176, 0.0343, 0.0471, 0.0096, -0.1186, -0.2673, -0.3645, -0.3743,
          -0.2899, -0.1527, -0.0381, 0.0306, 0.0708, 0.1053, 0.1385, 0.1598, 0.1645, 0.1520,
           0.1192, 0.0689, 0.0089, -0.0514, -0.1069, -0.1532, -0.1884, -0.2136, -0.2283, -0.2317,
          -0.2233, -0.2023, -0.1706, -0.1302, -0.0852, -0.0406, 0.0023, 0.0438, 0.0832, 0.1177,
        ],
        [ -0.2724, -0.1270, 0.1128, 0.2104, 0.2726, 0.2475, 0.1486, 0.0420, -0.0385, -0.1168,
          -0.2083, -0.2809, -0.2747, -0.2169, -0.1713, -0.1427, -0.1031, -0.0407, 0.0243, 0.0930,
           0.1562, 0.2021, 0.2231, 0.2215, 0.2043, 0.1784, 0.1508, 0.1225, 0.0923, 0.0617, 0.0346,
           0.0136, -0.0038, -0.0229, -0.0447, -0.0678, -0.0911, -0.1153, -0.1389, -0.1580,
        ],
        [  0.3046, 0.1525, -0.1486, -0.2677, -0.3386, -0.3284, -0.2392, -0.1453, -0.0921, -0.0505,
          -0.0289, -0.0277, -0.0160, 0.0077, 0.0304, 0.0450, 0.0608, 0.0842, 0.1104, 0.1387, 0.1609,
           0.1752, 0.1804, 0.1714, 0.1550, 0.1421, 0.1279, 0.1095, 0.0868, 0.0610, 0.0358, 0.0103,
          -0.0162, -0.0476, -0.0838, -0.1225, -0.1644, -0.2068, -0.2445, -0.2708,
        ],
        [ -0.5174, -0.1876, 0.0593, 0.0754, 0.0523, -0.0231, -0.1466, -0.2569, -0.2293, -0.0657, 0.1077,
           0.1717, 0.1685, 0.1611, 0.1463, 0.1061, 0.0533, 0.0090, -0.0429, -0.0868, -0.1188, -0.1250,
          -0.1158, -0.0940, -0.0757, -0.0525, -0.0271, 0.0104, 0.0473, 0.0785, 0.1050, 0.1249, 0.1241,
           0.0916, 0.0354, -0.0327, -0.1126, -0.1993, -0.2884, -0.3767,
        ],
    ]
)
# fmt: on


import singer
from singer import metadata

from funcy import project

from tap_exacttarget.util import sudsobj_to_dict

LOGGER = singer.get_logger()

def _get_catalog_schema(catalog):
    return catalog.get('schema', {}).get('properties')

class DataAccessObject():

    def __init__(self, config, state, auth_stub, catalog):
        self.config = config.copy()
        self.state = state.copy()
        self.catalog = catalog
        self.auth_stub = auth_stub

    @classmethod
    def matches_catalog(cls, catalog):
        return catalog.get('stream') == cls.TABLE

    def generate_catalog(self):
        cls = self.__class__

        mdata = metadata.new()
        metadata.write(mdata, (), 'inclusion', 'available')
        for prop in cls.SCHEMA['properties']: # pylint:disable=unsubscriptable-object
            metadata.write(mdata, ('properties', prop), 'inclusion', 'available')

        return [{
            'tap_stream_id': cls.TABLE,
            'stream': cls.TABLE,
            'key_properties': cls.KEY_PROPERTIES,
            'schema': cls.SCHEMA,
            'metadata': metadata.to_list(mdata)
        }]

    def filter_keys_and_parse(self, obj):
        to_return = sudsobj_to_dict(obj)

        return self.parse_object(to_return)

    def get_catalog_keys(self):
        return list(
            self.catalog.get('schema', {}).get('properties', {}).keys())

    def parse_object(self, obj):
        return project(obj, self.get_catalog_keys())

    def write_schema(self):
        singer.write_schema(
            self.catalog.get('stream'),
            self.catalog.get('schema'),
            key_properties=self.catalog.get('key_properties'))

    def sync(self):
        mdata = metadata.to_map(self.catalog['metadata'])
        if not metadata.get(mdata, (), 'selected'):
            LOGGER.info('{} is not marked as selected, skipping.'
                        .format(self.catalog.get('stream')))
            return None

        LOGGER.info('Syncing stream {} with accessor {}'
                    .format(self.catalog.get('tap_stream_id'),
                            self.__class__.__name__))

        self.write_schema()

        return self.sync_data()

    # OVERRIDE THESE TO IMPLEMENT A NEW DAO:

    SCHEMA = None
    TABLE = None
    KEY_PROPERTIES = None

    def sync_data(self):  # pylint: disable=no-self-use
        raise RuntimeError('sync_data is not implemented!')


import math
import cmath

def ft(x):
    N = len(x)
    y = []
    for k in range(0, N):
        yk = 0
        for i in range(0, N):
            yk += x[i] * cmath.exp(-(2j*i*k*math.pi/N))
        yk = (1/math.sqrt(N)) * yk
        y.append(yk)
        print('FT: ' + str(k) + '/' + str(N))
    return y

def ift(y):
    N = len(y)
    x = []
    for k in range(0, N):
        xk = 0
        for i in range(0, N):
            xk += y[i] * cmath.exp((2j*i*k*math.pi/N))
        xk = (1/math.sqrt(N)) * xk
        x.append(xk)
        print('IFT: ' + str(k) + '/' + str(N))
    return x


import numpy as np
import os
files_names = ["a_example", "b_read_on", "c_incunabula", "d_tough_choices",
               "e_so_many_books", "f_libraries_of_the_world"]
files_names = [os.path.join("input", fn) for fn in files_names]

class HashCode2020Solver:
    def __init__(self, input_file_name):
        self.days_to_scan = 0
        self.num_books = 0
        self.num_libs = 0
        self.all_books_scores = None
        self.libs_books_matrix = None
        self.num_books_in_lib = None
        self.libs_signup_time = None
        self.libs_scan_rate = None
        self.input_file_name = input_file_name + '.txt'
        self.output_file_name = input_file_name + '_sol.txt'
        self.parse_input()

        # Solution attributes - corresponding indices!
        self.sol_libs = []
        self.sol_books_lists = []

    @staticmethod
    def split_str_to_int_list(st):
        return list(map(int, st.strip().split()))

    def parse_input(self):
        with open(self.input_file_name, 'r') as f:
            lines = f.readlines()
            num_lines = len(lines)
            if lines[-1] == '\n':
                num_lines -= 1

        self.num_books, self.num_libs, self.days_to_scan = self.split_str_to_int_list(lines[0])
        self.libs_books_matrix = np.zeros((self.num_libs, self.num_books), dtype=np.int8)
        self.libs_signup_time = np.zeros(self.num_libs, dtype=np.int32)
        self.libs_scan_rate = np.zeros(self.num_libs, dtype=np.int32)
        print(
            f"number of books: {self.num_books}, number of libraries: {self.num_libs}, scanning days: {self.days_to_scan}")
        self.all_books_scores = np.array(self.split_str_to_int_list(lines[1]), dtype=np.int32).reshape((1, -1))

        line1_idx = 2
        line2_idx = 3
        for lib_id in range(self.num_libs):
            _, t, m = self.split_str_to_int_list(lines[line1_idx])
            lib_book_ids = np.array(self.split_str_to_int_list(lines[line2_idx]), dtype=np.int32)
            # lib_book_scores = np.zeros(self.num_books, dtype=np.uint16)
            # lib_book_scores[lib_book_ids] = all_book_scores[lib_book_ids]
            self.libs_books_matrix[lib_id, lib_book_ids] = 1
            self.libs_signup_time[lib_id] = t
            self.libs_scan_rate[lib_id] = m
            line1_idx += 2
            line2_idx += 2
        self.num_books_in_lib = np.sum(self.libs_books_matrix, axis=1)

    def calculate_max_possible_books_to_scan(self):
        # number of days left to scan after signup (per library)
        scan_days_per_lib = self.days_to_scan - self.libs_signup_time
        # number of books that can be scanned in the remaining time (per library)
        books_to_scan_per_lib = scan_days_per_lib * self.libs_scan_rate
        # return the minimum between the actual number of books to the number of books that can be scanned
        result = np.minimum(books_to_scan_per_lib, self.num_books_in_lib)
        return result

    def calc_libraries_scores(self, max_books):
        scores = (np.sum(self.libs_books_matrix * self.all_books_scores, axis=1) /
                  self.num_books_in_lib) * max_books
        return scores

    def update_status_and_solution(self, all_libs_scores, max_books):
        """
        return and update the VALID_BOOKS
        """
        max_score_lib_idx = np.nanargmax(all_libs_scores)  # best library
        lib_scores = self.libs_books_matrix[max_score_lib_idx] * self.all_books_scores
        top_books_idx = np.argsort(lib_scores.flatten())
        max_books_to_scan = max_books[max_score_lib_idx]
        books_to_scan = top_books_idx[-max_books_to_scan:][::-1]  # the first best max_books of lib_id

        self.sol_libs.append(max_score_lib_idx)
        self.sol_books_lists.append(books_to_scan.tolist())

        # zero the scanned books in all the libraries
        self.libs_books_matrix[:, books_to_scan] = 0
        # zero all books in the chosen library
        self.libs_books_matrix[max_score_lib_idx, :] = 0
        # update books count
        self.num_books_in_lib = np.sum(self.libs_books_matrix, axis=1)
        # update days left
        self.days_to_scan -= self.libs_signup_time[max_score_lib_idx]

    def solve_challenge(self):
        while self.days_to_scan > 0:
            max_books = self.calculate_max_possible_books_to_scan()
            libraries_scores = self.calc_libraries_scores(max_books)
            if np.all(max_books <= 0) or np.all(np.isnan(libraries_scores)):
                break
            self.update_status_and_solution(libraries_scores, max_books)
        self.output_solutions()

    def output_solutions(self):
        num_libs = len(self.sol_libs)
        with open(self.output_file_name, 'w') as f:
            f.write(f'{num_libs}\n')
            for i, lib_idx in enumerate(self.sol_libs):
                curr_books_list = self.sol_books_lists[i]
                lib_num_books = len(curr_books_list)
                f.write(f"{lib_idx} {lib_num_books}\n{' '.join(map(str, curr_books_list))}\n")

class Hashcode2020SolverV2(HashCode2020Solver):

    def __init__(self, input_file_name):
        super().__init__(input_file_name)
        self.tmp_scan_rate = np.zeros(self.num_libs, dtype=np.int32)
        self.avg_score = np.sum(self.libs_books_matrix * self.all_books_scores, axis=1) / self.num_books_in_lib

    def find_quickest_lib(self):
        self.num_books_in_lib = np.sum(self.libs_books_matrix, axis=1)
        self.avg_score = np.sum(self.libs_books_matrix * self.all_books_scores, axis=1) / self.num_books_in_lib
        quickest_lib = np.lexsort((self.avg_score, -self.libs_scan_rate, self.libs_signup_time))[0]
        return quickest_lib

    def update_solution(self, quickest_lib):
        lib_scores = self.libs_books_matrix[quickest_lib] * self.all_books_scores
        self.days_to_scan -= self.libs_signup_time[quickest_lib]
        scan_rate = self.libs_scan_rate[quickest_lib]
        books_left_in_lib = self.num_books_in_lib[quickest_lib]
        max_books = np.minimum(books_left_in_lib, scan_rate * self.days_to_scan)
        books_to_ship = np.argsort(lib_scores)[-max_books:][::-1]

        self.sol_libs.append(quickest_lib)
        self.sol_books_lists.append(list(books_to_ship.flatten()))
        self.libs_books_matrix[:, books_to_ship] = 0
        self.libs_books_matrix[quickest_lib, :] = 0
        self.num_books_in_lib = np.sum(self.libs_books_matrix, axis=1)

        self.libs_scan_rate[quickest_lib] = np.min(self.libs_scan_rate)
        self.libs_signup_time[quickest_lib] = np.max(self.libs_signup_time)

    def solve_challenge(self):
        while self.days_to_scan > 0 and np.any(self.num_books_in_lib != 0):
            quickest_lib = self.find_quickest_lib()
            self.update_solution(quickest_lib)
            if np.all(self.libs_signup_time == self.libs_signup_time[0]) and np.all(
                    self.libs_scan_rate == self.libs_scan_rate[0]):
                break
        self.output_solutions()


# -*- coding: utf-8 -*-

from servicedetecter import ServiceDetecter
import xml.etree.ElementTree as ET

class XmlServiceDetecter(ServiceDetecter):

    def __init__(self, path):
        super().__init__()
        self.root = ET.parse(path).getroot()
        self.parse()

    def parse(self):
        for host in self.root.iter('host'):
            address = host.find("address")
            for port in host.iter('port'):
                state = port.find('state')
                if (state != None):
                    if (state.attrib.get("state","") == "open"):
                        service = port.find('service')
                        if (service):
                            service_nombre = service.attrib.get("name","")
                            service_product = service.attrib.get("product","")
                            service_version = service.attrib.get("version","")
                            service_extrainfo = service.attrib.get("extrainfo","")
                            if (service_extrainfo != ""):
                                service_extrainfo = "(" + service_extrainfo + ")"
                        else:
                            service_nombre = ""
                            service_product = ""
                            service_version = ""
                            service_extrainfo = ""
                        self.add(address.attrib["addr"], port.attrib["portid"],
                                    port.attrib["protocol"],
                                    "{} {} {} {}".format(service_nombre,
                                    service_product, service_version, service_extrainfo))


# #!usr/bin/env python
# # -*- coding:utf-8 -*-
from django import template
import datetime
from django.utils.safestring import mark_safe
#由于没有计算时间差的过滤器 ，所以我们的自己定义一个，如下
register = template.Library()  # register的名字是固定的,不可改变
@register.filter  #过滤器
def yuanlingtime(createtime):
    #计算时间差：当前的时间减去创建的时间
    now_time = datetime.datetime.now()
    user_create_time = datetime.datetime(year=createtime.year,month=createtime.month,day=createtime.day,hour=createtime.hour,minute=createtime.minute,second=createtime.second)
    ret = now_time-user_create_time
    print("---------",ret)  #5 days, 15:51:47.855688
    print(type(ret))  #<class 'datetime.timedelta'>
    return mark_safe(str(ret)[:-17])  #mark_dafe只是做一个安全机制，和safe过滤器一样，安全之后返回
# print(datetime.datetime(year=2017,month=2,day=5))  #2017-02-05 00:00:00


import time
import configparser
import pandas as pd
import os
import json
import tweepy

#read config
config= configparser.ConfigParser()
config.read("config.ini")
Api_Key=config['twitter_api']['API_Key']
Api_Key_Secret=config['twitter_api']['API_Key_Secret']
Access_Token=config['twitter_api']['Access_Token']
Access_Token_Secret=config['twitter_api']['Access_Token_Secret']

auth=tweepy.OAuthHandler(Api_Key,Api_Key_Secret,Access_Token,Access_Token_Secret)
# auth.set_access_tokens(Access_Token,Access_Token_Secret)

api=tweepy.API(auth,wait_on_rate_limit=True)

num_tweets = 5000

# Define the number of requests to make
num_requests = int(num_tweets / 100)

container = []
ListOfKeywords=['Mario Draghi','Angela Merkel','Scott Morrison','Fumio Kishida','Pedro Sanchez','Emmanuel Macron','Jair Bolsonaro','Volodymyr Zelenskyy']
def func(myquery, myword):
    print("DEBUG:func:: myquery ",myquery)
    # Open the file to save the tweets
    filename = "tweets_" + myword.replace(" ", "-")
    try:
        with open(filename, 'w') as f:
            # Collect tweets using tweepy Cursor
            for i in range(num_requests):
                print("DEBUG:func:: request number ",i+1)
                tweets = tweepy.Cursor(api.search_tweets, q=myquery, tweet_mode="extended").items(100)
                for tweet in tweets:
                    # Write each tweet to the file as a separate line
                    f.write(json.dumps(tweet._json) + '\n')
    except Exception as e:
        print("DEBUG:func:: some Exception", e, "\n")
        print("DEBUG:func:: INCOMPLETE EXECUTION QUERY ", myquery)
        print("DEBUG:****Sleeping for 15 mins****")
        time.sleep(910)

for word in ListOfKeywords:
    query = word + " exclude:retweets" + " exclude:replies"
    print(" ******* DEBUG: query ",query, "******* \n")
    func(query, word)
    print(" ******* DEBUG: query complete",query,"*******  \n")


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 05 15:41:52 2020

@author: fuwei.zhao
"""

import struct
import math
import matplotlib.pyplot as plt
from datetime import datetime
import numpy as np
#import sys
import os
import scipy.interpolate as spi
import binascii
import multiprocessing  #add by FHC 2022/9/9
#import threading  #add by FHC 2022/9/9

global EKMAXPHSP  # 所有粒子的最大动能 add by FHC 2022/9/7
EKMAXPHSP = 0
global EKMINPHSPE  # 所有电子的最小动能 add by FHC 2022/9/7
EKMINPHSPE = 0.5
global SumElectron  # 电子数量 add by FHC 2022/9/9
SumElectron = 0
global NPHOTPHSP  # 光子数量  add by FHC 2022/9/9
NPHOTPHSP = 0
global NPPHSP  # 所有粒子的数量
NPPHSP = 0

def Cut(data, start, width):
    if not isinstance(data, int):
        print("Input is not int.")
        return
    cut_data = data >> start
    cut_data = cut_data % 2**width
    return cut_data

def Int2Binary(data, width):
    BASE=2
    quotient = data
    binstr=['0']*width
    for i in range(width):
        remainder = quotient%BASE
        quotient = quotient>>1
        binstr[width-i-1]=str(remainder)
    binstr=''.join(binstr)
    return binstr

class PhspVector:
    def __init__(self, PID, LATCH, Energy, X, Y, U, V,W_temp, WT, particle_type, ZLast=0,):
        self.LATCH = LATCH
        self.PID = PID  #粒子在相空间文件中的序号
        self.Bremsstrahlung = Cut(self.LATCH, 0, 1)      #粒子轫致辐射的信息
        self.InteractiveRegion = Cut(self.LATCH, 1, 23)  #粒子在相互作用区域的信息
        self.SecondaryParticle = Cut(self.LATCH, 24, 5)  #次级粒子的信息
        self.Charge = Cut(self.LATCH, 29, 2)             #粒子电荷的信息
        self.MultiScore = Cut(self.LATCH, 31, 1)         #多个score平面的信息
        self.ZLast = ZLast                               #粒子最后相互作用的Z坐标
        self.Energy = Energy                             #粒子的能量
        self.EkElectron = 0.5
        self.particle_type = particle_type
        global NPHOTPHSP
        #global SumElectron
        global EKMAXPHSP
        global EKMINPHSPE
        if self.particle_type == b'\x01':  #判断是否为光子
            self.EkAll = abs(Energy)
            NPHOTPHSP += 1
            self.Charge = 0
        elif self.particle_type == b'\x02':  #判断是否为电子
            self.EkAll = abs(Energy)
            self.EkElectron = self.EkAll
            self.Energy = abs(Energy) + 0.511  #IAEA输出的能量为动能
            self.Charge = 2
            #SumElectron += 1
        else:                       #判断是否为正电子
            self.EkAll = abs(Energy)
            self.Energy = abs(Energy) + 0.511   #IAEA输出的能量为动能
            self.Charge = 1
        self.LATCH += self.Charge << 29

        if self.EkAll > EKMAXPHSP:         #更新所有粒子的最大动能
            EKMAXPHSP = self.EkAll
        if self.EkElectron < EKMINPHSPE:   #更新所有电子的最小动能
            EKMINPHSPE = self.EkElectron

        self.X = X
        self.Y = Y
        self.U = U

        if not -1 <= self.U <= 1:
            print('NO.{:8d}:U is wrong:{:8.3f}'.format(self.PID, self.U))
            raise AssertionError
        self.V = V
        if not -1 <= self.V <= 1:
            print('NO.{:8d}:V is wrong:{:8.3f}'.format(self.PID, self.V))
            raise AssertionError
        self.W = math.sqrt(W_temp)
        # if self.particle_type == b'\xff':  #判断W是否要带负号 add by FHC 2022/9/2
        #     self.W = self.W * -1
        self.WT = WT
        if not 0 <= self.WT <= 1:
            print('NO.{:8d}:WT is wrong:{:8.3f}'.format(self.PID, self.WT))
            raise AssertionError
        #self.R = math.sqrt(X ** 2 + Y ** 2)

    def ShowTitle(self):
        print(
            "{:^12} |{:^1}|{:^2}|{:^5}|{:^23}|{:^1}"
            "|{:^8}|{:^8}|{:^8}|{:^8}|{:^8}|{:^8}|{:^10}|".format("ParticleID", "P", "Q", "SecP",
                                                                  "Interactive Region", "B",
                                                                  "X", "Y", "U", "V", "W", "Energy", "Weight"))

    def Show(self):
        print(
            "{:>12} |{:^1}|{:^2}|{:^5}|{:^23}|{:^1}|{:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}|{:>10.3e}|".format(self.PID, Int2Binary(self.MultiScore, 1), Int2Binary(self.Charge, 2), Int2Binary(self.SecondaryParticle, 5), Int2Binary(self.InteractiveRegion, 23), Int2Binary(self.Bremsstrahlung, 1), self.X, self.Y, self.U, self.V, self.W, self.Energy, self.WT))

class Phsp:

    def __init__(self, filename,headername):
        self.FileName = filename  #IAEA相空间文件名
        self.HeaderName = headername  #IAEAheader文件名
        self.NINCPHSP = 0  #History数量
        self.lock = multiprocessing.RLock()
        #读取相空间文件基本信息  add by FHC 2022/9/7
        with open(filename, 'rb') as fid, \
             open(headername, 'r', encoding='utf-8') as fid_header:
            while (self.NINCPHSP == 0):
                lines = fid_header.readline()  #整行读取数
                if ("$ORIG_HISTORIES:") in lines:
                    self.NINCPHSP = int(next(fid_header).replace("\n"," "))     #从IAEAheader读取History 数量
            fid_header.close()

            self.Mode = 'MODE2'   #IAEAphsp文件为默认MODE2
            self.TotalNumParticles = int(os.path.getsize(filename) / 33)
            if self.Mode == 'MODE2':
                self.offset = 33
            elif self.Mode == 'MODE0':
                self.offset = 28
        self.LoopNum = multiprocessing.cpu_count()   #获取进程数量 = 逻辑CPU数量
        self.MAXBuffer = math.ceil(self.TotalNumParticles / self.LoopNum)  #1000000  #单个进程内读取的粒子数量
        self.startTime = datetime.now()
        self.stopTime = datetime.now()
        self.NPHOTPHSP = 0

    def Loop(self, startID=1, stopID=-1, ptype='ope+-'):  #单个进程
        ptypebin = []   #可能存在的粒子种类
        if 'p' in ptype:  #光子
            ptypebin.append(0)
        if 'e+-' in ptype:  #正负电子
            ptypebin.append(2)
            ptypebin.append(1)
        elif 'e-' in ptype:  #电子
            ptypebin.append(2)
        elif 'e+' in ptype:  #正电子
            ptypebin.append(1)
        if 'o' in ptype:  #未知
            ptypebin.append(3)
        if len(ptypebin) == 0:
            print("No matched particle type.")
            return 0

        if stopID == -1:
            stopID = self.MAXBuffer
        phspList = []  #创建单个进程中读取的粒子相空间信息
        IDoffset = (startID - 1) * self.offset  #获取当前相空间文件的初始读取位置
        with open(self.FileName, 'rb') as fid:   #以二进制只读的方式，打开相空间文件
            fid.seek(IDoffset, 0)  #fid.seek(self.offset + IDoffset, 0)    将指针移动到当前进程应该在的初始读取位置
            if startID > self.TotalNumParticles:
                print("Start ID excessed the max particle number.")
                return []
            #typelist = []
            for i in range(min(self.TotalNumParticles - startID + 1, self.MAXBuffer, stopID - startID + 1)):  #对进程内部的粒子进行循环读取
                particle_type = struct.unpack('s', fid.read(1))[0]
                #typelist.append(particle_type)#粒子的type，代表Z方向速度的正负号
                Energy, X, Y, Z, U, V, WT = struct.unpack('7f', fid.read(28))  #粒子的能量、坐标（X, Y, Z）、水平速度（U, V）、权重（WT）
                LATCH = struct.unpack('i', fid.read(4))[0]  #粒子的LATCH信息

                W_temp = (1 - U ** 2 - V ** 2)  #获取Z方向速度的平方值，由于下一步的筛选，以免出现对负值开方的情况
                if particle_type != b'\xff' and abs(W_temp) > 1e-07:   #只选择Z方向速度为正 并且 Z速度平方为正的粒子进行后处理
                    p = PhspVector(i + startID, LATCH, Energy, X, Y, U, V, W_temp, WT, particle_type, Z)   # 对粒子的相空间信息进行后处理
                    # if p.Charge in ptypebin and p.SecondaryParticle in In_SecondaryParticle and p.InteractiveRegion in In_InteractiveRegion:
                    print("Particle Type: ", particle_type)
                    p.Show()
                    #if p.Charge in ptypebin:
                    phspList.append(p)  #添加粒子的相空间信息

                # if i+startID % 100000 == 1 or i+startID == self.TotalNumParticles:
                # self.stopTime = datetime.now()
                #     percent = (i + startID) % self.MAXBuffer / self.MAXBuffer
                #     timedelta = (self.stopTime-self.startTime).seconds
                #     print("Proceeding {:8.2%}, time used:{:4d} seconds, rest time:{:6.1f} seconds".format(percent, timedelta, (1-percent)*timedelta/percent), end='\r', flush=True)
                #     if i+startID == self.TotalNumParticles:
                #         print("\nDone.")
        return phspList

    def Show(self, phspList, pNum=0, printout=True):
        if len(phspList) > 0:
            phspList[0].ShowTitle()
        for i, p in enumerate(phspList):
            if printout:
                p.Show()
        return pNum

    def ProcessLoop(self,MAXBuffer, ptype, i):
        # 并行模块 开始
        #self.lock.acquire()  # 请求并行锁
        self.phspList = self.Loop(i * MAXBuffer + 1, (i + 1) * MAXBuffer, ptype)
        self.phspList = self.TranserEPID(self.phspList)
        self.SavePhspProcess(self.phspList, i)  #将当前进程的粒子相空间信息保存为二进制文件
        #self.lock.release()  # 释放并行锁
        # 并行模块 结束

    def TotalLoop(self, ptype='ope+-'):
        #LoopNum = math.ceil(self.TotalNumParticles / self.MAXBuffer)
        # self.EnergyFluence.reset()
        self.startTime = datetime.now()
        #targetevents=0

        ProcessList = []
        # #并行模块 开始
        # threading.Lock().acquire()  #请求并行锁

        for i in range(self.LoopNum):
            #phspList = self.Loop(i * self.MAXBuffer + 1, (i + 1) * self.MAXBuffer, ptype)
            exec('Process_'+ str(i)+ ' = multiprocessing.Process(target = self.ProcessLoop, args=(self.MAXBuffer, ptype, i))')
            eval('ProcessList.append(Process_'+str(i)+')')

        for t in ProcessList:
            #t.daemon(True)
            t.start()
        t.join()

        self.SavePhspAll()

            #targetevents = targetevents + len(phspList)
            #pNum = self.show(phspList, ptype, pNum, i, printout=False)
        #
        # threading.Lock().release()  #释放并行锁
        # #并行模块 结束
        self.stopTime = datetime.now()
        print("Total time: {:d} seconds".format((self.stopTime-self.startTime).seconds))

    def TranserEPID(self, phspList):
        Z_epid = 45.0
        for i in range(len(phspList)):
            phspList[i].X += (phspList[i].U / phspList[i].W) * (Z_epid - phspList[i].ZLast)
            phspList[i].Y += (phspList[i].V / phspList[i].W) * (Z_epid - phspList[i].ZLast) + 15.0
            phspList[i].ZLast = 0
        return phspList

    def SavePhspAll(self):    #将所有进程的粒子相空间信息进行整合
        self.FileName = self.FileName.split('.')[0] + ".egsphsp1"
        self.headtype = 'iifff'   #相空间文件Header 的数据类型
        #self.datatype = 'iffffff'  #相空间文件粒子的数据类型
        self.Nothing = b'\x00\x00\x00'  #Head信息与粒子信息之间的间隔
        #NPPHSP = len(phspList)  #获取粒子总数量
        global NPPHSP, NPHOTPHSP, EKMAXPHSP, EKMINPHSPE
        with open(self.FileName, 'wb') as fout:   #打开总相空间文件
            fout.write(b'MODE0')   #写入MODE信息
            phspHead = struct.pack(self.headtype, NPPHSP, NPHOTPHSP, EKMAXPHSP, EKMINPHSPE, self.NINCPHSP)
            fout.write(phspHead)  #写入Head信息
            fout.write(self.Nothing)  #写入Head信息与粒子信息之间的间隔
            for i in range(self.LoopNum):
                self.FinName = self.FileName.split('.')[0] + "_" + str(i) + ".egsphsp1"
                with open(self.FinName, 'rb') as fin:  #打开每一个进程的粒子相空间信息
                    singleHead = fin.read(4*4)
                    npphsp, nphotphsp, ekmaxphsp, ekminphspe = struct.unpack('iiff', singleHead)
                    NPPHSP += npphsp
                    NPHOTPHSP += nphotphsp
                    EKMAXPHSP = max(EKMAXPHSP, ekmaxphsp)
                    EKMINPHSPE = min(EKMINPHSPE, ekminphspe)
                    data = fin.read()
                    fout.write(data)
                fin.close()
        fout.close()
        with open(self.FileName, 'rb+') as fout:
            fout.seek(5)
            phspHead = struct.pack(self.headtype, NPPHSP, NPHOTPHSP, EKMAXPHSP, EKMINPHSPE, self.NINCPHSP)
            fout.write(phspHead)  # 写入Head信息
        fout.close()

    def SavePhspProcess(self,phspList, i):   #将当前进程的粒子相空间信息保存为二进制文件
        self.FileName = self.FileName.split('.')[0] + "_" + str(i) + ".egsphsp1"
        self.datatype = 'iffffff'  # 相空间文件粒子的数据类型
        self.headtype = 'iiff'  # 相空间文件Header 的数据类型
        global NPPHSP, NPHOTPHSP, EKMAXPHSP, EKMINPHSPE
        NPPHSP = len(phspList)  # 获取粒子总数量
        with open(self.FileName, 'wb') as fout:     #打开每一个进程的粒子相空间信息
            phspHead = struct.pack(self.headtype, NPPHSP, NPHOTPHSP, EKMAXPHSP, EKMINPHSPE)
            fout.write(phspHead)  # 写入Head信息
            for i in range(len(phspList)):
                self.binData = struct.pack(self.datatype, phspList[i].LATCH, phspList[i].Energy, phspList[i].X, phspList[i].Y, phspList[i].U, phspList[i].V, phspList[i].WT)
                fout.write(self.binData)
        fout.close()

class EnergyMap:
    def __init__(self, XSize = 512, YSize = 512, XMax = 20.48, XMin = -20.48, YMax = 20.48, YMin = -20.48):
        self.XSize = XSize
        self.YSize = YSize
        self.XMax = XMax
        self.XMin = XMin
        self.YMax = YMax
        self.YMin = YMin
        self.XRes = (self.XMax - self.XMin) / self.XSize
        self.YRes = (self.YMax - self.YMin) / self.YSize
        self.Fluence = np.zeros([self.XSize, self.YSize])
        self.EnergyFluence = np.zeros([self.XSize, self.YSize])
        self.Response = np.zeros([self.XSize, self.YSize])
        self.ipo3 = 0
        self.Statistics = 0
        self.EnergyResponse(r"E:\工作文档\资料\EGS\DetectorResponse.txt")

    def Deposite(self, p):
        if not isinstance(p, PhspVector):
            print("Unexpected input type.")
            return
        if self.XMin < p.X < self.XMax and self.YMin < p.Y < self.YMax:
            xbin = math.ceil((p.X - self.XMin) / self.XRes) - 1
            ybin = math.ceil((p.Y - self.YMin) / self.YRes) - 1
            Response = self.InterPolate(p.Energy)
            self.Fluence[xbin, ybin] = self.Fluence[xbin, ybin] + p.WT
            self.EnergyFluence[xbin, ybin] = self.EnergyFluence[xbin, ybin] + p.WT*p.Energy
            self.Response[xbin, ybin] = self.Response[xbin, ybin] + p.WT*Response
            self.Statistics = self.Statistics+p.WT
            # self.Fluence[xbin, ybin] = self.Fluence[xbin, ybin] + 1
            # self.EnergyFluence[xbin, ybin] = self.EnergyFluence[xbin, ybin] + p.Energy
            # self.Response[xbin, ybin] = self.Response[xbin, ybin] + Response
            # self.Statistics = self.Statistics+1

    def EnergyResponse(self, path):
        data = np.loadtxt(path)
        self.ipo3 = spi.splrep(data[:, 0], data[:, 1], k=3)

    def InterPolate(self, x):
        y = spi.splev(x, self.ipo3)
        return y

    def Show(self):
        # plt.figure(figsize=(12, 9))
        plt.matshow(self.Fluence)
        plt.title(''.join(['Fluence Statictics: ', str(self.Statistics)]))
        plt.matshow(self.EnergyFluence)
        plt.title(''.join(['EnergyFluence Statictics: ', str(self.Statistics)]))
        plt.matshow(self.Response)
        plt.title(''.join(['Response Statictics: ', str(self.Statistics)]))

    def Save(self, path):
        with open(path+"_Fluence.dat", 'wb') as fout:
            for i in range(self.XSize):
                for j in range(self.YSize):
                    bytes=struct.pack('f', self.Fluence[i, j])
                    fout.write(bytes)
        with open(path+"_EnergyFluence.dat", 'wb') as eout:
            for i in range(self.XSize):
                for j in range(self.YSize):
                    bytes=struct.pack('f', self.EnergyFluence[i, j])
                    eout.write(bytes)
        with open(path+"_Response.dat", 'wb') as rout:
            for i in range(self.XSize):
                for j in range(self.YSize):
                    bytes=struct.pack('f', self.Response[i, j])
                    rout.write(bytes)

class EnergyFluence:
    def __init__(self, nbin, Rmax, binshape='circle'):
        self.nbin = nbin
        self.Rmax = Rmax
        self.Rstep = 0
        self.x = []
        self.y = []
        self.yfin = np.zeros(self.nbin)
        self.ynum = []
        self.reset()

    def reset(self):
        self.Rstep = self.Rmax / math.sqrt(nbin)
        self.x = [(math.sqrt(i + 1)) * self.Rstep for i in range(self.nbin)]
        self.y = [0 for i in range(self.nbin)]
        self.yfin = np.zeros(self.nbin)
        self.ynum = [0 for i in range(self.nbin)]

    def accumulate(self, x, y):
        if 0 < x < self.Rmax:
            xbin = math.ceil((x/self.Rstep)**2) - 1
            # xbin = math.ceil(x * self.nbin / self.Rmax) - 1
            self.y[xbin] = self.y[xbin] + y
            self.ynum[xbin] = self.ynum[xbin]+1

    def show(self):
        plt.figure(figsize=(12, 9))
        self.yfin = [self.y[i]/(math.pi * self.Rstep**2) for i in range(self.nbin)]
        plt.plot(self.x, self.yfin)
        plt.xlabel("R/cm")
        plt.ylabel("Energy Fluence")
        plt.title("Energy Fluence vs Position")
        plt.show()

if __name__ == '__main__':
        # print("args:")
        # for n, arg in enumerate(sys.argv):
        #     print("NO.{:2d} arg: {:}".format(n, arg))
        # del n, arg
        # if len(sys.argv) == 2 and os.path.isfile(sys.argv[1]):
        #     P = Phsp(sys.argv[1])
        #     P.TotalLoop()
        # elif not os.path.isfile(sys.argv[1]):
        #     print("File doesn't exist.")

        path_phsp = "/home/uih/Head/Source21/IBL_FS27_UpHead_PE2.6/Source21_Phant_FS27.egsphsp1"
        path_header = "/home/uih/Head/Source21/IBL_FS27_UpHead_PE2.6/Source21_Phant_FS27.IAEAheader"
        ph = Phsp(path_phsp, path_header)
        ph.TotalLoop()
        # thread_1 = threading.Thread(target = ph.TotalLoop())
        # thread_2 = threading.Thread(target = ph.TotalLoop())
        #ph.TranserEPID(phspList)
        #ph.SavePhsp(phspList)


from matplotlib import pyplot as plt
import numpy as np

class MatplotlibBar(object):
    """柱状图"""
    def __init__(self):
        super(MatplotlibBar, self).__init__()
        self.n = 12
        self.X = np.arange(self.n)
        self.Y1 = (1 - self.X / float(self.n)) * np.random.uniform(0.5, 1.0, self.n)
        self.Y2 = (1 - self.X / float(self.n)) * np.random.uniform(0.5, 1.0, self.n)

    def generate_figure(self):
        plt.xlim(-.5, self.n)
        plt.ylim(-1.25, 1.25)

        plt.xticks(())
        plt.yticks(())

        # 绘制柱状图
        plt.bar(self.X, +self.Y1, facecolor='#9999ff', edgecolor='white')
        plt.bar(self.X, -self.Y2, facecolor='#ff9999', edgecolor='white')

        # 在柱状图的顶/底部添加对应数字标签
        # ha - horizotal alignment; va - vertical alignment
        for x, y in zip(self.X, self.Y1):
            plt.text(x, y, '%.2f' % y, ha='center', va='bottom')
        for x, y in zip(self.X, self.Y2):
            plt.text(x, -(y), '%.2f' % y, ha='center', va='top')

        plt.show()

if __name__ == "__main__":
    mb = MatplotlibBar()
    mb.generate_figure()


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 29 10:16:49 2018

@author: Akshay Jagadale
"""

class Solution:
    def nextGreaterElement(self, nums1, nums2):
        """
        :type nums1: List[int]
        :type nums2: List[int]
        :rtype: List[int]
        """

        newlist = []
        for i in range(len(nums1)):
            num = nums1[i]
            index = nums2.index(num)
            if index == (len(nums2) - 1) and i == (len(nums1)-1) :
                newlist.append(-1)
                break
            if index == (len(nums2) - 1) and i < (len(nums1)-1) :
                newlist.append(-1)
                continue
            for j in range(index+1,len(nums2)):
                if j == (len(nums2) - 1) and num >= nums2[j]:
                    newlist.append(-1)
                    break

                elif num < nums2[j]:
                    newlist.append(nums2[j])
                    break

                elif  j != (len(nums2)-1) and nums1[i] >= nums2[j]:
                    continue

        return newlist

#        cache, st = {}, []
#        for x in nums2:
#            if len(st) == 0:
#                st.append(x)
#            elif x <= st[-1]:
#                st.append(x)
#            else:
#                while st and st[-1] < x:
#                    cache[st.pop()] = x
#                st.append(x)
#        result = []
#        for x in nums1:
#            if x in cache:
#                result.append(cache[x])
#            else:
#                result.append(-1)
#        return result

myobj = Solution()
res = myobj.nextGreaterElement([2,4],[1,2,3,4])
print(res)

Leet Code Problems/496. Next Greater Element I.py


from typing import Union, List

from ..models import User, Video, Like, PostTag, Post
from django.db import connection

from ..sql.sql import SqlHandlerInterface

class SqlServerHandler(SqlHandlerInterface):
    # Initialize all objects in database
    def init_database(self) -> None:
        with connection.cursor() as cursor:
            # Create Users
            cursor.execute("""
            CREATE TABLE Users (
                UserId CHAR(32) NOT NULL PRIMARY KEY,
                UserName VARCHAR(255) NOT NULL UNIQUE,
                Password VARCHAR(255) NOT NULL,
                Email VARCHAR(255) NOT NULL UNIQUE,
                Created BIGINT NOT NULL,
                LastSeen BIGINT NOT NULL,
                Verified BIT DEFAULT 0
            );
            """)

            # Create Videos
            cursor.execute("""
            CREATE TABLE Videos (
                VideoId CHAR(32) NOT NULL PRIMARY KEY,
                UserId CHAR(32) NOT NULL FOREIGN KEY REFERENCES Users(UserId),
                Preset VARCHAR(255) NOT NULL,
                Created BIGINT NOT NULL,
            );
            """)

            # Create Posts
            cursor.execute("""
            CREATE TABLE Posts (
                PostId CHAR(32) NOT NULL PRIMARY KEY,
                VideoId CHAR(32) NOT NULL FOREIGN KEY REFERENCES Videos(VideoId) ON DELETE CASCADE,
                Title VARCHAR(255) NOT NULL,
                Description TEXT,
                Created BIGINT NOT NULL,
                Likes INT NOT NULL DEFAULT 0,
            );
            """)

            # Create Likes
            cursor.execute("""
            CREATE TABLE Likes (
                UserId CHAR(32) NOT NULL FOREIGN KEY REFERENCES Users(UserId) ON DELETE CASCADE,
                PostId CHAR(32) NOT NULL FOREIGN KEY REFERENCES Posts(PostId) ON DELETE CASCADE,
                Timestamp BIGINT NOT NULL
            );
            """)

            # Create Tags
            cursor.execute("""
            CREATE TABLE Tags (
                Name VARCHAR(225) NOT NULL,
                PostId CHAR(32) NOT NULL FOREIGN KEY REFERENCES Posts(PostId) ON DELETE CASCADE
            );
            """)

            # Create stored procedure for updating likes
            cursor.execute("""
            CREATE PROCEDURE UpdateSinglePostLikesCount @post_id char(32)
            AS
            UPDATE Posts
            SET Posts.Likes = ISNULL((SELECT COUNT(Likes.UserId) FROM Likes WHERE Likes.PostId = @post_id GROUP BY PostId), 0)
            WHERE Posts.PostId = @post_id
            """)

            # Create Trigger for Updating and Inserting values from the Likes table
            cursor.execute("""
            CREATE TRIGGER UpdateSinglePostLikesCountTriggerInsert
                ON Likes
                AFTER UPDATE, INSERT
            AS
                DECLARE @post_id CHAR(32)
                SELECT @post_id = inserted.PostId FROM inserted
                EXEC [dbo].[UpdateSinglePostLikesCount] @post_id;
            """)

            # Create Trigger for Deleting values from the Likes table
            cursor.execute("""
            CREATE TRIGGER UpdateSinglePostLikesCountTriggerDelete
                ON Likes
                AFTER DELETE
            AS
                DECLARE @post_id CHAR(32)
                SELECT @post_id = deleted.PostId FROM deleted
                EXEC [dbo].[UpdateSinglePostLikesCount] @post_id;
            """)

    # === USERS ===

    # Inserts the user into the database
    def insert_user(self, user: User) -> None:
        with connection.cursor() as cursor:
            query = "INSERT INTO Users VALUES (" \
                    f"'{user.user_id}', " \
                    f"'{user.user_name}', " \
                    f"'{user.password}', " \
                    f"'{user.email}', " \
                    f"'{int(user.created)}', " \
                    f"'{int(user.last_online)}', " \
                    "0)"
            cursor.execute(query)

    # Delete user information from database
    def delete_user(self, user_id: str) -> None:
        with connection.cursor() as cursor:
            query = f"DELETE FROM Users WHERE UserId = '{user_id}'"
            cursor.execute(query)

    # Get a User object from data from the database using the user_id
    def get_user(self, user_id: str) -> Union[User, None]:
        with connection.cursor() as cursor:
            query = f"SELECT * FROM Users WHERE UserId = '{user_id}'"
            cursor.execute(query)
            row = cursor.fetchone()
        if row:
            return User(row[1], row[2], row[3], row[0], row[4], row[5], row[6])
        return None

    # Gets a User object from data from the database using either the username or email
    def get_user_by_credential(self, username_or_email: str) -> Union[User, None]:
        with connection.cursor() as cursor:
            query = f"SELECT * FROM Users WHERE UserName = '{username_or_email}' OR Email = '{username_or_email}'"
            cursor.execute(query)
            row = cursor.fetchone()
        if row:
            return User(row[1], row[2], row[3], row[0], row[4], row[5], row[6])
        return None

    # === VIDEOS AND TAGS ===

    # Inserts video information to relational database. This does not do anything with the actual video.
    def insert_video(self, video: Video) -> None:
        with connection.cursor() as cursor:
            query = f"INSERT INTO Videos VALUES ('{video.video_id}', '{video.user_id}', '{video.preset}', '{video.created}')"
            cursor.execute(query)

    # Deletes video information from relational database. This does not do anything with the actual video.
    def delete_video(self, video_id: str) -> None:
        with connection.cursor() as cursor:
            query = f"DELETE FROM Videos WHERE VideoId = '{video_id}'"
            cursor.execute(query)

    # Gets video information if it exists. If it doesn't exist, return None
    def get_video(self, video_id: str) -> Union[Video, None]:
        with connection.cursor() as cursor:
            query = f"SELECT * FROM Videos WHERE VideoId = '{video_id}'"
            cursor.execute(query)
            row = cursor.fetchone()
        if row:
            return Video(row[1], row[0], row[2], row[3])
        return None

    def get_videos_by_user_id(self, user_id: str) -> List[Video]:
        with connection.cursor() as cursor:
            query = f"SELECT * FROM Videos WHERE UserId = '{user_id}'"
            cursor.execute(query)
            rows = cursor.fetchall()
        return [Video(row[1], row[0], row[2], row[3]) for row in rows]

    # === POSTS, TAGS, AND LIKES ===

    # Inserts a post to database, assuming the corresponding video information has already been inserted
    def insert_post(self, post: Post) -> None:
        with connection.cursor() as cursor:
            query = "INSERT INTO Posts VALUES (" \
                    f"'{post.post_id}', " \
                    f"'{post.video_id}', " \
                    f"'{self.escape_apostrophes(post.title)}', " \
                    f"'{self.escape_apostrophes(post.description)}', " \
                    f"'{post.created}', " \
                    f"0) "
            cursor.execute(query)

    # Deletes a post from database
    def delete_post(self, post_id: str) -> None:
        with connection.cursor() as cursor:
            query = f"DELETE FROM Posts WHERE PostId = '{post_id}'"
            cursor.execute(query)

    # Updates a post
    def update_post(self, post_id: str, title: str, description: str) -> None:
        with connection.cursor() as cursor:
            query = "UPDATE Posts SET " \
                    f"Title='{self.escape_apostrophes(title)}', Description='{self.escape_apostrophes(description)}' " \
                    f"WHERE PostId='{post_id}'"
            cursor.execute(query)

    # Gets a post from database
    def get_post(self, post_id: str) -> Union[Post, None]:
        with connection.cursor() as cursor:
            query = f"SELECT * FROM Posts WHERE PostId = '{post_id}'"
            cursor.execute(query)
            row = cursor.fetchone()
        if row:
            return Post(row[1], row[2], row[3], row[0], row[4], row[5])
        return None

    # Get all posts owned by a user
    def get_post_ids_by_user_id(self, user_id: str) -> list:
        with connection.cursor() as cursor:
            query = f"SELECT Posts.PostId FROM Posts INNER JOIN Videos ON Posts.VideoId = Videos.VideoId WHERE Videos.UserId = '{user_id}'"
            cursor.execute(query)
            rows = cursor.fetchall()
        return [row[0] for row in rows]

    # Gets all PostId's, ordered most recent first
    def get_all_post_ids(self) -> list:
        # TODO trim this function to only include a subset of all posts somehow
        with connection.cursor() as cursor:
            query = "SELECT PostId FROM Posts ORDER BY Created DESC"
            cursor.execute(query)
            postsdb = cursor.fetchall()

        if postsdb:
            postids = []
            for pdb in postsdb:
                postids.append(pdb[0])

            return postids

        return []

    # Adds a tag to a video. Forces tag to be lowercase
    def add_tag(self, post_tag: PostTag) -> None:
        with connection.cursor() as cursor:
            query = f"INSERT INTO Tags VALUES ('{post_tag.tag_name.lower()}', '{post_tag.post_id}')"
            cursor.execute(query)

    # Removes a tag from a video.
    def remove_tag(self, post_tag: PostTag) -> None:
        with connection.cursor() as cursor:
            query = f"DELETE FROM Tags WHERE Name = '{post_tag.tag_name.lower()}' AND PostId = '{post_tag.post_id}'"
            cursor.execute(query)

    # Returns whether a video has a video has a like on it
    def has_liked(self, user_id: str, post_id: str) -> bool:
        with connection.cursor() as cursor:
            query = f"SELECT * FROM Likes WHERE UserId = '{user_id}' AND PostId = '{post_id}'"
            cursor.execute(query)
            row = cursor.fetchone()
        return bool(row)

    # Toggles a like on a video, returns new like status
    def toggle_like(self, user_id: str, post_id: str) -> bool:
        print(user_id)
        print(post_id)
        if self.has_liked(user_id, post_id):
            query = f"DELETE FROM Likes WHERE UserId = '{user_id}' AND PostId = '{post_id}'"
            now_liked = False
        else:
            like = Like(user_id, post_id)
            query = f"INSERT INTO Likes VALUES ('{like.user_id}', '{like.post_id}', '{like.timestamp}')"
            now_liked = True
        with connection.cursor() as cursor:
            cursor.execute(query)
        return now_liked

    # Return number of likes on a video
    def likes_count(self, post_id: str) -> int:
        with connection.cursor() as cursor:
            query = f"SELECT Likes FROM Posts WHERE PostId = '{post_id}'"
            cursor.execute(query)
            row = cursor.fetchone()
        return int(row[0])

    # Get admin user, which is also the user which owns all
    def get_admin_user(self) -> User:
        with connection.cursor() as cursor:
            query = f"SELECT * FROM Users WHERE UserId = '{'0' * 32}'"
            cursor.execute(query)
            row = cursor.fetchone()
        return User(row[1], row[2], row[3], row[0], row[4], row[5], row[6])

    def escape_apostrophes(self, s: str) -> str:
        return s.replace('\'', '\'\'')

    # AGGREGATES AND SEQUENCES

    def get_users_post_count(self) -> list:
        with connection.cursor() as cursor:
            query = """
    SELECT Videos.UserID, COUNT(PostID) as NumPosts
    FROM Videos JOIN Posts ON Posts.VideoID = Videos.VideoID
    GROUP BY Videos.UserId
    ORDER BY NumPosts DESC
            """
            cursor.execute(query)
            rows = cursor.fetchall()
        return rows

    def tear_down_database(self) -> None:
        if input("""
    Are you sure you want to reset the entire database?
    This will delete all data and you will have to run init_database to reinitialize.
    Type "yes" to continue, or something else to exit.
        """).lower() == 'yes':
            with connection.cursor() as cursor:
                cursor.execute("""
    DROP TRIGGER UpdateSinglePostLikesCountTriggerDelete;
    DROP TRIGGER UpdateSinglePostLikesCountTrigger;
    DROP PROCEDURE UpdateSinglePostLikesCount;
    DROP TABLE Tags;
    DROP TABLE Likes;
    DROP TABLE Posts;
    DROP TABLE Videos;
    DROP TABLE Users;
            """)
                print('Database torn down')


import re
from typing import Iterable, Generator
import nltk.corpus
import unicodedata
import contractions
import nltk
nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()

USEFUL_NLTK_STOPWORDS = set(['nor',
                             'no',
                             'not',
                             'nothing',
                             'neither',
                             'never',
                             'none',
                             'up',
                             'down',
                             'latency',
                             'slot',
                             'standby',
                             'left',
                             'right',
                             'ise'])
NLTK_STOPWORDS = set(list(nltk.corpus.stopwords.words('english'))) - USEFUL_NLTK_STOPWORDS

def remove_urls(text: str) -> str:
    return re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*', '', text, flags=re.MULTILINE)

def remove_emails(text: str) -> str:
    return re.sub(r'\S*@\S*\s?', ' ', text)

def remove_multi_whitespaces(text: str) -> str:
    # Remove extra whitespace
    text = ' '.join(text.split())
    return text

def remove_times_and_dates(text: str) -> str:
    # Remove time formats Ex:  05:09:09
    text = re.sub(r'\s\d{1,2}:\d{1,2}:\d{1,2}', ' ', text)

    # Remove date formats Ex:  05/09 ,   05/09/2014 05-09-2014 05-Jan-2014
    text = re.sub(r'\s\d{1,4}/\d{1,4}/\d{1,4}', ' ', text)
    text = re.sub(r'\s\d{1,4}-\d{1,4}-\d{1,4}', ' ', text)
    text = re.sub(r'\s\d{1,2}/\d{1,2}', ' ', text)
    # text = re.sub(r'\d{1,4}-\w{1,4}-\d{1,4}', ' ', text)
    # Remove time formats Ex:  05:09
    text = re.sub(r'\s\d{1,2}:\d{1,2}', ' ', text)

    # Remove date formats Ex:  05-09
    text = re.sub(r'\s\d{1,2}-\d{1,2}', ' ', text)
    return text

def remove_punctuation(text: str) -> str:
    # Remove punctuation
    text = re.sub(r'[^\w\s]', '', text)
    return text

def remove_repetition(text: str) -> str:
    # Remove multiple instance of the repeating characters -=/_ Ex: Replace observation==== with observation
    regex = re.compile(r'(\=\=+)|(\/\/+)|(\_\_+)|(\-\-+)')
    text = re.sub(regex, '', text)

    # Remove repeating words fan fan tay = fan tray
    text = re.sub(r'\b(\w+)( \1\b)+', r'\1', text)
    return text

def customize_wf_anonymization(text: str) -> str:
    text = re.sub('wells fargo', 'wells_fargo', text)

    text = re.sub(r'x{1,9}\/x{1,9}\/x{1,9}', 'xxxx', text)
    # text = re.sub(r'xx/xx/xx', 'xx', text)

    # text = re.sub(r'xx/xx/xxxx', 'xx', text)
    text = re.sub(r'x{1,9}\/x{1,9}\/\d{1,4}', 'xxxx', text)
    text = re.sub(r' \{\$\d{1,9}.\d{1,9}\}', ' xxxx', text)

    text = re.sub(r'\d{1,4}', 'xxxx', text)
    return text

def preprocess_text(text: str) -> str:
    """
    Normalizes the text, the order of the operations is important
    """

    text = text.lower()

    text = remove_urls(text)
    text = remove_emails(text)
    text = remove_times_and_dates(text)

    text = customize_wf_anonymization(text)

    text = remove_punctuation(text)
    text = remove_multi_whitespaces(text)

    text = remove_repetition(text)

    # Remove special characters except /_%-
    # text = re.sub('[-]','', text)
    text = re.sub('[^a-zA-Z0-9?/_%\n-]', ' ', text)

    text = '' if text == 'none' else text.strip()

    return text

def expand_contractions(text: str) -> str:
    return contractions.fix(text)

def remove_accented_chars(text: str) -> str:
    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')
    return text

def preprocess_text_advanced(text: str, stem: bool = True) -> str:

    # remove extra newlines (often might be present in really noisy text)
    text = text.translate(text.maketrans("\n\t\r", "   "))

    text = remove_accented_chars(text)
    text = expand_contractions(text)

    text = ' '.join([word for word in text.split() if word not in USEFUL_NLTK_STOPWORDS and word != ''])
    if stem:
        # document = ' '.join([stemmer.stem(word) for word in document.split()])
        text = ' '.join([lemmatizer.lemmatize(word) for word in text.split()])

    # remove extra whitespace
    text = re.sub(' +', ' ', text)
    text = text.strip()

    return text

def preprocess_texts(texts: Iterable[str]) -> Generator[str, None, None]:
    for text in texts:
        text = preprocess_text(text)
        text = preprocess_text_advanced(text)
        yield text


#!/usr/bin/env python3

"""
MESH DESIGN FRAMEWORK

   >=>       >==>
 >=>  >=>  >=>
>=>   >=> >=>
 >=>  >=>  >=>
     >=>     >==>
  >=>
"""

from __future__ import annotations

import numpy as np
import os
import pickle

def point_boundary_intersection(
    point_of_interest: Coordinates,
    first_boundary_point: Coordinates,
    second_boundary_point: Coordinates,
) -> bool:
    """
    Checks if the horizontal line starting at the given point
    intersects the boundary
    """

    P = point_of_interest

    if first_boundary_point.y == second_boundary_point.y:
        A = first_boundary_point
        B = second_boundary_point
        if P.y == A.y and min(A.x, B.x) < P.x < max(A.x, B.x):
            return True
        else:
            return False

    elif first_boundary_point.y > second_boundary_point.y:
        B = first_boundary_point
        A = second_boundary_point
    else:
        A = first_boundary_point
        B = second_boundary_point

    if P.y == A.y or P.y == B.y:
        P = Coordinates(P.x, P.y + 1e-8)

    if P.y < A.y or P.y > B.y:
        return False
    elif P.x >= max(A.x, B.x):
        return False
    else:
        if P.x < min(A.x, B.x):
            return True
        else:
            if A.x != B.x:
                angle_XAB = (B.y - A.y) / (B.x - A.x)
            else:
                angle_XAB = 1e10
            if A.x != P.x:
                angle_XAP = (P.y - A.y) / (P.x - A.x)
            else:
                angle_XAP = 1e10

            if angle_XAP >= angle_XAB:
                return True
            else:
                return False

from my_typing import (
    Material,
    Coordinates,
    Boundary,
    Force,
    NodeDisplacement,
    MeshDimensions,
    MeshSupportDefinitions,
)

class Mesh:
    """Mesh object creation framework"""

    def __init__(self, material: Material, max_finite_element_size: float):
        """Initialize all of the used mesh attributes"""

        # (E, Poisson)
        self.material = material

        self.minimal_beam_width: float
        self.beam_height: float

        self.mechanism_volume: float

        self.max_element_size = max_finite_element_size

        # Node array init
        self.node_array = np.empty((0, 2), dtype=np.float64)
        self.non_removable_nodes = np.empty(0, dtype=np.int32)
        self.main_nodes = np.empty(0, dtype=np.int32)

        # Boundary init
        self.boundary_array = np.empty((0, 4), dtype=np.int32)

        # Beam array init
        self.beam_array = np.empty((0, 3), dtype=np.int32)
        self.beam_mid_nodes_array = np.empty((0), dtype=np.int32)

        self.beam_width_array = np.empty(shape=(0), dtype=np.float64)
        self.beam_width_beginning_map = np.empty(shape=(0), dtype=bool)

        # Start and end conditions init
        self.initial_displacement_array = np.empty((0, 3), dtype=np.float64)

        self.force_array = np.empty((0, 3), dtype=np.float64)

        self.final_displacement_array = np.empty((0, 3), dtype=np.float64)

    def _set_unremovable(self, node):
        """Sets node as non removable"""

        self.non_removable_nodes = np.append(self.non_removable_nodes, node)
        self.non_removable_nodes = np.unique(self.non_removable_nodes)

    def create_node(
        self, coordinates: Coordinates, main_node: bool = False, removable: bool = True
    ):
        """
        Creates a node.
        set if removable or main
        """

        current_node_index = int(np.shape(self.node_array)[0])

        self.node_array = np.append(
            self.node_array,
            np.array([[coordinates.x, coordinates.y]]),
            axis=0,
        )

        if not removable:
            self._set_unremovable(current_node_index)

        if main_node:
            self.main_nodes = np.append(self.main_nodes, current_node_index)

    def _near_node_index(self, node_def):
        """Near node coordinates to node index"""

        if type(node_def) == tuple:
            used_array = self.node_array

            closest_node_index = np.argmin(
                np.sqrt(
                    np.sum(
                        np.square(
                            used_array
                            - np.repeat(
                                np.array(node_def).reshape((1, 2)),
                                np.shape(used_array)[0],
                                axis=0,
                            )
                        ),
                        axis=1,
                    )
                ),
                axis=0,
            )

            return closest_node_index

        else:
            return node_def

    def node_laso(self, poly_points: list, only_main_nodes: bool = True) -> np.ndarray:
        """
        Collects points that are inside of a given polygon

        The polygon is constructed of consecutive points
        with the last given point connecting to the first given point.

        This function uses the "ray-casting-method"
        to determine if a node is inside of the given polygon.
        """

        polygon_array = np.append(
            np.array(poly_points), np.array(poly_points)[0].reshape(1, 2), axis=0
        )
        if only_main_nodes:
            used_array = self.node_array[self.main_nodes]
            contained_node_id = self.main_nodes
        else:
            used_array = self.node_array
            contained_node_id = np.arange(np.shape(used_array)[0])

        counter_array = []
        for point in used_array:
            counter = 0
            for first, second in zip(polygon_array[:-1], polygon_array[1:]):
                counter += point_boundary_intersection(
                    Coordinates(point[0], point[1]),
                    Coordinates(first[0], first[1]),
                    Coordinates(second[0], second[1]),
                )
            counter_array.append(counter)

        return contained_node_id[[result % 2 != 0 for result in counter_array]]

    def create_beam(self, first_node, last_node):
        """Creates a beam"""

        f_node = self._near_node_index(first_node)
        l_node = self._near_node_index(last_node)

        beam_size = np.sqrt(
            np.sum((self.node_array[f_node] - self.node_array[l_node]) ** 2)
        )

        no_nodes_per_beam = int(beam_size / self.max_element_size)

        self.beam_array = np.append(
            self.beam_array,
            np.array([[f_node, l_node, no_nodes_per_beam * 2 - 1]]),
            axis=0,
        )

        for new_node in np.linspace(
            self.node_array[first_node],
            self.node_array[last_node],
            no_nodes_per_beam * 2,
            False,
        )[1:]:

            self.create_node(Coordinates(new_node[0], new_node[1]))

            current_node_index = int(np.shape(self.node_array)[0] - 1)

            self.beam_mid_nodes_array = np.append(
                self.beam_mid_nodes_array, current_node_index
            )

    def _fetch_beam_nodes(self, beam_to_fetch: int) -> np.ndarray:
        """
        Fetches beam midnodes and outputs them
        together with the first and last node
        """

        out_nodes = self.beam_array[beam_to_fetch][:-1]
        mid_nodes_until_current = np.sum(self.beam_array[:beam_to_fetch, -1])

        beam_mid_nodes = self.beam_mid_nodes_array[
            mid_nodes_until_current : mid_nodes_until_current
            + self.beam_array[beam_to_fetch, -1]
        ]

        return np.insert(out_nodes, 1, beam_mid_nodes)

    def beam_laso(self, polygon_points: list, only_main_nodes: bool=False) -> np.ndarray:
        """
        Ouputs a list of beam ids
        """

        cought_nodes = self.node_laso(poly_points=polygon_points, only_main_nodes=only_main_nodes)
        cought_beams = np.empty(shape=(0), dtype=int)

        if only_main_nodes:
            for beam in range(self.beam_array.shape[0]):
                nodes_in_beam = self._fetch_beam_nodes(beam)
                if all(np.in1d(nodes_in_beam[[0, -1]], cought_nodes)):
                    cought_beams = np.append(cought_beams, beam)
        else:
            for beam in range(self.beam_array.shape[0]):
                nodes_in_beam = self._fetch_beam_nodes(beam)
                if any(np.in1d(nodes_in_beam, cought_nodes)):
                    cought_beams = np.append(cought_beams, beam)

        return cought_beams

    def create_boundary(self, node, bd_type: Boundary, set_unremovable: bool = False):
        """
        Creates a boundary
        x_trans = (1, 0 ,0)
        y_trans = (0, 1 ,0)
        z_rotat = (0, 0 ,1)
        """

        node_idx = self._near_node_index(node)

        self.boundary_array = np.append(
            self.boundary_array,
            np.array(
                [
                    [
                        node_idx,
                        bd_type.x_translation,
                        bd_type.y_translation,
                        bd_type.z_rotation,
                    ]
                ]
            ),
            axis=0,
        )

        if set_unremovable and node_idx not in self.non_removable_nodes:
            self._set_unremovable(node_idx)

    def create_force(self, node, force_vector: Force):
        """Creates a force"""

        node_idx = self._near_node_index(node)

        self.force_array = np.append(
            self.force_array,
            np.array([[node_idx, force_vector.x_component, force_vector.y_component]]),
            axis=0,
        )

        self._set_unremovable(node_idx)

    def create_initial_displacement(self, node, displacement_vector: NodeDisplacement):
        "Creates initial displacement definition"

        node_idx = self._near_node_index(node)

        self.initial_displacement_array = np.append(
            self.initial_displacement_array,
            np.array(
                [
                    [
                        node_idx,
                        displacement_vector.x_component,
                        displacement_vector.y_component,
                    ]
                ]
            ),
            axis=0,
        )

        self._set_unremovable(node_idx)

    def set_final_displacement(self, node, displacement_vector: NodeDisplacement):
        "Sets wanted displacement"

        node_idx = self._near_node_index(node)

        self.final_displacement_array = np.append(
            self.final_displacement_array,
            np.array(
                [
                    [
                        node_idx,
                        displacement_vector.x_component,
                        displacement_vector.y_component,
                    ]
                ]
            ),
            axis=0,
        )

        self._set_unremovable(node_idx)

    def calculate_mechanism_volume(self) -> float:
        """Calculates mechanism volume of used mesh beams"""

        length_array = np.empty((0), dtype=np.float64)

        for beam in self.beam_array:
            dx, dy = self.node_array[beam[0]] - self.node_array[beam[1]]
            length = np.sqrt(dx**2 + dy**2)
            length_array = np.append(length_array, length)
        mechanism_area = np.sum(length_array * self.beam_width_array)
        self.mechanism_volume = float(mechanism_area * self.beam_height)

        return self.mechanism_volume

    def remove_beams(self, polygon_points: list) -> None:
        removed_beams = self.beam_laso(polygon_points)
        if len(self.beam_width_beginning_map) == 0:
            self.beam_width_beginning_map = np.full(self.beam_array.shape[0], True)
        self.beam_width_beginning_map[removed_beams] = False

    def set_widths_array_with_interface(
            self,
            input_width: float | np.ndarray,
            interface_polygon_points: list,
            interface_width: float
    ):
        interface_beams = self.beam_laso(interface_polygon_points)
        self.beam_width_beginning_map[interface_beams] = False
        self.set_width_array(input_width)

        self.beam_width_beginning_map[interface_beams] = True

        self.beam_width_array[interface_beams] = interface_width

        self.calculate_mechanism_volume()

        return True  # Width assign terminated successfully

    def set_width_array(self, input_width: float | np.ndarray):
        """
        Sets mesh beam widths
        """

        if len(self.beam_width_beginning_map) == 0:
            self.beam_width_beginning_map = np.full(self.beam_array.shape[0], True)

        if self.beam_width_array.size == 0:
            self.beam_width_array = np.zeros_like(
                self.beam_width_beginning_map, dtype=float
            )

        out_width = np.empty((0), dtype=float)

        if type(input_width) == float:
            input_width_size = len(self.beam_width_beginning_map)
            out_width = np.full(input_width_size, input_width)
            out_width[[not state for state in self.beam_width_beginning_map]] = 0.0
        elif type(input_width) == np.ndarray:
            out_width = np.zeros_like(self.beam_width_beginning_map, dtype=float)
            out_width[self.beam_width_beginning_map] = input_width

        remove_beams_idx = np.arange(self.beam_width_array.size)[
            out_width < self.minimal_beam_width
        ]
        non_zero_beams_idx = np.arange(self.beam_width_array.size)[
            out_width >= self.minimal_beam_width
        ]

        remove_nodes = np.empty((0), dtype=np.int32)
        left_nodes = np.empty((0), dtype=np.int32)

        for beam in remove_beams_idx:
            remove_nodes = np.append(remove_nodes, self._fetch_beam_nodes(beam))
        for beam in non_zero_beams_idx:
            left_nodes = np.append(left_nodes, self._fetch_beam_nodes(beam))
        remove_nodes = np.unique(remove_nodes)
        left_nodes = np.unique(left_nodes)

        if (
            np.size(np.intersect1d(left_nodes, self.non_removable_nodes))
            != np.unique(self.non_removable_nodes).size
        ):
            return False  # Width assign terminated unsucessfully

        out_width[out_width < self.minimal_beam_width] = 0.0
        self.beam_width_array = out_width

        self.calculate_mechanism_volume()

        return True  # Width assign terminated successfully

    def write_beginning_state(self):

        """Writes beginning state of the construction"""

        with open(os.path.join(os.getcwd(), "mesh_setup.pkl"), "wb") as case_setup:
            pickle.dump(self, case_setup, pickle.HIGHEST_PROTOCOL)

def read_mesh_state(pickled_mesh_location: str | None) -> Mesh:
    """Reads the beginning mesh state from a .pkl file"""
    if pickled_mesh_location is None:
        with open(os.path.join(os.getcwd(), "mesh_setup.pkl"), "rb") as case_setup:
            return pickle.load(case_setup)
    else:
        with open(os.path.join(os.getcwd(), pickled_mesh_location), "rb") as case_setup:
            return pickle.load(case_setup)

def simple_mesh_creator(
    material: Material,
    max_el_size: float,
    dimensions: MeshDimensions,
    support_definition: MeshSupportDefinitions = None,
) -> Mesh:
    """Returns a rectangular mesh with optional diagonal supports"""

    mesh = Mesh(material, max_el_size)

    for vert_coord in np.linspace(
        0, dimensions.heigth, dimensions.heigth_divisions + 1, endpoint=True
    ):
        for hor_coord in np.linspace(
            0, dimensions.length, dimensions.length_divisions + 1, endpoint=True
        ):
            mesh.create_node(Coordinates(hor_coord, vert_coord), main_node=True)

    for y_node in range(dimensions.heigth_divisions + 1):
        for x_node in range(dimensions.length_divisions + 1):
            current_node_id = x_node + y_node * (dimensions.length_divisions + 1)

            if x_node < dimensions.length_divisions:
                mesh.create_beam(current_node_id, current_node_id + 1)
            if y_node < dimensions.heigth_divisions:
                mesh.create_beam(
                    current_node_id, current_node_id + (dimensions.length_divisions + 1)
                )

            if (
                support_definition == "fd"
                and y_node < dimensions.heigth_divisions
                and x_node < dimensions.length_divisions
            ):
                mesh.create_beam(
                    current_node_id,
                    current_node_id + 1 + (dimensions.length_divisions + 1),
                )

            if (
                support_definition == "bd"
                and y_node < dimensions.heigth_divisions
                and x_node > 0
            ):
                mesh.create_beam(
                    current_node_id,
                    current_node_id - 1 + (dimensions.length_divisions + 1),
                )

            if (
                support_definition == "x"
                and y_node < dimensions.heigth_divisions
                and x_node < dimensions.length_divisions
            ):
                node_x, node_y = np.average(
                        mesh.node_array[
                            np.array(
                                [
                                    current_node_id,
                                    current_node_id
                                    + 1
                                    + (dimensions.length_divisions + 1),
                                ]
                            ),
                            :,
                        ],
                        axis=0,
                )
                mesh.create_node(
                    Coordinates(node_x, node_y),
                    main_node=True,
                )

                created_mid_node_index = np.shape(mesh.node_array)[0] - 1

                mesh.create_beam(current_node_id, created_mid_node_index)
                mesh.create_beam(
                    created_mid_node_index,
                    current_node_id + 1 + (dimensions.length_divisions + 1),
                )
                mesh.create_beam(
                    current_node_id + (dimensions.length_divisions + 1),
                    created_mid_node_index,
                )
                mesh.create_beam(created_mid_node_index, current_node_id + 1)

    return mesh


prompt = "Welcome to Gherkins "
prompt +="my name is Henneritta."
print(prompt)
table_size = input("How many people in your party? ")

table_size = int(table_size)
if table_size < 8:
    print("We have a table right here.")
else:
    print("Please wait while I find you a table.")

number = table_size % 10

if number in range(0,5):
    print("I am happy to here this.")
elif number in range(5,7):
    print("this worrying.")
else:
    print("I can't deal with this!")


#!/usr/bin/env python
# coding: utf-8

# In[8]:

from selenium import webdriver

import numpy as np
import time
import requests
import cv2
import pandas as pd
from keras.models import load_model
from requests import get
import wave
import time
import librosa
import time
import matplotlib.pyplot as plt
class visual_test:
    def __init__(self):
        self.model=load_model("voice.h5")
        self.number_to_word={0: 'T', 1: '7', 2: 'Z', 3: 'M', 4: 'Y', 5: 'C', 6: 'R', 7: 'H', 8: '9', 9: '3',
                            10: 'Q', 11: 'F', 12: '4', 13: 'N', 14: '5', 15: 'A', 16: 'P', 17: '2', 18: 'K',19:'V',20:'G',21:'6',22:'D',23:'W'}
        self.url_one='https://irs.thsrc.com.tw/IMINT/?locale=tw'
        self.headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36"}
        self.threshold=0.045
        self.window=500
        self.n_fft=256
    def get_voice(self):
        driver=webdriver.Chrome()
        driver.maximize_window()
        driver.get(self.url_one)
        cookies=driver.get_cookies()
        sound_url=driver.find_element_by_xpath("//*[@id='action']/tbody/tr/td/a[2]").get_attribute("href")
        img_url=driver.find_element_by_xpath("//*[@id='BookingS1Form_homeCaptcha_passCode']").get_attribute("src")
        driver.find_element_by_xpath("/html/body/div[3]/div[2]/div/div/div[2]/table/tbody/tr/td/input").click()
        driver.close()
        request = requests.Session()
        request.headers.update(self.headers)
        for cookie in cookies:
            request.cookies.set(cookie['name'], cookie['value'])
        response_sound=request.get(sound_url)
        with open("current_voice.wav", 'wb') as f:
            f.write(response_sound.content)
        response_img=request.get(img_url)
        with open("current_voice.jpg", 'wb') as f:
            f.write(response_img.content)
    def split_voice(self,voice):
        for j in range(0,len(voice)):
            if abs(voice[j])<self.threshold:
                voice[j]=0
        data=[]
        done=True
        for j in range(0,len(voice)):
            avg=np.array(voice[j:j+self.window]).sum()
            if voice[j]!=0 and done:
                start=j
                done=False

            if not(done) and avg==0 and j-start>2500:
                done=True
                data.append(voice[start:j+1])
            if len(data)==4:
                break
        if done==False:
            data.append(voice[start:])
        return data

    def predict_code(self):
        f = wave.open('current_voice.wav','rb')
        params = f.getparams()
        nchannels, sampwidth, framerate, nframes = params[:4]
        strData = f.readframes(nframes)
        waveData = np.fromstring(strData,dtype=np.int16)
        waveData = waveData*1.0/(max(abs(waveData)))
        waveData=self.split_voice(waveData)
        inputdata=[]
        for j in range(0,len(waveData)):
            stft = np.abs(librosa.stft(waveData[j],n_fft=self.n_fft))
            stft=np.mean(stft,axis=1)
            inputdata.append(stft)
        inputdata=np.array(inputdata)
        shapes=inputdata.shape
        inputdata=inputdata.reshape((shapes[0],shapes[1],1))
        predict=self.model.predict(inputdata)
        answer=""
        for i in range(0,len(predict)):
            label=self.number_to_word[np.argmax(predict[i])]
            answer+=label
        img = plt.imread('current_voice.jpg')
        plt.imshow(img)
        plt.title(answer)
        plt.axis('off')
        plt.show()

V=visual_test()
for i in range(0,10):
    V.get_voice()
    V.predict_code()

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[39]:

# In[ ]: