from datetime import datetime
try:
    import ujson as json
except ImportError:
    import json

import logging
import pdb

try:
    # Test for mypy support (requires Python 3)
    from typing import List, Text
except:
    pass

class Encoder(object):
    """
    An encoder for the Collectd JSON format
    See https://collectd.org/wiki/index.php/JSON

    Sample measurements:
    """

    def encode(self, msg):
        # type: (bytes) -> List[Text]
        measurements = []

        for line in msg.decode().split("\n"):
            try:
                # Set flag for float precision to get the same
                # results for Python 2 and 3.
                json_object = self.parse_line(line)

            except ValueError as e:
                logging.debug("Error in encoder: %s", e)
                continue

            measurement = None
            tags = None
            value = None
            time = None

            try:
                time = time or Encoder.format_time(json_object)
                measurement = measurement or Encoder.format_measurement_name(json_object, ['name'])
                tags = tags or Encoder.format_tags(json_object, ['labels'])
                value = value or Encoder.format_value(json_object)

                if measurement and tags and value and time:
                   measurements.append(Encoder.compose_data(measurement, tags, value, time))

            except Exception as e:
                logging.debug("Error in input data: %s. Skipping.", e)
                continue

        return measurements

    @staticmethod
    def parse_line(line):
        # return json.loads(line, {'precise_float': True})
        # for influxdb version > 0.9, timestamp is an integer
        return json.loads(line)

    # following methods are added to support customizing measurement name, tags much more flexible
    @staticmethod
    def compose_data(measurement, tags, value, time):
        data = "{0!s},{1!s} {2!s} {3!s}".format(measurement, tags, value, time)
        return data

    @staticmethod
    def format_measurement_name(entry, args):
        name = []
        for arg in args:
            if arg in entry:
                # avoid to add extra _ if some entry value is None
                if entry[arg] != '':
                    name.append(entry[arg])
        return '_'.join(name)

    @staticmethod
    def format_tags(entry, args):
        tag = []
        for arg in args:
            if arg in entry:
                # to avoid add None as tag value
                prom_tags = entry[arg]
                if len(prom_tags) > 0:
                    for prom_tag, prom_tag_value in prom_tags.items():
                        tag.append("{0!s}={1!s}".format(prom_tag, prom_tag_value))
        return ','.join(tag)

    @staticmethod
    def format_time(entry):
        date = datetime.strptime(entry['timestamp'],'%Y-%m-%dT%H:%M:%SZ')
        return int(float(date.timestamp()))

    @staticmethod
    def format_value(entry):
        value = entry['value']
        #if len(values) == 1:
        return "value={0!s}".format(value)
        #else:
        #    # influxdb supports writing a record with multiple field values.
        #    # e.g: 'cpu_load_short,host=server01,region=us-west mem=0.1,cpu=0.2 1422568543702900257'
        #    field_pairs = []
        #    for key, value in zip(entry['dsnames'], values):
        #        field_pairs.append("{0!s}={1!s}".format(key, value))
        #    return ','.join(field_pairs)


"""
    2nd many binary searches LOL

	Time O(RlogC)
	144 ms, faster than 25.63%
"""

class Solution(object):
    def searchMatrix(self, matrix, target):
        """
        :type matrix: List[List[int]]
        :type target: int
        :rtype: bool
        """
        for i in range(len(matrix)):
            arr = matrix[i]
            if self.bsearch(arr, target):
                return True
        return False

    def bsearch(self, nums, target):
        left = 0
        right = len(nums)-1
        while left <= right:
            mid = (left + right)//2
            if target == nums[mid]:
                return True
            elif target < nums[mid]:
                right = mid - 1
            else:
                left = mid + 1
        return False


#!/usr/bin/python3

import sys

def add_arguments(arguments):
    result = 0
    for arg in arguments:
        result += int(arg)
    return result

if __name__ == "__main__":
    # Get all command-line arguments except the script name (sys.argv[0])
    arguments = sys.argv[1:]

    # Call the function to add the arguments
    result = add_arguments(arguments)

    # Print the result followed by a new line
    print(result)


from products.models import *
import pandas as pd
import unidecode
from django import db

# path = '/Users/noname/jff/santeh/santeh_scrape/test4.csv'
# path = '/Users/noname/jff/santeh_main/host/santeh_scrape/new_goods_main.csv'
path = 'new_goods_main.csv'

def createProducts():
    db.connections.close_all()
    deleteall()
    i = 0
    find_dest = 0
    not_find_dest = 0

    data = pd.read_csv(path)
    for index,item in data.iterrows():
        price = item['price']
        # just setting sale price
        if i % 2 == 0:
            sale_price = price / 2
        else:
            sale_price = 0
        # end setting sale price
        name = item['name']

        category_name = item['category']
        category_slug = unidecode.unidecode(category_name).lower().strip().replace(' ', '-')

        imgsrc = item['imgsrcnew']
        description = item['description']

        current_category = Category.objects.get_or_create(
            slug = category_slug,
            name = category_name,
        )[0]

        new_product = Product(
            category = current_category,
            name = name,
            price = price,
            sale_price = sale_price,
            description = description,
            imgsrc = imgsrc,
        )
        new_product.save()
        i += 1
        print('created', i, 'products')

        current_attributes = eval(item['attributes'])
        for key in current_attributes:
            current_attr = Attribute.objects.get_or_create(
                category = current_category,
                name = key,
            )[0]
            current_attr.product.add(new_product)

            # new_product.attributes.add(current_attr)
            current_key_value = current_attributes[key]
            if type(current_key_value) == list:
                for attr_name in current_key_value:
                    if len(attr_name) > 0:
                        current_attr_item = Attributeitem.objects.get_or_create(
                            attr = current_attr,
                            name = attr_name,
                        )[0]
                        current_attr_item.product.add(new_product)
                    # new_product.attributes.add(current_attr_item)
            if type(current_key_value) == str:
                if len(current_key_value) > 0:
                    current_attr_item = Attributeitem.objects.get_or_create(
                        attr = current_attr,
                        name = current_key_value,
                    )[0]
                    current_attr_item.product.add(new_product)
                # new_product.attributes.add(current_attr_item)

    print('created', i , 'products')

if __name__ == "__main__":
    createProducts()


from gymnasium import spaces

import matplotlib.pyplot as plt
import numpy as np
import gymnasium

class BitcoinTradingEnv(gymnasium.Env):
    metadata = {'render.modes': ['human']}

    def __init__(self, df, initial_balance=100, look_back_window=15):
        super(BitcoinTradingEnv, self).__init__()

        self.df = df.copy()
        self._prep_df()

        self.look_back_window = look_back_window
        self.initial_balance = initial_balance
        self.balance = initial_balance
        self.net_worth = initial_balance
        self.btc_held = 0
        self.transaction_fee_percent = 0.001  # e.g., 0.1% fee

        self.action_space = spaces.Box(
            low=0, high=1, shape=(2, 1), dtype=np.float32)

        # Observations: Open, Close, High, Low, Volume + Owned coins
        self.observation_space = spaces.Box(
            low=0, high=1, shape=(look_back_window, 8), dtype=np.float32)

        self.current_step = look_back_window

        self.history = {
            'net_worth': [],
            'balance': [],
            'btc_price': [],
            'btc_held': []
        }

    def _prep_df(self):
        self.df.loc[:, 'SMA'] = self.df['Close'].rolling(window=15).mean()
        self.df = self.df.fillna(0)

    def _next_observation(self):
        obs = self.df.iloc[self.current_step -
                           self.look_back_window: self.current_step, 1:].values
        obs = np.append(obs, [[self.btc_held]] * self.look_back_window, axis=1)
        mins = np.min(obs, axis=0)
        maxs = np.max(obs, axis=0) + 0.0001
        obs = (obs - mins) / (maxs - mins)
        return obs, {}

    def _append_history(self):
        self.history['net_worth'].append(self.net_worth)
        self.history['balance'].append(self.balance)
        self.history['btc_price'].append(self.df.iloc[self.current_step]['Close'])
        self.history['btc_held'].append(self.btc_held)

    def step(self, action):
        self.current_step += 1
        prev_net_worth = self.net_worth
        current_price = self.df.iloc[self.current_step]['Close']
        final_action = (action[0] - action[1])[0]

        if final_action >= 0:
            buy_amount_btc = self.balance * final_action / current_price
            self.btc_held += buy_amount_btc
            self.balance -= buy_amount_btc * current_price * (1 + self.transaction_fee_percent)
        else:
            sell_amount_btc = self.btc_held * final_action * current_price
            self.btc_held += sell_amount_btc
            self.balance += sell_amount_btc * current_price * (1 + self.transaction_fee_percent)

        self.net_worth = self.balance + self.btc_held * current_price
        reward = self.net_worth - self.initial_balance
        # reward = reward + (self.net_worth - self.initial_balance) if self.net_worth < self.initial_balance else reward

        done = self.net_worth <= 0 or self.current_step >= len(self.df) - 1

        if done:
            if not self.current_step >= len(self.df) - 1:
                reward -= 100

        # if self.current_step % 1000 == 0:
        #     print(f"Step: {self.current_step}, Action: {action}, Reward: {reward}")

        self._append_history()
        obs, info = self._next_observation()
        return obs, reward, done, done, info

    def render(self, mode='human'):
        if mode == 'human':
            plt.figure(figsize=(15, 6))
            plt.subplot(2, 2, 1)
            plt.plot(self.history['btc_price'], label='BTC Price')
            # plt.plot(self.history['net_worth'], label='Net Worth')
            plt.legend()
            plt.title('BTC Price & Net Worth Over Time')

            plt.subplot(2, 2, 2)
            plt.plot(self.history['balance'], label='Balance')
            plt.legend()
            plt.title('Balance Over Time')

            plt.subplot(2, 2, 3)
            plt.plot(self.history['btc_held'], label='BTC Held')
            plt.legend()
            plt.title('BTC Held Over Time')

            plt.subplot(2, 2, 4)
            plt.plot(self.history['net_worth'], label='Net Worth')
            plt.legend()
            plt.title('Net Worth Over Time')

            plt.tight_layout()
            plt.show()
        elif mode == 'rgb_array':
            return np.zeros((400, 600, 3))
        else:
            super(BitcoinTradingEnv, self).render()

    def reset(self, seed=42, options=None):
        self.balance = self.initial_balance
        self.net_worth = self.initial_balance
        self.btc_held = 0
        self.current_step = np.random.randint(self.look_back_window, len(self.df) - self.look_back_window)
        return self._next_observation()

    def close(self):
        pass


from __future__ import absolute_import, division, print_function
from six.moves import cStringIO as StringIO
import sys

class LoggingFramework:
  def __init__(self):
    self.k = StringIO()
    self.current_out = sys.stdout
    self.current_err = sys.stderr
    sys.stdout = self.k
    sys.stderr = self.k

  def __del__(self):
    sys.stdout = self.current_out
    sys.stderr = self.current_err
    self.k.flush()
    self.k.close()

  def getvalue(self): return self.k.getvalue()


import functools

from time import time

from django import forms

from projects.models import Project, RepositoryView

from . import data

SELECT_LINES = 20
MAX_ITEMS = 1000  # Implement pagination if there are more items

def perfdata(func):
    @functools.wraps(func)
    def decorator(self, *args, **kwargs):
        task_init = time()
        data = func(self, *args, **kwargs)
        print("%s: Total data collecting time ... %0.3f sec" %
              (self.__class__.__name__, time() - task_init))
        return data
    return decorator

class BestiaryEditorForm(forms.Form):

    widget = forms.Select(attrs={'size': SELECT_LINES, 'class': 'form-control'})

    # Hidden widgets to store the state of the BestiaryEditorForm
    eco_name_state = forms.CharField(required=False, max_length=50, widget=forms.HiddenInput())
    eco_id_state = forms.IntegerField(required=False, widget=forms.HiddenInput())
    projects_state = forms.CharField(required=False, max_length=50, widget=forms.HiddenInput())
    project_id_state = forms.IntegerField(required=False, widget=forms.HiddenInput())
    data_sources_state = forms.CharField(required=False, max_length=50, widget=forms.HiddenInput())
    repository_views_state = forms.CharField(required=False, max_length=50, widget=forms.HiddenInput())

    def is_empty_state(self):
        return self.state.is_empty() if self.state else True

    def __init__(self, *args, **kwargs):
        self.state = kwargs.pop('state') if 'state' in kwargs else None
        if self.state:
            if 'initial' in kwargs:
                kwargs['initial'].update(self.state.initial_state())
            else:
                kwargs['initial'] = self.state.initial_state()
        super(BestiaryEditorForm, self).__init__(*args, **kwargs)

        # The state includes the names of objects except for repository_views
        # in which ids are included because there is no name
        self.state_fields = [self['eco_name_state'],
                             self['eco_id_state'],
                             self['projects_state'],
                             self['project_id_state'],
                             self['data_sources_state'],
                             self['repository_views_state']
                             ]

class EcosystemForm(BestiaryEditorForm):

    @perfdata
    def __init__(self, *args, **kwargs):
        super(EcosystemForm, self).__init__(*args, **kwargs)

        eco_attrs = {'class': 'form-control', 'placeholder': 'Ecosystem name'}
        self.fields['ecosystem_name'] = forms.CharField(label='Ecosystem name', max_length=100)
        self.fields['ecosystem_name'].widget = forms.TextInput(attrs=eco_attrs)

class EcosystemsForm(BestiaryEditorForm):

    widget = forms.Select(attrs={'class': 'form-control', 'onclick': 'this.form.submit()'})

    @perfdata
    def __init__(self, *args, **kwargs):
        super(EcosystemsForm, self).__init__(*args, **kwargs)

        choices = [('', '')]  # Initial empty choice

        for eco in data.EcosystemsData(self.state).fetch():
            choices += ((eco.name, eco.name),)

        self.fields['name'] = forms.ChoiceField(label='Ecosystems', required=True,
                                                widget=self.widget, choices=choices)

class ProjectForm(BestiaryEditorForm):

    @perfdata
    def __init__(self, *args, **kwargs):
        super(ProjectForm, self).__init__(*args, **kwargs)

        self.fields['project_name'] = forms.CharField(label='Project name', max_length=100)
        self.fields['project_name'].widget = forms.TextInput(attrs={'class': 'form-control'})

class ProjectsForm(BestiaryEditorForm):

    @perfdata
    def __init__(self, *args, **kwargs):
        super(ProjectsForm, self).__init__(*args, **kwargs)

        choices = ()

        for project in data.ProjectsData(self.state).fetch():
            if (project.name, project.name) not in choices:
                choices += ((project.name, project.name),)

        choices = sorted(choices, key=lambda x: x[1])
        self.fields['name'] = forms.ChoiceField(label='Projects',
                                                widget=self.widget, choices=choices)

class DataSourceForm(BestiaryEditorForm):

    @perfdata
    def __init__(self, *args, **kwargs):
        super(DataSourceForm, self).__init__(*args, **kwargs)

        ds_attrs = {'class': 'form-control', 'placeholder': 'Data source type'}
        self.fields['data_source_name'] = forms.CharField(label='Data source name', max_length=100)
        self.fields['data_source_name'].widget = forms.TextInput(attrs=ds_attrs)

class DataSourcesForm(BestiaryEditorForm):

    @perfdata
    def __init__(self, *args, **kwargs):
        super(DataSourcesForm, self).__init__(*args, **kwargs)

        choices = ()

        for data_source in data.DataSourcesData(self.state).fetch():
            if (data_source.name, data_source.name) not in choices:
                choices += ((data_source.name, data_source.name),)

        choices = sorted(choices, key=lambda x: x[1])
        self.fields['name'] = forms.ChoiceField(label='DataSources',
                                                widget=self.widget, choices=choices)

class RepositoryViewsForm(BestiaryEditorForm):

    @perfdata
    def __init__(self, *args, **kwargs):
        super(RepositoryViewsForm, self).__init__(*args, **kwargs)

        choices = ()

        for view in data.RepositoryViewsData(self.state).fetch():
            choices += ((view.id, view),)
            if len(choices) > MAX_ITEMS:
                break

        print("Choices len", len(choices))
        self.fields['id'] = forms.ChoiceField(label='DataSource',
                                              widget=self.widget, choices=choices)

class RepositoryViewForm(BestiaryEditorForm):

    @perfdata
    def __init__(self, *args, **kwargs):
        self.repository_view_id = None
        self.repository_view_orm = None

        # First state process in order to fill initial values
        kwargs['initial'] = {}
        self.state = kwargs.get('state') if 'state' in kwargs else None
        if self.state:
            kwargs['initial'] = self.state.initial_state()

        if self.state and self.state.repository_views:
            self.repository_view_id = self.state.repository_views[0]

        if self.state and self.state.projects:
            project_orm = Project.objects.get(name=self.state.projects[0])
            kwargs['initial'].update({
                'project': project_orm.name
            })

        if self.repository_view_id:
            try:
                repository_view_orm = RepositoryView.objects.get(id=self.repository_view_id)
                kwargs['initial'].update({
                    'repository_view_id': self.repository_view_id,
                    'repository': repository_view_orm.repository.name,
                    'params': repository_view_orm.params
                })
            except RepositoryView.DoesNotExist:
                print(self.__class__, "Received repository view which does not exists", self.repository_view_id)
        super(RepositoryViewForm, self).__init__(*args, **kwargs)

        self.fields['repository_view_id'] = forms.CharField(label='repository_view_id', required=False, max_length=100)
        self.fields['repository_view_id'].widget = forms.HiddenInput()

        self.fields['repository'] = forms.CharField(label='repository', max_length=100, required=False)
        self.fields['repository'].widget = forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'URL'})

        self.fields['params'] = forms.CharField(label='params', max_length=100, required=False)
        self.fields['params'].widget = forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'Params'})

        choices = ()

        for ds in data.DataSourcesData(state=None).fetch():
            choices += ((ds.name, ds.name),)

        empty_choice = [('', '')]
        choices = empty_choice + sorted(choices, key=lambda x: x[1])

        self.widget = forms.Select(attrs={'class': 'form-control'})
        self.fields['data_source'] = forms.ChoiceField(label='Data Source', required=True,
                                                       widget=self.widget, choices=choices)

        self.fields['project'] = forms.CharField(label='project', max_length=100, required=False)
        self.fields['project'].widget = forms.HiddenInput(attrs={'class': 'form-control', 'readonly': 'True'})


"""
Cleans data on excel sheet for march madness dataset.
Used for CY305.
"""

import pandas as pd

excel_fn = 'NCAABasicStatsClean.xlsx'
out_fn = 'out.xlsx'
db_master_name = 'NCAAMStats'
db_tourney_name = 'NCAATourney'
db_master = pd.read_excel(excel_fn, sheet_name=db_master_name)
db_tourney = pd.read_excel(excel_fn, sheet_name=db_tourney_name, index_col=None)
# print(db_master.head())
# print(db_tourney.head())

db_out = db_master.copy()
master_count = db_master.shape[0]
tourney_count = db_tourney.shape[0]
print('\n')

# Find teams from tournament database that don't match names from master database
print('Searching for tournament names missing from master database...')
bad_tm_names = []
for j in range(0, tourney_count):
    print("Row {} of {}".format(j, tourney_count))
    team_nm = db_tourney.iloc[j, 3]
    team_in_master = False
    for i in range(0, master_count):
        if db_master.iloc[i, 2] == team_nm:
            team_in_master = True
    if not team_in_master:
        print(team_nm)
        bad_tm_names.append(team_nm)

print(bad_tm_names)
with open('listfile.txt', 'w') as filehandle:
    for listitem in bad_tm_names:
        filehandle.write('%s\n' % listitem)

if len(bad_tm_names) > 0:
    print('Fix names before moving on!')
    exit()

print('Team Names all match.')

print('Merging tournament data and master data into output database file...')
# Fill out tournament results data in output database
for i in range(0, master_count):
    print('Row {} of {}'.format(i, master_count))

    # zero out blank entries
    if pd.isnull(db_out.iloc[i, 4]):
        db_out.iloc[i, 4] = 0
    for k in range(5, 13):
        db_out.iloc[i, k] = False

    # test if team went to tournament before executing code
    if db_master.iloc[i, 3]:
        team_nm = db_master.iloc[i, 2]
        year = db_master.iloc[i, 1]

        # Iterate over every tournament instance if team made tournament
        for j in range(0, tourney_count):
            tour_year = db_tourney.iloc[j, 0]
            tour_ech = db_tourney.iloc[j, 1]
            tour_seed = db_tourney.iloc[j, 2]
            tour_team = db_tourney.iloc[j, 3]

            # Set values in output database for specific year and team
            if year == tour_year:
                if team_nm == tour_team:
                    db_out.iloc[i, 4] = tour_seed
                    db_out.loc[i, tour_ech] = True

db_out.to_excel(out_fn)
print('Mission complete!')


import json
import os
import random
from collections import namedtuple
import torch.utils.data as data
from PIL import Image
import numpy as np
import torch.nn.functional as F

class Cityscapes(data.Dataset):
    """Cityscapes <http://www.cityscapes-dataset.com/> Dataset.

    **Parameters:**
        - **root** (string): Root directory of dataset where directory 'leftImg8bit' and 'gtFine' or 'gtCoarse' are located.
        - **split** (string, optional): The image split to use, 'train', 'test' or 'val' if mode="gtFine" otherwise 'train', 'train_extra' or 'val'
        - **mode** (string, optional): The quality mode to use, 'gtFine' or 'gtCoarse' or 'color'. Can also be a list to output a tuple with all specified target types.
        - **transform** (callable, optional): A function/transform that takes in a PIL image and returns a transformed version. E.g, ``transforms.RandomCrop``
        - **target_transform** (callable, optional): A function/transform that takes in the target and transforms it.
    """

    # Based on https://github.com/mcordts/cityscapesScripts
    CityscapesClass = namedtuple('CityscapesClass', ['name', 'id', 'train_id', 'category', 'category_id',
                                                     'has_instances', 'ignore_in_eval', 'color'])
    classes = [
        CityscapesClass('unlabeled', 0, 255, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('ego vehicle', 1, 255, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('rectification border', 2, 255, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('out of roi', 3, 255, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('static', 4, 255, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('dynamic', 5, 25, 'void', 0, False, True, (111, 74, 0)),
        CityscapesClass('ground', 6, 24, 'void', 0, False, True, (81, 0, 81)),
        CityscapesClass('road', 7, 0, 'flat', 1, False, False, (128, 64, 128)),
        CityscapesClass('sidewalk', 8, 1, 'flat', 1, False, False, (244, 35, 232)),
        CityscapesClass('parking', 9, 23, 'flat', 1, False, True, (250, 170, 160)),
        CityscapesClass('rail track', 10, 22, 'flat', 1, False, True, (230, 150, 140)),
        CityscapesClass('building', 11, 2, 'construction', 2, False, False, (70, 70, 70)),
        CityscapesClass('wall', 12, 3, 'construction', 2, False, False, (102, 102, 156)),
        CityscapesClass('fence', 13, 4, 'construction', 2, False, False, (190, 153, 153)),
        CityscapesClass('guard rail', 14, 21, 'construction', 2, False, True, (180, 165, 180)),
        CityscapesClass('bridge', 15, 20, 'construction', 2, False, True, (150, 100, 100)),
        CityscapesClass('tunnel', 16, 19, 'construction', 2, False, True, (150, 120, 90)),
        CityscapesClass('pole', 17, 5, 'object', 3, False, False, (153, 153, 153)),
        CityscapesClass('polegroup', 18, 18, 'object', 3, False, True, (153, 153, 153)),
        CityscapesClass('traffic light', 19, 6, 'object', 3, False, False, (250, 170, 30)),
        CityscapesClass('traffic sign', 20, 7, 'object', 3, False, False, (220, 220, 0)),
        CityscapesClass('vegetation', 21, 8, 'nature', 4, False, False, (107, 142, 35)),
        CityscapesClass('terrain', 22, 9, 'nature', 4, False, False, (152, 251, 152)),
        CityscapesClass('sky', 23, 10, 'sky', 5, False, False, (70, 130, 180)),
        CityscapesClass('person', 24, 11, 'human', 6, True, False, (220, 20, 60)),
        CityscapesClass('rider', 25, 12, 'human', 6, True, False, (255, 0, 0)),
        CityscapesClass('car', 26, 13, 'vehicle', 7, True, False, (0, 0, 142)),
        CityscapesClass('truck', 27, 14, 'vehicle', 7, True, False, (0, 0, 70)),
        CityscapesClass('bus', 28, 15, 'vehicle', 7, True, False, (0, 60, 100)),
        CityscapesClass('caravan', 29, 13, 'vehicle', 7, True, True, (0, 0, 90)),
        CityscapesClass('trailer', 30, 13, 'vehicle', 7, True, True, (0, 0, 110)),
        CityscapesClass('train', 31, 16, 'vehicle', 7, True, False, (0, 80, 100)),
        CityscapesClass('motorcycle', 32, 17, 'vehicle', 7, True, False, (0, 0, 230)),
        CityscapesClass('bicycle', 33, 18, 'vehicle', 7, True, False, (119, 11, 32)),
        CityscapesClass('license plate', -1, 13, 'vehicle', 7, False, True, (0, 0, 142)),
    ]

    train_id_to_color = [c.color for c in classes if (c.train_id != -1 and c.train_id != 255)]
    train_id_to_color.append([0, 0, 0])
    train_id_to_color = np.array(train_id_to_color)
    id_to_train_id = np.array([c.train_id for c in classes])

    def __init__(self, root, cell_list, interested_classes, split='train', target_type='semantic', transform=None):
        self.root = os.path.expanduser(root)
        self.mode = 'gtFine'
        self.target_type = target_type
        self.images_dir = os.path.join(self.root, 'leftImg8bit', split)
        self.targets_dir = os.path.join(self.root, self.mode, split)
        self.transform = transform

        self.split = split
        self.images = []
        self.targets = []
        self.cell_list = cell_list
        self.interested_classes = interested_classes

        if split not in ['train', 'test', 'val']:
            raise ValueError('Invalid split for mode! Please use split="train", split="test"'
                             ' or split="val"')

        if not os.path.isdir(self.images_dir) or not os.path.isdir(self.targets_dir):
            raise RuntimeError('Dataset not found or incomplete. Please make sure all required folders for the'
                               ' specified "split" and "mode" are inside the "root" directory')

        for city in os.listdir(self.images_dir):
            img_dir = os.path.join(self.images_dir, city)
            target_dir = os.path.join(self.targets_dir, city)

            for file_name in os.listdir(img_dir):
                self.images.append(os.path.join(img_dir, file_name))
                target_name = '{}_{}'.format(file_name.split('_leftImg8bit')[0],
                                             self._get_target_suffix(self.mode, self.target_type))
                self.targets.append(os.path.join(target_dir, target_name))

    @classmethod
    def encode_target(cls, target):
        return cls.id_to_train_id[np.array(target)]

    @classmethod
    def decode_target(cls, target):
        target[target == 255] = 19
        # target = target.astype('uint8') + 1
        return cls.train_id_to_color[target]

    @classmethod
    def encode_cell(cls, target, cell_list, interested_classes):
        label_len = len(cell_list) * len(interested_classes)
        label_list = np.zeros(label_len, dtype=np.uint8)

        for num, region in enumerate(cell_list):
            cell = target[region[0][1]:region[1][1], region[0][0]:region[1][0]]
            value, count = np.unique(cell, return_counts=True)
            # print(value, count)

            last_class_idx = len(interested_classes) - 1
            for j, subclasses in enumerate(interested_classes):
                if isinstance(subclasses, int):
                    subclasses = (subclasses,)
                if j == last_class_idx:
                    if any(label_list[num * len(interested_classes):num * len(interested_classes) + last_class_idx]):
                        if np.size(value) == 2 and value[0] == 0 and value[1] == 255 and num >= 216:
                            label_list[num * len(interested_classes) + j] = 1
                        else:
                            label_list[num * len(interested_classes) + j] = 0
                    else:
                        if np.size(value) == 1 and value[0] == 255 and num <= 216:
                            label_list[num * len(interested_classes) + j] = 0
                        else:

                            label_list[num * len(interested_classes) + j] = 1
                else:
                    if any(subclass in value for subclass in subclasses):
                        label_list[num * len(interested_classes) + j] = 1
        return label_list

    def __getitem__(self, index):
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target) where target is a tuple of all target types if target_type is a list with more
            than one item. Otherwise target is a json object if target_type="polygon", else the image segmentation.
        """
        image = Image.open(self.images[index]).convert('RGB')
        filename, _ = os.path.splitext(os.path.basename(self.images[index]))
        target = Image.open(self.targets[index])
        cell_list = self.cell_list
        interested_classes = self.interested_classes
        # flip image and target image horizontally with 0.5 probability
        if self.split == 'train':
            if random.random() > 0.5:
                image = image.transpose(Image.FLIP_LEFT_RIGHT)
                target = target.transpose(Image.FLIP_LEFT_RIGHT)
            if random.random() > 0.5:
                new_width = 2198
                new_height = 1099
                image = image.resize((new_width, new_height), Image.ANTIALIAS)
                target = target.resize((new_width, new_height), Image.NEAREST)
                crop_width, crop_height = 2048, 1024
                left = random.randint(0, new_width - crop_width)
                upper = random.randint(0, new_height - crop_height)
                right = left + crop_width
                lower = upper + crop_height
                # Crop the image
                image = image.crop((left, upper, right, lower))
                target = target.crop((left, upper, right, lower))
        if self.transform:
            image = self.transform(image)
        target = self.encode_target(target)
        cell_label = self.encode_cell(target, cell_list, interested_classes)
        image = np.array(image)
        return image, cell_label

    def __len__(self):
        return len(self.images)

    def _load_json(self, path):
        with open(path, 'r') as file:
            data = json.load(file)
        return data

    def _get_target_suffix(self, mode, target_type):
        if target_type == 'instance':
            return '{}_instanceIds.png'.format(mode)
        elif target_type == 'semantic':
            return '{}_labelIds.png'.format(mode)
        elif target_type == 'color':
            return '{}_color.png'.format(mode)
        elif target_type == 'polygon':
            return '{}_polygons.json'.format(mode)
        elif target_type == 'depth':
            return '{}_disparity.png'.format(mode)


from django.db.models import signals
from django.dispatch import Signal, receiver

from apps.forums.models import Comment, Post
from apps.forums.services import forum_update
from apps.forums.services.forum_update import set_first_comment

new_opportunities_for_attorney = Signal(providing_args=('instance',))
new_opportunities_for_attorney.__doc__ = (
    'Signal which indicates that there are new opportunities for attorney'
)
new_comment_on_post = Signal(providing_args=('instance',))
new_comment_on_post.__doc__ = (
    'Signal which indicates that there are new comments on post'
    "and we should notify post's followers"
)
new_post_on_topic = Signal(providing_args=('instance',))
new_post_on_topic.__doc__ = (
    'Signal which indicates that there are new posts on topic'
    "and we should notify topic's followers"
)
new_comment_on_post_by_attorney = Signal(providing_args=('instance',))
new_comment_on_post_by_attorney.__doc__ = (
    'Signal which indicates that there are new comments on post'
    "made by attorney and we should notify attorney's followers"
)

@receiver(signals.post_save, sender=Post)
def new_post_created(instance: Post, created: bool, **kwargs):
    """Send signal about new post creation.
    """
    if not created:
        return

    new_post_on_topic.send(sender=Post, instance=instance)

@receiver(signals.post_save, sender=Comment)
def new_post(instance: Comment, created: bool, **kwargs):
    """Send signal about new comment.

    We send two different signals:
        new_comment_on_post is always sent
        new_comment_on_post_by_attorney is sent if author of post is attorney

    """
    if not created:
        return

    if not instance.post.first_comment:
        set_first_comment(instance=instance.post, first_comment=instance)

    new_comment_on_post.send(sender=Comment, instance=instance)
    if instance.author.is_attorney:
        new_comment_on_post_by_attorney.send(sender=Comment, instance=instance)

@receiver(signals.post_save, sender=Comment)
def post_creation(instance: Comment, created: bool, **kwargs):
    """Perform recalculation for statistic data in related models."""
    if not created:
        return

    # Update related topic
    forum_update.set_last_comment(instance.post, instance)
    forum_update.set_comments_count(instance.post, 1)

    # Update related category
    forum_update.set_last_comment(instance.post.topic, instance)
    forum_update.set_comments_count(instance.post.topic, 1)

    # Update post's author user stats
    forum_update.set_comments_count(instance.author, 1)

@receiver(signals.post_delete, sender=Comment)
def post_deletion(instance: Comment, **kwargs):
    """Perform recalculation for statistic data in related models.

    Update topic and category statistic in case of post deletion.

    """
    # Update related topic
    forum_update.set_comments_count(instance.post)
    forum_update.set_last_comment(instance.post)
    # Update related category
    forum_update.set_comments_count(instance.post.topic)
    forum_update.set_last_comment(instance.post.topic)
    # Update user statistic
    forum_update.set_comments_count(instance.author)


"""Main entry point into the application. This starts up each of the
subprocesses, which are capable of queueing jobs, then acts as a jobs worker.
This approach allows the loansbot to be parallelized on queries but
single-threaded for mutations.
"""
from multiprocessing import Process
import importlib
import time
from lblogging import Level
from lbshared.lazy_integrations import LazyIntegrations
import lbshared.retry_helper as retry_helper
import atexit
import signal

SUBPROCESSES = (
    'runners.comments', 'runners.rechecks', 'runners.links',
    'runners.new_lender', 'runners.borrower_request', 'runners.default_permissions',
    'runners.trust_loan_delays', 'runners.deprecated_alerts', 'runners.loans_stats',
    'runners.ban_unpaid', 'runners.lender_loan', 'runners.recheck_permission',
    'runners.lender_queue_trusts', 'runners.modlog', 'runners.modlog_cache_flush',
    'runners.mod_changes', 'runners.mod_offboarding', 'runners.mod_onboarding_claim',
    'runners.mod_onboarding', 'runners.mod_sync', 'runners.mod_onboarding_messages',
    'runners.flair_loan_threads_completed', 'runners.temp_ban_expired_cache_flush',
)

def subprocess_runner(name):
    """Runs the given submodule

    Arguments:
    - `name (str)`: The name of the module to run
    """
    mod = importlib.import_module(name)

    try:
        mod.main()
    except:  # noqa
        with LazyIntegrations(logger_iden='main.py#subprocess_runner') as itgs:
            itgs.logger.exception(
                Level.WARN,
                'Child process {} failed with an unhandled exception',
                name
            )

def main():
    """Spawn all of the subprocesses as daemons and then works jobs until one
    one them dies or a signal to shutdown is received."""
    retry_helper.handle()

    subprocs = []
    with LazyIntegrations(logger_iden='main.py#main') as itgs:
        itgs.logger.print(Level.DEBUG, 'Booting up..')
        for modnm in SUBPROCESSES:
            itgs.logger.print(Level.TRACE, 'Spawning subprocess {}', modnm)
            proc = Process(target=subprocess_runner, name=modnm, args=(modnm,), daemon=True)
            proc.start()
            subprocs.append(proc)

    shutting_down = False

    def onexit(*args, **kwargs):
        nonlocal shutting_down
        if shutting_down:
            return
        shutting_down = True
        try:
            with LazyIntegrations(logger_iden='main.py#main#onexit') as itgs:
                itgs.logger.print(Level.INFO, 'Shutting down')
        finally:
            for proc in subprocs:
                if proc.is_alive():
                    proc.terminate()

            for proc in subprocs:
                proc.join()

    atexit.register(onexit)
    signal.signal(signal.SIGINT, onexit)
    signal.signal(signal.SIGTERM, onexit)

    running = True
    while running and not shutting_down:
        for proc in subprocs:
            if not proc.is_alive():
                with LazyIntegrations(logger_iden='main.py#main') as itgs:
                    itgs.logger.print(Level.ERROR, 'A child process has died ({})! Terminating...', proc.name)
                running = False
                break
        if not running:
            break
        for _ in range(20):
            time.sleep(0.5)
            if shutting_down:
                break

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = "Xin YU"

import pandas as pd
import sys
import os
import csv

# extraire des données nécesaires afin de les importer dans l'outil Gargantext

# load dataset from csv
separateur = '\t'
filecsv = sys.argv[1] # '../data_scraping/NOM_DU_FICHIER.csv'
df = pd.read_csv(filecsv, sep =separateur, na_filter=False)

df['title'] = df['titreDelArticle'].replace(to_replace='[“”]', value='', regex=True)
df['abstract'] = df['résumé(fr)'].astype(str) + df['article']
df['publication_year'] = df['réréfenceDuNuméro'].replace(to_replace='/.*', value='', regex=True)
df.rename(columns={'auteur(s)':'authors', 'titreDuNuméro':'source'}, inplace=True)
df['publication_month'] = df['réréfenceDuNuméro'].str.extract(r'\d+/(\d)')
df['publication_day'] = '1'

df = df.drop(columns=['réréfenceDuNuméro', 'titreDelArticle', 'sous-titre', 'résumé(fr)', 'article', 'thème', 'urlDuNuméro', 'urlDelArticle', 'notes', 'motscles'])

print(df.columns, df.shape)
print(df.head(5))
# write
outcsv = sys.argv[2] # './data_gargantext/NOM_DU_FICHIER.csv'
df.to_csv(outcsv, sep=separateur, quotechar='"', quoting=csv.QUOTE_ALL, index=False)


import datetime
import databases
import sqlalchemy
import ormar
from environments import DATABASE_URI

class DateTimeFieldsMixins:
    created_at: datetime = ormar.DateTime(default=datetime.datetime.utcnow())
    updated_at: datetime = ormar.DateTime(default=datetime.datetime.utcnow(), onupdate=datetime.datetime.utcnow())

database = databases.Database(DATABASE_URI)
metadata = sqlalchemy.MetaData()

class Notification(ormar.Model, DateTimeFieldsMixins):
    class Meta:
        database = database
        metadata = metadata

    id: int = ormar.Integer(primary_key=True, autoincrement=True)
    name: str = ormar.String(max_length=256)
    state: int = ormar.Integer()
    responsed_text: str = ormar.Text(nullable=True)

async def with_connect(function, arg):
    async with database:
        await function(**arg)


# -*- coding: utf-8 -*-
import numpy as np

def events_to_mne(events, event_conditions=None):
    """**Create MNE-compatible events**

    Create `MNE <https://mne.tools/stable/index.html>`_ compatible events for integration with
    M/EEG.

    Parameters
    ----------
    events : list or ndarray or dict
        Events onset location. Can also be a dict obtained through :func:`.events_find`.
    event_conditions : list
        An optional list containing, for each event, for example the trial category, group or
        experimental conditions. Defaults to ``None``.

    Returns
    -------
    tuple
        MNE-formatted events and the event id, that can be added
        via ``raw.add_events(events)``, and a dictionary with event's names.

    See Also
    --------
    events_find

    Examples
    ----------
    .. ipython:: python

      import neurokit2 as nk

      signal = nk.signal_simulate(duration=4)
      events = nk.events_find(signal)
      events, event_id = nk.events_to_mne(events)
      events

      event_id

      # Conditions
      events = nk.events_find(signal, event_conditions=["A", "B", "A", "B"])
      events, event_id = nk.events_to_mne(events)
      event_id

    """

    if isinstance(events, dict):
        if "condition" in events.keys():
            event_conditions = events["condition"]
        events = events["onset"]

    event_id = {}

    if event_conditions is None:
        event_conditions = ["event"] * len(events)

    # Sanity check
    if len(event_conditions) != len(events):
        raise ValueError(
            "NeuroKit error: events_to_mne(): 'event_conditions' argument of different length than event onsets."
        )

    event_names = list(set(event_conditions))
    event_index = list(range(len(event_names)))
    for i in enumerate(event_names):
        event_conditions = [event_index[i[0]] if x == i[1] else x for x in event_conditions]
        event_id[i[1]] = event_index[i[0]]

    events = np.array([events, [0] * len(events), event_conditions]).T

    return events, event_id


from flask import Flask,render_template,request
import pickle
import pandas as pd
from model import result_predict

reccomendation_system=pickle.load(open('reccomendation_system_cosine_new.pickle', "rb"))

app = Flask(__name__)

@app.route("/",methods =["POST","GET"])
def home():
    if request.method == "POST":
        user_id = request.form.get("username")
        user_id=user_id.lower().strip()
        if len(user_id)==0:
            return render_template('Home.html') + 'User Id Empty'
        if user_id not in reccomendation_system.index:
            return render_template('Home.html') + 'Invalid User Id OR User Id Not Presnet'
        else:
            product_name=reccomendation_system.loc[user_id].sort_values(ascending=False)[0:20].index.tolist()
            result_df=pd.DataFrame(columns=['Product','Positive%','Negative%'])
            for prod in product_name:
                postivper,negativper=result_predict(prod)
                result_df = result_df.append({'Product':prod,'Positive%':postivper,'Negative%':negativper},ignore_index = True)
            result_df.sort_values(by=['Positive%'], inplace=True,ascending=False)
            return render_template('result.html',predict=result_df.head(5),user=user_id)

    else:
        return render_template('Home.html')

if __name__ == "__main__":
    app.run(debug=True)