# This script computes the scaled change in tas, hus, hur and wap averaged over the Caribbean region for the whole atmospheric column (Figures 18 and 19)

import xarray as xr
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import os
import pandas as pd
import datetime
from cftime import DatetimeNoLeap
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import cmocean.cm as cmo
from matplotlib import cm
from matplotlib.colors import ListedColormap,LinearSegmentedColormap
from scipy import stats
import matplotlib.lines as mlines
import matplotlib.patches as mpatches

#------------------------INPUT------------------------#

# SETTINGS PROGRAM

# -------------Things to choose:----------------------
# Variables

var = 'hur' # hur hus ta wap
n = 58986
ssp = 'ssp585'

# ----------------------------------------------------

# list things to do
season_list = ['annual','dry','wet']
number_list = [0,1,2]
year_period_list = ['2005', '2085']

# Paths variables
paths = {'hus':'/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/hus/*'+ssp+'*.nc' ,
        'ta':'/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/ta/*'+ssp+'*.nc',
        'wap': '/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/wap/*'+ssp+'*.nc',
        'hur': '/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/hur/*'+ssp+'*.nc'}

path_area_mean = {'tas_40N_40S': '/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6/03_area_mean/40N_40S/tas/*'+ssp+'*.nc'}

# SETTINGS FIGURES

period_date = {'2005': ['1-1-1991','31-12-2020'], '2085' : ['1-1-2071','31-12-2100']}

xlabel = {'hus': '$\Delta$ Specific humidity', 'ta': r'$\Delta$ Temperature [$^{\circ}$C/$^{\circ}$C]', 'wap': '$\Delta$ Omega [Pa/s $^{\circ}$C]', 'hur': '$\Delta$ Relative humidity [%/ $^{\circ}$C]'}

# Titles

title_var = {'hus': 'Specific humidity','ta': 'Air temperature', 'wap': 'Vertical motion','hur': 'Relative humidity'}
season_title = {'annual':'Annual','dry': 'Dry season (December-April)','wet':'Wet season (May-November)'}
title_ssp = {'ssp126':'SSP1-2.6', 'ssp245': 'SSP2-4.5', 'ssp585': 'SSP5-8.5'}

# Functions

def time_period(da,period):
    """
    Select only the months inside the season of interest ('period') and takes running mean 30 yr window
    """
    if period == 'annual':
        da_time = da
        #da_time = da_time.rolling(time=30*12,center=False).mean(dim='time')
    elif period == 'wet':
        da_time = da.where((da.time.dt.month>=5) & (da.time.dt.month<=11),drop=True)#.dropna(dim='time')
        #da_time = da_time.rolling(time=30*7,center=False).mean(dim='time')
    elif period == 'dry':
        da_time = da.where((da.time.dt.month<=4) | (da.time.dt.month>=12),drop=True)#.dropna(dim='time')
        #da_time = da_time.rolling(time=30*5,center=False).mean(dim='time')
    return da_time #.dropna(dim='time')

def boxmean(da):
    """Compute spatial mean"""
    weights = np.cos(np.deg2rad(da.lat))
    weights.name = 'weights'
    boxmean = da.weighted(weights).mean(dim=('lat','lon'))
    return boxmean

def preprocessing_vertical(ds):
    """
    Function to call when importing netcdf files with xr.open_mfdataset.
    - Sets all the datasets to the same datetime and time axis
    - Deletes from the files inconsistent dimensions/coordinates and saves models name
    - Saves the name of the model and the rpf number
    """
    if ds.indexes['time'].inferred_type != "datetime64":
            ds['time'] = ds.indexes['time'].to_datetimeindex()
    ds['time'] = pd.date_range(start=str(ds.time[0].dt.year.values)+'-0'+str(ds.time[0].dt.month.values)+'-01',end='2100-12-16',periods=len(ds.time))
    if 'plev_bnds' in ds:
        ds =  ds.drop('plev_bnds')
    if 'plev' in ds:
        ds = ds.drop('plev')
    if 'height' in ds:
        ds = ds.drop('height')
    if 'file_qf' in ds:
        ds = ds.drop('file_qf')
    ds = ds.assign_coords(model_name = ds.source_id)
    ds = ds.assign_coords(variant_label = ds.variant_label)
    return ds

def preprocessing_area_mean(ds):
    """
    Function to call when importing netcdf files with xr.open_mfdataset.
    - Sets all the datasets to the same datetime and time axis
    - Deletes from the files inconsistent dimensions/coordinates and saves models name
    """
    if ds.indexes['time'].inferred_type != "datetime64":
            ds['time'] = ds.indexes['time'].to_datetimeindex()
    ds['time'] = pd.date_range(start=str(ds.time[0].dt.year.values)+'-0'+str(ds.time[0].dt.month.values)+'-01',end='2100-12-16',periods=len(ds.time))
    if 'plev_bnds' in ds:
        ds =  ds.drop('plev_bnds')
    if 'plev' in ds:
        ds = ds.drop('plev')
    if 'height' in ds:
        ds = ds.drop('height')
    if 'file_qf' in ds:
        ds = ds.drop('file_qf')
    return ds

#------------------------CALCULATIONS------------------------#

# Import variables
breakpoint()

tas_40 = xr.open_mfdataset(path_area_mean['tas_40N_40S'],concat_dim = 'model',combine = 'nested', preprocess = preprocessing_area_mean)
tas_40 = tas_40.load()
tas_40 = tas_40.tas
tas_40 = tas_40.sortby(tas_40.model_name)

ds = xr.open_mfdataset(paths[var],combine = 'nested', concat_dim = 'model',preprocess = preprocessing_vertical)
ds = ds.load()
ds = ds.sortby(ds.model_name)

if var == 'ta': ds = ds.ta[:,:,:,0,0]-273.15; color_line =  'darkred'; color_range = 'lightcoral'; x_0 = 1; x_limit = (-3.5,2.5)
elif var == 'hus': ds = ds.hus[:,:,:,0,0]; color_line = 'steelblue'; color_range = 'lightskyblue'; x_0 = 0
elif var == 'wap': ds = ds.wap[:,:,:,0,0]; color_line = 'chocolate'; color_range =   'sandybrown'; x_0 = 0
elif var == 'hur': ds = ds.hur[:,:,:,0,0]; color_line ='steelblue'; color_range = 'lightskyblue'; x_0 = 0

plevs = [100000.,  92500.,  85000.,  70000.,  60000.,  50000.,  40000.,  30000., 25000.,  20000.,  15000.,  10000.,   7000.,   5000.,   3000.,   2000., 1000.,    500.,    100.]
ds['plev'] = plevs

#----------------------------------#

# Start computations

breakpoint()

for season,number in zip(season_list,number_list):

    ds_season = time_period(ds,season)

    #fig , ax = plt.subplots(ncols=1, nrows=1,figsize=(10,7),sharex=True, sharey =True,subplot_kw=dict(projection = ccrs.PlateCarree(central_longitude=180)))
    plt.figure(n+number)

    ds_2005 = ds_season.sel(time=slice(period_date['2005'][0], period_date['2005'][1])).mean(dim = 'time')
    ds_2085 = ds_season.sel(time=slice(period_date['2085'][0], period_date['2085'][1])).mean(dim = 'time')
    if (var == 'hus') or (var == 'hur'): ds_2005 = ds_2005.where((ds_2005 >=0), drop = True); ds_2085 = ds_2085.where((ds_2085 >=0), drop = True)
    tas_40_2005 = tas_40.sel(time=slice(period_date['2005'][0], period_date['2005'][1])).mean(dim = 'time')
    tas_40_2085 = tas_40.sel(time=slice(period_date['2085'][0], period_date['2085'][1])).mean(dim = 'time')
    ds_toplot = (ds_2085.mean(dim = 'model') - ds_2005.mean(dim = 'model'))/(tas_40_2085.mean(dim = 'model') - tas_40_2005.mean(dim = 'model'))
    ds_toquant = (ds_2085 - ds_2005)/(tas_40_2085 - tas_40_2005)

    plt.gca().invert_yaxis()
    plt.plot(ds_toplot,ds_toplot.plev*1e-2,color = color_line )
    plt.fill_betweenx(ds_toplot.plev*1e-2,ds_toquant.quantile(0.05,dim='model'),ds_toquant.quantile(0.95,dim='model'),color=color_range,linewidth=0)
    plt.axvline(x= x_0, color='black', ls = '--', lw = 0.5)
    # plt.axvline(x= 0, color='grey', ls = '--', lw = 0.5)
    # plt.xlim(x_limit)
    plt.gca().invert_yaxis()

    plt.xlabel(xlabel[var],fontsize=12)
    plt.ylabel('Pressure level [hPa]',fontsize=12)
    plt.title(season_title[season],loc='right',fontsize=11)
    plt.title(title_ssp[ssp],loc='left',fontsize=11)

    # fig.suptitle('30 years mean vertical profile. '+title_var[var])

    percent =  mpatches.Patch( color=color_range, label=r'$90\%$ CMIP6')
    line = mlines.Line2D([], [], color=color_line, label=r'CMIP6 mmmean')
    plt.legend(handles=[percent,line])

    plt.gca().invert_yaxis()
    # plt.savefig('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/definitivos/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png')
    # print('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/definitivos/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png')

    plt.savefig('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/report/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png',dpi = 300)
    print('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/report/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png')


#!/bin/python3

import math
import os
import random
import re
import sys

# Complete the minimumSwaps function below.
def minimumSwaps(arr):
    ''' Swap Loop
    arr[i]で取得した値が、評価したいindexと異なれば、本来のindexも異なるはずなのでswapを続ける
    そして欲しい値が出るまでloopをすれば、他のindexもsortされるので無駄なカウントにならない

    1 [0, 2, 4, 1, 3, 5, 7]
    1 [0, 4, 2, 1, 3, 5, 7] c
    1 [0, 3, 2, 1, 4, 5, 7] c
    1 [0, 1, 2, 3, 4, 5, 7] c
    2 [0, 1, 2, 3, 4, 5, 7]
    3 [0, 1, 2, 3, 4, 5, 7]
    4 [0, 1, 2, 3, 4, 5, 7]
    5 [0, 1, 2, 3, 4, 5, 7]
    6 [0, 1, 2, 3, 4, 5, 7]
    '''
    swaps = 0
    i = 0
    while i < n - 1:
        if arr[i] != i + 1:
            tmp = arr[i]
            arr[i], arr[tmp - 1] = arr[tmp - 1], arr[i]
            swaps += 1
        else:
            i += 1
    return swaps

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    n = int(input())

    arr = list(map(int, input().rstrip().split()))

    res = minimumSwaps(arr)

    fptr.write(str(res) + '\n')

    fptr.close()


import os
import cv2
import matplotlib.pyplot as plt
import numpy as np

# Set the input and output directories
input_dir = "image-processing-files/test_images/"
output_dir = "image-processing-files/results/"

for filename in os.listdir(input_dir):
    # Check if the file is an image
    if filename.endswith(".jpg"):
        image_path = os.path.join(input_dir, filename)
        img = cv2.imread(image_path)

        # Image processing

        processed_img = img

        ###INPAINTING
        # Using (similar) coordinates to inpaint
        mask = np.zeros_like(img[:,:,0])
        centre = (188, 212)
        radius = 22
        cv2.circle(mask, centre, radius, 255, -1)

        # Similar section
        section = img[120:160, 190:230]

        # Inpaint the corrupted region in the color image using the extracted section
        inpainted_color = cv2.inpaint(img, mask, 3, cv2.INPAINT_TELEA)

        # Resize the section to match the size of the inpainted region
        section_resized = cv2.resize(section, inpainted_color.shape[:2][::-1], interpolation=cv2.INTER_LANCZOS4)

        # Replace the inpainted region with the extracted section
        processed_img = inpainted_color.copy()
        processed_img[mask != 0] = section_resized[mask != 0]

        # Dewarping and removing borders
        gray = cv2.cvtColor(processed_img, cv2.COLOR_BGR2GRAY)
        ret, thresh = cv2.threshold(gray, 10, 255, 0)

        contours, _ = cv2.findContours(thresh, cv2.RETR_TREE,
                               cv2.CHAIN_APPROX_SIMPLE)

        # take the first contour
        cnt = contours[0]
        #https://www.geeksforgeeks.org/finding-minimum-enclosing-rectangle-in-opencv-python/
        rect = cv2.minAreaRect(cnt)
        box = cv2.boxPoints(rect)
        box = np.int0(box)
        box_img = cv2.drawContours(processed_img, [box], 0, (0, 0, 255), 2)

        extent = np.float32([[0,0], [255,0], [255,255], [0,255]])
        bounds = np.float32(box)

        M = cv2.getPerspectiveTransform(bounds, extent)

        # Apply the transformation to the image
        processed_img = cv2.warpPerspective(processed_img, M, (256, 256))

        #median (salt and pepper noise removal) (better after dewarping and stuff)
        # processed_img = cv2.medianBlur(processed_img, 5)
        kernel = np.ones((3,3), np.uint8)
        processed_img= cv2.dilate(processed_img, kernel, iterations=1)
        ###SHARPENING

        # Create a sharpening filter
        kernel = np.array([[0,-1,0], [-1,5,-1], [0,-1,0]])

        # Apply the sharpening filter to the blurred image
        processed_img= cv2.filter2D(processed_img, -1, kernel)
        processed_img = cv2.GaussianBlur(processed_img, (5,5), 30)

        #  #### REDUCING INTENSITY OF RED CHANNEL

        # b, g, r = cv2.split(processed_img)

        # # Reduce the intensity of the red channel
        # r = cv2.addWeighted(r, 0.8, 0, 0, 0)

        # # Merge the channels back together
        # processed_img = cv2.merge((b, g, r))

        ### Hist equalisation-  method 1- to equalise colour channels
        # using Lab (https://stackoverflow.com/questions/39308030/how-do-i-increase-the-contrast-of-an-image-in-python-opencv)
        # lab= cv2.cvtColor(processed_img, cv2.COLOR_BGR2LAB)
        # l_channel, a, b = cv2.split(lab)

        # # Applying CLAHE to L-channel
        # # feel free to try different values for the limit and grid size:
        # clahe = cv2.createCLAHE(clipLimit=1.5, tileGridSize=(8,8))
        # cl = clahe.apply(l_channel)

        # # merge the CLAHE enhanced L-channel with the a and b channel
        # limg = cv2.merge((cl,a,b))

        # # Converting image from LAB Color model to BGR color spcae
        # processed_img = cv2.cvtColor(limg, cv2.COLOR_LAB2BGR)

        ####Hist Equalisation- method 2- to equalise colour channels
        # img_yuv = cv2.cvtColor(processed_img, cv2.COLOR_RGB2YUV)

        # # Apply histogram equalization to the Y channel
        # img_yuv[:,:,0] = cv2.equalizeHist(img_yuv[:,:,0])

        # # Convert the image back to the original color space
        # processed_img = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2BGR)
        # processed_img = cv2.cvtColor(processed_img, cv2.COLOR_BGR2RGB)
        # # Exponential transform to increase contrast
        # gamma = 2.0
        # inv_gamma = 1.0 / gamma
        # table = np.array([((i / 255.0) ** inv_gamma) * 255 for i in np.arange(0, 256)]).astype('uint8')
        # # Apply the exponential transform to the image
        # processed_img = cv2.LUT(processed_img, table)

        ### BRIGHTNESS & CONTRAST

        # # Define the brightness and contrast adjustments
        # alpha = 2 # Contrast control (1.0-3.0)
        # beta = 50 # Brightness control (0-100)

        # # Apply the brightness and contrast adjustments
        # processed_img = cv2.convertScaleAbs(processed_img, alpha=alpha, beta=beta)

        ### Brightness and Contrast- but specific to image mean and std deviation
        # target_mean = 100
        # target_sd = 20
        # gray = cv2.cvtColor(processed_img, cv2.COLOR_BGR2GRAY)
        # mean_pixel = np.mean(gray)
        # sd_pixel = np.std(gray)
        # scale_factor = mean_pixel/target_mean
        # #scale_factor = target_sd / sd_pixel
        # difference = (target_mean- mean_pixel)*scale_factor
        # # Adjust the contrast and brightness
        # processed_img = cv2.convertScaleAbs(processed_img, alpha=scale_factor, beta=difference)

        # Save the processed image to the output directory
        output_path = os.path.join(output_dir, filename)
        cv2.imwrite(output_path, processed_img)

        print(f"Processed {filename}")


import uvicorn
from fastapi import FastAPI
from fastapi_sqlalchemy import DBSessionMiddleware
from starlette_exporter import PrometheusMiddleware, handle_metrics

from src.controllers import author_controller
from src.entities import DB_URL

def init_app():
    custom_app = FastAPI()
    custom_app.add_middleware(
        PrometheusMiddleware, app_name="bookshelf", group_paths=True
    )
    custom_app.add_middleware(DBSessionMiddleware, db_url=DB_URL)
    custom_app.add_route("/metrics", handle_metrics)
    custom_app.include_router(author_controller.router)

    return custom_app

def start_server():
    uvicorn.run(
        "main:app",
        host="0.0.0.0",
        port=5000,
        debug=True,
        reload=True,
        log_level="debug",
    )

app = init_app()

if __name__ == "__main__":
    start_server()


from numpy import array, zeros, arange, concatenate, searchsorted, where, unique

from pyNastran.bdf.fieldWriter import print_card_8
from pyNastran.bdf.bdfInterface.assign_type import (integer, integer_or_blank,
    double_or_blank, integer_double_or_blank, blank)

class PBAR(object):
    type = 'PBAR'
    def __init__(self, model):
        """
        Defines the PCOMP object.

        :param self: the PCOMP object
        :param model: the BDF object
        :param cards: the list of PCOMP cards
        """
        self.model = model
        self.n = 0
        self._cards = []
        self._comments = []

    def add(self, card, comment):
        self._cards.append(card)
        self._comments.append(comment)

    def build(self):
        cards = self._cards
        ncards = len(cards)
        self.n = ncards

        if ncards:
            #: Property ID
            self.property_id = zeros(ncards, 'int32')
            self.material_id = zeros(ncards, 'int32')
            self.area = zeros(ncards, 'float64')
            self.I1 = zeros(ncards, 'float64')
            self.I2 = zeros(ncards, 'float64')
            self.J = zeros(ncards, 'float64')
            self.nsm = zeros(ncards, 'float64')

            for i, card in enumerate(cards):
                #: property ID
                self.property_id[i] = integer(card, 1, 'property_id')

                #: material ID
                self.material_id[i] = integer(card, 2, 'material_id')

                #: material ID
                self.area[i] = double_or_blank(card, 3, 'area', 0.0)

                #: I1
                self.I1[i] = double_or_blank(card, 4, 'I1', 0.0)

                #: I2
                self.I2[i] = double_or_blank(card, 5, 'I2', 0.0)

                #: Polar Moment of Inertia J -> use J()
                #: default=1/2(I1+I2) for SOL=600, otherwise 0.0
                #: .. todo:: support SOL 600 default

                Jdefault = 0.5 * (self.I1[i] + self.I2[i])
                self.J[i] = double_or_blank(card, 6, 'J', Jdefault)
                self.nsm[i] = double_or_blank(card, 7, 'non-structural_mass', 0.0)

                if 0:
                    self.C1 = double_or_blank(card, 9, 'C1', 0.0)
                    self.C2 = double_or_blank(card, 10, 'C2', 0.0)
                    self.D1 = double_or_blank(card, 11, 'D1', 0.0)
                    self.D2 = double_or_blank(card, 12, 'D2', 0.0)
                    self.E1 = double_or_blank(card, 13, 'E1', 0.0)
                    self.E2 = double_or_blank(card, 14, 'E2', 0.0)
                    self.F1 = double_or_blank(card, 15, 'F1', 0.0)
                    self.F2 = double_or_blank(card, 16, 'F2', 0.0)

                    #: default=infinite; assume 1e8
                    self.K1 = double_or_blank(card, 17, 'K1', 1e8)
                    #: default=infinite; assume 1e8
                    self.K2 = double_or_blank(card, 18, 'K2', 1e8)
                    #: I12 -> use I12()
                    self.i12 = double_or_blank(card, 19, 'I12', 0.0)
                    if self.A == 0.0 and self.i12 == 0.0:
                        assert self.K1 is None, 'K1 must be blank if A=0.0 and I12=0.0; A=%r I12=%r K1=%r' % (self.A, self.i12, self.K1)
                        assert self.K2 is None, 'K2 must be blank if A=0.0 and I12=0.0; A=%r I12=%r K2=%r' % (self.A, self.i12, self.K2)
                    assert len(card) <= 20, 'len(PBAR card) = %i' % len(card)

            i = self.property_id.argsort()
            self.property_id = self.property_id[i]
            self.material_id = self.material_id[i]

            self.area = self.area[i]
            self.I1 = self.I1[i]
            self.I2 = self.I2[i]
            self.J = self.J[i]
            self.nsm = self.nsm[i]

            unique_pids = unique(self.property_id)

            if len(unique_pids) != len(self.property_id):
                raise RuntimeError('There are duplicate PCOMP IDs...')
            self._cards = []
            self._comments = []

    #=========================================================================
    def get_index(self, property_ids):
        if isinstance(property_ids, int):
            property_ids = array([property_ids])
        if property_ids is None:
            return arange(self.n)

        indexs = searchsorted(self.property_id, property_ids)
        assert len(indexs) == len(property_ids), 'indexs=%s pids=%s' % (indexs, property_ids)
        return indexs

    #=========================================================================
    def write_bdf(self, f, size=8, property_ids=None):
        if self.n:
            if property_ids is None:
                i = arange(self.n)
            else:
                i = searchsorted(self.property_id, property_ids)

            for (pid, mid, area, I1, I2, J) in zip(self.property_id[i], self.material_id[i],
                    self.area[i], self.I1[i], self.I2[i], self.J[i]):
                card = ['PBAR', pid, mid, area, I1, I2, J]
                f.write(print_card_8(card))


import contextlib
import os
import re
import subprocess
import sys

class Module:
    def __init__(self, name, arguments, signature):
        self.name = name
        self.arguments = arguments
        self.signature = signature

    def text(self, indent=""):
        arguments = ""
        if indent == "" and self.arguments:
            arguments = "".join(argument.text(indent) for argument in self.arguments)
        signature = ""
        if indent == "" or not self.signature.is_empty():
            signature = ": {}".format(self.signature.text(indent))
        return "{}module {}{}{}\n".format(indent, self.name, arguments, signature)

class FunctorArgument:
    def __init__(self, name, signature):
        self.name = name
        self.signature = signature

    def text(self, indent):
        return "({}: {})".format(self.name, self.signature.text(indent))

class Alias:
    def __init__(self, name, base):
        self.name = name
        self.base = base

    def text(self, indent):
        return "{}module {} = {}\n".format(indent, self.name, self.base)

class Name:
    def __init__(self, name):
        self.name = name

    def is_empty(self):
        return False

    def text(self, indent):
        return "{}".format(self.name)

class Signature:
    def __init__(self, elements):
        self.elements = elements

    def is_empty(self):
        return not self.elements

    def text(self, indent):
        return "sig\n{}{}end".format("".join(element.text("  " + indent) for element in self.elements), indent)

class ModuleType:
    def __init__(self, name, signature):
        self.name = name
        self.signature = signature

    def text(self, indent):
        return "{}module type {} = {}\n".format(indent, self.name, self.signature.text(indent))

class Atom:
    def __init__(self, definition):
        self.definition = definition

    def text(self, indent):
        return "{}{}\n".format(indent, " ".join(self.definition))

class UTop:
    def __init__(self, process):
        process.stdout.readline()
        process.stdout.readline()
        self.__process = process
        self.__tokens = []

    def show_module(self, module_name):
        self.__process.stdin.write("#show_module {};;\n".format(module_name))
        self.__process.stdin.flush()
        self.__assert_token("#")
        return self.__parse_module()

    def __refill_tokens(self, n):
        if len(self.__tokens) <= n:
            self.__tokens = re.split(r"\s+", self.__process.stdout.readline().strip().replace("(", "( ").replace(")", " )"))
            # print("Tokens:", self.__tokens)

    def __next_token(self):
        self.__refill_tokens(0)
        tok = self.__tokens[0]
        self.__tokens = self.__tokens[1:]
        return tok

    def __peek_token(self, n=0):
        self.__refill_tokens(n)
        return self.__tokens[n]

    def __assert_token(self, expected):
        tok = self.__next_token()
        if tok != expected:
            print("Expected '{}' but got '{}'".format(expected, tok))
            exit(1)

    kinds = {"module", "val", "type", "exception", "external"}
    ends = {None, "end"}
    atom_ends = kinds | ends

    def __parse_module(self):
        self.__assert_token("module")
        name = self.__next_token()
        if self.__peek_token() == ":":
            self.__assert_token(":")
            arguments = self.__parse_functor_arguments()
            signature = self.__parse_signature()
            return Module(name, arguments, signature)
        else:
            self.__assert_token("=")
            base = self.__next_token()
            return Alias(name, base)

    def __parse_functor_arguments(self):
        arguments = []
        if self.__peek_token() == "functor":
            self.__assert_token("functor")
            while self.__peek_token() == "(":
                self.__assert_token("(")
                name = self.__next_token()
                self.__assert_token(":")
                signature = self.__parse_signature()
                argument = FunctorArgument(name, signature)
                self.__assert_token(")")
                arguments.append(argument)
            self.__assert_token("->")
        return arguments

    def __parse_signature(self):
        if self.__peek_token() == "sig":
            self.__assert_token("sig")
            if self.__peek_token() == "...":
                self.__assert_token("...")
                elements = []
            else:
                elements = self.__parse_elements()
            self.__assert_token("end")
            return Signature(elements)
        else:
            return Name(self.__next_token())

    def __parse_elements(self):
        elements = []
        while self.__peek_token() not in self.ends:
            element = self.__parse_element()
            elements.append(element)
        return elements

    def __parse_element(self):
        assert self.__peek_token() in self.kinds, "Unkwnown kind: {}".format(self.__peek_token())
        if self.__peek_token() == "module":
            if self.__peek_token(1) == "type":
                return self.__parse_module_type()
            else:
                return self.__parse_module()
        else:
            return self.__parse_atom()

    def __parse_module_type(self):
        self.__assert_token("module")
        self.__assert_token("type")
        name = self.__next_token()
        self.__assert_token("=")
        signature = self.__parse_signature()
        return ModuleType(name, signature)

    def __parse_atom(self):
        definition = [self.__next_token()]
        while self.__peek_token() not in self.atom_ends:
            definition.append(self.__next_token())
        return Atom(definition)

@contextlib.contextmanager
def utop(*options):
    process = subprocess.Popen(["utop"] + list(options), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)
    yield UTop(process)
    process.communicate()

if __name__ == "__main__":
    subprocess.run(["dune", "build", "src/.General.objs/byte/General.cmi"], check=True)
    with utop("-I", "_build/default/src/.General.objs/byte") as utop:
        def show(module_name):
            # print(module_name)
            module = utop.show_module(module_name)
            with open("{}/{}.txt".format(sys.argv[1], module_name), "w") as f:
                f.write(module.text().replace("( ", "(").replace(" )", ")").replace(" :", ":"))

            if module_name not in [
                "General.Pervasives.OCamlStandard",
                "General.Abbr",
                "General.Standard",
            ]:
                for element in module.signature.elements:
                    if isinstance(element, Module) and element.signature.is_empty():
                        show("{}.{}".format(module_name, element.name))

        show("General")


from bokeh.io import save, output_file
from bokeh.models.widgets import DataTable, TableColumn
from bokeh.models import ColumnDataSource
from bokeh.layouts import widgetbox
import pandas as pd
import modules.channel_module as channel_module

channel_list_file_path = "./list.txt"
channel_list = channel_module.get_channel_list(channel_list_file_path)

output_file("make_table_posting_interval.html")
df = pd.read_csv("./output/posting_interval.csv")
source = ColumnDataSource(df)

columns = [
    TableColumn(field="Channel", title="Channel"),
    TableColumn(field="ViewCount", title="ViewCount"),
    TableColumn(field="SubscriberCount", title="SubscriberCount"),
    TableColumn(field="VideCount", title="VideCount"),
    TableColumn(field="PostingInterval", title="PostingInterval"),
]

data_table = DataTable(
    source=source, columns=columns, fit_columns=True, width=1300, height=800
)
save(widgetbox([data_table], sizing_mode="scale_both"))


import cv2

# Load image
image_path = 'images/salt_pepper_noise.png'
image = cv2.imread(image_path)

# Gaussian blur
k = 5
blur = cv2.medianBlur(image, k)

# Show
cv2.imshow('Original', image)
cv2.imshow('Filtered', blur)
cv2.waitKey(0)
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
# Part of Softhealer Technologies.

from odoo import models, fields, api

class ResCompany(models.Model):
    _inherit = 'res.company'

    category = fields.Boolean('Category')
    sub_category = fields.Boolean('Sub Category')
    customer_rating = fields.Boolean('Customer Rating')
    auto_close_ticket = fields.Boolean('Auto Close Ticket')
    close_days = fields.Integer('No of Days')
    new_stage_id = fields.Many2one(
        'helpdesk.stages', string="Draft/New Stage")
    done_stage_id = fields.Many2one('helpdesk.stages', string="Resolved Stage")
    cancel_stage_id = fields.Many2one('helpdesk.stages', string="Cancel Stage")
    allocation_mail_template_id = fields.Many2one(
        'mail.template', string='Ticket Allocation To User Mail Template')
    reply_mail_template_id = fields.Many2one(
        'mail.template', string='Ticket Reply Mail Template')
    dashboard_filter = fields.Many2many(
        'helpdesk.stages', 'rel_company_stage_counter', string="Dashbaord Filter", required=True)
    dashboard_tables = fields.Many2many(
        'helpdesk.stages', 'rel_company_stage_tables', string="Dashbaord Tables", required=True)
    reopen_stage_id = fields.Many2one(
        'helpdesk.stages', string="Re-Opened Stage")
    close_stage_id = fields.Many2one('helpdesk.stages', string="Closed Stage")
    sh_default_team_id = fields.Many2one(
        'helpdesk.team', string="Default Team")
    sh_default_user_id = fields.Many2one(
        'res.users', string="Default Assign User")
    sh_display_multi_user = fields.Boolean('Display Multi Users ?')
    sh_configure_activate = fields.Boolean(
        'Manage Products')
    sh_display_ticket_reminder = fields.Boolean('Ticket Reminder ?')

class HelpdeskSettings(models.TransientModel):
    _inherit = 'res.config.settings'

    company_id = fields.Many2one('res.company', string='Company', required=True,
                                 default=lambda self: self.env.company)
    category = fields.Boolean(
        string='Category', related='company_id.category', readonly=False)
    sub_category = fields.Boolean(
        string='Sub Category', related='company_id.sub_category', readonly=False)
    customer_rating = fields.Boolean(
        string='Customer Rating', related='company_id.customer_rating', readonly=False)
    auto_close_ticket = fields.Boolean(
        string='Auto Close Ticket', related='company_id.auto_close_ticket', readonly=False)
    close_days = fields.Integer(
        string='No of Days', related='company_id.close_days', readonly=False)
    new_stage_id = fields.Many2one('helpdesk.stages', string="Draft/New Stage",
                                   related='company_id.new_stage_id', readonly=False)
    done_stage_id = fields.Many2one(
        'helpdesk.stages', string="Resolved Stage", related='company_id.done_stage_id', readonly=False)
    cancel_stage_id = fields.Many2one(
        'helpdesk.stages', string="Cancel Stage", related='company_id.cancel_stage_id', readonly=False)
    allocation_mail_template_id = fields.Many2one(
        'mail.template', string='Ticket Allocation To User Mail Template', related='company_id.allocation_mail_template_id', readonly=False)
    reply_mail_template_id = fields.Many2one(
        'mail.template', string='Ticket Reply Mail Template', related='company_id.reply_mail_template_id', readonly=False)
    dashboard_filter = fields.Many2many('helpdesk.stages', 'rel_company_stage_counter',
                                        string="Dashbaord Filter", related="company_id.dashboard_filter", readonly=False, required=True)
    dashboard_tables = fields.Many2many('helpdesk.stages', 'rel_company_stage_tables',
                                        string="Dashbaord Tables", related="company_id.dashboard_tables", readonly=False, required=True)
    reopen_stage_id = fields.Many2one(
        'helpdesk.stages', string="Re-Opened Stage", readonly=False, related='company_id.reopen_stage_id')
    close_stage_id = fields.Many2one(
        'helpdesk.stages', string="Closed Stage", readonly=False, related='company_id.close_stage_id')
    sh_default_team_id = fields.Many2one(
        'helpdesk.team', string="Default Team", readonly=False, related='company_id.sh_default_team_id')
    sh_default_user_id = fields.Many2one(
        'res.users', string="Default Assign User", readonly=False, related='company_id.sh_default_user_id')
    sh_display_multi_user = fields.Boolean(
        'Display Multi Users ?', related='company_id.sh_display_multi_user', readonly=False)
    sh_configure_activate = fields.Boolean(
        'Manage Products', related='company_id.sh_configure_activate', readonly=False)
    sh_display_ticket_reminder = fields.Boolean('Ticket Reminder ?',related='company_id.sh_display_ticket_reminder',readonly=False)

    @ api.onchange('sh_default_team_id')
    def onchange_sh_default_team_id(self):
        if self.sh_default_team_id:
            domain = {}
            domain = {'sh_default_user_id': [
                ('id', 'in', self.sh_default_team_id.team_members.ids)]}
            return {'domain': domain}


import math
import rclpy
from rclpy.executors import ExternalShutdownException
from rclpy.node import Node
from ros2_example_msgs.srv import Circle

class CircleServer(Node):
    """
    円サービスServerノード
    """

    def __init__(self) -> None:
        """
        ノードの初期化
        """
        super().__init__("circle_server")

        # ロガー取得
        self.logger = self.get_logger()

        # circleサービスserverの定義
        self.srv = self.create_service(
            Circle,
            "circle",
            self._calc_circle,
        )

    def _calc_circle(self, req, rsp):
        """
        circleサービスコールバック
        """
        # 受信requestの中身をログ出力
        self.logger.info("＜リクエスト＞")
        self.logger.info("  - 半径:[{:.2f}]".format(req.radius))

        # 円周の計算(直径×π)
        rsp.circumference = req.radius * 2 * math.pi
        # 円面積の計算(半径^2×π)
        rsp.area = req.radius**2 * math.pi

        self.logger.info("＜レスポンス＞")
        self.logger.info("  - 円周:[{:.2f}]".format(rsp.circumference))
        self.logger.info("  - 円面積:[{:.2f}]".format(rsp.area))

        return rsp

def main(args: list[str] | None = None) -> None:
    # ROSの初期化
    rclpy.init(args=args)
    # ohlc_serverノードの作成
    cs = CircleServer()
    cs.get_logger().info("circle_server start!")

    try:
        # ノードの実行開始
        rclpy.spin(cs)
    except (KeyboardInterrupt, ExternalShutdownException):
        pass
    else:
        # ROSのシャットダウン
        rclpy.shutdown()
    finally:
        # ノードの破棄
        cs.destroy_node()


import warnings
warnings.filterwarnings("ignore")
import re
import sqlite3
from matplotlib.backends.backend_pdf import PdfPages
import time
import os
import csv
import pickle
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sys
plt.ion() # take away popups - putting plot images in pdf
#import pdfplumber

from nltk import FreqDist
from nltk.stem.snowball import SnowballStemmer
from nltk import word_tokenize
from nltk import sent_tokenize
from nltk.util import ngrams
from nltk.corpus import stopwords
import nltk
from nltk.sentiment import SentimentIntensityAnalyzer

#from sklearn.cluster import KMeans
#from sklearn.metrics.pairwise import cosine_similarity
#from sklearn.cluster import AgglomerativeClustering
#from sklearn.linear_model import LogisticRegression
#from sklearn.linear_model import LinearRegression
#from sklearn import feature_extraction # br
#from sklearn.decomposition import TruncatedSVD
#from sklearn.svm import SVC
#from sklearn.metrics import accuracy_score

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer # lsi
from sklearn.decomposition import TruncatedSVD
from sklearn.cluster import MiniBatchKMeans
from sklearn.model_selection import train_test_split

from collections import defaultdict
from sklearn.decomposition import PCA
from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD

from gensim.models import LsiModel
from gensim.models import LdaModel
from gensim.models.coherencemodel import CoherenceModel
from gensim.matutils import Sparse2Corpus
from gensim.corpora import Dictionary

from sklearn.metrics import silhouette_samples, silhouette_score
from yellowbrick.cluster import SilhouetteVisualizer

from sklearn.cluster import DBSCAN
import random
import seaborn as sns
from pprint import pprint # lda link
#import pyLDAvis
#import pyLDAvis.gensim

#from scipy.cluster.hierarchy import dendrogram
#from scipy.cluster import hierarchy
#import pyLDAvis
#import pyLDAvis.sklearn
#pyLDAvis.enable_notebook()
sns.set() #365 link
paper_count = 3769874

#vocab = dict()

# def get_model_topics(model, vectorizer, topics, n_top_words=15): # towardsdatascience article
#     word_dict = {}
#     feature_names = vectorizer.get_feature_names()
#     for topic_idx, topic in enumerate(model.components_):
#         top_features_ind = topic.argsort()[:-n_top_words - 1:-1]
#         top_features = [feature_names[i] for i in top_features_ind]
#         #word_dict[topics[topic_idx]] = top_features
#         word_dict[topic_idx] = top_features

#     return pd.DataFrame(word_dict)

# def get_inference(model, vectorizer, topics, text, threshold): # towards data science article
#     v_text = vectorizer.transform([text])
#     score = model.transform(v_text)

#     labels = set()
#     for i in range(len(score[0])):
#         if score[0][i] > threshold:
#             labels.add(topics[i])

#     if not labels:
#         return 'None', -1, set()

#     return topics[np.argmax(score)], score, labels
#def process_entries(path, output_dir, test_pred_dir, figure_dir):

def process_entries(path, results_dir, folder_dir, size, cluster):
    print("in here")
    global paper_count
    conn = sqlite3.connect(path)
    cur = conn.cursor()
    #cur.execute("SELECT count(*) FROM astro_papers_2;")
    #paper_count = cur.fetchone()[0]
    #print(type(cur.fetchone()[0]))
    #conn.close()
    #print("connected")
    #run = 0
    #while True:
    if size == 0:
        print("am doing")
        df_1 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 1980 AND year <1990 LIMIT 25;", conn)
        df_2 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 1990 AND year <2000 LIMIT 25;", conn)
        df_3 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 2000 AND year <2010 LIMIT 25;", conn)
        df_4 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 2010 AND year <2022 LIMIT 25;", conn)
        df = pd.concat([df_1, df_2, df_3, df_4], axis=0)
        kmeans_clustering(df, results_dir, folder_dir)
    elif size == 1:
        df = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2;", conn)
        #df = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers;", conn)
        #print("got different frames")
        #df = pd.concat([df_1, df_2, df_3, df_4], axis=0)
        #kmeans_cluster(df, results_dir)
        #df['bibcode'] = df['bibcode'].astype(str)
        #df['title'] = df['title'].astype(str)
        #df['author'] = df['author'].astype(str)
        #df['abstract'] = df['abstract'].astype(str)
        #print(df.columns)
        #print("----")
        #print(df.dtypes)
        #df.to_csv('/home/maleeha/research/data.csv', index=False)
        #print(df.info(verbose=False, memory_usage="deep"))
        kmeans_clustering(df, results_dir, folder_dir)
    elif size == 2:
        df_1 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 1980 AND year <1990 LIMIT 100;", conn)
        df_2 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 1990 AND year <2000 LIMIT 100;", conn)
        df_3 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 2000 AND year <2010 LIMIT 100;", conn)
        df_4 = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 WHERE year >= 2010 AND year <2022 LIMIT 100;", conn)
        df = pd.concat([df_1, df_2, df_3, df_4], axis=0)
        print("DF INFO --------->")
        print(df.info(verbose=False, memory_usage="deep"))
        print()
        kmeans_clustering(df, results_dir, folder_dir)

    conn.close()
    #return df

def parse_text(text):
    return re.findall("[a-zA-Z0-9']{2,}", text)

def tokenize(text):
    return [word for sent in sent_tokenize(text) for word in word_tokenize(sent)]

def tokenize_bigram(text):
    tokens = [word for sent in sent_tokenize(text) for word in word_tokenize(sent)]
    return list(ngrams(tokens, 2))

def tokenize_and_stem(text):
    stemmer = SnowballStemmer('english')
    tokens = [word for sent in sent_tokenize(text) for word in word_tokenize(sent)]

    filtered_tokens = []

    for token in tokens:
        if re.search('[a-zA-Z]', token):
            filtered_tokens.append(token)
    stems = [stemmer.stem(t) for t in filtered_tokens]
    return stems  # taken from: http://brandonrose.org/clustering_mobile

def process_text(df):
    #stop_words = stopwords.words('english')
    abs_text = df['abstract']
    abs_text = abs_text.str.lower()
    #abs_text = abs_text.apply(parse_text)
    #print(abs_text)

    abs_text = abs_text.apply(parse_text)
    abs_text = abs_text.apply(' '.join)

    #abs_text = abs_text.apply(stemmer.stem())

    abs_text_stem = abs_text.apply(tokenize_and_stem)
    abs_text_token = abs_text.apply(tokenize)
    abs_text_bigram = abs_text.apply(tokenize_bigram)
    #abs_text_trigram = abs_text.apply(tokenize_trigram)

    #print(df.columns)
    #print(abs_text_bigram)
    #df['abs_text'] = abs_text_stem #?
    df['abs_text'] = abs_text_token
    #df['abs_text'] = abs_text_bigram
    #sia = SentimentIntensityAnalyzer()

    # tokenized = []
    # for abs_text in df['abs_text']:
    #     abstract_str = " ".join(abs_text)
    #     tokenized.append(abstract_str)

    # df['abs_tokenized'] = tokenized

    # for abs_text in abs_text_token:
    #     for term in abs_text:
    #         vocab_tokens[term] +=1

    # for abs_text in abs_text_stem:
    #     for term in abs_text:
    #         vocab_stems[term] +=1

    # for abs_text in abs_text_bigram:
    #     for term in abs_text:
    #         vocab_bigrams[term] += 1
    return df

def trunc(number): #so
    text = f"{number:.3f}"
    return float(text)

# def stats(df):

#     stats_dict = {}

#     Qmin = np.min(df['citation_count'])
#     Q1 = np.percentile(df['citation_count'], 25, interpolation = 'midpoint')
#     Q3 = np.percentile(df['citation_count'], 75, interpolation = 'midpoint')
#     Q2 = np.median(df['citation_count'])
#     Qmax = np.max(df['citation_count'])

#     stats_dict.update({"Qmin": Qmin})
#     stats_dict.update({"Q1":Q1})
#     stats_dict.update({"Q3":Q3})
#     stats_dict.update({"Q3":Q3})
#     stats_dict.update({"Qmax":Qmax})

#     return stats_dict

# def get_data_set(vect_no, data, test):
#     if vect_no == 0:
#         vect = TfidfVectorizer(stop_words='english', strip_accents='unicode', token_pattern=r"[a-zA-Z0-9']{2,}", ngram_range=(1,2))
#         d = vect.fit_transform(data)
#         terms = vect.get_feature_names()
#         return (d, terms)

#     elif vect_no == 1:
#         count_vect = CountVectorizer(stop_words='english',strip_accents='unicode',token_pattern=r"[a-zA-Z0-9']{2,}", ngram_range=(1,2))
#         dc = count_vect.fit_transform(data)
#         terms = count_vect.get_feature_names()
#         return (dc, terms)

#     #terms = vect.get_feature_names()

# no == 0 --> lda, no == 1 --> lsi,
def topic_model(tm_no, data):
    if tm_no == 0:
        svd = TruncatedSVD(n_components=5, n_iter=7, random_state=42)
        svd.fit(data)
        data2D = svd.transform(data)
        return data2D

    elif tm_no == 1:
        lda = LatentDirichletAllocation(n_components=2, random_state=1)
        lda.fit(data)
        data_LDA = lda.transform(data)
        return data_LDA
    #X_tr = X_tr.todense()
            #pca = PCA(n_components=3).fit(X_tr)
            #d = pd.DataFrame(X_tr_c.toarray(),index=train['abstract'],columns=count_vect.get_feature_names()) # assn pdf
            #print(d.head())
            #X_tr_c = X_tr_c.todense()
            #print(X_tr_c)
            #print(len(list(count_vect.get_feature_names())))
            #print(len(count_vect.vocabulary))
            #corpus_vect = Sparse2Corpus(X_tr_c, documents_columns=False)
            #print(count_vect.vocabulary)
            #lsa_model = LsiModel(corpus_vect, id2word=count_vect.vocabulary)
            #print(lsa_model.print_topics())

def see_clusters(model, data, pdf):
    f = plt.figure()
    visualizer = SilhouetteVisualizer(model,  colors='yellowbrick')

    visualizer.fit(data)       # Fit the data to the visualizer
    #visualizer.set_title(title)
    visualizer.show()
    pdf.savefig(f)

def get_sent(df):

    sia = SentimentIntensityAnalyzer()
    neg_scores = []
    neu_scores = []
    pos_scores = []
    comp_scores = []
    abs_text = df['abstract']
    for text in abs_text:
        neg_scores.append(sia.polarity_scores(text)['neg'])
        neu_scores.append(sia.polarity_scores(text)['neu'])
        pos_scores.append(sia.polarity_scores(text)['pos'])
        comp_scores.append(sia.polarity_scores(text)['compound'])
    df['abs_sent_neg'] = neg_scores
    df['abs_sent_neu'] = neu_scores
    df['abs_sent_pos'] = pos_scores
    df['abs_sent_comp'] = comp_scores
    return df

def get_lda(df):
    df_2 = process_text(df)
    #df_2 = get_sent(df_2)
    print(df_2.head())
    train_2, test_2 = train_test_split(df_2, train_size=0.75)
    temp = df
    #df = df_2
    #getting data

    texts = list(train_2['abs_text'])
    tagged = []
    for text in texts:
        tag = nltk.pos_tag(text)
        tagged.append(tag)
    print(tagged[0])
    id2word = Dictionary(texts)
    corpus = [id2word.doc2bow(text) for text in texts]
    lda_model = LdaModel(corpus=corpus,
                        id2word=id2word,
                        num_topics=10,
                        random_state=100,
                        update_every=1,
                        chunksize=100,
                        passes=10,
                        alpha='auto',
                        per_word_topics=True)
    # Compute Perplexity
    print('\nPerplexity: ', lda_model.log_perplexity(corpus))  # a measure of how good the model is. lower the better.

    # Compute Coherence Score
    coherence_model_lda = CoherenceModel(model=lda_model, texts=texts, dictionary=id2word, coherence='c_v')
    coherence_lda = coherence_model_lda.get_coherence()
    print('\nCoherence Score: ', coherence_lda)

    #plot_sent(df_2)
    #pyLDAvis.enable_notebook()
    #vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)

    lda = LatentDirichletAllocation(n_components=7, random_state=1)
    #lda.fit(data)
    lda.fit(X_tr_count)
    for i,topic in enumerate(lda.components_):
        print(f'Top 10 words for topic #{i}:')
        print([count_vect.get_feature_names()[i] for i in topic.argsort()[-10:]])
        print('\n')
    pprint(lda_model.print_topics())

    #data_LDA_tr = lda.transform(X_tr_count)
    #data_LDA_te = lda.transform(X_te_count)

# model_no == 0 --> kmeans, model_no == 1 --> dbscan
def kmeans_clustering(df, results_dir, kmeans_dir):
    # preprocessing

    # print("Stats:")
    # r =  df['citation_count'].max() - df['citation_count'].min()
    # print("\trange of citation count:", r)

    Qmin = np.min(df['citation_count'])
    Q1 = np.percentile(df['citation_count'], 25, interpolation = 'midpoint')
    Q3 = np.percentile(df['citation_count'], 75, interpolation = 'midpoint')
    Q2 = np.median(df['citation_count'])
    Qmax = np.max(df['citation_count'])

    # print("\tmin: ", Qmin)
    # print("\tQ1: ", Q1)
    # print("\tQ2: ", Q2)
    # print("\tQ3: ", Q3)
    # print("\tmax: ", Qmax)
    # print()
    # try_size(df)
    # a_file = open(results_dir + "stats.csv", "w")
    # stats = {'range':r, "min":Qmin, "q1":Q1, "q2":Q2, "q3":Q3, "max":Qmax}
    # writer = csv.writer(a_file) #kite
    # for key, value in stats.items():
    #     writer.writerow([key, value])
    # a_file.close()

    vect = TfidfVectorizer(stop_words='english', strip_accents='unicode', token_pattern=r"[a-zA-Z0-9']{2,}", ngram_range=(1,2))
    count_vect = CountVectorizer(stop_words='english',strip_accents='unicode',token_pattern=r"[a-zA-Z0-9']{2,}", ngram_range=(1,2))

    df = get_sent(df)
    train, test = train_test_split(df, train_size=0.75)

    X_tr_tfidf = vect.fit_transform(train['abstract'])
    tfidf_terms = vect.get_feature_names()
    X_tr_count = count_vect.fit_transform(train['abstract'])
    count_terms = count_vect.get_feature_names()
    X_te_tfidf = vect.transform(test['abstract'])
    X_te_count = count_vect.transform(test['abstract'])

    svd = TruncatedSVD(n_components=5, n_iter=7, random_state=42)
    data2D_tr = svd.fit_transform(X_tr_tfidf)
    data2D_te = svd.transform(X_te_tfidf)

    # print("Data Shapes:")
    # print("X_tr_tfidf shape: ", X_tr_tfidf.shape)
    # print("X_tr_count shape: ", X_tr_count.shape)
    # print("X_te_tfidf shape: ", X_te_tfidf.shape)
    # print("X_te_count shape: ", X_te_count.shape)
    # print()
    c_nums = [3, 6, 8, 10, 15, 25]

    #clustering
    cluster_plots_pdf = PdfPages(results_dir + "cluster_plots.pdf")
    test_pred_pdf = PdfPages(results_dir + "test_pred.pdf")
    silhouette_pdf = PdfPages(results_dir + "silhouette.pdf")
    #df = temp
    #with PdfPages(results_dir + "figures.pdf") as pdf:
    with PdfPages(results_dir + "cluster_plots.pdf") as cluster_plots_pdf,\
        PdfPages(results_dir + "test_pred.pdf") as test_pred_pdf,\
        PdfPages(results_dir + "silhouette.pdf") as silhouette_pdf:
        for c in c_nums:
            print("Cluster={}".format(c))
            print()

            model_p = kmeans_dir + "model_" + str(c) + ".pkl"
            model_ori_p = kmeans_dir + "model_ori_" + str(c) + ".pkl"

            model = MiniBatchKMeans(n_clusters=c,random_state=0,batch_size=2000)
            model_ori = MiniBatchKMeans(n_clusters=c,random_state=0,batch_size=2000)

            model_ori.fit(X_tr_tfidf)
            #see_clusters(model_ori, X_tr_tfidf, "model_ori", silhouette_pdf)
            pickle.dump(model_ori, open(model_ori_p,'wb')) # machine learning mastery

            #data2D = topic_model(0, X_tr_tfidf)
            model.fit(data2D_tr)
            see_clusters(model, data2D_tr, silhouette_pdf)
            pickle.dump(model, open(model_p, 'wb'))

            new_train = pd.DataFrame(columns=train.columns)
            new_train = pd.concat([train.reset_index(drop=True), pd.DataFrame(data2D_tr)], axis=1)
            new_train['cluster'] = model.labels_

            test_pred = model_ori.predict(X_te_tfidf)
            test_pred_data2D = model.predict(data2D_te)
            #print("Test Pred _ X_te_tfidf")
            #print(test_pred)
            #print("ORI")
            silhouette_avg = silhouette_score(X_te_tfidf, test_pred) #scikitlearn silhouette
            #print("For n_clusters =", c,
            #        "The average silhouette_score is :", silhouette_avg)
            #print()
            #print("DATA2D")
            silhouette_avg = silhouette_score(data2D_te, test_pred_data2D) #scikitlearn silhouette
            #print("For n_clusters =", c,
            #       "The average silhouette_score is :", silhouette_avg)

            #print("Prediction scores on test data ", test_pred)
            #print(test.loc[0])
            test_dict = {}
            for i in range(X_te_tfidf.shape[0]):
                #print("updating test_dict ...at value {x} out of {y}".format(x=i, y=len(X_te)))
                curr = test.iloc[i]
                #print(curr)
                #break
                # print(curr, test_pred[i])
                # print(type(curr.values))
                # print(type(tuple(curr.values)))
                #x = tuple(curr.values)
                #print(x)
                # print(type(x))
                #print(type(x), type(test_pred[i]))
                test_dict.update({tuple(curr.values):test_pred[i]})
                #test_dict.update({tuple(curr.values):test_pred[i]})
                test_dict.update({(test.iloc[i]['bibcode'], test.iloc[i]['title']):test_pred[i]})
            #return
            #print("test dict ", test_dict)
            centers2D = model.cluster_centers_
            centers2D_ori = model_ori.cluster_centers_
            #print("centers 2d", centers2D, centers2D.shape)
            #print("centers 2d ori", centers2D_ori, centers2D_ori.shape)
            order_centroids = centers2D.argsort()[:, ::-1]
            order_centroids_ori = centers2D_ori.argsort()[:, ::-1]
            #print("oc " , order_centroids)
            #print("oc ori ", order_centroids_ori)
            cluster_terms = []
            #cluster_terms_ori = []
            cluster_terms_ori = {}
            for i in range(c):
                c_terms = " ".join([tfidf_terms[ind] for ind in order_centroids[i, :c]])
                cluster_terms.append(c_terms)
                c_terms_ori = " ".join([tfidf_terms[ind] for ind in order_centroids_ori[i, :c]])
                #cluster_terms_ori.append((i, c_terms_ori))
                cluster_terms_ori[i] = c_terms_ori

            c_file = open(results_dir + "c_terms" + str(c) + ".csv", "w")
            #stats = {'range':r, "min":Qmin, "q1":Q1, "q2":Q2, "q3":Q3, "max":Qmax}
            writer = csv.writer(c_file) #kite
            for key, value in cluster_terms_ori.items():
                writer.writerow([key, value])
            c_file.close()

            new_train.rename(columns={0: 'Component 1', 1 : 'Component 2', 2: 'Component 3'}, inplace=True)
            #print("columns b4 means", new_train.columns)
            #print(new_train.groupby(['cluster']).head())
            #means = new_train.groupby(['cluster']).mean()
            #print("means columns", means.columns)
            #print(means.head())
            #print("new_train.columns", new_train.columns)
            #for x in new_train.columns:
            #    print(new_train[x].dtype)
            new_train = new_train.astype({'citation_count': 'float64'})
            means = new_train.groupby(['cluster'], as_index=False)
            #means = df_new.groupby(['cluster']).mean()
            #print(means.head())
            means = means.mean()
            print("means.columns", means.columns)
            #print(means.head())

            means['citation_count'] = means['citation_count'].apply(trunc)
            means['abs_sent_neg'] = means['abs_sent_neg'].apply(trunc)
            means['abs_sent_neu'] = means['abs_sent_neu'].apply(trunc)
            means['abs_sent_pos'] = means['abs_sent_pos'].apply(trunc)
            means['abs_sent_comp'] = means['abs_sent_comp'].apply(trunc)

            print(means.head())
            #print("means", list(means.index))
            #means['cluster'] = means.index

            test_pred_df = pd.DataFrame(columns=list(df.columns) + [ 'predicted_cluster', 'expected_cit_count', 'pred_str'])

            #test_pred_df = pd.DataFrame(columns=['bibcode', 'title', 'predicted_cluster', 'expected_cit_count', 'pred_str'])
            print(test_pred_df.columns)
            #break
            p_count = 0
            arr = []
            for x in test_dict.items():
                #print(x)
                #print("here")
                m = means.loc[x[1]]
                #print(x, m)
                #print("making test_pred dictionary...at paper {p}".format(p=p_count))
                pred_str = ""
                exp_cit_count = m['citation_count']
                if exp_cit_count < Q1:
                    pred_str = "low"
                elif exp_cit_count >= Q1 and exp_cit_count <= Q3:
                    pred_str = 'medium'
                elif exp_cit_count > Q3:
                    pred_str = "high"

                try:
                    # print("testing...")
                    # print("bibcode: ", x[0][0])
                    # print("title: ",x[0][1])
                    # print("year: ", x[0][2])
                    # print("author: ",x[0][3])
                    # print("abstract: ", x[0][4])
                    # print("citation_count: ", x[0][5])
                    # print("predicted_cluster: ", x[1])
                    # print("expected_cit_count: ", exp_cit_count)
                    # print("pred_str: ", pred_str)
                    # print()

                    test_pred_df = test_pred_df.append({'bibcode':x[0][0], 'title':x[0][1], \
                        'year':x[0][2], 'author':x[0][3], 'abstract':x[0][4], 'citation_count':x[0][5],\
                            'predicted_cluster':x[1],'expected_cit_count':exp_cit_count, 'pred_str':pred_str},\
                                ignore_index=True)
                except IndexError:
                    #print(len(x))
                    arr.append(len(x))
                    test_pred_df = test_pred_df.append({'bibcode':x[0][0], 'title':x[0][1], \
                        'predicted_cluster':x[1],\
                    'expected_cit_count':exp_cit_count, 'pred_str':pred_str}, ignore_index=True)
                    #pass
                #print(x[0], "predicted cluster: ", x[1], "avg citation count of cluster {x}: ".format(x=x[1]), m['citation_count'])
                #print(m)
                p_count+=1

            # print("test pred pdf head")
            # print(test_pred_df.head())
            # print()
            #print(set(arr))

            #print("Cluster Prediction of Test Set Plus Expected Citation Count")
            #print()
            #print(test_pred_df)
            #print()
            #test_file = results_dir + "test_pred" + str(c) + ".csv"
            #test_file = results_dir + "test_pred" + str(c) + "run_{r}".format(r=run) + ".csv"
            #test_pred_df.to_csv(test_file)

            #with PdfPages("results.pdf") as pdf:
                    #name = os.path.join(path, filename)
            #print(test_pred_df.columns)
            f = plt.figure()
            # d = pd.read_csv(name, error_bad_lines=False) # skipping bad entries
            #         #d = pd.read_csv(name)
            #         print(d.head())
            #         print(d.columns)
                    #ax = sns.histplot(data=pd.read_csv(name), x='year', hue='pred_str')
            ax = sns.histplot(data=test_pred_df, x='year', hue='pred_str')
            ax.set_title("File: cluster_" + str(c) + " test prediction results")
            plt.show()
            test_pred_pdf.savefig(f)

            cluster_colors = dict()
            for l in model.labels_:
                cluster_colors[l] = (random.random(), random.random(), random.random())
            #print("creating plot 1 ....")
            #print(centers2D)

            plot_clusters(cluster_plots_pdf, c, new_train, 0, cluster_colors, centers2D, cluster_terms_ori)
            plot_clusters(cluster_plots_pdf, c, new_train, 1, cluster_colors, centers2D, cluster_terms_ori)
            plot_clusters(cluster_plots_pdf, c, new_train, 2, cluster_colors, centers2D, cluster_terms_ori)

# def dbs_cluster(df, results_dir):
#     folder_str = results_dir + "/dbs/"
#     model = DBSCAN(eps=3, min_samples=2)

def plot_clusters(pdf, c, new_train, ax_val, cluster_colors, centers2D, cluster_terms_ori):
    f = plt.figure()
    #print(centers2D)
    if ax_val == 0:
        x_axis = new_train['Component 2']
        y_axis = new_train['Component 1']
        title = "PCA Components 2 v. 1"
    elif ax_val == 1:
        x_axis = new_train['Component 3']
        y_axis = new_train['Component 2']
        title = "PCA Components 3 v. 2"
    elif ax_val == 2:
        x_axis = new_train['Component 3']
        y_axis = new_train['Component 1']
        title = "PCA Components 3 v. 1"

    #plt.figure(figsize=(10,8))
    ax = sns.scatterplot(x_axis, y_axis, hue = new_train['cluster'], palette=list(cluster_colors.values()))
    ax.legend(fontsize=6)
    xs = [p[1] for p in centers2D] # link
    ys = [p[0] for p in centers2D]
    #print(xs, ys)
    #print(cluster_terms_ori.items())
    if c == 3:
        for i, txt in enumerate(cluster_terms_ori.items()): # link
            #print(txt[1], type(txt[1]))
            text = "    " + str(i) + " " + txt[1]
            plt.annotate(text, (xs[i], ys[i]), (xs[i], ys[i]),fontsize='medium',c='k', weight='bold') #link
        plt.scatter(centers2D[:,1], centers2D[:,0], marker='*', s=125, linewidths=3, c='k')
    plt.title(("{x}-means Clusters -" + title).format(x=c))
    plt.show()
    #f.savefig("component2v1.pdf", bbox_inches='tight')
    pdf.savefig(f)

def plot_sent(df):
    plt.figure()
    df['abs_sent_neg'].plot(title='neg scores', kind='kde')
    plt.show()
    df['abs_sent_neu'].plot(title='neu scores', kind='kde')
    print
    plt.show()
    df['abs_sent_pos'].plot(title='pos scores', kind='kde')
    plt.show()
    df['abs_sent_comp'].plot(title='comp scores', kind='kde')
    plt.show()

    print("Stats: neg")
    r =  df['abs_sent_neg'].max() - df['abs_sent_neg'].min()
    print("\trange of citation count:", r)

    Qmin = np.min(df['abs_sent_neg'])
    Q1 = np.percentile(df['abs_sent_neg'], 25, interpolation = 'midpoint')
    Q3 = np.percentile(df['abs_sent_neg'], 75, interpolation = 'midpoint')
    Q2 = np.median(df['abs_sent_neg'])
    Qmax = np.max(df['abs_sent_neg'])

    print("\tmin: ", Qmin)
    print("\tQ1: ", Q1)
    print("\tQ2: ", Q2)
    print("\tQ3: ", Q3)
    print("\tmax: ", Qmax)
    print()

    print("Stats: neu")
    r =  df['abs_sent_neu'].max() - df['abs_sent_neu'].min()
    print("\trange of citation count:", r)

    Qmin = np.min(df['abs_sent_neu'])
    Q1 = np.percentile(df['abs_sent_neu'], 25, interpolation = 'midpoint')
    Q3 = np.percentile(df['abs_sent_neu'], 75, interpolation = 'midpoint')
    Q2 = np.median(df['abs_sent_neu'])
    Qmax = np.max(df['abs_sent_neu'])

    print("\tmin: ", Qmin)
    print("\tQ1: ", Q1)
    print("\tQ2: ", Q2)
    print("\tQ3: ", Q3)
    print("\tmax: ", Qmax)
    print()

    print("Stats: pos")
    r =  df['abs_sent_pos'].max() - df['abs_sent_pos'].min()
    print("\trange of citation count:", r)

    Qmin = np.min(df['abs_sent_pos'])
    Q1 = np.percentile(df['abs_sent_pos'], 25, interpolation = 'midpoint')
    Q3 = np.percentile(df['abs_sent_pos'], 75, interpolation = 'midpoint')
    Q2 = np.median(df['abs_sent_pos'])
    Qmax = np.max(df['abs_sent_pos'])

    print("\tmin: ", Qmin)
    print("\tQ1: ", Q1)
    print("\tQ2: ", Q2)
    print("\tQ3: ", Q3)
    print("\tmax: ", Qmax)
    print()

    print("Stats: comp")
    r =  df['abs_sent_comp'].max() - df['abs_sent_comp'].min()
    print("\trange of citation count:", r)

    Qmin = np.min(df['abs_sent_neu'])
    Q1 = np.percentile(df['abs_sent_comp'], 25, interpolation = 'midpoint')
    Q3 = np.percentile(df['abs_sent_comp'], 75, interpolation = 'midpoint')
    Q2 = np.median(df['abs_sent_comp'])
    Qmax = np.max(df['abs_sent_comp'])

    print("\tmin: ", Qmin)
    print("\tQ1: ", Q1)
    print("\tQ2: ", Q2)
    print("\tQ3: ", Q3)
    print("\tmax: ", Qmax)
    print()

    #for sent in df['abs_sent']:

def plot_results(path):
    #path = '/Users/Mal/Documents/results'
    with PdfPages("results.pdf") as pdf:
        for filename in os.listdir(path):
            if "test_pred" in filename:
                name = os.path.join(path, filename)
                f = plt.figure()
                d = pd.read_csv(name, error_bad_lines=False) # skipping bad entries
                #d = pd.read_csv(name)
                print(d.head())
                print(d.columns)
                #ax = sns.histplot(data=pd.read_csv(name), x='year', hue='pred_str')
                ax = sns.histplot(data=d, x='year', hue='pred_str')
                ax.set_title("File: " + name + " test prediction results")
                plt.show()
                pdf.savefig(f)

def main():
    #results_dir = "/home/maleeha/research/results/"
    run_type = sys.argv[1]
    results_dir = "/Users/Mal/Desktop/results/"
    kmeans_dir = "/Users/Mal/Desktop/results/kmeans/"
    #results_dir = "/Users/Mal/Documents/results/"

    if(os.path.exists(results_dir)==0):
        os.mkdir(results_dir)

    if(os.path.exists(kmeans_dir)==0):
        os.mkdir(kmeans_dir)
    # if(os.path.exists(kmeans_dir)==0):
    #     os.mkdir(kmeans_dir)
    folder_dir = kmeans_dir # kmeans == 0

    #path = '/Users/Mal/Desktop/sqlite/research.db'
    path = '/Users/Mal/Documents/research.db'
    #path = '/home/maleeha/research/research.db'
    #process_entries(path, results_dir, 0)
    process_entries(path, results_dir, folder_dir, int(run_type), 0)
    #path = '/Users/Mal/Desktop/resu'
    #plot_results(results_dir)
    #process_entries(path, results_dir, 1)

    # import sys
    # from resource import getrusage, RUSAGE_SELF
    # process_entries(path, results_dir, 2)
    # print("Peak memory (MiB):",
    #   int(getrusage(RUSAGE_SELF).ru_maxrss / 1024))
    # #df = process_text(df)
    # print(np.polyfit([100, 1000], [1558488, 3593720], 1))

    #print(2.26e-03 * (3769874/ 1024) + 1.33e+06)
    #kmeans_cluster(df, output_dir_, test_pred_dir, figure_dir)
    #t0 = time.time()
    #plot_words(df)
    #print("time elapsed for plot words ", time.time()-t0)
    #agg_cluster(df)

if __name__ == '__main__':
    main()

#-------------------------BELOW FROM CLUSTERING FUNCITON---------------------------------
   #        new_train = pd.concat([new_train.reset_index(drop=True), mini_df])
                    #model_ori = mini_kmeans(vect, mini_df, model_ori, 1)
            #while i <= len(train['abstract']):
            #    try:
            #        mini_df = train['abstract'][i:i+2000]
            #        X_tr = vect.fit_transform(mini_df)
            #        X_tr = X_tr.todense()
            #        pca = PCA(n_components=3).fit(X_tr)
            #        data2D = pca.transform(X_tr)
                    #model = mini_kmeans(vect, mini_df, model, 0)
            #        print("not data2d", i, X_tr.shape)
            #        print(i, data2D.shape)
            #        model = model.partial_fit(data2D)
            #        model_ori = model_ori.partial_fit(X_tr)
            #        mini_df = pd.concat([mini_df.reset_index(drop=True), pd.DataFrame(data2D)], axis=1)
            #        new_train = pd.concat([new_train.reset_index(drop=True), mini_df])
                    #model_ori = mini_kmeans(vect, mini_df, model_ori, 1)
            #        i += 2000
            #    except IndexError:
            #        break
            #df_new = pd.concat([train.reset_index(drop=True), pd.DataFrame(data2D)], axis=1)
            #df_new = pd.concat([train.reset_index(drop=True), pd.DataFrame(data2D)], axis=1)
            #print(new_train.columns)
            #print(new_train.head())
            #print("new_train.shape", new_train.shape)

    # i = 0
    # count = 0
    # while i != train.shape[0]:
    #     X_tr = vect.fit_transform(train.iloc[i:i+50]['abstract'])
    #     print("run ... ",count )
    #     print(vect.get_feature_names()[:50])
    #     count+=1
    #     i+=50

    #print(vect.get_feature_names()[:25])

    #d = pd.DataFrame(lsa.components_,index = ["component_1","component_2"],columns = count_vect.get_feature_names())
            #print(d.head())

            #pyLDAvis.sklearn.prepare(lda, X_tr_c, count_vect)

            #print(get_model_topics(lda_tfidf, vect, []))
            #print(get_model_topics(lda, count_vect, []))

            #topic, score, _ = get_inference(nmf, tfidf_vectorizer, nmf_topics, text, 0)
            #print(topic, score)
            #print(train.columns)
            #for x in train.columns:
            #    print(x, type(x))

# def try_size(df):
#     vect = TfidfVectorizer(stop_words='english', strip_accents='unicode', token_pattern=r"[a-zA-Z0-9']{2,}", ngram_range=(1,2))
#     train, test = train_test_split(df, train_size=0.90)
#     X_te = vect.fit_transform(test['abstract'])

# def mini_kmeans(vect, df, model, data):
#     X_tr = vect.fit_transform(df)
#     X_tr = X_tr.todense()
#     if data == 0:
#         pca = PCA(n_components=3).fit(X_tr)
#         data2D = pca.transform(X_tr)
#         model = model.partial_fit(data2D)
#     elif data == 1:
#         model = model.partial_fit(X_tr)
#     return model

'''
f = plt.figure()
x_axis = new_train['Component 2']
y_axis = new_train['Component 1']
#plt.figure(figsize=(10,8))
ax = sns.scatterplot(x_axis, y_axis, hue = new_train['cluster'], palette=list(cluster_colors.values()))
ax.legend(fontsize=6)
xs = [p[1] for p in centers2D] # link
ys = [p[0] for p in centers2D]
#print(xs, ys)
#print(cluster_terms_ori.items())
if c == 3:
    for i, txt in enumerate(cluster_terms_ori.items()): # link
        print(txt[1], type(txt[1]))
        text = "    " + str(i) + " " + txt[1]
        plt.annotate(text, (xs[i], ys[i]), (xs[i], ys[i]),fontsize='medium',c='k', weight='bold') #link
    plt.scatter(centers2D[:,1], centers2D[:,0], marker='*', s=125, linewidths=3, c='k')
plt.title("{x}-means Clusters - PCA components 2 v. 1".format(x=c))
plt.show()
#f.savefig("component2v1.pdf", bbox_inches='tight')
pdf.savefig(f)

#print("creating plot 2 ....")
#print(centers2D)
f2 = plt.figure()
x_axis = new_train['Component 3']
y_axis = new_train['Component 2']
#plt.figure(figsize=(10,8))
ax = sns.scatterplot(x_axis, y_axis, hue = new_train['cluster'], palette=list(cluster_colors.values()))
ax.legend(fontsize=6)
xs = [p[2] for p in centers2D] # link
ys = [p[1] for p in centers2D]
#print(xs, ys)
if c == 3:
    for i, txt in enumerate(cluster_terms_ori.items()): # link
        text = "    " + str(i) + " " + txt[1]
        plt.annotate(text, (xs[i], ys[i]), (xs[i], ys[i]),fontsize='medium',c='k', weight='bold') #link
    plt.scatter(centers2D[:,2], centers2D[:,1], marker='*', s=125, linewidths=3, c='k')
plt.title("{x}-means Clusters - PCA components 3 v. 2".format(x=c))
plt.show()
pdf.savefig(f2)
#f.savefig("component3v2.pdf", bbox_inches='tight')
#print("creating plot 3 .....")
f3 = plt.figure()
x_axis = new_train['Component 3']
y_axis = new_train['Component 1']
#plt.figure(figsize=(10,8))
ax = sns.scatterplot(x_axis, y_axis, hue = new_train['cluster'], palette=list(cluster_colors.values()))
ax.legend(fontsize=6)
xs = [p[2] for p in centers2D] # link
ys = [p[0] for p in centers2D]
#print(xs, ys)
if c == 3:
    for i, txt in enumerate(cluster_terms_ori.items()): # link
        text = "    " + str(i) + " " + txt[1]
        plt.annotate(text, (xs[i], ys[i]), (xs[i], ys[i]),fontsize='medium',c='k', weight='bold') # link
    plt.scatter(centers2D[:,2], centers2D[:,0], marker='*', s=125, linewidths=3, c='k')
plt.title("{x}-means Clusters - PCA components 3 v. 1".format(x=c))
plt.show()
pdf.savefig(f3)
#f.savefig("component3v1.pdf", bbox_inches='tight')
#print("done making plots ....")

run = 0
cmd = cur.execute("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2;")
#df_count = 0
run = 0
while True:
results = cmd.fetchmany(size=10000)
if len(results) == 0:
break
print("Paper count left ... {x}, current run: {r}".format(x=paper_count, r=run))
#cmd = cur.execute("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 LIMIT 200;")
df = pd.DataFrame(results)
df.columns = list(map(lambda x: x[0], cur.description)) #so
#df = pd.read_sql_query("SELECT  bibcode, title, year, author, abstract, citation_count FROM astro_papers_2 LIMIT 200;", conn)
t0 = time.time()
kmeans_cluster(df, run, output_dir, test_pred_dir, figure_dir)
print("time taken for 10000 ", time.time() - t0)
paper_count -= 10000
run +=1
'''


import basf2 as b2
import os

# suppress messages and warnings during processing:
# set_log_level(LogLevel.ERROR)
basf2datadir = os.path.join(os.environ.get('BELLE2_LOCAL_DIR', None), 'data')

# Register modules
particlegun = b2.register_module('ParticleGun')
# particlegun.param('randomSeed', 3452346)
# particlegun.param('randomSeed', 345)
particlegun.param('randomSeed', 346)
# The particle we are shooting
particlegun.param('pdgCodes', [11])
particlegun.param('nTracks', 1)
particlegun.param('momentumGeneration', 'uniformPt')
# particlegun.param('momentumGeneration', 'uniform')
particlegun.param('momentumParams', [1.0, 1.0])
particlegun.param('thetaGeneration', 'uniform')
# particlegun.param('thetaParams', [35, 127])
particlegun.param('thetaParams', [45, 45])
particlegun.param('phiGeneration', 'uniform')
particlegun.param('phiParams', [0, 360])
particlegun.param('vertexGeneration', 'fixed')
particlegun.param('vertexGeneration', 'normal')
particlegun.param('xVertexParams', [0, 0.0])
particlegun.param('yVertexParams', [0, 0.0])
particlegun.param('zVertexParams', [0, 0.0])

# Register modules
eventinfosetter = b2.register_module('EventInfoSetter')
eventinfoprinter = b2.register_module('Progress')
# eventinfoprinter = fw.register_module("EventInfoPrinter")
paramloader = b2.register_module('Gearbox')
geobuilder = b2.register_module('Geometry')
# geobuilder.log_level = LogLevel.INFO
g4sim = b2.register_module('FullSim')
cdcdigitizer = b2.register_module('CDCDigitizer')
out = b2.register_module('SimpleOutput')
cdctrg = b2.register_module("TRGCDC")
tsstream = b2.register_module("TRGCDCTSStream")
# mcparticle  = fw.register_module('PrintMCParticles')

# ...EventInfoSetter...
eventinfosetter.param({'EvtNumList': [10], 'RunList': [1]})

# ...CDC Trigger...
cdctrg.param('ConfigFile',
             os.path.join(basf2datadir,
                          "trg/TRGCDCConfig_0_20101111_1051.dat"))
cdctrg.param('DebugLevel', 0)
cdctrg.param('CurlBackStop', 1)
cdctrg.param('HoughFinderPerfect', 1)

# ...CDC Trigger TS Stream...
tsstream.param('DebugLevel', 2)
tsstream.param('Mode', 0)
tsstream.param('OutputStreamFile', "TRGCDCTSStream.dat")

# set mcprinter
mcparticleprinter = b2.register_module('PrintMCParticles')
mcparticleprinter.param('maxLevel', -1)

# set geometry(geobuilder)
geobuilder.param('Components', ['MagneticField', 'CDC'
                                ])

# set digitizer to no smearing
param_cdcdigi = {'Fraction': 1, 'Resolution1': 0.00, 'Resolution2': 0.0}
cdcdigitizer.param(param_cdcdigi)

# Create paths
main = b2.create_path()

# Add modules to paths
main.add_module(eventinfosetter)
main.add_module(eventinfoprinter)
main.add_module(paramloader)
main.add_module(geobuilder)
main.add_module(particlegun)
main.add_module(mcparticleprinter)
main.add_module(g4sim)
main.add_module(cdcdigitizer)
main.add_module(cdctrg)
main.add_module(tsstream)
# main.add_module(out)

# Process events
b2.process(main)

# Print call statistics
print(b2.statistics)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb  16 19:25:11 2019

@author: YASH
"""

import tweepy
from textblob import TextBlob
import pandas as pd
import re
import matplotlib.pyplot as plt

# Step 1 - Authenticate
def authenticate():
    consumer_key= 'CONSUMER_KEY'
    consumer_secret= 'CONSUMER_KEY_SECRET'
    access_token='ACCESS_TOKEN'
    access_token_secret='ACCESS_TOKEN_SECRET'
    try:
        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token, access_token_secret)
        api = tweepy.API(auth)
        return api
    except:
        print("Authentication Failed")

def preprocess_tweet(tweet):
	return re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)|(http:\S+)|(RT)", '', tweet)

def get_sentiment(score):
    if score>0:
        return 'Positive'
    elif score<0:
        return 'Negative'
    else:
        return 'Neutral'

def create_report(data):
    neg_tweets=0
    pos_tweets=0
    neu_tweets=0
    for index, row in data.iterrows():
        if row['Polarity']=='Positive':
            pos_tweets=pos_tweets+1
        elif row['Polarity']=='Negative':
            neg_tweets=neg_tweets+1
        else:
            neu_tweets=neu_tweets+1
    summ_data = {'Class':['Positive','Negative', 'Neutral'],
                 'Count':[pos_tweets, neg_tweets,neu_tweets]}
    df = pd.DataFrame(summ_data)
    plt.figure(figsize=(16,8))
    # plot chart
    ax1 = plt.subplot(121, aspect='equal')
    df.plot(kind='pie', ax=ax1, autopct='%1.1f%%',
     startangle=90, shadow=False, labels=df['Class'], legend = False, fontsize=14, y='Count')
    plt.title("Sentiment Distribution Chart")
    plt.show()

#Take input from command line
api = authenticate()
#Step 3 - Retrieve Tweets
query=input("Please enter search term:")
public_tweets = api.search(q=query, count=500)
tweets =[]
polarity=[]
scores=[]
tweets_analysis=dict()
for tweet in public_tweets:
    #clean tweet
    tweet_txt=preprocess_tweet(tweet.text)
    tweets.append(tweet_txt)
    analysis= TextBlob(tweet.text)
    scores.append(analysis.sentiment.polarity)
    polarity.append(get_sentiment(analysis.sentiment.polarity))
tweets_analysis={'Tweet':tweets,'Score':scores, 'Polarity':polarity}
data =pd.DataFrame(tweets_analysis)
data.to_csv('report.csv',header=True, index=True)
create_report(data)


#Игра в рыцаря и дракончиков с полным управлением по осям х и y
import random
from tkinter import *
window = Tk()

w = 600
h = 600
window.geometry(str(w) + 'x' + str(h))

canvas = Canvas(window, width=w, height=h)
canvas.place(in_=window, x=0, y=0)

bg_photo = PhotoImage(file='bg_2.png')

class Knight:
    def __init__(self):
        self.x = 150
        self.y = h//2
        self.yv = 0
        self.xv = 0
        self.photo = PhotoImage(file='knight.png')

    def up(self, event):
        self.yv = -5

    # Движение вниз
    def down(self, event):
        self.yv = 5

    # Остановка
    def stop(self, event):
        self.yv = 0
        self.xv = 0

    def right(self, event):
        self.xv = 5

    def left(self, event):
        self.xv = -5

class Dragon:
    def __init__(self):
        self.x = 750
        self.y = random.randint(100, 500)
        self.v = random.randint(1, 3)
        self.photo = PhotoImage(file='dragon.png')

knight = Knight()

dragons = []
for i in range(3):
    dragons.append(Dragon())

def game():
    canvas.delete('all')
    canvas.create_image(300, 300, image=bg_photo)
    canvas.create_image(knight.x, knight.y, image=knight.photo)
    knight.y += knight.yv
    knight.x += knight.xv
    if knight.y < 50:
        knight.y = 51
    elif knight.y > 565:
        knight.y = 564

    current_dragon = 0

    dragon_to_kill = -1

    for dragon in dragons:
        canvas.create_image(dragon.x, dragon.y, image=dragon.photo)
        dragon.x -= dragon.v

        if ((dragon.x - knight.x) ** 2) + ((dragon.y - knight.y) ** 2) <= (96) ** 2:
            dragon_to_kill = current_dragon

        current_dragon += 1

        if dragon.x <= 0:
            canvas.delete('all')
            canvas.create_text(w // 2, h // 2, text="You lose!", font="Verdana 42", fill='red')
            break

    if dragon_to_kill >= 0:
        del dragons[dragon_to_kill]

    if len(dragons) == 0:
        canvas.delete('all')
        canvas.create_text(w // 2, h // 2, text="You win!", font="Verdana 42", fill='green')
    else:
        window.after(5, game)

game()

window.bind('<Key-Up>', knight.up)
window.bind('<Key-Down>', knight.down)
window.bind('<KeyRelease>', knight.stop)
window.bind('<Key-Right>', knight.right)
window.bind('<Key-Left>', knight.left)

window.mainloop()

Модуль 2. Урок 4. Анимация в Tkinter.py

Модуль 2. Урок 4. Анимация в Tkinter.py


alphabet = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']

def caesen(text, shift, dir):
  textt = ""

  if dir == 'de':
    shift *= -1

  for i in text:
    pos = alphabet.index(i)

    new_pos = pos + shift
    textt += alphabet[new_pos]

  print(f"The {dir} text is {textt}")

dir = input("Type encode to encypt and decode to decrypt: ")
text = input("Type your message: ").lower()
shift = int (input("Type shift num: "))

caesen(text, shift, dir)


## Ex 5-1. QPushButton.
## https://wikidocs.net/21934

import sys
from PyQt5.QtWidgets import QApplication, QWidget, QPushButton, QVBoxLayout

class MyApp(QWidget):

    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        btn1 = QPushButton('&Button1', self)
        btn1.setCheckable(True)
        btn1.toggle()
        # btn1.toggled.connect(self.activateBtn)

        btn2 = QPushButton(self)
        btn2.setText('Button&2')

        btn3 = QPushButton('Button3', self)
        btn3.setEnabled(False)

        vbox = QVBoxLayout()
        vbox.addWidget(btn1)
        vbox.addWidget(btn2)
        vbox.addWidget(btn3)

        self.setLayout(vbox)
        self.setWindowTitle('QPushButton')
        self.setGeometry(300, 300, 300, 200)
        self.show()

    # def activateBtn(self, state):
    #     if self.btn1.isChecked == 1:
    #         self.btn3.setEnabled(True)
    #     else:
    #         self.btn3.setEnabled(False)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = MyApp()
    sys.exit(app.exec_())


t = int(input())

for i in range(t):
    n = int(input())
    arr = list(map(int, input().split()))
    arr.sort()
    left, right = 1, len(arr) - 1
    blueSum = arr[0] + arr[1]
    redSum = arr[-1]

    result = "NO"

    while left < right:
        if blueSum < redSum:
            result = "YES"
            break
        left += 1
        right -= 1
        blueSum += arr[left]
        redSum += arr[right]

    print(result)


import requests
from rdflib import DCTERMS, RDF, SKOS, Graph, Literal, Namespace, URIRef

from dawe_vocabs.schemas import LUTSchema
from dawe_vocabs.vocabs.common import get_local_uuid_name

class NoDataInAPIException(Exception):
    """This is raised when no data is received from the API."""

    pass

class UnexpectedDataShapeException(Exception):
    """This is raised when the shape of the data (JSON) is not the expected shape."""

def create(
    base_uri: Namespace, g: Graph, config: LUTSchema, parent_collection_uri: str
) -> None:
    r = requests.get(config.endpoint_url)
    r.raise_for_status()

    data = r.json()

    if isinstance(data, list):
        # Strapi3
        rows = data
    else:
        if data.get("data"):
            # Strapi4
            rows = data["data"]
        else:
            rows = None
            # raise UnexpectedDataShapeException(
            #     "Expected the JSON object to have a 'data' key but none was found."
            # )

    if not rows:
        raise NoDataInAPIException(
            f"API endpoint {config.endpoint_url} returned no data."
        )

    concepts = list()

    for row in rows:
        if row.get("attributes") and row["attributes"].get("uuid"):
            local_name = row["attributes"]["uuid"]
        else:
            # Use this temporarily until the "uuid" field is available from the API.
            local_name = get_local_uuid_name(f"{config.uuid_namespace}-{row['id']}")
        concept_uri = base_uri[local_name]
        concepts.append(concept_uri)

        if row.get("attributes"):
            label = row["attributes"]["label"]
            symbol = row["attributes"]["symbol"]
            if row["attributes"].get("tree_description"):
                description = (
                    "Tree description: "
                    + row["attributes"]["tree_description"]
                    + ".  "
                    + row["attributes"]["shrub_description"]
                    + ". "
                )
            else:
                description = row["attributes"]["description"]
        else:
            label = row["label"]
            description = row["description"]
            symbol = row["symbol"]

        g.add((concept_uri, RDF.type, SKOS.Concept))
        g.add((concept_uri, SKOS.prefLabel, Literal(label)))
        g.add((concept_uri, SKOS.definition, Literal(description)))
        g.add((concept_uri, SKOS.notation, Literal(symbol)))
        g.add((concept_uri, DCTERMS.identifier, Literal(row["id"])))
        g.add(
            (
                concept_uri,
                DCTERMS.source,
                URIRef(config.endpoint_url),
            )
        )

    # Create the collection
    collection_uri = base_uri[config.collection_uuid]
    g.add((collection_uri, DCTERMS.source, URIRef(config.endpoint_url)))
    g.add((URIRef(parent_collection_uri), SKOS.member, collection_uri))
    g.add((collection_uri, RDF.type, SKOS.Collection))
    g.add((collection_uri, SKOS.prefLabel, Literal(config.label)))
    g.add(
        (
            collection_uri,
            DCTERMS.description,
            Literal(config.description),
        )
    )
    for concept in concepts:
        g.add((collection_uri, SKOS.member, concept))


import torch
import torch.nn as nn

class LatentEmbeddingModel(nn.Module):
    def __init__(self, n_user, n_item, n_embed):
        super(LatentEmbeddingModel, self).__init__()

        # embedding layer (B, 2) -> (B, 2, N_EMBED)
        self.embedding_user = nn.Embedding(n_user, n_embed)
        self.embedding_item = nn.Embedding(n_item, n_embed)

        # global avg, bias
        self.mu = nn.Embedding(1, 1)
        self.b_user = nn.Embedding(n_user, 1)
        self.b_item = nn.Embedding(n_item, 1)

    def forward(self, x):
        # x (B, 2)
        x_user = self.embedding_user(x[:, 0].long())                # (B, N_EMBED)
        x_item = self.embedding_item(x[:, 1].long())                # (B, N_EMBED)

        mu = self.mu(torch.tensor([0]*x.shape[0])).view(-1)         # (B)
        b_user = self.b_user(x[:, 0].long()).view(-1)               # (B)
        b_item = self.b_item(x[:, 1].long()).view(-1)               # (B)

        # x = torch.sum(x_user * x_item, dim=1)  # (B)
        x = mu + b_user + b_item + torch.sum(x_user*x_item, dim=1)  # (B)
        return x


from typing import List

import requests

from slackhooks.blocks.block import Block

class Message:
    def __init__(self, text: str = None, blocks: List[Block] = None):
        self.text: str = text
        self.blocks: List[Block] = blocks or []

    def serialize(self):
        return {
            "text": self.text,
            "blocks": list(
                map(
                    lambda block: block.serialize(),
                    self.blocks,
                )
            ),
        }

    def send(self, webhook_url: str):
        return requests.post(
            url=webhook_url,
            json=self.serialize(),
        )


# -*- coding: utf-8 -*-
import matplotlib
import matplotlib.pyplot as plt
import numpy as np

matplotlib.rcParams['axes.unicode_minus'] = False
# np.random.seed(0)  有这一行，程序每次生成相同的随机数
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

vmin_law = 60

vmin_driver = vmin_law+1

def Run1d(path=5000, n=100, v0=80, ltv=120, p_wander=0, p_slow_down=0.3,rou=0, times=3000):
    #  p_slow_down叫做随机慢速的概率
    '''
    path = 5000.0 # 道路长度,每个元胞长度7m,因此道路长度模拟的是35km的一段
    n = 100 # 车辆数目
    v0 = 80 # 初始速度,高速公路不得低于60km/h
    ltv = 120 # 最大限速
    p = 0.3 # 减速概率
    rou = 0 # 自动车的比例
    times = 3000 # 模拟的时刻数目,每个时间单位代表现实世界当中的25.2s
    '''

    # x保存每辆车在道路上的位置，随机初始化
    x = np.random.randint(0,path,n)
    x.sort()
    # v保存每辆车的速度，初速度相同

    # 添加自动车的下标数组
    auto_list=np.random.randint(0,n-1,int(n*rou))

    v = np.ones((n,),dtype=int) * v0

    plt.figure(figsize=(5, 4), facecolor='w')
    # 模拟每个时刻

    for t in range(times):

        plt.scatter(x*7, [t*25.2] * n, s=1, c='k', alpha=0.05)
        # 模拟每辆车
        for i in range(n):
            # 计算当前车与前车的距离
            if x[(i + 1) % n] > x[i]:
                d = x[(i + 1) % n] - x[i]
            else:
                d = path - x[i] + x[(i + 1) % n]
            # 根据距离计算下一秒的速度

            temp_p_slow_down=p_slow_down
            if (auto_list == i).any():
                temp_p_slow_down=0
            if v[i] < d:
                if np.random.rand() > temp_p_slow_down:
                    v[i] += 1
                else:
                    v[i] -= 1
            else:

                v[i] = d - 1

        # 对速度进行限制
        v = v.clip(vmin_driver, ltv)

        # if np.random.rand() < p_wander:
        #     v[i]-=1

        # 一秒后，车辆的位置发生了变化
        x += v
        # 注意,周期型边界条件
        x = x % path

    # 展示
    plt.xlim(0, path*7)
    plt.ylim(0, times*25.5)
    plt.xlabel(u'车辆位置m')
    plt.ylabel(u'模拟时间s')
    plt.title(u'交通模拟(车道长度%dm,车辆数%d,初速度%skm/h,减速概率%s)' % (path*7, n, v0, p_slow_down))

    plt.savefig('shikongtu.png')

def Run1d_withoutplot(path=5000, n=100, v0=60, ltv=120, p_wander=0, p_slow_down=0.3, rou=0, times=300):
    # 可以假设随机慢速均匀分布   p_slow_down叫做随机慢速
    '''
    path = 5000.0 # 道路长度
    n = 100 # 车辆数目
    v0 = 60 # 初始速度
    ltv = 120 # 最大限速
    p = 0.3 # 减速概率
    rou = 0 # 自动车的比例
    times = 3000 # 模拟的时刻数目
    '''

    # x保存每辆车在道路上的位置，随机初始化
    x = np.random.randint(0, path, n).astype('float64')
    x.sort()
    # v保存每辆车的速度，初速度相同

    # 添加自动车的下标数组
    auto_list = np.random.randint(0, n - 1, int(n * rou))

    v = np.ones(n) * v0

    # v_mean = np.zeros_like(v)

    q_num=0
    # 运行公路一段时间
    for t in range(times):
        # print(x[1])

        # 模拟每辆车的行为，包括自动车与普通车

        for i in range(n):

            # 计算当前车与前车的距离
            if x[(i + 1) % n] > x[i]:
                d = x[(i + 1) % n] - x[i]
            else:
                d = path - x[i] + x[(i + 1) % n]
            # 根据距离计算下一秒的速度

            # 自动车的行为是不会随机慢速
            temp_p_slow_down = p_slow_down

            if (auto_list == i).any():
                temp_p_slow_down = 0
            if v[i] < d:
                if np.random.rand() > temp_p_slow_down:
                    v[i] += 1
                else:
                    v[i] -= 1
            else:

                v[i] = d - 1

        # 对速度进行限制
        v = v.clip(vmin_driver, ltv)

        # if np.random.rand() < p_wander:
        #     v[i]-=1

        # 一秒后，车辆的位置发生了变化
        x += v

        # q_num是每个时间出车的数目，即流量

        q_sum_array=np.where(x>path,1,0)

        q_num +=np.sum(q_sum_array)

        # 注意,周期型边界条件
        x = x % path

        # v_mean+=v

    # v_mean/=times

    q_num/=times

    # q=np.mean(v_mean)

    return q_num

class Car:
    x=0
    y=0
    v=0

def Run2d_withoutplot(path=5000, n=200, v0=60, ltv=120,p_left_and_right=0.5,p_wander=0, p_slow_down=0.3, rou=0, times=300,p_huan=0.2):
    # 可以假设随机慢速均匀分布   p_slow_down叫做随机慢速
    '''
    path = 5000.0 # 道路长度
    n = 200 # 车辆数目
    v0 = 60 # 初始速度
    ltv = 120 # 最大限速
    p = 0.3 # 减速概率
    rou = 0 # 自动车的比例
    times = 3000 # 模拟的时刻数目
    '''

    # x保存每辆车在道路上的x轴位置坐标
    x = np.random.randint(0, path, n)
    x.sort()
    # 每辆车在道路上的y轴位置坐标随机给出
    y = np.random.binomial(1,p_left_and_right,size=n)

    #保存道路快照
    car_distribution = np.zeros((path,2))

    car_distribution[x,y]=1

    # 添加自动车的下标数组
    auto_list = np.random.randint(0, n, int(n * rou))
    # v保存每辆车的速度，初速度相同
    v = np.ones((n,),dtype=int) * v0

    # v_mean = np.zeros_like(v)

    q_num=0

    # 运行公路一段时间
    for t in range(times):

        # 模拟每辆车的行为，包括自动车与普通车

        for i in range(n):

            # x.sort()

            auto_flag=(auto_list == i).any()#为真是自动车

            # if x[(i + 1) % n] > x[i]:
            #     d = x[(i + 1) % n] - x[i]
            # else:
            #     d = path - x[i] + x[(i + 1) % n]

            # #首先进行特例的判断
            # if x[i]+d>path-1:

            #像人的视线一样计算当前车与前车的距离以及和另一个车道上前车的距离，如果另一个车道上前车距离更大那么有一定几率换道

            d1=1 #当前道路

            d2=1 #另一个道路

            while car_distribution[(x[i]+d1)%path,y[i]]==0 and d1<=120:

                d1=d1+1

            while car_distribution[(x[i]+d2)%path,1-y[i]]==0 and d2<=120:

                d2=d2+1

            if d2>d1:

                # 有一定几率按换道规则进行换道

                if np.random.rand()<p_huan:

                    y[i]=1-y[i]

                    d=d2

                else:

                    d=d1

            else:

                d=d1

            # 根据距离计算下一秒的速度

            # 自动车的行为是不会随机慢速
            temp_p_slow_down = p_slow_down

            if auto_flag:
                temp_p_slow_down = 0
            if v[i] < d:
                if np.random.rand() > temp_p_slow_down:
                    v[i] += 1
                else:
                    v[i] -= 1
            else:

                v[i] = d - 1

        # 对速度进行限制
        v = v.clip(vmin_driver, ltv)

        # if np.random.rand() < p_wander:
        #     v[i]-=1

        # 一秒后，车辆的位置发生了变化

        x += v

        # q_num是每个时间出车的数目，即流量

        q_sum_array=np.where(x>path,1,0)

        q_num +=np.sum(q_sum_array)

        # 周期型边界条件
        x = x % path

        # 更新快照
        car_distribution = np.zeros((path,2))

        car_distribution[x,y]=1

        # v_mean+=v

    # v_mean/=times

    q_num/=times

    # q=np.mean(v_mean)

    return q_num


def manipulate(s):
	plus = s.find("+")
	at = s.find("@")
	if plus == -1:
		return s[0:at].replace('.', '') + s[at:]
	return s[0:plus].replace('.', '') + s[at:]

def count_users(input_file="input.txt"):
    with open(input_file) as f:
        lines = f.readlines()
    number_of_lines = int(lines[0])
    users = set(manipulate(email) for email in lines[1:])
    return len(users)