import datetime
import requests
import json
import re
import os
import csv
from src.models.Uploads import Uploads
from src.models import Emails as Email
from src.views import Logger

BASE = os.getcwd()
LOGGER = Logger.init_logger(__name__)

def clean_filename(course_id: str, fname: str):
    """Cleans up given filename, removing undesired punctuation
    Args:
        course_id (str): course id
        fname (str): file name to clean up
    Returns:
        str: either cleaned filename or course_id if filename is over 160 chars long
    """
    LOGGER.debug(f"{course_id}, {fname}")
    handle_types(*[(course_id, ""), (fname, "")])

    file = re.sub(r"[^\w\s]", "", fname)
    filename = file.replace("recording", "").rstrip().replace(" ", "-")
    return course_id if len(filename) > 160 else filename

def create_dir_if_not_exists(path: str):
    """Creates directory if it doesn't exist already
    Args:
        path (str): path to directory
    Returns:
        str: path to directory
    """
    LOGGER.debug(f"{path}")
    handle_types(*[(path, "")])
    try:
        if not os.path.exists(path):
            os.makedirs(path)
    except Exception as e:
        msg = str(e)
        LOGGER.error(msg)
        raise e

    return path

def calculate_time(secs: int):
    """Converts given number of seconds into "%H:%M:%S" format
    Args:
        secs (int): number of seconds
    Returns:
        str: "%H:%M:%S" time format
    """
    LOGGER.debug(f"{secs}")
    handle_types(*[(secs, 0)])
    mins, secs = divmod(secs, 60)
    hours, mins = divmod(mins, 60)
    days, hours = divmod(hours, 24)
    time_in_session = datetime.time(hours, mins, secs)
    return time_in_session.strftime("%H:%M:%S")

def convert_date(date: str):
    """Converts given date to "%b %d,%Y" format. Example : Sep 03,2021
    Args:
        date (str): date
    Returns:
        str: date in "%b %d,%Y" format
    """
    LOGGER.debug(f"{date}")
    handle_types(*[(date, "")])
    try:
        date_obj = datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ")
    except ValueError:
        msg = "Incorrect data format, should be %Y-%m-%dT%H:%M:%S.%fZ"
        LOGGER.error(msg)
        raise ValueError(msg)
    return date_obj.strftime("%b %d,%Y")

def set_search_start_date(weeks: int):
    """Returns date {weeks} from now
    Args:
        weeks (int): number of weeks to go back
    Returns:
        str: date in "%Y-%m-%d" format
    """
    LOGGER.debug(f"Setting search from {weeks} weeks ago")
    handle_types(*[(weeks, 0)])

    start_date = datetime.datetime.now() - datetime.timedelta(weeks=int(weeks))
    return start_date.strftime("%Y-%m-%d")

def get_headers(auth_or_bearer):
    """Helper function - template for returning headers to make API calls depending on auth or bearer being required
    Args:
        auth_or_bearer (str): String variable for header
    Returns:
        [dict]: Header for making requests
    """
    LOGGER.debug(f"{auth_or_bearer}")
    handle_types(*[(auth_or_bearer, "")])

    return {
        "Authorization": auth_or_bearer,
        "Content-Type": "application/json",
        "Accept": "application/json",
    }

def clean_chat_message(message):
    """Removes unwanted whitespace and characters from Collab chat entries prior to saving
    Args:
        message (str): Chat message on given recording
    Returns:
        [str]: Cleaned message
    """
    LOGGER.debug(f"{message}")
    handle_types(*[(message, "")])
    clean = re.compile("<.*?>")
    return re.sub(clean, "", message)

def save_chatfile_as_csv(fname, json_info, header):
    file = open(fname, "w", encoding="utf-8")
    writer = csv.writer(file)
    writer.writerow(header)
    for json_row in json_info:
        writer.writerow(
            [
                json_row["userName"],
                clean_chat_message(json_row["body"]),
                json_row["relativeEventTime"],
                json_row["id"],
            ]
        )
    file.close()

def list_entry(results, size=None, recording_type=0):
    """Generates download report entry data depending on recording found or not found
    Returns:
        [dict]: Dict of data for entry to Collab download report
    """
    LOGGER.debug(f"{results}, {size}, {recording_type}")
    handle_types(*[(results, {}), (recording_type, 0)])

    if not type(size) is int and size != None:
        msg = f"Recording size should be type(int) and not None"
        LOGGER.error(msg)
        raise TypeError(msg)

    try:
        if recording_type == 1:
            list_data = {
                "recording_id": results["id"],
                "recording_name": results["name"],
                "duration": results["duration"],
                "storage_size": size,
                "created": results["created"],
            }
        else:
            list_data = {
                "recording_id": results["id"],
                "recording_name": results["name"],
                "msg": 403,
            }
    except KeyError as ke:
        raise KeyError(f"Expected key in dict: {ke}")

    return list_data

def recording_storage_size(url: str):
    """returns content length for given recording url
    Args:
        url (str): recording url
    Returns:
        int: size of content length
    """
    LOGGER.debug(f"{url}")
    handle_types(*[(url, "")])

    try:
        r = requests.get(url, stream=True, headers={"Accept-Encoding": None})
    except requests.exceptions.MissingSchema as e:
        LOGGER.error(str(e))
        raise requests.exceptions.MissingSchema(str(e))

    return int(r.headers.get("content-length", 0))

def return_filename(course_id, rec, filetype, chat=""):
    LOGGER.debug(f"{course_id}, {rec}, {filetype}, {chat}")
    handle_types(*[(course_id, ""), (rec, {}), (filetype, ""), (chat, "")])

    try:
        return (
            f"Collab-2-Panopto-",
            f"{chat}"
            f"{course_id}-{rec['recording_id']}_"
            f"{clean_filename(course_id, rec['recording_name'])}"
            f"{filetype}"
        )

    except KeyError as ke:
        raise KeyError(f"Expected key in dict: {ke}")

def get_courseId_folderId_pairs(uuid_id_pairs):
    """Takes dict of uuid:course_id mappings from Collab data, uses course_id to search
        for corresponding Panopto folder_id
    Args:
        uuid_id_pairs ([dict]): uuid to course_id mappings
    Returns:
        [dict]: Course_id to Panopto folder_id mappings
    """
    LOGGER.debug(f"{uuid_id_pairs}")
    handle_types(*[(uuid_id_pairs, {})])

    uploader = Uploads()
    courseId_folderId_pairs = check_json_exists()

    missing_folders_list = []
    for courseId in uuid_id_pairs.values():
        if courseId in courseId_folderId_pairs.keys():
            continue
        else:
            folderId = uploader.get_folder_id_from_course_id(courseId)
            if folderId is not None:
                courseId_folderId_pairs[courseId] = folderId
            else:
                missing_folders_list.append(courseId)

    save_as_json(courseId_folderId_pairs, f"{BASE}/data")

    # message = Email.create_email()
    # email = Email.attach_missing_folders_csv(message, missing_folders_list)
    # Email.send_info_email(email)

    return courseId_folderId_pairs, missing_folders_list

def save_as_json(courseId_folderId_pairs, json_dir_path):
    """Saves or updates courseId_folderId_pairs as json to /data folder
    Args:
        courseId_folderId_pairs (dict): Dict of course_id to Panopto folder_id mappings
        json_dir_path (path): Path to /data folder
    """
    LOGGER.debug(f"{courseId_folderId_pairs}, {json_dir_path}")
    handle_types(*[(courseId_folderId_pairs, {}), (json_dir_path, "")])
    dir = create_dir_if_not_exists(json_dir_path)
    with open(f"{dir}/courseId_folderId_pairs.json", "w") as fp:
        json.dump(courseId_folderId_pairs, fp, indent=4)

    return courseId_folderId_pairs

def get_downloads_list(downloads_path):
    """Gets list of mp4 files in Collab /downloads folder
    Args:
        downloads_path (path): Path to Collab downloads folder
    Returns:
        [list]: List of paths to each mp4 download file
    """
    LOGGER.debug(f"{downloads_path}")
    handle_types(*[(downloads_path, "")])

    downloads_list = []
    for file in os.scandir(downloads_path):
        if file.path.endswith(".mp4"):
            downloads_list.append(file.path)

    return downloads_list

def get_folder_download_matches(courseIds_folderIds, downloads_list):
    """ Matches course_id strings from mp4 files in Collab /downloads to Panopto folder_id
        Adds {folder_id: [download]} to matches dict if not exists, else appends download
    Args:
        courseIds_folderIds (dict): Dict of course_id:folder_id mappings
        downloads_list (list(mp4)): List of mp4 recordings in Collab /downloads
    Returns:
        [dict]: Mapping of course downloads to corresponding Panopto folder_ids
    """
    LOGGER.debug(f"{courseIds_folderIds}, {downloads_list}")
    handle_types(*[(courseIds_folderIds, {}), (downloads_list, [])])

    matches = {}
    for download in downloads_list:
        for course_id, folder_id in courseIds_folderIds.items():
            if course_id in download:
                if folder_id in matches:
                    matches[folder_id].append(download)
                else:
                    matches[folder_id] = [download]
    return matches

def logger_msg(*vars):
    return [str(type(var)) for var in vars]

def handle_response(r):
    """Generic function for API response handling and error logging.
    Args:
        r (res): Response from API call
    Returns:
        Parsed json or None, depending on status_code
    """
    if r.status_code == 200:
        res = json.loads(r.text)
        return res
    elif r.status_code == 403:
        LOGGER.warning(f"Status code 403: (might be private recording!) - {r}")
        return None
    elif r.status_code == 404:
        LOGGER.warning(f"Status code 404: Not Found - {str(r)}")
        return None
    else:
        LOGGER.warning(f"Unknown response for: {str(r)}")
        return None

def handle_types(*received_expected):
    """Generic helper function for handling TypeErrors
    Args:
        rec_exp (list(tuples)): Unpackable list of (rec, exp) tuples of arbitrary length
    Raises:
        TypeError: if type(rec) != type(exp)
    """
    for pair in received_expected:
        if not type(pair[0]) is type(pair[1]):
            msg = f"Type {type(pair[1])} expected, received: {type(pair[0])}"
            LOGGER.error(msg)
            raise TypeError(msg)

def handle_res_content(key, json):
    if not key in json:
        raise KeyError(f"Expected key {key} in response json")

def check_json_exists():
    """Loads courseId_folderId_pairs.json as dict if exists, returns empty dict if not
    Returns:
        [dict]: dict(courseId_folderId_pairs)
    """
    try:
        with open(f"{BASE}/data/courseId_folderId_pairs.json") as f:
            courseId_folderId_pairs = json.load(f)
    except:
        courseId_folderId_pairs = {}

    return courseId_folderId_pairs

def return_absent_present(present_on_ppto, list_of_recordings):
    absent, present = [], []

    # Turn list of dicts into a single dict
    dict_present_recordings = {}
    for item in present_on_ppto:
        dict_present_recordings[list(item.keys())[0]] = list(item.values())[0]

    # Check if recording is already present on panopto
    for rec in list_of_recordings:
        rec_short_name = get_new_name(os.path.basename(rec))

        # Check for both the short name and the full name.
        # Full name can be on panopto when we are checking before the recording has been renamed
        if not rec_short_name in list(
            dict_present_recordings.keys()
        ) and not os.path.basename(rec) in list(dict_present_recordings.keys()):
            # Neither were found, the recording is absent on panopto
            absent.append(rec)
        else:
            # Present with short name
            if rec_short_name in dict_present_recordings:
                present.append(
                    {rec_short_name: dict_present_recordings[rec_short_name]}
                )
            # Present with full name
            else:
                present.append(
                    {
                        os.path.basename(rec): dict_present_recordings[
                            os.path.basename(rec)
                        ]
                    }
                )

    return absent, present

def check_upload_results(failed_uploads, recordings_not_uploaded):
    if len(recordings_not_uploaded) > 0:
        LOGGER.warning(
            f"Some recordings weren't uploaded properly : {recordings_not_uploaded }"
        )
    if failed_uploads != recordings_not_uploaded:
        LOGGER.warning(
            f"failed_uploads and recordings_not_uploaded should be identical - they aren't!!!"
        )

def get_date_created(path, rec_ids_date_created):
    for ele in rec_ids_date_created:
        for id, date in ele.items():
            if id in path:
                return date.rsplit("T", 1)

def get_new_name(old_name):
    return old_name.rsplit("_", 1)[1]


import torch.nn as nn

class ModelCNN(nn.Module):
    def __init__(self, in_channel = 1):
        super(ModelCNN, self).__init__()

        self.in_dim = 28 * 28 * 3
        self.out_dim = 10
        self.keep_prob = 0.5

        c1 = 6
        c2 = 16
        c3 = 64

        self.layer1 = 0
        if in_channel == 3:
            self.layer1 = nn.Sequential(
                nn.Conv2d(in_channels=in_channel, out_channels= c1, kernel_size=3, stride=1, padding=1), # 28 * 28 * 6
                nn.BatchNorm2d(c1),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2) # 14 * 14 * 6
                )
        else:
            self.layer1 = nn.Sequential(
                nn.Conv2d(in_channels=in_channel, out_channels= c1, kernel_size=3, stride=1, padding=1), # 28 * 28 * 6
                nn.BatchNorm2d(c1),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2) # 14 * 14 * 6
                )

        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels=c1, out_channels= c2, kernel_size=3, stride=1), # 12 * 12 * 16
            nn.BatchNorm2d(c2),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2) # 6 * 6 * 16
            )

        self.layer3 = nn.Sequential(
            nn.Conv2d(in_channels=c2, out_channels= c3, kernel_size=3, stride=1), # 4 * 4 * 64
            nn.BatchNorm2d(c3),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2) # 2 * 2 * 64
            )

        self.fc4 = nn.Linear(2 * 2 * 64, 120, bias=True)
        nn.init.xavier_uniform_(self.fc4.weight)
        self.layer4 = nn.Sequential(
            self.fc4,
            nn.ReLU()
            ,nn.Dropout(p = 1 - self.keep_prob)
        )

        self.fc5 = nn.Linear(120, 80, bias=True)
        nn.init.xavier_uniform_(self.fc5.weight)
        self.layer5 = nn.Sequential(
            self.fc5,
            nn.ReLU()
            ,nn.Dropout(p = 1 - self.keep_prob)
        )

        self.fc6 = nn.Linear(80, self.out_dim, bias=True)
        nn.init.xavier_uniform_(self.fc6.weight)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = self.layer3(out)
        out = out.view(out.size(0), -1)

        out = self.layer4(out)
        out = self.layer5(out)

        out = self.fc6(out)
        return out


import numpy as np
import pandas as pd

SH_index = np.genfromtxt('SH_index.csv', dtype=float,delimiter=',')
SH_index_date = np.genfromtxt('SH_index_date.csv', dtype=str,delimiter=',')
'''
for i in range(0,len(SH_index)):
    print(SH_index_date[i],"today's index is", SH_index[i])
'''

def generate(data, d):

    p = pd.DataFrame({
    "Price": data
    })
    p["Event"] = ''
    event = 'upturn'
    ph = p['Price'][0] # highest price
    pl = ph # lowest price
    Buy = np.zeros(3426,int)
    Sell = np.zeros(3426,int)
    Transactiontime = 0
    Profit = 0
    Loss = 0
    maxdrawdown = 0
    maxprofit = 0
    for i in range(0, len(p)):
        if event is 'upturn':
            if p['Price'][i] <= (ph * (1 - d)):
                event = 'downturn'
                pl = p['Price'][i]
                Sell[i] = 1
                Transactiontime = Transactiontime + 1
            else:
                if ph < p['Price'][i]:
                    ph = p['Price'][i]
        else:
            if p['Price'][i] >= (pl * (1 + d)):
                event = 'upturn'
                ph = p['Price'][i]
                Buy[i] = 1
                Transactiontime = Transactiontime + 1
            else:
                if pl > p['Price'][i]:
                    pl = p['Price'][i]
        '''print('Asset is',Asset,'Total Asset is',TotalAsset)'''
    winorlose = 0
    bet = 0
    wintime = 0
    time= 0
    Return = np.zeros(Transactiontime)
    for i in range(0, len(p)-1):
        if (Buy[i]==1 or Sell[i]==1) :
            winorlose = winorlose - Buy[i] * p['Price'][i] + Sell[i] * p['Price'][i]
            bet = bet + 1
        if(bet==2):
            Return[time] = winorlose
            time= time + 1
            if (winorlose>=0):
                wintime=wintime+1
                Profit = Profit + winorlose
                maxprofit=max(maxprofit,winorlose)
            else:
                maxdrawdown=min(maxdrawdown,winorlose)
                Loss = Loss + winorlose
            bet=0
            winorlose=0
    print("the Transaction time and wintime is",Transactiontime,wintime)
    print("At the end, the total return is",Profit+Loss)
    print("the maximum drawdown is",-(maxdrawdown-maxprofit)/maxprofit)
    print("the winning rate is",wintime*2/Transactiontime)
    print("the profit factor is",-Profit/Loss)
    '''
    np.savetxt('return.csv', Return, fmt='%f', delimiter=',')
    '''
    return p['Event']

generate(SH_index, 0.2)


# Hackos : 50
from collections import OrderedDict

n = int(input())
ordereddict = OrderedDict()
for i in range (n):
    name = input()
    if name not in ordereddict.keys():
        ordereddict[name] = 1
    else :
        ordereddict[name] += 1

print(len(ordereddict))
for v in ordereddict.values():
    print (v, end = ' ')


import sys
import csv

def main():
    if len(sys.argv) < 3:
        sys.exit("Too few comand-line arguments!")
    elif len(sys.argv) > 3:
        sys.exit("Too many comand-line arguments!")
    elif not sys.argv[1].endswith(".csv"):
        sys.exit("Not a python file")
    else:
        before = sys.argv[1]
        after = sys.argv[2]
        convert_file(before, after)

def convert_file(x, y):
    try:
        with open(x, "r") as input_file:
            reader = csv.DictReader(input_file)
            with open(y, "w", newline="" ) as output_file:
                writer = csv.DictWriter(output_file, fieldnames= ["first", "last", "house"])
                writer.writeheader()
                for row in reader:
                    l_name, f_name = row["name"].split(", ")
                    house = row["house"]
                    writer.writerow({"first": f_name, "last": l_name, "house": house})
    except (FileNotFoundError, PermissionError, ValueError, UnicodeDecodeError, csv.Error) as e:
            sys.exit(f"Could not read {x} {e}")

if __name__ == "__main__":
    main()


import os
import errno
import fs
from fs.walk import Walker
from fnmatch import fnmatch
from fs.mountfs import MountFS, MountError
from fs.multifs import MultiFS
from fs.path import abspath, forcedir, normpath
from psslib.contentmatcher import ContentMatcher
from psslib.utils import istextfile
from psslib.driver import _build_match_context_dict, LINE_MATCH, LINE_CONTEXT
from typing import Any, List, Optional, Union, Tuple
from flask import send_file, stream_with_context, Response
from airflow_code_editor.utils import read_mount_points_config, get_plugin_config, get_plugin_int_config

__all__ = [
    "RootFS",
]

STAT_FIELDS = [
    "st_mode",
    "st_ino",
    "st_dev",
    "st_nlink",
    "st_uid",
    "st_gid",
    "st_size",
    "st_atime",
    "st_mtime",
    "st_ctime",
]

SEND_FILE_CHUNK_SIZE = 8192

def split(pathname: str):
    "Split a pathname, returns tuple (head, tail)"
    pathname = pathname.rstrip("/")
    i = pathname.rfind("/") + 1
    if i == 0:
        return ("/", pathname)
    else:
        return pathname[: i - 1], pathname[i:]

class RootFS(MountFS):
    "Root filesystem with mountpoints"

    def __init__(self):
        super().__init__()
        mounts = read_mount_points_config()
        # Set default fs (root)
        self.default_fs = MultiFS()
        self.tmp_fs = fs.open_fs("mem://")
        self.default_fs.add_fs("tmp", self.tmp_fs, write=False, priority=0)
        self.root_fs = [fs.open_fs(v.path) for v in mounts.values() if v.default][0]
        self.default_fs.add_fs("root", self.root_fs, write=True, priority=1)
        # Mount other fs
        for k, v in mounts.items():
            if not v.default:
                self.mount("/~" + k, fs.open_fs(v.path))

    def mount(self, path, fs_):
        "Mounts a host FS object on a given path"
        if isinstance(fs_, str):
            fs_ = fs.open_fs(fs_)
        path_ = forcedir(abspath(normpath(path)))
        for mount_path, _ in self.mounts:
            if path_.startswith(mount_path):
                raise MountError("mount point overlaps existing mount")
        self.mounts.append((path_, fs_))
        # Create mountpoint on the temporary filesystem
        self.tmp_fs.makedirs(path_, recreate=True)

    def path(self, *parts: List[str]):
        "Return a FSPath instance for the given path"
        return FSPath(*parts, root_fs=self)

    def find_files(
        self,
        path: str = "/",
        filter: Union[List[str], str, None] = None,
        exclude: Optional[List[str]] = None,
        max_depth: Optional[int] = None,
    ):
        "Walk a filesystem, yielding FSPAth"
        if exclude == []:
            exclude = None
        if isinstance(filter, str):
            filter = [filter]
        walker = Walker(
            ignore_errors=True,
            filter=filter,
            exclude=exclude,
            exclude_dirs=exclude,
            max_depth=max_depth,
        )
        fs = self.root_fs
        for filename in walker.files(fs=fs, path=path):
            yield self.path(filename)

    def search(
        self,
        query: str,
        search_context: Optional[int] = None,
        path: str = "/",
        filter: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        max_depth: Optional[int] = None,
    ):
        "Search for pattern in files"
        if search_context is None:
            search_context = get_plugin_int_config("search_context")
        if exclude is None:
            exclude = get_plugin_config("ignored_entries").split(",")

        result = []
        matcher = ContentMatcher(pattern=query.encode("utf-8"))
        for path in self.find_files(filter=filter, exclude=exclude, max_depth=max_depth):
            try:
                with path.open("rb") as f:
                    if istextfile(f):
                        f.seek(0)
                        matches = list(matcher.match_file(f))
                        if not search_context:
                            for match in matches:
                                context = match[0].decode("utf-8")
                                row_number = match[1]
                                result.append(
                                    {
                                        "row_number": row_number,  # matching row number
                                        "context_first_row": row_number,  # context first row number
                                        "context": context,  # context (matching row)
                                        "path": path.path,  # file path
                                    }
                                )
                        else:
                            for row, context, context_first_row in prepare_search_context(f, matches, search_context):
                                result.append(
                                    {
                                        "row_number": row,  # matching row number
                                        "context_first_row": context_first_row,  # context first row number
                                        "context": context,  # context
                                        "path": path.path,  # file path
                                    }
                                )
            except (OSError, IOError):
                pass

        return result

def prepare_search_context(f, matches, search_context) -> Tuple[int, str, int]:
    f.seek(0)
    match_context_dict = _build_match_context_dict(matches, search_context, search_context)
    lines = []
    context_first_row = 0
    row_number = 0

    prev_was_blank = False
    had_context = False
    for current_row_number, current_line in enumerate(f, 1):
        current_line = current_line.decode("utf-8")
        result, match = match_context_dict.get(current_row_number, (None, None))
        if result is None:
            prev_was_blank = True
            continue

        elif result == LINE_MATCH:
            row_number = current_row_number
            if current_row_number < context_first_row or context_first_row == 0:
                context_first_row = current_row_number
            lines.append(current_line)

        elif result == LINE_CONTEXT:
            if prev_was_blank and had_context:
                context = "".join(lines)
                yield row_number, context, context_first_row
                lines = []
                context_first_row = current_row_number
            if context_first_row < context_first_row or context_first_row == 0:
                context_first_row = current_row_number
            lines.append(current_line)
            had_context = True
        prev_was_blank = False

    if lines:
        context = "".join(lines)
        yield row_number, context, context_first_row

class FSPath(object):
    def __init__(self, *parts: List[str], root_fs: RootFS) -> None:
        self.root_fs = root_fs
        self.path = os.path.join("/", *parts)

    def open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None):
        "Open the file pointed by this path and return a file object"
        return self.root_fs.open(
            self.path,
            mode=mode,
            buffering=buffering,
            encoding=encoding,
            errors=errors,
            newline=newline,
        )

    @property
    def name(self) -> str:
        "The final path component"
        return split(self.path)[1]

    @property
    def parent(self):
        "The logical parent of the path"
        return self.root_fs.path(split(self.path)[0])

    def touch(self, mode=0o666, exist_ok=True):
        "Create this file"
        return self.root_fs.touch(self.path)

    def rmdir(self) -> None:
        "Remove this directory"
        self.root_fs.removedir(self.path)

    def unlink(self, missing_ok: bool = False) -> None:
        "Remove this file"
        try:
            self.root_fs.remove(self.path)
        except fs.errors.ResourceNotFound:
            if not missing_ok:
                raise FileNotFoundError(self.path)

    def delete(self) -> None:
        "Remove this file or directory"
        if self.is_dir():
            self.rmdir()
        else:
            self.unlink()

    def stat(self):
        "File stat"
        info = self.root_fs.getinfo(self.path, namespaces=["stat"])
        if not info.has_namespace("stat"):
            return os.stat_result([None for _ in STAT_FIELDS])
        return os.stat_result([info.raw["stat"].get(field) for field in STAT_FIELDS])

    def is_dir(self) -> bool:
        "Return True if this path is a directory"
        try:
            return self.root_fs.isdir(self.path)
        except Exception:
            return False

    def resolve(self):
        "Make the path absolute"
        return self.root_fs.path(os.path.realpath(self.path))

    def exists(self):
        "Check if this path exists"
        return self.root_fs.exists(self.path)

    def iterdir(self, show_ignored_entries=False):
        "Iterate over the files in this directory"
        ignored_entries = get_plugin_config("ignored_entries").split(",")
        mount_points = [x[0].rstrip("/") for x in self.root_fs.mounts]
        for name in sorted(self.root_fs.listdir(self.path)):
            skip = False
            if not show_ignored_entries:
                fullpath = os.path.join(self.path, name)
                # Skip mount points
                if fullpath in mount_points:
                    skip = True
                # Ship hidden files
                for patter in ignored_entries:
                    if fnmatch(fullpath if patter.startswith("/") else name, patter.strip()):
                        skip = True
            if not skip:
                yield self.root_fs.path(self.path, name)

    def size(self) -> Optional[int]:
        "Return file size for files and number of files for directories"
        try:
            if self.is_dir():
                return len(self.root_fs.listdir(self.path))
            else:
                return self.root_fs.getsize(self.path)
        except fs.errors.FSError:
            return None

    def move(self, target) -> None:
        "Move/rename a file or directory"
        target = self.root_fs.path(target)
        if target.is_dir():
            self.root_fs.move(self.path, (target / self.name).path)
        else:
            self.root_fs.move(self.path, target.path)

    def read_file_chunks(self, chunk_size: int = SEND_FILE_CHUNK_SIZE):
        "Read file in chunks"
        with self.root_fs.openbin(self.path) as f:
            while True:
                buffer = f.read(chunk_size)
                if buffer:
                    yield buffer
                else:
                    break

    def send_file(self, as_attachment: bool):
        "Send the contents of a file to the client"
        if not self.exists():
            raise FileNotFoundError(errno.ENOENT, "File not found", self.path)
        elif self.root_fs.hassyspath(self.path):
            # Local filesystem
            if as_attachment:
                # Send file as attachment (set Content-Disposition: attachment header)
                try:
                    # flask >= 2.0 -  download_name replaces the attachment_filename
                    return send_file(
                        self.root_fs.getsyspath(self.path),
                        as_attachment=True,
                        download_name=self.name,
                    )
                except TypeError:
                    return send_file(
                        self.root_fs.getsyspath(self.path),
                        as_attachment=True,
                    )
            else:
                return send_file(self.root_fs.getsyspath(self.path))
        else:
            # Other filesystems
            response = Response(stream_with_context(self.read_file_chunks()))
            if as_attachment:
                content_disposition = "attachment;filename={}".format(self.name)
                response.headers["Content-Disposition"] = content_disposition
            return response

    def write_file(self, data: Union[str, bytes], is_text: bool) -> None:
        "Write data to a file"
        self.root_fs.makedirs(self.parent.path, recreate=True)
        if is_text:
            self.root_fs.writetext(self.path, data)
        else:
            self.root_fs.writebytes(self.path, data)

    def read_text(self, encoding=None, errors=None) -> str:
        "Get the contents of a file as a string"
        return self.root_fs.readtext(self.path, encoding=encoding, errors=errors)

    def read_bytes(self) -> bytes:
        "Get the contents of a file as bytes"
        return self.root_fs.readbytes(self.path)

    def __str__(self) -> str:
        return self.path

    def __repr__(self) -> str:
        return self.path

    def __truediv__(self, key):
        try:
            path = os.path.join(self.path, key)
            return self.root_fs.path(path)
        except TypeError:
            return NotImplemented

    def __eq__(self, other) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path == other.path

    def __hash__(self) -> int:
        try:
            return self._hash
        except AttributeError:
            self._hash = hash(self.path)
            return self._hash

    def __lt__(self, other: Any):
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path < other.path

    def __le__(self, other: Any) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path <= other.path

    def __gt__(self, other: Any) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path > other.path

    def __ge__(self, other: Any) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path >= other.path


def encryption(message, shifter_key):
    encryption_result = ""

    # go through every character

    for i in range(len(message)):
        letter = message[i]

        if (letter.isupper()):
            encryption_result += chr((ord(letter) + shifter_key - 65)%26 + 65)
        else:
            encryption_result += chr((ord(letter) + shifter_key - 97)%26 + 97)

    return encryption_result

message = input("mesajınızı giriniz:")
shifter_key = int(input("key seçiniz:"))
print("Encrypted message:", encryption(message, shifter_key))


import os
import sys
import copy
import re
from collections import deque
import operator

from utils.modify import Modify
from utils.infos import Infos
from utils.varCheck import VarCheck
from utils.replace import Replace
from utils.slicing import Slicing

class Merge:

	def mergeReduce(self, filename, fileContent, argvs):
	    if argvs:
	        customTag = argvs.split(',')
	    else:
	        customTag = []
	    modifyF = Modify()

	    fileContent = modifyF.deleteEmit(fileContent)
	    fileContent = modifyF.deleteMulDiv(fileContent)

	    # STEP 1 Merge
	    getInfo = Infos()
	    contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
	    #print(mainContract)
	    isAbnormal = False
	    if mainContract == "NULLNULLNULL":
	    	isAbnormal = True
	    if isAbnormal:
	    	return isAbnormal, False

	    contractGraph = {}
	    for item in contractList:
	        contractGraph[item['contract']] = item['inherit']

	    searchPath = getInfo.inheritChain(contractList, contractGraph)

	    fileContent = modifyF.replaceSuper(searchPath, contractFunctionDict, fileContent)

	    modSearchOrder, functionSearchOrder = getInfo.subChain(modDict, contractFunctionDict, searchPath)
	    fileContent = modifyF.eventList(fileContent, searchPath, contractDict)

	    #update dict info due to delete of event statement
	    contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
	    fileContent = modifyF.moveConstructFunc(contractFunctionDict, mainContract, fileContent)

	    contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
	    statementDict, contractGlobalVar = getInfo.varStatement(contractList, contractDict, modDict, contractFunctionDict, contractConstructorDict, fileContent)

	    isHighRisk = False
	    checkVar = VarCheck()
	    isHighRisk = checkVar.check(contractDict, contractFunctionDict, statementDict, searchPath, fileContent)
	    if isHighRisk:
	    	return isAbnormal, isHighRisk

	    fileContent = modifyF.EPluribusUnum(modSearchOrder, functionSearchOrder, contractList, contractDict, modDict, contractFunctionDict, mainContract, contractConstructorDict, statementDict, searchPath, fileContent)

	    #delete constructor
	    contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
	    fileContent = modifyF.delConstructor(fileContent, contractConstructorDict, mainContract)

	    contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
	    contractElemLibDict = getInfo.libBindingDict(fileContent, libDict, libFunctionDict, contractDict)

	    replaceF = Replace()
	    fileContent = replaceF.functionReplace(contractDict, fileContent, modSearchOrder, modDict, contractFunctionDict, searchPath, contractElemLibDict, libFunctionDict)

	    contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
	    fileContent, saveList = modifyF.saveRelContract(mainContract, contractDict, fileContent)

	    contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
	    # STEP 2 Slicing
	    Slicer = Slicing()
	    reservedList = Slicer.reserve(fileContent, mainContract, contractDict, contractList, contractFunctionDict, customTag, saveList)

	    #step 4

	    newFile = "merged_" + filename
	    op = open(newFile, "w+")

	    # Write
	    op.write(str(contractList.index(mainContract)) + "\n")
	    op.write(mainContract['contract']+"\n")

	    for i in reservedList:
	        if i['line'].strip():
	            op.write(i['line'])

	    #for i in fileContent:
	    #    op.write(i)

	    #Close files
	    op.close()

	    return False, False


from typing import Tuple, Dict, List

import numpy as np
import pandas as pd
import torch

from BiLSTM.bilstm import get_all_masked
from DST.loader import Loader
from DataConstants import DataConstants
from STISE.module import improved_ses_all, improved_idw, ElmNet

def stise(eTag):
    dc = DataConstants(eTag)
    cities, indicator, stations = dc.getPublicData()

    for city in cities:
        loader = Loader(city, indicator, eTag)
        _, __, ___, ____, _____, ______, X_test_row_index, X_test_column_index = loader.train_test_loader(
            0.3, 128)

        masked_data = pd.read_csv(r'./data/' + eTag + '/' + city + '-' + indicator + '-masked.csv')
        extracted_data = pd.read_csv(r'./data/' + eTag + '/' + city + '-' + indicator + '-extracted.csv')
        # the furthest station is removed in the experiment 'e84'
        if eTag in ['e84', 'e86']:
            location_data = pd.read_csv(r'./data/' + city + "-station-location-selected.csv")
        else:
            location_data = pd.read_csv(r'./data/' + city + "-station-location.csv")

        # get indexes of training data and test data respectively
        X_test_row_index, X_test_column_index = \
            X_test_row_index.astype(int).tolist(), X_test_column_index.astype(int).tolist()
        X_test_index = [(r, c) for r, c in zip(X_test_row_index, X_test_column_index)]
        X_train_index = list(set(get_all_masked(masked_data)) - set(X_test_index))

        # Transform indexes into training sets and test sets
        """Obtain trainable or predictable samples according to the row and column index"""
        site_list = [p for p in masked_data.columns if 'A' in p]
        temp_result = improved_ses_all(masked_data, site_list=site_list)
        spat_result = improved_idw(raw_data=masked_data, sites_location=location_data, point_list=site_list)
        x_train, y_train = [], []
        for x_idx in X_train_index:
            row_index, col_index = x_idx
            if np.isnan(temp_result.iloc[row_index, col_index]) or np.isnan(spat_result.iloc[row_index, col_index]):
                continue
            x_train.append([temp_result.iloc[row_index, col_index], spat_result.iloc[row_index, col_index]])
            y_train.append(extracted_data.iloc[row_index, col_index])
        x_train, y_train = torch.tensor(x_train).to(torch.float), torch.tensor(y_train).to(torch.float)

        elm_net = ElmNet()
        elm_net.fine_ture(x_train, y_train)
        elm_net.eval()

        with torch.no_grad():
            for x_idx in X_test_index:
                row_index, col_index = x_idx
                x = torch.tensor([temp_result.iloc[row_index, col_index], spat_result.iloc[row_index, col_index]])
                y = float(elm_net.forward(x))
                masked_data.iloc[row_index, col_index] = y

        site_list = [p for p in masked_data.columns if 'A' in p]
        for i, r in masked_data.iterrows():
            nan_list, un_nan_list = [], []
            for site in site_list:
                if np.isnan(r[site]):
                    nan_list.append(site)
                else:
                    un_nan_list.append(site)
            if len(nan_list) == 0 or len(un_nan_list) == 0:
                continue
            new_r = r.copy(deep=True)
            new_r[nan_list] = float(new_r[un_nan_list].values.mean())
            masked_data.iloc[i, :] = new_r
        # masked_data = masked_data.interpolate(method='linear', limit_direction='both')

        for train_index in X_train_index:
            row_index, col_index = train_index
            masked_data.iloc[row_index, col_index] = np.nan

        infilled_file = "./results/" + eTag + '/' + city + "-" + indicator + "-stise-filling.csv"
        masked_data.to_csv(infilled_file, index=False)


import numpy as np
import random
from matrixvariables import main_metrix  # This is the matrix which contain all the characters like "a","b"... etc.
from matrixvariables import indexing_metrix  # This is the matrix which contain the numbers 0,1,2,3,4,5,6,7,8 in a form of 3*3 matrix

# This function basically returns three parameters i.e. a rearranged version of the indexing metrix and the order of the rows and columns
def indexencoding1():
    i = [0, 1, 2]
    random.shuffle(i)
    res = indexing_metrix[:, i]
    j = [0, 1, 2]
    random.shuffle(j)
    res = res[j]
    return i, j, res

# encodestep1:-
# This function basically shuffle the matrix in a random way to get different outputs every single time.
# This function first rearrange the columns of the main matrix (res = metrix[:, i]) then rearrange the rows of the matrix(result = res[x]).
# Everytime the function is called it rearranges the main matrix in a new form.
# tis function returns three parameters i.e. the order of the shuffled column and row with the rearranged matrix as result
def encodestep1():
    i = [0, 1, 2, 3, 4, 5, 6, 7, 8]
    random.shuffle(i)
    res = main_metrix[:, i]
    x = [0, 1, 2, 3, 4, 5, 6, 7, 8]
    random.shuffle(x)
    result = res[x]
    return i, x, result

# encodestep2:-
# This function takes two parameters one is the coordinates or in other word a list that contains the indexes of the elements you wnt to find and the second one is the matrix.
# It basically take the indexes from the list and search the elements at that indexes in the provided matrix.
# It returns a string which is made out off up the sticking the elements found at that index in to a string
def encodestep2(codinates, ciphermetrix):
    cipherlist = []
    for a in codinates:
        temp = str(ciphermetrix[a[0], a[1]])
        cipherlist.append(temp)
    cipherstring = "".join([str(x) for x in cipherlist])
    return cipherstring

# coordinates:-
# This function basically returns the coordinates of the element(arr) present in the given matrix(metrix1)
# cod is a list of the coordinates of the elements
def coordinates(arr, metrix1):
    cod = []
    for i in arr:
        temp = np.argwhere(metrix1 == i)
        tempa = temp.tolist()
        cod.append(*tempa)
    return cod

message = list(str(input("Enter the message: ")))  # this will take the message as input from the user
cl, ro, met = encodestep1()  # This function is called to get a random version of the main matrix and the order of the rows and column
co = coordinates(message, met)  # This function is called to get the initial coordinates of the individual character of the message
cl1, ro1, met1 = encodestep1()  # This encodestep1 is again called to get a new matrix
ciphertext = encodestep2(co, met1)  # This function is called to get the cipher text by getting the elements at same coordinates where the message elements were there
# This next part have used mainly to add a extra bit of encryption by changing the places of the indexes of column and rows which are present in the starting and ending of the cipher text
incls, inros, inmex = indexencoding1()  # This line calls the indexencoding1 function to return the shuffled form of the indexing matrix and the way the 3*3 matrix is arranged (column nad row)
# These next four lines basically do the same thing i.e. these lines will call the coordinates function which will return the coordinates of the stc(start cipher) and enc(end cipher)
cocls = coordinates(cl, inmex)  # The belong to start cipher
coros = coordinates(ro, inmex)  # The belong to start cipher
cocle = coordinates(cl1, inmex)  # The belong to end cipher
coroe = coordinates(ro1, inmex)  # The belong to end cipher
incle, inroe, inmexe = indexencoding1()  # This line calls the indexencoding1 function to return a new shuffled matrix so that the coordinates of the previous element we have will be applied and gather random numbers
# These next four lines basically takes the coordinates of the previous elements with the reshuffled matrix and returns the element that are present in the exact coordinates of the previous elements in the new matrix
encodcls = encodestep2(cocls, inmexe)  # This is the encoded form of cls
encodros = encodestep2(coros, inmexe)  # This is the encoded form of ros
encodcle = encodestep2(cocle, inmexe)  # This is the encoded form of cle
encodroe = encodestep2(coroe, inmexe)  # This is the encoded form of roe
# This next two lines will convert the four list into two separate strings that we will attach later
secretcodes = "".join([str(x) for x in incls]) + "".join([str(x) for x in inros])  # This line stitches the two lists i.e. incls and inros into one string
secretcodee = "".join([str(x) for x in incle]) + "".join([str(x) for x in inroe])  # This line stitches the two lists i.e. incle and inroe into one string
# This next two lines will convert the four list into two separate strings that we will attach later
stc = "".join([str(x) for x in encodcls]) + "".join([str(x) for x in encodros])    # This line stitches the two lists i.e. encodcls and encodros into one string
enc = "".join([str(x) for x in encodcle]) + "".join([str(x) for x in encodroe])    # This line stitches the two lists i.e. encodcle and encodroe into one string
# The next line prints the text, then first the stc(encodes start cipher) then the cipher text then anc(encoded end cipher)
print("Your encoded message is : ", stc + ciphertext + enc)
# This next line prints a secret key (The order in which after arranging the indexing_matrix you could recode the start cipher and the end cipher)
print("Your private key is : ", secretcodes + secretcodee)


# -*- coding: utf-8 -*-
__author__ = 'Paul'

from zope.interface import implements,Interface
from zope.schema.interfaces import IContextSourceBinder
from zope.schema.interfaces import IVocabularyFactory
from zope.schema.vocabulary import SimpleVocabulary,SimpleTerm
from Products.CMFCore.utils import getToolByName
#from arcas.content.coleccion import IColeccion
from zope.site.hooks import getSite

class IListaColecciones(Interface):
    """Marca para la utilidad """

class ColecAsignadasVocab(object):
    implements(IVocabularyFactory)
    def __init__(self):
        pass

    def __call__(self,context):
        contexto=getSite()
        catalogo= getToolByName(contexto, 'portal_catalog')
        #results=catalogo.unrestrictedSearchResults({'object_provides': IColeccion.__identifier__,'review_state':('SetUp','Publicado')})
        results=catalogo.unrestrictedSearchResults({'portal_types': "arcas.coleccion",'review_state':('SetUp','Publicado')})
        terms = []
        autenti=getSite().portal_membership.getAuthenticatedMember().id

        if len(results)>0:
            for colec in results:
                terms.append(SimpleVocabulary.createTerm(colec.id, str(colec.id), colec.Title))

        if len(terms)==0:
            terms.append(SimpleVocabulary.createTerm("", "", ""))

        return SimpleVocabulary(terms)

class ListaColecciones(object):
    """
    Devuelve un listado con las colecciones activas
    """
    implements(IVocabularyFactory)
    def __init__(self):
        pass

    def __call__(self,context):
        contexto=getSite()
        catalogo= getToolByName(contexto, 'portal_catalog')
        #results=catalogo.unrestrictedSearchResults({'object_provides': IColeccion.__identifier__,'review_state':('SetUp','Publicado')})
        results=catalogo.unrestrictedSearchResults({'portal_types': "arcas.coleccion",'review_state':('SetUp','Publicado')})
        terms = []
        autenti=getSite().portal_membership.getAuthenticatedMember().id

        if len(results)>0:
            for colec in results:
                terms.append(SimpleVocabulary.createTerm(colec.id, str(colec.id), colec.Title))
        else:
            terms.append(SimpleVocabulary.createTerm("", "", ""))

        return SimpleVocabulary(terms)

ColecAsignadaVocabFactory=ColecAsignadasVocab()
ColeccionesVocabFactory = ListaColecciones()


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def invertTree(self, root: TreeNode) -> TreeNode:
        if root == None:
            return None

        queue = []
        queue.append(root)

        while queue:
            current = queue.pop(0)
            if current.left:
                queue.append(current.left)
            if current.right:
                queue.append(current.right)
            tmp = current.left
            current.left = current.right
            current.right = tmp

        return root

def printTree(node: TreeNode, levelShift: str=''):
    print(levelShift, node.val, ":")
    if node.left:
        printTree(node.left, levelShift + "  ")
    if node.right:
        printTree(node.right, levelShift + "  ")

    return

s = Solution()
t = TreeNode(4,TreeNode(2,TreeNode(1,None,None),TreeNode(3,None,None)),TreeNode(7,TreeNode(6,None,None),TreeNode(9,None,None)))
res = s.invertTree(t)
printTree(res)


class Node:
    def __init__(self, data):
        self.data = data
        self.left = None
        self.right = None

class Stack:
    def __init__(self):
        self.stacklist = []

    def push(self, data):
        self.stacklist.append(data)

    def pop(self):
        return self.stacklist.pop()

exp = "3 2 + 5 3 - *".split()
stack = Stack()

for i in exp:
    if i in "+-*/":
        node = Node(i)
        node.right = stack.pop()
        node.left = stack.pop()
    else:
        node = Node(int(i))
    stack.push(node)

def calc(node):
    if node.data == "+":
        return calc(node.left) + calc(node.right)
    elif node.data == "-":
        return calc(node.left) - calc(node.right)
    elif node.data == "*":
        return calc(node.left) * calc(node.right)
    elif node.data == "/":
        return calc(node.left) / calc(node.right)
    else:
        return node.data

root = stack.pop()
ans = calc(root)
print(ans)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Written by Gregory Norton
# July 28 2018

import argparse
import os
import serial
import sys

from options import process

if sys.version_info[0] >= 3:
    raw_input = input

#port_name = '/dev/ttyACM0'  # Arduino Mega 2560 and leonardo
#port_name = '/dev/ttyUSB0'  # Arduino Uno R3

def main(args):
    parser = argparse.ArgumentParser(description='Load bytes into computer memory')
    parser.add_argument('script', nargs='?', help='The script file to execute')
    parser.add_argument('-port', '-p', nargs=1, default='/dev/ttyACM0', help='The serial port to send commands to')
    #parser.print_help()

    args = parser.parse_args()
    #print args

    port = serial.Serial(args.port)
    port.read(size=1)
    #port.write(chr(0xFF))
    #print(ord(port.read(size=1)))
    #if args.script != None and not os.path.isfile(args.script):

    #lineno = 0

    if args.script == None:
        while True:
            try:
                command = raw_input('? ').strip()
            except EOFError:
                break
            if process(command, port) == -1:
                break;
    else:
        if not file_exists(args.script):
            print('Script does not exist')
            port.close()
            return 1
        with open(args.script) as f:
            for command in f:
                if process(command.strip(), port) == -1:
                    break;
    port.close()
    return 0

if __name__ == '__main__':
    import sys
    sys.exit(main(sys.argv))


from datetime import datetime
from elasticsearch import Elasticsearch as ES, RequestsHttpConnection
from requests_aws4auth import AWS4Auth

class ESSearch:
    def __init__(self, connection, es_index='covid_tweets'):
        aws_auth = AWS4Auth(connection['ACCESS_KEY'], connection['SECRET_KEY'], 'us-east-2', 'es')
        self.es = ES(
            hosts=[{'host': connection['AWS_HOST'], 'port': 443}],
            http_auth=aws_auth,
            use_ssl=True,
            verify_certs=True,
            connection_class=RequestsHttpConnection,
            timeout=200
        )
        self.es_index = es_index

    def format_query(self, keywords, startDateString=None, endDateString=None, tweettype=None, user=None):
        if len(keywords) == 0:
            keywords=None
        queries = []
        time_range = {}
        if startDateString:
            startDate = datetime.strptime(startDateString, '%m/%d/%Y')
            time_range['gte'] = startDate.strftime('00-%d-%m-%Y')
        if endDateString:
            endDate = datetime.strptime(endDateString, '%m/%d/%Y')
            time_range['lte'] = endDate.strftime('23-%d-%m-%Y')
        if time_range:
            queries.append({
                'range': {
                    'date': time_range
                }
            })
        if keywords:
            queries.append({
                "query_string": {
                    "query": keywords,
                    "fields" : ["text"],
                    "default_operator" : "or"
                }
            })
        if user:
            queries.append({
                "match": {
                    "user_name" : user
                }
                })
        if tweettype:
            queries.append({
                "terms": {
                    "tweet_type" : tweettype,
                }
            })
        print(queries)
        return queries

    def get_doc(self, tweet_id):
        retval = self.es.get(index=self.es_index, id=tweet_id, doc_type='document')
        return retval

    def get_user_tweet(self, keywords, startDateString=None, endDateString=None, tweettype=None, user=None):
        queries = self.format_query(keywords, startDateString=startDateString, endDateString=endDateString, tweettype=tweettype, user=user)
        total_qry = { "size" : 250, 'query' : {'bool': {'must' : queries
        }}}
        retval = self.es.search(index=self.es_index, doc_type='document', body=total_qry)
        return retval

    def count(self, keywords, startDateString=None, endDateString=None, tweettype=None, user=False):
        queries = self.format_query(keywords, startDateString, endDateString, tweettype)
        total_qry = {}
        retval = {}
        if user:
            total_qry = {"size" : 0,
                        'query': {'bool': {'must': queries }},
                        "aggs" : {
                            "users_count" : {
                                "cardinality" : {
                                    "field" : "user_name"
                                }
                            }
                        }}
            retval = self.es.search(index=self.es_index, doc_type='document', body=total_qry)
        else:
            total_qry = { 'query' : {'bool': {'must' : queries}}}
            retval = self.es.count(index=self.es_index, body=total_qry)
        return retval

    def agg_qry(self, keywords, startDateString=None, endDateString=None, tweettype=None):
        return retval

    def query(self, keywords, startDateString=None, endDateString=None, size=None, tweettype=None):
        queries = self.format_query(keywords, startDateString, endDateString, tweettype)
        if size:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': size, 'query': {'bool': {'must': queries }}})
            for tw in retval['hits']['hits']:
                yield tw
        else:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': 1000, 'query': {'bool': {'must': queries }}})
            total = retval["hits"]["total"]
            print(total)
            sid = retval['_scroll_id']
            scroll_size = len(retval['hits']['hits'])
            while scroll_size > 0:
                print("Scrolling...")
                for tw in retval['hits']['hits']:
                    yield tw
                if not size:
                    retval = self.es.scroll(scroll_id=sid, scroll='1m')
                    sid = retval['_scroll_id']
                    scroll_size = len(retval['hits']['hits'])
            self.es.clear_scroll(scroll_id=sid)

    def sizequery(self, keywords, startDateString=None, endDateString=None, size=None, tweettype=None, random=True):
        queries = self.format_query(keywords, startDateString, endDateString, tweettype)
        queries = {'bool': {'must': queries }}
        print(queries)
        if random:
            queries = {"function_score": {
                "query": queries,
                "random_score": {},
            }}
        if size < 1000:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': size, 'query': queries})
            for tw in retval['hits']['hits']:
                yield tw
        else:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': 1000, 'query': queries})
            total = retval["hits"]["total"]
            so_far = len(retval['hits']['hits'])
            if size == None:
                size = total
            sid = retval['_scroll_id']
            scroll_size = len(retval['hits']['hits'])
            while scroll_size > 0 and so_far <= size:
                print(so_far)
                print(size)
                print(total)
                print("Scrolling...")
                for tw in retval['hits']['hits']:
                    yield tw
                retval = self.es.scroll(scroll_id=sid, scroll='1m')
                sid = retval['_scroll_id']
                scroll_size = len(retval['hits']['hits'])
                so_far = so_far + scroll_size
            self.es.clear_scroll(scroll_id=sid)
        """count = 0
        retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
            body={'size': 1000, 'query': {'bool': {'must': queries }}})
        total = retval["hits"]["total"]
        print(total)
        sid = retval['_scroll_id']
        scroll_size = len(retval['hits']['hits'])
        #count += scroll_size
        while count < size:
            print("Scrolling...")
            for tw in retval['hits']['hits']:
                yield tw

            retval = self.es.scroll(scroll_id=sid, scroll='1m')
            sid = retval['_scroll_id']
            scroll_size = len(retval['hits']['hits'])
            count += scroll_size
        self.es.clear_scroll(scroll_id=sid)"""

if __name__ == "__main__":
    from test_credentials import AWS_PROFILE
    import json
    e = ESSearch(AWS_PROFILE)
    count=0
    future_of_work_string="employee employees employed employing employ jobless job jobs work working works worked unemployable unemployed wfh"
    test_string = '"self care"'
    test_string = '"covid"'
    retarr = []
    #print(e.count(test_string, tweettype=["original", "quote", "reply"]))
    for doc in e.sizequery(test_string, tweettype=["original"], size=10000):
        #print(doc["_source"])
        retarr.append(doc["_source"])
        count+=1
    f = open("lindsay_tweets.json", "w")

    json.dump(retarr, f)
    print(count)


from django.contrib import messages
from django.core.mail import send_mail
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.shortcuts import redirect, render
from djeym.models import Placemark

from gotohome.settings import EMAIL_HOST_USER
from users.models import CustomUser

def map(request):
    if not request.user.is_authenticated or not request.user.is_shelter:
        messages.info(request,
                      'Только приюты могут пользоваться этой картой')
        return redirect('homepage:home')

    template_name = 'maps/shelter_maps.html'

    return render(request, template_name)

@receiver(post_save, sender=Placemark)
def notify_email(instance, **kwargs):
    if instance.is_user_marker and not instance.is_sended_admin_email:
        subject = 'Text subject'
        message = 'Text message - Url: ' + \
            'http(s)://your.domain/admin/djeym/placemark/{}/change/'.format(
                instance.pk
            )
        from_email = EMAIL_HOST_USER
        staff_users = list(
            i['email'] for i in CustomUser.objects.filter(
                is_staff=True
            ).values('email').distinct()
        )
        recipient_list = staff_users
        send_mail(subject, message, from_email,
                  recipient_list, fail_silently=False)

        instance.is_sended_admin_email = True
        instance.save()
    elif instance.active and instance.is_user_marker \
            and not instance.is_sended_user_email:
        subject = 'Text subject'
        message = 'Text message'
        from_email = EMAIL_HOST_USER
        recipient_list = [instance.user_email]
        send_mail(subject, message, from_email,
                  recipient_list, fail_silently=False)
        instance.is_sended_user_email = True
        instance.save()


from typing import Optional
import c4d
import json
import os

doc: c4d.documents.BaseDocument  # The active document
op: Optional[c4d.BaseObject]  # The active object, None if unselected

#exemple extraction initiale via og2ogr
#ogr2ogr -clipdst 2567613.1657820013 1108343.8926543007 2570413.1657820013 1111143.8926543007 "/Users/olivierdonze/switchdrive/Mandats/Parc_naturel_regional_vallee_Trient/c4d/maquettes_villages/Vernayaz/vernayaz_routes_TLM.geojson" "/Volumes/My Passport Pro/swisstopo/SWISSTLM3D_2022_LV95_LN02.gdb" "TLM_STRASSE"

CONTAINER_ORIGIN = 1026473

def main() -> None:
    fn = '/Users/olivierdonze/switchdrive/Mandats/Parc_naturel_regional_vallee_Trient/c4d/maquettes_villages/Vernayaz/vernayaz_routes_TLM.geojson'
    origin = doc[CONTAINER_ORIGIN]

    res = c4d.BaseObject(c4d.Onull)
    res.SetName(os.path.basename(fn))
    doc.InsertObject(res)
    with open(fn,encoding='utf-8') as f :
        data = json.loads(f.read())

        for feat in data['features']:
            objektart = feat['properties']['OBJEKTART']

            if feat['geometry']['type'] =='LineString':
                nb_seg = 1
                pts = [c4d.Vector(x,y,z) - origin for x,z,y in feat['geometry']['coordinates']]

            if feat['geometry']['type'] =='MultiLineString':
                coord = feat['geometry']['coordinates']
                nb_seg = len(coord)
                pts = []
                segs = []
                for i,points in enumerate(coord):
                    pts.extend([c4d.Vector(x,y,z) - origin for x,z,y in points])
                    segs.append(len(points))

            nb_pts = len(pts)
            sp = c4d.SplineObject(nb_pts,c4d.SPLINETYPE_LINEAR)
            sp.SetName(objektart)
            sp.SetAllPoints(pts)
            if nb_seg>1:
                sp.ResizeObject(nb_pts, nb_seg)
                for i,n in enumerate(segs):
                    sp.SetSegment(i,n,closed = False)

            sp.Message(c4d.MSG_UPDATE)
            sp.InsertUnderLast(res)

    c4d.EventAdd()

"""
def state():
    # Defines the state of the command in a menu. Similar to CommandData.GetState.
    return c4d.CMD_ENABLED
"""

if __name__ == '__main__':
    main()

swistopo/import geojson TLM3D_strasse after clip.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import copy

from ..register import registers
from ..operation import Operation

def traverse_postorder(operation):

    nodes_postorder = []
    identifiers = []

    def recurse(node):
        if isinstance(node, Operation):
            for input_node in node.input_nodes:
                recurse(input_node)
        if id(node) not in identifiers:
            nodes_postorder.append(node)
            identifiers.append(id(node))

    recurse(operation)

    return nodes_postorder

def create_variable(node_name, node_params_: dict):
    """"""
    node_params = copy.deepcopy(node_params_)
    node_type = node_params.pop("type", None)
    assert node_type is not None, f"{node_name} has no type."
    node_template = node_params.pop("template", None)
    # -- special keywords
    #random_seed = node_params.pop("random_seed", None)
    #if random_seed is not None:
    #    rng = np.random.default_rng(random_seed)
    #    node_params.update(rng=rng)

    # --
    node = None
    if node_template is not None:
        node_params.update(**node_template)
    node_cls = registers.get("variable", node_type, convert_name=True)
    node = node_cls(**node_params)

    return node

def create_operation(op_name, op_params_: dict):
    """"""
    op_params = copy.deepcopy(op_params_)
    op_type = op_params.pop("type", None)
    assert op_type is not None, f"{op_name} has no type."
    op_template = op_params.pop("template", None)
    # -- special keywords
    #random_seed = op_params.pop("random_seed", None)
    #if random_seed is not None:
    #    rng = np.random.default_rng(random_seed)
    #    op_params.update(rng=rng)

    # --
    op_cls = registers.get("operation", op_type, convert_name=False)
    operation = op_cls(**op_params)

    return operation

if __name__ == "__main__":
    ...


from nl4dv.utils import constants, helpers
import re
import nltk
from nltk import word_tokenize
from si_prefix import si_parse
from nltk.corpus import stopwords
import copy

class DialogGenie:
    def __init__(self, nl4dv_instance):
        self.nl4dv_instance = nl4dv_instance
        self.previous_response = None

    def renew_fillter(self, current_response, mode = 'replace filter which has the same field'):
        previous_transform = copy.deepcopy(self.previous_response['visList'][0]['vlSpec']['transform'])
        current_transform = copy.deepcopy(current_response['visList'][0]['vlSpec']['transform'])
        if mode == 'just add filter':
            self.previous_response['visList'][0]['vlSpec']['transform'].extend(current_transform)
            return

        elif mode == 'replace filter which has the same field':
            ft_pre = dict() # ft: field map to filter
            ft_cur = dict()
            for trans_pre in previous_transform:
                ft_pre[trans_pre['filter']['field']] = trans_pre
            for trans_cur in current_transform:
                if list(trans_cur.keys())[0] == 'filter':
                    ft_cur[trans_cur['filter']['field']] = trans_cur

            for field_cur in ft_cur:
                ft_pre[field_cur] = ft_cur[field_cur]
            self.previous_response['visList'][0]['vlSpec']['transform'] = list(ft_pre.values())
            return

        elif mode == 'OR with fillter which has the same field':
            ft_pre = dict() # ft: field map to filter
            ft_cur = dict()
            for trans_pre in previous_transform:
                ft_pre[trans_pre['filter']['field']] = trans_pre
            for trans_cur in current_transform:
                if list(trans_cur.keys())[0] == 'filter':
                    ft_cur[trans_cur['filter']['field']] = trans_cur

            p_newed_by_c = copy.deepcopy(ft_pre)
            c_newed_by_p = copy.deepcopy(ft_cur)
            for field_cur in ft_cur:
                p_newed_by_c[field_cur] = ft_cur[field_cur]
            for field_pre in ft_pre:
                c_newed_by_p[field_pre] = ft_pre[field_pre]

            for field in p_newed_by_c:
                  if(p_newed_by_c[field]['filter']['oneOf']!=c_newed_by_p[field]['filter']['oneOf']):
                      p_newed_by_c[field]['filter']['oneOf'].extend(c_newed_by_p[field]['filter']['oneOf'])
            self.previous_response['visList'][0]['vlSpec']['transform'] = list(p_newed_by_c.values())
            return

        else:
            raise ValueError("bad mode")


from datetime import datetime

import sqlalchemy as sa

from .base import Model

class User(Model):
    __tablename__ = 'users'

    join_date = sa.Column(sa.DateTime, default=datetime.now())
    deep_link = sa.Column(sa.String(255))

    def __repr__(self) -> str:
        return self._repr(id=self.id,
                          join_date=self.join_date,
                          deep_link=self.deep_link)


"""
Train on Emotion dataset
"""
import os
import time
import numpy as np
import torch
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
import Utils
import math
import Config
from tqdm import trange

def emotrain(model, data_loader, args, focus_emo):
	"""
	:data_loader input the whole field
	"""
	# start time
	time_st = time.time()
	decay_rate = args.decay

	# Load in the training set and validation set
	train_loader = data_loader['train']
	dev_loader = data_loader['dev']

	labels = train_loader['label']
	audio_feats = train_loader['audio']
	raw_audio_feats = train_loader['raw_audio']

	# Optimizer
	lr = args.lr
	model_opt = optim.Adam(model.parameters(), lr=lr)

	print("Dataset : {} \n Emotion rates {}".format(args.dataset,Config.data_count))

	# Raise the .train() flag before training
	model.train()

	over_fitting = 0
	cur_best = -1e10
	cur_best_loss = 100
	glob_steps = 0
	report_loss = 0
	for epoch in range(1, args.epochs + 1):
		model_opt.param_groups[0]['lr'] *= decay_rate
		labels, audio_feats, raw_audio_feats = Utils.shuffle_lists(labels, audio_feats, raw_audio_feats)
		print("===========Epoch {}==============".format(epoch))
		print("-{}-{}".format(epoch, Utils.timeSince(time_st)))
		for bz in trange(len(labels)):
			# Tensorize a dialogue, a dialogue is a batch

			label = Utils.ToTensor(labels[bz])
			audio_feat = torch.from_numpy(np.array(audio_feats[bz])).float()
			raw_audio_feat = torch.from_numpy(np.array(raw_audio_feats[bz])).float()

			audio_len = raw_audio_feats[bz][:,:,0]
			mask_len = torch.from_numpy(np.array(audio_len)).float()
			audio_lens = Utils.ToAudioLens(audio_len)

			label = Variable(label)
			audio_feat = Variable(audio_feat)
			raw_audio_feat = Variable(raw_audio_feat)

			if args.gpu != None:
				os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
				device = torch.device("cuda: 0")
				model.cuda(device)
				label = label.cuda(device)
				audio_feat = audio_feat.cuda(device)
				raw_audio_feat = raw_audio_feat.cuda(device)
				mask_len = mask_len.cuda(device)

			log_prob = model(raw_audio_feat, audio_lens, mask_len)
			loss = comput_class_loss(log_prob, label)
			loss.backward()
			report_loss += loss.item()
			glob_steps += 1

			# gradient clip
			torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5)

			model_opt.step()
			model_opt.zero_grad()

			if glob_steps % args.report_loss == 0:
				print("Steps: {} Loss: {} LR: {}".format(glob_steps, report_loss/args.report_loss, model_opt.param_groups[0]['lr']))
				report_loss = 0

		# validate
		pAccs, y_true, y_pred = emoeval(model=model,
										data_loader=dev_loader,
										args=args,
										focus_emo=focus_emo)
		print("Evaluation Metric [{}, {}, {}, {}, {}, {}]".format('happy', 'anger', 'sad', 'neutral', 'WAcc', 'UWAcc'))
		print("Validate: ACCs-WA-UWA {}".format(pAccs[:-1]))

		last_best = pAccs[-1]  # loss
		if last_best < cur_best_loss:
			Utils.model_saver(model, args.save_dir, args.dataset)
			cur_best_loss = last_best
			over_fitting = 0
		else:
			over_fitting += 1
		if over_fitting >= args.patience:
			break

def comput_class_loss(log_prob, target):
	""" Weighted loss function """
	loss = F.nll_loss(log_prob, target.view(target.size(0)), reduction='sum')
	loss /= target.size(0)

	return loss

def emoeval(model, data_loader, args, focus_emo):
	""" data_loader only input 'dev' """
	model.eval()

	TP = np.zeros([args.class_num], dtype=np.long) # recall
	TP_FN = np.zeros([args.class_num], dtype=np.long) # gold
	focus_idx = [Config.label_index[emo] for emo in focus_emo]

	audio_feats = data_loader['audio']
	labels = data_loader['label']
	raw_audio_feats = data_loader['raw_audio']

	val_loss = 0
	y_true=[]
	y_pred=[]

	for bz in range(len(labels)):
		label = Utils.ToTensor(labels[bz])
		audio_feat = torch.from_numpy(np.array(audio_feats[bz])).float()
		raw_audio_feat = torch.from_numpy(np.array(raw_audio_feats[bz])).float()

		audio_len = raw_audio_feats[bz][:,:,0]
		mask_len = torch.from_numpy(np.array(audio_len)).float()
		audio_lens = Utils.ToAudioLens(audio_len)

		label = Variable(label)
		audio_feat = Variable(audio_feat)
		raw_audio_feat = Variable(raw_audio_feat)

		if args.gpu != None:
			os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
			device = torch.device("cuda: 0")
			model.cuda(device)
			label = label.cuda(device)
			audio_feat = audio_feat.cuda(device)
			raw_audio_feat = raw_audio_feat.cuda(device)
			mask_len = mask_len.cuda(device)

		log_prob = model(raw_audio_feat, audio_lens, mask_len)

		# print(log_prob, label)
		# val loss
		loss = comput_class_loss(log_prob, label)
		val_loss += loss.item()

		# accuracy
		emo_predidx = torch.argmax(log_prob, dim=1)
		emo_true = label.view(label.size(0))

		a = emo_true.cpu().tolist()
		for i in a:
			y_true.append(i)
		b = emo_predidx.cpu().tolist()
		for j in b:
			y_pred.append(j)

		for lb in range(emo_true.size(0)):
			idx = emo_true[lb].item()
			TP_FN[idx] += 1
			if idx in focus_idx:
				if emo_true[lb] == emo_predidx[lb]:
					TP[idx] += 1

	f_TP = [TP[Config.label_index[w]] for w in focus_emo]
	f_TP_FN = [TP_FN[Config.label_index[w]] for w in focus_emo]
	Recall = [np.round(tp/tp_fn*100, 2) if tp_fn>0 else 0 for tp,tp_fn in zip(f_TP,f_TP_FN)]
	wRecall = sum([r * w / sum(f_TP_FN) for r,w in zip(Recall, f_TP_FN)])
	uRecall = sum(Recall) / len(Recall)

	# Accuracy of each class w.r.t. the focus_emo, the weighted acc, and the unweighted acc
	Total = Recall + [np.round(wRecall,2), np.round(uRecall,2)] + [np.round(val_loss,3)]

	# Return to .train() state after validation
	model.train()

	return Total, y_true, y_pred


import codecademylib
import pandas as pd

#columns: user_id, utm_source, day, ad_click_timestamp, experimental_group

#reads csv into DataFrame
ad_clicks = pd.read_csv('ad_clicks.csv')
print(ad_clicks.head())
#new Df with amount of users per source
print(ad_clicks.groupby("utm_source").user_id.count().reset_index())

#adds a new boolean column informing if ad has been clicked or not
ad_clicks["is_click"] = ~ad_clicks.ad_click_timestamp.isnull()

#displays amount of clicks per source
clicks_by_source = ad_clicks.groupby(("utm_source", "is_click")).user_id.count().reset_index()

print(clicks_by_source)
#pivots that DateFrame for better readability
clicks_pivot = clicks_by_source.pivot(
  columns = "is_click",
  index = "utm_source",
  values = "user_id"
)

#adds a column with the percentage or clicks per sources
clicks_pivot["percent_clicked"] = \
  clicks_pivot[True] / (clicks_pivot[True] + clicks_pivot[False])
print(clicks_pivot)

#displays if both experimental groups received equal amount of ads
print(ad_clicks.groupby("experimental_group").user_id.count().reset_index())

#counts the amount of clicks per group
print(ad_clicks.groupby(["experimental_group", "is_click"]).user_id.count().reset_index().pivot(
  index = 'experimental_group',
  columns = 'is_click',
  values = 'user_id'
).reset_index())

a_clicks = ad_clicks[ad_clicks.experimental_group == "A"]
b_clicks = ad_clicks[ad_clicks.experimental_group == "B"]
#print(b_clicks)

a_clicks_pivot = a_clicks.groupby(["is_click", "day"]).user_id.count().reset_index().pivot(
  index = "day",
  columns = "is_click",
  values = "user_id"
).reset_index()

b_clicks_pivot = b_clicks.groupby(["is_click", "day"]).user_id.count().reset_index().pivot(
  index = "day",
  columns = "is_click",
  values = "user_id"
).reset_index()

print(a_clicks_pivot)
a_clicks_pivot["Percentage"] = a_clicks_pivot[True] / (a_clicks_pivot[False] + a_clicks_pivot[True])
print(a_clicks_pivot)

b_clicks_pivot["Percentage"] = b_clicks_pivot[True] / (b_clicks_pivot[False] + b_clicks_pivot[True])
print(b_clicks_pivot)


shardsList = [0]
xpList = [50]
for i in range (1,5):
    shardsList.append(1300)
    xpList.append(200+i*100)
for i in range(1,7):
    shardsList.append(2400)
    xpList.append(800+i*100)
for i in range(1,7):
    shardsList.append(3200)
    xpList.append(1600+i*100)
for i in range(1,8):
    shardsList.append(3200)
    xpList.append(2400+i*100)
xpList.append(6800)
shardsList.append(9700)
print(xpList)
print(shardsList)
heroLevel = int(input("Input current hero level"))
desiredLevel = int(input("Input desired hero level"))
xpTotal = (sum(xpList[heroLevel:desiredLevel]))
shardsEarned = sum(shardsList[heroLevel:desiredLevel])
print(xpTotal, " XP required.")
print(shardsEarned, " shards earned.")
print(xpTotal/100, " accounts required.")
print(xpTotal/600, " hours.")


import traceback
import math
import os.path
import pandas as pd
import numpy as np
import datetime
import plotly.graph_objects as go
from statsmodels.tsa.stattools import adfuller
from statsmodels.tsa.ar_model import AutoReg
from sklearn.metrics import mean_squared_error
from pandas_profiling import ProfileReport
from financepy.utils import from_datetime
from financepy.products.bonds import *

root_path = 'd:/ProjectRicequant/fxincome/'
chinabond_ttm_labels = ['6m', '1y', '2y', '3y', '4y', '5y', '6y', '7y', '8y', '9y', '10y']
predicted_ttm_labels = ['pred_6m', 'pred_1y', 'pred_2y', 'pred_3y', 'pred_4y', 'pred_5y', 'pred_6y', 'pred_7y',
                        'pred_8y', 'pred_9y', 'pred_10y']
maturities = [0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

def to_bond_freq(freq):
    if freq == 1:
        return FrequencyTypes.ANNUAL
    elif freq == 2:
        return FrequencyTypes.SEMI_ANNUAL
    elif freq == 4:
        return FrequencyTypes.QUARTERLY
    else:
        raise NotImplementedError("Unknown frequency")

def plot_bond_ylds(yld_date: datetime, bonds: list, ylds: list):
    plt.figure(figsize=(12, 6))
    title = yld_date.strftime('%Y%m%d')
    plt.title(f'CDB YTM @{title}')
    yld_date = from_datetime(yld_date.date())
    x = []
    for bond in bonds:
        years_to_maturity = (bond._maturity_date - yld_date) / 365
        x.append(years_to_maturity)
    bond_ylds_scaled = [n * 100 for n in ylds]
    plt.plot(x, bond_ylds_scaled, 'o')

    plt.xlabel('Time to Maturity (years)')
    plt.ylabel('Yield To Maturity (%)')
    plt.ylim((min(bond_ylds_scaled) - 0.3, max(bond_ylds_scaled) * 1.1))
    plt.grid(True)
    plt.savefig(f'./results/EXCEPTION_ytm_{title}.jpg', dpi=600)
    plt.close()

def plot_ns_params(df: pd.DataFrame):
    df = df.sort_values(by='date')
    first_date = df.date.iloc[0]
    last_date = df.date.iloc[len(df) - 1]
    title = f"{first_date.strftime('%Y%m%d')}-{last_date.strftime('%Y%m%d')}"
    df = df.rename(columns={'sum_beta1_beta2': 'beta1+beta2'})
    df.plot(kind='line', x='date', y=['beta1', 'beta2', 'beta1+beta2'], figsize=(15, 8))
    plt.grid(True)
    plt.legend(loc='upper left', bbox_to_anchor=(1, 1))
    plt.savefig(f'./results/ns_params_{title}.jpg')
    plt.close()

def preprocess():
    source = 'cdb_valuation.csv'
    result = 'cdb_valuation_processed.csv'
    df = pd.read_csv(os.path.join(root_path, source), parse_dates=['date', 'issue_date', 'maturity_date'],
                     encoding='gb2312')
    df = df[df.coupon_type == '固定利率']
    # df = df[(df.coupon_info == '附息') | (df.coupon_info == '到期一次还本付息')]
    #  Wind的付息次数不正确，到期一次还本付息的付息次数显示为0，应修改为1
    df.loc[df.coupon_info == '到期一次还本付息', 'coupon_freq'] = 1
    df_2019_2022 = df[(df.outstanding_balance >= 800) & (df.date.dt.year >= 2019)]
    df_2017_2018 = df[(df.outstanding_balance >= 500) & (df.date.dt.year >= 2017) & (df.date.dt.year <= 2018)]
    df_2015_2016 = df[(df.outstanding_balance >= 400) & (df.date.dt.year >= 2015) & (df.date.dt.year <= 2016)]
    df_2014 = df[(df.outstanding_balance >= 350) & (df.date.dt.year == 2014)]
    df_2013 = df[(df.outstanding_balance >= 300) & (df.date.dt.year == 2013)]
    df_2010_2012 = df[(df.outstanding_balance >= 250) & (df.date.dt.year >= 2010) & (df.date.dt.year <= 2012)]

    df = pd.concat([df_2010_2012, df_2013, df_2014, df_2015_2016, df_2017_2018, df_2019_2022])
    df = df[df.bond_code.str.len() == 9]
    df = df[(df.maturity_date - df.date).dt.days / 365 <= 10]  # only bonds with maturity <= 10 years
    df.to_csv(os.path.join(root_path, result), index=False, encoding='gb2312')

def check_csv():
    # source = 'cdb_valuation_processed.csv'
    # bond_df = pd.read_csv(root_path + source, parse_dates=['date', 'issue_date', 'maturity_date'],
    #                       encoding='gb2312')
    # bond_dates = set(bond_df.date.dt.date.to_list())
    # ytm_df = pd.read_csv(root_path + 'cdb_ytm.csv', parse_dates=['date'], encoding='gb2312')
    # ytm_dates = set(ytm_df.date.dt.date.to_list())
    # difference = list(ytm_dates - bond_dates)
    # difference.sort()
    # print(difference)
    # print(len(difference))
    # df = pd.read_csv('./results/ns_params_20100104_20220729.csv', parse_dates=['date'])
    # report = ProfileReport(df)
    # report.to_file('./results/ns_params_report.html')
    df = pd.DataFrame(
        {"value": np.random.randn(36)},
        index=pd.date_range("2011-01-01", freq="M", periods=36),
    )
    table = pd.pivot_table(
        df, index=df.index.month, columns=df.index.year, values="value", aggfunc="sum"
    )
    print(df)
    print(table)

def gen_curves(fit_type: str, start: datetime.datetime, end: datetime.datetime, plot: bool) -> pd.DataFrame:
    source = 'cdb_valuation_processed.csv'
    df_src = pd.read_csv(os.path.join(root_path, source), parse_dates=['date', 'issue_date', 'maturity_date'],
                         encoding='gb2312')
    if fit_type == 'ns':
        ns_stats = pd.DataFrame(columns=['date', 'beta1', 'beta2', 'sum_beta1_beta2', 'beta3', 'tau'])
    elif fit_type == 'nss':
        nss_stats = pd.DataFrame(
            columns=['date', 'beta1', 'beta2', 'sum_beta1_beta2', 'beta3', 'beta4', 'tau1', 'tau2'])
    else:
        raise NotImplementedError("Unknown curve fit type")
    ytm_date = start
    while ytm_date <= end:
        df = df_src[df_src.date == ytm_date]
        if len(df) == 0:  # Today is not a buisness day.
            ytm_date += datetime.timedelta(days=1)
            continue
        bonds = []
        ytms = []
        for bond in df.itertuples():
            issue_date = from_datetime(bond.issue_date)
            maturity_date = from_datetime(bond.maturity_date)
            coupon = bond.coupon / 100
            freq = to_bond_freq(bond.coupon_freq)
            accrual_type = DayCountTypes.ACT_ACT_ISDA
            ytm = bond.ytm / 100
            b = Bond(issue_date, maturity_date, coupon, freq, accrual_type)
            bonds.append(b)
            ytms.append(ytm)

        try:
            if fit_type == 'ns':
                ns_bounds = [(-1, -1, -1, 1.3), (1, 1, 1, 3)]
                ns_stats = ns_fit(bonds, ns_stats, ytm_date, ytms, ns_bounds, plot=plot)
            elif fit_type == 'nss':
                nss_bounds = [(0, -1, -1, -1, 1.3, 1.3), (1, 1, 1, 1, 3, 3)]
                nss_stats = nss_fit(bonds, nss_stats, ytm_date, ytms, nss_bounds, plot=plot)
            # curve_function = CurveFitPolynomial(5)
            # fitted_curve = BondYieldCurve(eval_date, bonds, ytms, curve_function)
            # fitted_curve.plot('CDB Yield Curve by Quintic Polynomial')
            # curve_function = CurveFitBSpline()
            # fitted_curve = BondYieldCurve(eval_date, bonds, ytms, curve_function)
            # fitted_curve.plot('CDB Yield Curve by B Spline')
            # curve_function = CurveFitBSpline(4)
            # fitted_curve = BondYieldCurve(eval_date, bonds, ytms, curve_function)
            # fitted_curve.plot('CDB Yield Curve by B Spline power of 4')

        except Exception as e:
            print(f'Exception:{str(e)} {ytm_date}')
            traceback.print_exc()
            plot_bond_ylds(ytm_date, bonds, ytms)

        ytm_date += datetime.timedelta(days=1)
    start = start.strftime('%Y%m%d')
    end = end.strftime('%Y%m%d')
    if fit_type == 'ns':
        ns_stats.to_csv(os.path.join('./results/', f'ns_params_{start}_{end}.csv'), index=False, encoding='utf-8')
        return ns_stats
    elif fit_type == 'nss':
        nss_stats.to_csv(os.path.join('./results/', f'nss_params_{start}_{end}.csv'), index=False, encoding='utf-8')
        return nss_stats
    else:
        raise NotImplementedError("Unknown curve fit type")

def ns_fit(bonds, ns_stats, ytm_date, ytms, bounds, plot=False):
    curve_function = CurveFitNelsonSiegel(bounds=bounds)
    fitted_curve = BondYieldCurve(from_datetime(ytm_date), bonds, ytms, curve_function)
    beta1 = curve_function._beta1
    beta2 = curve_function._beta2
    beta3 = curve_function._beta3
    tau = curve_function._tau
    ns_stats.loc[len(ns_stats)] = [ytm_date, beta1, beta2, beta1 + beta2, beta3, tau]
    if plot:
        today = ytm_date.strftime('%Y%m%d')
        fitted_curve.plot(f'CDB Yield Curve by Nelson Siegel @{today}')
        address = f'./results/images/ns_cdb_ytm_{today}.jpg'
        plt.savefig(address, dpi=600)
        plt.close()
    return ns_stats

def nss_fit(bonds, nss_stats, ytm_date, ytms, bounds, plot=False):
    bounds = bounds
    curve_function = CurveFitNelsonSiegelSvensson(bounds=bounds)
    fitted_curve = BondYieldCurve(from_datetime(ytm_date), bonds, ytms, curve_function)
    beta1 = curve_function._beta1
    beta2 = curve_function._beta2
    beta3 = curve_function._beta3
    beta4 = curve_function._beta4
    tau1 = curve_function._tau1
    tau2 = curve_function._tau2
    nss_stats.loc[len(nss_stats)] = [ytm_date, beta1, beta2, beta1 + beta2, beta3, beta4, tau1, tau2]
    if plot:
        today = ytm_date.strftime('%Y%m%d')
        fitted_curve.plot(f'CDB Yield Curve by Nelson Siegel Svensson @{today}')
        address = f'./results/images/nss_cdb_ytm_{today}.jpg'
        plt.savefig(address, dpi=600)
        plt.close()
    return nss_stats

def curves_stats():
    df = pd.read_csv(root_path + 'cdb_ytm.csv', parse_dates=['date'], encoding='gb2312').set_index('date')
    df['slope'] = df['10y'] - df['6m']
    df['curvature'] = df['10y'] + df['6m'] - 2 * df['2y']
    df.to_csv('./results/chinabond_ytm.csv', encoding='utf-8')
    corr_df = pd.DataFrame(columns=['maturity', 'p_1M', 'p_1Y', 'p_2.5Y'])
    for col in df.columns:
        #  autocorrelation, 1M=21 trade days, 1Y=250 trade days, 2.5Y=625 trade days
        corr_df.loc[len(corr_df)] = [col, df[col].autocorr(21), df[col].autocorr(250), df[col].autocorr(625)]
    corr_df = corr_df.set_index('maturity').round(3)
    chinabond_stats = df.describe().T.round(3)
    chinabond_stats = chinabond_stats.join(corr_df)
    chinabond_stats.to_csv('./results/chinabond_stats.csv', encoding='utf-8')
    ns_df = pd.read_csv('./results/ns_params_20100104_20220729.csv', parse_dates=['date']).set_index('date')
    b1_adfuller = adfuller(ns_df.beta1.to_numpy())
    b2_adfuller = adfuller(ns_df.beta2.to_numpy())
    b3_adfuller = adfuller(ns_df.beta3.to_numpy())
    b1_adf = b1_adfuller[0]
    b2_adf = b2_adfuller[0]
    b3_adf = b3_adfuller[0]
    b1_pval = b1_adfuller[1]
    b2_pval = b2_adfuller[1]
    b3_pval = b3_adfuller[1]
    ns_stats = ns_df[['beta1', 'beta2', 'beta3']].describe().T
    #  Beta值需要放大100倍
    for col in ns_stats.columns:
        if col is not 'count':
            ns_stats[col] = ns_stats[col] * 100
    ns_stats = ns_stats.round(3)
    corr_df = pd.DataFrame(columns=['beta', 'p_1M', 'p_1Y', 'p_2.5Y'])
    for col in ['beta1', 'beta2', 'beta3']:
        #  autocorrelation, 1M=21 trade days, 1Y=250 trade days, 2.5Y=625 trade days
        corr_df.loc[len(corr_df)] = [col, ns_df[col].autocorr(21), ns_df[col].autocorr(250), ns_df[col].autocorr(625)]
    corr_df = corr_df.set_index('beta').round(3)
    ns_stats = ns_stats.join(corr_df)
    ns_stats.at['beta1', 'ADF'] = b1_adf
    ns_stats.at['beta1', 'P_Value'] = b1_pval
    ns_stats.at['beta2', 'ADF'] = b2_adf
    ns_stats.at['beta2', 'P_Value'] = b2_pval
    ns_stats.at['beta3', 'ADF'] = b3_adf
    ns_stats.at['beta3', 'P_Value'] = b3_pval
    ns_stats.to_csv('./results/ns_params_stats.csv')

def fit_stats():
    chinabond = pd.read_csv('./results/' + 'chinabond_ytm.csv', parse_dates=['date'], encoding='gb2312').set_index(
        'date')
    ns = pd.read_csv('./results/' + 'ns_params_20100104_20220729.csv', parse_dates=['date'],
                     encoding='gb2312').set_index(
        'date')
    combined = chinabond.join(ns)
    combined = combined[chinabond.columns.values.tolist() + ['beta1', 'beta2', 'beta3', 'tau']]
    for curve in combined.itertuples():
        ns_curve = CurveFitNelsonSiegel(tau=curve.tau)
        ns_curve._beta1 = curve.beta1
        ns_curve._beta2 = curve.beta2
        ns_curve._beta3 = curve.beta3
        for chinabond_label, pred_label, ttm in zip(chinabond_ttm_labels, predicted_ttm_labels, maturities):
            combined.at[curve.Index, pred_label] = round(ns_curve._interpolated_yield(ttm) * 100, 2)
    combined.to_csv('./results/fit_stat.csv')
    real_fitted_ytm = pd.DataFrame(columns=['Indicator'] + chinabond_ttm_labels)
    real_avgs = [chinabond[t].mean() for t in chinabond_ttm_labels]
    fitted_avgs = [combined[t].mean() for t in predicted_ttm_labels]
    errors = [real - fitted for real, fitted in zip(real_avgs, fitted_avgs)]
    real_fitted_ytm.loc[0] = ['Average Chinabond YTM'] + real_avgs
    real_fitted_ytm.loc[1] = ['Average Fitted YTM'] + fitted_avgs
    real_fitted_ytm.loc[2] = ['Average YTM Errors'] + errors
    real_fitted_ytm = real_fitted_ytm.round(2).set_index('Indicator').T
    real_fitted_ytm.to_csv('./results/chinabond_vs_fitted.csv')
    corr_real_beta = combined[['beta1', 'beta2', 'beta3', '10y', 'slope', 'curvature']].corr().round(2)
    corr_real_beta.to_csv('./results/corr_real_beta.csv')
    real_fitted_ytm['maturity'] = maturities
    ax = real_fitted_ytm.plot(kind='line', x='maturity', y='Average Chinabond YTM', color='red',
                              label='Average Chinabond YTM')
    real_fitted_ytm.plot(kind='scatter', x='maturity', y='Average Fitted YTM', ax=ax, style='*',
                         label='Average Fitted YTM')
    plt.xlabel('Time to Maturity (years)')
    plt.ylabel('Yield To Maturity (%)')
    plt.grid(True)
    plt.savefig(f'./results/chinabond_vs_fitted.jpg')
    plt.close()

def plot_ytm_3d_surface():
    df = pd.read_csv(root_path + 'cdb_ytm.csv', parse_dates=['date'], encoding='gb2312')
    df = df.set_index('date').sort_index()
    x = df.columns
    y = df.index
    z = df.to_numpy()
    fig = go.Figure(data=[go.Surface(z=z, x=x, y=y)])
    fig.update_layout(
        title=dict(
            text='CDB YTM between 2010 and 2022',
            x=0.5,
            y=0.9),
        scene=dict(
            xaxis=dict(
                title='maturity',
                backgroundcolor="rgb(200, 200, 230)",
                gridcolor="white",
                showbackground=True,
                zerolinecolor="white"),
            yaxis=dict(
                title='date',
                backgroundcolor="rgb(230, 200,230)",
                gridcolor="white",
                showbackground=True,
                zerolinecolor="white"),
            zaxis=dict(
                title='ytm',
                backgroundcolor="rgb(230, 230,200)",
                gridcolor="white",
                showbackground=True,
                zerolinecolor="white"),
            aspectratio=dict(
                x=1,
                y=1,
                z=0.6
            )
        )
    )

    fig.show()

def forecast():
    factors = pd.read_csv('d:/ProjectRicequant/fxincome/eco_factors_2010_202207.csv', parse_dates=['date']).set_index(
        'date')
    factors = factors[factors.index >= datetime.datetime(2015, 12, 1)]
    ns_params = pd.read_csv('./results/fit_stat.csv', parse_dates=['date']).set_index('date')
    ns_params = ns_params[['beta1', 'beta2', 'beta3']]
    ns_params = ns_params * 100
    # pd.plotting.autocorrelation_plot(ns_params.beta3)
    X = ns_params.beta1.values
    print(X)
    train, test = X[3000: len(X) - 44], X[len(X) - 44:]
    model = AutoReg(train, lags=22)
    model_fit = model.fit()
    print(f'Coefficients: {model_fit.params}')
    predictions = model_fit.predict(start=len(train), end=len(train) + len(test) - 1, dynamic=False)
    for i in range(len(predictions)):
        print(f'predicted={predictions[i]}, expected={test[i]}')
    rmse = math.sqrt(mean_squared_error(test, predictions))
    print(f'RMSE:{rmse:.3f}')
    plt.plot(test)
    plt.plot(predictions, color='red')
    plt.show()

    ns_params = ns_params.groupby(pd.Grouper(freq='MS')).nth(10)  # 假设每个月的第11个交易日或之后才出上个月的经济数据
    ns_params = ns_params[ns_params.index >= datetime.datetime(2015, 12, 1)]
    ppi_adf = adfuller(factors.ppi.to_numpy())
    cpi_adf = adfuller(factors.cpi.to_numpy())
    m2_adf = adfuller(factors.m2.to_numpy())
    delta_adf = adfuller(factors.delta_fin_m2.to_numpy())
    # print(f'ppi adf: {ppi_adf[0]}, p: {ppi_adf[1]}')
    # print(f'cpi adf: {cpi_adf[0]}, p: {cpi_adf[1]}')
    # print(f'm2 adf: {m2_adf[0]}, p: {m2_adf[1]}')
    # print(f'delta adf: {delta_adf[0]}, p: {delta_adf[1]}')
    df = ns_params.merge(factors, on='date')
    df['beta1_lag1'] = df.beta1.shift(-1)
    df['beta2_lag1'] = df.beta2.shift(-1)
    df['beta3_lag1'] = df.beta3.shift(-1)
    df.corr().to_csv('./results/param_factors.csv', index=True)

if __name__ == '__main__':
    # preprocess()
    check_csv()
    # gen_curves('ns', start=datetime.datetime(2010, 1, 4), end=datetime.datetime(2022, 7, 29), plot=True)
    # ns_df = pd.read_csv('./results/ns_params_20100104_20220729.csv', parse_dates=['date'], encoding='gb2312')
    # plot_ns_params(ns_df)
    # curves_stats()
    # fit_stats()
    # plot_ytm_3d_surface()
    # forecast()


PATH = '08. Files/file.txt'

# files = open(PATH, 'w')

# print(
#     f'files: {files}\nName: {files.name}\nIs closed: {files.closed}\nMode: {files.mode}')
# files.write(f'This is a test!\n')
# files.write(f'Python the best programming language.\n')
# files.close()

# files = open(PATH, 'a')
# files.write(f'A also like C#!\n')
# files.close()

files = open(PATH, 'r+')
text = files.readline()
# read = files.read(9)
read = files.read()
files.close()

print(text)
# print(read)


import cv2

#import numpy as np

#img=np.zeros((512,512))
img=cv2.imread('/home/avani/Desktop/dogs.jpg')

#cutimg=img[0:200,0:300]

cv2.line(img,(0,0),(20,30),(0,0,255),3)
cv2.circle(img,(50,50),24,(0,255,0),2)
cv2.putText(img,"heya",(50,60),cv2.FONT_HERSHEY_SIMPLEX,1,(255,0,0),4)
cv2.imshow("cute dog",img)
#cv2.imshow("cut",cutimg)
cv2.waitKey(0)
cv2.destroyAllWindows()