"""Toy implementation of a motion correction algorithm similar to the
one in Warp (https://doi.org/10.1038/s41592-019-0580-y)
"""
from datetime import datetime

import torch
import numpy as np
import mrcfile
import einops

import torch.nn.functional as F
from torch_cubic_spline_grids import CubicBSplineGrid3d

from libtilt.coordinate_utils import array_to_grid_sample
from libtilt.shift import phase_shift_dft_2d
from libtilt.shapes.shapes_2d import circle
from libtilt.grids import patch_grid

IMAGE_FILE = 'data/TS_01_000_0.0.mrc'
GT_DEFORMATION_FIELD_RESOLUTION = (3, 3, 3)  # (t, h, w)
LEARNED_DEFORMATION_FIELD_RESOLUTION = (5, 5, 5)  # (t, h, w)
PATCH_SIDELENGTH = 64
N_ITERATIONS = 200
N_PATCHES_PER_BATCH = 15
LEARNING_RATE = 0.05

# load data, bin 8x, reduce to 2D and normalise
image = torch.tensor(mrcfile.read(IMAGE_FILE)).float()
image = F.avg_pool2d(image, kernel_size=8)
image = einops.reduce(image, 't h w -> h w', reduction='mean')
image = (image - torch.mean(image)) / torch.std(image)

# simulate continuous beam induced motion on the image
# make image multi-frame, initialise a deformation field with random shifts
multi_frame_micrograph = einops.repeat(image, 'h w -> 10 h w').clone()
gt_deformation_field_data = np.random.uniform(
    low=-5, high=5, size=(2, *GT_DEFORMATION_FIELD_RESOLUTION)
)
gt_deformation_field = CubicBSplineGrid3d.from_grid_data(
    torch.tensor(gt_deformation_field_data).float()
)

# evaluate deformation field at every pixel
t, h, w = multi_frame_micrograph.shape[-3:]
_t, _y, _x = torch.linspace(0, 1, t), torch.linspace(0, 1, h), torch.linspace(0, 1, w)
tt, yy, xx = torch.meshgrid(_t, _y, _x, indexing='ij')
tyx = einops.rearrange([tt, yy, xx], 'tyx t h w -> t h w tyx')
shifts = gt_deformation_field(tyx)
array_coordinates = torch.tensor(np.indices(image.shape[-2:]))
array_coordinates = einops.rearrange(array_coordinates, 'yx ... -> ... yx')

# find coordinates to sample (after deforming by shifts)
deformed_coordinates = array_coordinates + shifts

# sample image at deformed positions
multi_frame_micrograph = F.grid_sample(
    input=einops.rearrange(multi_frame_micrograph, 't h w -> t 1 h w'),
    grid=array_to_grid_sample(deformed_coordinates, array_shape=(h, w)),
    mode='bicubic',
    padding_mode='zeros',
    align_corners=True,
)
multi_frame_micrograph = einops.rearrange(multi_frame_micrograph, 't 1 h w -> t h w')

# prepare data for learning the motion...
reference = image.clone()

# extract patches from data and reference
ph, pw = (PATCH_SIDELENGTH, PATCH_SIDELENGTH)
data_patches, data_patch_centers = patch_grid(
    images=multi_frame_micrograph,
    patch_shape=(1, ph, pw),
    patch_step=(1, ph // 2, pw // 2),
    distribute_patches=True,
)
data_patches = einops.rearrange(data_patches, 't gh gw 1 ph pw -> t gh gw ph pw')
reference_patches, _ = patch_grid(
    images=reference,
    patch_shape=(PATCH_SIDELENGTH, PATCH_SIDELENGTH),
    patch_step=(PATCH_SIDELENGTH // 2, PATCH_SIDELENGTH // 2),
    distribute_patches=True,
)  # (grid_h, grid_w, ph, pw)
gh, gw = data_patch_centers.shape[1:3]

# apply a soft circular mask on both reference and data patches
mask = circle(
    radius=PATCH_SIDELENGTH / 4,
    image_shape=(PATCH_SIDELENGTH, PATCH_SIDELENGTH),
    smoothing_radius=PATCH_SIDELENGTH / 8,
)
data_patches *= mask
reference_patches *= mask

# rfft the data and the reference
data_patches = torch.fft.rfftn(data_patches, dim=(-2, -1))
reference_patches = torch.fft.rfftn(reference_patches, dim=(-2, -1))

# initialise the deformation field with learnable parameters and normalise
# patch centers to [0, 1] for evaluation of shifts.
deformation_field = CubicBSplineGrid3d(
    resolution=LEARNED_DEFORMATION_FIELD_RESOLUTION,
    n_channels=2
)
data_patch_centers = data_patch_centers / torch.tensor([t - 1, h - 1, w - 1])

# initialise optimiser and detach data
motion_optimiser = torch.optim.Adam(
    params=deformation_field.parameters(),
    lr=LEARNING_RATE,
)
data_patches = data_patches.detach()
reference_patches=reference_patches.detach()

# optimise shifts at grid points on deformation field
start = datetime.now()
for i in range(N_ITERATIONS):
    # take a random subset of the patch grid over spatial dimensions
    subset_idx = np.random.randint(
        low=(0, 0), high=(gh, gw), size=(N_PATCHES_PER_BATCH, 2)
    )
    patch_idx_h, patch_idx_w = einops.rearrange(subset_idx, 'b idx -> idx b')
    patch_subset = data_patches[:, patch_idx_h, patch_idx_w]
    patch_subset_centers = data_patch_centers[:, patch_idx_h, patch_idx_w]
    reference_patch_subset = reference_patches[patch_idx_h, patch_idx_w]

    # predict the shifts at patch centers
    predicted_shifts = deformation_field(patch_subset_centers)

    # shift the patches by the predicted shifts
    shifted_patches = phase_shift_dft_2d(
        dft=patch_subset,
        image_shape=(ph, pw),
        shifts=predicted_shifts,
        rfft=True,
    )  # (b, ph, pw, h, w)

    # calculate the loss, MSE between data patches and reference patches
    loss = torch.mean((reference_patch_subset - shifted_patches).abs() ** 2)

    # zero gradients, backpropagate loss and step optimiser
    motion_optimiser.zero_grad()
    loss.backward()
    motion_optimiser.step()

    if i % 20 == 0:
        print(loss.item())
end = datetime.now()
delta = (end - start).total_seconds()
print(f'time taken: {delta}s')

# quantify how well we're doing over whole field
gt = gt_deformation_field(tyx)
learned = deformation_field(tyx)
print(torch.mean(torch.abs(gt - learned)))

# invert the motion to reconstruct a 'motion corrected' image
predicted_shifts = -1 * deformation_field(tyx)
sample_coords = array_coordinates + predicted_shifts
corrected_image_thw = F.grid_sample(
    input=einops.rearrange(multi_frame_micrograph, 't h w -> t 1 h w'),
    grid=array_to_grid_sample(sample_coords, array_shape=(h, w)),
    mode='bicubic',
    padding_mode='zeros',
    align_corners=True,
)
corrected_image_thw = einops.rearrange(corrected_image_thw, 't 1 h w -> t h w')
corrected_image = einops.reduce(corrected_image_thw, 't h w -> h w', reduction='mean')

# visualise results
import napari
viewer = napari.Viewer()
viewer.add_image(reference.detach().numpy(), name='reference')
viewer.add_image(multi_frame_micrograph.detach().numpy(), name='beam induced motion (simulated)')
viewer.add_image(corrected_image_thw.detach().numpy(), name='motion corrected (2d+t)')
viewer.add_image(corrected_image.detach().numpy(), name='motion corrected (2d)')
napari.run()


# coding: utf-8

import xair
import gym
import xsim
import numpy as np
import xtools as xt
import pandas as pd
import argparse
import matplotlib.pyplot as plt
from tqdm import tqdm
from dataclasses import dataclass
from feles.controller import sfb
from feles.controller.se import SystemEstimator3
from feles.utils.buffer import ReplayBuffer, Experience

COLOR_ZERO = "black"
COLOR_DATA = ["gray", "forestgreen", "hotpink"]
WIDTH_ZERO = "1"

@dataclass
class Log:
    state: np.ndarray
    action: np.ndarray
    next_state: np.ndarray

    @staticmethod
    def format(buffer):
        state = np.vstack([exp.state for exp in buffer]).astype(np.float32)
        action = np.vstack([exp.action for exp in buffer]).astype(np.float32)
        next_state = np.vstack([exp.next_state for exp in buffer]).astype(np.float32)
        return Log(state, action, next_state)

def dev(args):
    xt.info("development mode of SystemEstimator2")
    cf = xt.Config(args.config)

    # plant
    ecf = cf.env
    env = build_env(cf)
    dim_state = 2
    dim_action = 1
    xt.info("env", env)

    # controller
    scf = cf.controller.sfb
    Q = np.diag(scf.Q).astype(float)
    R = np.diag(scf.R).astype(float)
    K = sfb.compute_sfb_gain(env, Q, R)
    K = np.squeeze(K)

    # estimator
    se3 = SystemEstimator3(dim_state, dim_action, cf.controller.se.lr)

    # logger
    logger = xsim.Logger()
    rb = ReplayBuffer(int(1.0 / ecf.dt))

    # prepare simulation
    loss = 0.0

    xt.info("run simulation")
    for time in tqdm(xsim.generate_step_time(ecf.due, ecf.dt)):
        obs = env.observation
        state = env.state
        action = compute_action(env, K)

        # simulation update
        env.step(action)

        rb.add(Log(state, action[1], env.state))
        if len(rb) >= 100:
            data = rb.buffer
            loss = se3.update(data.state, data.action, data.next_state)
        predict = se3.predict(state, action[1:])

        logger.store(time=time, obs=obs, act=action, pre=predict, loss=loss).flush()

    print(se3.A())
    exit()
    # retrieve result
    xt.info("make result")
    res = xsim.Retriever(logger)
    res = pd.DataFrame(dict(
        time=res.time(),
        pitch=xt.r2d(res.obs(env.IX_T)),
        reference=xt.r2d(res.obs(env.IX_C)),
        predict=xt.r2d(res.pre(env.IX_T)),
        elevator=xt.r2d(res.act(env.IX_de)),
        loss=res.loss()
    ))

    if not args.show_fig and not cf.result.fig.show:
        return args

    xt.info("plot result")
    delim = xt.round(xt.r2d([
        env.action_space.low[env.IX_de],
        env.action_space.high[env.IX_de],
    ]), 3)

    fig, axes = plt.subplots(nrows=3, sharex=True, figsize=(8, 8))
    # draw zero-lines
    axes[0].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)
    axes[1].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)

    res.plot(
        x="time", y=["reference", "pitch", "predict"], ax=axes[0],
        style=["--", "-", "-"], color=COLOR_DATA,
        xlim=[0, ecf.due], ylim=ecf.reference.range, grid=True,
        yticks=np.arange(ecf.reference.range[0], ecf.reference.range[1]+1, 1)
    )
    res.plot(
        x="time", y="elevator", ax=axes[1],
        style="-", color="orange",
        ylim=delim, grid=True,
        yticks=np.arange(delim[0], delim[1]+1, 1)
    )
    res.plot(
        x="time", y="loss", ax=axes[2],
        style="-", color="orange",
        ylim=[0, np.max(res.loss) * 1.2], grid=True
    )

    plt.show()

def build_env(cf):
    ecf = cf.env
    env = gym.make(
        ecf.name, dt=ecf.dt,
        target_range=xt.d2r(ecf.reference.range),
        target_period=ecf.reference.interval
    )
    env.reset()
    return env

def compute_action(env, K):
    x = env.state
    r = env.target[:2]
    e = r - x
    u = e.dot(K)
    u = np.array([0, u], dtype=env.dtype)
    u = np.clip(u, env.action_space.low, env.action_space.high)
    return u

def run(args):
    xt.info("run to dev system estimation")
    cf = xt.Config(args.config)

    # prepare env
    ecf = cf.env
    env = gym.make(
        ecf.name, dt=ecf.dt,
        target_range=xt.d2r(ecf.reference.range),
        target_period=ecf.reference.interval
    )
    env.reset()
    xt.info("env", env)

    # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    # controllers
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # state F/B
    #
    scf = cf.controller.sfb
    Q = np.diag(scf.Q).astype(float)
    R = np.diag(scf.R).astype(float)
    K = sfb.compute_sfb_gain(env, Q, R)
    K = np.squeeze(K)

    # logger
    logger = xsim.Logger()
    rb = ReplayBuffer(int(ecf.due / ecf.dt + 1))

    xt.info("run simulation")
    for time in tqdm(xsim.generate_step_time(ecf.due, ecf.dt)):
        state = env.state
        action = compute_action(env, K)
        logger.store(time=time, x=state, u=action).flush()

        # simulation update
        env.step(action)

        rb.add(state, action, env.state)

    # retrieve result
    xt.info("make result")
    res = xsim.Retriever(logger)
    res = pd.DataFrame(dict(
        time=res.time(),
        pitch=xt.r2d(res.state(env.IX_T)),
        reference=xt.r2d(res.state(env.IX_C)),
        elevator=xt.r2d(res.action(env.IX_de))
    ))

    if not args.show_fig and not cf.result.fig.show:
        return args

    xt.info("plot result")
    delim = xt.round(xt.r2d([
        env.action_space.low[env.IX_de],
        env.action_space.high[env.IX_de],
    ]), 3)

    fig, axes = plt.subplots(nrows=2, sharex=True)
    # draw zero-lines
    axes[0].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)
    axes[1].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)

    res.plot(
        x="time", y=["reference", "pitch"], ax=axes[0],
        style=["--", "-"], color=COLOR_DATA,
        xlim=[0, ecf.due], ylim=ecf.reference.range, grid=True,
        yticks=np.arange(ecf.reference.range[0], ecf.reference.range[1]+1, 1)
    )
    res.plot(
        x="time", y="elevator", ax=axes[1],
        style="-", color="orange",
        ylim=delim, grid=True,
        yticks=np.arange(delim[0], delim[1]+1, 1)
    )

    plt.show()

def prepare_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--dev", action="store_true")
    parser.add_argument("--config", default="configs/se_no_fail.yaml", type=str)
    # result export
    parser.add_argument("--show-fig", action="store_true")

    return parser.parse_args()

if __name__ == '__main__':
    args = prepare_args()
    if args.dev:
        dev(args)
        exit(0)
    run(args)


#!/usr/bin/python

import urllib
from bs4 import BeautifulSoup
import pandas as pd
from datetime import datetime
import os.path
import time

urls = ["http://www.bbc.co.uk/sport/football/scottish-championship/table?print=true",
        "http://www.bbc.co.uk/sport/football/scottish-league-one/table?print=true",
        "http://www.bbc.co.uk/sport/football/scottish-league-two/table?print=true"]

def getTables():
    date = str(datetime.now().strftime("%Y%m%d-%H%M%S"))
    current_url = 0

    for url in urls:
        html = urllib.urlopen(url)
        soup = BeautifulSoup(html, "html.parser")

        column_headers = [th.getText() for th in soup.findAll('tr', limit=2)[0].findAll('th')]

        data_rows = soup.findAll('tr')[2:12]
        team_data = [[td.getText() for td in data_rows[i].findAll('td')] for i in range(len(data_rows))]

        df = pd.DataFrame(team_data, columns=column_headers)

        # Drop unnecessary columns
        df = df.drop('Last 10 games results', 1)
        df = df.drop('Match status', 1)
        df = df.drop(df.columns[[0]], axis=1)

        # Replace unnecessary strings with simple integers
        for x in range(0,10):
            df['Position'][x] = x + 1

        # Pick a filename depending on the URL we're scraping
        if current_url == 0:
            filename = "/home/richusx/graded-unit/app/tables/archive/championship_%s.html" % (date)
        elif current_url == 1:
            filename = "/home/richusx/graded-unit/app/tables/archive/league-one_%s.html" % (date)
        elif current_url == 2:
            filename = "/home/richusx/graded-unit/app/tables/archive/league-two_%s.html" % (date)

        # Return HTML table
        df.to_html(filename, index=False)

        if current_url == 0:
            filename = "/home/richusx/graded-unit/app/tables/championship.html"
        elif current_url == 1:
            filename = "/home/richusx/graded-unit/app/tables/league-one.html"
        elif current_url == 2:
            filename = "/home/richusx/graded-unit/app/tables/league-two.html"

        if os.path.exists(filename):
            os.remove(filename)

        df.to_html(filename, index=False, classes='table table-striped table-bordered')
        #print ""
        #print df.head(10)
        current_url += 1

def mergeHTML():
    timestamp = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

    # Make sure the file doesn't already exist
    if os.path.exists('/home/richusx/graded-unit/app/templates/tables.html'):
        os.remove('/home/richusx/graded-unit/app/templates/tables.html')

    # Merge all 3 HTML files into one
    tm = open('/home/richusx/graded-unit/app/templates/tables.html', 'a')
    t1 = open('/home/richusx/graded-unit/app/tables/championship.html')
    t2 = open('/home/richusx/graded-unit/app/tables/league-one.html')
    t3 = open('/home/richusx/graded-unit/app/tables/league-two.html')

    tm.write('{% extends "base.html" %}\n{% block content %}')

    tm.write("

Championship League
")
    for line in t1.readlines():
        tm.write(line)
    t1.close()
    tm.write("

League One
")
    for line in t2.readlines():
        tm.write(line)
    t2.close()
    tm.write("

League Two
")
    for line in t3.readlines():
        tm.write(line)
    t3.close()
    tm.write("
Last Updated: %s
Tables get updated automatically every 2 hours\n{%% endblock %%}" % (timestamp))
    tm.close()

if __name__ == "__main__":
    getTables()
    mergeHTML()


import wx, wx.aui, wx.lib.hyperlink
import os
import utils

RELEASE_ARTIST_WIDTH=150
RELEASE_RELEASE_WIDTH=100
RELEASE_DATE_WIDTH=100
RELEASE_FONT_SIZE=8

RELATED_ARTIST_WIDTH=150

ALLOW_AUI_FLOATING = False
LEFT_WIDTH = 160

class ReleasePage( wx.Panel ):
    def __init__( self, parent, id, width, main ):

        wx.Panel.__init__( self, parent, id )

        self.main = main

        self.__uiInit( parent,id,width )

    def __uiInit( self, parent, id, width ):
        self.artistWidth = LEFT_WIDTH
        self.leftWidth = LEFT_WIDTH
        self.rightWidth = width - LEFT_WIDTH - 7
        self.sizer = wx.BoxSizer( wx.VERTICAL )

        self.lrSplitter = wx.SplitterWindow( self, wx.ID_ANY,
                style = wx.SP_LIVE_UPDATE, size=(-1, 480))
        self.lrSplitter.SetMinimumPaneSize( 160 )
        self.lrSplitter.Bind( wx.EVT_SPLITTER_SASH_POS_CHANGING,
                self.OnVertSash )
        self.lrSplitter.Bind( wx.EVT_SPLITTER_SASH_POS_CHANGED,
                self.OnVertSashEnd )
        self.tbSplitter = wx.SplitterWindow( self.lrSplitter, wx.ID_ANY,
                style = wx.SP_LIVE_UPDATE, size=(-1, 480))
        self.tbSplitter.SetMinimumPaneSize( 60 )

        # Create the  left panel with an artist list and a search box
        leftPanel = wx.Panel( self.lrSplitter, wx.ID_ANY,
                style = wx.CLIP_CHILDREN )
        self.searchItems = {}

        self.artistList = wx.ListCtrl( leftPanel, wx.ID_ANY, wx.DefaultPosition,
                                       wx.DefaultSize,
                                       wx.LC_REPORT|wx.LC_SINGLE_SEL )
        self.artistList.Bind( wx.EVT_LIST_COL_END_DRAG, self.OnArtistColDrag,
                self.artistList )

        self.filter = wx.SearchCtrl(leftPanel, style=wx.TE_PROCESS_ENTER)
        self.filter.ShowCancelButton(True)
        self.filter.Bind(wx.EVT_TEXT, self.RecreateArtists)
        self.filter.Bind(wx.EVT_SEARCHCTRL_CANCEL_BTN, self.OnSearchCancelBtn)
        self.filter.Bind(wx.EVT_TEXT_ENTER, self.OnSearch)

        searchMenu = wx.Menu()
        item = searchMenu.AppendRadioItem(-1, "Artist")
        self.Bind(wx.EVT_MENU, self.OnSearchMenu, item)
        item = searchMenu.AppendRadioItem(-1, "Year")
        self.Bind(wx.EVT_MENU, self.OnSearchMenu, item)
        self.filter.SetMenu(searchMenu)

        self.RecreateArtists()
        self.artistList.Bind( wx.EVT_LIST_ITEM_SELECTED, self.OnItemSelected)

        # Create the top pane with the list of releases
        topPanel = wx.Panel( self.tbSplitter, style=wx.CLIP_CHILDREN )
        self.releases = wx.ListCtrl( topPanel, wx.ID_ANY,
                style=wx.LC_REPORT|wx.LC_SINGLE_SEL )
        self.RecreateReleases()

        topBox = wx.BoxSizer( wx.VERTICAL )
        topBox.Add( self.releases, 1, wx.EXPAND )
        topPanel.SetSizer( topBox )
        topPanel.SetAutoLayout( True )

        # Create static text placeholder
        bottomPanel = wx.Panel( self.tbSplitter, style=wx.CLIP_CHILDREN )
        #self.placeholder = wx.StaticText( bottomPanel, wx.ID_ANY,
            #"\n\n Placeholder \n\n" )
        self.link = wx.lib.hyperlink.HyperLinkCtrl( parent=bottomPanel )
        self.whatLink = wx.lib.hyperlink.HyperLinkCtrl( parent=bottomPanel )

        bottomBox = wx.BoxSizer( wx.VERTICAL )
        #bottomBox.Add( self.placeholder, 1, wx.EXPAND )
        bottomBox.Add( self.link, 1, wx.EXPAND )
        bottomBox.Add( self.whatLink, 1, wx.EXPAND )
        bottomPanel.SetSizer( bottomBox )
        bottomPanel.SetAutoLayout( True )

        # Create top/bottom splitter

        self.tbSplitter.SplitHorizontally( topPanel, bottomPanel )

        # Create the left pane
        leftBox = wx.BoxSizer( wx.VERTICAL )
        leftBox.Add( self.artistList, 1, wx.EXPAND )
        leftBox.Add( wx.StaticText( leftPanel, label="Filter artists:" ), 0,
                wx.TOP|wx.LEFT, 5 )
        leftBox.Add( self.filter, 0, wx.EXPAND|wx.ALL, 5 )
        leftPanel.SetSizer( leftBox )
        leftPanel.SetAutoLayout( True )

        # Split left / right
        self.lrSplitter.SplitVertically( leftPanel, self.tbSplitter )

        self.sizer.Add( self.lrSplitter, 1, wx.EXPAND )
        self.SetSizer( self.sizer )
        self.SetAutoLayout( True )
        self.sizer.Fit( self )
        self.Show( True )

    def OnVertSash( self, event ):
        if self.artistList.GetColumnWidth( 0 ) < event.GetSashPosition():
            self.artistList.SetColumnWidth( 0, event.GetSashPosition() )
            self.artistWidth = event.GetSashPosition()

    def OnVertSashEnd( self, event ):
        self.Refresh()

    def OnSearchCancelBtn( self, event ):
        self.filter.SetValue('')
        self.OnSearch()
        self.RecreateReleases()

    def OnSearch( self, item=None ):
        self.RecreateArtists()

    def OnSearchMenu( self, item ):
        x = 1

    def OnItemSelected( self, event ):
        self.RecreateReleases( event )

    def RecreateArtists( self, event=None ):
        self.artistList.DeleteAllColumns()
        self.artistList.DeleteAllItems()
        self.artistList.InsertColumn( 0, "Artist" )
        self.artistList.SetColumnWidth( 0, self.artistWidth )

        index = 0
        for junk,artist in sorted( self.main.library.artists.items() ):
            if( artist['name'].lower().find( self.filter.GetValue().lower() ) > -1 ):
                self.artistList.InsertStringItem( index, artist['name'] )
                index+=1

    def RecreateReleases( self, event=None ):
        self.releases.DeleteAllColumns()
        self.releases.DeleteAllItems()
        self.releases.InsertColumn( 0, "Release" )
        self.releases.InsertColumn( 1, "Have" )
        self.releases.InsertColumn( 2, "Date" )
        self.releases.SetColumnWidth( 0, self.rightWidth * .70 )
        self.releases.SetColumnWidth( 1, self.rightWidth * .09 )
        self.releases.SetColumnWidth( 2, self.rightWidth * .21 )

        if event:
            index = 0
            for junk,artist in self.main.library.artists.items() :
                if( artist['name'] == event.GetItem().GetText() ) :
                    self.link.SetURL( str(artist['aid']) )
                    self.link.SetLabel( artist['name'] )
                    self.link.SetToolTipString( str(artist['aid']) )
                    self.link.SetVisited(False)
                    whatURL = "http://what.cd/artist.php?name=" + str(artist['name'])
                    self.whatLink.SetURL( str(whatURL) )
                    self.whatLink.SetLabel('Search on What.CD' )
                    self.whatLink.SetToolTipString( str(whatURL) )
                    self.whatLink.SetVisited(False)
                    if( artist['aid'] == 0 ):
                        # queue a high priority lookup of this artist
                        self.main.library.lookupReleases( junk, 6 )
                    for rel in artist['releases']:
                        self.releases.InsertStringItem( index, rel.title )
                        self.releases.SetStringItem( index, 1, rel.have and "Y" or "N" )
                        self.releases.SetStringItem( index, 2, rel.date )
                        index += 1

    def InitSize( self ):
        self.lrSplitter.SetSashPosition( LEFT_WIDTH )
        self.tbSplitter.SetSashPosition( -100 )
        self.artistList.SetColumnWidth( 0, LEFT_WIDTH )

    def OnArtistColDrag( self, event ):
        self.artistWidth = event.GetPosition()[0]

    def FormatItem( self, itemNum, list, bg=wx.WHITE, fg=wx.BLACK,
                    size=RELEASE_FONT_SIZE,family=wx.FONTFAMILY_DEFAULT,
                    weight=wx.FONTWEIGHT_NORMAL ):
        item =  list.GetItem( itemNum )
        font = wx.Font( size, family, wx.FONTSTYLE_NORMAL, weight )
        item.SetFont( font )
        item.SetBackgroundColour( bg )
        item.SetTextColour( fg )
        list.SetItem( item )

    def OnResize( self, event ):
        if self.GetAutoLayout():
            self.Layout()


import threading

from pywb.webapp.pywb_init import create_wb_router
from pywb.framework.wsgi_wrappers import init_app

# disable is_hop_by_hop restrictions
import wsgiref.handlers
wsgiref.handlers.is_hop_by_hop = lambda x: False

class ServerThreadRunner(object):
    def __init__(self, make_httpd, config_file=None):

        if config_file:
            self.app = init_app(create_wb_router,
                                load_yaml=True,
                                config_file=config_file)
        else:
            self.app = None

        self.httpd = make_httpd(self.app)
        self.port = self.httpd.socket.getsockname()[1]

        proxy_str = 'http://localhost:' + str(self.port)
        self.proxy_str = proxy_str
        self.proxy_dict = {'http': proxy_str,
                           'https': proxy_str}

        def run():
            self.httpd.serve_forever()

        self.thread = threading.Thread(target=run)
        self.thread.daemon = True
        self.thread.start()

    def stop_thread(self):
        self.httpd.shutdown()


#! /usr/bin/env python

import roslib
roslib.load_manifest('eye_tracking_client')
import rospy
import actionlib

from eye_tracking_client.msg import GoToAction, GoToGoal
from eye_tracking_client.srv import nbOfPosition

class RobotArmController():

    def __init__(self, resume=0):
        self.__currentIdx = resume
        self.__nbOfPosition = 1

        rospy.init_node('GoToClient')
        self.client = actionlib.SimpleActionClient('GoTo', GoToAction)
        self.client.wait_for_server()

        rospy.wait_for_service('nbOfPosition')
        try:
            getNPosition = rospy.ServiceProxy('nbOfPosition', nbOfPosition)
            resp1 = getNPosition()
            self.__nbOfPosition = resp1.nbOfPosition
        except rospy.ServiceException as e:
            print("Service call failed: %s"%e)

    def goto(self,idx):
        if idx < self.__nbOfPosition and idx >= 0:
            goal = GoToGoal(idx)
            self.client.send_goal(goal)
            success = self.client.wait_for_result(rospy.Duration.from_sec(5.0))
            print('Result received, success:{}'.format(success))
            print('Number of positions:{}'.format(self.__nbOfPosition))

            self.__currentIdx = idx
            print(self.__currentIdx, file=open('LastPosition', 'w'))
            return True
        else:
            return False

    def position(self):
        return self.__currentIdx

    def nbPositions(self):
        # A topic subscription.
        return self.__nbOfPosition

if __name__ == '__main__':
    rc = RobotArmController()
    rc.goto(13)


def digitDegree(n):
    degree = 0
    num = str(n)
    while len(num) > 1:
        new = 0
        for i in range(len(num)):
            new += int(num[i])
        num = str(new)
        degree +=1
    return degree

digit = 99
print(digitDegree(digit))

'''
Let's define digit degree of some positive integer as the number of times we need to replace this number with the sum of its digits until we get to a one digit number.

Given an integer, find its digit degree.

Example

    For n = 5, the output should be
    digitDegree(n) = 0;
    For n = 100, the output should be
    digitDegree(n) = 1.
    1 + 0 + 0 = 1.
    For n = 91, the output should be
    digitDegree(n) = 2.
    9 + 1 = 10 -> 1 + 0 = 1.
'''


#Jogo da advinhação
from random import randint

computador = randint(0, 10)
tentativas = 0
acertou = False
while not acertou:
    jogador = int(input('Qual numero eu pensei? '))
    tentativas += 1
    if jogador == computador:
        acertou = True
    else:
        if jogador > computador:
            print('Menos...')
        elif jogador < computador:
            print('Mais...')

print('Muito bem você acertou em {} tentativas!'.format(tentativas))


# fa19-516-162: Cloudmesh Common
# E.Cloudmesh.Common.2
# Develop a program that demonstrates the use of dotdict.

from cloudmesh.common.dotdict import dotdict

student = {"Name": "Shivani",
           "Program": "ISE",
           "University": "IU Bloomington"}

student = dotdict(student)

print(f"My name is {student.Name} and I study at {student.University}.")


# -*- coding: utf-8 -*-
"""
In this code,
1) as in the previous practical we import raster data as our agents' environment
and let our created agents interact with it-move around it, eat and and store eaten- using
the Agent class methods stored in agentframework.
2) we let the agents communicate with each other and change each other variables using
'.share_with_neighbours' method. This method looks for nearest neighbours of agent in specified
distance from it and shares the food with them by splitting their total amount of food evenly.
More details of the method are given in the source code - agentframework.py
3) introduce artifacts concept - patterns or mistakes that result from the model's
operation, not from how well it represents reality -  and avoid them by randomizing the order=shuffling.

@author: Mira
version: 1.0
"""
#import libraries
import random
import time
import agentframework       # module that stores Agent class
import csv

#check how long it takes to run code
start = time.process_time()

#Open environment data file
in_data=open('data.txt', newline='')
reader = csv.reader(in_data, quoting=csv.QUOTE_NONNUMERIC)

#Fill 'environment' list with the data
environment = []
for row in reader:
    rowlist = []
    for value in row:
        rowlist.append(value)
    environment.append(rowlist)
in_data.close()

# Create nested list of agents
agents=[]

#Set number of agents, iterations,neighbourhood
num_of_agents = 10
num_of_iterations=100
neighbourhood=20 #max distance from the agent that is considered its neighbourhood

#Create agents with their random coord-s
for i in range(num_of_agents):
    agents.append(agentframework.Agent(i, agents, environment, neighbourhood))
    #print("Agents' initial coords:", agents[i]) #check the starting locations of the agents

#Move the agents around, let them eat and share with neighbours the eaten(stored)
# for number of iterations
for j in range(num_of_iterations):
    random.shuffle(agents) #shuffle to avoid artifacts
    for i in range(num_of_agents):
        agents[i].move()
        agents[i].eat()
        agents[i].share_with_neighbours(agents)

# Check the new locations of the agents & how much they store
# for i in range(num_of_agents):
# print("Agents' coords after all the actions:", agents[i])

#Check time taken
end = time.process_time()
print("time = " + str(end - start))


import os
import shutil
from pathlib import Path

import pandas as pd
import streamlit as st

from dataset_splitting_by_language import main as split_dataset_by_language

@st.cache_data
def convert_to_csv(df):
    # IMPORTANT: Cache the conversion to prevent computation on every rerun
    return df.to_csv().encode('utf_8_sig')

@st.cache_data
def convert_to_jsonl(df):
    # IMPORTANT: Cache the conversion to prevent computation on every rerun
    return df.to_json(orient='records', lines=True,
                      force_ascii=False).encode('utf_8_sig')

def split_dataset(dataset_file, target_dir):

    del_file = False

    if dataset_file is not None:
        file_contents = dataset_file.getvalue()
        with open(dataset_file.name, 'wb') as f:
            f.write(file_contents)
        dataset_path = dataset_file.name
        del_file = True
    else:
        dataset_path = st.session_state.get('default_demo_dataset')
    split_dataset_by_language(dataset_path, target_dir)

    if del_file:
        os.remove(dataset_file.name)

class Visualize:

    @staticmethod
    def setup():
        st.set_page_config(
            page_title='Data-Juicer',
            page_icon=':smile',
            layout='wide',
            # initial_sidebar_state="expanded",
        )

        readme_link = 'https://github.com/alibaba/data-juicer'
        st.markdown(
            '
  Data-Juicer \

',
            unsafe_allow_html=True,
        )
        st.markdown(
            f'
 A One-Stop Data Processing System for \
                Large Language Models, \
                see more details in our Github
',
            unsafe_allow_html=True,
        )

    @staticmethod
    def split():
        col1, col2 = st.columns(2)
        with col1:
            dataset_file = st.file_uploader(
                label='Upload your custom dataset(jsonl/parquet)',
                type=['json', 'jsonl', 'parquet'])

        with col2:
            st.text_input(label='Default Demo dataset',
                          disabled=True,
                          key='default_demo_dataset',
                          value='data/demo-dataset.jsonl')

        start_btn = st.button('Start to split dataset by language ',
                              use_container_width=True)

        with st.expander('Splitted Results', expanded=True):

            target_dir = 'outputs'
            if start_btn:
                with st.spinner('Wait for splitting...'):
                    shutil.rmtree(target_dir, ignore_errors=True)
                    split_dataset(dataset_file, target_dir)

            splitted_datasets = Path(target_dir).glob('*.jsonl')
            datasets_dict = {
                ds_file.stem: ds_file
                for ds_file in splitted_datasets
            }
            count = len(datasets_dict)
            if count > 1:
                st.header(f'There are :red[{len(datasets_dict)}] languages:\
                         {[ lang for lang in datasets_dict.keys()]}')
            elif count == 1:
                st.header(f'There is :red[{len(datasets_dict)}] language: \
                        {[ lang for lang in datasets_dict.keys()]}')

            for k, v in datasets_dict.items():
                df = pd.read_json(str(v), lines=True)
                count = len(df.index)
                be = 'is' if count <= 1 else 'are'
                samples = 'sample ' if count <= 1 else 'samples'
                st.subheader(
                    f'There {be} :red[ {count} ] {samples} in :red[{k}]')
                st.dataframe(df, use_container_width=True)
                st.download_button(f'Download {k} as JSONL',
                                   data=convert_to_jsonl(df),
                                   file_name=v.name)

            shutil.rmtree(target_dir, ignore_errors=True)

    @staticmethod
    def visualize():
        Visualize.setup()
        Visualize.split()

def main():
    Visualize.visualize()

if __name__ == '__main__':
    main()


with open("input.txt", 'r') as f:
    a = [i.rstrip() for i in f.readlines()]

def p1():
    epsilon = ''
    gamma = ''
    nlines = len(a)
    linelen = len(a[0])
    for j in range(linelen):
        count = 0
        for i in range(nlines):
            count += int(a[i][j])
        if count/nlines > 0.5:
            gamma += '1'
            epsilon += '0'
        if count/nlines < 0.5:
            epsilon += '1'
            gamma += '0'
    return int(gamma, 2)*int(epsilon, 2)

def p2():
    def counter(inputlines, index, co2count):
        if len(inputlines) == 1:
            return inputlines[0]
        keep = []
        numbers = []
        for line in inputlines:
            numbers += (line[index])
        if numbers.count('1') >= numbers.count('0'):
            number = '0' if co2count else '1'
        else:
            number = '1' if co2count else '0'
        for line in inputlines:
            if line[index] == number:
                keep.append(line)
        return counter(keep, index+1, co2count)

    ogr = counter(a, 0, False)
    csr = counter(a, 0, True)
    return int(ogr, 2)*int(csr, 2)

print(1, p1())
print(2, p2())


import pandas as pd
import os
from config import *
import matplotlib.pyplot as plt
import numpy as np
from pydts.data_generation import EventTimesSampler

final_dfs = []
n_patients_list = [250, 500]
for n_patients in n_patients_list:
    case = f'n{n_patients}_beta_results_comparison.csv'

    df = pd.read_csv(os.path.join(OUTPUT_DIR, case))
    df.index = [r'$\beta_{11}$', r'$\beta_{12}$', r'$\beta_{13}$', r'$\beta_{14}$', r'$\beta_{15}$',
                 r'$\beta_{21}$', r'$\beta_{22}$', r'$\beta_{23}$', r'$\beta_{24}$', r'$\beta_{25}$']

    df.columns = pd.MultiIndex.from_tuples([('', 'True'),
                                            ('Lee et al.', 'Estimate'),
                                            ('Lee et al.', 'Estimated SE'),
                                            ('two-step', 'Estimate'),
                                            ('two-step', 'Estimated SE'),
                                            ('two-step', 'Empirical SE'),
                                            ('two-step', 'Coverage Rate')])

    final_dfs.append(df.astype(float).round(3))

final_df = pd.concat(final_dfs, keys=n_patients_list)
print(final_df.to_latex(escape=False))

filename = 'alpha_small_dataset.png'

d_times = 7
n_cov = 5
j_events = 2

real_coef_dict = {
    "alpha": {
        1: lambda t: -1.4 + 0.4 * np.log(t),
        2: lambda t: -1.3 + 0.4 * np.log(t)
    },
    "beta": {
        1: -0.7*np.log([0.8, 3, 3, 2.5, 2]),
        2: -0.6*np.log([1, 3, 4, 3, 2])
    }
}

ets = EventTimesSampler(d_times=d_times, j_event_types=j_events)
covariates = [f'Z{i + 1}' for i in range(n_cov)]

first_model_name = 'Lee et al.'
second_model_name = 'two-step'
times = range(1, d_times + 1)

lee_colors = ['tab:blue', 'tab:green']
two_step_colors = ['navy', 'darkgreen']
true_colors = ['tab:blue', 'tab:green']

fig, axes = plt.subplots(1, 2, figsize=(12, 5))

for idn, n_patients in enumerate(n_patients_list):
    case = f'n{n_patients}_results_comparison.csv'

    np.random.seed(idn)
    patients_df = pd.DataFrame(data=pd.DataFrame(data=np.random.uniform(0, 1, size=[n_patients, n_cov]),
                                                 columns=covariates))

    patients_df = ets.sample_event_times(patients_df, hazard_coefs=real_coef_dict, seed=idn)
    patients_df = ets.sample_independent_lof_censoring(patients_df,
                                                       prob_lof_at_t=0.02 * np.ones(d_times))
    patients_df = ets.update_event_or_lof(patients_df)
    patients_df.index.name = 'pid'
    patients_df = patients_df.reset_index()
    counts = patients_df.groupby(['J', 'X'])['pid'].count().unstack('J').fillna(0)

    alpha_res_df = pd.read_csv(os.path.join(OUTPUT_DIR, case))

    ax = axes[int(idn % 2)]
    ax.set_title(f'n={n_patients}', fontsize=15)
    ax.tick_params(axis='both', which='major', labelsize=15)
    ax.tick_params(axis='both', which='minor', labelsize=15)

    for j in [1, 2]:
        tmp_lee_alpha = alpha_res_df.iloc[((d_times+n_cov)*(j-1)):((d_times+n_cov)*(j-1)+d_times), 1]
        ax.scatter(times, tmp_lee_alpha.values,
                   label=f'J={j} ({first_model_name})', color=lee_colors[j - 1], marker='o', alpha=0.4, s=40)

        tmp_two_step_alpha = alpha_res_df.iloc[((d_times+n_cov)*(j-1)):((d_times+n_cov)*(j-1)+d_times), 3]
        ax.scatter(times, tmp_two_step_alpha.values,
                   label=f'J={j} ({second_model_name})', color=two_step_colors[j - 1], marker='*', alpha=0.7, s=20)

        true_values = [real_coef_dict['alpha'][j](t) for t in times]
        ax.plot(times, true_values, label=f'J={j} (True)', ls='--', color=true_colors[j - 1])

        ax.set_xlabel(r'Time', fontsize=18)
        ax.set_ylabel(r'$\alpha_{jt}$', fontsize=18)
        ax.legend(loc='upper right', fontsize=12)
        ax.set_ylim([-2.2, 1])
        ax.set_xlim([0, d_times+1])

    ax2 = ax.twinx()
    ax2.bar(counts.index, counts[1].values.squeeze(), label='J=1', color='tab:red', alpha=0.4, width=0.5)
    ax2.bar(counts.index, counts[2].values.squeeze(), label='J=2', color='tab:brown', alpha=0.6, align='edge',
            width=0.5)
    ax2.legend(loc='upper left', fontsize=12)
    ax2.set_ylabel('Number of observed events', fontsize=16, color='red')
    ax2.tick_params(axis='y', colors='red')
    ax2.set_ylim([0, 150])
    ax2.tick_params(axis='both', which='major', labelsize=15)
    ax2.tick_params(axis='both', which='minor', labelsize=15)
fig.tight_layout()

if filename is not None:
    fig.savefig(os.path.join(OUTPUT_DIR, filename), dpi=300)


import logging
import socket
import threading
import argparse
import pickle

from rich.console import Console
from rich.logging import RichHandler

from hmg.word_utils import get_word_from_internet

FORMAT_LOGGING = "%(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT_LOGGING, handlers=[RichHandler()])

logger = logging.getLogger("hmg-server")

MAX_PLAYERS = 2

console = Console()

class Server:
    def __init__(self, host, port, secret_word=None):
        self.host = host
        self.port = port
        self.secret_word = secret_word
        self._using_word_arg = secret_word is not None
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._players = {}

    def _update_secret_word(self):
        with console.status("[blue]Searching for word, wait..."):
            self.secret_word = get_word_from_internet()
        logger.info("Secret word=%s", self.secret_word)

    def serve_forever(self):
        self._socket.bind((self.host, self.port))
        self._socket.listen(2)
        logger.info("Server ready and listening on %s:%s", self.host, self.port)
        self._listen_connection()

    def _listen_connection(self):
        is_listening = True
        while is_listening:
            if len(self._players) == MAX_PLAYERS:
                if not self._using_word_arg:
                    self._update_secret_word()

                for _, sock in self._players.items():
                    sock.send(pickle.dumps({"msg": self.secret_word}))

                player1, player2 = self._players.values()
                payload = {"turn": False, "win": False, "msg": ""}
                player1.send(pickle.dumps(payload))
                payload = {"turn": True, "win": False, "msg": ""}
                player2.send(pickle.dumps(payload))

            conn, addr = self._socket.accept()
            incoming_message = conn.recv(1024)
            if not incoming_message:
                continue
            data = pickle.loads(incoming_message)
            username = data["user"]

            if username in self._players.keys():
                username += "_x"

            logger.info("Username=%s with address=%s connected", username, addr)
            self._players[username] = conn

            client_thread = threading.Thread(
                target=self._handle_client, args=(conn, addr, username)
            )
            client_thread.start()

    def _handle_client(self, client, addr, username):
        while True:
            logger.info("Waiting for client message...")
            msg_client = client.recv(1024)
            if not msg_client:
                logger.warning("Client=%s disconnected", username)
                del self._players[username]
                break

            data = pickle.loads(msg_client)
            logger.info("Sending data=%s", data)

            for user, sock in self._players.items():
                if user != username:
                    data["turn"] = True
                else:
                    data["turn"] = False
                sock.send(pickle.dumps(data))

def get_cli_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-H", "--host", default=socket.gethostname())
    parser.add_argument("-p", "--port", type=int, default=8889)
    parser.add_argument(
        "-w",
        "--word",
        default=None,
        help="Word to use instead of internet (usefull for testing)",
    )

    return parser

def main():
    args = get_cli_parser().parse_args()

    server = Server(host=args.host, port=args.port, secret_word=args.word)
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        logger.info("Exiting...")

if __name__ == "__main__":
    main()


import os
from shutil import copyfile

import lxml.etree as etree

class config:

    def __init__(self, os=None, device=None, h=None, w=None, output=None, num_update_steps=None,
                 num_level_per_agent=None, num_worker=None, multiagent=None, training_attempts_per_level=None,
                 memory_size=None, memory_type=None, singleagent=None, lr=None, train_time_per_ep=None,
                 train_time_rise=None, train_batch=None, gamma=None, eps_start=None, eps_test=None, network=None,
                 reward_type=None, eval_freq=None, train_template=None, test_template=None, simulation_speed=None,
                 online_training=None, test_steps=None):
        # operating system
        self.test_steps = test_steps
        self.online_training = online_training
        self.os = os

        # pytorch parameters
        self.device = device

        # image network parameters
        self.h = h
        self.w = w
        self.output = output

        # multiagent trainning parameters
        self.num_update_steps = num_update_steps
        self.num_level_per_agent = num_level_per_agent
        self.num_worker = num_worker
        self.multiagent = multiagent
        self.training_attempts_per_level = training_attempts_per_level
        self.memory_size = memory_size
        self.memory_type = memory_type

        # single agent training parameters
        self.singleagent = singleagent

        # general trainning parameters

        self.lr = lr
        self.train_time_per_ep = train_time_per_ep
        self.train_time_rise = train_time_rise
        self.train_batch = train_batch
        self.gamma = gamma
        self.eps_start = eps_start
        self.eps_test = eps_test  # randomeness in testing
        self.network = network
        self.reward_type = reward_type  # 'score' / 'passing'
        self.simulation_speed = simulation_speed
        self.eval_freq = eval_freq

        self.train_template = train_template  # ['1_1_1', '1_1_2'] level 1 capability 1 template 1 and 2 for training
        self.test_template = test_template  # ['1_1_3', '1_1_4'] level 1 capability 1 template 3 and 4 for testing

        self.train_level_list = []
        self.test_level_list = []
        self.total_level = 0
        self.target_level_path = '../tasks/generated_tasks/'
        self.origin_level_path = '../buildgame/{}/9001_Data/StreamingAssets/Levels/novelty_level_1/type1/Levels/'.format(
            self.os)
        self.game_level_path = '9001_Data/StreamingAssets/Levels/novelty_level_1/type1/Levels/'.format(self.os)
        self.game_config_path = '../buildgame/{}/config.xml'.format(self.os)
        self.update_level_index()

    def update_level_index(self):
        '''
        by taking the TRAIN_TEMPLATE and TEST_TEMPLATE,
        this function create the config list for training and testing

        :return:
        '''

        # remove all the levels in self.origin_level_path
        old_levels = os.listdir(self.origin_level_path)
        for old_level in old_levels:
            os.remove(os.path.join(self.origin_level_path, old_level))

        if len(self.train_template) == 1 and self.train_template == self.test_template:  # within template training and testing
            # by default using 80% of levels to train and 20% for testing
            train_percent = 0.8
            level, capability, template_idx = self.train_template[0].split('_')
            template_path = os.path.join(self.target_level_path, level, capability, template_idx)

            # copy all target levels to self.origin_level_path
            new_levels = sorted(os.listdir(template_path), key=lambda x: int(x.split('.')[0].split("_")[-1]))
            total_template_level_path = []
            for new_level in new_levels:
                src_path = os.path.join(template_path, new_level)
                dst_path = os.path.join(self.origin_level_path, new_level)
                copyfile(src_path, dst_path)
                total_template_level_path.append(os.path.join(self.game_level_path, new_level))
            self.train_level_list = list(range(1, int(len(total_template_level_path) * train_percent) + 1))
            self.test_level_list = list(
                range(int(len(total_template_level_path) * train_percent) + 1, len(total_template_level_path) + 1))

            # write configs
            parser = etree.XMLParser(encoding='UTF-8')
            game_config = etree.parse(self.game_config_path, parser=parser)
            config_root = game_config.getroot()
            # remove old level path
            for level in list(config_root[1][0][0]):
                config_root[1][0][0].remove(level)
            # add new level path
            for l in total_template_level_path:
                new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
                new_level.set('level_path', l)

            # add a repeated level for the weird not loadding last level bug
            new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
            new_level.set('level_path', l)

            game_config.write(self.game_config_path)

        else:  # when more than one template is required to train
            # cross template testing
            total_template_level_path = []

            for template in self.train_template:
                level, capability, template_idx = template.split('_')
                template_path = os.path.join(self.target_level_path, level, capability, template_idx)

                # move all levels to folder
                new_levels = sorted(os.listdir(template_path), key=lambda x: int(x.split('.')[0].split("_")[-1]))
                for new_level in new_levels:
                    src_path = os.path.join(template_path, new_level)
                    dst_path = os.path.join(self.origin_level_path, new_level)
                    copyfile(src_path, dst_path)
                    total_template_level_path.append(os.path.join(self.game_level_path, new_level))

            self.train_level_list = [i for i in range(1, len(total_template_level_path) + 1)]

            for template in self.test_template:
                level, capability, template_idx = template.split('_')
                template_path = os.path.join(self.target_level_path, level, capability, template_idx)

                # move all levels to folder
                new_levels = sorted(os.listdir(template_path), key=lambda x: int(x.split('.')[0].split("_")[-1]))

                # takes only the last 20 levels per each template for testing
                new_levels = new_levels[80:]

                for new_level in new_levels:
                    src_path = os.path.join(template_path, new_level)
                    dst_path = os.path.join(self.origin_level_path, new_level)
                    copyfile(src_path, dst_path)
                    total_template_level_path.append(os.path.join(self.game_level_path, new_level))

            self.test_level_list = [i for i in range(self.train_level_list[-1] + 1, len(total_template_level_path) + 1)]

            # write config
            # write configs
            parser = etree.XMLParser(encoding='UTF-8')
            game_config = etree.parse(self.game_config_path, parser=parser)
            config_root = game_config.getroot()
            # remove old level path
            for level in list(config_root[1][0][0]):
                config_root[1][0][0].remove(level)
            # add new level path
            for l in total_template_level_path:
                new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
                new_level.set('level_path', l)

            # add a repeated level for the weird not loadding last level bug
            new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
            new_level.set('level_path', l)

            game_config.write(self.game_config_path)

if __name__ == '__main__':
    from LearningAgents.DQNImageAgent import DQNImageAgent
    from LearningAgents.RL.DQNImageDueling import DQNImageDueling

    test_config = {
        # operating system
        'os': "Linux",
        # pytorch parameters
        'device': "cuda:0",

        # image network parameters
        'h': 224,
        'w': 224,
        'output': 180,

        # multiagent trainning parameters
        'num_update_steps': 100,
        'num_level_per_agent': 20,
        'num_worker': 30,
        'multiagent': DQNImageAgent,

        # single agent training parameters
        'singleagent': None,

        # general trainning parameters

        'train_time_per_ep': 32,
        'train_time_rise': 1.5,
        'train_batch': 32,
        'gamma': 0.99,
        'eps_start': 0.95,
        'eps_decay': 0.99,

        'network': DQNImageDueling,

        'train_template': ['1_1_1'],  # ['1_1_1', '1_1_2'] level 1 capability 1 template 1 and 2 for training
        'test_template': ['1_1_3'],  # ['1_1_3', '1_1_4'] level 1 capability 1 template 3 and 4 for testing
    }
    c = config(**test_config)

    print(c.train_level_list)
    print(c.test_level_list)


import sqlite3
import json
from tqdm import tqdm

class DatabaseConnection:
    def __init__(self, db_file, topic_json, heading_json, alltree_json):
        self.db_file = db_file
        self.connection = None
        self.topic_json = json.loads(open(topic_json, "r").read())
        self.heading_json = json.loads(open(heading_json, "r").read())
        self.alltree_json = json.loads(open(alltree_json, "r").read())
        self.connect()

    def connect(self):
        try:
            self.connection = sqlite3.connect(self.db_file)
            print("Connected to the database")
        except sqlite3.Error as e:
            print(f"Error connecting to the database: {e}")

    def disconnect(self):
        if self.connection:
            self.connection.close()
            print("Disconnected from the database")
        else:
            print("No active connection to disconnect")

    def __table(self):
        cursor = self.connection.cursor()
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
        print(cursor.fetchall())

    def insert_topic(self, table_name, json_topic):
        cursor = self.connection.cursor()
        for index, data_topic in enumerate(json_topic):
            cursor.execute(
                f"INSERT INTO {table_name} VALUES ('{data_topic['Value']}', '{data_topic['Text']}');")
        self.connection.commit()

    def insert_heading(self, table_name, json_heading):
        cursor = self.connection.cursor()
        for index, value in enumerate(json_heading):
            id_topic = value['ChuDe']
            id_heading = value['Value']
            rank = value['STT']
            data = value['Text']

            # get topic from id_topic
            cursor.execute(
                "SELECT * FROM app_topic WHERE topic_id=?", (id_topic,))
            topic = cursor.fetchone()

            # insert heading
            if topic:
                cursor.execute(
                    f"INSERT INTO {table_name} VALUES (?, ?, ?, ?);", (id_heading, data, rank, topic[0]))
            else:
                print(f"Topic with id {id_topic} not found.")

        self.connection.commit()

    def insert_article(self, table_name, json_article):
        cursor = self.connection.cursor()
        for index, value in enumerate(json_article):
            id_article = value['ID']
            rank = value['ChiMuc']
            mapc = value['MAPC']
            name_article = value['TEN']
            id_heading = value['DeMucID']

            # Retrieve heading object
            cursor.execute(
                "SELECT * FROM app_heading WHERE heading_id=?", (id_heading,))
            heading = cursor.fetchone()

            if heading:
                # Instead of passing the entire heading object, pass its primary key
                cursor.execute(
                    f"INSERT INTO {table_name} VALUES (?, ?, ?, ?, ?);",
                    (id_article, rank, mapc, name_article, heading[0]),
                )
            else:
                print(f"Heading with id {id_heading} not found.")

        self.connection.commit()

    def article(self, table_name, json_article, len_MAPC=20):
        print(f"INSERT DATA {len_MAPC}")
        cursor = self.connection.cursor()
        json_data = dict()
        for index, value in enumerate(json_article):
            if len(value["MAPC"]) not in json_data.keys():
                json_data[len(value["MAPC"])] = []
            json_data[len(value["MAPC"])].append(value)

        json_data = json_data[len_MAPC]
        for index, value in tqdm(enumerate(json_data)):
            id_article = value['ID']
            rank = value['ChiMuc']
            mapc = value['MAPC']
            name_article = value['TEN']
            id_topic = value['ChuDeID']
            id_heading = value['DeMucID']

            cursor.execute(
                "SELECT * FROM app_topic WHERE topic_id=?", (id_topic,))
            topic = cursor.fetchone()
            cursor.execute(
                "SELECT * FROM app_heading WHERE heading_id=?", (id_heading,))
            heading = cursor.fetchone()

            if topic and heading:

                cursor.execute(
                    f"SELECT * FROM {table_name} WHERE mapc = ?", (mapc[:-20],))
                mapc_parent = cursor.fetchone()
                if mapc_parent is not None:
                    mapc_parent = mapc_parent[0]
                cursor.execute(
                    f"INSERT INTO {table_name} VALUES (?, ?, ?, ?, ?, ?, ?);",
                    (id_article, rank, mapc, name_article,
                     heading[0], mapc_parent, topic[0]),
                )
        self.connection.commit()

database = DatabaseConnection("db.sqlite3",
                              "static/JSON/topics_json.json",
                              "static/JSON/headings_json.json",
                              "static/JSON/allTree_json.json")

database.insert_topic("app_topic", database.topic_json)
database.insert_heading("app_heading", database.heading_json)
database.article("app_article", database.alltree_json, 20)
database.article("app_article", database.alltree_json, 40)
database.article("app_article", database.alltree_json, 60)
database.article("app_article", database.alltree_json, 80)
database.article("app_article", database.alltree_json, 100)
database.article("app_article", database.alltree_json, 59)
database.article("app_article", database.alltree_json, 79)


import os
import sys
import argparse
from pathlib import Path
from pdf2image import convert_from_path
from shutil import copyfile

def convert(input_pdf_path, output_images_dir_path):
    print(input_pdf_path, output_images_dir_path)

    # poppler/binを環境変数PATHに追加する
    # poppler_dir = Path(__file__).parent.absolute() / "poppler/bin"
    # os.environ["PATH"] += os.pathsep + str(poppler_dir)

    # PDFファイルのパス
    pdf_path = Path("./pdf_file/pdf3009p.pdf")

    # PDF -> Image に変換（150dpi）
    pages = convert_from_path(input_pdf_path, 150)

    # 画像ファイルを１ページずつ保存
    image_dir = Path(output_images_dir_path)
    for i, page in enumerate(pages):
        file_name = pdf_path.stem + "_{:02d}".format(i + 1) + ".jpeg"
        image_path = image_dir / file_name
        # JPEGで保存
        page.save(str(image_path), "JPEG")

def main():
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument('input', help='Relative or absolute path of the input PDF file')
    parser.add_argument('-o', '--out', help='Relative or absolute path of the output directory of images')
    args = parser.parse_args()

    # In case no output directory is specified, store in ./temp
    if not args.out:
        args.out = './temp'
        copyfile(args.input, args.input.replace(".pdf", "_BACKUP.pdf"))

    # Run
    convert(args.input, args.out)

if __name__ == '__main__':
    main()


import re

print('*** EMAIL VALIDATOR ***')
print('Type in an email you want to check and I will tell you if it is valid.')
while(True):
    email = input('Your email: ')
    if re.match('\S+@[a-z]+\.[a-z]+',email):
        print('Your e-mail is correct!')
    else:
        print('Your e-mail is not correct!')
    end = input('If you want to exit, press \'x\' then \'Enter\'\nIf you want to check another e-mail, type any other key and then \'Enter\'\n')
    if end == 'x':
        break


from helper import *

class expect_triggerscope_error:
    def __enter__(self):
        pass

    def __exit__(self, type, e, traceback):
        if isinstance(e, TriggerScopeError):
            print(" - Error as expected : %s" % e)
            return True
        else:
            raise RuntimeError("EPError not thrown")

with expect_triggerscope_error():
    RawTrigger()

@EUDFunc
def f_add(a, b):
    return a + b

@EUDFunc
def f_mul(a, b):
    return a * b

@EUDFunc
def f_div(a, b):
    return a // b, a % b

@EUDFunc
def f_addmul(a, b):
    return a + b, a * b

noretCheck = EUDVariable()

@EUDFunc
def testnoret():
    noretCheck << 1

@TestInstance
def test_dfptr():
    p = EUDFuncPtr(2, 1).alloc()

    # Test - Simple pattern
    p << f_add
    a = p(7, 9)

    p << f_mul
    b = p(7, 9)

    with expect_eperror():
        p << f_div  # Should trigger error

    test_equality("Simple dfptr call", [a, b], [16, 63])

    # Test2 - multi return
    q = EUDFuncPtr(2, 2).alloc()
    q << f_div
    a1, a2 = q(12, 5)
    q << f_addmul
    b1, b2 = q(12, 5)
    q << f_div
    c1, c2 = q(17, 3)
    test_equality("Function dpointer test", [a1, a2, b1, b2, c1, c2], [2, 2, 17, 60, 5, 2])

    # Test 3 - no arg no ret
    r = EUDFuncPtr(0, 0).alloc()
    r << testnoret
    r()
    test_assert("dFunction pointer test (0.0 Func)", noretCheck == 1)

    # Test 4 - transfer fptr to fptr
    r = EUDFuncPtr(2, 1).cast(p)
    p << f_mul
    a, b = p(7, 9), r(7, 9)
    r << f_add
    c, d = p(7, 9), r(7, 9)
    test_equality("Indirect dfptr call", [a, b, c, d], [63, 63, 16, 16])

    # Test 5 - fptr cloning
    s = p.copy()
    p << f_mul
    e, f = p(7, 9), s(7, 9)
    test_equality("dFptr cloning", [e, f], [63, 16])

# Nested fptr call
@EUDTypedFunc([EUDFuncPtr(1, 1)])
def f1(f):
    return 4 + f(4) + 6

@EUDFunc
def f(x):
    return x * x

@TestInstance
def test_nested_dfptr():
    k = EUDFuncPtr(1, 1).alloc()
    k << f1
    test_equality("Nested dfptr", k(EUDFuncPtr(1, 1)(f)), 26)


from django.shortcuts import render
from django.urls import reverse_lazy,reverse
# from django.core.urlresolvers import reverse
# Create your views here.
from django.http import HttpResponse
from django.views.generic.edit import FormView
from django.views.generic.edit import CreateView, DeleteView, UpdateView
from django.views.generic import DetailView,ListView
from Products.models import Product
from Products.forms import ProductForm

class ProductCreate(CreateView):
    form_class = ProductForm
    template_name="Products/product_create.html"
    success_url = '/'
    def form_valid(self, form):

        form.save()
        return super(ProductCreate, self).form_valid(form)

    def form_invalid(self, form):
        # print "form is invalid"
        return HttpResponse("form is invalid.. this is just an HttpResponse object")

    def get_success_url(self):
        return reverse("Product-list")

    def get_context_data(self, **kwargs):
        kwargs["object_list"] = Product.objects.all()
        return super(ProductCreate, self).get_context_data(**kwargs)

class ProductUpdate(UpdateView):
    model = Product
    form_class = ProductForm
    template_name_suffix = '_update_form'
    def get_success_url(self):
        return reverse("Product-list")

class ProductDelete(DeleteView):
    # form_class = ProductForm
    model= Product
    success_url = reverse_lazy('Product-list')
    items_to_delete = []

class ProductDetailView(DetailView):
    model= Product
class ProductListView(ListView):
    queryset= Product.objects.all()


from os import read
import time

def readInputFile(fileName) :
  with open(f"{__file__.rstrip('code.py')}{fileName}",'r') as f :
    return [line.strip().replace('x','0').split(",") for line in f.read().splitlines()]

def findIDOfEarliestBus(inputlist) :
  estTimestamp = int(inputlist[0][0])
  earlTimeStamp = estTimestamp
  earlbus = 0

  for bus in list(map(int, inputlist[1])):
    if bus > 0:
      bus = int(bus)
      bustime = bus * (estTimestamp//bus + 1)

      if earlbus == 0:
        earlTimeStamp = bustime
        earlbus = bus
      elif bustime < earlTimeStamp :
        earlTimeStamp = bustime
        earlbus = bus

  # print(earlTimeStamp, earlbus)
  return earlbus * (earlTimeStamp - estTimestamp)

def findTimestampForSubsequent(inputlist):
  earlTimeStamp, increment = int(inputlist[0][0]), 1

  for i, bus in enumerate(list(map(int, inputlist[1]))):
    if bus > 0 :
      while (earlTimeStamp + i) % bus != 0:
          earlTimeStamp += increment
      increment *= bus

  return earlTimeStamp

def test():
  inputlist = readInputFile("example.txt")
  assert findIDOfEarliestBus(inputlist) == 295
  assert findTimestampForSubsequent(inputlist) == 1068781

def main():
  inputlist = readInputFile("input.txt")
  print("Earliest Bus ID * Wait Time - [" ,findIDOfEarliestBus(inputlist),"]")
  print("Earliest Timestamp for all buses to depart Subsequently - [",findTimestampForSubsequent(inputlist),"]")

if __name__ == "__main__" :
  test_start_time = time.time()
  test()
  print("Total Execution Time [{}] seconds for Test".format(time.time() - test_start_time))
  main_start_time = time.time()
  main()
  print("Total Execution Time [{}] seconds for Main".format(time.time() - main_start_time))


from qwerqerqwerqweqwerqwerqwerqwerqwerwewqrwrerewre import get_date
from random import randint
from aiogram import Router, Dispatcher
from aiogram.types import Message,InlineKeyboardMarkup,InlineKeyboardButton,CallbackQuery
from aiogram import Bot
from aiogram.types.dice import DiceEmoji
from aiogram import F
from aiogram.types import ContentType
from aiogram.filters import Text
from aiogram.filters.command import Command
from asyncio import run, sleep
from aiogram.types import Poll, PollAnswer
from abc import ABC,abstractclassmethod
from xixi import my_token
#from asyncio import run
City=InlineKeyboardMarkup(
    inline_keyboard= [
    [
    InlineKeyboardButton(text="Tashkent",callback_data="Tashkent"),
    InlineKeyboardButton(text="Fergana",callback_data="Fergana"),
    InlineKeyboardButton(text="Andijon",callback_data="Andijon"),
    ],
    [
    InlineKeyboardButton(text="Namangan",callback_data="Namangan"),
    InlineKeyboardButton(text="Qashqadaryo",callback_data="Qashqadaryo"),
    InlineKeyboardButton(text="Sirdaryo",callback_data="Sirdaryo"),
    ],
    [
    InlineKeyboardButton(text="Surhondaryo",callback_data="Termez"),
    InlineKeyboardButton(text="Samarqand",callback_data="Samarqand"),
    InlineKeyboardButton(text="Xorazm",callback_data="Urgench"),
    ],
    [
    InlineKeyboardButton(text="Buxoro",callback_data="Buxoro"),
    InlineKeyboardButton(text="Navoiy",callback_data="Navoiy"),
    InlineKeyboardButton(text="Jizzax",callback_data="Jizzax"),
    ],
    [
    InlineKeyboardButton(text="Qoraqolpog'iston",callback_data="Karakalpakstan"),
    ],
    ],
)
great=Router()
@great.message(Command(commands=["start"]))
async def great_message(msg:Message,bot:Bot):
    await msg.answer("Bu 12 viloyat ob-havo ma'lumoti boti",reply_markup=City)

@great.callback_query()
async def get_weather(call:CallbackQuery):
    city=call.data
    print(city)
    weather=get_date(city)
    await call.answer(f"{weather} in C")

async def start():
    dp=Dispatcher()
    kb=Bot(my_token)#token
    dp.include_router(great)
    try:
        await dp.start_polling(kb)
    except Exception as f:
        await kb.session.close()
if __name__=="__main__":
    run(start())


import os
import json
import time
import flask
import openai

from flask import (
    request, jsonify
)
from datetime import datetime
from dotenv import load_dotenv

load_dotenv()

openai.api_key = os.getenv('OPENAI_API_KEY')

app = flask.Flask(__name__)
app.config["DEBUG"] = True

SP = """You are an AI assitant that takes in a piece of text and tries to summarize it in 2-5 sentences."""

TP = """You are an AI assitant that takes in a piece of text and extracts topics from the text."""

def complete(system_prompt: str, user_prompt: str) -> str:
    start = time.time()
    completion = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=[
            {
                "role": "system",
                "content": system_prompt
            },
            {
                "role": "user",
                "content": user_prompt
            }
        ],
        max_tokens=400
    )
    time_to_complete = time.time() - start
    return completion.choices[0].message.content, time_to_complete

@app.route('/', methods=['POST'])
def home():
    return "
Hello from Azure Hackthon! :)
"

@app.route('/api/v1/summarize', methods=['POST'])
def summarize():
    content_type = request.headers.get('Content-Type')
    if content_type == 'application/json':
        data = json.loads(request.data)
        if data.get('context'):
            summary, ttc = complete(
                system_prompt=SP,
                user_prompt=data.get('context')
            )

            return {
                "status": "success",
                "message": summary,
                "timeToComplete": ttc,
                "timeStamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            }
        else:
          return {
            "status": "error",
            "message": "Context to summarize must be provided."
        }
    else:
        return {
            "status": "error",
            "message": "Unsupported content type."
        }

@app.route('/api/v1/topics', methods=['POST'])
def topcis():
    content_type = request.headers.get('Content-Type')
    if content_type == 'application/json':
        data = json.loads(request.data)
        if data.get('context'):
            if data.get('numberTopics'):
                topics, ttc = complete(
                    system_prompt=TP,
                    user_prompt=f"Generate {data.get('numberTopics')} topics, not numbered, seperated by a newline for the following text:\n{data.get('context')}\n\nTopics:\n"
                )

                return {
                    "status": "success",
                    "message": topics.split('\n'),
                    "timeToComplete": ttc,
                    "timeStamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                }
            else:
                return {
                    "status": "error",
                    "message": "Number of topics must be provided."
                }
        else:
          return {
            "status": "error",
            "message": "Context to summarize must be provided."
        }
    else:
        return {
            "status": "error",
            "message": "Unsupported content type."
        }

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=7860)


# -*- coding: utf-8 -*-
"""
Created on Tue Jan 14 01:14:25 2020

@author: SHOBHIT SAINI
"""
import numpy as np
import cv2
# Import the image
img = cv2.imread('D:\IIT_BBS_CLG_WORK\CV_Practice\island.jpeg')
cv2.imshow('BGR_img_default', img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
cv2.imshow('RGB_IMG', img)
cv2.waitKey(0)
cv2.destroyAllWindows()


from django.contrib import admin
from import_export import resources
from import_export.admin import ImportExportModelAdmin
from django.conf import settings
from django.conf.urls import url
from django.contrib import messages
from django.shortcuts import redirect
from pleethai.models import Word, SysWordJapanese, SysWordConnector, WordClass, Example, Constituent, Tag, TaggedItem
from pleethai.common import Common
from taggit.utils import _parse_tags
from django.db import models

class WordResource(resources.ModelResource):

    class Meta:
        model = Word

class WordClassResource(resources.ModelResource):

    class Meta:
        model = WordClass

class ExampleResource(resources.ModelResource):

    class Meta:
        model = Example

class ConstituentResource(resources.ModelResource):

    class Meta:
        model = Constituent

class TagResource(resources.ModelResource):

    class Meta:
        model = Tag

class WordAdmin(ImportExportModelAdmin):
    resource_class = WordResource
    # Add "UPDATE SYS_WORD TABLES" button
    change_list_template = "admin/pleethai/Word/change_list.html"

    # URL for update system tables
    def get_urls(self):
        urls = super().get_urls()
        my_urls = [
            url(r'^updatesystables/$', self.admin_site.admin_view(self.update_system_tables)),
            url(r'^appendsystables/$', self.admin_site.admin_view(self.append_system_tables)),
        ]
        return my_urls + urls

    # Condition of search
    def exist_japanese(self, word1, word2) -> bool:
        return word1.japanese == word2.japanese and word1.hiragana == word2.hiragana

    def delete_all(self, model: models.Model):
        while model.objects.count():
            ids = model.objects.values_list('pk', flat=True)[:100]
            model.objects.filter(pk__in = ids).delete()

    # Create SysWordJapanse table and SysWordConnector table from Word table
    def update_system_tables(self, request):
        all_words = Word.objects.all()
        sys_japanese = []
        sys_word_con = []
        temp_constituent = []
        try:
            for word in all_words:
                # Add a Word record (japanese & hiragana unique) to SysWordJapanese table
                if not Common.contains(sys_japanese, word, self.exist_japanese):
                    sys_japanese.append(SysWordJapanese.create(word))
                # Add a Word record to SysWordConnector table
                tempConnect = SysWordConnector.create(word, sys_japanese)
                if tempConnect != None:
                    sys_word_con.append(tempConnect)
            # delete and recreate
            for con_dict in Constituent.objects.values():
                temp_constituent.append(Constituent(**con_dict))
            self.delete_all(Constituent)
            self.delete_all(SysWordConnector)
            self.delete_all(SysWordJapanese)
            SysWordJapanese.objects.bulk_create(sys_japanese)
            SysWordConnector.objects.bulk_create(sys_word_con)
            Constituent.objects.bulk_create(temp_constituent)
            self.update_tags()
            messages.info(request, "Succeeded to update system tables")
        except Exception as e:
            messages.error(request, "Failed to update system tables")
            messages.error(request, str(e))
        return redirect(request.META['HTTP_REFERER'])

    # Update tags
    def update_tags(self):
        all_tags = Tag.objects.all().values_list('name', flat=True)
        for word_con in SysWordConnector.objects.all():
            for tag in _parse_tags(Word.objects.filter(id=word_con.id).first().tags):
                # skip the invalid tag
                if tag not in all_tags:
                    continue
                word_con.tags.add(tag)

    # Append SysWordJapanse table and SysWordConnector table
    # (append words number max is defined in settings.py)
    def append_system_tables(self, request):
        all_words = Word.objects.all()
        exist_sys_japs = list(SysWordJapanese.objects.all())
        sys_japanese = []
        exist_word_ids = SysWordConnector.objects.all().values_list("id", flat=True).distinct()
        sys_word_con = []
        all_tags = Tag.objects.all().values_list('name', flat=True)
        try:
            for word in all_words:
                # skip existed word
                if word.id in exist_word_ids:
                    continue

                # check max append
                if len(sys_word_con) >= settings.APPEND_TABLES_MAX_NUM:
                    break

                # Add a Word record (japanese & hiragana unique) to SysWordJapanese table
                if not Common.contains(exist_sys_japs + sys_japanese, word, self.exist_japanese):
                    sys_japanese.append(SysWordJapanese.create(word))
                # Add a Word record to SysWordConnector table
                tempConnect = SysWordConnector.create(word, exist_sys_japs + sys_japanese)
                if tempConnect != None:
                    sys_word_con.append(tempConnect)
                    # create tag
                    for tag in _parse_tags(word.tags):
                        # skip the invalid tag
                        if tag not in all_tags:
                            continue
                        tempConnect.tags.add(tag)
            # append
            SysWordJapanese.objects.bulk_create(sys_japanese)
            SysWordConnector.objects.bulk_create(sys_word_con)

            if len(sys_japanese) == 0 and len(sys_word_con) == 0:
                messages.info(request, "Already all data appended.")
            else:
                messages.info(request, "Succeeded to append {0} system tables(id from {1} to {2})"
                    .format(len(sys_word_con), sys_word_con[0].id, sys_word_con[-1].id))
                messages.info(request, "Not-appended data may remain. Try to 'append system tables' again.")
        except Exception as e:
            messages.error(request, "Failed to append system tables")
            messages.error(request, str(e))
        return redirect(request.META['HTTP_REFERER'])

class WordClassAdmin(ImportExportModelAdmin):
    resource_class = WordClassResource

class ExampleAdmin(ImportExportModelAdmin):
    resource_class = ExampleResource

class ConstituentAdmin(ImportExportModelAdmin):
    resource_class = ConstituentResource

class TaggedItemInline(admin.StackedInline):
    model = TaggedItem

class TagAdmin(ImportExportModelAdmin):
    resource_class = TagResource
    inlines = [TaggedItemInline]

admin.site.register(Word, WordAdmin)
admin.site.register(SysWordJapanese)
admin.site.register(SysWordConnector)
admin.site.register(WordClass,WordClassAdmin)
admin.site.register(Example, ExampleAdmin)
admin.site.register(Constituent, ConstituentAdmin)
admin.site.register(Tag, TagAdmin)
admin.site.register(TaggedItem)


import json
import logging
import sys

from passivetotal.common.utilities import is_ip

log = logging.getLogger('passivetotal')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

misperrors = {'error': 'Error'}
mispattributes = {
    'input': ['hostname', 'domain', 'ip-src', 'ip-dst',
              'x509-fingerprint-sha1', 'email-src', 'email-dst',
              'target-email', 'whois-registrant-email',
              'whois-registrant-phone', 'text', 'whois-registrant-name',
              'whois-registrar', 'whois-creation-date'],
    'output': ['hostname', 'domain', 'ip-src', 'ip-dst',
               'x509-fingerprint-sha1', 'email-src', 'email-dst',
               'target-email', 'whois-registrant-email',
               'whois-registrant-phone', 'text', 'whois-registrant-name',
               'whois-registrar', 'whois-creation-date', 'md5', 'sha1',
               'sha256', 'link']
}
moduleinfo = {
    'version': '1.0',
    'author': 'Brandon Dixon',
    'description': 'The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register',
    'module-type': ['expansion', 'hover']
}
moduleconfig = ['username', 'api_key']
query_playbook = [
    {'inputs': ['ip-src', 'ip-dst', 'hostname', 'domain'],
     'services': ['whois', 'ssl', 'dns', 'enrichment'],
     'name': 'generic'},
    {'inputs': ['whois-registrant-email', 'whois-registrant-phone',
                'whois-registrant-name', 'email-src', 'email-dst',
                'target-email'],
     'services': ['whois'],
     'name': 'reverse-whois'},
    {'inputs': ['x509-fingerprint-sha1'],
     'services': ['ssl'],
     'name': 'ssl-history'},
]

def query_finder(request):
    """Find the query value in the client request."""
    for item in mispattributes['input']:
        if not request.get(item, None):
            continue

        playbook = None
        for x in query_playbook:
            if item not in x['inputs']:
                continue
            playbook = x
            break

        return {'type': item, 'value': request.get(item), 'playbook': playbook}

def build_profile(request):
    """Check the incoming request for a valid configuration."""
    output = {'success': False}
    config = request.get('config', None)
    if not config:
        misperrors['error'] = "Configuration is missing from the request."
        return output

    for item in moduleconfig:
        if config.get(item, None):
            continue
        misperrors['error'] = "PassiveTotal authentication is missing."
        return output

    profile = {'success': True, 'config': config}
    profile.update(query_finder(request))

    return profile

def _generate_request_instance(conf, request_type):
    """Automatically generate a request instance to use.

    In the end, this saves us from having to load each request class in a
    explicit way. Loading via a string is helpful to reduce the code per
    call.

    :param request_type: Type of client to load
    :return: Loaded PassiveTotal client
    """
    pt_username = conf.get('username')
    pt_api_key = conf.get('api_key')

    class_lookup = {'dns': 'DnsRequest', 'whois': 'WhoisRequest',
                    'ssl': 'SslRequest', 'enrichment': 'EnrichmentRequest',
                    'attributes': 'AttributeRequest'}
    class_name = class_lookup[request_type]
    mod = __import__('passivetotal.libs.%s' % request_type,
                     fromlist=[class_name])
    loaded = getattr(mod, class_name)
    headers = {'PT-INTEGRATION': 'MISP'}
    authenticated = loaded(pt_username, pt_api_key, headers=headers)
    return authenticated

def _has_error(results):
    """Check to see if there's an error in place and log it."""
    if 'error' in results:
        msg = "%s - %s" % (results['error']['message'],
                           results['error']['developer_message'])
        misperrors['error'] = msg
        return True

    return False

def process_ssl_details(instance, query):
    """Process details for a specific certificate."""
    log.debug("SSL Details: starting")
    values = list()
    details = instance.get_ssl_certificate_details(query=query)
    err = _has_error(details)
    if err:
        raise Exception("We hit an error, time to bail!")
    if details.get('message') and details['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    values = {value for value in details.values() if value}
    txt = [{'types': ['ssl-cert-attributes'], 'values': list(values)}]
    log.debug("SSL Details: ending")

    return txt

def process_ssl_history(instance, query):
    """Process the history for an SSL certificate."""
    log.debug("SSL History: starting")

    type_map = {
        'ip': ['ip-src', 'ip-dst'],
        'domain': ['domain', 'hostname'],
        'sha1': ['x509-fingerprint-sha1']
    }

    hits = {'ip': list(), 'sha1': list(), 'domain': list()}
    history = instance.get_ssl_certificate_history(query=query)
    err = _has_error(history)
    if err:
        raise Exception("We hit an error, time to bail!")
    if history.get('message') and history['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in history.get('results', []):
        hits['ip'] += item.get('ipAddresses', [])
        hits['sha1'].append(item['sha1'])
        hits['domain'] += item.get('domains', [])

    tmp = list()
    for key, value in hits.items():
        txt = {'types': type_map[key], 'values': list(set(value))}
        tmp.append(txt)

    log.debug("SSL Details: ending")

    return tmp

def process_whois_details(instance, query):
    """Process the detail from the WHOIS record."""
    log.debug("WHOIS Details: starting")
    tmp = list()
    details = instance.get_whois_details(query=query, compact_record=True)
    err = _has_error(details)
    if err:
        raise Exception("We hit an error, time to bail!")
    if details.get('message') and details['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    if details.get('contactEmail', None):
        tmp.append({'types': ['whois-registrant-email'], 'values': [details.get('contactEmail')]})
    phones = details['compact']['telephone']['raw']
    tmp.append({'types': ['whois-registrant-phone'], 'values': phones})
    names = details['compact']['name']['raw']
    tmp.append({'types': ['whois-registrant-name'], 'values': names})
    if details.get('registrar', None):
        tmp.append({'types': ['whois-registrar'], 'values': [details.get('registrar')]})
    if details.get('registered', None):
        tmp.append({'types': ['whois-creation-date'], 'values': [details.get('registered')]})
    log.debug("WHOIS Details: ending")

    return tmp

def process_whois_search(instance, query, qtype):
    """Process a WHOIS search for a specific field value."""
    log.debug("WHOIS Search: starting")
    if qtype in ['whois-registrant-email', 'email-src', 'email-dst', 'target-email']:
        field_type = 'email'
    if qtype in ['whois-registrant-phone']:
        field_type = 'phone'
    if qtype in ['whois-registrant-name']:
        field_type = 'name'

    domains = list()
    search = instance.search_whois_by_field(field=field_type, query=query)
    err = _has_error(search)
    if err:
        raise Exception("We hit an error, time to bail!")
    if search.get('message') and search['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in search.get('results', []):
        domain = item.get('domain', None)
        if not domain:
            continue
        domains.append(domain)

    tmp = [{'types': ['hostname', 'domain'], 'values': list(set(domains))}]
    log.debug("WHOIS Search: ending")

    return tmp

def process_passive_dns(instance, query):
    """Process passive DNS data."""
    log.debug("Passive DNS: starting")
    tmp = list()
    pdns = instance.get_unique_resolutions(query=query)
    err = _has_error(pdns)
    if err:
        raise Exception("We hit an error, time to bail!")
    if pdns.get('message') and pdns['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    if is_ip(query):
        tmp = [{'types': ['domain', 'hostname'], 'values': pdns.get('results', [])}]
    else:
        tmp = [{'types': ['ip-src', 'ip-dst'], 'values': pdns.get('results', [])}]
    log.debug("Passive DNS: ending")

    return tmp

def process_osint(instance, query):
    """Process OSINT links."""
    log.debug("OSINT: starting")
    urls = list()
    osint = instance.get_osint(query=query)
    err = _has_error(osint)
    if err:
        raise Exception("We hit an error, time to bail!")
    if osint.get('message') and osint['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in osint.get('results', []):
        urls.append(item['sourceUrl'])

    tmp = [{'types': ['link'], 'values': urls}]
    log.debug("OSINT: ending")

    return tmp

def process_malware(instance, query):
    """Process malware samples."""
    log.debug("Malware: starting")
    content = {'hashes': list(), 'urls': list()}
    malware = instance.get_malware(query=query)
    err = _has_error(malware)
    if err:
        raise Exception("We hit an error, time to bail!")
    if malware.get('message') and malware['message'].startswith('quota_exceeded'):
        raise Exception("API quota exceeded.")
    for item in malware.get('results', []):
        content['hashes'].append(item['sample'])
        content['urls'].append(item['sourceUrl'])

    tmp = [{'types': ['link'], 'values': content['urls']}]
    hashes = {'md5': list(), 'sha1': list(), 'sha256': list()}
    for h in content['hashes']:
        if len(h) == 32:
            hashes['md5'].append(h)
        elif len(h) == 41:
            hashes['sha1'].append(h)
        elif len(h) == 64:
            hashes['sha256'].append(h)
    tmp += [{'types': ['md5'], 'values': hashes['md5']}]
    tmp += [{'types': ['sha1'], 'values': hashes['sha1']}]
    tmp += [{'types': ['sha256'], 'values': hashes['sha256']}]
    log.debug("Malware: ending")

    return tmp

def handler(q=False):
    if not q:
        return q

    request = json.loads(q)
    profile = build_profile(request)
    if not profile['success']:
        log.error(misperrors['error'])
        return misperrors

    output = {'results': list()}

    instances = dict()
    for service in profile['playbook']['services']:
        instances[service] = _generate_request_instance(
            profile['config'], service)

    play_type = profile['playbook']['name']
    query = profile['value']
    qtype = profile['type']
    try:
        if play_type == 'generic':
            results = process_passive_dns(instances['dns'], query)
            output['results'] += results
            results = process_whois_details(instances['whois'], query)
            output['results'] += results
            results = process_ssl_history(instances['ssl'], query)
            output['results'] += results
            results = process_osint(instances['enrichment'], query)
            output['results'] += results
            results = process_malware(instances['enrichment'], query)
            output['results'] += results
        elif play_type == 'reverse-whois':
            results = process_whois_search(instances['whois'], query, qtype)
            output['results'] += results
        elif play_type == 'ssl-history':
            results = process_ssl_details(instances['ssl'], query)
            output['results'] += results
            results = process_ssl_history(instances['ssl'], query)
            output['results'] += results
        else:
            log.error("Unsupported query pattern issued.")
    except Exception as e:
        misperrors['error'] = e.__str__()
        return misperrors

    return output

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo


def mergeNodes(head):
    dummy,tail = ListNode()
    total = 0
    cur = head.next
    while cur:
        if cur.val == 0:
            node = ListNode(total)
            tail.next = node
            tail = tail.next
            total = 0
        else:
            total += cur.val
        cur = cur.next
    return dummy.next


import unittest

import torch
from torch_quant.observer import HistogramObserver

class TestHistogramObserver(unittest.TestCase):
    def test_basic(self):
        obs = HistogramObserver()
        obs.fake_quant = False
        inp = torch.tensor([1, 2, 3, 4, 5], dtype=torch.float)
        oup = obs(inp)
        obs.calculate_qparams()
        self.assertTrue(torch.equal(oup, inp))
        target_scale = torch.tensor(0.01960018463432789)
        target_zp = torch.tensor(0, dtype=torch.int32)
        self.assertTrue(torch.equal(target_scale, obs.scale))
        self.assertTrue(torch.equal(target_zp, obs.zero_point))
        target_fake_quant_output = torch.fake_quantize_per_tensor_affine(
            inp, target_scale, target_zp, 0, 255
        )
        obs.fake_quant = True
        obs.observe = False
        oup = obs(inp)
        self.assertTrue(torch.equal(target_fake_quant_output, oup))

    def test_per_channel(self):
        with self.assertRaises(NotImplementedError):
            HistogramObserver(qscheme=torch.per_channel_symmetric)

if __name__ == "__main__":
    unittest.main()


from pyspark.sql import SparkSession
from pyspark.sql import Row
from pyspark.sql.functions import avg, min, max

spark = SparkSession.builder.appName('filter-example').getOrCreate()
sc = spark.sparkContext
sc.setLogLevel("ERROR")

def run():
    print()
    student = spark.createDataFrame([
        Row(name='Song', subject='English', term='Spring', score=80),
        Row(name='Song', subject='English', term='Fall', score=90),
        Row(name='Song', subject='Math', term='Spring', score=90),
        Row(name='Song', subject='Math', term='Fall', score=100),
        Row(name='Donald trump', subject='English', term='Spring', score=20),
        Row(name='Donald trump', subject='English', term='Fall', score=40),
        Row(name='Donald trump', subject='Math', term='Spring', score=30),
        Row(name='Donald trump', subject='Math', term='Fall', score=25),
    ])

    student.printSchema()
    student.show()

    print('Get the aggregations')
    df = student.groupBy('name', 'subject').agg(
        avg('score').alias("avg_score"),
        min('score').alias("min_score"),
        max('score').alias("max_score")
    )

    df.show()

if __name__ == '__main__':
    run()


import os
import numpy as np
import pickle
import matplotlib.pyplot as plt
import hdbscan
from tqdm import tqdm
from sklearn.cluster import DBSCAN
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import itertools
from random import sample

class ClusterEmbedding():
    def __init__(self, cluster_img_dir, embeddings_dir, cluster_algorithm,
                    cluster_dir, eps=0.5, min_samples=5, n_clusters=15, fps=30, sample_rate=3, dim=30):
        self.cluster_img_dir = cluster_img_dir
        self.embeddings_dir = embeddings_dir
        self.cluster_algorithm = cluster_algorithm
        self.cluster_dir = cluster_dir
        self.eps = eps
        self.min_samples= min_samples
        self.n_clusters = n_clusters
        self.fps = fps
        self.sample_rate = sample_rate
        self.dim = dim

    def cluster_helper(self, embeddings):
        labels = None
        if self.cluster_algorithm.upper() == "DBSCAN":
            cluster = DBSCAN(eps=self.eps, min_samples=self.min_samples, n_jobs=-1).fit(embeddings)
            labels = cluster.labels_
        if self.cluster_algorithm.upper() == "KMEANS":
            cluster = KMeans(n_clusters=self.n_clusters).fit(embeddings)
            labels = cluster.labels_
        if self.cluster_algorithm.upper() == "HDBSCAN":
            cluster = hdbscan.HDBSCAN(min_cluster_size=self.n_clusters)
            labels = cluster.fit_predict(embeddings)
        self.plottr(labels, embeddings)
        print(len(set(labels)))
        return labels

    def plottr(self, labels, embeddings):
        if not os.path.isdir(self.cluster_img_dir):
            os.mkdir(self.cluster_img_dir)

        embeddings_arr = np.vstack(embeddings)
        plt.scatter(embeddings[:, 0], embeddings[:, 1], c=labels, cmap="plasma")
        # plt.scatter(embeddings_arr[:, 0], embeddings_arr[:, 1], c=labels, cmap="plasma")
        plt.xlabel("Feature 1")
        plt.ylabel("Feature 2")
        plt.title("Unique Events")
        plt.savefig(os.path.join(self.cluster_img_dir, self.cluster_algorithm + ".png"))
        plt.show()

    def subsample(self, length, embed_dict):
        window = self.fps
        rate = self.fps//self.sample_rate
        samples = []
        for i in range(0, length, window):
            if i + window < length:
                value = [*range(i, i + window)]
                samples.append(sample(value, rate))
            else:
                value = [*range(i, length)]
                samples.append(value)
        samples = list(itertools.chain.from_iterable(samples))
        samples.sort()

        frame_keys = list(embed_dict.keys())
        frame_keys_ = [frame_keys[i] for i in samples]
        frame_values_ = [embed_dict[i] for i in frame_keys_]
        return frame_keys_, frame_values_

    def reduce_dim(self, frame_values_):
        frames_arr = np.vstack(frame_values_)
        reduced = PCA(n_components=self.dim).fit(frames_arr).transform(frames_arr)
        print(reduced[0])
        return reduced

    def cluster(self):
        if not os.path.isdir(self.cluster_dir):
            os.mkdir(self.cluster_dir)
        embeddings = []
        frame_names = []

        cluster_dict = {}

        # get embeddings
        embed_dict = pickle.load(open(os.path.join(self.embeddings_dir, "frame_embeddings.pkl"), 'rb'))

        sample_frames, sample_embeddings = self.subsample(len(embed_dict), embed_dict)
        reduced_embeddings = self.reduce_dim(sample_embeddings)

        labels = self.cluster_helper(reduced_embeddings)
        for i in tqdm(range(len(sample_frames)), desc="Mapping frames to clusters", ncols=100):
            cluster_dict[sample_frames[i]] = labels[i]

        with open(os.path.join(self.cluster_dir, "frame_to_cluster.pkl"), 'wb') as file:
            pickle.dump(cluster_dict, file)