import hudmodule
from matplotlib.backends.backend_tkagg import (
    FigureCanvasTkAgg, NavigationToolbar2Tk)
from matplotlib.figure import Figure
import tkinter as tk
from tkinter import *
import networkx as nx

class interface_analysis:
    def __init__(self):
        self.root = tk.Tk()
        self.fig = Figure(figsize=(7, 7), dpi=72)
        self.canvas = FigureCanvasTkAgg(self.fig, master=self.root)
        self.toolbar = NavigationToolbar2Tk(self.canvas, self.root)
        self.root.wm_title("Анализ репозиторий Github")
        self.canvas.draw()
        self.canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)
        self.toolbar.update()
        self.canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)
        self.ax = self.fig.add_subplot(111)
        self.githubGraph = hudmodule.githubStars()

    def example(self):
        self.ax.clear()
        self.githubGraph.setToken('2cb6efd6c6628ecd7e5e359d191feb5a2ea1e495')
        self.githubGraph.setUser('MNYudina')
        self.githubGraph.setRepo('Seminars')
        g = self.githubGraph.getGraph()
        g = self.githubGraph.getMoreGraph(g)
        nx.draw(g, node_color=self.githubGraph.color_map, with_labels=True, ax=self.ax)
        self.canvas.draw()

    def enter(self):
        self.ax.clear()
        self.githubGraph.setToken(self.e1.get())
        self.githubGraph.setUser(self.e2.get())
        self.githubGraph.setRepo(self.e3.get())
        g = self.githubGraph.getGraph()
        g = self.githubGraph.getMoreGraph(g)
        nx.draw(g, node_color=self.githubGraph.color_map, with_labels=True, ax=self.ax)
        self.canvas.draw()

    def start(self):
        self.button = tk.Button(master=self.root, text="Пример", command=self.example)
        self.button.pack(side=tk.BOTTOM)
        self.button = tk.Button(master=self.root, text="Расчитать", command=self.enter)
        self.button.pack(side=tk.BOTTOM)
        self.labl = Label(text="Токен", fg="#eee", bg="#333")
        self.labl.pack()
        self.e1 = Entry(width=50)
        self.e1.pack()
        self.labl2 = Label(text="Пользователь", fg="#eee", bg="#333")
        self.labl2.pack()
        self.e2 = Entry(width=50)
        self.e2.pack()
        self.labl3 = Label(text="Названия репозитория", fg="#eee", bg="#333")
        self.labl3.pack()
        self.e3 = Entry(width=50)
        self.e3.pack()
        tk.mainloop()

if __name__ == '__main__':
    prog = interface_analysis()
    prog.start()


#import library
import pathlib
import numpy as np
from tensorflow import keras
from keras import Sequential
from keras.layers import Dense
import  csv
import  tensorflow as tf

#Train Data to make a prediction about total sold based on price
def predictionmodel():
    # Preprocessing Data
    filename = 'Dataset.csv'
    reader = csv.reader(open(filename, "rt"), delimiter=",")
    header = next(reader)
    sold = []
    price = []
    for row in reader:
        price.append(row[1])
        sold.append(row[2])
    price = np.array(price, dtype=float)
    sold = np.array(sold, dtype=float)
    PRICE = price / 1000000
    SOLD = sold / 100

    # Split Data
    train_ratio = 0.8
    Y_TRAIN_SIZE = int(len(SOLD) * train_ratio)
    X_TRAIN_SIZE = int(len(PRICE) * train_ratio)
    xtrain_data = PRICE[:X_TRAIN_SIZE]
    xtest_data = PRICE[X_TRAIN_SIZE:]
    ytrain_data = SOLD[:Y_TRAIN_SIZE]
    ytest_data = SOLD[Y_TRAIN_SIZE:]
    X_train = [item for item in xtrain_data]
    X_test = [item for item in xtest_data]
    y_train = [item for item in ytrain_data]
    y_test = [item for item in ytest_data]

    # build model
    model = Sequential([Dense(10,input_shape=[1],input_dim=1),
                        Dense(10),
                        Dense(1)])

    # compile the model
    model.compile(optimizer=keras.optimizers.SGD(learning_rate=1e-4), loss='mean_squared_error',metrics=['mae'])

    # train the model
    model.fit(X_train, y_train, epochs=100,validation_data=(X_test,y_test))
    return model

#make a simple case
model = predictionmodel()
inpt = int(input('Masukkan Harga Untuk Prediksi jumlah yang akan terjual(jt) : '))
new_x = inpt
prediction = model.predict([new_x])[0][0]
print('Prediksi Jumlah Rata-Rata yang akan Terjual :',int(prediction * 100) ,'unit' )
model.save("model.h5")

#save the model
# export_dir ='model/1'
# if '__main__':
#     tf.saved_model.save(model,export_dir=export_dir)
#     mode = 'Speed'
#     if mode =='Storage':
#         optimization=tf.lite.Optimize.OPTIMIZE_FOR_SIZE
#     elif mode =='Speed':
#         optimization =tf.lite.Optimize.OPTIMIZE_FOR_LATENCY
#     else:
#         optimization = tf.lite.Optimize.DEFAULT
#     converter = tf.lite.TFLiteConverter.from_keras_model(export_dir)
#     converter.optimizations=[optimization]
#     tflite_model = converter.convert();
#     tflite_model_file = pathlib.Path('./model.tflite')
#     tflite_model_file.write_bytes(tflite_model)


import cPickle
import numpy as np
import os
import scipy.stats as spstats

class Workspace(object):

    def __init__(self, movies, ratings, users, k=1, cwd='dat/pkl'):
        self.cwd = cwd
        self.k = k

        self.movies = movies
        self.ratings = ratings
        self.users = users
        self.tags = {}
        self.decades = {}
        self.age_groups = {}
        self.genders = {}
        self.occupations = {}

    def summarize_movies(self):
        def compute():
            movie_ratings = self.ratings.for_movie(m.ID)
            return Workspace.summary_stats(movie_ratings)

        for m in self.movies.movies:
            pickle_fn = os.path.join(self.cwd, 'movies', str(m.ID))
            self.movies[m.ID].summarize(*Workspace.load_or_compute(pickle_fn, compute))

    def summarize_users(self):
        def compute():
            movie_ratings = self.ratings.for_user(u.ID)
            return Workspace.summary_stats(movie_ratings)

        for u in self.users.users:
            pickle_fn = os.path.join(self.cwd, 'users', str(u.ID))
            self.users[u.ID].summarize(*Workspace.load_or_compute(pickle_fn, compute))

    def summarize_tags(self):
        def compute():
            movie_ratings = self.ratings.for_tag(t, movies=self.movies)
            return Workspace.summary_stats(movie_ratings)

        for t in self.movies.tags:
            pickle_fn = os.path.join(self.cwd, 'tags', str(t))
            self.tags[t] = Workspace.load_or_compute(pickle_fn, compute)

    def summarize_decades(self):
        def compute():
            movie_ratings = self.ratings.for_decade(d, movies=self.movies)
            return Workspace.summary_stats(movie_ratings)

        for d in self.movies.decades:
            pickle_fn = os.path.join(self.cwd, 'decades', str(d))
            self.decades[d] = Workspace.load_or_compute(pickle_fn, compute)

    def summarize_age_groups(self):
        def compute():
            movie_ratings = self.ratings.for_age_group(a, users=self.users)
            return Workspace.summary_stats(movie_ratings)

        for a in self.users.age_groups:
            pickle_fn = os.path.join(self.cwd, 'age_groups', str(a))
            self.age_groups[a] = Workspace.load_or_compute(pickle_fn, compute)

    def summarize_genders(self):
        def compute():
            movie_ratings = self.ratings.for_gender(g, users=self.users)
            return Workspace.summary_stats(movie_ratings)

        for g in self.users.genders:
            pickle_fn = os.path.join(self.cwd, 'genders', str(g))
            self.genders[g] = Workspace.load_or_compute(pickle_fn, compute)

    def summarize_occupations(self):
        def compute():
            movie_ratings = self.ratings.for_occupation(o, users=self.users)
            return Workspace.summary_stats(movie_ratings)

        for o in self.users.occupations:
            pickle_fn = os.path.join(self.cwd, 'occupations', str(o))
            self.occupations[o] = Workspace.load_or_compute(pickle_fn, compute)

    def extract_components(self):
        def compute():
            raise Exception('run features.py to extract components')

        for m in self.movies.movies:
            if m.count > 0:
                pickle_fn = os.path.join(self.cwd, 'components', '%02d' % self.k, str(m.ID))
                components = Workspace.load_or_compute(pickle_fn, compute)
                self.movies[m.ID].components = np.ones(components.size + 1)
                self.movies[m.ID].components[1:] = components  # components[0] === scale factor

    @staticmethod
    def load_or_compute(pickle_fn, compute):
        if os.path.isfile(pickle_fn):
            result = cPickle.load(open(pickle_fn))
        else:
            result = compute()
            cPickle.dump(result, open(pickle_fn, 'wb'))
        return result

    @staticmethod
    def summary_stats(movie_ratings):
        r_ids = [r.ID for r in movie_ratings]
        count = len(movie_ratings)
        if count > 0:
            amean = np.mean([r.rating for r in movie_ratings])
            hmean = spstats.hmean([r.rating for r in movie_ratings])
            var = np.var([r.rating for r in movie_ratings])
        else:
            amean = np.NaN
            hmean = np.NaN
            var = np.NaN
        return r_ids, count, amean, hmean, var


from igraph import Graph
from random import random

def chance_to_activate_1(g, source, target):
    '''
    First model approach on the paper. The probability of
    u activating v is c_(u,v)/d_v.

    Parameters:
        @param source: the source of the edge.
        @param target: the target of the edge.

    Returns:
        The probability of source activating target.
    '''
    edges = g.es.select(_source=source, _target=target)
    c = len(edges)
    p = edges[0]['prob']
    return 1-(1-p)**c

def chance_to_activate_2(g, target):
    '''
    Second model approach. The probability of u activating v is
    given by 1/d_v. Note we don't need the initial probabilities for
    this approach.

    Parameters:
        @param target: the target of the edge.

    Returns:
        The probability of activation.
    '''
    vertex = g.vs.select(target)[0]
    d = vertex.outdegree() + vertex.indegree()
    return 1/d

def activation(g, verbosity, source, target):
    '''
    Calls for one or the other activation function depending on the user's
    input.

    Parameters:
        @param verbosity: 1 for the first model approach, 2 for the second one.
        @param source: the source of the edge.
        @param target: the target of the edge.

    Returns:
        Either first model approach probability of source activating target or
        the second one depending on the verbosity chosen.
    '''
    if verbosity == 1:
        return chance_to_activate_1(g, source, target)
    else:
        if verbosity == 2:
            return chance_to_activate_2(g, target)
        else:
            raise Exception('Incorrect activation. Please select 1 or 2.')

def init_prob(g, p):
    '''
    Initiallizes the probability of the edges.

    Parameters:
        @param g: the graph.
        @param p: the probability.
    '''
    for e in g.es:
        e.update_attributes({'prob': p})

def one_step(g, seed, old_edges, verbosity, visited):
    '''
    Performs one step of the algorithm.

    Parameters:
        @param g: the graph.
        @param seed: the seed or set of current active nodes.
        @param old_edges: the active edges.

    Returns:
        A list of active edges, active nodes, and the updated seed set.
    '''
    active_nodes = set()
    active_edges = set()
    prob_active = set()

    for v in seed:
        #import ipdb; ipdb.set_trace()
        for nb in g.successors(v):
            if nb in seed or (v, nb) in old_edges or (v, nb) in active_edges:
                continue
            prob = random()
            if prob <= activation(g, verbosity, v, nb):
                active_nodes.add(nb)
                prob_active.add((nb, prob))
            active_edges.add((v, nb))
            visited.add(nb)
    seed = seed + list(active_nodes)
    return active_edges, active_nodes, seed, prob_active, visited

def independent_cascade(g, seed, p, st, verbosity):
    '''
    The main function. Computes the independent cascade model.

    Parameters:
        @param g: the graph.
        @param seed: the seed or initial set of active nodes.
        @param p: the probability to initialize the edges.
        @param st: the number of steps.

    Returns:
        A set of active nodes and a list of the active nodes at each
        step. Also a list of visited nodes.
    '''
    init_prob(g, p)
    old_edges = set()
    B = []
    gvcount = len(set([i for (i, _) in g.get_edgelist()] + [i for (_, i) in g.get_edgelist()]))
    visited = set(seed)
    #If st set to zero then we want the algorithm to run until it runs out of nodes. To reset st=gvount+1000
    #is a simple way of achieving that.
    if st == 0:
        st = gvcount + 1000
    #import ipdb; ipdb.set_trace()
    while st>0 and gvcount>len(seed):
        alen = len(seed)
        active_edges, active_nodes, seed, prob_active, visited = one_step(g, seed, old_edges, verbosity, visited)
        B = B + [prob_active]
        old_edges = old_edges.union(active_edges)
        if len(seed) == alen:
            break
        st = st - 1
    return seed, B, visited


import types

class foo:
  def method():
    return('a method')

f=foo()

x = f.fun

if type(x) == types.MethodType:
  print("x is a MethodType")
else:
  print("x is not a MethodType")


def majorityElement(nums):
    m = nums[0]
    count = 1
    for i in range(1, len(nums)):
        if count == 0:
            count += 1
            m = nums[i]
        elif m == nums[i]:
            count += 1
        else:
            count -= 1
    return m

print(majorityElement([1, 2, 3, 4, 4, 5]))


# Method-1

def getSumAbsoluteDifferences(nums):
    answer = []
    res = 0
    for i in range(1, len(nums)):
        res += nums[i] - nums[0]

    answer.append(res)

    for i in range(1, len(nums)):
        diff = nums[i] - nums[i - 1]
        res += diff * (i - 1)
        res -= diff * (len(nums) - i - 1)

        answer.append(res)

    return answer

# Time Complexity : O(n)
# Space Complexity : O(n)

328.Sum of Absolute Differences in a Sorted Array/Method-1.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
GitHub key downloader.
We use this for academic research on SSH keys entropy.
"""

import os
import sys
import inspect
import resource

from requests.auth import HTTPBasicAuth

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)

import requests
import logging
import coloredlogs
import traceback
import json
import argparse
import re
import math
import random
import time
import shutil
import multiprocessing
import time
import signal
import utils
import databaseutils
import collections
import threading
from threading import Lock as Lock
import types
import Queue
from blessed import Terminal
from cmd2 import Cmd
from evt_dequeue import EvtDequeue
import dbutil

from github_base import AccessResource, RateLimitHit
from database import GitHubKey, GitHubUser as GitHubUserDb, GitHubUserKeys
from database import Base as DB_Base
from sqlalchemy.orm import scoped_session
import sqlalchemy as salch
from trace_logger import Tracelogger

import gc
import mem_top
from pympler.tracker import SummaryTracker
from collections import OrderedDict, namedtuple
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey

logger = logging.getLogger(__name__)
coloredlogs.install(level=logging.INFO)

# GitHub User ID & Name
GitHubUser = namedtuple('GitHubUser', ['user_id', 'user_name', 'user_type', 'user_url'])

class DownloadJob(object):
    """
    Represents link to download
    """

    TYPE_USERS = 1
    TYPE_KEYS = 2

    __slots__ = ['url', 'type', 'user', 'fail_cnt', 'last_fail', 'priority', 'time_added']

    def __init__(self, url=None, jtype=TYPE_USERS, user=None, priority=0, time_added=None, *args, **kwargs):
        self.url = url
        self.type = jtype
        self.user = user
        self.fail_cnt = 0
        self.last_fail = 0
        self.priority = priority
        self.time_added = time.time() if time_added is None else time_added

    def to_json(self):
        js = collections.OrderedDict()
        js['url'] = self.url
        js['type'] = self.type
        js['fail_cnt'] = self.fail_cnt
        js['last_fail'] = self.last_fail
        js['priority'] = self.priority
        js['time_added'] = self.time_added
        if self.user is not None:
            js['user_id'] = self.user.user_id
            js['user_name'] = self.user.user_name
            js['user_type'] = self.user.user_type
            js['user_url'] = self.user.user_url
        return js

    @classmethod
    def from_json(cls, js):
        tj = cls()
        tj.url = js['url']
        tj.type = js['type']
        tj.fail_cnt = js['fail_cnt']
        tj.last_fail = js['last_fail']
        tj.priority = utils.defvalkey(js, 'priority', 0)
        tj.time_added = utils.defvalkey(js, 'time_added', 0)
        if 'user_id' in js:
            user_url = js['user_url'] if 'user_url' in js else None
            tj.user = GitHubUser(user_id=js['user_id'], user_name=js['user_name'], user_type=js['user_type'], user_url=user_url)
        return tj

    @staticmethod
    def cmp(self, other):
        """
        Comparator
        :param self:
        :param other:
        :return:
        """
        # Inside the category: fail cnt, time added.
        if self.type == other.type:
            if self.fail_cnt == other.fail_cnt:
                if self.time_added == other.time_added:
                    return int(other.priority - self.priority)
                else:
                    return int(self.time_added - other.time_added)
            else:
                return int(self.fail_cnt - other.fail_cnt)
        else:
            # Outside the category - priority ordering. Higher the priority, sooner will be picked
            if self.priority == other.priority:
                return int(self.time_added - other.time_added)
            else:
                return int(other.priority - self.priority)

    def __cmp__(self, other):
        """
        Compare operation for priority queue.
        :param other:
        :return:
        """
        return self.cmp(self, other)

class GitHubLoader(Cmd):
    """
    GitHub SSH keys loader
    """
    prompt = '$> '

    LINK_FACTOR = 70
    USER_URL = 'https://api.github.com/users/%s'
    USERS_URL = 'https://api.github.com/users?since=%s'
    KEYS_URL = 'https://api.github.com/users/%s/keys'
    KEYS_ID_URL = 'https://api.github.com/user/%s/keys'

    def __init__(self, attempts=5, threads=1, state=None, state_file=None, config_file=None, audit_file=None,
                 max_mem=None, users_only=False, merge=False, update_keys=False, *args, **kwargs):

        Cmd.__init__(self, *args, **kwargs)
        self.t = Terminal()
        self.trace_logger = Tracelogger(logger=logger)

        self.attempts = int(attempts)
        self.total = None
        self.terminate = False
        self.since_id = 0
        self.last_users_count = None
        self.user_lock = Lock()
        self.processed_user_set = set()
        self.processed_user_set_lock = Lock()

        self.max_mem = max_mem
        self.merge = merge
        self.update_keys = update_keys
        self.users_only = users_only
        self.users_per_page = 30
        self.users_bulk_load_pages = 500
        self.user_load_bulk = 5000
        self.user_refill_lock = Lock()
        self.state = state
        self.state_file_path = state_file
        self.rate_limit_reset = None
        self.rate_limit_remaining = None

        self.config = None
        self.config_file = config_file

        self.audit_file = audit_file
        self.audit_records_buffered = []
        self.audit_lock = Lock()

        self.stop_event = threading.Event()
        self.threads = int(threads)
        self.link_queue = Queue.PriorityQueue()  # Store links to download here
        self.worker_threads = []

        self.state_thread = None
        self.state_thread_lock = Lock()

        self.resources_list = []
        self.resources_queue = Queue.PriorityQueue()
        self.local_data = threading.local()

        self.new_users_events = EvtDequeue()
        self.new_keys_events = EvtDequeue()

        self.db_config = None
        self.engine = None
        self.session = None

        self.mem_tracker = None

    def signal_handler(self, signal, frame):
        """
        Signal handler - terminate gracefully
        :param signal:
        :param frame:
        :return:
        """
        logger.info('CTRL+C pressed')
        self.trigger_stop()

    def trigger_stop(self):
        """
        Sets terminal conditions to true
        :return:
        """
        self.terminate = True
        self.stop_event.set()

    def trigger_quit(self):
        """
        Terminal condition & file change
        :return:
        """
        self.trigger_stop()
        utils.try_touch('.github-quit')

    #
    # CMD handlers
    #

    def do_quit(self, arg):
        self.trigger_quit()
        logger.info('Waiting for thread termination')

        time.sleep(1)
        logger.info('Quitting')
        return Cmd.do_quit(self, arg)

    def do_reset(self, line):
        print('\033c')

    def do_gc(self, line):
        gc.collect()

    def do_mem_top(self, line):
        print(mem_top.mem_top())

    def do_mem_track_init(self, line):
        self.mem_tracker = SummaryTracker()

    def do_mem_track_diff(self, line):
        print(self.mem_tracker.print_diff())

    def do_mem_track_deinit(self, line):
        self.mem_tracker = None

    def do_mem(self, line):
        print('Memory usage: %s kB' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

    def do_state(self, line):
        js = self.state_gen()

        if line is None or len(line) == 0:
            del js['link_queue']
            del js['resource_stats']
        elif line == '1':
            del js['link_queue']

        print(json.dumps(js, indent=2))

    def do_deq_enable(self, line):
        self.new_keys_events.disabled = False
        self.new_users_events.disabled = False

    def do_deq_disable(self, line):
        self.new_keys_events.disabled = True
        self.new_users_events.disabled = True

    #
    # Init
    #

    def init_config(self):
        """
        Loads config & state files
        :return:
        """
        if self.state_file_path is not None and os.path.exists(self.state_file_path):
            with open(self.state_file_path, 'r') as fh:
                self.state = json.load(fh, object_pairs_hook=OrderedDict)
                logger.info('State loaded: %s' % os.path.abspath(self.state_file_path))

        with open(self.config_file, 'r') as fh:
            self.config = json.load(fh, object_pairs_hook=OrderedDict)
            logger.info('Config loaded: %s' % os.path.abspath(self.config_file))

            if 'since_id' in self.config:
                self.since_id = self.config['since_id']

            # Process resources - randomized
            if 'res' in self.config:
                res_tmp = self.config['res']
                random.shuffle(res_tmp)
                for idx, res in enumerate(res_tmp):
                    r = AccessResource(usr=res['usr'], token=res['token'], idx=idx)
                    self.resources_list.append(r)
                    self.resources_queue.put(r)
                    logger.info('Resource %02d loaded: %s' % (idx, r.usr))
            else:
                # unauth
                r = AccessResource(usr=None, token=None)
                self.resources_list.append(r)
                self.resources_queue.put(r)

    def init_db(self):
        """
        Initializes database engine & session.
        Has to be done on main thread.
        :return:
        """
        self.db_config = databaseutils.process_db_config(self.config['db'])

        from sqlalchemy import create_engine
        from sqlalchemy.orm import sessionmaker, scoped_session
        self.engine = create_engine(self.db_config.constr, pool_recycle=3600)
        self.session = scoped_session(sessionmaker(bind=self.engine))

        # Make sure tables are created
        DB_Base.metadata.create_all(self.engine)

    def init_workers(self):
        """
        Initialize worker threads
        :return:
        """
        logger.info('Starting %d working threads' % self.threads)
        for idx in range(self.threads):
            t = threading.Thread(target=self.work_thread_main, args=(idx, ))
            self.worker_threads.append(t)

        # Kick-off all threads
        for t in self.worker_threads:
            t.start()

        logger.info('Worker threads started')

    def cli(self):
        """
        CLI thread
        :return:
        """
        logger.info('CLI thread started')
        self.cmdloop()
        logger.info('Terminating CLI thread')

    #
    # Operation
    #

    def work(self):
        """
        Main thread work method
        :return:
        """
        # Interrupt signals
        signal.signal(signal.SIGINT, self.signal_handler)

        self.init_config()
        self.init_db()

        # Resume last state
        self.state_resume()

        # Monitor threads.
        self.state_thread = threading.Thread(target=self.state_main, args=())
        self.state_thread.start()

        # If there is no link to process - create from since.
        if self.link_queue.qsize() == 0:
            self.kickoff_links()

        # Worker threads
        self.init_workers()

        logger.info('Main thread started %s %s %s' % (os.getpid(), os.getppid(), threading.current_thread()))

        # CLI thread
        cli_thread = threading.Thread(target=self.cli, args=())
        cli_thread.setDaemon(True)
        cli_thread.start()

        # Join on workers
        self.after_loop()
        logger.info('Terminating main thread')
        return None

    def after_loop(self, wait_for_state=True):
        """
        After work loop finishes
        :return:
        """
        logger.info('Waiting termination of slave threads')

        # Wait here for termination of all workers and monitors.
        try:
            for t in self.worker_threads:
                t.join()

            if wait_for_state:
                self.state_thread.join()
        except:
            logger.error('Exception during thread join')
            logger.error(traceback.format_exc())

        logger.info('All threads terminates, last state save')
        self.state_save()

    def work_thread_main(self, idx):
        """
        Worker thread main loop
        :return:
        """
        self.local_data.idx = idx
        logger.info('Working thread %d started' % idx)

        while not self.terminate and not self.stop_event.is_set():
            self.interruptible_sleep_delta(0.1)

            # Get credential to process link with
            resource = self.resource_allocate()
            if resource is None:
                continue

            # We have resource, now get the job
            job = None
            try:
                job = self.link_queue.get(True, timeout=1.0)
            except Queue.Empty:
                self.resource_return(resource)
                continue

            # If job last fail is too recent - put again back to queue
            if time.time() - job.last_fail < 3.0:
                self.link_queue.put(job)  # re-insert to the back of the queue for later processing
                self.resource_return(resource)
                continue

            # Job processing starts here - fetch data page with the resource.
            js_data = None
            try:
                self.local_data.job = job
                self.local_data.resource = resource
                js_data, headers, raw_response = self.load_page_local()

            except RateLimitHit as e:
                logger.error('[%d] Rate limit hit: %s, failcnt: %d, res: %s, exception: %s'
                             % (idx, job.url, job.fail_cnt, resource.usr, e))
                continue

            except Exception as e:
                logger.error('[%d] Exception in processing job: %s, failcnt: %d, res: %s, exception: %s'
                             % (idx, job.url, job.fail_cnt, resource.usr, e))

                self.on_job_failed(job)
                continue

            finally:
                self.resource_return(resource)
                self.local_data.resource = None
                self.local_data.last_usr = resource.usr
                self.local_data.last_remaining = resource.remaining

            # Process downloaded data here.
            try:
                if js_data is None:
                    self.audit_log('404', job.url, jtype=job.type, job=job)
                    self.flush_audit()
                    continue

                if job.type == DownloadJob.TYPE_USERS:
                    self.process_users_data(job, js_data, headers, raw_response)
                else:
                    self.process_keys_data(job, js_data, headers, raw_response)

            except Exception as e:
                logger.error('[%d] Unexpected exception, processing type %s, link %s: cnt: %d, res: %s, %s'
                             % (idx, job.type, job.url, job.fail_cnt, resource.usr, e))

                self.trace_logger.log(e)
                self.on_job_failed(job)
            finally:
                self.local_data.resource = None
                self.local_data.job = None
                self.local_data.last_usr = None
                self.local_data.last_remaining = None
                resource = None
                job = None
                headers = None
                raw_response = None

        pass
        logger.info('Terminating worker thread %d' % idx)

    def on_job_failed(self, job):
        """
        If job failed, this teaches it how to behave
        :param job:
        :return:
        """
        job.fail_cnt += 1
        job.last_fail = time.time()

        # if failed too many times - log and discard.
        if job.fail_cnt > 35:
            logger.warning('Job failed too many times %s' % job.url)
            self.audit_log('too-many-fails', job.url, jtype=job.type, job=job)
            self.flush_audit()
        else:
            self.link_queue.put(job)  # re-insert to the queue for later processing

    def load_page_local(self):
        """
        Loads page stored in thread local
        :return:
        """

        auth = None
        resource = self.local_data.resource
        if resource.usr is not None:
            auth = HTTPBasicAuth(resource.usr, resource.token)

        job = self.local_data.job

        res = requests.get(job.url, timeout=10, auth=auth)
        headers = res.headers

        resource.reset_time = utils.try_float(headers.get('X-RateLimit-Reset'))
        resource.remaining = utils.try_int(headers.get('X-RateLimit-Remaining'))
        resource.last_used = time.time()
        resource.used_cnt += 1

        if res.status_code == 403 and resource.remaining is not None and resource.remaining < 10:
            resource.fail_cnt += 1
            raise RateLimitHit

        if res.status_code == 404:
            resource.fail_cnt += 1
            logger.warning('URL not found: %s' % job.url)
            return None, None, None

        if res.status_code // 100 != 2:
            resource.fail_cnt += 1
            res.raise_for_status()

        data = res.content
        if data is None:
            resource.fail_cnt += 1
            raise Exception('Empty response')

        js = json.loads(data, object_pairs_hook=OrderedDict)
        return js, headers, res

    def resource_allocate(self, blocking=True, timeout=1.0):
        """
        Takes resource from the pool.
        If the resource has low remaining credit, thread is suspended to re-charge.
        :return: resource or None if not available in the time
        """
        try:
            resource = self.resources_queue.get(True, timeout=1.0)
            if resource.remaining is not None and resource.remaining <= self.threads + 2:
                sleep_sec = resource.reset_time - time.time() if resource.reset_time is not None else 15*60
                sleep_sec += 120  # extra 2 minutes to avoid problems with resources

                logger.info('Rate limit exceeded on resource %s, remaining: %s, sleeping till: %s, it is %d seconds, '
                            '%d minutes'
                            % (resource.usr, resource.remaining, resource.reset_time, sleep_sec, sleep_sec / 60.0))
                self.sleep_interruptible(time.time() + sleep_sec)
                logger.info('Resource sleep finished %s' % resource.usr)

                # Reset estimations, needs to be refreshed
                resource.remaining = None
                resource.reset_time = None

            return resource

        except Queue.Empty:
            return None

    def resource_return(self, res):
        """
        Returns resource to the pool
        :param res:
        :return:
        """
        self.resources_queue.put(res)

    def sleep_interruptible(self, until_time):
        """
        Interruptible sleep - sleep until given time.
        :param until_time:
        :return:
        """
        while time.time() <= until_time:
            time.sleep(1.0)
            if self.terminate or self.stop_event.is_set():
                return

    def interruptible_sleep_delta(self, sleep_time):
        """
        Sleeps the current thread for given amount of seconds, stop event terminates the sleep - to exit the thread.
        :param sleep_time:
        :return:
        """
        if sleep_time is None:
            return

        sleep_time = float(sleep_time)

        if sleep_time == 0:
            return

        sleep_start = time.time()
        while not self.stop_event.is_set() and not self.terminate:
            time.sleep(0.1)
            if time.time() - sleep_start >= sleep_time:
                return

    #
    # Parser and processing logic
    #

    def kickoff_links(self):
        """
        Kick off the scrapping by adding initial links to the queue
        :return:
        """
        if self.update_keys:
            self.fill_user_key_links()

        else:
            job = DownloadJob(url=self.USERS_URL % self.since_id, jtype=DownloadJob.TYPE_USERS)
            self.link_queue.put(job)
            logger.info('Kickoff link added: %s' % job.url)

    def process_users_data(self, job, js, headers, raw_response):
        """
        Process user data - produce keys links + next user link
        :param job:
        :param js:
        :param headers:
        :param raw_response:
        :return:
        """
        max_id = 0
        github_users = []
        cur_time = int(time.time())
        for user in js:
            if 'id' not in user:
                logger.error('Field ID not found in user')
                continue

            github_user = GitHubUser(user_id=int(user['id']), user_name=user['login'],
                                     user_type=user['type'], user_url=user['url'])
            github_users.append(github_user)

            if github_user.user_id > max_id:
                max_id = github_user.user_id

            if self.users_only:
                continue

            key_url = '%s/keys' % github_user.user_url
            new_job = DownloadJob(url=key_url, jtype=DownloadJob.TYPE_KEYS, user=github_user,
                                  priority=random.randint(0, 1000), time_added=cur_time)
            self.link_queue.put(new_job)

        # Link with the maximal user id
        users_url = self.USERS_URL % max_id
        new_job = DownloadJob(url=users_url, jtype=DownloadJob.TYPE_USERS, time_added=cur_time)

        # Optimizing the position of this link in the link queue
        queue_size = self.link_queue.qsize()
        queue_size_max = self.LINK_FACTOR * self.threads
        fill_up_ratio = queue_size / float(queue_size_max)

        # Key jobs are uniformly distributed on priorities 0...1000.
        # To increase queue size pick priority closer to 1000, do decrease, closer to 0
        priority = random.randint(0, 500)
        if queue_size < queue_size_max:
            priority = int((1 - fill_up_ratio) * 5000) + 500
        if queue_size > 3*queue_size_max:
            priority = 0

        new_job.priority = priority
        lucky_one = False
        with self.user_lock:
            if self.since_id < max_id:
                self.since_id = max_id
                self.link_queue.put(new_job)
                lucky_one = True

                # Bulk user optimisation - add more users at once, multithreading
                if self.users_only:
                    self.bulk_user_only_load(max_id=max_id, cur_time=cur_time, priority=priority)

        logger.info('[%02d, usr=%20s, remaining=%5s] Processed users link %s, Next since: %3s. ResQSize: %4d, '
                    'LQSize: %4d, fill-up: %0.4f, priority: %4s, ram: %s kB, new=%s, New users: [%s]'
                    % (self.local_data.idx, self.local_data.last_usr, self.local_data.last_remaining,
                       len(github_users)+1, max_id, self.resources_queue.qsize(),
                       queue_size, fill_up_ratio, priority,
                       resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, lucky_one,
                       ', '.join([str(x.user_name) for x in github_users])))

        # Store all users.
        self.store_users_list(github_users)

    def bulk_user_only_load(self, max_id, cur_time, priority):
        """
        Fills the queue with multiple user links
        :param max_id:
        :param cur_time:
        :param priority:
        :return:
        """
        new_jobs = []
        for page_idx in range(1, self.users_bulk_load_pages + 1):
            users_url = self.USERS_URL % (max_id + self.users_per_page * page_idx)
            new_job = DownloadJob(url=users_url, jtype=DownloadJob.TYPE_USERS,
                                  time_added=cur_time, priority=priority - page_idx)
            new_jobs.append(new_job)

        self.since_id = max_id + self.users_per_page * self.users_bulk_load_pages
        logger.info('[%02d] Bulk load, new max=%s' % (self.local_data.idx, self.since_id))

        for job in new_jobs:
            self.link_queue.put(job)

    def store_users_list(self, users):
        """
        Stores all user in the list
        :param users
        :return:
        """
        # Handling gaps in the user space ID. With user-only optimization it causes
        # overlaps.
        reduced_by = 0
        with self.processed_user_set_lock:
            ids = [user.user_id for user in users]
            ids_ok = []
            for id in ids:
                if id in self.processed_user_set:
                    reduced_by += 1
                    continue
                self.processed_user_set.add(id)
                ids_ok.append(id)
            users = [user for user in users if user.user_id in ids_ok]

        # Bulk user load
        s = self.session()
        id_list = sorted([user.user_id for user in users])
        db_users = s.query(GitHubUserDb).filter(GitHubUserDb.id.in_(id_list)).all()
        db_user_map = {user.id: user for user in db_users}

        for user in users:
            self.new_users_events.insert()

            # Store user to the DB
            try:
                db_user = utils.defvalkey(db_user_map, key=user.user_id)
                self.store_user(user, s, db_user=db_user, db_user_loaded=True)

            except Exception as e:
                logger.warning('[%02d] Exception in storing user %s' % (self.local_data.idx, e))
                self.trace_logger.log(e)
                logger.info('[%02d] idlist: %s' % (self.local_data.idx, id_list))
                self.trigger_quit()
                break

        try:
            s.commit()
            # logger.info('[%02d] Commited, reduced by: %s' % (self.local_data.idx, reduced_by))
        except Exception as e:
            logger.warning('[%02d] Exception in storing bulk users' % self.local_data.idx)
            logger.warning(traceback.format_exc())
            logger.info('[%02d] idlist: %s' % (self.local_data.idx, id_list))
            self.trigger_quit()
        finally:
            utils.silent_close(s)

    def process_keys_data(self, job, js, headers, raw_response):
        """
        Processing key loaded data
        :param job:
        :param js:
        :param headers:
        :param raw_response:
        :return:
        """
        js_empty = js is None or len(js) == 0

        # Expect failures, commit everything before
        if self.merge and not js_empty:
            try:
                s = self.session()
                s.commit()
            except Exception as e:
                logger.warning('Could not pre-commit: %s' % e)

        # Store each key.
        key_ids = []
        for key in js:
            s = None
            self.new_keys_events.insert()

            try:
                s = self.session()
                self.store_key(job.user, key, s)
                key_ids.append(int(key['id']))
                s.commit()

                self.assoc_key(job.user.user_id, key['id'], s)
                s.commit()

                s.flush()        # writes changes to DB
                s.expunge_all()  # removes objects from session

            except Exception as e:
                logger.warning('Exception in storing key %s' % e)
                self.trace_logger.log(e)

            finally:
                utils.silent_close(s)
                s = None

        # Deassoc lost keys
        try:
            s = self.session()
            self.deassoc_lost_keys(job.user.user_id, key_ids, s)
            s.commit()

        except Exception as e:
            logger.warning('Exception in deassoc for users %s : %s' % (job.user.user_id, e))
            self.trace_logger.log(e)

        finally:
            utils.silent_close(s)
            s = None

        self.on_keys_processed()

    def on_keys_processed(self):
        """
        Event called on keys have been processed.
        Currently used to re-fill the link queue in the update key scenario.
        :return:
        """
        if not self.update_keys:
            return

        with self.user_refill_lock:
            qsize = self.link_queue.qsize()
            if qsize < 30:
                self.fill_user_key_links()

    def store_user(self, user, s, db_user=None, db_user_loaded=False):
        """
        Stores username to the database.
        :param user:
        :return:
        """
        type_id = 0
        if user.user_type == 'User':
            type_id = 1
        elif user.user_type == 'Organization':
            type_id = 2

        try:
            if not db_user_loaded:
                db_user = s.query(GitHubUserDb).filter(GitHubUserDb.id == user.user_id).one_or_none()
            if db_user is not None:
                db_user.date_last_check = salch.func.now()
                db_user.usr_type = type_id
                s.merge(db_user)
                return 0

        except Exception as e:
            self.trace_logger.log(e)
            logger.warning('User query problem: %s' % e)

        # Store a new user here
        try:
            db_user = GitHubUserDb()
            db_user.id = user.user_id
            db_user.username = user.user_name
            db_user.usr_type = type_id
            s.add(db_user)
            return 0

        except Exception as e:
            self.trace_logger.log(e)
            logger.warning('[%02d] Exception during user store: %s' % (self.local_data.idx, e))
            if db_user_loaded:
                raise
            return 1

    def load_existing_key(self, key, s):
        """
        Loads existing key if exists
        :param key:
        :param s:
        :return:
        """
        key_id = int(key['id'])
        return s.query(GitHubKey).filter(GitHubKey.id == key_id).one_or_none()

    def store_key(self, user, key, s):
        """
        Stores user key to the database.
        :param user:
        :param key:
        :param s: current DB session
        :return:
        """

        # Loading phase
        existing_key = None
        try:
            if self.merge or self.update_keys:
                existing_key = self.load_existing_key(key, s)

        except Exception as e:
            logger.warning('Exception: %s' % e)

        # Storing phase
        try:
            if existing_key is not None:
                existing_key.date_last_check = salch.func.now()
                s.merge(existing_key)
                return existing_key

            key_id = int(key['id'])
            key_raw = key['key']

            key_type, key_val = [utils.strip(x) for x in key_raw.split(' ', 1)]

            db_key = GitHubKey()
            db_key.id = key_id
            db_key.key_id = key_id
            db_key.key_type = key_type
            db_key.key_user_found = user.user_name
            db_key.key_user_id_found = user.user_id
            db_key.text_raw = key_raw

            if key_type == 'ssh-rsa':
                try:
                    key_obj = utils.load_ssh_pubkey(key_raw)
                    if isinstance(key_obj, RSAPublicKey):
                        db_key.key_size = key_obj.key_size
                        numbers = key_obj.public_numbers()
                        db_key.key_modulus_hex = '%x' % numbers.n
                        db_key.key_exponent = numbers.e
                except Exception as e:
                    logger.info('Exception during processing the key[%s]: %s' % (key_type, e))

            s.add(db_key)
            return db_key

        except Exception as e:
            utils.silent_rollback(s)
            logger.warning('Exception during key store: %s' % e)
            return 1

    def assoc_key(self, user_id, key_id, s):
        """
        Association user <-> key
        :param user_id:
        :param key_id:
        :param s:
        :return:
        """
        try:
            uassoc = None
            if self.merge:
                uassoc = s.query(GitHubUserKeys)\
                    .filter(GitHubUserKeys.user_id == user_id)\
                    .filter(GitHubUserKeys.key_id == key_id)\
                    .one_or_none()

            if uassoc is not None:
                uassoc.lost_at = None
                return

            if uassoc is None:
                uassoc = GitHubUserKeys()
                uassoc.user_id = user_id
                uassoc.key_id = key_id
                uassoc.fount_at = salch.func.now()
                uassoc.lost_at = None
                s.add(uassoc)

            return 0

        except Exception as e:
            utils.silent_rollback(s)
            logger.warning('Exception during key assoc: %s' % e)
            return 1

    def deassoc_lost_keys(self, user_id, key_id, s):
        """
        Mark all lost keys as lost for the given user (not in key_id)
        :param user_id:
        :param key_id:
        :param s:
        :return:
        """
        try:
            q = s.query(GitHubUserKeys)\
                .filter(GitHubUserKeys.user_id == user_id)

            if len(key_id) > 0:
                q = q.filter(GitHubUserKeys.key_id.notin_(key_id))

            q.update({GitHubUserKeys.lost_at: salch.func.now()}, synchronize_session='fetch')
            return 0

        except Exception as e:
            utils.silent_rollback(s)
            logger.warning('Exception during key deassoc: %s' % e)
            return 1

    def flush_state(self):
        """
        Flushes state/config to the state file
        :return:
        """
        self.state['since_id'] = self.since_id
        self.state['rate_limit_remaining'] = self.rate_limit_remaining
        self.state['rate_limit_reset'] = self.rate_limit_reset
        utils.flush_json(self.state, self.state_file_path)

    #
    # DB
    #

    def fill_user_key_links(self):
        """
        Loads next X users from the database, advances since_id
        :return:
        """
        # self.since_id
        s = self.session()
        try:
            db_users = s.query(GitHubUserDb)\
                .filter(GitHubUserDb.id > self.since_id)\
                .order_by(GitHubUserDb.id)\
                .limit(self.user_load_bulk)\
                .all()

            for user in db_users:
                key_url = self.KEYS_ID_URL % user.id
                github_user = GitHubUser(user_id=user.id, user_name=user.username,
                                         user_type=user.usr_type, user_url=self.USER_URL % user.username)
                new_job = DownloadJob(url=key_url, jtype=DownloadJob.TYPE_KEYS, user=github_user,
                                      priority=random.randint(0, 1000), time_added=time.time())
                self.link_queue.put(new_job)

                if user.id > self.since_id:
                    self.since_id = user.id

        except Exception as e:
            logger.warning('Exception in loading users: %s' % e)
            utils.silent_rollback(s)

        finally:
            utils.silent_close(s)

    #
    # Auditing - errors, problems for further analysis
    #

    def audit_log(self, evt=None, link=None, jtype=None, job=None):
        """
        Appends audit log to the buffer. Lock protected.
        :param evt:
        :param link:
        :return:
        """
        log = collections.OrderedDict()
        log['time'] = time.time()
        log['evt'] = evt
        log['jtype'] = jtype
        log['link'] = link

        if job is not None and isinstance(job, DownloadJob):
            log['job'] = job.to_json()

        with self.audit_lock:
            self.audit_records_buffered.append(log)

    def flush_audit(self):
        """
        Flushes audit logs to the JSON append only file.
        Routine protected by the lock (no new audit record can be inserted while holding the lock)
        :return:
        """
        if self.audit_file is None:
            self.audit_records_buffered = []
            return

        self.audit_lock.acquire()
        try:
            if len(self.audit_records_buffered) == 0:
                return
            with open(self.audit_file, 'a') as fa:
                for x in self.audit_records_buffered:
                    fa.write(json.dumps(x) + "\n")
            self.audit_records_buffered = []
        except Exception as e:
            logger.error('Exception in audit log dump %s' % e)
        finally:
            self.audit_lock.release()

    #
    # State save / resume
    #

    def state_main(self):
        """
        State thread - periodical dump of the queues.
        :return:
        """
        logger.info('State thread started %s %s %s' % (os.getpid(), os.getppid(), threading.current_thread()))
        try:
            while not self.stop_event.is_set() and not self.terminate:
                # Dump stats each x seconds
                # Sleep is here because of dumping the state for the last time just before program quits.
                self.interruptible_sleep_delta(2)
                self.state_save()

                # Check memory conditions
                self.state_ram_check()

        except Exception as e:
            self.trace_logger.log(e)
            logger.error('Exception in state: %s' % e)

        finally:
            pass

        logger.info('State loop terminated')

    def state_ram_check(self):
        """
        Checks memory terminating conditions
        :return:
        """

        if self.max_mem is None:
            return

        cur_ram = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        if cur_ram <= self.max_mem:
            return

        logger.warning('Maximum memory threshold reached: %s kB = %s MB, threshold = %s kB'
                       % (cur_ram, cur_ram / 1024.0, self.max_mem))
        self.trigger_stop()

    def state_gen(self):
        """
        Dumps state
        :return:
        """
        try:
            js_q = collections.OrderedDict()
            js_q['gen'] = time.time()
            js_q['link_size'] = self.link_queue.qsize()
            js_q['since_id'] = self.since_id
            js_q['memory'] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

            # Dequeues
            self.new_users_events.maintain()
            self.new_keys_events.maintain()

            users_in_5min = self.new_users_events.under_limit(5*60)
            keys_in_5min = self.new_keys_events.under_limit(5*60)

            js_q['users_dequeue_size'] = self.new_users_events.len()
            js_q['keys_dequeue_size'] = self.new_keys_events.len()
            js_q['users_5min'] = users_in_5min
            js_q['keys_5min'] = keys_in_5min
            js_q['users_1min'] = users_in_5min / 5.0
            js_q['keys_1min'] = keys_in_5min / 5.0

            # link queue structure
            qdata = list(self.link_queue.queue)
            qdata.sort(cmp=DownloadJob.cmp)
            js_q['link_structure'] = ''.join(['.' if x.type == DownloadJob.TYPE_KEYS else 'U' for x in qdata])

            # Stats.
            js_q['resource_stats'] = [x.to_json() for x in list(self.resources_list)]

            # Finally - the queue
            js_q['link_queue'] = [x.to_json() for x in qdata]
            return js_q

        except Exception as e:
            self.trace_logger.log(e)
            logger.error('Exception in state: %s', e)

    def state_save(self):
        """
        saves the state
        :return:
        """
        try:
            js_q = self.state_gen()
            utils.flush_json(js_q, self.state_file_path)

        except Exception as e:
            self.trace_logger.log(e)
            logger.error('Exception in state: %s', e)

    def state_resume(self):
        """
        Attempts to resume the queues from the monitoring files
        :return:
        """
        try:
            if self.state is None:
                return

            if 'since_id' in self.state:
                self.since_id = self.state['since_id']

            if 'link_queue' in self.state:
                for rec in self.state['link_queue']:
                    job = DownloadJob.from_json(rec)
                    self.link_queue.put(job)
                logger.info('Link queue resumed, entries: %d' % len(self.state['link_queue']))

        except Exception as e:
            self.trace_logger.log(e)
            logger.warning('Exception in resuming the state %s' % e)
            logger.error('State resume failed, exiting')
            sys.exit(1)

def main():
    args_src = sys.argv
    parser = argparse.ArgumentParser(description='Downloads GitHub SSH keys')
    parser.add_argument('-c', dest='config', default=None, help='JSON config file')
    parser.add_argument('-s', dest='status', default=None, help='JSON status file')
    parser.add_argument('-t', dest='threads', default=1, help='Number of download threads to use')
    parser.add_argument('--max-mem', dest='max_mem', default=None, type=int,
                        help='Maximal memory threshold in kB when program terminates itself')
    parser.add_argument('--users-only', dest='users_only', default=False, action='store_const', const=True,
                        help='Load only users list')
    parser.add_argument('--merge', dest='merge', default=False, action='store_const', const=True,
                        help='Merge DB operation - merge instead of add. slower, updates if exists')
    parser.add_argument('--update-keys', dest='update_keys', default=False, action='store_const', const=True,
                        help='Update keys for existing users')

    args = parser.parse_args(args=args_src[1:])
    config_file = args.config

    audit_file = os.path.join(os.getcwd(), 'audit.json')
    state_file = args.status if args.status is not None else os.path.join(os.getcwd(), 'state.json')
    if os.path.exists(state_file):
        utils.file_backup(state_file, backup_dir='.')

    if os.path.exists('.github-quit'):
        os.remove('.github-quit')

    sys.argv = [args_src[0]]
    logger.info('GitHub loader started, args: %s' % args)
    l = GitHubLoader(state_file=state_file, config_file=config_file, audit_file=audit_file, threads=args.threads,
                     max_mem=args.max_mem, users_only=args.users_only, merge=args.merge, update_keys=args.update_keys)
    l.work()
    sys.argv = args_src

# Launcher
if __name__ == "__main__":
    main()


#Enunciado:
# Escribir un programa en el cual Dados 5 números enteros solicitados al usuario,
# determinar cuál de los 4 números enteros es más cercano al primero.

# Declaración de variables
int_PrimerNumero = 0
int_SegundoNumero = 0
int_TercerNumero = 0
int_CuartoNumero= 0
int_QuintoNumero= 0
int_Diferencia = 0
int_Comparador = 0
int_MasCercano = 0

#ingreso de Datos
int_PrimerNumero =  int(input("Ingrese el primer número: "))
int_SegundoNumero = int(input("Ingrese el segundo número: "))
int_TercerNumero =  int(input("Ingrese el tercer número: "))
int_CuartoNumero =  int(input("Ingrese el cuarto número: "))
int_QuintoNumero =  int(input("Ingrese el quinto número: "))

#Comparacion
int_MasCercano = int_SegundoNumero

int_Diferencia = int_PrimerNumero - int_SegundoNumero
if (int_Diferencia < 0):
    int_Diferencia = int_Diferencia * (-1)

int_Comparador = int_PrimerNumero - int_TercerNumero
if(int_Comparador < 0):
    int_Comparador = int_Comparador * -1
if (int_Comparador < int_Diferencia):
    int_MasCercano = int_TercerNumero
    int_Diferencia = int_Comparador

int_Comparador = int_PrimerNumero - int_CuartoNumero
if(int_Comparador < 0):
    int_Comparador = int_Comparador * -1
if (int_Comparador < int_Diferencia):
    int_MasCercano = int_CuartoNumero
    int_Diferencia = int_Comparador

int_Comparador = int_PrimerNumero - int_QuintoNumero
if(int_Comparador < 0):
    int_Comparador = int_Comparador * -1
if (int_Comparador < int_Diferencia):
    int_MasCercano =  int_QuintoNumero

print(f"El numero mas cercano a {str(int_PrimerNumero)} es el {str(int_MasCercano)}")


from AlteryxPythonSDK import AlteryxEngine, FieldType, RecordCreator, RecordInfo

from ayx_blackbird.proxies import FieldProxy
from ayx_blackbird.utilities.constants import NULL_VALUE_PLACEHOLDER

def test_field_proxy():
    record_info = RecordInfo(AlteryxEngine())
    record_info.add_field(
        field_name="float",
        field_type=FieldType.float,
        size=1,
        scale=1,
        source="test.txt",
        description="Test description",
    )

    field_proxy = FieldProxy(record_info.fields[0])

    record_creator = RecordCreator(record_info)
    assert field_proxy.name == "float"

    field_proxy.set(record_creator, "10.0")
    assert field_proxy.get(record_creator.finalize_record()) == 10.0

    field_proxy.set(record_creator, NULL_VALUE_PLACEHOLDER)
    assert field_proxy.get(record_creator.finalize_record()) is None

    field_proxy.set_null(record_creator)
    assert field_proxy.get(record_creator.finalize_record()) is None


import sys
import os

non_native_directory_path = sys.argv[1]
native_directory_path = sys.argv[2]
OutputFileName = sys.argv[3]

OutputFile = open(OutputFileName,"w", encoding="utf-8")
nativeFileList = os.listdir(native_directory_path)
nonNativeFileList = os.listdir(non_native_directory_path)
sentenceArray = []
wordArray = []
wordFreqDictionary = {}

for file in nonNativeFileList:
    nonNativeFile = open(non_native_directory_path + "\\" + file, "r", encoding = "utf8",  errors = "ignore")
    print("Running file " + str(file))
    for line in nonNativeFile:
        sentenceArray.append(line)

for file in nativeFileList:
    nativeFile = open(native_directory_path + "\\" + file, "r", encoding = "utf8",  errors = "ignore")
    print("Running file " + str(file))
    for line in nativeFile:
        sentenceArray.append(line)

counter = 1
for sentence in sentenceArray:
    wordsArray = sentence.split( )
    for word in wordsArray:
        if not word.isalpha():
            continue
        key = word
        if key in wordFreqDictionary.keys():
            wordFreqDictionary[key] = wordFreqDictionary[key] + 1
        else:
           wordFreqDictionary[key] = counter

wordFreqArray = []
for key in wordFreqDictionary:
    line = str(wordFreqDictionary[key]) + " " + key + "\n"
    wordFreqArray.append(line)

for str in wordFreqArray:
    OutputFile.write(str)
OutputFile.close()


"""
Model
-----
This module defines AI-dependent functions.
"""
import json
from pathlib import Path
import random
import numpy as np
import torch
from transformers import AutoTokenizer

from src.models.contriever import Contriever

SEED = 42
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)

contriever = Contriever.from_pretrained("facebook/contriever")
tokenizer = AutoTokenizer.from_pretrained("facebook/contriever")

with open(Path(__file__).parents[1] / "dataset\qa_df.json", "r", encoding="utf-8") as file:
    faq = json.load(file)

questions = [item["question"] for item in faq]

def get_answer(question: str):
    """Return an aswer and a context to the question from the datanase."""
    answer_and_context = [(item["text"], item["context"]) for item in faq if item["question"] == question]
    return answer_and_context

def find_similar_questions(question: str):
    """Return a list of similar questions from the database."""
    query = tokenizer(question, padding=True, truncation=True, return_tensors="pt")
    embeddings_query = contriever(**query)
    questions_toks = tokenizer(questions, padding=True, truncation=True, return_tensors="pt")
    embeddings_question = contriever(**questions_toks)
    dot_product = []
    for idx, _ in enumerate(embeddings_question):
        dot_product.append((embeddings_query @ embeddings_question[idx]).item())
    ids_max_score = sorted(range(len(dot_product)), key=lambda x: dot_product[x], reverse=True)[:3]
    result = []
    for i in ids_max_score:
        result.append(questions[i])
    return result


#sottomultipli e multipli del piede
f_y= 1/3
f_in = 12
f_m = 1/5280

#Chiede una misurazione in piedi all'utente
ft = int(input("Numero di piedi: "))

#Convertire
inches = ft*f_in
yards = ft*f_y
miles = ft*f_m

#Risultati
print(ft, " piedi corrispondono a ", inches, " pollici, ", yards, " yards e ", miles, " miglia.")


"""
    This module provides an AzureTranslator class for translating text using the Azure Cognitive Services Translator Text API.

    Classes:
    - AzureTranslator: A translator class that uses the Azure Cognitive Services Translator Text API for text translation.

        Methods:
        - __init__(): Initializes the AzureTranslator class by setting up the necessary credentials and endpoint.
        - preprocess(text): Preprocesses the text for translation, including language detection and language pair selection.
        - postprocess(response): Postprocesses the translation response and returns the translated text.
        - translate(text): Translates the given text and returns the translated text.

"""

import requests
import uuid
from langdetect import detect
import os
from src.translation.base import Translator

class AzureTranslator(Translator):
    def __init__(self):
        self.key = os.environ["azure_trans_key"]
        self.location = "eastus"
        self.endpoint = "https://api.cognitive.microsofttranslator.com"
        self.path = "/translate"
        self.constructed_url = self.endpoint + self.path
        self.headers = {
            "Ocp-Apim-Subscription-Key": self.key,
            "Ocp-Apim-Subscription-Region": self.location,
            "Content-type": "application/json",
            "X-ClientTraceId": str(uuid.uuid4()),
        }

    def preprocess(self, text):
        language = detect(text)
        if language == "ar":
            self.from_lang, self.to_lang = "ar", "en"
        else:
            self.from_lang, self.to_lang = "en", "ar"

        params = {"api-version": "3.0", "from": self.from_lang, "to": self.to_lang}

        body = [{"text": text}]

        response = requests.post(
            self.constructed_url, params=params, headers=self.headers, json=body
        )
        response = response.json()
        return response

    def postprocess(self, response):
        translated_text = response[0]["translations"][0]["text"]
        return translated_text

    def translate(self, text):
        response = self.preprocess(text)
        translation = self.postprocess(response)
        return translation

if __name__ == "__main__":
    # text_to_translate = "Hello, how are you?"

    with open("src/rasa/data/english-questions.txt", "r") as f:
        text_to_translate = f.read()

    translator = AzureTranslator()
    translation = translator.translate(text_to_translate)

    with open("src/rasa/data/azure-arabic-questions.txt", "w") as f:
        f.write(translation)

    # with open('src/rasa/data/example-arabic.txt', 'r') as f:
    #     text_to_translate = f.read()

    # translator = AzureTranslator()
    # translation = translator.translate(text_to_translate)

    # with open('src/rasa/data/azure-example-english-after-arabic.txt', 'w') as f:
    #     f.write(translation)

    # translation = translator.translate(text_to_translate)

    # print("Translation: {}".format(translation))


import sys
from collections import deque

n = int(sys.stdin.readline().rstrip())
n_ori = n
now = n
res = []
min_cnt = 4

def lagrangian(n, res):
    global min_cnt

    if res:
        if res[-1] < int(n ** 0.5):
            return

    if len(res) > 4:
        return

    if n and len(res) >= min_cnt:
        return

    if min_cnt == 1 or min_cnt == 2:
        return

    if not n:
        min_cnt = min(len(res), min_cnt)
        return

    now = int(n ** 0.5)
    for i in range(now):
        res.append(now - i)
        lagrangian(n - (now - i) ** 2, res)
        res.pop()

lagrangian(n, res)

print(min_cnt)


#!/usr/bin/env.python
# -*- coding:utf-8 -*-
'''
Dijkstra 最短路径算法. 需要记录路径, 沿途需要操作, 更新需要携带的车的数量. 主权是最短路, 第二权是带出量, 第三权是带回量
比如一条路上bike数为 3 10，是send2，back5，而不是back3。
次数不能只用Dijkstra算法, 因为不是一路进,也不是一路出,路上有进有出,相当于有负权的情况. Dijkstra和DFS结合
题目示例中的&gt有问题,应该是->

Dijkstra() {
  初始化;
  for(循环n次) {
    u = 使dis[u]最小的还未被访问的顶点的编号;
    记u为确定值;
    for(从出发能到达的所有顶点v){
      for(v未被访问 && 以u为中介点使s到顶点v的最短距离更优)
        优化dis[v];
    }
  }
}
'''
import heapq

C, N, p, M = map(int, input().split())

capacity = map(int, input().split())
diff = [0] + [int(i - C/2) for i in capacity]

table = [[] for i in range(N + 1)]
for _ in range(M):
    i, j, w = map(int, input().split())
    table[i].append([w, j])
    table[j].append([w, i])

# Dijkstra算法找出最短路的大小
inf = float('inf')
visited = [0] * (N + 1)
path = [[] for i in range(N + 1)]
dist = [inf] * (N + 1)
dist[0] = 0
h = [(dist[0], 0)]
# Dijkstra算法找到所有最短路径
while len(h)>0:
    _, i = heapq.heappop(h)  # 未收录的最近的点
    if visited[i] == 1:
        continue
    if i == p: # 找到目标结点,直接跳出
        break
    visited[i] = 1
    # 更新i的邻接点
    for w, j in table[i]:
        if visited[j]:
            continue
        if dist[j] >= dist[i] + w:
            if dist[j] > dist[i] + w:
                dist[j] = dist[i] + w
                path[j] = [i]
                heapq.heappush(h, (dist[j], j))
            else:
                path[j].append(i)  # 搜集所有最短路径

# path代表的二维数组是一个由起点与终点的间所有最短路径构成的图,他是原图的子图, 且可以看成是一个反向有向图,因为记录的是前驱结点的编号
# 以第二权和第三权来考察这些最短路径
# 重新建图, 正向思考比回溯思考来得容易
t2 = [[] for i in range(N + 1)]
for i in range(N+1):
    for j in path[i]: # j是i的前驱结点
        t2[j].append(i)

Need = inf
Back = inf
need, back = 0, 0
R = None
def dfs(s, need, back, road:list):
    global Need, Back, R
    if s == p:  # 因为t2中包含了一些无关的点的路径,故一定是走到目标点p才算.别的无关路径会在函数中自然走到头.
        if need<Need:
            Need = need
            Back = back
            R = road
        if need == Need:
            if back < Back:
                Back = back
                R = road
        return
    for e in t2[s]:
        r = list(road)
        need2, back2 = need, back
        if diff[e] < 0:
            if back2 + diff[e] >= 0:
                back2 += diff[e]
            else:
                need2 = -(back2+diff[e])+need2
                back2 = 0
        if diff[e] > 0:
            back2 += diff[e]
        r.append(e)
        dfs(e, need2, back2, r)

dfs(0, need, back, [0])

num = len(R) - 1
road = '0' + '->{}' * num

print(Need, road.format(*R[1:]), Back)

Advanced/1018 Public Bike Management (30).py


import math
import random
import json
import csv

from src.preprocessor import preprocessor
from src.ngram import Ngrams

class speech_classification:
    def __init__(self):
        return

    def get_model(self, n, threshold):
        return Ngrams({"n": n, "threshold": threshold})

    def compute(self, data, ngram):  # the smaller total_log_prob is, the more probable the sentence is
        total_log_prob_arr = [];
        for i in range(len(data)):
            test_sentence = data[i]
            total_log_prob = 0
            for x in range(0, len(test_sentence) - ngram.n + 1):
                key = ngram.to_key(test_sentence[x: x + ngram.n - 1]) if ngram.n > 1 else ngram.placeholder
                target = test_sentence[x + ngram.n - 1]
                log_prob = math.log(ngram.lookup_dist(key, target)) if ngram.lookup_dist(key, target) > 0 else 0
                total_log_prob -= log_prob
            total_log_prob_arr.append(total_log_prob)
        return total_log_prob_arr

    def classify(self):
        obama_model = preprocessor("../Assignment1_resources/train/obama.txt", 0).data
        trump_model = preprocessor("../Assignment1_resources/train/trump.txt", 0).data
        test_data = (preprocessor("../Assignment1_resources/test/test.txt", 0).data)[5]
        ngram_obama = self.get_model(3, 100)
        ngram_trump = self.get_model(3, 100)
        ngram_obama.dist_table_smoothed_kneser_ney(obama_model)
        ngram_trump.dist_table_smoothed_kneser_ney(trump_model)
        #ngram_obama.dist_table_add_one_smooth(obama_model, 2)
        #ngram_trump.dist_table_add_one_smooth(trump_model, 2)
        #print ngram_obama.count_table
        obama_arr = self.compute(test_data, ngram_obama)
        trump_arr = self.compute(test_data, ngram_trump)
        result_arr = []

        for i in range(len(obama_arr)):
            if obama_arr[i] < trump_arr[i]:
                result_arr.append('0')
            else:
                result_arr.append('1')

        with open('../output/speech_classification.csv', mode='w') as speech_output:
            speech_writer = csv.writer(speech_output, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            speech_writer.writerow(['Id', 'Prediction'])
            for i in range(len(result_arr)):
                speech_writer.writerow([i, result_arr[i]])

        #print result_arr

my_speech_classification = speech_classification()
my_speech_classification.classify()


# Laura Durant
# Measures For Justice Coding Eval
# Parse Data
# 4/23/19

# In this exercise I use double quotes to escape ambiguous commas

import io
import re

def readFile(filename):
    file = open(filename, 'r')
    return file.read()

# The fixes for various lengths are largely guesswork based on patterns.
# A better way to do this is to track the properly formatted names and
# identify them within the incorrect data. The set was small enough for
# me to use patterns to rough out a solution rather than attempt some
# kind of light AI to identify name groupings.
def fix(fields):
    count = len(fields)
    if count is 0:
        return
    if count is 1 and not fields[0]:
        return
    isNum = []
    notNum = []
    for field in fields:
        if re.search("[0-9][0-9][0-9]", field):
            isNum += [field]
        else:
            notNum += [field]
    #isNum is always length 3
    if len(notNum) is 2:
        #no attorney listed
        fields.insert(3,"")
        return fields
    if len(notNum) is 6:
        # most likely "last, first, suffix, attorney last, attorney first, desc"
        # handle suffixing in the 4 case instead of rewriting the code here
        notNum = [notNum[1] + ' ' + notNum[0],notNum[2],notNum[4] + ' ' + notNum[3], notNum[5]]
    if len(notNum) is 4:
        # if both have spaces, assume it's a suffix for the first name
        if ' ' in notNum[0] and ' ' in notNum[2]:
            notNum = ['"' + notNum[0] + ', ' + notNum[1] + '"',notNum[2],notNum[3]]
        # if only the first name has a space, assume it's part of the attorney name
        elif ' ' in notNum[0]:
            notNum = [notNum[0],notNum[2] + ' ' + notNum[1],notNum[3]]
        # if only the second name has a space, assume it's part of the first name
        elif ' ' in notNum[2]:
            notNum = [notNum[1] + ' ' + notNum[0], notNum[2],notNum[3]]
        # no other cases occur in this set
    if len(notNum) is 5:
        # it's probably two names in "last, first" style
        notNum = [notNum[1] + ' ' + notNum[0],notNum[3] + ' ' + notNum[2],notNum[4]]
    #notNum is never >6
    if len(isNum) is not 3 or len(notNum) is not 3:
        # I'd have to add more handling since this is more manual fixing than algorithmic
        print("Something has gone amiss...")
    # put the fields together
    return [isNum[0],isNum[1],notNum[0],notNum[1],isNum[2],notNum[2]]

content = readFile("fakedata.csv").split("\n")
lines = []
for line in content:
    fields = []
    start = 0
    inQuote = False
    for i in range(len(line)):
        if line[i] is '"':
            if inQuote:
                if line[i+1] is '"':
                    i += 1
                else:
                    inQuote = False
            else:
                inQuote = True
        elif line[i] is ',':
            if not inQuote:
                fields += [line[start:i]]
                start = i+1
    fields += [line[start:]]
    if len(fields) is not 6:
        fields = fix(fields)
    if fields:
        lines += [fields]

f = open("valid.csv", 'w')
for line in lines:
    f.write(str.join(",", line)+"\n")


#!/usr/bin/env python3

import loader
import sys
import util
from random import randint
import time

"""
Title: Mad House
Author: Keith Campbell
Translator: David Sarkies
Version: 0.1
Date:
Source: https://ia601902.us.archive.org/3/items/Creepy_Computer_Games_1983_Usborne_Publishing/Creepy_Computer_Games_1983_Usborne_Publishing.pdf
This game can be found on page 12 of Creepy Computer Games, and it a python3 translation.

In this game you control two doors, and you have to line it up with the third, that will move randomly every twenty-five turns.
"""

instructions = "You're trapped in a weird house where everything moves including the walls. If\n"
instructions = "{}the doorways would line up, even for a moment, you could make a dash for\n".format(instructions)
instructions = "{}freedom. You've found a console which appears to control some of the\n".format(instructions)
instructions = "{}movements in the house. Keys X and C make the doorway nearest to you\n".format(instructions)
instructions = "{}(top of the screen) change direction. Keys N and M have the same effect\n".format(instructions)
instructions = "{}on the door way furthest from you. There doesn't seem to be any way for\n".format(instructions)
instructions = "{}controlling the centre doorway.".format(instructions)
instructions = "{}As you frantically wrestle with the knobs on the console, you can hear\n".format(instructions)
instructions = "{}footsteps pounding down the corridor behind you. The number top left\n".format(instructions)
instructions = "{}of the screen shows their progress towards you. If you can't escape\n".format(instructions)
instructions = "{} before the phantom footsteps catch up with you ...".format(instructions)

def main_game():

	width = 31
	footsteps = 240
	door_position = [randint(1,(width-4)+1),randint(1,(width-4)+1),randint(1,(width-4)+1)]
	door_direction = [1,0,-1]
	finished = 0

	#Main game loop
	while (finished == 0):
		display(footsteps,width,door_position)
		print(door_direction)
		action = get_action()
		if (action != 'q'):
			door_direction = process_action(action,door_direction)
		else:
			finished = 3

		door_position = move_doors(door_position,door_direction,width,footsteps)
		footsteps -=1
		finished = check_results(finished,footsteps,door_position)

	end_game(finished)

#Displays the screen
def display(footsteps,width,door_position):

	util.clear_screen()
	print(footsteps)
	for x in range(3):
		print(display_walls(width,door_position[x]))
		print()

	#Commands
	print("\t\tx <-   Top Wall  -> c")
	print("\t\tm <- Bottom Wall -> n")
	print("\t\t\tq - quit")

#Builds each of the doors and positions them
def display_walls(width,door):

	line = "\t"

	for x in range(width):

		if (x==door):
			line = "{} ".format(line)
		elif (x==door-1):
			line = "{}>".format(line)
		elif (x==door+1):
			line = "{}<".format(line)
		else:
			line = "{}*".format(line)

	return line

#Function to get user command
def get_action():

	action = util.input_with_timeout_no_comment("",2)
	return action

#Processes the player action
def process_action(action,door_direction):

	if (action =="x"):
		door_direction[0] = -1
	elif (action == "c"):
		door_direction[0] = 1
	elif (action == "m"):
		door_direction[2] = -1
	elif (action == "n"):
		door_direction[2] = 1

	return door_direction

def move_doors(door_position,door_direction,width,footsteps):

	#Checks if the doors have reached the end of the row
	#If not, moves the door in the set directions
	if (((door_position[0]>1) and door_direction[0]==-1)
		or ((door_position[0]<width-2) and door_direction[0]==1)):
		door_position[0] += door_direction[0]
	if (((door_position[2]>1) and (door_direction[2] == -1))
		or ((door_position[2]<width-2) and (door_direction[2]==1))):
		door_position[2] += door_direction[2]

	#Middle door moves every 5 moves. If it moves
	#beyond the width it goes back to the beginning
	if (footsteps/25==int(footsteps/25)):
		door_position[1] +=randint(0,20)-10

		if (door_position[1] >width-2):
			door_position[1] = 2

	return door_position

#Checks if the game has ended.
def check_results(finished,footsteps,door_position):

	if (footsteps == 0):
		finished = 1
	elif ((door_position[0] == door_position[1]) and (door_position[1] == door_position[2])):
		finished = 2

	return finished

def end_game(finished):

	if (finished == 1):
		print("Too Late! The footsteps have stopped")
		print("AGGGHHHH!!!!")
	elif (finished == 2):
		print("You are free!")

#Passes the current file as a module to the loader
if __name__ == '__main__':
	loader.start_game("Mad House",sys.modules[__name__])


import os
import torch
import json
import pandas as pd
import matplotlib.pyplot as plt
import sys
import cv2
import numpy as np
import glob
from skimage import io, transform

from utils.utils import json_load
from utils.constants import IMG_EXT

import torch
import torchvision
from torch.utils.data import Dataset
import torchvision.transforms as transforms

sample_input = 'input/iwildcam-2020-fgvc7'
train_data_path = f'{sample_input}/train'
json_path = f'{train_data_path}/sample_train.json'
train_annotations = json_load(json_path=f'{sample_input}/iwildcam2020_train_annotations.json')
num_class = len(train_annotations['categories'])
labels = [cls['id'] for cls in train_annotations['categories']]

transform = transforms.Compose(
    [
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
        transforms.Resize((224, 224))
    ]
)

class ClassificationDataset(Dataset):
    """Face Landmarks dataset."""

    def __init__(self, json_path, root_dir, transform=None):
        """
        Arguments:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.root_dir = root_dir
        self.transform = transform

        image_dict = json_load(json_path=json_path)
        self.data = pd.DataFrame(image_dict.items(),
                       columns=['image_path', 'label'])

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_path = os.path.join(self.root_dir,
                                self.data.iloc[idx, 0])
        img = io.imread(img_path)
        label = self.data.iloc[idx, 1]
        label = torch.tensor(labels.index(label))
        # label =  torch.nn.functional.one_hot(torch.tensor(labels.index(label)), num_classes=self.num_classes)

        if self.transform:
            img = self.transform(img)

        return img, label

class InferenceDataset(Dataset):
    def __init__(self, root_dir, transform=None):
        """
        Inference dataset without labels
        Arguments:
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.root_dir = root_dir
        self.transform = transform

        image_paths = [path for path in glob.glob(f'{root_dir}/*') if path.split('.')[-1].lower() in IMG_EXT]
        image_paths = [os.path.basename(path) for path in image_paths]
        image_dict = {'image_path': image_paths}
        self.data = pd.DataFrame(data=image_dict)

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_path = os.path.join(self.root_dir,
                                self.data.iloc[idx, 0])

        img = io.imread(img_path)

        if self.transform:
            img = self.transform(img)

        return img, img_path

def main():
    # img_dataset = ClassificationDataset(
    #     json_path=json_path,
    #     root_dir=train_data_path,
    #     transform=transform
    #     )

    img_dataset = InferenceDataset(
        root_dir=train_data_path,
        transform=transform
        )

    for i in range(len(img_dataset)):
        img, label  = img_dataset[i]

        # print(i, img.shape, label)
        # img = sample['image']
        # img = cv2.imread(sample['image'])
        # print(img.shape)
        showImg = np.rollaxis(np.array(img), 0, 3)
        plt.imshow(showImg)
        plt.show()

if __name__ == '__main__':
    main()


"""This script finds the correlation between ratemaps for each cell for each task type."""

from ripple_heterogeneity.utils import (
    functions,
    loading,
    add_new_deep_sup,
)
import nelpy as nel
from ripple_heterogeneity.place_cells import maps
import pandas as pd
import numpy as np
from itertools import combinations
from typing import Union

def find_repeated_strings(lst: list) -> dict:
    """
    Find all repeated strings in a list.
    Parameters
    ----------
    lst : list or list-like object
        List of strings to search for repeated strings.
    Returns
    -------
    repeated_strings : dict
        Dictionary with repeated strings as keys and list of indices where
        the string is found as values.
    """
    if not isinstance(lst, list):
        try:
            lst = list(lst)
        except:
            raise TypeError("lst must be a list or a list-like object")
    repeated_strings = {}
    for i in range(len(lst)):
        if lst[i] in repeated_strings:
            repeated_strings[lst[i]].append(i)
        elif lst.count(lst[i]) > 1:
            repeated_strings[lst[i]] = [i]
    return repeated_strings

def ratemap_cor(A: np.ndarray, B: np.ndarray) -> float:
    # for given ratemaps A & B, find their correlation
    x1 = A.flatten()
    x2 = B.flatten()
    bad_idx = np.isnan(x1) | np.isnan(x2)
    return np.corrcoef(x1[~bad_idx], x2[~bad_idx])[0, 1]

def run(
    basepath: str, n_shuff: int = 500, parallel_shuff: bool = True
) -> Union[pd.DataFrame, None]:

    position_df = loading.load_animal_behavior(basepath)
    position_df_no_nan = position_df.query("not x.isnull() & not y.isnull()")

    if position_df_no_nan.shape[0] == 0:
        return pd.DataFrame()

    st, cm = loading.load_spikes(basepath, putativeCellType="Pyr", brainRegion="CA1")

    if st.isempty:
        return pd.DataFrame()

    cm = add_new_deep_sup.deep_sup_from_distance(cm)

    pos = nel.PositionArray(
        data=position_df_no_nan[["x", "y"]].values.T,
        timestamps=position_df_no_nan.timestamps.values,
    )
    beh_df = loading.load_epoch(basepath)
    beh_df = beh_df.query("environment != 'sleep'")

    beh_epochs = nel.EpochArray(np.array([beh_df.startTime, beh_df.stopTime]).T)

    # find all repeated strings in beh_df.environment
    tasks = find_repeated_strings(list(beh_df.environment.values))

    results_df = pd.DataFrame()
    for task in tasks:

        results_df_temp = pd.DataFrame()
        # linearize position if necessary
        if "linear" in task:
            for i in tasks[task]:
                idx = functions.in_intervals(pos.abscissa_vals, beh_epochs[i].data)
                x, y = pos.data[:, idx]
                if len(x) == 0:
                    continue
                pos._data[:, idx] = functions.linearize_position(x, y)

        # pull out coordinates for for this task type to get spatial extent
        current_pos = pos[beh_epochs[tasks[task]]]
        if current_pos.isempty:
            continue
        xminmax = [np.nanmin(current_pos.data[0, :]), np.nanmax(current_pos.data[0, :])]
        yminmax = [np.nanmin(current_pos.data[1, :]), np.nanmax(current_pos.data[1, :])]

        # find ratemaps for each cell
        # for reduced complexity, using 2d spatial maps and not splitting by direction (linear track)
        ratemaps = []
        spatial_pval = []
        n_spikes = []
        peak_firing_rate = []
        epoch = []
        for i in tasks[task]:
            current_pos = pos[beh_epochs[i]]
            current_st = st[beh_epochs[i]]

            if current_pos.isempty | current_st.isempty:
                continue

            spatial_maps = maps.SpatialMap(
                current_pos,
                current_st,
                dim=2,
                s_binsize=3,
                x_minmax=xminmax,
                y_minmax=yminmax,
                tuning_curve_sigma=3,
                n_shuff=n_shuff,
                parallel_shuff=parallel_shuff,
            )
            # store ratemaps
            ratemaps.append(spatial_maps.ratemap)
            # store p-values for spatial information shuffle
            spatial_pval.append(spatial_maps.shuffle_spatial_information())
            # store n spikes per cell
            n_spikes.append(current_st.n_events)
            # store peak firing rate for each cell
            peak_firing_rate.append(spatial_maps.max(axis=1))
            # store epoch n for each cell
            epoch.append(np.ones((len(current_st.data), 1)) * i)

        # find correlation between all pairs of ratemaps for each cell
        # find all pairs of ratemaps
        pairs = np.array(list(combinations(np.arange(0, len(ratemaps)), 2)))
        # set up arrays to store metadata
        metadata = {
            "spatial_pval": spatial_pval,
            "n_spikes": n_spikes,
            "peak_firing_rate": peak_firing_rate,
            "epoch": epoch,
        }
        metadata_ref_target = {}
        for key in metadata:
            metadata_ref_target["ref_" + key] = np.zeros((len(st.data), len(pairs)))
            metadata_ref_target["target_" + key] = np.zeros((len(st.data), len(pairs)))

        between_corr = np.zeros((len(st.data), len(pairs)))

        # loop through pairs
        for pair_i, s in enumerate(pairs):
            # loop through cells
            for cell_i in range(len(st.data)):
                # find correlation between ratemaps
                between_corr[cell_i, pair_i] = ratemap_cor(
                    ratemaps[s[0]][cell_i, :, :], ratemaps[s[1]][cell_i, :, :]
                )
                # store metadata for each cell for each pair
                for key in metadata:
                    metadata_ref_target["ref_" + key][cell_i, pair_i] = metadata[key][s[0]][cell_i]
                    metadata_ref_target["target_" + key][cell_i, pair_i] = metadata[key][s[1]][cell_i]

        # store results in dataframe
        results_df_temp["correlation"] = between_corr.flatten("F")

        for key in metadata_ref_target:
            results_df_temp[key] = metadata_ref_target[key].flatten("F")

        results_df_temp["UID"] = np.tile(cm.UID.values, len(pairs))
        results_df_temp["deepSuperficial"] = np.tile(
            cm.deepSuperficial.values, len(pairs)
        )
        results_df_temp["task"] = task
        results_df = pd.concat([results_df, results_df_temp], ignore_index=True)

    results_df["basepath"] = basepath
    return results_df


import timeit
import copy

def solve_pt1(vstup):
    def playGame(p1, p2):
        end = len(p1) + len(p2)
        while len(p1) != end and len(p2) != end:
            r1 = p1.pop(0)
            r2 = p2.pop(0)
            if r1 > r2:
                p1.append(r1)
                p1.append(r2)
            else:
                p2.append(r2)
                p2.append(r1)
        return p1 if len(p1) == end else p2

    def calcResult(winner):
        res = 0
        for i, card in enumerate(winner[::-1], start=1):
            res += card * i
        return res

    with open(vstup, "r") as file:
        k = []
        file.readline()
        for line in file:
            if "Player" in line:
                p1 = copy.deepcopy(k)
                k.clear()
                continue
            if line == "\n":
                continue
            k.append(int(line.strip()))
        p2 = copy.deepcopy(k)
        winner = playGame(p1, p2)
        res = calcResult(winner)

    return res

def solve_pt2(vstup):
    class Player:
        def __init__(self, desk):
            self.desk = desk
            self.configurations = set()

    def playGame(p1, p2):
        end = len(p1.desk) + len(p2.desk)
        while len(p1.desk) != end and len(p2.desk) != end:
            if tuple(p1.desk) in p1.configurations or tuple(p2.desk) in p2.configurations:
                return (p1, 1)
            p1.configurations.add(tuple(p1.desk))
            p2.configurations.add(tuple(p2.desk))
            r1 = p1.desk.pop(0)
            r2 = p2.desk.pop(0)
            if len(p1.desk) >= r1 and len(p2.desk) >= r2:
                _, idx = playGame(Player(p1.desk[:r1]), Player(p2.desk[:r2]))
                if idx == 1:
                    p1.desk.append(r1)
                    p1.desk.append(r2)
                else:
                    p2.desk.append(r2)
                    p2.desk.append(r1)
            else:
                if r1 > r2:
                    p1.desk.append(r1)
                    p1.desk.append(r2)
                else:
                    p2.desk.append(r2)
                    p2.desk.append(r1)

        return (p1, 1) if len(p1.desk) == end else (p2, 2)

    def calcResult(winner):
        res = 0
        for i, card in enumerate(winner[::-1], start=1):
            res += card * i
        return res

    with open(vstup, "r") as file:
        k = []
        file.readline()
        for line in file:
            if "Player" in line:
                p1 = copy.deepcopy(k)
                k.clear()
                continue
            if line == "\n":
                continue
            k.append(int(line.strip()))
        p2 = copy.deepcopy(k)

        winner, _ = playGame(Player(p1), Player(p2))

        res = calcResult(winner.desk)
        return res

vstup = "vstup.txt"

start = timeit.default_timer()
result1 = solve_pt1(vstup)
print(result1)
print(f"Cas vykonavania pt1:{timeit.default_timer() - start} sec")

start = timeit.default_timer()
result2 = solve_pt2(vstup)
print(result2)
print(f"Cas vykonavania pt2:{timeit.default_timer() - start} sec")


import datetime
import re

# 3.3
def remove_text_garbage(
        text, remove_apos=True, remove_parens=True, remove_punct=True):
    if remove_parens:
        text = re.sub(r"\(([^)]*)\)", r"\1", text)
    if remove_apos:
        text = re.sub(r"([^\s']+)'[\S]+", r"\1", text)
    if remove_punct:
        text = text.replace(",", "")
        text = text.replace(".", "")
        text = text.replace("?", "")
    if text.startswith('"') and text.endswith('"'):
        text = text[1:-1]
    return text

# 4.1
# When I originally wrote my word search functions, I made them split the
# question sentence into each word, and then turned the set of words into a set.
# Then, I turned the list of words to search for into a set, and checked whether
# the words to search for were a subset of the words in the question sentence
# resser = df_word_search(jeopardy, "way", "hell", exact=False)
# print(jeopardy[resser].question.count())
# resser = df_word_search(jeopardy, "blue"] exact=False)
# print(jeopardy[resser].question.count())
# resser = df_word_search(jeopardy, "poo", "pool", exact=False)
# print(jeopardy[resser].question.count())
# resser = df_word_search(jeopardy, "they", "there", exact=False)
# print(jeopardy[resser].question.count())
# resser = df_word_search(jeopardy, "'s", exact=False)
# print(jeopardy[resser].question.count())
# resser = df_word_search(jeopardy, "'re", exact=False)
# print(jeopardy[resser].question.count())
# resser = df_word_search(jeopardy, "'ve", exact=False)
# print(jeopardy[resser].question.count())

# 5.1
def money_to_float(money):
    if money is None or money == "None":
        return 0.0
    if money.startswith("$"):
        money = money[1:]
    if money.find(",") >= 0:
        monies = money.split(",")
        money = "".join(monies)
    return float(money)

# 7.1
def to_date(date):
    date_things = date.split("-")
    #if len(date_things) < 3:
        #print("Date:", date)
    #date_things += ["1"] * (3 - len(date_things))
    return datetime.date(*map(int, date_things))

# Helper - set up Jeopardy DataFrame for analysis
def preprocess(jeopardy):
    jeopardy.rename(columns={
    "Show Number": "show_number",
    " Air Date": "air_date",
    " Round": "round",
    " Category": "category",
    " Value": "value",
    " Question": "question",
    " Answer": "answer"
    }, inplace=True)
    jeopardy["value_float"] = jeopardy.value.apply(money_to_float)
    jeopardy.air_date = jeopardy.air_date.apply(to_date)
    jeopardy["air_year"] = jeopardy.air_date.apply(lambda d: d.year)


from Global_Value import *
from Network import sexualNetwork

class NetworkPopulation():
	def __init__(self, pop_size, network_size):
		t0 = str(datetime.now()).split(" ")[1]
		self.size = pop_size
		self.population = [sexualNetwork(network_size,1) for _ in range(self.size)]
		self.mutation = mutation
		self.crossing_over = crossing_over
		self.fitnessmean = []
		self.Subfitness = None
		self.SubfitnessMean = [[],[],[]]
		self.actualMinFitIndice = None
		t1 = str(datetime.now()).split(" ")[1]
		print("--> TIME INITIATING POP" ,Timepassed(t0,t1))
	def Save_pop(self):
		for index in range(len(self.population)):
			with open("Population\essai"+str(index), 'wb') as f:
				nx.write_adjlist(pop.population[index],f)

	def Save_best(self):
		all_fitness = []
		GraphWithFitness = []
		for index in range(len(self.population)):
			if self.population[index].fitness != None:
				all_fitness.append(self.population[index].fitness)
				GraphWithFitness.append(self.population[index])
		index_min = all_fitness.index(min(all_fitness))
		with open("Best_graph1000_1", 'wb') as f:
			nx.write_adjlist(GraphWithFitness[index_min],f)

	def Selection(self,t):
		#return list of index of graph that will be selectionned for mutations
		t0 = str(datetime.now()).split(" ")[1]
		weight = []
		NonePos = []
		OtherPos = []
		self.Subfitness = [[],[],[]]
		for index in range(self.size):
			fitness = self.population[index].fitness
			self.Subfitness[0].append(self.population[index].Mydeg_rel)
			self.Subfitness[1].append(self.population[index].MyD_rel)
			self.Subfitness[2].append(self.population[index].Mycc_rel)
			if fitness == None:
				NonePos.append(index)
			else :
				OtherPos.append(index)
				weight.append(fitness)
		try :
			self.fitnessmean.append(min(weight))
			self.actualMinFitIndice = weight.index(min(weight))
		except :
			self.fitnessmean.append(None)
			self.actualMinFitIndice = None
		if len(NonePos)>= self.size/2:
			t1 = str(datetime.now()).split(" ")[1]
			print("--> TIME SELECTION" ,Timepassed(t0,t1))
			return(NonePos)
		else:
			weight_p = [w**(1+t/1000) for w in weight]
			"""
			if (t%100==0):
				plt.subplot(211)
				plt.pie(weight)
				plt.subplot(212)
				plt.pie(weight_p)
				plt.show()
			"""
			ToReturn = list(np.random.choice(OtherPos,floor(self.size/2)+1-len(NonePos), p = np.array(weight_p)/sum(weight_p), replace = False))
			"""
			sortedIndex = np.argsort(np.array(weight))
			indicesToChange = sortedIndex[-floor(self.size/2)+1-len(NonePos):]
			ToReturn = [OtherPos[i] for i in indicesToChange]
			"""
			t1 = str(datetime.now()).split(" ")[1]
			print("--> TIME SELECTION" ,Timepassed(t0,t1))
			return(np.array(ToReturn + NonePos))

	def Evolution(self,t) :
		t0 = str(datetime.now()).split(" ")[1]
		selected = self.Selection(t)
		self.FilterSubFitness()
		for i in range(3):
			if self.actualMinFitIndice != None :
				self.SubfitnessMean[i].append(self.Subfitness[i][self.actualMinFitIndice])
			else :
				self.SubfitnessMean[i].append(None)
		for s in selected :
			#print("|||||||||||||||||||| Le graph " + str(s) )
			self.population[s].Update_graph(mutation,crossing_over,self.population)
		t1 = str(datetime.now()).split(" ")[1]
		print("--> TIME EVOLUTION" ,Timepassed(t0,t1))

	def EvoluNGeneration(self,n) :
		for i in range (n):
			#print([g.fitness for g in self.population])
			print("|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| J'evolue",i)
			self.Evolution(i)

	def FilterSubFitness(self):
		t0 = str(datetime.now()).split(" ")[1]
		ToDelete = []
		for i in range (3):
			for j in range (len(self.Subfitness[i])):
				if (self.Subfitness[i][j])==None or (self.Subfitness[i][j])=="nan":
					if not j in ToDelete :
						ToDelete.append(j)
		ToDelete = sorted(ToDelete,reverse = True)
		for i in range(3):
			for j in range(len(ToDelete)):
				del self.Subfitness[i][ToDelete[j]]
		t1 = str(datetime.now()).split(" ")[1]
		print("--> TIME FILTERSUBFITNESS" ,Timepassed(t0,t1))

pop = NetworkPopulation(100,1000)

nbrgen = 1000

pop.EvoluNGeneration(nbrgen)

"""
plt.plot(pop.fitnessmean)
plt.show()
"""
"""
fig = plt.figure()
plt.plot(pop.fitnessmean)
plt.savefig("myfig.png")
"""

pop.Save_best()

t = range(0,nbrgen,1)

#deg/D/cc
fig = plt.figure()
plt.plot(t,pop.SubfitnessMean[0],"r",t,pop.SubfitnessMean[1],"b",t,pop.SubfitnessMean[2],"g",t,pop.fitnessmean,"c--")
plt.savefig("Fitness1000_1.png")
plt.show()
"""
plt.subplot(311)
nx.draw_circular(pop.population[0], with_labels=True, font_weight='bold')

plt.subplot(312)
nx.draw_circular(pop.population[1], with_labels=True, font_weight='bold')

pop.population[0].CrossOver(1,5,pop.population)
plt.subplot(313)
nx.draw_circular(pop.population[0], with_labels=True, font_weight='bold')

plt.show()
"""