"""PRE gen hook
"""
import logging
import os
import shutil
import sys
import git

_logger = logging.getLogger()
TMP_DIR = './tmp/'

def pre_hook():
    """Flag the PRE gen HOOK execution"""
    _logger.warning("PRE Gen hook executed")

    try:

        # Default value = Volt
        theme = "{{cookiecutter.theme}}"
        repo  = 'https://github.com/app-generator/django-dashboard-volt.git'

        if 'material-dashboard' == theme:
            repo  = 'https://github.com/app-generator/django-dashboard-material.git'

        if 'soft-ui' == theme:
            repo  = 'https://github.com/app-generator/django-soft-ui-dashboard.git'

        if 'datta-able' == theme:
            repo  = 'https://github.com/app-generator/django-datta-able.git'

        if 'star-admin' == theme:
            repo  = 'https://github.com/app-generator/django-star-admin.git'

        print(' Cloning theme: <' + theme + '>, REPO: ' + repo )

        repo = git.Repo.clone_from( repo, TMP_DIR, branch='master')
        print(' ...done')

    except OSError as e:
        _logger.warning(f"Error: {e}")
        sys.exit(0)

    # Exit with success
    sys.exit(0)

if __name__ == "__main__":
    pre_hook()


# Joshua Arulsamy
# Assignment 7
# 03/09/2020
# COSC 2049 500
import random

# Unicode characters did not work on windows in CMD.
# They worked fine in a linux KDE Konsole session.

# Unicode representation of dice
one = "\u2680"
two = "\u2681"
three = "\u2682"
four = "\u2683"
five = "\u2684"
six = "\u2685"
ucodes = [one, two, three, four, five, six]

# Count wins/losses/points
WINS = 0
LOSSES = 0

def roll():
    # Roll the two dice
    # Between 1 and 6 so point math is easy
    roll1 = random.randint(1, 6)
    roll2 = random.randint(1, 6)
    roll_sum = roll1 + roll2
    return roll_sum, f"{ucodes[roll1 - 1]} {ucodes[roll2 - 1]}"

# Keep going until the user says no.
# Anything other than N/n assumes yes.
while input("Roll? [Y/N] ").lower() != "n":
    roll_sum, ucode_dice = roll()

    # Win conditions
    if roll_sum == 7 or roll_sum == 11:
        WINS += 1
        print(f"\t{ucode_dice}: {roll_sum} Win")
    # Loss conditions
    elif roll_sum == 2 or roll_sum == 3 or roll_sum == 12:
        LOSSES += 1
        output_str = "Loss"
        print(f"\t{ucode_dice}: {roll_sum} Loss")
    # Everything else is just a point
    else:
        done = False
        # Reroll until 7 (loss) or their original roll (win).
        while not done:
            new_sum, ucode_dice = roll()
            if new_sum == 7:
                done = True
                LOSSES += 1
                print(f"\t{ucode_dice}: {new_sum} Loss")
            elif new_sum == roll_sum:
                done = True
                WINS += 1
                print(f"\t{ucode_dice}: {new_sum} Win")
            # Show the number of rerolls (debug)
            # else:
            #     print(".", end="")

# Show total wins/losses
print(f"WINS: {WINS}")
print(f"LOSSES: {LOSSES}")


# shoebox.main

import os
from tkinter import *
from tkinter import ttk
from shoebox import services
from shoebox.medit import Medit
from shoebox.px import Px
from shoebox.cx import Cx
from shoebox.nailer import Nailer
from shoebox.sweeper import Sweeper
import shoebox

root = Tk()
root.title("Shoebox")
lab = ttk.Label(root, text="Shoebox Launcher")
lab.pack(side=TOP, expand=True, ipady=10)

def launch_px():
    Px()
pxButton = ttk.Button(root, text="Px", command=launch_px)
pxButton.pack(side=LEFT, fill=X, expand=True)

def launch_cx():
    Cx()
cxButton = ttk.Button(root, text="Cx", command=launch_cx)
cxButton.pack(side=LEFT, fill=X, expand=True)

def launch_medit():
    Medit()
meditButton = ttk.Button(root, text="Medit", command=launch_medit)
meditButton.pack(side=LEFT, fill=X, expand=True)

def launch_nailer():
    Nailer(".")
nailerButton = ttk.Button(root, text="Nailer", command=launch_nailer)
nailerButton.pack(side=LEFT, fill=X, expand=True)

def launch_sweeper():
    Sweeper(".")
sweeperButton = ttk.Button(root, text="Sweeper", command=launch_sweeper)
sweeperButton.pack(side=LEFT, fill=X, expand=True)

def close_all():
    for px in list(shoebox.px.instances):
        px.destroy()
    for cx in list(shoebox.cx.instances):
        cx.destroy()
    for medit in list(shoebox.medit.instances):
        medit.destroy()
    for nailer in list(shoebox.nailer.instances):
        nailer.destroy()
    for sweeper in list(shoebox.sweeper.instances):
        sweeper.destroy()
    for viewer in list(shoebox.viewer.instances):
        viewer.destroy()
closeAllButton = ttk.Button(root, text="Close All", command=close_all)
closeAllButton.pack(side=LEFT, fill=X, expand=True)

def on_exit():
    root.destroy()
root.protocol("WM_DELETE_WINDOW", on_exit)

root.mainloop()


import FreeCAD

def buildVerticalStabilizer():
    from PyOCCLevelUtils import RibMakerTranslatePlane
    import OCCUtils
    import Part
    from BSplineSketch import InteractiveBSplineCurve

    sectionSketch = FreeCAD.ActiveDocument.getObjectsByLabel('wingRoot008')[0]
    leadingEdge= FreeCAD.ActiveDocument.getObjectsByLabel('leadingEdge')[0]
    trailingEdge= FreeCAD.ActiveDocument.getObjectsByLabel('trailingEdge')[0]
    widthEdge= FreeCAD.ActiveDocument.getObjectsByLabel('widthEdge')[0]

    leadingEdge = OCCUtils.Topo(Part.__toPythonOCC__(leadingEdge.Shape)).edges().next()
    trailingEdge = OCCUtils.Topo(Part.__toPythonOCC__(trailingEdge.Shape)).edges().next()
    widthEdge = OCCUtils.Topo(Part.__toPythonOCC__(widthEdge.Shape)).edges().next()

    sectionICurve = InteractiveBSplineCurve(sectionSketch)
    pointsList = sectionICurve.getPolesAs_gp_Pnt()

    rm = RibMakerTranslatePlane(pointsList,leadingEdge,trailingEdge,widthEdge)

    #rm.uPeriodic = True
    #rm.UStartMultiplicity = 1
    #rm.UEndMultiplicity = 1

    return rm.TopoDS_Face()

def buildFoil(foilLabel,sectionSketchLabel,leadingEdgeSketchLabel,trailingEdgeSketchLabel,widthEdgeSketchLabel):

    from PyOCCLevelUtils import RibMakerTranslatePlane
    import OCCUtils
    import Part
    from BSplineSketch import InteractiveBSplineCurve

    sps = FreeCAD.ActiveDocument.getObjectsByLabel(sectionSketchLabel)[0]
    sectionPoints = InteractiveBSplineCurve(sps).getPolesAs_gp_Pnt()

    leadingEdgeSketch = FreeCAD.ActiveDocument.getObjectsByLabel(leadingEdgeSketchLabel)[0]
    trailingEdgeSketch = FreeCAD.ActiveDocument.getObjectsByLabel(trailingEdgeSketchLabel)[0]
    widthEdgeSketch = FreeCAD.ActiveDocument.getObjectsByLabel(widthEdgeSketchLabel)[0]

    leadingEdge = OCCUtils.Topo(Part.__toPythonOCC__(InteractiveBSplineCurve(leadingEdgeSketch).getShape())).edges().next()
    trailingEdge = OCCUtils.Topo(Part.__toPythonOCC__(InteractiveBSplineCurve(trailingEdgeSketch).getShape())).edges().next()
    widthEdge = OCCUtils.Topo(Part.__toPythonOCC__(InteractiveBSplineCurve(widthEdgeSketch).getShape())).edges().next()

    rm = RibMakerTranslatePlane(sectionPoints,leadingEdge,trailingEdge,widthEdge)
    shapeobj = FreeCAD.ActiveDocument.addObject("Part::Feature",foilLabel)
    shapeobj.Shape = Part.__fromPythonOCC__(rm.TopoDS_Face())
    FreeCAD.ActiveDocument.recompute()


import sys

import os
from BNComplexes import *
from CobComplexes import *
from Drawing import *
from CrossingTangle import *

# This is used solely to execute the files in the examples subdirectory

def main(data):
    print("----------------")
    print("KhT, version ???")
    print("----------------")
    BNComplexes.filename = filename
    BNComplexes.filepath = filepath
    Drawing.filename = filename
    Drawing.filepath = filepath
    exec(data)
    print("-------------------------")
    print("KhT executed successfully")
    print("-------------------------")

if __name__ == "__main__":
    filename=sys.argv[1]
    filepath="examples/"+filename+"/"
    # filename = name of example file called by kht
    # filepath = examples/<filename>/

    try:# assuming KhT calls file without including the '.py'-ending
        with open("examples/"+filename+".py", "r") as text_file:
            data = text_file.read()
    except: # assuming KhT calls file with the '.py'-ending or the file does not have an ending
        with open("examples/"+filename, "r") as text_file:
            data = text_file.read()
            filename=filename[:-3]

    paths=["","/PSTricks","/BNComplexes","/CobComplexes"]
    for path in paths:
        if not os.path.exists('examples/'+filename+path):
            os.makedirs('examples/'+filename+path)

    main(data)


"""
Aggregate feature subsets from multiple runs from feature generation
code in gen_mol_fea.py.

Example:
    python src/agg_runs.py --res_dir out/BL2/2020-4-24 --fea dd --prfx BL2
"""
import warnings
warnings.filterwarnings('ignore')

import os
import sys
from pathlib import Path
from time import time
import argparse
from glob import glob

import numpy as np
import pandas as pd

filepath = Path(__file__).resolve().parent

# Utils
from utils.utils import load_data

def parse_args(args):
    parser = argparse.ArgumentParser(description='Aggregate feature subsets from multiple runs.')
    parser.add_argument('--res_dir', required=True, default=None, type=str,
                        help='Global dir where multiple runs were dumped (default: None).')
    parser.add_argument('--fea', required=True, default='dd', type=str,
                        help='Feature type (default: dd).')
    parser.add_argument('--prfx', default=None, type=str,
                        help='Name prefix for the output file (default: None).')
    args = parser.parse_args(args)
    return args

def run(args):
    res_dir = Path( args['res_dir']).resolve()
    fea_name = args['fea']
    prfx = args['prfx']

    run_dirs = glob(str(res_dir/'*ids*'))
    fea_dfs = []
    for i, r in enumerate(run_dirs):
        dpath = sorted(Path(r).glob(f'*{fea_name}*'))[0]
        if not dpath.exists():
            continue
        df = load_data(dpath)

        if 'smiles' in df.columns:
            df = df.rename(columns={'smiles': 'SMILES'})
        if 'name' in df.columns:
            df = df.rename(columns={'name': 'TITLE'})

        fea_dfs.append(df)
    fea_df = pd.concat(fea_dfs, axis=0)
    del fea_dfs

    print(fea_df.shape)
    fea_df = fea_df.drop_duplicates(subset=['TITLE']).reset_index(drop=True)
    print(fea_df.shape)

    fea_df.to_parquet(f'{res_dir}/{prfx}.{fea_name}.parquet')
    print('Done.')

def main(args):
    args = parse_args(args)
    args = vars(args)
    run(args)

if __name__ == '__main__':
    main(sys.argv[1:])


#Author: Mayank Mandava

def make_symbol_table(forest):
    tbl = {
            'SP'    : 0,
            'LCL'   : 1,
            'ARG'   : 2,
            'THIS'  : 3,
            'THAT'  : 4,
            'SCREEN':16384,
            'KBD'   :24576
            }
    for i in range(16):
        #R0 to R15
        tbl['R%d'%i] = i
    line = 0  #line numbers
    free = 16 #current free memory location
    for tree in forest:
        if tree[0] == 'L':
            tbl[tree[1]] = line
        else:
            line += 1
    for tree in forest:
        if tree[0] == 'A' and tree[1][0] == 'SYM':
            if not tbl.has_key(tree[1][1]):
                tbl[tree[1][1]] = free
                free += 1
    return tbl


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@time    : 2020/3/30 10:30
@file    : convert_pictures.py
@author  : yitong.feng
@contact: yitong.feng@majorbio.com
"""

import fitz
from skimage import io
import cairosvg
import sys

def trim(img):
    img2 = img.sum(axis=2)
    img2_r = img2.sum(axis=1)
    img2_c = img2.sum(axis=0)
    (row,col) = img2.shape
    tempr0=0
    tempr1=0
    tempc0=0
    tempc1=0
    for r in range(0,row):
        if img2_r[r]!=765*col:
            tempr0=r
            break
    for r in range(row-1,0,-1):
        if img2_r[r]!=765*col:
            tempr1=r
            break
    for c in range(0,col):
        if img2_c[c]!=765*row:
            tempc0=c
            break
    for c in range(col-1,0,-1):
        if img2_c[c]!=765*row:
            tempc1=c
            break
    # print((tempr0, tempr1, tempc0, tempc1))
    new_img=img[tempr0:tempr1+3,tempc0:tempc1+3,0:3]
    return new_img

def convert(pic, *args, **kargs):
    if pic.endswith('.svg'):
        pdf = pic.split('.svg')[0] + '.pdf'
        cairosvg.svg2pdf(url=pic, write_to=pdf)
        pic = pdf
    pdfDoc = fitz.open(pic)
    zoom_x = 2
    zoom_y = 2
    mat = fitz.Matrix(zoom_x, zoom_y)
    pix = pdfDoc[0].getPixmap(matrix = mat)
    png = pic.split('.pdf')[0] + '.png'
    if 'out' in kargs:
        png = kargs['out']
    pix.writePNG(png)
    trim_ = True
    try:
        trim_ = kargs['trim']
    except:
        pass
    if trim_:
        im = io.imread(png)
        im = trim(im)
        io.imsave(png, im)

if __name__ == '__main__':
    if len(sys.argv) == 2:
        _, pic = sys.argv
        png = pic.split('.pdf')[0] + '.png'
        convert(pic, trim=False)
        convert(pic, out=png + '_trim.png')
    elif len(sys.argv) == 3:
        _, pic, out = sys.argv
        convert(pic, out=out, trim=False)
        convert(pic, out=out + '_trim.png', trim=True)
    else:
        exit("USAGE:python $s picture out")


import torch.nn as nn
from typing import Tuple, Optional, Iterable, List

def build_mlp(
    _in: int,
    hidden: Tuple[int, ...],
    out: Optional[int] = None,
    add_batchnorm: bool = False,
    activations: Iterable[nn.Module] = [nn.LeakyReLU()]
) -> List[nn.Module]:
    layers: List[nn.Module] = []
    h_prev = _in
    for h in hidden:
        layers.extend([
            nn.Linear(in_features=h_prev, out_features=h),
        ])
        layers.extend(activations)
        if add_batchnorm:
            layers.append(nn.BatchNorm1d(h))

        h_prev = h

    if out is not None:
        layers.append(nn.Linear(in_features=h_prev, out_features=out))

    return layers


import math
import traceback
from typing import List

import cv2
import d3dshot
import numpy as np
from cv2 import DMatch
from numpy import ndarray

class CustomErr(Exception):
    pass

d3dshot_mgr = d3dshot.create(capture_output='numpy')

color_board_img = cv2.imread('imgs/cb.png')
draw_board_img = cv2.imread('imgs/db.png')
pen_setting_img = cv2.imread('imgs/ps.png')

class Color:
    RED = (255, 0, 0)
    GREEN = (0, 255, 0)
    BLUE = (0, 0, 255)

def to_int(data):
    if isinstance(data, (list, tuple, set, ndarray)):
        return [int(item) for item in data]
    elif isinstance(data, (str, float, int)):
        return int(data)
    else:
        raise NotImplementedError(f'{type(data)} is not supported')

def distance(p1, p2):
    return math.sqrt(
        ((p1[0] - p2[0]) ** 2) + ((p1[1] - p2[1]) ** 2)
    )

def draw_rectangle(img, pt1, pt2, color=Color.GREEN, thickness=2, text=None):
    cv2.rectangle(img, pt1, pt2, color=color, thickness=thickness)
    if text:
        cv2.putText(img, text, pt1, fontFace=3, fontScale=1, color=color)

def find_pos(sub_img, src_img, text, tolerance=0.5, method='sift', grey_mode=False, preview=1):
    if grey_mode:
        src = cv2.cvtColor(src_img.copy(), cv2.COLOR_RGB2GRAY)
        sub = cv2.cvtColor(sub_img.copy(), cv2.COLOR_RGB2GRAY)
    else:
        src = src_img.copy()
        sub = sub_img.copy()

    src_h, src_w, *_ = src.shape
    sub_h, sub_w, *_ = sub.shape

    if method == 'legacy':
        # Legacy template matching
        result = cv2.matchTemplate(sub, src, cv2.TM_CCOEFF_NORMED)
        y, x = np.unravel_index(result.argmax(), result.shape)
        cv2.rectangle(src, (x, y), (x + sub_w, y + sub_h), color=(0, 255, 0), thickness=5)
        preview = src_img
        pt1 = (x, y)
        pt2 = (x + src_w, y + src_h)
    elif method == 'sift':
        # SIFT
        sift = cv2.SIFT_create()
        src_kp, src_des = sift.detectAndCompute(src, None)
        sub_kp, sub_des = sift.detectAndCompute(sub, None)
        bf = cv2.BFMatcher()
        matches = bf.knnMatch(sub_des, src_des, k=2)
        good: List[List[DMatch]] = []
        for m, n in matches:
            if m.distance < tolerance * n.distance:
                good.append([m])
        try:
            # Calculate offset and Draw final point
            if len(good) < 2:
                raise CustomErr(f'{text} feature point not enough ({len(good)})')

            x, y = to_int(src_kp[good[0][0].trainIdx].pt)
            offset_x, offset_y = to_int(sub_kp[good[0][0].queryIdx].pt)
            factor = np.divide(
                distance(to_int(src_kp[good[0][0].trainIdx].pt), to_int(src_kp[good[1][0].trainIdx].pt)),
                distance(to_int(sub_kp[good[0][0].queryIdx].pt), to_int(sub_kp[good[1][0].queryIdx].pt))
            )
            if np.isnan(x) or np.isnan(y):
                raise CustomErr(f'{text}:Invalid raw position {x, y}')
            if np.isnan(offset_x) or np.isnan(offset_y):
                raise CustomErr(f'{text}:Invalid offset {offset_x, offset_y}')
            if np.isnan(factor):
                raise CustomErr(f'{text}:Invalid factor {factor}')

            pt1 = (x - offset_x * factor), (y - offset_y * factor)
            pt2 = np.add(pt1, [sub_w * factor, sub_h * factor])
            draw_rectangle(src_img, to_int(pt1), to_int(pt2), thickness=5, text=text)

            # Draw feature point
            if preview > 1:
                preview = cv2.drawMatchesKnn(sub_img, sub_kp, src_img, src_kp, good[:2], None, flags=2)
            else:
                preview = src_img
        except CustomErr as e:
            print(e)
            preview = src_img
            pt1 = pt2 = 0
        except Exception:
            print(traceback.format_exc())
            preview = src_img
            pt1 = pt2 = 0
    else:
        raise NotImplementedError

    # Show preview
    show(preview)
    return pt1, pt2

def show(img, width=1280, height=720, title='Image'):
    cv2.imshow(title, cv2.resize(img, (width, height)))

def screenshot():
    screen_shot = d3dshot_mgr.screenshot()
    screen_shot = cv2.cvtColor(screen_shot, cv2.COLOR_BGR2RGB)
    return screen_shot

if __name__ == '__main__':
    while True:
        screen = screenshot()
        find_pos(color_board_img, screen, text='Color', preview=2)
        # find_pos(draw_board_img, screen, tolerance=0.9, text='Draw', grey_mode=True)
        # find_pos(pen_setting_img, screen, tolerance=0.9, text='Pen', grey_mode=True)
        cv2.waitKey(1)


#!/usr/bin/env python
# coding: utf-8

# In[12]:

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

test= pd.read_csv("C:\\Users\\akshansh\\Desktop\\python programs\\test.csv")
print(test)

# In[15]:

test=sns.load_dataset("titanic")
test.head()

# In[16]:

sns.lineplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[17]:

sns.lineplot(x="age",y="fare",data=test,hue="sex",style="sex")
plt.show()

# In[28]:

sns.lineplot(x="age",y="fare",data=test,hue="sex",marker='o')
plt.show()

# In[30]:

sns.barplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[31]:

sns.barplot(x="age",y="fare",data=test,hue="sex",palette="rocket")
plt.show()

# In[54]:

plt.figure(figsize=(10,6))
sns.barplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[65]:

plt.figure(figsize=(10,6))
sns.histplot(x="age",y="fare",data=test,hue="sex",bins=10)
plt.show()

# In[66]:

plt.figure(figsize=(10,6))
sns.scatterplot(x="age",y="fare",data=test,hue="sex")
plt.show()

# In[ ]:


# Restaurant Rater
# Author: Ryan Berns
# Date Created: 2017-08-14

from urllib.request import urlopen
from bs4 import BeautifulSoup
import urllib.request
import re
import requests
import pprint
import json
import csv
from twilio.rest import Client

# Information for the yelp API
accountSID = '*******************************'
authToken = '*******************************'
client = Client(accountSID, authToken)

# Phone number used to text users
myTwilioNumber = '+99999999'

# Use a browser emulator so google allows scrapes
class AppURLopener(urllib.request.FancyURLopener):
    version = "Mozilla/5.0"
appId ='**************'
appSecret ='************'
data = {'grant_type': 'client_credentials',
        'client_id': appId,
        'client_secret': appSecret}
token = requests.post('https://api.yelp.com/oauth2/token', data=data)
accessToken = token.json()['access_token']
url = 'https://api.yelp.com/v3/businesses/search'
headers = {'Authorization': 'bearer %s' % accessToken}

# Start app and ask for input information from user
print("Welcome to my restaurant rating app!\n")
userName = input("Please enter your name. (First name and first initial of last name):\t")

# App persists user information in a csv.  When a user logs in again, those same preferences are found and do not need to be entered again.
try:
    reader = csv.reader(open(r'C:\localData\userData.csv'))
    userDirectory={}
    for row in reader:
        if len(row) > 0:
            userDirectory[row[0]] = row[1:]
except FileNotFoundError:
    userDirectory={}

if userName in userDirectory:
    print("Welcome back",userName)
    myCellPhone = userDirectory[userName][0]
    googlePref = int(userDirectory[userName][1])
    yelpPref = int(userDirectory[userName][2])
else:
    phoneNumberMessage = str("Hello, "+userName+". Can you please input your mobile number?  Just numbers! (no dashes please)\t")
    myCellPhone = input(phoneNumberMessage)
    googlePref = input("On a scale of 1 - 3 (3 being the best), how much do you value google reviews?\t")

    # Handle errors if user enters invalid scaling for google
    try:
        googlePref = int(googlePref)
    except ValueError:
        googlePref = input("Please enter as a number. (1,2 or 3). Try again:\t")
    if int(googlePref) > 3:
        googlePref = input("You can only choose 1,2 or 3.  how much do you value google reviews?\t")
    elif int(googlePref) < 1:
        googlePref = input("You can only choose 1,2 or 3.  how much do you value google reviews?\t")

    # Add yelp review value
    yelpPref = input("On a scale of 1 - 3 (3 being the best), how much do you value yelp reviews?\t")

    # Handle errors if user enters invalid scaling for yelp
    try:
        yelpPref = int(yelpPref)
    except ValueError:
        yelpPref = input("Please enter as a number. (1,2 or 3). Try again:\t")
    if int(yelpPref) > 3:
        yelpPref = input("You can only choose 1,2 or 3.  how much do you value yelp reviews?\t")
    elif int(yelpPref) < 1:
        yelpPref = input("You can only choose 1,2 or 3.  how much do you value yelp reviews?\t")

    userValues = str(myCellPhone+" "+str(googlePref)+" "+str(yelpPref)+" ").split()
    userDirectory[userName] = userValues
    with open(r'C:\localData\userData.csv', 'w') as outfile:
        csv_writer = csv.writer(outfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        for k,v in userDirectory.items():
            csv_writer.writerow([k] + v)
# Extract information from Google and Yelp while ensuring we get a rating back
restart="Y"
while restart=="Y":
    restart="N"
    nm = input("What is the name of the restaurant?\t")
    ask = str("What is the location of "+nm+"?\t")
    loct = input(ask)
    print("")
    nmUp=nm.replace(" ","+")
    loctUp = loct.replace(" ","+")
### Google ###
    query = ("https://www.google.com/search?safe=off&q="+nmUp+"+"+loctUp+"+google+review")
    opener = AppURLopener()

    response = opener.open(query)
    soup = BeautifulSoup(response,'html.parser')

    # Rating
    output = soup.find('span', {'class':'_kgd'})

    if output==None:
        print("We can\'t find that restaurant.  Please try again.")
        restart="Y"
    else:
        restart="N"
    # Number of Reviews
    output2 = soup.find('span', {'style': 'color:#777'})
    # Error handling in case the app cannot find the restaurant on google reviews
    try:
        output2Tmp = ''.join(re.findall(r'\d',str(output2.get_text())))
    except AttributeError:
        print('Sorry. We can\'t find that restaurant on google. Please check to make sure you inputted the restaurant correctly.\n')

### YELP ###
    params = {'location': loct,
              'term': nm,
              'limit': '1'
             }

    resp = requests.get(url=url, params=params, headers=headers)

    outputY = resp.json()
    subsetList = outputY['businesses']
    # Error handling in case the app cannot find the restaurant on yelp
    try:
        subsetDict = dict(subsetList[0])
    except IndexError:
        print('Sorry. We can\'t find that restaurant on yelp. Please check to make sure you inputted the restaurant correctly.\n')
        restart="Y"

def yourRating(yelpPref=yelpPref,googlePref=googlePref):
    ratio = float(googlePref/yelpPref)
    googleNum = (float(output.get_text())*float(output2Tmp)*ratio)
    yelpNum = (float(subsetDict['rating'])*float(subsetDict['review_count']))
    den = ((float(output2Tmp)*ratio)+float(subsetDict['review_count']))
    yourRtng = round(((googleNum+yelpNum)/den),2)
    return yourRtng

yourRtng = yourRating()
# Print Output to screen
print("Based on your preferences, our recommended rating for", subsetDict['name'],"is",yourRtng,".\n\n")
print("########## Google ##########")
print("Rating:\t",output.get_text())
print("Total reviews:\t",output2Tmp)
print("")
print("######### Yelp ##########")
print("Rating:\t",subsetDict['rating'])
print("Total reviews:\t",subsetDict['review_count'])
print("")
print("Address:")
for i in subsetDict['location']['display_address']:
    print(i)
print(subsetDict['display_phone'])
if subsetDict['is_closed']==True:
    print("Unfortunately",subsetDict['name'],"closed and no longer exists.")

# Build Text Output for text message
restaurant_nm = subsetDict['name']
yourRtngFormatted = str(yourRtng)
googleRtng = output.get_text()
yelpRtng = str(subsetDict['rating'])
yelpCnt = str(subsetDict['review_count'])
restaurantPhone = subsetDict['display_phone']
address = ""
for i in subsetDict['location']['display_address']:
    address += str(i+"\n")
txtOutput = str(restaurant_nm+"\nRecommended Rating:\t"+yourRtngFormatted+"\nGoogle\n"+"Rating:\t"+googleRtng+"\nTotal reviews:\t"+output2Tmp+"\n\nYelp\n"+"Rating:\t"+yelpRtng+"\nTotal reviews:\t"+yelpCnt+"\n\n"+address+"\n"+restaurantPhone)

# Send Text Message
try:
    message = client.messages.create(body=txtOutput, from_=myTwilioNumber, to=myCellPhone)
except:
    print("If the app creator wasn\'t so cheap, you would\'ve been texted this information to your mobile phone as well.\nHope you enjoy your meal!")


# -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import train_test_split
from bert import Bert
from data_loader import full_generator
from model import get_model
import json

# prepare y
# open data
dataframe = pd.read_csv("input/occupation_seq.csv", encoding = "cp1257").fillna(0)
LPKdict=pd.read_excel("input/ISCO_relationship.xlsx").fillna(0)#, encoding = "cp1257"

LPKdict.lpk4=LPKdict.lpk4.astype(int)

# create hierarchical structure

lpkindexmap = {}

for i in range(1,5):
    # to split 4 digit code into additional 1, 2, 3 digits codes
    dataframe[f'LPK_{i}_kodas'] = dataframe['s_veik_prof'].astype(str).str[0:i].astype(int)
    # add names to professions
    dataframe = dataframe.join(LPKdict.set_index('lpk4'), on=f'LPK_{i}_kodas')
    dataframe.rename(columns={'lpk_pav': f'LPK_{i}_pav'}, inplace=True)

    # dictionary for prediction
    lpkindexmap[i] = sorted(dataframe.set_index(f'LPK_{i}_kodas').to_dict()[f'LPK_{i}_pav'].items())

dataframe=dataframe.drop(['s_veik_prof'], axis=1)

with open('lpkindexmap.json', 'w') as f:
    json.dump(lpkindexmap, f)

#make sparse matrix as we need to save memory
mlb = MultiLabelBinarizer(sparse_output=True)

y = pd.DataFrame()
for i in range(1,5):
    d = (dataframe[['id', f'LPK_{i}_kodas']]
        .drop_duplicates()
        .groupby('id')[f'LPK_{i}_kodas']
        .apply(lambda g: g.values.tolist())
        .to_dict())

    df = pd.DataFrame(d.items(), columns=['id', f'lpk{i}'])
    d = pd.DataFrame(mlb.fit_transform(df[f'lpk{i}']), columns=[f'sparselpk{i}'])
    if y.empty:
        y = df
    y = y.join(pd.DataFrame(d, index=df.index))

# prepare x

#open file
edu = pd.read_csv("input/education_seq.csv", encoding = "cp1257").fillna("")
abstract=pd.read_csv("input/descriptions.csv", encoding = "cp1257").fillna("NA")

# make id for program
edu = edu.assign(ind=(edu['kodas']).astype('category').cat.codes)
edu=pd.merge(edu, abstract, on='kodas',  how='left').fillna("NA")

# prepare sparse dataset
edu_sparse=edu.groupby('id')['ind'].apply(lambda g: g.values.tolist()).to_dict()
edu_sparse=pd.DataFrame(edu_sparse.items(), columns=['id', 'programos_kodai'])

mlb = MultiLabelBinarizer(sparse_output=True)
mlb_transform=pd.DataFrame(mlb.fit_transform(edu_sparse['programos_kodai']), columns=['sparse_programoskodai'])

x_sp=edu_sparse.join(pd.DataFrame(mlb_transform, index=edu_sparse.index))

# prepare text dataset
x_txt=edu.groupby(['id'])['trans_abstract'].apply(','.join).reset_index()
dataframe=pd.merge(x_txt, x_sp, on='id',  how='left')

# join x and y variables for final dataset and drop cases where a person
# has education but has no occupation
dataframe=pd.merge(dataframe, y, on='id',  how='left')

dataframe=dataframe.dropna()
print(dataframe.iloc[1])

#make BERT from text input

bert = Bert()

translated_abstracts_df = dataframe[['id', 'trans_abstract']]

embeddings_df = pd.DataFrame()

for embedding in bert.make_embeddings(translated_abstracts_df[['trans_abstract']]):
    embeddings_df.append(pd.DataFrame(embedding), ignore_index=True)

result = pd.concat([translated_abstracts_df, embeddings_df], axis=1, sort=False)

#final data for training
X=result.values[:,2:]
Y = dataframe.values[:,5:9]

X_train, X_test, y_train, y_test = train_test_split(X, Y, random_state=42, test_size=0.2)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, random_state=42, test_size=0.2)
print(X_train.shape)
print(y_train.shape)

batch_size = 128
number_of_batches = int(np.ceil(X_train.shape[0]/batch_size))
val_batch_size = int(np.ceil(X_val.shape[0]/number_of_batches))
number_of_batches

#training
model = get_model(X_train, y_train)

history = model.fit_generator(
    generator=full_generator(X_train, y_train[:,1], y_train[:,2], y_train[:,3], y_train[:,0], batch_size, number_of_batches, True),
    steps_per_epoch=number_of_batches,
    validation_data=full_generator(X_val, y_val[:,1], y_val[:,2], y_val[:,3], y_val[:,0], val_batch_size, number_of_batches, False),
    validation_steps=number_of_batches,
    epochs=30, verbose=1)

gen_data=full_generator(X_test, y_test[:,1], y_test[:,2], y_test[:,3], y_test[:,0],val_batch_size, number_of_batches, False, infinite=False)
res = model.evaluate_generator(gen_data)
print(res)


import datetime
from datetime import *
from models import Scale, Observation

def age_check(agent_behavior, observation, scale):
    """
    :param agent_behavior:  Metrics to be used the agent.
    :type agent_behavior: dict
    :param observation: Content and metadata of message received and on which the trust is calculated.
    :type observation: Observation
    :param scale: The trust scale used by the agent
    :type scale: Scale
    :return: An age punishment value that is equal to the scale maximum value for recent publications and falls within
    [default, max) if it exceeded the allowed lifetime.
    :rtype: float or int
    """

    now = datetime.utcnow().timestamp()
    age = observation.details['content_trust.publication_date'] + \
        agent_behavior['content_trust.max_lifetime_seconds']

    if now < age:  # within allowed lifetime
        return scale.maximum_value()
    else:  # exceeded lifetime
        if 'content_trust.age_grace_period_seconds' in agent_behavior:
            grace_value = (
                now - age) / agent_behavior['content_trust.age_grace_period_seconds']
            if grace_value < 1.0:  # within grace period
                return (1-grace_value) * scale.maximum_value()

        return scale.default_value()  # no grace period or grace period exceeded


#!/usr/bin/env python3

import pandas as pd
import os
import glob
import shutil
# import multiprocessing
# from subprocess import call
import logging
import sys
import json
import scipy.cluster.hierarchy
import scipy.spatial.distance as ssd
import numpy as np
import pickle
import time
import glob

import drep
import drep.d_filter
import drep.d_bonus

# This is to make pandas shut up with it's warnings
pd.options.mode.chained_assignment = None

"""
Bdb = pandas DataFrame with the columns genome and location
Mdb = pandas containing raw MASH information
Ndb = pandas containing raw ANIn information
Cdb = pandas containing clustering information (both MASH and ANIn)
"""

""" Program architecture:

*   The main method is cluster_genomes(). This method will take all necessary arguments
    for a normal clustering pipeline, and return all normal outputs

*   The method called by the main program is d_cluster_wrapper(). This method will take
    the raw arguments and workDirectory, parse and extract the needed info, and then call
    cluster_genomes(). It will then take the output information and save it to the
    WorkDirectory object.
"""

def d_cluster_wrapper(workDirectory, **kwargs):

    # Load the WorkDirectory.
    logging.debug("Loading work directory")
    workDirectory = drep.WorkDirectory.WorkDirectory(workDirectory)
    logging.debug(str(workDirectory))

    # Parse arguments
    Bdb, data_folder, kwargs = parse_arguments(workDirectory, **kwargs)
    if kwargs['n_PRESET'] != None:
        kwargs['n_c'], kwargs['n_maxgap'], kwargs['n_noextend'], kwargs['n_method'] \
        = nucmer_preset(kwargs['n_PRESET'])

    # Run the main program
    Cdb, Mdb, Ndb = cluster_genomes(Bdb, data_folder, wd=workDirectory, **kwargs)

    # Save the output
    data_dir = workDirectory.location + '/data_tables/'
    if not os.path.exists(data_dir):
        os.makedirs(data_dir)

    logging.debug("Main program run complete- saving output to {0}".format(data_dir))

    Cdb.to_csv(os.path.join(data_dir,'Cdb.csv'),index=False)
    Mdb.to_csv(os.path.join(data_dir,'Mdb.csv'),index=False)
    Ndb.to_csv(os.path.join(data_dir,'Ndb.csv'),index=False)
    Bdb.to_csv(os.path.join(data_dir,'Bdb.csv'),index=False)

    # Log arguments
    cluster_log = workDirectory.location + '/log/cluster_arguments.json'
    with open(cluster_log, 'w') as fp:
        json.dump(kwargs, fp)
    fp.close()

def cluster_genomes(Bdb, data_folder, **kwargs):

    """
    Takes a number of command line arguments and returns a couple pandas dataframes

    Required Input:
    * Bdb           - pandas dataframe with the columns "genome" and "location"
    * data_folder   - location where MASH and ANIn data will be stored

    Optional Input:

    ***** Clustering Arguments *****
    * skipMash      - If true, skip MASH altogether (all_vs_all for ANIn)
    * P_Lmethod      - MASH method of determining linkage
                    (to be passed to scipy.cluster.hierarchy.linkage)
    * P_Lcutoff      - MASH linkage cutoff
    * MASH_ANI      - ANI threshold for simple MASH clustering

    * SkipSecondary - If true, skip secondary clustering altogether
    * ANIn_cov      - Coverage threshold for secondary clustering
    * S_Lmethod     - ANIn method of determining linkage
                    (to be passed to scipy.cluster.hierarchy.linkage)
    * S_Lcutoff     - ANIn linkage cutoff

    ***** Comparison Algorithm Arguments *****
    * MASH_s        - MASH sketch size

    * n_c           - nucmer argument c
    * n_maxgap      - nucmer argument maxgap
    * n_noextend    - nucmer argument noextend
    * n_method      - nucmer argument method

    * n_preset      - preset nucmer arrangements: {tight,normal}

    ***** Other Arguments *****
    * dry           - don't actually do anything, just print commands
    * threads       - number of threads to use for multithreading steps
    * overwrite     - overwrite existing data

    Returned Output:
    * Mdb = MASH comparison specifics
    * Ndb = ANIn comparison specifics
    * Cdb = Clustering information
    """

    logging.info("Step 1. Parse Arguments")

    # Deal with nucmer presets
    if kwargs.get('n_preset', None) != None:
        kwargs['n_c'], kwargs['n_maxgap'], kwargs['n_noextend'], kwargs['n_method'] \
        = nucmer_preset(kwargs['n_PRESET'])

    logging.debug("kwargs: {0}".format(kwargs))

    logging.info("Step 2. Perform MASH (primary) clustering")
    if not kwargs.get('SkipMash', False):

        logging.info("2a. Run pair-wise MASH clustering")
        Mdb = all_vs_all_MASH(Bdb, data_folder, **kwargs)

        logging.info("2b. Cluster pair-wise MASH clustering")
        Cdb = cluster_mash_database(Mdb, data_folder= data_folder, **kwargs)

    else:
        logging.info("2. Nevermind! Skipping Mash")
        Cdb = gen_nomash_cdb(Bdb)
        Mdb = pd.DataFrame({'Blank':[]})

    logging.info("{0} primary clusters made".format(len(Cdb['MASH_cluster'].unique())))

    logging.info("Step 3. Perform secondary clustering")

    Ndb, Cdb = run_secondary_clustering(Bdb, Cdb, data_folder, Mdb = Mdb, \
                **kwargs)

    logging.info(
    "Step 4. Return output")

    return Cdb, Mdb, Ndb

def run_secondary_clustering(Bdb, Cdb, data_folder, **kwargs):
    SkipSecondary = kwargs.get('SkipSecondary', False)
    algorithm = kwargs.get('S_algorithm', 'ANIn')

    # This should only happen when called from the wrapper, and Mdb need to be provided
    if SkipSecondary:
        Mdb = kwargs.pop('Mdb')

        Cdb = gen_nomani_cdb(Cdb, Mdb, data_folder = data_folder, **kwargs)
        Ndb = pd.DataFrame({'Blank':[]})

        return Ndb, Cdb

    # Estimate time
    comps = 0
    for bdb, name in iteratre_clusters(Bdb,Cdb):
        g = len(bdb['genome'].unique())
        comps += (g * g)
    time = estimate_time(comps, algorithm)
    time = time / int(kwargs.get('processors'))
    logging.info("Running {0} {1} comparisons- should take ~ {2:.1f} min".format(\
            comps, algorithm, time))

    # Run comparisons
    Ndb = pd.DataFrame()
    for bdb, name in iteratre_clusters(Bdb,Cdb):
        ndb = compare_genomes(bdb, algorithm, data_folder, **kwargs)
        ndb['MASH_cluster'] = name
        Ndb = pd.concat([Ndb,ndb], ignore_index= True)

    # Clear out clustering folder
    c_folder = data_folder + 'Clustering_files/'
    if not os.path.exists(data_folder):
        os.makedirs(data_folder)
    logging.debug('clobbering {0}'.format(data_folder))
    if kwargs.get('overwrite',False):
        for fn in glob.glob(data_folder + 'secondary_linkage_cluster*'):
            os.remove(fn)

    # Run clustering
    Cdb = pd.DataFrame()
    for ndb, name in iteratre_clusters(Ndb,Ndb):
        cdb = genome_hierarchical_clustering(ndb, c_folder, algorithm,\
                cluster=name, **kwargs)
        cdb['primary_cluster'] = name
        Cdb = pd.concat([Cdb,cdb], ignore_index=True)

    return Ndb, Cdb

'''
Description
'''
def genome_hierarchical_clustering(Ndb, data_folder, comp_method, **kwargs):
    logging.debug('Clustering ANIn database')
    S_Lmethod = kwargs.get('clusterAlg', 'single')
    S_Lcutoff = 1 - kwargs.get('S_ani', .99)
    cov_thresh = float(kwargs.get('cov_thresh',0.5))

    cluster = kwargs.get('cluster','')

    Table = {'genome':[],'secondary_cluster':[]}

    # Handle the case where there's only one genome
    if len(Ndb['reference'].unique()) == 1:
        Table['genome'].append(os.path.basename(Ndb['reference'].unique().tolist()[0]))
        Table['secondary_cluster'].append("{0}_0".format(cluster))

    else:
        # 2) Make a linkage-db in algorithm-specific manner
        Ldb = make_linkage_Ndb(Ndb,comp_method,**kwargs)

        # 3) Cluster the linkagedb
        Gdb, linkage = cluster_hierarchical(Ldb, linkage_method= S_Lmethod, \
                                    linkage_cutoff= S_Lcutoff)

        # 4) Extract secondary clusters
        for clust in Gdb['cluster'].unique():
            d = Gdb[Gdb['cluster'] == clust]
            for genome in d['genome'].tolist():
                Table['genome'].append(genome)
                Table['secondary_cluster'].append("{0}_{1}".format(cluster,clust))

        # 5) Save the linkage
        arguments = {'linkage_method':S_Lmethod,'linkage_cutoff':S_Lcutoff,\
                    'comparison_algorithm':comp_method,'minimum_coverage':cov_thresh}
        pickle_name = "secondary_linkage_cluster_{0}.pickle".format(cluster)
        logging.debug('Saving secondary_linkage pickle {1} to {0}'.format(data_folder,\
                                                            pickle_name))
        with open(data_folder + pickle_name, 'wb') as handle:
            pickle.dump(linkage, handle)
            pickle.dump(Ldb,handle)
            pickle.dump(arguments,handle)

    # Return the database
    Gdb = pd.DataFrame(Table)
    Gdb['threshold'] = S_Lcutoff
    Gdb['cluster_method'] = S_Lmethod
    Gdb['comparison_algorithm'] = comp_method

    return Gdb

'''
Filter the Ndb folder in accordance with the kwargs, depending on the algorithm
'''
def make_linkage_Ndb(Ndb,algorithm,**kwargs):
    d = Ndb.copy()
    cov_thresh = float(kwargs.get('cov_thresh',0.5))

    # Remove values without enough coverage
    d.loc[d['alignment_coverage'] <= cov_thresh, 'ani'] = 0

    # Make a linkagedb by averaging values and setting self-compare to 1
    #d['av_ani'] = d.apply(lambda row: average_ani (row,d),axis=1)
    add_avani(d)
    d['dist'] = 1 - d['av_ani']
    db = d.pivot("reference", "querry", "dist")

    return db

def iteratre_clusters(Bdb, Cdb, id='MASH_cluster'):
    Bdb = pd.merge(Bdb,Cdb)
    for cluster in Bdb[id].unique():
        d = Bdb[Bdb[id] == cluster]
        yield d, cluster

def estimate_time(comps, alg):
    if alg == 'ANIn':
        time = comps * .33
    elif alg == 'gANI':
        time = comps * .1
    elif alg == 'ANImf':
        time = comps * .5
    return time

def parse_arguments(workDirectory, **kwargs):

    # Make sure you have the required program installed
    loc = shutil.which('mash')
    if loc == None:
        logging.error('Cannot locate the program {0}- make sure its in the system path'\
            .format('mash'))
    kwargs['mash_exe'] = loc

    # If genomes are provided, load them
    if kwargs.get('genomes',None) != None:
        assert workDirectory.hasDb("Bdb") == False, \
        "Don't provide new genomes- you already have them in the work directory"
        Bdb = load_genomes(kwargs['genomes'])
    # If genomes are not provided, don't load them
    if kwargs.get('genomes',None) == None:
        assert workDirectory.hasDb("Bdb") != False, \
        "Must either provide a genome list, or run the 'filter' operation with the same work directory"
        Bdb = workDirectory.data_tables['Bdb']

    # Make sure this isn't going to overwrite old data
    overwrite = kwargs.get('overwrite',False)
    for db in ['Cdb','Mdb','Ndb']:
        if workDirectory.hasDb(db):
            if not overwrite:
                logging.error("clustering already exists; run with --overwrite to continue")
                sys.exit()
            logging.debug("THIS WILL OVERWRITE {0}".format(db))

    # Make data_folder
    data_folder = os.path.join(workDirectory.location, 'data/')

    return Bdb, data_folder, kwargs

def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10):

    # Save names
    names = list(db.columns)

    # Generate linkage dataframe
    arr =  np.asarray(db)
    try:
        arr = ssd.squareform(arr)
    except:
        logging.error("The database passed in is not symmetrical!")
        logging.error(arr)
        sys.exit()
    linkage = scipy.cluster.hierarchy.linkage(arr, method= linkage_method)

    # Form clusters
    fclust = scipy.cluster.hierarchy.fcluster(linkage,linkage_cutoff, \
                                                                    criterion='distance')

    # Make Cdb
    Cdb = gen_cdb_from_fclust(fclust,names)

    return Cdb, linkage

def gen_cdb_from_fclust(fclust,names):

    Table={'cluster':[],'genome':[]}
    for i, c in enumerate(fclust):
        Table['cluster'].append(c)
        Table['genome'].append(names[i])

    return pd.DataFrame(Table)

def cluster_anin_database(Cdb, Ndb, data_folder = False, **kwargs):

    logging.debug('Clustering ANIn database')

    cov_thresh = float(kwargs.get('cov_thresh',0.5))
    S_Lmethod = kwargs.get('clusterAlg', 'single')
    S_Lcutoff = 1 - kwargs.get('S_ani', .99)
    overwrite = kwargs.get('overwrite', False)

    if (data_folder != False):
        data_folder = data_folder + 'Clustering_files/'
        if not os.path.exists(data_folder):
            os.makedirs(data_folder)

        # Delete all existing pickles
        logging.debug('clobbering {0}'.format(data_folder))
        if kwargs.get('overwrite',False):
            for fn in glob.glob(data_folder + 'secondary_linkage_cluster*'):
                os.remove(fn)

    Table = {'genome':[],'ANIn_cluster':[]}

    # For every MASH cluster-
    for cluster in Cdb['MASH_cluster'].unique():
        # Filter the database to this cluster
        d = Ndb[Ndb['reference'].isin(Cdb['genome'][Cdb['MASH_cluster'] == cluster].tolist())]

        # Remove values without enough coverage
        d.loc[d['alignment_coverage'] <= cov_thresh, 'ani'] = 0

        # Handle case where cluster has one member
        if len(d['reference'].unique()) == 1:
            Table['genome'].append(d['reference'].unique().tolist()[0])
            Table['ANIn_cluster'].append("{0}_0".format(cluster))
            continue

        # Make a linkagedb
        #d['av_ani'] = d.apply(lambda row: average_ani (row,d),axis=1)
        add_avani(d)
        d['dist'] = 1 - d['av_ani']
        db = d.pivot("reference", "querry", "dist")

        Gdb, linkage = cluster_hierarchical(db, linkage_method= S_Lmethod, \
                                    linkage_cutoff= S_Lcutoff)

        # For every ANIn cluster
        for clust in Gdb['cluster'].unique():

            # Filter the database to this cluster
            d = Gdb[Gdb['cluster'] == clust]

            # For every genome in this cluster
            for genome in d['genome'].tolist():

                # Save cluster information
                Table['genome'].append(genome)
                Table['ANIn_cluster'].append("{0}_{1}".format(cluster,clust))

        # Save the linkage
        if (data_folder != False):
            arguments = {'linkage_method':S_Lmethod,'linkage_cutoff':S_Lcutoff,\
                        'comparison_algorithm':'ANIn','minimum_coverage':cov_thresh}
            pickle_name = "secondary_linkage_cluster_{0}.pickle".format(cluster)
            logging.debug('Saving secondary_linkage pickle {1} to {0}'.format(data_folder,\
                                                                pickle_name))
            with open(data_folder + pickle_name, 'wb') as handle:
                pickle.dump(linkage, handle)
                pickle.dump(db,handle)
                pickle.dump(arguments,handle)

    Gdb = pd.DataFrame(Table)
    Cdb = pd.merge(Gdb, Cdb)
    Cdb['threshold'] = S_Lcutoff
    Cdb['cluster_method'] = S_Lmethod
    Cdb['comparison_algorithm'] = 'ANIn'
    Cdb = Cdb.rename(columns={'MASH_cluster':'primary_cluster',\
                        'ANIn_cluster':'secondary_cluster'})

    return Cdb

def cluster_anin_simple(Cdb, Ndb, ANIn=.99, cov_thresh=0.5):

    Table = {'genome':[],'ANIn_cluster':[]}

    # For every MASH cluster-
    for cluster in Cdb['MASH_cluster'].unique():
        # Filter the database to this cluster
        d = Ndb[Ndb['reference'].isin(Cdb['genome'][Cdb['MASH_cluster'] == cluster].tolist())]

        # Make a graph of the genomes in this cluster based on Ndb
        g = make_graph_anin(d,cov_thresh=cov_thresh,anin_thresh=ANIn)
        df = cluster_graph(g)

        # For every ANIn cluster in this graph -
        for clust in df['cluster'].unique():
            # Filter the database to this cluster
            d = df[df['cluster'] == clust]

            # For every genome in this cluster-
            for genome in d['genome'].tolist():

                # Save the cluster information
                Table['genome'].append(genome)
                Table['ANIn_cluster'].append("{0}_{1}".format(cluster,clust))

    Gdb = pd.DataFrame(Table)

    return pd.merge(Gdb,Cdb)

def run_anin_on_clusters(Bdb, Cdb, data_folder, **kwargs):

    """
    For each cluster in Cdb, run pairwise ANIn
    """

    n_c = kwargs.get('n_c', 65)
    n_maxgap = kwargs.get('n_maxgap', 90)
    n_noextend = kwargs.get('n_noextend', False)
    n_method = kwargs.get('method', 'mum')
    p = kwargs.get('processors', 6)
    dry = kwargs.get('dry',False)
    overwrite = kwargs.get('overwrite', False)

    # Set up folders
    ANIn_folder = data_folder + 'ANIn_files/'
    if not os.path.exists(ANIn_folder):
        os.makedirs(ANIn_folder)

    # Add cluster information to Cdb
    Bdb = pd.merge(Bdb,Cdb)
    logging.info("{0} MASH clusters were made".format(len(Bdb['MASH_cluster']\
                .unique().tolist())))

    # Step 1. Make the directories and generate the list of commands to be run
    cmds = []
    files = []
    for cluster in Bdb['MASH_cluster'].unique():
        d = Bdb[Bdb['MASH_cluster'] == cluster]
        genomes = d['location'].tolist()
        for g1 in genomes:
            for g2 in genomes:
                file_name = "{0}{1}_vs_{2}".format(ANIn_folder, \
                            get_genome_name_from_fasta(g1),\
                            get_genome_name_from_fasta(g2))
                files.append(file_name + '.delta')

                # If the file doesn't already exist, add it to what needs to be run
                if not os.path.isfile(file_name + '.delta'):
                    cmds.append(gen_nucmer_cmd(file_name,g1,g2,c=n_c,noextend=n_noextend,\
                                maxgap=n_maxgap,method=n_method))

    # Step 2. Run the nucmer commands

    if not dry:
        if len(cmds) > 0:
            thread_nucmer_cmds_status(cmds,p)

    # Step 3. Parse the nucmer output

    org_lengths = {y:dm.fasta_length(x) for x,y in zip(Bdb['location'].tolist(),Bdb['genome'].tolist())}
    Ndb = process_deltadir(files, org_lengths)
    Ndb['MASH_cluster'] = None
    for cluster in Bdb['MASH_cluster'].unique():
        d = Bdb[Bdb['MASH_cluster'] == cluster]
        Ndb['MASH_cluster'][Ndb['reference'].isin(d['genome'].tolist())] = cluster

    return Ndb

def gen_nucmer_commands(genomes,outf,c=65,maxgap=90,noextend=False,method='mum'):
    cmds = []
    for g1 in genomes:
        for g2 in genomes:
            out = "{0}{1}_vs_{2}".format(outf,get_genome_name_from_fasta(g1),get_genome_name_from_fasta(g2))
            cmds.append(gen_nucmer_cmd(out,g1,g2,c=c,noextend=noextend,maxgap=maxgap,method=method))

    return cmds

def run_nucmer_genomeList(genomes,outf,b2s,c=65,maxgap=90,noextend=False,method='mum',dry=False):
    """
    genomes is a list of locations of genomes in .fasta file. This will do pair-wise
    comparisons of those genomes using the nucmer settings given
    """

    # Run commands on biotite
    cmds = []
    for g1 in genomes:
        for g2 in genomes:
            out = "{0}{1}_vs_{2}".format(outf,get_genome_name_from_fasta(g1),get_genome_name_from_fasta(g2))
            cmds.append(gen_nucmer_cmd(out,g1,g2,c=c,noextend=noextend,maxgap=maxgap,method=method))
    if not dry:
        thread_nucmer_cmds_status(cmds, t=p)

    # Parse resulting folder
    data = process_deltadir(outf, b2s)
    data['c'] = c
    data['maxgap'] = maxgap
    data['noextend'] = noextend
    data['method'] = method

    return data

def all_vs_all_MASH(Bdb, data_folder, **kwargs):
    """
    Run MASH pairwise within all samples in Bdb
    """

    MASH_s = kwargs.get('MASH_sketch',1000)
    dry = kwargs.get('dry',False)
    overwrite = kwargs.get('overwrite', False)
    mash_exe = kwargs.get('mash_exe', 'mash')
    p = kwargs.get('processors',6)

    # set up logdir
    if 'wd' in kwargs:
        logdir = kwargs.get('wd').get_dir('cmd_logs')
    else:
        logdir = False

    # Set up folders
    MASH_folder = data_folder + 'MASH_files/'
    if not os.path.exists(MASH_folder):
        os.makedirs(MASH_folder)

    sketch_folder = MASH_folder + 'sketches/'
    if not os.path.exists(sketch_folder):
        os.makedirs(sketch_folder)

    # Make the MASH sketches
    cmds = []
    for fasta in Bdb['location'].unique():
        genome = Bdb['genome'][Bdb['location'] == fasta].tolist()[0]
        file = sketch_folder + genome
        if not os.path.isfile(file + '.msh'):
            cmd = [mash_exe, 'sketch', fasta, '-s', str(MASH_s), '-o',
                file]
            cmds.append(cmd)

    if not dry:
        if len(cmds) > 0:
            dm.thread_cmds(cmds, logdir=logdir, t=int(p))

    # Combine MASH sketches
    cmd = [mash_exe, 'paste', MASH_folder + 'ALL.msh'] + glob.glob(sketch_folder+ '*')
    # cmd = ' '.join(cmd)
    dm.run_cmd(cmd, dry, shell=False, logdir=logdir)

    # Calculate distances
    all_file = MASH_folder + 'ALL.msh'
    cmd = [mash_exe, 'dist', all_file, all_file, '>', MASH_folder
            + 'MASH_table.tsv']
    cmd = ' '.join(cmd)
    dm.run_cmd(cmd, dry, shell=True, logdir=logdir)

    # Make Mdb based on all genomes in the MASH folder
    Mdb = pd.DataFrame()
    file = MASH_folder + 'MASH_table.tsv'

    table = pd.read_csv(file,sep='\t',header = None)
    table.columns = ['genome1','genome2','dist','p','kmers']
    table['genome1'] = table['genome1'].apply(get_genome_name_from_fasta)
    table['genome2'] = table['genome2'].apply(get_genome_name_from_fasta)
    Mdb = pd.concat([Mdb,table],ignore_index=True)
    Mdb['similarity'] = 1 - Mdb['dist'].astype(float)

    # Filter out those genomes that are in the MASH folder but shouldn't be in Mdb
    genomes = Bdb['genome'].unique()
    Mdb = Mdb[Mdb['genome1'].isin(genomes)]
    Mdb = Mdb[Mdb['genome2'].isin(genomes)]

    return Mdb

def cluster_mash_database(db, data_folder= False, **kwargs):

    logging.debug('Clustering MASH database')

    P_Lmethod = kwargs.get('clusterAlg','single')
    P_Lcutoff = 1 - kwargs.get('P_ani',.9)
    dry = kwargs.get('dry',False)
    overwrite = kwargs.get('overwrite', False)

    if data_folder != False:
        data_folder = data_folder + 'Clustering_files/'
        if not os.path.exists(data_folder):
            os.makedirs(data_folder)

    db['dist'] = 1 - db['similarity']
    linkage_db = db.pivot("genome1","genome2","dist")
    Cdb, linkage = cluster_hierarchical(linkage_db, linkage_method= P_Lmethod, \
                                linkage_cutoff= P_Lcutoff)
    Cdb = Cdb.rename(columns={'cluster':'MASH_cluster'})

    if (data_folder != False):
        arguments = {'linkage_method':P_Lmethod,'linkage_cutoff':P_Lcutoff,\
                        'comparison_algorithm':'MASH'}
        logging.debug('Saving primary_linkage pickle to {0}'.format(data_folder))
        with open(data_folder + 'primary_linkage.pickle', 'wb') as handle:
            pickle.dump(linkage, handle)
            pickle.dump(linkage_db, handle)
            pickle.dump(arguments, handle)

    return Cdb

def get_genome_name_from_fasta(fasta):
    return str(os.path.basename(fasta))

# Parse NUCmer delta file to get total alignment length and total sim_errors
def parse_delta(filename):
    """Returns (alignment length, similarity errors) tuple from passed .delta.
    - filename - path to the input .delta file
    Extracts the aligned length and number of similarity errors for each
    aligned uniquely-matched region, and returns the cumulative total for
    each as a tuple.
    """
    aln_length, sim_errors = 0, 0
    for line in [l.strip().split() for l in open(filename, 'rU').readlines()]:
        if line[0] == 'NUCMER' or line[0].startswith('>'):  # Skip headers
            continue
        # We only process lines with seven columns:
        if len(line) == 7:
            aln_length += abs(int(line[1]) - int(line[0]))
            sim_errors += int(line[4])
    return aln_length, sim_errors

def gen_gANI_cmd(file, g1, g2, dir, exe):
    # Handle self comparison
    # Did a pretty exhaustive test- same comaprisons always give 100% ANI
    if g1 == g2:
        # make a copy of g1
        #g1T = g1 + '.GANI_TEMP'
        #shutil.copyfile(g1,g1T)

        #cmd = [exe,'-genome1fna',g1T,'-genome2fna',g2,'-outfile',file,'-outdir',file + 'TEMP']
        return []
    else:
        cmd = [exe,'-genome1fna',g1,'-genome2fna',g2,'-outfile',file,'-outdir',file + 'TEMP']
    #cmd = [exe,'-genome1fna',g1,'-genome2fna',g2,'-outfile',file,'-outdir',dir]
    return cmd

def process_deltadir(deltafiles, org_lengths, logger=None):
    """Returns a tuple of ANIm results for .deltas in passed directory.
    - delta_dir - path to the directory containing .delta files
    - org_lengths - dictionary of total sequence lengths, keyed by sequence
    Returns the following pandas dataframes in a tuple; query sequences are
    rows, subject sequences are columns:
    - alignment_lengths - symmetrical: total length of alignment
    - percentage_identity - symmetrical: percentage identity of alignment
    - alignment_coverage - non-symmetrical: coverage of query and subject
    - similarity_errors - symmetrical: count of similarity errors
    May throw a ZeroDivisionError if one or more NUCmer runs failed, or a
    very distant sequence was included in the analysis.
    """
    # Process directory to identify input files
    #deltafiles = glob.glob(delta_dir + '*.delta')

    Table = {'querry':[],'reference':[],'alignment_length':[],'similarity_errors':[],
            'ref_coverage':[],'querry_coverage':[],'ani':[], 'reference_length':[],
            'querry_length':[],'alignment_coverage':[]}

    # Process .delta files assuming that the filename format holds:
    # org1_vs_org2.delta
    zero_error = False  # flag to register a divide-by-zero error
    for deltafile in deltafiles:
        qname, sname = os.path.splitext(os.path.split(deltafile)[-1])[0].split('_vs_')
        tot_length, tot_sim_error = parse_delta(deltafile)
        if tot_length == 0 and logger is not None:
            logging.info("Total alignment length reported in " +
                               "%s is zero!" % deltafile)
        query_cover = float(tot_length) / org_lengths[qname]
        sbjct_cover = float(tot_length) / org_lengths[sname]
        # Calculate percentage ID of aligned length. This may fail if
        # total length is zero.
        # The ZeroDivisionError that would arise should be handled
        # Common causes are that a NUCmer run failed, or that a very
        # distant sequence was included in the analysis.
        try:
            perc_id = 1 - float(tot_sim_error) / tot_length
        except ZeroDivisionError:
            #print("Alignment between {0} and {1} has 0 alignment!".format(qname,sname))
            perc_id = 0  # set arbitrary value of zero identity
            zero_error = True

        Table['querry'].append(qname)
        Table['querry_length'].append(org_lengths[qname])
        Table['reference'].append(sname)
        Table['reference_length'].append(org_lengths[sname])
        Table['alignment_length'].append(tot_length)
        Table['similarity_errors'].append(tot_sim_error)
        Table['ani'].append(perc_id)
        Table['ref_coverage'].append(sbjct_cover)
        Table['querry_coverage'].append(query_cover)
        Table['alignment_coverage'].append((tot_length * 2)/(org_lengths[qname]\
                                                             + org_lengths[sname]))

    df = pd.DataFrame(Table)
    return df

def process_deltafiles(deltafiles, org_lengths, logger=None, **kwargs):

    Table = {'querry':[],'reference':[],'alignment_length':[],'similarity_errors':[],
            'ref_coverage':[],'querry_coverage':[],'ani':[], 'reference_length':[],
            'querry_length':[],'alignment_coverage':[]}

    # Process .delta files assuming that the filename format holds:
    # org1_vs_org2.delta
    coverage_method = kwargs.get('coverage_method')
    #logging.debug('coverage_method is {0}'.format(coverage_method))

    for deltafile in deltafiles:
        qname, sname = os.path.splitext(os.path.split(deltafile)[-1])[0].split('_vs_')
        # Handle naming of "filtered"
        if sname.endswith('.delta'):
            sname = sname[:-6]

        tot_length, tot_sim_error = parse_delta(deltafile)
        if tot_length == 0 and logger is not None:
            logging.info("Total alignment length reported in " +
                               "%s is zero!" % deltafile)
        query_cover = float(tot_length) / org_lengths[qname]
        sbjct_cover = float(tot_length) / org_lengths[sname]

        try:
            perc_id = 1 - float(tot_sim_error) / tot_length
        except ZeroDivisionError:
            #print("Alignment between {0} and {1} has 0 alignment!".format(qname,sname))
            perc_id = 0  # set arbitrary value of zero identity
            zero_error = True

        Table['querry'].append(qname)
        Table['querry_length'].append(org_lengths[qname])
        Table['reference'].append(sname)
        Table['reference_length'].append(org_lengths[sname])
        Table['alignment_length'].append(tot_length)
        Table['similarity_errors'].append(tot_sim_error)
        Table['ani'].append(perc_id)
        Table['ref_coverage'].append(sbjct_cover)
        Table['querry_coverage'].append(query_cover)

        if coverage_method == 'total':
            Table['alignment_coverage'].append((tot_length * 2)/(org_lengths[qname]\
                                                             + org_lengths[sname]))
        elif coverage_method == 'larger':
            Table['alignment_coverage'].append(max((tot_length/org_lengths[qname]),\
                (tot_length/org_lengths[sname])))

    df = pd.DataFrame(Table)
    return df

### MAKE IT SO THAT YOU REMOVE THE _TEMP MARKER FROM THE GENOMES, AND DELTE THE
### FILE WHILE YOU'RE AT IT
def process_gani_files(files):
    Table = {'querry':[],'reference':[],'ani':[],'alignment_coverage':[]}
    for file in files:
        results = parse_gani_file(file)

        # calculate coverage using the "larger" method
        cov = max(results['rq_coverage'], results['qr_coverage'])

        # Add forward results
        Table['reference'].append(results['reference'])
        Table['querry'].append(results['querry'])
        Table['ani'].append(float(results['rq_ani'])/100)
        Table['alignment_coverage'].append(cov)
        #Table['alignment_coverage'].append(results['rq_coverage'])

        # Add reverse results [they're the same]
        Table['reference'].append(results['querry'])
        Table['querry'].append(results['reference'])
        Table['ani'].append(float(results['qr_ani'])/100)
        Table['alignment_coverage'].append(cov)
        #Table['alignment_coverage'].append(results['qr_coverage'])

    # Add self comparisons
    for g in set(Table['reference']):
        Table['reference'].append(g)
        Table['querry'].append(g)
        Table['ani'].append(1)
        Table['alignment_coverage'].append(1)

    Gdb = pd.DataFrame(Table)
    return Gdb

'''
This method takes in bdb (a table with the columns location and genome), runs
pair-wise comparisons between all genomes in the sample, and returns a table
with at least the columns 'reference', 'querry', 'ani','coverage', depending
on what algorithm is called
'''
def compare_genomes(bdb, algorithm, data_folder, **kwargs):
    # To handle other versions of this method which passed in a WorkDirectory
    # instead of data_folder string
    if isinstance(data_folder,drep.WorkDirectory.WorkDirectory):
        data_folder = data_folder.location + '/data/'

    if algorithm == 'ANIn':
        genome_list = bdb['location'].tolist()
        working_data_folder = data_folder + 'ANIn_files/'
        df = run_pairwise_ANIn(genome_list, working_data_folder, **kwargs)
        return df

    elif algorithm == 'ANImf':
        genome_list = bdb['location'].tolist()
        working_data_folder = data_folder + 'ANImf_files/'
        df = run_pairwise_ANImf(genome_list, working_data_folder, **kwargs)
        return df

    elif algorithm == 'gANI':
        working_data_folder = data_folder + 'gANI_files/'
        prod_folder = data_folder + 'prodigal/'
        df = run_pairwise_gANI(bdb, working_data_folder, \
                prod_folder = prod_folder, **kwargs)
        return df

    elif algorithm == 'mauve':
        working_data_folder = data_folder + 'mauve_files/'
        df = run_pairwise_mauve(bdb, working_data_folder, **kwargs)
        return df

    else:
        logging.error("{0} not supported".format(algorithm))
        sys.exit()

def run_pairwise_ANIn(genome_list, ANIn_folder, **kwargs):
    p = kwargs.get('processors',6)
    genomes = genome_list

    # Make folder
    if not os.path.exists(ANIn_folder):
        os.makedirs(ANIn_folder)

    # Gen commands
    cmds = []
    files = []
    for g1 in genomes:
        for g2 in genomes:
            file_name = "{0}{1}_vs_{2}".format(ANIn_folder, \
                        get_genome_name_from_fasta(g1),\
                        get_genome_name_from_fasta(g2))
            files.append(file_name)

            # If the file doesn't already exist, add it to what needs to be run
            if not os.path.isfile(file_name + '.delta'):
                cmds.append(gen_nucmer_cmd(file_name,g1,g2))

    # Run commands
    if len(cmds) > 0:
        for c in cmds:
            logging.debug(' '.join(c))

        if 'wd' in kwargs:
            logdir = kwargs.get('wd').get_dir('cmd_logs')
        else:
            logdir = False
        dm.thread_cmds(cmds, logdir=logdir, t=int(p))
        #thread_nucmer_cmds_status(cmds,p,verbose=False)

    # Parse output
    org_lengths = {}
    for genome in genomes:
        #logging.debug("getting genome length of {0}".format(genome))
        org_lengths[get_genome_name_from_fasta(genome)] = dm.fasta_length(genome)

    deltafiles = ["{0}.delta".format(file) for file in files]
    df = process_deltafiles(deltafiles, org_lengths, **kwargs)

    return df

def run_pairwise_ANImf(genome_list, ANIn_folder, **kwargs):
    p = kwargs.get('processors',6)
    genomes = genome_list

    # Make folder
    if not os.path.exists(ANIn_folder):
        os.makedirs(ANIn_folder)

    # Gen commands
    cmds = []
    files = []
    for g1 in genomes:
        for g2 in genomes:
            file_name = "{0}{1}_vs_{2}".format(ANIn_folder, \
                        get_genome_name_from_fasta(g1),\
                        get_genome_name_from_fasta(g2))
            files.append(file_name)

            # If the file doesn't already exist, add it to what needs to be run
            if not os.path.isfile(file_name + '.delta.filtered'):
                cmds.append(gen_animf_cmd(file_name,g1,g2))

    # Run commands
    if len(cmds) > 0:
        for c in cmds:
            logging.debug(' '.join(c))

        if 'wd' in kwargs:
            logdir = kwargs.get('wd').get_dir('cmd_logs')
        else:
            logdir = False
        dm.thread_cmds(cmds, shell=True, logdir=logdir, t=int(p))

    # Parse output
    org_lengths = {}
    for genome in genomes:
        #logging.debug("getting genome length of {0}".format(genome))
        org_lengths[get_genome_name_from_fasta(genome)] = dm.fasta_length(genome)

    deltafiles = ["{0}.delta.filtered".format(file) for file in files]
    df = process_deltafiles(deltafiles, org_lengths, **kwargs)

    return df

def run_pairwise_mauve(bdb, data_folder, **kwargs):
    p = kwargs.get('processors',6)

    # Make folder
    if not os.path.exists(data_folder):
        os.makedirs(data_folder)

    # Gen commands
    cmds = []
    files = []
    for g1 in genomes:
        for g2 in genomes:
            '''
            file_name = "{0}{1}_vs_{2}".format(ANIn_folder, \
                        get_genome_name_from_fasta(g1),\
                        get_genome_name_from_fasta(g2))
            files.append(file_name)

            # If the file doesn't already exist, add it to what needs to be run
            if not os.path.isfile(file_name + '.delta'):
                cmds.append(gen_nucmer_cmd(file_name,g1,g2))
            '''

    # Run commands
    if len(cmds) > 0:
        thread_nucmer_cmds_status(cmds,p,verbose=False)

    # Parse output
    '''
    org_lengths = {get_genome_name_from_fasta(y):dm.fasta_length(x) \
                    for x,y in zip(genomes,genomes)}
    deltafiles = ["{0}.delta".format(file) for file in files]
    df = process_deltafiles(deltafiles, org_lengths)
    '''

    return df

def run_pairwise_gANI(bdb, gANI_folder, verbose = False, **kwargs):
    # gANI_exe = kwargs.get('gANI_exe','ANIcalculator')
    # loc = shutil.which('ANIcalculator')
    loc, works = dBonus.find_program('ANIcalculator')
    if loc == None:
        logging.error('Cannot locate the program {0}- make sure its in the system path'\
            .format('ANIcalculator (for gANI)'))
    if works == False:
        logging.error('Program {0} is not working'\
            .format('ANIcalculator (for gANI)'))
    gANI_exe = loc

    p = kwargs.get('processors',6)
    prod_folder = kwargs.get('prod_folder')
    genomes = bdb['location'].tolist()

    # Make folder
    if not os.path.exists(gANI_folder):
        os.makedirs(gANI_folder)

    # Remove crap folders- they shouldn't exist and if they do it messes things up
    crap_folders = glob.glob(gANI_folder + '*.gANITEMP')
    for crap_folder in crap_folders:
        if os.path.exists(crap_folder):
            logging.debug("CRAP FOLDER EXISTS FOR gANI- removing {0}".format(crap_folder))
            shutil.rmtree(crap_folder)

    if not os.path.exists(prod_folder):
        os.makedirs(prod_folder)

    # Run prodigal
    if verbose:
        logging.info("Running prodigal...")
    dFilter.run_prodigal(bdb, prod_folder, verbose=verbose, **kwargs)

    # Gen gANI commands
    if verbose:
        logging.info("Running gANI...")
    cmds = []
    files = []
    for i, g1 in enumerate(genomes):
        for j, g2 in enumerate(genomes):
            if i > j:
                name1= get_genome_name_from_fasta(g1)
                name2= get_genome_name_from_fasta(g2)
                file_name = "{0}{1}_vs_{2}.gANI".format(gANI_folder, \
                            name1, name2)
                files.append(file_name)

                # If the file doesn't already exist, add it to what needs to be run
                if not os.path.isfile(file_name):
                    fna1 = "{0}{1}.fna".format(prod_folder,name1)
                    fna2 = "{0}{1}.fna".format(prod_folder,name2)
                    cmds.append(gen_gANI_cmd(file_name,fna1,fna2,gANI_folder,gANI_exe))

    # Run commands
    if len(cmds) > 0:
        logging.debug('Running gANI commands: {0}'.format('\n'.join([' '.join(x) for x in cmds])))
        if 'wd' in kwargs:
            logdir = kwargs.get('wd').get_dir('cmd_logs')
        else:
            logdir = False
        dm.thread_cmds(cmds, logdir=logdir, t=int(p))

    else:
        if verbose:
            logging.info("gANI already run- will not re-run")

    # Parse output
    df = process_gani_files(files)

    # Handle self-comparisons
    #df['reference'] = [i.replace('.fna.GANI_','') for i in df['reference']]
    #df['querry'] = [i.replace('.fna.GANI_','') for i in df['querry']]
    #df = df.drop_duplicates()

    # Add self-comparisons if there is only one genome
    if len(genomes) == 1:
        Table = {'querry':[],'reference':[],'ani':[],'alignment_coverage':[]}
        for g in genomes:
            Table['reference'].append(get_genome_name_from_fasta(g))
            Table['querry'].append(get_genome_name_from_fasta(g))
            Table['ani'].append(1)
            Table['alignment_coverage'].append(1)
        d = pd.DataFrame(Table)
        df = pd.concat([df,d],ignore_index=True)

    return df

def parse_gani_file(file):
    try:
        x = pd.read_table(file)
    except:
        logging.error('gANI file {0} does not exist. The most likely reason is that '.format(file)\
            + 'one of the genomes has a .fasta header than gANI doesnt like.'\
            + ' Known issues include having a header over 160 characaters long, or any '\
            + 'kind of special character besides "_" (including ".", ":", and "-").'\
            + ' To fix this error either use ANIn (which is not as picky with fasta headers)'\
            + ', or fix the .fasta headers to conform to those rules.')
        sys.exit()
    x = x.rename(columns={'GENOME1':'reference','GENOME2':'querry','AF(1->2)':'rq_coverage',\
                        'AF(2->1)':'qr_coverage','ANI(1->2)':'rq_ani','ANI(2->1)':'qr_ani'})
    dict = x.to_dict(orient='list')
    dict = {k:v[0] for k,v in dict.items()}
    dict['reference'] = dict['reference'][:-4]
    dict['querry'] = dict['querry'][:-4]
    return dict

def gen_nucmer_cmd(prefix,ref,querry,c='65',noextend=False,maxgap='90',method='mum'):
    cmd = ['nucmer','--' + method,'-p',prefix,'-c',str(c),'-g',str(maxgap)]
    if noextend: cmd.append('--noextend')
    cmd += [ref,querry]
    return cmd

def gen_animf_cmd(prefix,ref,querry, **kwargs):
    '''
    return animf command. It will be a single string, with the format:
    "nucmer cmd ; filter cmd"
    '''

    nucmer_cmd = gen_nucmer_cmd(prefix,ref,querry, **kwargs)

    delta = prefix + '.delta'
    out = delta + '.filtered'

    filter_cmd = gen_filter_cmd(prefix + '.delta', out)
    return ' '.join(nucmer_cmd) + '; ' + ' '.join(filter_cmd)

def gen_filter_cmd(delta, out):
    cmd = ["delta-filter", '-r', '-q', delta, '>', out]
    return cmd

def thread_nucmer_cmds(cmds,t=10):
    pool = multiprocessing.Pool(processes=t)
    pool.map(run_nucmer_cmd,cmds)
    pool.close()
    pool.join()
    return

def thread_nucmer_cmds_status(cmds,t=10,verbose=True):
    total = len(cmds)
    if verbose:
        minutes = ((float(total) * (float(0.33))) /float(t))
        logging.info("Running {0} mummer comparisons: should take ~ {1:.1f} min".format(total,minutes))
    pool = multiprocessing.Pool(processes=int(t))
    rs = pool.map_async(run_nucmer_cmd,cmds)
    pool.close()

    pool.join()
    return

def thread_mash_cmds_status(cmds,t=10):
    total = len(cmds)
    pool = multiprocessing.Pool(processes=t)
    rs = pool.map_async(run_nucmer_cmd,cmds)
    pool.close()

    while(True):
        done = total - (rs._number_left * rs._chunksize)
        percR = (done/total) * 100
        sys.stdout.write('\r')
        sys.stdout.write("[{0:20}] {1:3.2f}%".format('='*int(percR/5), percR))
        sys.stdout.flush()
        if (rs.ready()):
            sys.stdout.write('\n')
            break
        time.sleep(0.5)

    pool.join()
    return

def gen_nomash_cdb(Bdb):
    Cdb = Bdb.copy()
    Cdb['MASH_cluster'] = 0
    return Cdb

# def run_nucmer_cmd(cmd,dry=False,shell=False):
#     devnull = open(os.devnull, 'w')
#     if shell:
#         if not dry: call(cmd,shell=True, stderr=devnull,stdout=devnull)
#         else: print(cmd)
#     else:
#         if not dry: call(cmd, stderr=devnull,stdout=devnull)
#         else: print(' '.join(cmd))
#     return
#
# def run_mash_cmd(cmds):
#     cmd = ' '.join(cmd)
#     dm.run_cmd(cmd,dry=False,shell=True)

def load_genomes(genome_list):
    Table = {'genome':[],'location':[]}

    for genome in genome_list:
        assert os.path.isfile(genome), "{0} is not a file".format(genome)
        Table['genome'].append(os.path.basename(genome))
        Table['location'].append(os.path.abspath(genome))

    Bdb = pd.DataFrame(Table)
    return Bdb

def average_ani(row,db):
    g1 = row['querry']
    g2 = row['reference']
    if g1 == g2:
        return 1
    else:
        ani1 = float(row['ani'])
        ani2 = float(db['ani'][(db['reference'] == g1) & (db['querry'] == g2)].tolist()[0])
        avg = np.mean([ani1,ani2])
        return avg

def add_avani(db):
    '''
    add a column titled 'av_ani' to the passed in dataframe

    dataframe must have rows reference, querey, and ani.
    '''

    logging.debug('making dictionary for average_ani')
    combo2value = {}
    for i, row in db.iterrows():
        combo2value["{0}-vs-{1}".format(row['querry'], row['reference'])] \
            = row['ani']

    logging.debug('list comprehension for average_ani')
    db['av_ani'] = [np.mean([combo2value["{0}-vs-{1}".format(q,r)], \
                        combo2value["{0}-vs-{1}".format(r,q)]]) if r != q else 1\
                        for q, r in zip(db['querry'].tolist(), \
                        db['reference'].tolist())]

    logging.debug('averageing done')

def nucmer_preset(preset):
   #nucmer argument c, n_maxgap, n_noextend, n_method

    assert preset in ['tight','normal']

    if preset == 'tight':
        return 65, 1, True, 'mum'

    elif preset == 'normal':
        return 65, 90, False, 'mum'

def gen_nomani_cdb(Cdb, Mdb, **kwargs):
    c_method = kwargs.get('clusterAlg','single')
    threshold = 1 - kwargs.get('P_ani',.9)
    data_folder = kwargs.get('data_folder') + 'Clustering_files/'

    # Make Cdb look like Cdb
    cdb = Cdb.copy()
    cdb['secondary_cluster'] = ["{0}_0".format(i) for i in cdb['MASH_cluster']]
    cdb.rename(columns={'MASH_cluster': 'primary_cluster'}, inplace=True)
    cdb['threshold'] = threshold
    cdb['cluster_method'] = c_method
    cdb['comparison_algorithm'] = 'MASH'

    # Delete any only secondary clusters
    if kwargs.get('overwrite',False):
        for fn in glob.glob(data_folder + 'secondary_linkage_cluster*'):
            os.remove(fn)

    return cdb

def test_clustering():
    # Get test genomes
    test_genomes = glob.glob(str(os.getcwd()) + '/../test/genomes/*')
    names = [os.path.basename(g) for g in test_genomes]
    assert names == ['Enterococcus_faecalis_T2.fna',
	                 'Escherichia_coli_Sakai.fna',
	                 'Enterococcus_casseliflavus_EC20.fasta',
	                 'Enterococcus_faecalis_TX0104.fa',
	                 'Enterococcus_faecalis_YI6-1.fna']
    Bdb = load_genomes(test_genomes)

    # Set test directory
    test_directory = str(os.getcwd()) + '/../test/test_backend/'
    dm.make_dir(test_directory,overwrite=True)

    # Perform functional test
    Cdb, Mdb, Ndb = cluster_genomes(Bdb,test_directory)

    # Confirm it's right by showing there is one group of 3 and two groups of one
    group_lengths = sorted([len(Cdb['genome'][Cdb['ANIn_cluster'] == x].tolist()) for x in Cdb['ANIn_cluster'].unique()])
    assert group_lengths == [1, 1, 3]

    print("Functional test success!")

if __name__ == '__main__':

    test_clustering()


"""
Unit test classes for modules module.
"""
from vmaas.reposcan.repodata.modules import ModuleMD

class TestModuleMD:
    """Test ModuleMD class."""

    def test_modules_loading(self):
        """Test modules.yaml loading and parsing."""
        mod = ModuleMD("test_data/repodata/modules.yaml")
        assert len(mod.modules) == 2
        keys = ("name", "stream", "version", "context", "arch", "artifacts", "default_stream", "profiles", "requires")
        assert len(keys) == len(mod.modules[0])
        assert all(key in mod.modules[0] for key in keys)


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

class RNN(nn.Module):
	name = 'rnn_unroll'
	# def __init__(self, hidden_size=128, forecast=5, relu=False, deep=False):
	def __init__(self, hidden_size=256, forecast=5, deep=True, lag=6):
		super(RNN, self).__init__()
		# self.relu = relu
		self.lag = lag # temporal dimension
		# self.lag = 5 # temporal dimension
		self.steps = 10 # spatial dimension (optional ?)
		self.hidden_size = hidden_size
		self.forecast = forecast

		hsize = hidden_size
		if deep:
			self.inp = nn.Sequential(
				nn.Linear(self.lag, hsize),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				# nn.Dropout(0.5),
			)
			self.out = nn.Sequential(
				# nn.Dropout(0.5),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				nn.Linear(hsize, hsize),
				nn.ReLU(),
				nn.Linear(hsize, self.lag),
			)
		else:
			self.name += '_short'
			self.inp = nn.Sequential(
				nn.Linear(self.lag, hsize),
			)
			self.out = nn.Sequential(
				nn.Linear(hsize, self.lag),
			)

		self.rnn = nn.LSTM(hidden_size, hidden_size, 1)
		self.bsize = 32

	def step(self, input, hidden=None):
		# seqlen = 1 for stepwise eval
		# in: batch x inputsize
		if input is not None:
			input = self.inp(input)
			input = input.unsqueeze(0)
		else:
			input = torch.zeros(1, self.bsize, self.hidden_size).to(self.device)
		# in: seqlen x batch x inputsize
		output, hidden = self.rnn(input, hidden)
		# out: seqlen x batch x hiddensize
		output = self.out(output.squeeze(0))
		return output, hidden

	def forward(self, inputs, hidden=None, dump=False):
		steps = len(inputs)
		# lastKnown = steps - self.forecast - 1
		outputs = []

		for ii in range(steps):
			output, hidden = self.step(inputs[ii], hidden)
			outputs.append(output)

		outputs = torch.stack(outputs, dim=0)
		# return outputs, hidden
		if dump:
			return outputs, hidden
		else:
			return outputs

	def params(self, lr=0.001):
		criterion = nn.MSELoss().cuda()
		opt = optim.SGD(self.parameters(), lr=lr)
		sch = optim.lr_scheduler.StepLR(opt, step_size=20, gamma=0.5)
		return criterion, opt, sch

	def format_batch(self, data, wrap=True, normalize=1):
		# raw   : batch x timelen x seqlen
		data /= normalize
		data = torch.transpose(torch.transpose(data, 2, 1), 1, 0)
		# fmt   : seqlen x batch x timelen
		sequence = list(torch.split(data, 1, dim=0))

		for ti in range(len(sequence)):
			sequence[ti] = sequence[ti].to(self.device).float().squeeze(0)

		Xs = list(reversed(sequence[1:]))
		Ys = list(reversed(sequence[:-1])) # predict 1 stop back
		Ys = torch.stack(Ys, dim=0)

		return Xs, Ys


# leet494

from collections import defaultdict
from typing import List

def init(dims, filled_value = 0):
    if not dims: return filled_value
    return [init(dims[1:]) for _ in range(dims[0])]

class Solution:
    def findTargetSumWays(self, A: List[int], target: int) -> int:
        f = {0: 1}
        for x in A:
            f, g = defaultdict(int), f
            for t, cnt in g.items():
                f[t - x] += cnt
                f[t + x] += cnt
        return f[target]

print(Solution().findTargetSumWays([1,1,1,1,1], 3))


with open('data.txt') as fp:
    datas = fp.read()
#each testcase is seprated by '\n\n'
data = datas.split('\n\n')
suma = 0
for elem in data:
    quests = set()
    # different lines contain no information...
    answ = elem.replace('\n','').replace(' ','')
    for char in answ:
        #if this question has not answered, it is added to the set
        quests.add(char)
    suma = len(quests) + suma
print(suma)