import enchant
import os
import xml.etree.ElementTree as ET
import matplotlib.pyplot as plt
import argparse

#The English dictionary
d = enchant.Dict("en_US")

def DictLookup(input_path, output_path):
	'''input: dictionary file from the output of running concept mapper,
		output: list of terms that are in the english dictionary - canonical form or synonyms '''

	#Initialize variables
	Problem_terms = {} #dictionary name to problem terms
	total_words_all = [] #all the counts of the total number of single words
	total_english_words_all = [] #all the counts of the total number of English words
	labels = [] #name of ontology dictionary

	#For each dictionary file in the input_path: find the single words and check if they are in the English dictionary
	for file in os.listdir(input_path):
		if 'cmDict-' in file: #Dictionary file names (cm = ConceptMapper)
			total_words = 0 #count of how many single words
			total_english_words = 0 #count of how many English words

			#Output file
			Variants_InEnDict = open(output_path +'%s' %(str(file).split('.xml')[0]), 'a+')
			Variants_InEnDict.write('%s\t%s\t%s\n' %('Ontology Term', 'Concept', 'Variant'))

			#open the dictionary file and check if the single words are in the English dictionary
			with open(input_path+file,'r'):
				tree = ET.parse(input_path+file) #tree = token id, canonical, variant base (first one is same as canonical)
				root = tree.getroot() #synonyms
				for child in root: #for each ontology term
					ont_term = str(child.attrib['id']).split('obo/')[-1] #ontology id

					for i,variant in enumerate(child.iter('variant')): #for each variant (synonym)
						if ' ' not in variant.attrib['base'] and '/' not in variant.attrib['base']: #only single words so no spaces in between words
							total_words += 1 #update the total number of single words
							if i > 0: #synonyms since the first variant is the same as the canonical form
								if d.check(variant.attrib['base']) == True: #check if the word is in the English dictionary
									total_english_words += 1 #update the total number of English words
									if i == 0: #if the variant is just the canonical form
										Variants_InEnDict.write('%s\t%s\t%s\t%s\n' %('canonical',ont_term, child.attrib['canonical'], variant.attrib['base'])) #write canonical so we know
									else:
										Variants_InEnDict.write('%s\t%s\t%s\n' %(ont_term, child.attrib['canonical'], variant.attrib['base'])) #otherwise it is not the canonical form and synonyms

							else: #canonical form only
								if d.check(child.attrib['canonical']) == True: #check if the word is in the English dictionary
									total_english_words += 1 #update the total number of English words
									Variants_InEnDict.write('%s\t%s\t%s\t%s\n' %('canonical',ont_term, child.attrib['canonical'], variant.attrib['base'])) #write canonical so we know
						else: #mutliwords
							pass
			labels += [file.split('-')[-1].split('.')[0]] #name of ontology dictionary
			total_words_all += [total_words] #all the counts of the total number of single words
			total_english_words_all += [total_english_words] #all the counts of the total number of English words

	# print labels
	# print total_words_all
	# print total_english_words_all
	Fraction_english_words = [float(total_english_words_all[f])/float(total_words_all[f]) for f in range(0,len(labels))] #the fraction of the number of english words over the number of total single words for all ontologies
	# print Fraction_english_words

	#Create a histogram of Fraction_english_words for all ontologies
	plt.bar(range(len(Fraction_english_words)), Fraction_english_words, align='center')
	plt.xticks(range(len(Fraction_english_words)), labels, size='small')
	plt.xlabel('Ontology')
	plt.ylabel('# English words/# Total words')	# plt.set_xticks(labels[:-1])
	# plt.show()
	plt.savefig(output_path+'Ontology_hardness.pdf')

#add def main() that inputs the path of where the files are

## Run the main program and print out the time it takes to run it
if __name__ == '__main__':
	### Argparse - parse arguments in command line: need training set, optional: minPrecision, minRecall, and test_set
	parser = argparse.ArgumentParser()
	parser.add_argument('input_path',type=str,help='Input the path to the ontology dictionary files that start with "cmDict"')
	parser.add_argument('output_path',type=str,help='Input the path to where you would like the output')
	args = parser.parse_args()
	DictLookup(args.input_path, args.output_path)


from django.shortcuts import render
from django.http import JsonResponse,HttpResponse,Http404
from .models import Main
from .form import IdeaForm
# Create your views here.

def bet_list(response):
    obj = Main.objects.all()
    data = {}
    status = 200
    try:
       li = [{"Id":i.id ,"Title":i.Title} for i in obj]
       data['response'] = li
    except:
       status = 404
       data['response'] = "Error Fetching"
    return JsonResponse(data,status=status)

def home(response):
    return render(response,"homepage/home.html")

def form(response):
    form = IdeaForm(response.POST or None)
    if(form.is_valid()):
        obj = form.save(commit=False)
        obj.save()
        form = IdeaForm()
    return render(response,"form.html",{"form":form})


from krita import Krita
from krita import QMdiArea, QAbstractScrollArea

def ScaleToZoomFunction(*args, **kwargs):
  instance = Krita.instance()
  documement = instance.activeDocument()
  window = instance.activeWindow()
  scroll = (
    window.qwindow()
    .findChild(QMdiArea)
    .currentSubWindow()
    .findChild(QAbstractScrollArea)
    )

  s_horizontal = scroll.horizontalScrollBar().value()
  s_vertical = scroll.verticalScrollBar().value()

  c_zoom = window.activeView().canvas().zoomLevel()
  d_resolution = documement.resolution()
  d_width = documement.width()
  d_height = documement.height()
  w_devicePixelRatioF = window.qwindow().devicePixelRatioF()

  _zoom = c_zoom * w_devicePixelRatioF / (d_resolution/72)

  if round(_zoom, 2) == 1.0:
    return (False, "No need to scale zoom level is 100%.")

  _width = int(d_width * _zoom)
  _height = int(d_height * _zoom)

  documement.scaleImage(_width, _height, d_resolution, d_resolution, 'bicubic')
  window.activeView().canvas().setZoomLevel(1.0)
  scroll.horizontalScrollBar().setValue(s_horizontal)
  scroll.verticalScrollBar().setValue(s_vertical)

  documement.refreshProjection()

  return (True, f'Scaled {_zoom}')


"""
Import typing, numpy, matplotlib scipy
"""
from typing import List
import numpy as np
import matplotlib.pyplot as plt
import scipy.fft as fft
fourier_data = list()  # initialize fourier list
frequencies_of_partitions = list()  # initialize frequencies list
# function to plot a list of fourrier transform in frequency domain
def fourier_plot(signal_data: List, frequency: int, index: int) -> None:
    """
    :param signal_data: the signal data
    :param frequency: the sampling frequency
    :param index: iterator for each signal
    :return:
    """
    # calculate fourier transform of the input signal
    fourier_data.append(fft.fft(signal_data))
    # calculate the corresponding fourier magnitude
    fourier_mag = abs(fourier_data[index])  # of all coeffs
    # split frequencies axis
    frequencies_of_partitions.append(np.linspace(0, frequency, len(signal_data) // 2))
    # plot the data
    plt.plot(frequencies_of_partitions[index], fourier_mag[0:len(signal_data) // 2])
    plt.title(f'Fourrier of signal: {index + 1}') # add a title
    plt.xlabel('frequency in Hz')
    plt.ylabel('Fourier magnitude ')
    plt.show() # show the plot


# This program is used to create a multithreaded server that can display a webpage
# Run this program first before the client and connecting to server from browser
# To run the program use this command: python HTTP_Server.py <port number>
# Authors: Vincent Guevara, Tianhui Xu, Toan Tran

#import socket module
import socket
# In order to terminate the program
import sys

#import thread module
import thread

#thread creation routine
def clientthread(conn):
    while True:
        try:
            #client request file
            message = conn.recv(2048)
            if not message:
                break
            #read in file requested
            filename = message.split()[1]
            f = open(filename[1:])
            outputdata = f.read()
            f.close()
            #send the OK file was found to client
            conn.send('HTTP/1.1 200 OK\r\n\r\n'.encode())
            #send all information of file
            conn.sendall(outputdata)
            conn.send('\r\n')
            conn.close()
        except IOError:
            conn.close()
            break
    sys.exit()

def main():
    #Prepare a sever socket allowing for reusing of address
    serverPort = int(sys.argv[1])
    serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    try:
        serverSocket.bind(('', serverPort))
    except socket.error:
        print("binding failed")
        sys.exit()
    serverSocket.listen(10)
    while True:
        #Establish the connection
        print('Ready to serve...')
        connectionSocket, addr = serverSocket.accept()#Fill in start #Fill in end
        print("Connected with "+ addr[0]+":"+str(addr[1]))
        #create a new thread
        thread.start_new_thread(clientthread, (connectionSocket,))

main()


x = input('请输入参数x ')
n = input('请输入参数n ')
x = float(x)
n = float(n)
def powernx(x,n):
	if not (isinstance(x,(int,float)) and isinstance(n,(int,float))):
		raise TypeError('TypeError')
	else:
		s = 1
		while n>0:
			n=n-1
			s = s*x
		return s
y = powernx(x,n)
print(y)


# %% --------------------------------------- Imports -------------------------------------------------------------------
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.optimizers import Adam
from sklearn.metrics import accuracy_score
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.initializers import glorot_uniform

# %% ----------------------------------- Hyper Parameters --------------------------------------------------------------
LR = 0.2
N_NEURONS = 2
N_EPOCHS = 3000

# %% ----------------------------------- Helper Functions --------------------------------------------------------------
def acc(x, y):
    """ Simple function to get the accuracy. The label with the highest prob is chosen """
    # model.evaluate can also be used, this is just to illustrate the argmax operation
    probs = model.predict(x)  # (n_examples, n_labels) --> Need to operate on columns, so axis=1 on argmax
    pred_labels = np.argmax(probs, axis=1)
    return 100*accuracy_score(np.argmax(y, axis=1), pred_labels)  # Uses argmax again to come back from one-hot-encoded

# %% -------------------------------------- Data Prep ------------------------------------------------------------------
p = np.array([[0, 0], [1, 1], [0, 1], [1, 0]])
t = np.array([0, 0, 1, 1])
# One-hot-encodes the targets to use the softmax and categorical cross-entropy performance index
t = to_categorical(t, num_classes=2)

# %% -------------------------------------- Training Prep --------------------------------------------------------------
model = Sequential([
    Dense(N_NEURONS, input_dim=2, kernel_initializer=glorot_uniform(42)),
    Activation("sigmoid"),
    Dense(2, kernel_initializer=glorot_uniform(42)),  # Output layer with softmax to map to the two classes
    Activation("softmax")
])
# Compiles using categorical cross-entropy performance index and tracks the accuracy during training
model.compile(optimizer=Adam(lr=LR), loss="categorical_crossentropy", metrics=["accuracy"])

# %% -------------------------------------- Training Loop --------------------------------------------------------------
model.fit(p, t, batch_size=len(p), epochs=N_EPOCHS)

# %% -------------------------------------- Check Approx ---------------------------------------------------------------
print("The MLP accuracy is", acc(p, t), "%")
# Gets the weights and biases from the first layer
w, b = model.layers[0].get_weights()
# Gets some points on the interval [0, 1] and the two decision boundaries
a = np.arange(0, 1.1, 0.1)
db1 = -w[0, 0]*a/w[1, 0] - b[0]/w[1, 0]  # Note than on Keras the shape of the weight matrix
db2 = -w[0, 1]*a/w[1, 1] - b[1]/w[1, 1]  # is RxS (n_inputs x n_neurons)
# Plots the points and the decision boundaries
plt.scatter(p[:2, 0], p[:2, 1], marker="*", c="b")
plt.scatter(p[2:, 0], p[2:, 1], marker=".", c="b")
plt.plot(a, db1, c="b")
plt.plot(a, db2, c="b")

plt.show()


#!/usr/bin/python2.7
# coding=utf-8
"""
Created on 2015-12-22

@author: Jay
"""

from lib.common import *
from mmm_da.lib.filter.control import ApplyPayFilter
import time
from mmm_da.lib.help_pay.setting import APYS_PAY_WAIT

apply_pay_max_days = 1
days_diff_fun = lambda d1, d2: int((d1 - d2) / 1)
id_2_dic = {}

ot_ls = []
def cb_fun(uid):
    ot_ls.append(uid)

def clear_tmp():
    global ot_ls
    global id_2_dic
    ot_ls= []
    id_2_dic.clear()

def get_apply_pay_ls():
    return id_2_dic.values()

apc = ApplyPayFilter()
filter_params = {}
filter_params.setdefault("apply_pay_max_days", apply_pay_max_days)
filter_params.setdefault("days_diff_fun", days_diff_fun)
apc.init(get_apply_pay_ls, cb_fun, filter_params)

class ApplyPayCheckerTest(unittest.TestCase):

    # 初始化工作
    def setUp(self):
        pass

    # 退出清理工作
    def tearDown(self):
        pass

    @unittest_adaptor()
    def test_apply_pay_nomal(self):
        clear_tmp()
        apc.filter()
        self.assertTrue(not ot_ls)

    @unittest_adaptor()
    def test_apply_pay_ot(self):
        clear_tmp()
        uid2 = "2"
        id_2_dic[uid2] = {'apply_sorder':uid2,
                          'apply_mtime': (time.time() - apply_pay_max_days - 1),
                          'apply_pstat': APYS_PAY_WAIT}

        apc.filter()
        self.assertTrue(ot_ls == [uid2])

    @unittest_adaptor()
    def test_apply_pay_ot_much(self):
        clear_tmp()
        uid2 = "2"
        id_2_dic[uid2] = {'apply_sorder':uid2,
                          'apply_mtime': (time.time() - apply_pay_max_days - 1),
                          'apply_pstat': APYS_PAY_WAIT}

        uid3 = "3"
        id_2_dic[uid3] = {'apply_sorder':uid3,
                          'apply_mtime': (time.time() - apply_pay_max_days - 2),
                          'apply_pstat': APYS_PAY_WAIT}

        uid4 = "4"
        id_2_dic[uid4] = {'apply_sorder':uid4,
                          'apply_mtime': (time.time() - apply_pay_max_days),
                          'apply_pstat': APYS_PAY_WAIT}

        uid5 = "5"
        id_2_dic[uid5] = {'apply_sorder':uid5,
                          'apply_mtime': (time.time()),
                          'apply_pstat': APYS_PAY_WAIT}

        apc.filter()
        #self.assertTrue(set(ot_ls) == set([uid2, uid3]))
        self.assertTrue(set(ot_ls) == {uid2, uid3, uid4})


import os
import numpy as np
import matplotlib.pyplot as plt
import keras
from trainingdata import *
from model import *

os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
#np_config.enable_numpy_behavior()

plt.close('all')
plt.ioff()

def main():

    loaddata = True
    current_directory = os.path.join(os.path.dirname(__file__))
    data_path         = os.path.join(current_directory, "training_data")
    if loaddata:
        x_train  = np.load(os.path.join(data_path , "x_train.npy"))
        xt_train = np.load(os.path.join(data_path , "xt_train.npy"))
    else:
        N         = 1500
        x0        = np.array([3*np.pi/7, 3*np.pi/4, 0, 0], dtype=np.float32)
        t         = np.arange(N, dtype=np.float32) # time steps 0 to N

        trainingdata  = TrainingDataDoublePendulum()
        data         = trainingdata.createdata(x0,t)

        trainingdata.savedata(data_path,data)

    'Create model'
    optimiser = keras.optimizers.Adam(learning_rate=1e-3)
    epochs = 10000
    dim = 4
    model = nn_model(dim,optimiser)
    model.train(x_train[:100,:],xt_train[:100,:],epochs)

if __name__ == "__main__":
    main()


from ssdaq.data import MonitorData
import datetime
import os
import zmq
import asyncio

class ReceiverMonSender:
    def __init__(self, name, loop, zmqcontext):
        """Summary

        Args:
            name (TYPE): Description
            loop (TYPE): Description
            zmqcontext (TYPE): Description
        """
        # monitoring socket
        self._context = zmqcontext
        self._monitor_sock = self._context.socket(zmq.PUSH)
        self._monitor_sock.connect("tcp://127.0.0.101:10002")
        self.total_data_counter = 0
        self.current_data_counter = 0
        self.got_data = False
        self.data_timeout = None
        self.mon_wait = 1.0
        self.name = name
        self.past = None
        loop.create_task(self.sendmon())
        self.loop = loop

    def register_data_packet(self):
        """Summary
        """
        self.total_data_counter += 1
        self.current_data_counter += 1
        self.got_data = True

    def _compute_rates(self):
        """Summary

        Returns:
            TYPE: Description
        """
        now = datetime.datetime.now().timestamp()

        if self.past is None:
            self.past = now
            self.current_data_counter = 0
            return
        dt = now - self.past
        self.past = now
        rate = self.current_data_counter / dt
        self.current_data_counter = 0

        return rate

    async def sendmon(self):
        """Summary
        """
        while True:
            await asyncio.sleep(self.mon_wait)

            mdata = MonitorData()
            # constructing timestamp
            tstamp = datetime.datetime.utcnow().timestamp()
            mdata.time.sec = int(tstamp)
            mdata.time.nsec = int((tstamp - mdata.time.sec) * 1e9)
            # Constructing monitoring message
            mdata.reciver.pid = os.getpid()
            mdata.reciver.name = self.name
            rate = self._compute_rates()
            if rate is None:
                continue
            mdata.reciver.data_rate = rate
            mdata.reciver.recv_data = self.got_data
            self.got_data = False
            # Putting it into a monitor data message

            self._monitor_sock.send(mdata.SerializeToString())


"""
    1588. Sum of All Odd Length Subarrays

    Given an array of positive integers arr, return the sum of all possible odd-length subarrays of arr.
    A subarray is a contiguous subsequence of the array.

    :type arr: List[int]
    :rtype: int

    This following codes are modified for use in local editor. It may be slighly different from submission code.
"""
def sumOddLengthSubarrays(arr):
    ans=0
    for i in range(len(arr)):
        for j in range(i,len(arr),2):
            ans+=sum(arr[i:j+1])
    return ans

print(sumOddLengthSubarrays([1,4,2,5,3]))


import psycopg2 as ps
import config

def exec_query(query: str) -> list:
    with ps.connect(dbname=config.db_name, user=config.user) as conn:
        with conn.cursor() as cur:
            cur.execute(query)
            res = cur.fetchall()
    return res


import numpy as np
from utils.datasets import DataLoader
from evaluation.wrapper import WrapperEvaluation
from fitness.function import WrapperFitnessFunction

class SubsetSizePenalty(WrapperFitnessFunction):
    """
    Objective function that penalizes large subsets of features.

    Rashid, A.N.M Bazlur, et al. "A novel penalty-based wrapper objective function for feature
    selection in Big Data using cooperative co-evolution." IEEE Access 8 (2020): 150113-150129.

    Attributes
    ----------
    w1: float
        Predictive performance weight.
    w2: float
        Penalty weight.
    """

    def __init__(self, evaluator: WrapperEvaluation, weights: list):
        super().__init__(evaluator)
        # Check the number of weights
        if len(weights) != 2:
            raise AssertionError(
                f"'{SubsetSizePenalty.__name__}' fitness function has only two components "
                "(predictive performance and penalty). Therefore, it requires only two weights."
            )
        # Check the sum of the weights
        if sum(weights) != 1:
            raise AssertionError(
                f"The sum of weights is {sum(weights)} but must be 1."
            )
        self.w1 = weights[0]
        self.w2 = weights[1]

    def evaluate(self, context_vector: np.ndarray, data: DataLoader):
        """
        Evaluate the given context vector using the fitness function.

        Parameters
        ----------
        context_vector: np.ndarray
            Solution of the complete problem.
        data: DataLoader
            Container with process data and training and test sets.

        Returns
        -------
        fitness: float
            Quality of the context vector.
        """
        penalty = context_vector.sum()/data.n_features
        evaluations = self._evaluate_predictive_performance(context_vector, data)
        evaluation = evaluations[self.evaluator.eval_function]
        fitness = self.w1*evaluation - self.w2*penalty
        return fitness


import os
import utils
import torch
import tqdm
import random
import matplotlib
import numpy as np
from torchvision import transforms
from evaluator.evaluator import KinshipEvaluator
from datasets.kinfacew_loader_gen import KinFaceWLoaderGenerator
from models.small_siamese_face_model import SmallSiameseFaceModel
from sklearn.metrics import accuracy_score
matplotlib.use('TkAgg')

seed = 990411
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)

dataset_name = "KinFaceW-I"
dataset_path = "/home/manuel/Documents/masters/Computer Vision/YGYME/data/"

model_path = f"logs/triplet_bn_auc/"
# model_path = f"logs/no_triplet_bn_auc/"
loader_gen = KinFaceWLoaderGenerator(dataset_name=dataset_name, dataset_path=dataset_path, color_space_name="rgb")

pair_type = "fd"
fold = 1
gpu_id = 0

# ms fold 3

device = torch.device(f"cuda:{gpu_id}" if torch.cuda.is_available() else "cpu")
# device = "cpu"
distances = list()

model = SmallSiameseFaceModel()
model.eval()
state_dict = torch.load(os.path.join(model_path, f"best_model_{pair_type}_fold_{fold}.pth"))
model.load_state_dict(state_dict)
model.to(device)
transformer = transforms.Compose([transforms.ToPILImage(),
                                  transforms.Resize((64, 64)),
                                  transforms.ToTensor()])

for fold in [1, 2, 3, 4, 5]:
    model = SmallSiameseFaceModel()
    model.eval()
    state_dict = torch.load(os.path.join(model_path, f"best_model_{pair_type}_fold_{fold}.pth"))
    model.load_state_dict(state_dict)
    model.to(device)

    loader_test, loader_train = loader_gen.get_data_loader(fold, 16, pair_type, transformer, transformer)

    evaluator = KinshipEvaluator(set_name="Training", pair=pair_type, log_path="logs/testing", fold=fold)
    evaluator.reset()
    parent_features = list()
    child_features = list()
    model_out = list()
    targets = list()

    for sample in tqdm.tqdm(loader_test, total=len(loader_test), desc=f"Val epoch {1}"):
        parent_image, children_image = sample["parent_image"].to(device), \
                                       sample["children_image"].to(device)
        labels = sample["kin"].to(device).float()
        output, p_f, c_f = model(parent_image.float(), children_image.float())
        output, p_f, c_f = output.squeeze(1), p_f.squeeze(1).detach().cpu().numpy(), c_f.squeeze(1).detach().cpu().numpy()
        parent_features.append(p_f)
        child_features.append(c_f)
        output = torch.sigmoid(output)
        model_out += list(output.detach().cpu().numpy())
        targets += list(labels.detach().cpu().numpy())
    best_threshold = utils.load_json(f"{model_path}/{pair_type}_testing_fold_{fold}.json")["best_threshold"]

    targets = np.array(targets)
    y_hat = np.zeros_like(model_out)
    y_hat[np.array(model_out) > 0.5338352918624878] = 1
    acc = accuracy_score(targets, y_hat)
    print("Accuracy ", acc)


from dataclasses import dataclass
from typing import Optional

from app.store.bot.state import EventTypes

@dataclass
class Message:
    user_id: int
    peer_id: int
    text: str
    payload: Optional[dict]
    event_id: Optional[str]

    @classmethod
    def from_message_new(cls, message: dict) -> "Message":
        return cls(
            user_id=message["from_id"],
            peer_id=message["peer_id"],
            text=message["text"],
            payload=message["payload"] if "payload" in message.keys() else None,
            event_id=None,
        )

    @classmethod
    def from_message_event(cls, message: dict) -> "Message":
        if "payload" in message.keys():
            return cls(
                user_id=message["user_id"],
                peer_id=message["peer_id"],
                text=None,
                payload=message["payload"],
                event_id=message["event_id"],
            )

@dataclass
class UpdateObject:
    id: int
    body: Message

    @classmethod
    def from_message_new(cls, update_obj: dict) -> "UpdateObject":
        return cls(
            id=update_obj["message"]["id"],
            body=Message.from_message_new(update_obj["message"]),
        )

    @classmethod
    def from_message_event(cls, update_obj: dict) -> "UpdateObject":
        return cls(id=None, body=Message.from_message_event(update_obj))

@dataclass
class Update:
    type: str
    object: UpdateObject

    @classmethod
    def from_dict(cls, update: dict) -> "Update":
        if update["type"] == EventTypes.MESSAGE_NEW:
            return cls(
                type=update["type"],
                object=UpdateObject.from_message_new(update["object"]),
            )
        elif update["type"] == EventTypes.MESSAGE_EVENT:
            return cls(
                type=update["type"],
                object=UpdateObject.from_message_event(update["object"]),
            )

    @property
    def user_id(self):
        return self.object.body.user_id

    @property
    def peer_id(self):
        return self.object.body.peer_id

    @property
    def text(self):
        return self.object.body.text

    @property
    def payload(self):
        return self.object.body.payload


# 딕셔너리 선언
dict = {
    "name" : "DB-HDW",
    "type" : "computer",
    "element" : ["cpu", "gpu", "ram", "mainboard"]
}
print("딕셔너리 선언(고정) : dict =", dict, "\n")

# 딕셔너리 요소 제거
del dict["name"]
print('del dict["name"]')
print(dict, "\n")

dict = {
    "name" : "DB-HDW",
    "type" : "computer",
    "element" : ["cpu", "gpu", "ram", "mainboard"]
}

del dict[input("제거할  키값 입력 : ")]
print('del dict[input("제거할 키값 입력 : ")]')
print(dict)

dict = {
    "name" : "DB-HDW",
    "type" : "computer",
    "element" : ["cpu", "gpu", "ram", "mainboard"]
}

print('asd = input("딕셔너리 이름 :")')
asd = input("딕셔너리 이름 :")
del dict[asd]
print('del dict[asd]')
print(dict)


#!/bin/python3
import math
import os
import random
import re
import sys

#
# Complete the 'pickingNumbers' function below.
#
# The function is expected to return an INTEGER.
# The function accepts INTEGER_ARRAY a as parameter.
#

def pickingNumbers(a):
    counts = {i:0 for i in range(1, 100)}
    max_ = max(a)
    for i in a:
        counts[i] += 1
    max_count = 0
    for i in range(1, max_):
        if max_count < counts[i] + counts[i+1]:
            max_count = counts[i] + counts[i+1]
    return max_count

    x = sorted(counts.items(), key=lambda x: x[1], reverse=True)
    print(x)
    # Write your code here

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    n = int(input().strip())

    a = list(map(int, input().rstrip().split()))

    result = pickingNumbers(a)

    fptr.write(str(result) + '\n')

    fptr.close()


def processTextFile(inputFilePath, outputFilePath):
    with open(inputFilePath, 'r', encoding='utf-8') as inputFile:
        data = inputFile.read()

    lineCount = len(data.split('\n'))

    wordCount = len(data.split())

    charCount = len(data)

    resultStr = f"行数: {lineCount}\n单词数: {wordCount}\n字符数: {charCount}"

    with open(outputFilePath, 'w', encoding='utf-8') as outputFile:
        outputFile.write(resultStr)

    print("处理完成，结果已写入到{}文件中。".format(outputFilePath))

inputFilePath = 'Homework\Lecture8-Homework（files readind and writing)\input1.txt'
outputFilePath = 'Homework\Lecture8-Homework（files readind and writing)\output1.txt'
processTextFile(inputFilePath, outputFilePath)

Homework/Lecture8-Homework（files readind and writing)/1.py


#!env/bin/python

from slackclient import SlackClient
from importlib import import_module
from time import sleep
import config
import os

class Bot():
    def __init__(self, config):

        self.config = config
        self.connect()

    def connect(self):
        """ connect to slack """

        self.sc = SlackClient(self.config.slack["api_key"])
        self.users = self.sc.api_call("users.list")
        self.channel = "#general"
        self.message_text = "It’s noisy in here! :mega:"

        self.bot_user = self.get_user_id_by_name(
            self.config.slack["bot_name"])

        self.sc.api_call(
            "chat.postMessage",
            channel=self.channel,
            text=self.message_text,
            as_user=self.bot_user
        )

    def get_user_id_by_name(self, user_name):
        for user in self.users["members"]:
            if user["name"] == user_name:
                return user["id"]

noisebot = Bot(config)


#!/usr/bin/env python
# -*- coding: utf8 -*-
import  plac

from 	contextlib 		import closing

import 	sys

import	os

import 	json

from 	pwd 			import getpwnam

import	colorama

import	shared_infrastructure

import	nginx_configuration_fs

import	agnostic_configuration

import	ssl_configuration

import	url2entity_configuration

import	hook_server_configuration

__NGINX_CONFIGURATION_FS_CONFIG_TEMPLATE__ 	= 'nginx_configuration_fs.config.template'
__ROOT_AGNOSTIC_CONFIGURATION__			= 'root_agnostic_configuration'
__ROOT_SSL_CONFIGURATION__			= 'root_ssl_configuration'
__ROOT_URL2ENTITY_CONFIGURATION__		= 'root_url2entity_configuration'
__ROOT_HOOK_SERVER_CONFIGURATION__		= 'root_hook_server_configuration'
__USER_OWNER__					= 'user_owner'
__GROUP_OWNER__					= 'group_owner'
__RESOLVER_CONF__				= 'resolver_conf'
__MOUNT_FILENAME__				= 'mount_filename'
__UNMOUNT_FILENAME__				= 'unmount_filename'
__REDIRECT_FILENAME__				= 'redirect_filename'
__ERROR_STATUS_FILENAME__			= 'error_status_filename'
__RESTART_NGINX__				= 'restart_nginx'
__SSL_CERTIFICATE_FILENAME__			= 'ssl_certificate_filename'
__SSL_CERTIFICATE_KEY_FILENAME__		= 'ssl_certificate_key_filename'
__URL2ENTITY_FILENAME__				= 'url2entity_filename'
__HOOK_SERVER_FILENAMES_LIST__			= 'hook_server_filenames_list'

@plac.annotations(
    configuration_path	= 						\
        'path to formated file as %s' % 				\
            ( __NGINX_CONFIGURATION_FS_CONFIG_TEMPLATE__ ),
    mountpoint				= 				\
        'mountpoint, must be empty',
    named_mount_options			= 				\
        (
            'fs_mntops formated option (see man fstab)',
            'option',
            'o',
            None,
            None,
            'fs_mntops'
        ),
    nodaemon				= 				\
        (
            'no daemonize twisted FS. Default: False',
            'flag',
            'nodaemon',
        ),
)
def main_verify(
    configuration_path,
    mountpoint,
    named_mount_options		= '',
    nodaemon			= False,
    ):
    """
    - Analyse la presence de tous les parametres necessaires
    au programme.
    - Verifie des regles fontionnelles (repertoire non montee...).
    - Modifie les valeurs de fs_mntops passees et les rend coherentes.
    - Le parametre nodaemon a True permet de ne pas detacher les
    processus de la commande de depart. Ce parametre n'est pas
    acessible lors d'un montage par /etc/fstab.
    - renvoit sous forme de dictionnaire le parametres nodaemon
    et tous les parametres necessaires a l'execution du / des
    Daemon demarrer pat twisted.
    """

    if not os.path.isfile( configuration_path ):
        print( '%s not found' % ( configuration_path ) )
        sys.exit(1)

    if not os.path.isdir( mountpoint ):
        print( '%s is not a directory' % ( mountpoint ) )
        sys.exit(2)

    if os.listdir( mountpoint ):
        print( '%s is not empty' % ( mountpoint ) )
        sys.exit(3)

    d_config = {}

    with closing( open( configuration_path ) ) as f_configuration:
        d_config = json.load( f_configuration )

    if not d_config.has_key( __ROOT_AGNOSTIC_CONFIGURATION__ ) :
        print( '%s not found in %s' % ( __ROOT_AGNOSTIC_CONFIGURATION__, __NGINX_CONFIGURATION_FS__ ) )
        sys.exit(3)

    if not os.path.isdir( d_config[ __ROOT_AGNOSTIC_CONFIGURATION__ ] ):
        print( '%s is not a directory' % ( d_config[ __ROOT_AGNOSTIC_CONFIGURATION__ ] ) )
        sys.exit(4)

    if not d_config.has_key( __USER_OWNER__ ) :
        print( '%s not found in %s' % ( __USER_OWNER__, configuration_path )  )
        sys.exit(6)

    if not d_config.has_key( __GROUP_OWNER__ ) :
        print( '%s not found in %s' % ( __GROUP_OWNER__, configuration_path )  )
        sys.exit(7)

    try:
        uid_owner, gid_owner = 					\
            getpwnam( d_config[ __USER_OWNER__ ] )[ 2 ], 	\
            getpwnam( d_config[ __GROUP_OWNER__ ])[ 3 ]
    except:
        print( '%s.%s does not exist' % ( d_config[ __USER_OWNER__ ], d_config[ __GROUP_OWNER__ ] )  )
        sys.exit(8)

    if not d_config.has_key( __RESOLVER_CONF__ ) :
        print( '%s not found in %s' % ( __RESOLVER_CONF__, configuration_path )  )
        sys.exit(9)

    if not os.path.isfile( d_config[ __RESOLVER_CONF__ ] ):
        print( '%s not a file' % ( d_config[ __RESOLVER_CONF__ ] ) )
        sys.exit(10)

    if not d_config.has_key( __MOUNT_FILENAME__ ) :
        print( '%s not found in %s' % ( __MOUNT_FILENAME__, configuration_path )  )
        sys.exit(11)

    if not d_config.has_key( __UNMOUNT_FILENAME__ ) :
        print( '%s not found in %s' % ( __UNMOUNT_FILENAME__, configuration_path )  )
        sys.exit(12)

    if not d_config.has_key( __REDIRECT_FILENAME__ ) :
        print( '%s not found in %s' % ( __REDIRECT_FILENAME__, configuration_path )  )
        sys.exit(13)

    if not d_config.has_key( __ERROR_STATUS_FILENAME__ ) :
        print( '%s not found in %s' % ( __ERROR_STATUS_FILENAME__, configuration_path )  )
        sys.exit(14)

    if not d_config.has_key( __RESTART_NGINX__ ) :
        print( '%s not found in %s' % ( __RESTART_NGINX__, configuration_path )  )
        sys.exit(15)

    if not d_config.has_key( __ROOT_SSL_CONFIGURATION__ ) :
        print( '%s not found in %s' % ( __ROOT_SSL_CONFIGURATION__, configuration_path )  )
        sys.exit(16)

    if not os.path.isdir( d_config[ __ROOT_SSL_CONFIGURATION__ ] ):
        print( '%s is not a directory' % ( d_config[ __ROOT_SSL_CONFIGURATION__ ] ) )
        sys.exit(21)

    if not d_config.has_key( __SSL_CERTIFICATE_FILENAME__ ) :
        print( '%s not found in %s' % ( __SSL_CERTIFICATE_FILENAME__, configuration_path )  )
        sys.exit(17)

    if not d_config.has_key( __SSL_CERTIFICATE_KEY_FILENAME__ ) :
        print( '%s not found in %s' % ( __SSL_CERTIFICATE_KEY_FILENAME__, configuration_path )  )
        sys.exit(18)

    if not d_config.has_key( __ROOT_URL2ENTITY_CONFIGURATION__ ) :
        print( '%s not found in %s' % ( __ROOT_URL2ENTITY_CONFIGURATION__, configuration_path )  )
        sys.exit(19)

    if not os.path.isdir( d_config[ __ROOT_URL2ENTITY_CONFIGURATION__ ] ):
        print( '%s is not a directory' % ( d_config[ __ROOT_URL2ENTITY_CONFIGURATION__ ] ) )
        sys.exit(21)

    if not d_config.has_key( __URL2ENTITY_FILENAME__ ) :
        print( '%s not found in %s' % ( __URL2ENTITY_FILENAME__, configuration_path )  )
        sys.exit(20)

    if not d_config.has_key( __ROOT_HOOK_SERVER_CONFIGURATION__ ) :
        print( '%s not found in %s' % ( __ROOT_HOOK_SERVER_CONFIGURATION__, configuration_path )  )
        sys.exit(22)

    if not os.path.isdir( d_config[ __ROOT_HOOK_SERVER_CONFIGURATION__ ] ):
        print( '%s is not a directory' % ( d_config[ __ROOT_HOOK_SERVER_CONFIGURATION__ ] ) )
        sys.exit(24)

    if not d_config.has_key( __HOOK_SERVER_FILENAMES_LIST__ ) :
        print( '%s not found in %s' % ( __HOOK_SERVER_FILENAMES_LIST__, configuration_path )  )
        sys.exit(23)

    if not isinstance( d_config[ __HOOK_SERVER_FILENAMES_LIST__ ], list ):
        print( '%s must be a list in %s' % ( __HOOK_SERVER_FILENAMES_LIST__, configuration_path )  )
        sys.exit(25)

    return 							\
        {							\
            'uid_owner'			: uid_owner,		\
            'gid_owner'			: gid_owner,		\
            'd_config'			: d_config,		\
            'configuration_path'	: configuration_path,   \
            'mountpoint'		: mountpoint,		\
            'named_mount_options'	: named_mount_options,	\
            'nodaemon'			: nodaemon,		\
        }

def main_process(
    uid_owner,
    gid_owner,
    d_config,
    configuration_path,
    mountpoint,
    named_mount_options,
    **kwargs
):

    ssl_conf 	= 					\
        ssl_configuration.SSLConfiguration(
            d_config[ __ROOT_SSL_CONFIGURATION__ ],
            d_config[ __RESOLVER_CONF__ ],
            d_config[ __SSL_CERTIFICATE_FILENAME__ ],
            d_config[ __SSL_CERTIFICATE_KEY_FILENAME__ ],
            d_config[ __RESTART_NGINX__ ],
        )

    fuse 		= 					\
        shared_infrastructure.ContextualizedFUSE(
            nginx_configuration_fs.NGINXConfigurationFS(
                agnostic_configuration.AgnosticConfiguration(
                    d_config[ __ROOT_AGNOSTIC_CONFIGURATION__ ],
                    d_config[ __RESOLVER_CONF__ ],
                    d_config[ __MOUNT_FILENAME__ ],
                    d_config[ __UNMOUNT_FILENAME__ ],
                    d_config[ __REDIRECT_FILENAME__ ],
                    d_config[ __RESTART_NGINX__ ],
                    ssl_conf
                ),
                mountpoint,
                uid_owner,
                gid_owner,
                d_config[ __RESOLVER_CONF__ ],
                d_config[ __MOUNT_FILENAME__ ],
                d_config[ __UNMOUNT_FILENAME__ ],
                d_config[ __REDIRECT_FILENAME__ ],
                d_config[ __ERROR_STATUS_FILENAME__ ],
                d_config[ __RESTART_NGINX__ ],
                ssl_conf,
                url2entity_configuration.URL2EntityConfiguration(
                    d_config[ __ROOT_URL2ENTITY_CONFIGURATION__ ],
                    d_config[ __RESOLVER_CONF__ ],
                    d_config[ __URL2ENTITY_FILENAME__ ],
                    d_config[ __RESTART_NGINX__ ],
                    ssl_conf
                ),
                d_config[ __URL2ENTITY_FILENAME__ ],
                hook_server_configuration.HookServerConfiguration(
                    d_config[ __ROOT_HOOK_SERVER_CONFIGURATION__ ],
                    d_config[ __RESOLVER_CONF__ ],
                    d_config[ __HOOK_SERVER_FILENAMES_LIST__ ],
                    d_config[ __RESTART_NGINX__ ],
                ),
            ),
            mountpoint,
            named_mount_options,
            **kwargs
        )

if __name__ == '__main__':

    shared_infrastructure.DaemonRunner(
        le_preApplication	= 				\
            lambda: 						\
                plac.call( 					\
                    main_verify, 				\
                    eager 	= False				\
                    # eager = Falase sinon le tableau renovye
                    # devient une liste
                ),						\
        l_le_startApplication	= 				\
            [							\
                lambda startApplicationParams, l_process: 	\
                    shared_infrastructure.TwistedDaemon( 	\
                        '''
import  nginx_configuration_fs.main

kwargs = %r

nginx_configuration_fs.main.main_process(
   **kwargs
)''',								\
                        l_process,				\
                        **startApplicationParams 		\
                    ).run(),					\
            ]
    ).run()


from src.reseau.types.ObjectItemQuantityPriceDateEffects import ObjectItemQuantityPriceDateEffects
from src.reseau.types.ObjectItemQuantityPriceDateEffects import ObjectItemQuantityPriceDateEffects

class ExchangeOfflineSoldItemsMessage:
   def __init__(self,input):
      self.bidHouseItems = []
      self.merchantItems = []
      _item1 = None
      _item2 = None
      _bidHouseItemsLen = input.readUnsignedShort()
      for _i1 in range(0,_bidHouseItemsLen):
         _item1 = ObjectItemQuantityPriceDateEffects(input)
         self.bidHouseItems.append(_item1)
      _merchantItemsLen = input.readUnsignedShort()
      for _i2 in range(0,_merchantItemsLen):
         _item2 = ObjectItemQuantityPriceDateEffects(input)
         self.merchantItems.append(_item2)

   def resume(self):
      for e in self.bidHouseItems:
         e.resume()
      for e in self.merchantItems:
         e.resume()


#!/usr/bin/python
# -*- coding: UTF-8 -*

from config import Config as conf
from model import XYL2RGB
from scipy.io import loadmat as load
from data_pro import decode_from_tfrecords,get_batch
from utils import conv2d, deconv2d, linear, batch_norm, lrelu
import tensorflow as tf
import scipy.io as sio
import numpy as np
import time
import sys
import os

def train():
    dict = load(r'F:\WH\Test\center_max.mat')

    xyl_train = dict["input_train"]
    rgb_train = dict["output_train"]
    xyl_test = dict["input_test"]
    rgb_test = dict["output_test"]

    print(xyl_train.shape)
    print(rgb_train.shape)
    print(xyl_test.shape)
    print(rgb_test.shape)
#    xyl_train = np.reshape(xyl_train,[train_shape,1,3,1])
#    rgb_train =np.reshape(rgb_train,[train_shape,1,3,1])
#
#    xyl_test = np.reshape(xyl_test,[test_shape,1,3,1])
#    rgb_test =np.reshape(rgb_test,[test_shape,1,3,1])

    global_step=tf.Variable(0,trainable=False)
    counter = global_step.assign_add(1)
    lr=tf.train.exponential_decay(0.1,global_step,100000,0.1,staircase=True)
    tf.summary.scalar('lr',lr)

    model = XYL2RGB()

    # xyls,rgbs=get_batch(xyl_train,rgb_train,28)

    opt = tf.train.GradientDescentOptimizer(lr).minimize(model.loss)

    saver = tf.train.Saver()

    start_time = time.time()
    if not os.path.exists(conf.data_path + "/checkpoint"):
        os.makedirs(conf.data_path + "/checkpoint")

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    with tf.Session(config=config) as sess:
        if conf.model_path == "":
            sess.run(tf.global_variables_initializer())
        else:
            saver.restore(sess, conf.model_path)
        merged_summary_op = tf.summary.merge_all()
        summary_writer = tf.summary.FileWriter('./logs/t_56', sess.graph)

        # coord = tf.train.Coordinator()
        # threads = tf.train.start_queue_runners(sess, coord)
        for i in range(100000):
            # xyl_batch_v, rgb_batch_v = sess.run([xyls, rgbs])
            # _, m = sess.run([opt, model.loss], feed_dict={model.xyl:xyl_batch_v, model.rgb:rgb_batch_v})
            _, m = sess.run([opt, model.loss], feed_dict={model.xyl: xyl_train, model.rgb: rgb_train})
            step = sess.run(counter)
            if (i+1) % 100  == 0:

                print("Iterate [%d]: time: %4.4f, train_loss: %.8f" % (i+1, time.time() - start_time, m))
                m_test = sess.run(model.loss, feed_dict={model.xyl:xyl_test, model.rgb:rgb_test})

                print("Iterate [%d]: time: %4.4f, test_loss: %.8f" % (i + 1, time.time() - start_time, m_test))

                summary_str = sess.run(merged_summary_op,feed_dict={model.xyl:xyl_test, model.rgb:rgb_test})
                summary_writer.add_summary(summary_str, i+1)
            if (i+1) % 1000  == 0:
                save_path = saver.save(sess, conf.data_path + "/checkpoint56/" + "model_%d.ckpt" % (i+1))
        # coord.request_stop()
        # coord.join(threads)

def train_shuffle_batch():
    dict = load(r'F:\WH\Test\single_8_14_x.mat')

    xyl = dict["input_train"]
    rgb = dict["output_train"]
    xyl_test = dict["input_test"]
    rgb_test = dict["output_test"]

    xyls,rgbs=get_batch(xyl,rgb,conf.batch_size)#batch

    predict = predictor(xyls)
    loss = tf.reduce_mean(tf.reduce_sum(tf.square(rgb - predict),1))
    tf.summary.scalar('loss',loss)

    opt = tf.train.GradientDescentOptimizer(0.1).minimize(loss)

    saver = tf.train.Saver()

    #counter = 0
    start_time = time.time()
    if not os.path.exists(conf.data_path + "/checkpoint"):
        os.makedirs(conf.data_path + "/checkpoint")

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    with tf.Session(config=config) as sess:
        if conf.model_path == "":
            sess.run(tf.global_variables_initializer())
        else:
            saver.restore(sess, conf.model_path)
        merged_summary_op = tf.summary.merge_all()
        summary_writer = tf.summary.FileWriter('./logs/t_1', sess.graph)

        coord = tf.train.Coordinator()  #创建一个协调器，管理线程
        #启动QueueRunner, 此时文件名队列已经进队。
        threads=tf.train.start_queue_runners(sess=sess,coord=coord)

        for epoch in xrange(2000000):
            _, m = sess.run([opt, loss])

            if (epoch + 1) % 100  == 0:
                summary_str = sess.run(merged_summary_op)
                summary_writer.add_summary(summary_str, epoch+1)
                print("Iterate [%d]: time: %4.4f, loss: %.8f" % (epoch+1, time.time() - start_time, m))
            if (epoch + 1) % 100000  == 0:
                save_path = saver.save(sess, conf.data_path + "/checkpoint/" + "model_%d.ckpt" % (epoch+1))

        coord.request_stop()
        coord.join(threads)

if __name__ == "__main__":
    if len(sys.argv) > 1 and sys.argv[1] == 'gpu=':
        os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
        os.environ["CUDA_VISIBLE_DEVICES"]=str(sys.argv[1][4:])
    else:
        os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
        os.environ["CUDA_VISIBLE_DEVICES"]=str(0)
    train()
    # train_shuffle_batch()


import pandas as pd
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error

home_data_path = 'data/train.csv'

home_data = pd.read_csv(home_data_path)

# explore the data basic properties

# print(home_data.describe())

# print(home_data.columns)

# print(home_data.head())
# home_data = home_data.dropna(axis = 0)

# print(home_data.dropna(axis=0))
# selecting the prediction Target

y = home_data.SalePrice

house_features = ['MSSubClass', 'MSZoning', 'LotFrontage', 'LotArea', 'Street',
       'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig',
       'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType',
       'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd',
       'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType',
       'MasVnrArea', 'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual',
       'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1',
       'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'Heating',
       'HeatingQC', 'CentralAir', 'Electrical', '1stFlrSF', '2ndFlrSF',
       'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
       'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual',
       'TotRmsAbvGrd', 'Functional', 'Fireplaces', 'FireplaceQu', 'GarageType',
       'GarageYrBlt', 'GarageFinish', 'GarageCars', 'GarageArea', 'GarageQual',
       'GarageCond', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF',
       'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'PoolQC',
       'Fence', 'MiscFeature', 'MiscVal', 'MoSold', 'YrSold', 'SaleType',
       'SaleCondition']
X = home_data[house_features]

home_model = DecisionTreeRegressor()

X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)

home_model.fit(X_train, y_train)

y_prediction = home_model.predict(X_test)

print(mean_absolute_error(y_test, y_prediction))


import torch
from torch import optim
from torch import nn
from torch.autograd import Variable
from torchvision import datasets, transforms, models
from collections import OrderedDict

OUTPUT_SIZE = 102
PRINT_EVERY = 40

def get_loaders(data_dir):
    """
    Return dataloaders for training, validation and teting datasets.
    """
    train_dir = data_dir + '/train/'
    valid_dir = data_dir + '/valid/'
    test_dir = data_dir + '/test/'

    data_transforms = {
        'training' : transforms.Compose([transforms.RandomResizedCrop(224),
                                        transforms.RandomHorizontalFlip(),transforms.RandomRotation(30),
                                        transforms.ToTensor(),
                                        transforms.Normalize([0.485, 0.456, 0.406],
                                                             [0.229, 0.224, 0.225])]),

        'validation' : transforms.Compose([transforms.Resize(256),
                                          transforms.CenterCrop(224),
                                          transforms.ToTensor(),
                                          transforms.Normalize([0.485, 0.456, 0.406],
                                                               [0.229, 0.224, 0.225])]),

        'testing' : transforms.Compose([transforms.Resize(256),
                                          transforms.CenterCrop(224),
                                          transforms.ToTensor(),
                                          transforms.Normalize([0.485, 0.456, 0.406],
                                                               [0.229, 0.224, 0.225])])
    }

    image_datasets = {
        'training' : datasets.ImageFolder(train_dir, transform=data_transforms['training']),
        'testing' : datasets.ImageFolder(test_dir, transform=data_transforms['testing']),
        'validation' : datasets.ImageFolder(valid_dir, transform=data_transforms['validation'])
    }

    dataloaders = {
        'training' : torch.utils.data.DataLoader(image_datasets['training'], batch_size=64, shuffle=True),
        'testing' : torch.utils.data.DataLoader(image_datasets['testing'], batch_size=64, shuffle=False),
        'validation' : torch.utils.data.DataLoader(image_datasets['validation'], batch_size=64, shuffle=True)
    }

    class_to_idx = image_datasets['training'].class_to_idx
    return dataloaders, class_to_idx

def get_model(arch, hidden_units):
    '''
    Load pretrained model
    '''
    if arch == 'vgg13_bn':
        model = models.vgg13_bn(pretrained=True)
        input_size = model.classifier[0].in_features
    elif arch == 'vgg16_bn':
        model = models.vgg16_bn(pretrained=True)
        input_size = model.classifier[0].in_features
    elif arch == 'vgg19_bn':
        model = models.vgg19_bn(pretrained=True)
        input_size = model.classifier[0].in_features
    elif arch == 'densenet121':
        model = models.densenet121(pretrained=True)
        input_size = model.classifier.in_features
    elif arch == 'densenet161':
        model = models.densenet161(pretrained=True)
        input_size = model.classifier.in_features
    elif arch == 'densenet201':
        model = models.densenet201(pretrained=True)
        input_size = model.classifier.in_features
    elif arch == 'resnet18':
        model = models.resnet18(pretrained=True)
        input_size = model.fc.in_features
    elif arch == 'resnet34':
        model = models.resnet34(pretrained=True)
        input_size = model.fc.in_features
    elif arch == 'resnet50':
        model = models.resnet50(pretrained=True)
        input_size = model.fc.in_features
    else:
        raise Exception("Unknown model")

    for param in model.parameters():
        param.requires_grad = False

    output_size = OUTPUT_SIZE

    classifier = nn.Sequential(OrderedDict([
        ('fc1', nn.Linear(input_size, hidden_units)),
        ('relu', nn.ReLU()),
        ('fc2', nn.Linear(hidden_units, output_size)),
        ('output', nn.LogSoftmax(dim=1))
    ]))

    if 'vgg' in arch:
        model.classifier = classifier
    elif 'densenet' in arch:
        model.classifier = classifier
    elif 'resnet' in arch:
        model.fc = classifier

    return model

def build_model(arch, hidden_units, learning_rate):
    '''
    Build the pretrained model
    '''
    model = get_model(arch, hidden_units)
    print("Pretrained model retrieved.")

    # Set the parameters
    parameters = filter(lambda p: p.requires_grad, model.parameters())
    optimizer = optim.Adam(parameters, lr=learning_rate)
    optimizer.zero_grad()
    criterion = nn.NLLLoss()
    return model, optimizer, criterion

def train_model(model, epochs, criterion, optimizer, train_loader,
                val_loader, use_gpu):
    '''
    Train the model
    '''
    print("Beginning model training")
    model.train()
    print_every = PRINT_EVERY
    steps = 0

    for epoch in range(epochs):
        running_loss = 0

        # Get inputs and labels
        for inputs, labels in iter(train_loader):
            steps += 1

            # Move to GPU
            if use_gpu:
                inputs = Variable(inputs.float().cuda())
                labels = Variable(labels.long().cuda())
            else:
                inputs = Variable(inputs)
                labels = Variable(labels)

            optimizer.zero_grad()
            output = model.forward(inputs)
            loss = criterion(output, labels)
            loss.backward()
            optimizer.step()
            running_loss += loss.data[0]
            if steps % print_every == 0:
                val_loss, val_accuracy = validate(model, criterion, val_loader, use_gpu)
                model.train()
                print("Epoch: {}/{} ".format(epoch+1, epochs),
                      "Train Loss: {:.3f} ".format(running_loss/print_every),
                      "Val. Loss: {:.3f} ".format(val_loss),
                      "Val. Acc.: {:.3f}".format(val_accuracy))
                running_loss = 0

def validate(model, criterion, data_loader, use_gpu):
    '''
    Validate the model
    '''
    print("Validating the model")
    model.eval()
    accuracy = 0
    test_loss = 0
    for inputs, labels in iter(data_loader):
        if use_gpu:
            inputs = Variable(inputs.float().cuda(), volatile=True)
            labels = Variable(labels.long().cuda(), volatile=True)
        else:
            inputs = Variable(inputs, volatile=True)
            labels = Variable(labels, volatile=True)
        output = model.forward(inputs)
        test_loss += criterion(output, labels).data[0]
        ps = torch.exp(output).data
        measure = (labels.data == ps.max(1)[1])
        accuracy += measure.type_as(torch.FloatTensor()).mean()

    return test_loss/len(data_loader), accuracy/len(data_loader)


from sklearn import linear_model
import numpy as np
import math
import matplotlib.pyplot as plt
from datetime import datetime
from sklearn.svm import l1_min_c
from pylab import scatter, show, legend, xlabel, ylabel

# https://www.youtube.com/watch?v=-BQCB6Uch1g
# from __future__ import division

def logistic_func(theta, x):
  return float(1) / (1 + math.e**(-x.dot(theta)))

def log_gradient(theta, x, y):
  first_calc = logistic_func(theta, x) - np.squeeze(y)
  final_calc = first_calc.T.dot(x)
  return final_calc

def cost_func(theta, x, y):
  log_func_v = logistic_func(theta,x)
  y = np.squeeze(y)
  step1 = y * np.log(log_func_v)
  step2 = (1-y) * np.log(1 - log_func_v)
  final = -step1 - step2
  return np.mean(final)

def grad_desc(theta_values, X, y, lr=.001, limit=10):
  #normalize
  X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
  #setup cost iter
  cost_iter = []
  i = 0
  while(i < limit):
    # old_cost = cost
    theta_values = theta_values - (lr * log_gradient(theta_values, X, y))
    cost = cost_func(theta_values, X, y)
    cost_iter.append([i, cost])
    i+=1
  return theta_values, np.array(cost_iter)

def pred_values(theta, X, hard=True):
  #normalize
  X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
  pred_prob = logistic_func(theta, X)
  pred_value = np.where(pred_prob >= .5, 1, 0)
  if hard:
    return pred_value
  return pred_prob

def logistic_regression(Y, X, X_validation=[],Y_validation=[], limit=100):
  shape = X.shape[1]
  print("Dimension = (",shape,")")
  betas = np.zeros(shape)
  fitted_values, cost_iter = grad_desc(betas, X, Y,limit=limit)
  # print("Fit values =",fitted_values)

  predicted_y = pred_values(fitted_values, X)
  print("Y =",Y)
  print("Predict Y =",predicted_y)

  score_y = np.sum(Y)

  print("Y =",score_y)
  print("Predict Y =", np.sum(predicted_y))
  print("Compare predict_y and Y =", np.sum(Y == predicted_y), " values are equal")

  print(cost_iter)
  plt.plot(cost_iter[:,0], cost_iter[:,1])
  plt.ylabel("Cost")
  plt.xlabel("Iteration")
  plt.show()
  plt.savefig('cost_iter.png')

  #normalize data
  # norm_X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
  # myargs = (norm_X, Y)
  # betas = np.zeros(norm_X.shape[1])

  # logreg = linear_model.LogisticRegression()
  # logreg.fit(norm_X, Y)
  # print("From 'sklearn' --> logreg.predict == Y =", sum(Y == logreg.predict(norm_X,)))
  # print(logreg.score(norm_X, Y))

  # print("Computing regularization path ...")
  # start = datetime.now()
  # clf = linear_model.LogisticRegression(C=1.0, penalty='l1', tol=1e-6)
  # # coefs_ = []
  # # cs = l1_min_c(X, Y, loss='log') * np.logspace(0, 3)
  #
  # cost_iter = []
  # i = 0
  # # for c in cs:
  # clf.fit(X, Y)
  # while (i < limit):
  #   # clf.set_params(C=c)
  #   # coefs_.append(clf.coef_.ravel().copy())
  #   cost = float(sum(Y_validation == clf.predict(X_validation,)))/float(score_y)
  #   cost_iter.append([i, cost])
  #   print(i, cost, flush=True)
  #   i += 1
  # print("This took ", datetime.now() - start)

  # coefs_ = np.array(coefs_)
  # plt.plot(np.log10(cs), coefs_)
  # ymin, ymax = plt.ylim()
  # plt.xlabel('log(C)')
  # plt.ylabel('Coefficients')
  # plt.title('Logistic Regression Path')
  # plt.axis('tight')
  # plt.show()
  # plt.savefig("cost_iter_second.png")

  # print(cost_iter)
  # cost_iter = np.array(cost_iter)
  # plt.plot(cost_iter[:,0], cost_iter[:,1],'-', linewidth=3)
  # plt.ylabel("Cost")
  # plt.xlabel("Iteration")
  # plt.xlim(0, len(cost_iter[:,0]))
  # # plt.ylim(0, 100)
  # plt.show()
  # plt.savefig('cost_iter.png')

  # fitted_values, cost_iter = grad_desc(betas, norm_X, Y)
  # predicted_y = pred_values(fitted_values, norm_X)
  # print("From 'my function' --> Predict Y =",sum(predicted_y == Y))
  #
  # plt.plot(cost_iter[:,0], cost_iter[:,1],'-', linewidth=3)
  # plt.ylabel("Cost")
  # plt.xlabel("Iteration")
  # plt.show()


this_script = __file__

#calculate the [CII] column density, channel-by-channel
import numpy as np
import pandas as pd
import upGREATUtils
from scipy.interpolate import interp1d
from scipy.special import erf
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import seaborn as sns

plt.rcParams['font.family']='serif'
plt.rcParams['mathtext.rm'] = 'serif'
plt.rcParams['mathtext.fontset'] = 'cm'
plt.rcParams['font.size'] = 14

#define some key functions

def B_T(T,SigRuln):
	#Crawford+1985, Goldsmith+2012, Tarantino+2021
	Aul = 2.3E-6 #s^-1
	return 2*np.exp(-91.2/T)/(1+2*np.exp(-91.2/T)+Aul/SigRuln)

def SigRuln(T,n_CNM):
	Rul_H0 = 4.0E-11*(16+0.35*T**0.5+48*T**-1) #Goldsmith+2012
	return 1.038*Rul_H0*n_CNM #Draine+2011

def NCp_from_TpCII(Tp_CII,dv,B):
	#Crawford+1985, Goldsmith+2012, Tarantino+2021
	return dv*Tp_CII/B*3.0E15

def NHI_CII(NCp_from_TpCII,f_CNM):
	C_H = 1.5E-4 #C/H, Gerin+2015
	#assumes all C singly-ionized and all H atomic
	return NCp_from_TpCII/C_H/f_CNM

def MCp_from_NCp(N_Cp,A_pix_cm2):
	m_Cp = 1.99E-23 #g
	Msun = 1.99E33 #g
	return N_Cp*A_pix_cm2*m_Cp/Msun #Msun

#define some fixed parameters
D_pix = 14.1 #arcsec
d = 3.63E6 #pc
A_pix = np.pi*(D_pix/2/206265*d)**2 #pc^2
A_pix_cm2 = A_pix*(3.086E18)**2 #cm^2
T = 100. #K, T_CNM
n_CNM = 100. #cm^-3
dv = 10. #km/s
f_CNM = 0.5
n_pixels = 7
n_rows = int(np.ceil(np.sqrt(n_pixels)))
n_cols = int(np.floor(n_pixels/n_rows)*n_rows)
pix_order_labels =  np.array([1,6,2,0,5,3,4])

#set up plotting colors, same as in pltCompositeImage.py
cii_color = sns.color_palette('crest',as_cmap=True)(150)
cii_color_dark = sns.color_palette('crest',as_cmap=True)(225)
co_color = '0.33'
hi_color = sns.color_palette('flare',as_cmap=True)(150)

fig = plt.figure(1,figsize=(8,8))
plt.clf()
gs = gridspec.GridSpec(n_rows,n_cols)

start_cols = [1,3,0,2,4,1,3]

#EVENTUALLY DONT HARD CODE THIS
rms = np.array([8.3, 9.4, 9.6, 6.9, 6.8, 16.8, 11.5])/1E3 #K

Nn_cii = []
Mm_cii = []
eu_Mm_cii = []
el_Mm_cii = []
med_ratios = []
peak_ratios = []

for i in range(n_pixels):
	this_row = int(np.floor((i+1)/n_rows))

	#load the CII data from the fits file
	cii_spec,cii_vel,header,_ = upGREATUtils.load_upgreat_data('../Data/Outflow_Pointings/CII/outflow1_pix'+str(pix_order_labels[i])+'.fits')
	Vclip_red,Vclip_blue = upGREATUtils.get_bad_edge_chans(1,pix_order_labels[i])
	cii_vel,cii_spec = upGREATUtils.rm_bad_edge_chans(cii_vel,cii_spec,which_end='both',Vred=Vclip_red,Vblue=Vclip_blue)

	cii_vel = np.flip(cii_vel)
	cii_spec = np.flip(cii_spec)
	#cii_spec_norm = cii_spec/np.nanmax(cii_spec)
	cii_dv = header['CDELT1']/header['RESTFREQ']*2.9979E5 #km/s
	cii_vix = np.where((cii_vel>=-100) & (cii_vel <= 400))
	cii_max = np.nanmax(cii_spec[cii_vix])

	n_edge = 6
	cii_vel = cii_vel[0:-n_edge]
	cii_spec = cii_spec[0:-n_edge]
	cii_vmax = np.nanmax(cii_vel)
	cii_vmin = np.nanmin(cii_vel)

	#load the HI data
	hi_dat = pd.read_csv('../Data/Outflow_Pointings/HI/outflow1_pix'+str(pix_order_labels[i])+'_smo_summed.csv')
	hi_vel = np.flip(hi_dat['Velocity_kms'].values)
	hi_spec = np.flip(hi_dat['Intensity_K'].values)
	idx = np.where((hi_vel <= cii_vmax) & (hi_vel >= cii_vmin))
	hi_vel = hi_vel[idx]
	hi_spec = hi_spec[idx]
	#interpolate hi onto cii velocity grid
	hi_int = interp1d(hi_vel,hi_spec,fill_value='extrapolate')
	hi_spec = hi_int(cii_vel)

	#now calculate the CII column density at each channel
	N_cii = NCp_from_TpCII(cii_spec,dv,B_T(T,SigRuln(T,n_CNM))) #cm^-2
	N_hi_cii = NHI_CII(N_cii,f_CNM)

	#get uncertinaties
	N_cii_max = NCp_from_TpCII(cii_spec+rms[pix_order_labels[i]],dv,B_T(T,SigRuln(T,n_CNM))) #cm^-2
	N_hi_cii_max = NHI_CII(N_cii_max,f_CNM)
	N_cii_min = NCp_from_TpCII(cii_spec-rms[pix_order_labels[i]],dv,B_T(T,SigRuln(T,n_CNM))) #cm^-2
	N_hi_cii_min = NHI_CII(N_cii_min,f_CNM)

	#sum the CII column
	N_cii_sum = np.nansum(N_cii)
	N_cii_max_sum = np.nansum(N_cii_max)
	N_cii_min_sum = np.nansum(N_cii_min)
	Nn_cii.append(N_cii_sum)
	this_M_cii = MCp_from_NCp(N_cii_sum,A_pix_cm2) #Msun
	this_M_cii_max = MCp_from_NCp(N_cii_max_sum,A_pix_cm2) #Msun
	this_M_cii_min = MCp_from_NCp(N_cii_min_sum,A_pix_cm2) #Msun
	Mm_cii.append(this_M_cii) #Msun
	eu_Mm_cii.append(this_M_cii_max-this_M_cii)
	el_Mm_cii.append(this_M_cii-this_M_cii_min)

	#convert the HI intensity back to a column density
	N_hi = hi_spec*dv*1.823E18 #cm^-2

	#take the ratio
	ratio_N_hi_cii = N_hi/N_hi_cii
	#only find the ratio where there's signal in the HI
	vix = np.where((cii_vel > 25.) & (cii_vel < 300.))
	med_ratio_N_hi_cii = np.nanmedian(ratio_N_hi_cii[vix])
	med_ratios.append(med_ratio_N_hi_cii)

	#find the ratio at the CII peak
	tt = N_hi_cii.copy()
	tt[cii_vel < 25] = np.nan
	tt[cii_vel > 300] = np.nan
	ixp = np.nanargmax(tt)
	peak_ratio = np.nanmax(N_hi)/N_hi_cii[ixp]
	peak_ratios.append(peak_ratio)

	#
	norm = 1E20

	#also compare for different values of f_CNM
	N_hi_cii_70 = NHI_CII(N_cii,0.7)/dv
	N_hi_cii_30 = NHI_CII(N_cii,0.3)/dv
	d_fCNM = np.nanmedian(N_hi_cii_30[vix]/norm-N_hi_cii_70[vix]/norm)
	d_fCNM_mid = np.nanmedian(N_hi_cii_30[vix]/norm-N_hi_cii[vix]/dv/norm)

	#plot the data
	ax = plt.subplot(gs[this_row,start_cols[i]:start_cols[i]+2],)
	c2=ax.fill_between(cii_vel[vix],N_hi[vix]/dv/norm,color=hi_color,alpha=0.1,step='pre')
	c0=ax.fill_between(cii_vel[vix],N_hi_cii[vix]/dv/norm,color=cii_color,alpha=0.5,step='pre')
	c3,=ax.step(cii_vel,N_hi/dv/norm,color=hi_color,lw=1.75,label='$N_{\mathrm{HI}}$')#\\%.1e' %(1/hi_sf))
	c1,=ax.step(cii_vel,N_hi_cii/dv/norm,color=cii_color,lw=1.5,label='$N_{\mathrm{HI}}^{\mathrm{[CII]}}$')

	#add errorbars for different f_cnm values
	xx = -120; yy = 3.25;
	ax.plot([xx,xx],[yy,yy-d_fCNM],color=cii_color,lw=1.5)
	ax.plot([xx-10,xx+10],[yy-d_fCNM_mid,yy-d_fCNM_mid],color=cii_color,lw=1.5)
	ax.plot([xx-5,xx+5],[yy,yy],color=cii_color,lw=1.5)
	ax.plot([xx-5,xx+5],[yy-d_fCNM,yy-d_fCNM],color=cii_color,lw=1.5)
	if i==0:
		ax.text(xx+10,yy,' $f_\mathrm{CNM}$=0.3',color=cii_color,
			ha='left',va='center',fontsize=plt.rcParams['font.size']-8)
		ax.text(xx+10,yy-d_fCNM_mid,' $f_\mathrm{CNM}$=0.5',color=cii_color,
			ha='left',va='center',fontsize=plt.rcParams['font.size']-8)
		ax.text(xx+10,yy-d_fCNM,' $f_\mathrm{CNM}$=0.7',color=cii_color,
			ha='left',va='center',fontsize=plt.rcParams['font.size']-8)

	ax.text(0.05,0.95,pix_order_labels[i],ha='center',va='top',transform=ax.transAxes)
	ax.text(0.975,0.975,'$\mathrm{\\frac{max(N_{HI})}{max(N_{HI}^{[CII]})}=%.1f}$' %(peak_ratio),
		ha='right',va='top',fontsize=plt.rcParams['font.size']-4,transform=ax.transAxes)

	if i == 1:
		leg = plt.legend([(c0,c1),(c2,c3)],[c1.get_label(),c3.get_label()],
			handlelength=1.25,fontsize=plt.rcParams['font.size']-2,loc='center left',bbox_to_anchor=(1.025,0.65,0.2,0.2))

	ax.set_xlim(-150,450)
	ax.set_ylim(-0.5,4.)
	ax.minorticks_on()
	ax.set_yticks([0,1,2,3,4])
	ax.grid(which='major',axis='y',alpha=0.5)

	if i==5:
		ax.set_xlabel('V$_{\mathrm{LSRK}}$ (km s$^{-1}$)')
		#ax.set_ylabel('N$_{\mathrm{HI}}$ ($\\times10^{21}$ cm$^{-2}$)')
	elif (i==0):
		ax.set_ylabel('$N\\times10^{%i}$\n(cm$^{-2}$ (km s$^{-1}$)$^{-1}$)' %np.log10(norm))
	elif (i==1) | (i==3) | (i==4) | (i==6):
		#ax.xaxis.set_ticklabels([])
		ax.yaxis.set_ticklabels([])
plt.savefig('../Plots/Outflow_Spectra/M82_ColumnDensity_CII_HI_Outflow1.pdf',bbox_inches='tight',metadata={'Creator':this_script})

Mmm_cii = Mm_cii.copy()
eu_Mmm_cii = eu_Mm_cii.copy()
el_Mmm_cii = el_Mm_cii.copy()
Mmm_cii[4]=0.
Mmm_cii[5]=0.
eu_Mmm_cii[4]=0.
el_Mmm_cii[5]=0.
eu_Mmm_cii[4]=0.
el_Mmm_cii[5]=0.
print('The total C+ mass in the outflow is %.2e Msun' %np.nansum(Mmm_cii))

tab = {'Pixel Number':pix_order_labels, 'ColumnDensity_Cp_cm2': Nn_cii,
	'Mass_Cp_Msun':Mm_cii,'e_Mass_Cp_Msun_upper':eu_Mm_cii,'e_Mass_Cp_Msun_lower':el_Mm_cii,
	'med_NHI_NHICII': med_ratios, 'peak_NHI_NHICII': peak_ratios}
df = pd.DataFrame.from_dict(tab)
df.to_csv('../Data/Outflow_Pointings/CII/outflow1_Mass_ColumnDensities.csv',index=False)

#now add the CO to the column density plot
del(leg)
for i in range(n_pixels):
	this_row = int(np.floor((i+1)/n_rows))

	#open the CO data
	co_dat = pd.read_csv('../Data/Outflow_Pointings/CO_TP/outflow1_pix'+str(pix_order_labels[i])+'_smo.csv')
	co_vel = co_dat['Velocity_kms'].values
	co_spec = co_dat['Intensity_K'].values
	idx = np.where((co_vel <= cii_vmax) & (co_vel >= cii_vmin))
	co_vel = co_vel[idx]
	co_spec = co_spec[idx]
	#interpolate hi onto cii velocity grid
	co_int = interp1d(co_vel,co_spec,fill_value='extrapolate')
	co_spec = co_int(cii_vel)

	#now convert the CO intensity to a H2 column density
	XCO = 0.5E20 #cm-2/(K km/s), starburst value from Bolatto+2013
	#XCO = 2E20 #cm-2/(K km/s), MW value from Bolatto+2013
	N_h2_co = XCO*co_spec*dv #cm^-2

	ax = plt.subplot(gs[this_row,start_cols[i]:start_cols[i]+2],)
	c4=ax.fill_between(cii_vel[vix],N_h2_co[vix]/dv/norm,color=co_color,alpha=0.1,step='pre')
	c5,=ax.step(cii_vel,N_h2_co/dv/norm,color=co_color,lw=1.75,label='$N_{\mathrm{H_{2}}}^{\mathrm{CO}}$')#\\%.1e' %(1/hi_sf))

	if i == 1:
		leg = plt.legend([(c0,c1),(c2,c3),(c4,c5)],[c1.get_label(),c3.get_label(),c5.get_label()],
			handlelength=1.25,fontsize=plt.rcParams['font.size']-2,loc='center left',bbox_to_anchor=(1.025,0.625,0.2,0.2))
plt.savefig('../Plots/Outflow_Spectra/M82_ColumnDensity_CII_HI_CO_Outflow1.pdf',bbox_inches='tight',metadata={'Creator':this_script})


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import regex as re
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import numpy as np
from numpy import random
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
import pandas as pd
from Preprocessing import *

# In[76]:

class models:

    def __init__(self, train_x, train_y, label_IDs, dev_x = None, dev_y = None, test_x = None, test_y = None):

        assert len(train_x) == len(train_y), "ERROR: train_x and train_y are of different length!"

        if dev_x is not None and dev_y is not None:
            assert len(dev_x) == len(dev_y), "ERROR: dev_x and dev_y are of different length!"

        if test_x is not None and test_y is not None:
            assert len(test_x) == len(test_y), "ERROR: test_x and test_y are of different length!"

        self.train_x = train_x
        self.train_y = train_y
        self.label_IDs = label_IDs

        self.dev_x = dev_x
        self.dev_y = dev_y

        self.test_x = test_x
        self.test_y = test_y

        self.Word2Vec = None
        self.MaxEnt = None

    def assert_len(self, x, y, split):
        if x is not None and y is not None:
            assert len(x) == len(y), "ERROR: %s_x and %s_y datasets are of different length!"%(split, split)

    def set_train_data(self, train_x, train_y):

        self.assert_len(train_x, train_y, "train")

        self.train_x = train_x
        self.train_y = train_y

    def set_test_data(self, test_x, test_y):

        self.assert_len(test_x, test_y, "test")

        self.test_x = test_x
        self.test_y = test_y

    def set_dev_data(self, dev_x, dev_y):

        self.assert_len(dev_x, dev_y, "dev")

        self.dev_x = dev_x
        self.dev_y = dev_y

    def train_Word2Vec(self):
        tagged_docs = [TaggedDocument(words = x, tags = str(y)) for x in self.train_x for y in self.train_y]

        x_merged = []

        for lst in self.train_x:
            x_merged.extend(lst)

        x_dict = {x: x_merged.count(x) for x in set(x_merged)}
        model_dm = Doc2Vec(dm = 1, vector_size = 300, epochs = 50)
        model_dm.build_vocab_from_freq(x_dict)
        model_dm.train(tagged_docs, total_examples=len(tagged_docs), epochs=model_dm.epochs)

        self.Word2Vec = model_dm

    def train_MaxEnt(self):

        assert self.Word2Vec is not None, "ERROR: Word2Vec model not initialized yet! Call models.train_Word2Vec() to initialize."

        try:
            train_x_vectorized = [self.Word2Vec.infer_vector(x) for x in self.train_x]
        except AssertionError:
            print("ERROR: Word2Vec model not initialized yet! Call models.train_Word2Vec() to initialize.")
            return

        logreg = LogisticRegression(max_iter = 1000, C=1e5)
        logreg.fit(train_x_vectorized, self.train_y)

        self.MaxEnt = logreg

    def save_Word2Vec(self):
        return self.Word2Vec

    def load_Word2Vec(self, Word2Vec):
        self.Word2Vec = Word2Vec

    def predict_sentence(self, sentence_list):

        assert self.Word2Vec is not None, "ERROR: Word2Vec model undefined! Train it by calling model.train_Word2Vec()"
        assert self.MaxEnt is not None, "ERROR: MaxEnt model undefined! Train it by calling model.train_MaxEnt()"

        sentence_vector = self.Word2Vec.infer_vector(sentence_list).reshape(1, -1)
        predict = self.MaxEnt.predict_proba(sentence_vector)[0]

        for i in range(0, len(predict)):
            print("Label %s (%s) probability: "%(i, self.label_IDs[i]))
            print(np.around(predict[i], decimals = 5))

    def test_MaxEnt(self, test_x = None, test_y = None, overwrite = True):

        self.assert_len(test_x, test_y, "test")

        assert self.test_x is not None and self.test_y is not None, "ERROR: test data undefined! Either set test data by calling models.set_test_data(self, test_x, test_y), or specify them as parameters in this method. (test_MaxEnt(self, test_x = test_x, test_y = test_y, overwrite = True/False.)) Set overwrite to True or False to overwrite existing test data."

        if test_x is not None and overwrite:
            self.test_x = test_x

        if test_y is not None and overwrite:
            self.test_y = test_y

        cur_test_x = None
        cur_test_y = None

        if overwrite:
            cur_test_x = self.test_x
            cur_test_y = self.test_y
        else:
            cur_test_x = test_x
            cur_test_y = test_y

        predicted = []
        actual = cur_test_y

        for x in range(0, len(cur_test_x)):
            #print(logreg.predict_proba(model_dm.infer_vector(x).reshape(1, -1)))
            predict = self.MaxEnt.predict(self.Word2Vec.infer_vector(cur_test_x[x]).reshape(1, -1))[0]
            predicted.append(predict)

        total_labels = set(cur_test_y)

        for label in total_labels:
            print("Stats for label %s (%s): " %(label, self.label_IDs[label]))
            print()

            TP = 0
            FP = 0
            FN = 0

            for i in range(0, len(predicted)):

                predicted_ID = int(predicted[i])

                if predicted_ID == label:
                    if predicted_ID == actual[i]:
                        TP += 1
                    else:
                        FP += 1
                elif predicted_ID == label:
                    FN += 1

            #print('TP: ', TP)
            #print('FP: ', FP)
            #print('FN: ', FN)

            precision = 0
            recall = 0
            F1 = 0

            if TP is not 0:

                precision = TP/(TP+FP)
                recall = TP/(TP+FN)
                F1 = 2*(precision*recall)/(precision+recall)

            print("Precision: ", precision)
            print("Recall: ", recall)
            print("F1: ", F1)
            print("_______________________________")

# In[68]:

#train_x_vectorized = [model_dm.infer_vector(x) for x in train_x]

# In[69]:

# logreg = LogisticRegression(max_iter = 1000, C=1e5)
# logreg.fit(train_x_vectorized, train_y)

# In[72]:

# for x in range(0, len(test_x)):
#     #print(logreg.predict_proba(model_dm.infer_vector(x).reshape(1, -1)))
#     predict = logreg.predict(model_dm.infer_vector(test_x[x]).reshape(1, -1))[0]
#     actual = test_y[x]

#     print(predict, ' ', actual)


from flask import Flask, render_template, request, flash, redirect, url_for
from forms import ContactForm
from flask_mail import Message, Mail

import sys
import pandas as pd
import numpy as np

from forex_python.converter import CurrencyRates

mail = Mail()

application = Flask(__name__)
application.secret_key = "SECRETKEY"

#Config mail for Feedback page
application.config["MAIL_SERVER"] = "smtp.gmail.com"
application.config["MAIL_PORT"] = 465
application.config["MAIL_USE_SSL"] = True
application.config["MAIL_USERNAME"] = 'ysivekum@gmail.com'
application.config["MAIL_PASSWORD"] = 'ocsxjlfzgnqjrmyt'

mail.init_app(application)

#route for the Home page
@application.route("/")
def home():
    return render_template('home.html')

#route for the Upload page
@application.route("/upload", methods=['GET', 'POST'])
def upload():
    if request.method == 'POST':
        print(request.files['file'])
        f = request.files['file']
        currency = request.form['currency']
        download = request.form.getlist('download')
        display = request.form.getlist('display')

        #check if file was uploaded
        if f.filename == '':
            flash(u'Please upload a file', 'confirmation')
            return redirect(url_for('upload'))

        #check if file is correct file type (csv)
        if f.filename.rsplit('.', 1)[1].lower() != 'csv':
            flash(u'Please upload a csv file', 'confirmation')
            return redirect(url_for('upload'))

        #checks if user wants to download and/or display results in browser
        downloadCheck = bool(download)
        displayCheck = bool(display)

        #checks if both download and display were not selected
        if downloadCheck is False and displayCheck is False:
            flash(u'Please select at least one: Download or Display in Browser', 'confirmation')
            return redirect(url_for('upload'))

        data_xls = pd.read_csv(f)

        trans_ordervalue = 0    #initialize transaction fee variable
        trans_shipping = 0      #initialize shipping fee variable
        order_net_updated = 0   #initialize net order value variable

        c = CurrencyRates()
        rate = c.get_rate('USD', currency) #get exchange rate of selected currency to USD
        listing_fee = round(rate*0.20, 2)

        trans1 = []            #stores transaction fees for all orders
        trans2 = []            #stores shopping fees for all orders
        orderNetUpdated = []   #stores net order values for all orders
        listingfees =[]        #stores listing fees for all orders

        #Calculates fees for each order in file
        for index, row in data_xls.iterrows():
            trans_ordervalue = row['Order Value']*0.05    #5% transaction fee
            trans_shipping = row['Delivery']*0.05         #5% shipping fee

            #update total value after taking into account all the fees
            order_net_updated = float(row['Order Value']) + float(row['Delivery']) - float(row['Discount Amount']) - float(row['Delivery Discount']) - float(row['Card Processing Fees']) - trans_ordervalue - trans_shipping - listing_fee
            trans1.append('%.2f' % round(trans_ordervalue,2))
            trans2.append('%.2f' % round(trans_shipping,2))
            orderNetUpdated.append('%.2f' % round(order_net_updated, 2))
            listingfees.append('%.2f' % listing_fee)

        #Creates 4 new columns in csv file and populates with the appropriate arrays
        data_xls['Transaction Fee'] = trans1
        data_xls['Shipping Transaction Fee'] = trans2
        data_xls['Listing Fee'] = listingfees
        data_xls['Order Net Updated'] = orderNetUpdated

        #Replace n/a with blanks
        data_xls = data_xls.replace(np.nan, '', regex=True)

        #display results and download updated file
        if downloadCheck is True and displayCheck is True:
            data_xls.to_csv('output.xlsx', index = False, header = True)
            flash(u'Download Complete!', 'confirmation')
            return render_template('table.html', tables=[data_xls.to_html(classes='data')], titles= data_xls.columns.values)

        #download updated file only
        if downloadCheck is True:
            data_xls.to_csv('output.xlsx', index = False, header = True)
            flash(u'Download Complete!', 'confirmation')
            return redirect(url_for('upload'))

        #display results in browserr only
        if displayCheck is True:
            return render_template('table.html', tables=[data_xls.to_html(classes='data')], titles= data_xls.columns.values)

    return render_template('upload.html')

#Route for the Feedback page
@application.route("/feedback",methods=['GET', 'POST'])
def feedback():
    form = ContactForm(request.form)

    if request.method == 'POST':
        #check if there are any errors in the input
        if form.validate() is False:
            return render_template('feedback.html', form=form)
        else:
            flash(u'Feedback Received. Thank you!', 'confirmation')
            #format email
            msg = Message(form.subject.data, sender='contact@example.com', recipients=['ysivekum@gmail.com'])
            msg.body = """
            From: %s <%s>
            %s
            """ % (form.name.data, form.email.data, form.message.data)
            mail.send(msg)
            form.name.data = ""
            form.email.data = ""
            form.subject.data = ""
            form.message.data = ""
            return render_template('feedback.html', form = form)

    elif request.method == 'GET':
        return render_template('feedback.html', form=form)

if __name__ == "__main__":
    application.run(host='0.0.0.0', port=int(sys.argv[1]))


from typing import Optional
from typing import List, Dict

from app.schemas.ner import NerQuery, NerEntities, NerEntity

from app.main import nlp_model

def get_entities(query: NerQuery) -> NerEntities:
    query_dict = query.dict(exclude_unset=True)
    doc = nlp_model(query_dict["sentence"])
    entities = [
        NerEntity(
            text=str(x), entity=x.label_, pos=i, start=x.start_char, end=x.end_char
        )
        for i, x in enumerate(doc.ents)
    ]

    ner = nlp_model._meta["name"] + "_" + nlp_model._meta["version"]
    return NerEntities(entities=entities, ner=ner)


[3, 3, 4],
[3, 6, 4],
[5, 6, 2]

U =[
[21, 32, 14, 8, 6, 9, 11, 3, 5],
[17, 2, 8, 14, 55, 23, 19, 1, 6],
[41, 23, 13, 5, 11, 22, 26, 7, 9],
[12, 11, 5, 8, 3, 15, 7, 25, 19],
[14, 7, 3, 5, 11, 23, 8, 7, 9],
[2, 8, 5, 7, 1, 13, 23, 11, 17],
[11, 7, 9, 5, 3, 8, 26, 13, 17],
[23, 1, 5, 19, 11, 7, 9, 4, 16],
[31, 5, 12, 7, 13, 17, 24, 3, 11]
]

L = [
[1, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 1, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 1]
]

b = [[2],
     [5],
     [7],
     [1],
     [6],
     [9],
     [4],
     [8],
     [3]]

utest = [[4,2,3],[4,5,6],[7,-8,2]]
ltest = [[1,0,0],[0,1,0],[0,0,1]]
btest = [[3],[2],[1]]
size = 9

i = 0
j = 0
o = 0
while i < size: # the row being used to judge

    o = i + 1

    while o < size: #the other rows
        lnumber = -1 * U[o][i]
        multiplyer = lnumber / U[i][i]
        L[o][i] = -1 * multiplyer
        while j < size:
            addto = U[o][j]
            multiplyto = U[i][j]
            newsum = addto + (multiplyer * multiplyto)
            U[o][j] = newsum

            j = j + 1

        j = 0
        o = o + 1

    o = 0
    i = i + 1

i = 0
j = 0
o = 0
j2 = 0
while i < size: # the row being used to judge
    if i == j and L[i][j] != 1 and L[i][j] != 0:
        divider = L[i][j]

        tempb = b[i][0]
        tempb = tempb / divider
        b[i][0] = tempb
        u = 0
        while u < size:
            temp = L[i][u]
            if temp != 0:
                temp = temp / divider
                L[i][u] = temp
            u = u + 1

    while o < size: #the other rows
        if i == o:
            o = o + 1
        else:
            newadder = -1 * L[o][j]

            baddto = b[o][0]
            bmultiply = b[i][0] * newadder
            bsum = baddto + bmultiply
            b[o][0] = bsum

            while j2 < size: # we must do all the additions and subtractions
                    #a[o][j] is the number we want to make 0
                addto = L[o][j2]
                multiplyto = L[i][j2]
                newsum = addto + (newadder * multiplyto)
                L[o][j2] = newsum
                j2 = j2 + 1
            j2 = 0
            o = o + 1

    o = 0
    i = i + 1
    j = j + 1

i = 0
j = 0
o = 0
j2 = 0
while i < size: # the row being used to judge
    if i == j and U[i][j] != 1 and U[i][j] != 0:
        divider = U[i][j]

        tempb = b[i][0]
        tempb = tempb / divider
        b[i][0] = tempb
        u = 0
        while u < size:
            temp = U[i][u]
            if temp != 0:
                temp = temp / divider
                U[i][u] = temp
            u = u + 1

    while o < size: #the other rows
        if i == o:
            o = o + 1
        else:
            newadder = -1 * U[o][j]

            baddto = b[o][0]
            bmultiply = b[i][0] * newadder
            bsum = baddto + bmultiply
            b[o][0] = bsum

            while j2 < size: # we must do all the additions and subtractions
                    #a[o][j] is the number we want to make 0
                addto = U[o][j2]
                multiplyto = U[i][j2]
                newsum = addto + (newadder * multiplyto)
                U[o][j2] = newsum
                j2 = j2 + 1
            j2 = 0
            o = o + 1

    o = 0
    i = i + 1
    j = j + 1

t = 0
while t < size:
   print(f'X{t} = {b[t][0]}')
   t = t + 1


# python3 LFTP_client.py lsend localhost ./client_input/out
# python3 LFTP_client.py lsend localhost example
# python3 LFTP_client.py lget localhost example

import os
import sys
import socket

from LFTP_config import SERVER_PORT, log, MTU, \
  SEND_REQUEST_STR, RECV_REQUEST_STR, warn
from LFTP_Connection import LFTP_Connection

def hint():
  print('[error] usage: LFTP ' + SEND_REQUEST_STR + ' <myserver> <mylargefile>' + '\n'
    + '           or: LFTP ' + RECV_REQUEST_STR + '  <myserver> <mylargefile>')

def main():
  if len(sys.argv) != 4 \
    or (sys.argv[1] != 'lget' and sys.argv[1] != 'lsend'):
    hint()
    return

  port = SERVER_PORT
  if sys.argv[2].find(':') == -1:
    host = sys.argv[2]
  else:
    (host, port_s) = (sys.argv[2].split(':'))
    port = int(port_s)

  path = sys.argv[3]
  if sys.argv[1] == 'lsend' and not os.path.exists(path):
    print('the sending file: "{}" do not exist'.format(path))
    return

  serverIP = socket.gethostbyname(host)

  serverAddress = (serverIP, port)

  connections = LFTP_Connection(isServer=False)
  connections.newConnectionTo(addr=serverAddress, connectionType=sys.argv[1], path=sys.argv[3])

if __name__ == "__main__":
  main()


#!/usr/bin/python

############################################################
# ExtractLines.py
#
# This script reads in range data from a csv file, and
# implements a split-and-merge to extract meaningful lines
# in the environment.
############################################################

# Imports
import numpy as np
from PlotFunctions import *

############################################################
# functions
############################################################

#-----------------------------------------------------------
# ExtractLines
#
# This function implements a split-and-merge line
# extraction algorithm
#
# INPUT: RangeData - (x_r, y_r, theta, rho)
#                x_r - robot's x position (m)
#                y_r - robot's y position (m)
#              theta - (1D) np array of angle 'theta' from data (rads)
#                rho - (1D) np array of distance 'rho' from data (m)
#           params - dictionary of parameters for line extraction
#
# OUTPUT: (alpha, r, segend, pointIdx)
#         alpha - (1D) np array of 'alpha' for each fitted line (rads)
#             r - (1D) np array of 'r' for each fitted line (m)
#        segend - np array (N_lines, 4) of line segment endpoints.
#                 each row represents [x1, y1, x2, y2]
#      pointIdx - (N_lines,2) segment's first and last point index

def ExtractLines(RangeData, params):

    # Extract useful variables from RangeData
    x_r = RangeData[0]
    y_r = RangeData[1]
    theta = RangeData[2]
    rho = RangeData[3]

    ### Split Lines ###
    N_pts = len(rho)
    r = np.zeros(0)
    alpha = np.zeros(0)
    pointIdx = np.zeros((0, 2), dtype=np.int)

    # This implementation pre-prepartitions the data according to the "MAX_P2P_DIST"
    # parameter. It forces line segmentation at sufficiently large range jumps.
    rho_diff = np.abs(rho[1:] - rho[:(len(rho)-1)])
    LineBreak = np.hstack((np.where(rho_diff > params['MAX_P2P_DIST'])[0]+1, N_pts))
    startIdx = 0
    for endIdx in LineBreak:
        alpha_seg, r_seg, pointIdx_seg = SplitLinesRecursive(theta, rho, startIdx, endIdx, params)
        N_lines = r_seg.size

        ### Merge Lines ###
        if (N_lines > 1):
            alpha_seg, r_seg, pointIdx_seg = MergeColinearNeigbors(theta, rho, alpha_seg, r_seg, pointIdx_seg, params)
        r = np.append(r, r_seg)
        alpha = np.append(alpha, alpha_seg)
        pointIdx = np.vstack((pointIdx, pointIdx_seg))
        startIdx = endIdx

    N_lines = alpha.size

    ### Compute endpoints/lengths of the segments ###
    segend = np.zeros((N_lines, 4))
    seglen = np.zeros(N_lines)
    for i in range(N_lines):
        rho1 = r[i]/np.cos(theta[pointIdx[i, 0]]-alpha[i])
        rho2 = r[i]/np.cos(theta[pointIdx[i, 1]-1]-alpha[i])
        x1 = rho1*np.cos(theta[pointIdx[i, 0]])
        y1 = rho1*np.sin(theta[pointIdx[i, 0]])
        x2 = rho2*np.cos(theta[pointIdx[i, 1]-1])
        y2 = rho2*np.sin(theta[pointIdx[i, 1]-1])
        segend[i, :] = np.hstack((x1, y1, x2, y2))
        seglen[i] = np.linalg.norm(segend[i, 0:2] - segend[i, 2:4])

    ### Filter Lines ###
    # Find and remove line segments that are too short
    goodSegIdx = np.where((seglen >= params['MIN_SEG_LENGTH']) &
                          (pointIdx[:, 1] - pointIdx[:, 0] >= params['MIN_POINTS_PER_SEGMENT']))[0]
    pointIdx = pointIdx[goodSegIdx, :]
    alpha = alpha[goodSegIdx]
    r = r[goodSegIdx]
    segend = segend[goodSegIdx, :]

    # change back to scene coordinates
    segend[:, (0, 2)] = segend[:, (0, 2)] + x_r
    segend[:, (1, 3)] = segend[:, (1, 3)] + y_r

    return alpha, r, segend, pointIdx

#-----------------------------------------------------------
# SplitLineRecursive
#
# This function executes a recursive line-splitting algorithm,
# which recursively sub-divides line segments until no further
# splitting is required.
#
# INPUT:  theta - (1D) np array of angle 'theta' from data (rads)
#           rho - (1D) np array of distance 'rho' from data (m)
#      startIdx - starting index of segment to be split
#        endIdx - ending index of segment to be split
#        params - dictionary of parameters
#
# OUTPUT: alpha - (1D) np array of 'alpha' for each fitted line (rads)
#             r - (1D) np array of 'r' for each fitted line (m)
#           idx - (N_lines,2) segment's first and last point index

def SplitLinesRecursive(theta, rho, startIdx, endIdx, params):

    ##### TO DO #####
    # Implement a recursive line splitting function
    # It should call 'FitLine()' to fit individual line segments
    # It should call 'FindSplit()' to find an index to split at
    #################
    theta_s = theta[startIdx:endIdx]
    rho_s = rho[startIdx:endIdx]

    # No more splits possible if single point
    if startIdx == endIdx:
        return np.empty((0)), np.empty((0)), np.empty((0,2))

    # Attempt to split the fitted line segment
    alpha, r = FitLine(theta_s, rho_s)
    splitIdx_s = FindSplit(theta_s, rho_s, alpha, r, params)
    if splitIdx_s == -1:
        idx = np.array([[startIdx, endIdx]])
    else:
        # Fit line segments to each of the split data sets
        splitIdx = splitIdx_s + startIdx
        alpha_l, r_l, idx_l = SplitLinesRecursive(theta, rho, startIdx, splitIdx, params)
        alpha_r, r_r, idx_r = SplitLinesRecursive(theta, rho, splitIdx, endIdx, params)

        alpha = np.hstack((alpha_l, alpha_r))
        r = np.hstack((r_l, r_r))
        idx = np.vstack((idx_l, idx_r))
    return alpha, r, idx

#-----------------------------------------------------------
# FindSplit
#
# This function takes in a line segment and outputs the best
# index at which to split the segment
#
# INPUT:  theta - (1D) np array of angle 'theta' from data (rads)
#           rho - (1D) np array of distance 'rho' from data (m)
#         alpha - 'alpha' of input line segment (1 number)
#             r - 'r' of input line segment (1 number)
#        params - dictionary of parameters
#
# OUTPUT: SplitIdx - idx at which to split line (return -1 if
#                    it cannot be split)

def FindSplit(theta, rho, alpha, r, params):

    ##### TO DO #####
    # Implement a function to find the split index (if one exists)
    # It should compute the distance of each point to the line.
    # The index to split at is the one with the maximum distance
    # value that exceeds 'LINE_POINT_DIST_THRESHOLD', and also does
    # not divide into segments smaller than 'MIN_POINTS_PER_SEGMENT'
    # return -1 if no split is possiple
    #################
    # Calculate distance from line and apply threshold filters
    n = len(theta)
    num_pts = np.array(range(0,n))   # Assume split point assigned to left segment
    d = np.abs(np.multiply(rho, np.cos(theta-alpha)) - r)
    filt = (d > params["LINE_POINT_DIST_THRESHOLD"]) & \
           (num_pts >= params["MIN_POINTS_PER_SEGMENT"]) & \
           ((n-num_pts) >= params["MIN_POINTS_PER_SEGMENT"])

    # Choose largest distance that satisfies thresholds
    if np.any(filt):
        org_to_max = np.argsort(d)
        idx = np.where(filt[org_to_max])[0][-1]  # Max element is last since sorted in ascending order
        splitIdx = org_to_max[idx]   # Map back to unsorted data index
    else:
        splitIdx = -1
    return splitIdx

#-----------------------------------------------------------
# FitLine
#
# This function outputs a best fit line to a segment of range
# data, expressed in polar form (alpha, r)
#
# INPUT:  theta - (1D) np array of angle 'theta' from data (rads)
#           rho - (1D) np array of distance 'rho' from data (m)
#
# OUTPUT: alpha - 'alpha' of best fit for range data (1 number) (rads)
#             r - 'r' of best fit for range data (1 number) (m)

def FitLine(theta, rho):

    ##### TO DO #####
    # Implement a function to fit a line to polar data points
    # based on the solution to the least squares problem (see Hw)
    #################
    n = len(theta)
    alpha_num = 0.0
    alpha_den = 0.0
    for i in range(0, n):
        for j in range(0, n):
            alpha_num += rho[i]*rho[j]*np.cos(theta[i])*np.sin(theta[j])
            alpha_den += rho[i]*rho[j]*np.cos(theta[i] + theta[j])
    alpha_num = np.sum(np.multiply(rho**2, np.sin(2.0*theta))) - (2.0/n)*alpha_num
    alpha_den = np.sum(np.multiply(rho**2, np.cos(2.0*theta))) - (1.0/n)*alpha_den

    alpha = 0.5*np.arctan2(alpha_num, alpha_den) + np.pi/2
    r = (1.0/n)*np.sum(np.multiply(rho, np.cos(theta-alpha)))
    return alpha, r

#---------------------------------------------------------------------
# MergeColinearNeigbors
#
# This function merges neighboring segments that are colinear and outputs
# a new set of line segments
#
# INPUT:  theta - (1D) np array of angle 'theta' from data (rads)
#           rho - (1D) np array of distance 'rho' from data (m)
#         alpha - (1D) np array of 'alpha' for each fitted line (rads)
#             r - (1D) np array of 'r' for each fitted line (m)
#      pointIdx - (N_lines,2) segment's first and last point indices
#        params - dictionary of parameters
#
# OUTPUT: alphaOut - output 'alpha' of merged lines (rads)
#             rOut - output 'r' of merged lines (m)
#      pointIdxOut - output start and end indices of merged line segments

def MergeColinearNeigbors(theta, rho, alpha, r, pointIdx, params):

    ##### TO DO #####
    # Implement a function to merge colinear neighboring line segments
    # HINT: loop through line segments and try to fit a line to data
    #       points from two adjacent segments. If this line cannot be
    #       split, then accept the merge. If it can be split, do not merge.
    #################
    alphaOut = alpha.copy()
    rOut = r.copy()
    pointIdxOut = pointIdx.copy()

    i = 0
    N_lines = pointIdxOut.shape[0]
    while i < (N_lines - 1):
        startIdx = pointIdx[i,0]   # Start of first segment
        endIdx = pointIdx[i+1,1]   # End of next segment

        # Fit line to data from adjacent segments and attempt to split
        alpha_fit, r_fit = FitLine(theta[startIdx:endIdx], rho[startIdx:endIdx])
        splitIdx = FindSplit(theta[startIdx:endIdx], rho[startIdx:endIdx], alpha_fit, r_fit, params)

        # Accept merge if fitted line cannot be split
        if splitIdx == -1:
            # Update with new segment
            alphaOut[i] = alpha_fit
            rOut[i] = r_fit
            pointIdxOut[i,1] = endIdx

            # Delete old segment that was merged
            alphaOut = np.delete(alphaOut, i+1)
            rOut = np.delete(rOut, i+1)
            pointIdxOut = np.delete(pointIdxOut, i+1, axis=0)
            N_lines = N_lines - 1
        else:
            i = i + 1

    return alphaOut, rOut, pointIdxOut

#----------------------------------
# ImportRangeData
def ImportRangeData(filename):

    data = np.genfromtxt('./RangeData/'+filename, delimiter=',')
    x_r = data[0, 0]
    y_r = data[0, 1]
    theta = data[1:, 0]
    rho = data[1:, 1]
    return (x_r, y_r, theta, rho)
#----------------------------------

############################################################
# Main
############################################################
def main():
    # parameters for line extraction (mess with these!)
    MIN_SEG_LENGTH = 0.05  # minimum length of each line segment (m)
    LINE_POINT_DIST_THRESHOLD = 0.025  # max distance of pt from line to split
    MIN_POINTS_PER_SEGMENT = 3  # minimum number of points per line segment
    MAX_P2P_DIST = 0.54  # max distance between two adjent pts within a segment

    # Data files are formated as 'rangeData_<x_r>_<y_r>_N_pts.csv
    # where x_r is the robot's x position
    #       y_r is the robot's y position
    #       N_pts is the number of beams (e.g. 180 -> beams are 2deg apart)

    filename = 'rangeData_5_5_180.csv'   # Best params = [0.05, 0.065, 2, 0.80]
    # filename = 'rangeData_4_9_360.csv'     # Best params = [0.05, 0.025, 3, 0.54]
    # filename = 'rangeData_7_2_90.csv'    # Best params = [0.05, 0.165, 2, 0.42]

    # Import Range Data
    RangeData = ImportRangeData(filename)

    params = {'MIN_SEG_LENGTH': MIN_SEG_LENGTH,
              'LINE_POINT_DIST_THRESHOLD': LINE_POINT_DIST_THRESHOLD,
              'MIN_POINTS_PER_SEGMENT': MIN_POINTS_PER_SEGMENT,
              'MAX_P2P_DIST': MAX_P2P_DIST}

    alpha, r, segend, pointIdx = ExtractLines(RangeData, params)

    ax = PlotScene()
    ax = PlotData(RangeData, ax)
    ax = PlotRays(RangeData, ax)
    ax = PlotLines(segend, ax)

    plt.show(ax)

############################################################

if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        import traceback
        traceback.print_exc()