get_ipython().magic('matplotlib inline')

from astroML.plotting          import hist
from astropy.io                import fits
from astropy.modeling          import models, fitting
from datetime                  import datetime
from image_registration        import cross_correlation_shifts
from glob                      import glob
from matplotlib.ticker         import MaxNLocator
from matplotlib                import style
from os                        import listdir
from pandas                    import DataFrame, read_csv, read_pickle, scatter_matrix
from photutils                 import CircularAperture, CircularAnnulus, aperture_photometry, findstars
from least_asymmetry           import actr, moments, fitgaussian
from pylab                     import ion, gcf, sort, linspace, indices, median, mean, std, empty, figure, transpose, ceil
from pylab                     import concatenate, pi, sqrt, ones, diag, inf, rcParams, isnan, isfinite, array, nanmax
from numpy                     import min as npmin, max as npmax, zeros, arange, sum, float, isnan, hstack
from numpy                     import int32 as npint, round as npround, nansum as sum, nanstd as std
from seaborn                   import *
from scipy.special             import erf
from scipy                     import stats
from sklearn.externals         import joblib
from socket                    import gethostname
from statsmodels.robust        import scale
from statsmodels.nonparametric import kde
from sys                       import exit
from time                      import time, localtime

from numpy                     import zeros, nanmedian as median, nanmean as mean, nan
from sys                       import exit
from sklearn.externals         import joblib
from least_asymmetry           import actr

import numpy as np

rcParams['image.interpolation'] = 'None'
rcParams['image.cmap']          = 'Blues_r'
rcParams['axes.grid']           = False

dataDir = '/path/to/fits/files/main/directory/'
fitsFileDir = 'path/to/fits/subdirectories/'

fitsFilenames = glob(dataDir + fitsFileDir + '*slp.fits')
fitsFilenames

def get_julian_date_from_gregorian_date(*date):
    """gd2jd.py converts a UT Gregorian date to Julian date.

    Functions for JD <-> GD conversion,
      courtesy of Ian Crossfield at
      http://www.astro.ucla.edu/~ianc/python/_modules/date.html

    Downloaded from Marshall Perrin Github at
        https://github.com/mperrin/misc_astro/blob/master/idlastro_ports/gd2jd.py

    Usage: gd2jd.py (2009, 02, 25, 01, 59, 59)

    To get the current Julian date:
        import time
        gd2jd(time.gmtime())

    Hours, minutes and/or seconds can be omitted -- if so, they are
    assumed to be zero.

    Year and month are converted to type INT, but all others can be
    type FLOAT (standard practice would suggest only the final element
    of the date should be float)
    """
    verbose=False
    if verbose: print(date)

    date = list(date)

    if len(date)<3:
        print("You must enter a date of the form (2009, 02, 25)!")
        return -1
    elif len(date)==3:
        for ii in range(3): date.append(0)
    elif len(date)==4:
        for ii in range(2): date.append(0)
    elif len(date)==5:
        date.append(0)

    yyyy = int(date[0])
    mm = int(date[1])
    dd = float(date[2])
    hh = float(date[3])
    min = float(date[4])
    sec = float(date[5])

    UT=hh+min/60+sec/3600

    total_seconds=hh*3600+min*60+sec
    fracday=total_seconds/86400

    if (100*yyyy+mm-190002.5)>0:
        sig=1
    else:
        sig=-1

    JD = 367*yyyy - int(7*(yyyy+int((mm+9)/12))/4) + int(275*mm/9) + dd + 1721013.5 + UT/24 - 0.5*sig +0.5

    months=["January", "February", "March", "April", "May", "June", "July", "August",
                "September", "October", "November", "December"]

    # Now calculate the fractional year. Do we have a leap year?
    daylist=[31,28,31,30,31,30,31,31,30,31,30,31]
    daylist2=[31,29,31,30,31,30,31,31,30,31,30,31]
    if (yyyy%4 != 0):
        days=daylist2
    elif (yyyy%400 == 0):
        days=daylist2
    elif (yyyy%100 == 0):
        days=daylist
    else:
        days=daylist2

    daysum=0
    for y in range(mm-1):
        daysum=daysum+days[y]
    daysum=daysum+dd-1+UT/24

    if days[1]==29:
        fracyear=yyyy+daysum/366
    else:
        fracyear=yyyy+daysum/365
    if verbose:
        print(yyyy,mm,dd,hh,min,sec)
        print("UT="+UT)
        print("Fractional day: %f" % fracday)
        print("\n"+months[mm-1]+" %i, %i, %i:%i:%i UT = JD %f" % (dd, yyyy, hh, min, sec, JD), end= " ")
        print(" = " + fracyear+"\n")

    return JD

def get_julian_date_from_header(header):
    # These are specific to STScI standards -- may vary on the ground
    fitsDate    = header['DATE-OBS']
    startTimeStr= header['TIME-OBS']
    endTimeStr  = header['TIME-END']

    yyyy, mm , dd   = fitsDate.split('-')

    hh1 , mn1, ss1  = array(startTimeStr.split(':')).astype(float)
    hh2 , mn2, ss2  = array(endTimeStr.split(':')).astype(float)

    yyyy  = float(yyyy)
    mm    = float(mm)
    dd    = float(dd)

    hh1   = float(hh1)
    mn1   = float(mn1)
    ss1   = float(ss1)

    hh2   = float(hh2)
    mn2   = float(mn2)
    ss2   = float(ss2)

    startDate   = get_julian_date_from_gregorian_date(yyyy,mm,dd,hh1,mn1,ss1)
    endDate     = get_julian_date_from_gregorian_date(yyyy,mm,dd,hh2,mn2,ss2)

    return startDate, endDate

def flux_weighted_centroid(image, ypos, xpos, bSize = 7):
    '''
        Flux-weighted centroiding (Knutson et al. 2008)
        xpos and ypos are the rounded pixel positions of the star
    '''

    ## extract a box around the star:
    #im = a[ypos-bSize:ypos+bSize, xpos-bSize:xpos+bSize].copy()
    subImage = image[ypos-bSize:ypos+bSize, xpos-bSize:xpos+bSize].transpose().copy()

    y,x = 0,1

    ydim = subImage.shape[y]
    xdim = subImage.shape[x]

    ## add up the flux along x and y
    xflux = zeros(xdim)
    xrng  = arange(xdim)

    yflux = zeros(ydim)
    yrng  = arange(ydim)

    for i in range(xdim):
        xflux[i] = sum(subImage[i,:])

    for j in range(ydim):
        yflux[j] = sum(subImage[:,j])

    ## get the flux weighted average position:
    ypeak = sum(yflux * yrng) / sum(yflux) + ypos - float(bSize)
    xpeak = sum(xflux * xrng) / sum(xflux) + xpos - float(bSize)

    return (ypeak, xpeak)

def fit_gauss(subFrameNow, xinds, yinds, initParams, print_compare=False):
    # initParams = (height, x, y, width_x, width_y, offset)
    fit_lvmq = fitting.LevMarLSQFitter()
    model0  = models.Gaussian2D(amplitude=initParams[0], x_mean=initParams[1], y_mean=initParams[2],
                                x_stddev=initParams[3], y_stddev=initParams[4], theta=0.0) + \
              models.Const2D(amplitude=initParams[5])

    model1  = fit_lvmq(model0, xinds, yinds, subFrameNow)
    model1  = fit_lvmq(model1, xinds, yinds, subFrameNow)

    if print_compare:
        print(model1.amplitude_0 - initParams[0], end=" ")
        print(model1.x_mean_0    - initParams[1], end=" ")
        print(model1.y_mean_0    - initParams[2], end=" ")
        print(model1.x_stddev_0  - initParams[3], end=" ")
        print(model1.y_stddev_0  - initParams[4], end=" ")
        print(model1.amplitude_1 - initParams[5])

    return model1.parameters

class wanderer(object):
    print('\n\n** Not all who wander are lost **\n\n')
    def __init__(self, fitsFileDir = './', filetype = 'slp.fits',
                 yguess=None, xguess=None, npix=10, method='mean'):

        y,x = 0,1

        if method == 'mean':
            self.metric  = mean
        elif method == 'median':
            self.metric  = median
        else:
            raise Exception("`method` must be from the list ['mean', 'median']")

        self.fitsFileDir  = fitsFileDir
        self.fitsFilenames = glob(self.fitsFileDir + '/*' + filetype)
        self.nSlopeFiles  = len(self.fitsFilenames)

        if self.nSlopeFiles == 0:
            print('Pipeline found no Files in ' + self.fitsFileDir + ' of type /*' + filetype)
            exit(-1)

        self.centering_df   = DataFrame()
        self.background_df  = DataFrame()
        self.flux_TSO_df    = DataFrame()
        self.noise_TSO_df   = DataFrame()

        testfits            = fits.open(self.fitsFilenames[0])[0]

        self.imageCube      = np.zeros((self.nSlopeFiles, testfits.data[0].shape[0], testfits.data[0].shape[1]))
        self.noiseCube      = np.zeros((self.nSlopeFiles, testfits.data[0].shape[0], testfits.data[0].shape[1]))
        self.timeCube       = np.zeros(self.nSlopeFiles)

        if yguess == None:
            self.yguess = self.imageCube.shape[y]//2
        else:
            self.yguess = yguess
        if xguess == None:
            self.xguess = self.imageCube.shape[x]//2
        else:
            self.xguess = xguess

        self.npix = npix

    def load_data_from_fits_files(self):

        nGroups        = len(self.fitsFilenames)
        for kframe, fname in enumerate(self.fitsFilenames):

            fitsNow = fits.open(fname)

            self.imageCube[kframe] = fitsNow[0].data[0]
            self.noiseCube[kframe] = fitsNow[0].data[1]

            # re-write these 4 lines into `get_julian_date_from_header`
            day2sec        = 86400.
            startJD,endJD     = get_julian_date_from_header(fitsNow[0].header)
            timeSpan          = (endJD - startJD)*day2sec/nGroups
            self.timeCube[kframe]  = startJD  + timeSpan*(kframe+0.5) / day2sec - 2450000.

            del fitsNow[0].data
            fitsNow.close()
            del fitsNow

    def load_data_from_save_files(self, savefiledir=None, saveFileNameHeader=None, saveFileType='.pickle.save'):

        if saveFileNameHeader is None:
            raise Exception('`saveFileNameHeader` should be the beginning of each save file name')

        if savefiledir is None:
            savefiledir = './'

        print('Loading from Master Files')
        self.centering_df   = read_pickle(savefiledir  + saveFileNameHeader + '_centering_dataframe'  + saveFileType)
        self.background_df  = read_pickle(savefiledir + saveFileNameHeader + '_background_dataframe' + saveFileType)
        self.flux_TSO_df    = read_pickle(savefiledir   + saveFileNameHeader + '_flux_TSO_dataframe'   + saveFileType)

        try:
            self.noise_TSO_df   = read_pickle(savefiledir   + saveFileNameHeader + '_noise_TSO_dataframe'   + saveFileType)
        except:
            self.noise_TSO_df   = None

        self.imageCube        = joblib.load(savefiledir  + saveFileNameHeader + '_image_cube_array' + saveFileType)
        self.noiseCube        = joblib.load(savefiledir  + saveFileNameHeader + '_noise_cube_array' + saveFileType)
        self.timeCube         = joblib.load(savefiledir  + saveFileNameHeader + '_time_cube_array'  + saveFileType)

        self.imageBadPixMasks = joblib.load(savefiledir  + saveFileNameHeader + '_image_bad_pix_cube_array' + saveFileType)

        self.save_dict        = joblib.load(savefiledir + saveFileNameHeader + '_save_dict' + saveFileType)

        print('Assigning Parts of `self.save_dict` to individual data structures')
        for key in self.save_dict.keys():
            exec("self." + key + " = self.save_dict['" + key + "']")

        # self.fitsFileDir              = self.save_dict['fitsFileDir']
        # self.fitsFilenames             = self.save_dict['fitsFilenames']

        # self.background_Annulus       = self.save_dict['background_Annulus']
        # self.background_CircleMask    = self.save_dict['background_CircleMask']
        # self.background_GaussMoment   = self.save_dict['background_GaussMoment']
        # self.background_GaussianFit   = self.save_dict['background_GaussianFit']
        # self.background_KDEUniv       = self.save_dict['background_KDEUniv']
        # self.background_MedianMask    = self.save_dict['background_MedianMask']
        # self.centering_FluxWeight     = self.save_dict['centering_FluxWeight']
        # self.centering_GaussianFit    = self.save_dict['centering_GaussianFit']
        # self.centering_GaussianMoment = self.save_dict['centering_GaussianMoment']
        # self.centering_LeastAsym      = self.save_dict['centering_LeastAsym']
        # self.fitsFileDir              = self.save_dict['fitsFileDir']
        # self.heights_GaussianFit      = self.save_dict['heights_GaussianFit']
        # self.heights_GaussianMoment   = self.save_dict['heights_GaussianMoment']
        # # self.imageCubeMAD             = self.save_dict['imageCubeMAD']
        # # self.imageCubeMedian          = self.save_dict['imageCubeMedian']
        # self.fitsFilenames             = self.save_dict['fitsFilenames']
        # self.widths_GaussianFit       = self.save_dict['widths_GaussianFit']
        # self.widths_GaussianMoment    = self.save_dict['widths_GaussianMoment']

    def save_data_to_save_files(self, savefiledir=None, saveFileNameHeader=None, saveFileType='.pickle.save'):

        if saveFileNameHeader is None:
            raise Exception('`saveFileNameHeader` should be the beginning of each save file name')

        if savefiledir is None:
            savefiledir = './'

        date            = localtime()

        year            = date.tm_year
        month           = date.tm_mon
        day             = date.tm_mday

        hour            = date.tm_hour
        minute          = date.tm_min
        sec             = date.tm_sec

        date_string     = '_' + str(year) + '-' + str(month)  + '-' + str(day) + '_' +                                 str(hour) + 'h' + str(minute) + 'm' + str(sec) + 's'

        saveFileTypeBak = date_string + saveFileType

        initiate_save_dict()

        print('Saving to Master File -- Overwriting Previous Master')
        self.centering_df.to_pickle(savefiledir  + saveFileNameHeader + '_centering_dataframe'  + saveFileType)
        self.background_df.to_pickle(savefiledir + saveFileNameHeader + '_background_dataframe' + saveFileType)
        self.flux_TSO_df.to_pickle(savefiledir   + saveFileNameHeader + '_flux_TSO_dataframe'   + saveFileType)

        joblib.dump(self.imageCube, savefiledir  + saveFileNameHeader + '_image_cube_array' + saveFileType)
        joblib.dump(self.noiseCube, savefiledir  + saveFileNameHeader + '_noise_cube_array' + saveFileType)
        joblib.dump(self.timeCube , savefiledir  + saveFileNameHeader + '_time_cube_array'  + saveFileType)

        joblib.dump(self.imageBadPixMasks, savefiledir  + saveFileNameHeader + '_image_bad_pix_cube_array' + saveFileType)

        joblib.dump(self.save_dict, savefiledir + saveFileNameHeader + '_save_dict' + saveFileType)

        print('Saving to New TimeStamped File -- These Tend to Pile Up!')
        self.centering_df.to_pickle(savefiledir  + saveFileNameHeader + '_centering_dataframe'  + saveFileTypeBak)
        self.background_df.to_pickle(savefiledir + saveFileNameHeader + '_background_dataframe' + saveFileTypeBak)
        self.flux_TSO_df.to_pickle(savefiledir   + saveFileNameHeader + '_flux_TSO_dataframe'   + saveFileTypeBak)

        joblib.dump(self.imageCube, savefiledir  + saveFileNameHeader + '_image_cube_array' + saveFileTypeBak)
        joblib.dump(self.noiseCube, savefiledir  + saveFileNameHeader + '_noise_cube_array' + saveFileTypeBak)
        joblib.dump(self.timeCube , savefiledir  + saveFileNameHeader + '_time_cube_array'  + saveFileTypeBak)

        joblib.dump(self.imageBadPixMasks, savefiledir  + saveFileNameHeader + '_image_bad_pix_cube_array' + saveFileTypeBak)

        joblib.dump(self.save_dict, savefiledir + saveFileNameHeader + '_save_dict' + saveFileTypeBak)

    def initiate_save_dict(self):

        self.save_dict  = {} # DataFrame() -- test if this works later

        self.save_dict['fitsFileDir']               = self.fitsFileDir
        self.save_dict['fitsFilenames']              = self.fitsFilenames

        self.save_dict['background_Annulus']        = self.background_Annulus
        self.save_dict['background_CircleMask']     = self.background_CircleMask
        self.save_dict['background_GaussMoment']    = self.background_GaussMoment
        self.save_dict['background_GaussianFit']    = self.background_GaussianFit
        self.save_dict['background_KDEUniv']        = self.background_KDEUniv
        self.save_dict['background_MedianMask']     = self.background_MedianMask
        self.save_dict['centering_FluxWeight']      = self.centering_FluxWeight
        self.save_dict['centering_GaussianFit']     = self.centering_GaussianFit
        self.save_dict['centering_GaussianMoment']  = self.centering_GaussianMoment
        self.save_dict['centering_LeastAsym']       = self.centering_LeastAsym
        self.save_dict['fitsFileDir']               = self.fitsFileDir
        self.save_dict['heights_GaussianFit']       = self.heights_GaussianFit
        self.save_dict['heights_GaussianMoment']    = self.heights_GaussianMoment
        # self.save_dict['']                          = self.imageCubeMAD
        # self.save_dict['']                          = self.imageCubeMedian
        self.save_dict['method']                    = self.method
        self.save_dict['npix']                      = self.npix
        self.save_dict['fitsFilenames']              = self.fitsFilenames
        self.save_dict['yguess']                    = self.yguess
        self.save_dict['xguess']                    = self.xguess
        self.save_dict['widths_GaussianFit']        = self.widths_GaussianFit
        self.save_dict['widths_GaussianMoment']     = self.widths_GaussianMoment

    def copy_instance(self):

        temp = wanderer()
        temp.saveFileNameHeader = self.saveFileNameHeader
        temp.savefiledir = self.savefiledir

        temp.centering_df = self.centering_df
        temp.background_df = self.background_df
        temp.flux_TSO_df = self.flux_TSO_df
        temp.noise_TSO_df = self.noise_TSO_df

        temp.imageCube = self.imageCube
        temp.noiseCube = self.noiseCube
        temp.timeCube = self.timeCube

        temp.imageBadPixMasks = self.imageBadPixMasks = joblib.load(savefiledir  + saveFileNameHeader + '_image_bad_pix_cube_array' + saveFileType)

        print('Assigning Parts of `temp.save_dict` to from `self.save_dict`')
        temp.save_dict = self.save_dict
        for key in self.save_dict.keys():
            exec("temp." + key + " = temp.save_dict['" + key + "']")

        # temp.fitsFileDir              = temp.save_dict['fitsFileDir']
        # temp.fitsFilenames             = temp.save_dict['fitsFilenames']

        # temp.background_Annulus       = temp.save_dict['background_Annulus']
        # temp.background_CircleMask    = temp.save_dict['background_CircleMask']
        # temp.background_GaussMoment   = temp.save_dict['background_GaussMoment']
        # temp.background_GaussianFit   = temp.save_dict['background_GaussianFit']
        # temp.background_KDEUniv       = temp.save_dict['background_KDEUniv']
        # temp.background_MedianMask    = temp.save_dict['background_MedianMask']
        # temp.centering_FluxWeight     = temp.save_dict['centering_FluxWeight']
        # temp.centering_GaussianFit    = temp.save_dict['centering_GaussianFit']
        # temp.centering_GaussianMoment = temp.save_dict['centering_GaussianMoment']
        # temp.centering_LeastAsym      = temp.save_dict['centering_LeastAsym']
        # temp.fitsFileDir              = temp.save_dict['fitsFileDir']
        # temp.heights_GaussianFit      = temp.save_dict['heights_GaussianFit']
        # temp.heights_GaussianMoment   = temp.save_dict['heights_GaussianMoment']
        # # temp.imageCubeMAD             = temp.save_dict['imageCubeMAD']
        # # temp.imageCubeMedian          = temp.save_dict['imageCubeMedian']
        # temp.fitsFilenames             = temp.save_dict['fitsFilenames']
        # temp.widths_GaussianFit       = temp.save_dict['widths_GaussianFit']
        # temp.widths_GaussianMoment    = temp.save_dict['widths_GaussianMoment']

    def find_bad_pixels(self, nSig=5):
        # we chose 5 arbitrarily, but from experience
        self.imageCubeMedian  = median(self.imageCube,axis=0)
        self.imageCubeMAD     = scale.mad(self.imageCube,axis=0)

        self.imageBadPixMasks = abs(self.imageCube - self.imageCubeMedian) > nSig*self.imageCubeMAD

        print("There are " + str(sum(self.imageBadPixMasks)) + " 'Hot' Pixels")

        self.imageCube[self.imageBadPixMasks] = nan

    def fit_gaussian_fitting_centering(self, method='la', initc='fw', print_compare=False):
        y,x = 0,1

        yinds0, xinds0 = indices(self.imageCube[0].shape)

        ylower = self.yguess - self.npix
        yupper = self.yguess + self.npix
        xlower = self.xguess - self.npix
        xupper = self.xguess + self.npix

        ylower, xlower, yupper, xupper = np.int32([ylower, xlower, yupper, xupper])

        yinds = yinds0[ylower:yupper, xlower:xupper]
        xinds = xinds0[ylower:yupper, xlower:xupper]

        self.centering_GaussianFit    = zeros((self.imageCube.shape[0], 2))
        self.centering_GaussianMoment = zeros((self.imageCube.shape[0], 2))
        self.widths_GaussianFit       = zeros((self.imageCube.shape[0], 2))
        self.widths_GaussianMoment    = zeros((self.imageCube.shape[0], 2))

        self.heights_GaussianFit      = zeros(self.imageCube.shape[0])
        self.heights_GaussianMoment   = zeros(self.imageCube.shape[0])
        # self.rotation_GaussianFit     = zeros(self.imageCube.shape[0])
        self.background_GaussMoment   = zeros(self.imageCube.shape[0])
        self.background_GaussianFit   = zeros(self.imageCube.shape[0])

        for kframe in range(self.imageCube.shape[0]):
            subFrameNow = self.imageCube[kframe][ylower:yupper, xlower:xupper]
            subFrameNow[isnan(subFrameNow)] = median(~isnan(subFrameNow))

            cmom    = np.array(moments(subFrameNow))  # H, Xc, Yc, Xs, Ys, O

            if method == 'ap':
                if initc == 'fw' and self.centering_FluxWeight.sum():
                    FWCNow    = self.centering_FluxWeight[kframe]
                    FWCNow[y] = FWCNow[y] - ylower
                    FWCNow[x] = FWCNow[x] - xlower
                    gaussI    = hstack([cmom[0], FWCNow, cmom[3:]])
                if initc == 'cm':
                    gaussI  = hstack([cmom[0], cmom[1], cmom[2], cmom[3:]])

                gaussP  = fit_gauss(subFrameNow, xinds, yinds, gaussI) # H, Xc, Yc, Xs, Ys, Th, O

            if method == 'la':
                gaussP  = fitgaussian(subFrameNow)#, xinds, yinds, np.copy(cmom)) # H, Xc, Yc, Xs, Ys, Th, O

            self.centering_GaussianFit[kframe][x]     = gaussP[1] + xlower
            self.centering_GaussianFit[kframe][y]     = gaussP[2] + ylower
            self.centering_GaussianMoment[kframe][x]  = cmom[1]   + xlower
            self.centering_GaussianMoment[kframe][y]  = cmom[2]   + ylower

            self.widths_GaussianFit[kframe][x]        = gaussP[3]
            self.widths_GaussianFit[kframe][y]        = gaussP[4]
            self.widths_GaussianMoment[kframe][x]     = cmom[3]
            self.widths_GaussianMoment[kframe][y]     = cmom[4]

            self.heights_GaussianFit[kframe]          = gaussP[0]
            self.heights_GaussianMoment[kframe]       = cmom[0]

            self.background_GaussianFit[kframe]       = gaussP[5]
            self.background_GaussMoment[kframe]       = cmom[5]

            if print_compare:
                print('Finished Frame ' + str(kframe) + ' with Yc = ' +                       str(self.centering_GaussianFit[kframe][y] - self.centering_GaussianMoment[kframe][y]) + '; Xc = ' +                       str(self.centering_GaussianFit[kframe][x] - self.centering_GaussianMoment[kframe][x]))

            del gaussP, cmom

        self.centering_df = DataFrame()
        self.centering_df['Gaussian_Fit_Y_Centers'] = self.centering_GaussianFit.T[y]
        self.centering_df['Gaussian_Fit_X_Centers'] = self.centering_GaussianFit.T[x]
        self.centering_df['Gaussian_Mom_Y_Centers'] = self.centering_GaussianFit.T[y]
        self.centering_df['Gaussian_Mom_X_Centers'] = self.centering_GaussianFit.T[x]

        self.centering_df['Gaussian_Fit_Y_Widths']  = self.widths_GaussianFit.T[y]
        self.centering_df['Gaussian_Fit_X_Widths']  = self.widths_GaussianFit.T[x]
        self.centering_df['Gaussian_Mom_Y_Widths']  = self.widths_GaussianMoment.T[y]
        self.centering_df['Gaussian_Mom_X_Widths']  = self.widths_GaussianMoment.T[x]

        self.centering_df['Gaussian_Fit_Heights']   = self.heights_GaussianFit
        self.centering_df['Gaussian_Mom_Heights']   = self.heights_GaussianMoment

        self.centering_df['Gaussian_Fit_Offset']    = self.background_GaussianFit
        self.centering_df['Gaussian_Mom_Offset']    = self.background_GaussMoment

        # self.centering_df['Gaussian_Fit_Rotation']    = self.rotation_GaussianFit

    def fit_flux_weighted_centering(self):
        y,x = 0,1

        yinds0, xinds0 = indices(self.imageCube[0].shape)

        ylower = self.yguess - self.npix
        yupper = self.yguess + self.npix
        xlower = self.xguess - self.npix
        xupper = self.xguess + self.npix

        ylower, xlower, yupper, xupper = np.int32([ylower, xlower, yupper, xupper])

        yinds = yinds0[ylower:yupper, xlower:xupper]
        xinds = xinds0[ylower:yupper, xlower:xupper]

        nFWCParams                = 2 # Xc, Yc
        self.centering_FluxWeight = np.zeros((self.nSlopeFiles, nFWCParams))

        for kframe in range(self.nSlopeFiles):
            subFrameNow = self.imageCube[kframe][ylower:yupper, xlower:xupper]
            subFrameNow[isnan(subFrameNow)] = median(~isnan(subFrameNow))

            self.centering_FluxWeight[kframe] = flux_weighted_centroid(self.imageCube[kframe],
                                                                       self.yguess, self.xguess, bSize = 7)
            self.centering_FluxWeight[kframe] = self.centering_FluxWeight[kframe][::-1]

        self.centering_df['FluxWeighted_Y_Centers'] = self.centering_FluxWeight.T[y]
        self.centering_df['FluxWeighted_X_Centers'] = self.centering_FluxWeight.T[x]

    def fit_least_asymmetry_centering(self):

        y,x = 0,1

        yinds0, xinds0 = indices(self.imageCube[0].shape)

        ylower = self.yguess - self.npix
        yupper = self.yguess + self.npix
        xlower = self.xguess - self.npix
        xupper = self.xguess + self.npix

        ylower, xlower, yupper, xupper = np.int32([ylower, xlower, yupper, xupper])

        yinds = yinds0[ylower:yupper, xlower:xupper]
        xinds = xinds0[ylower:yupper, xlower:xupper]

        nAsymParams = 2 # Xc, Yc
        self.centering_LeastAsym  = np.zeros((self.nSlopeFiles, nAsymParams))

        for kframe in range(self.nSlopeFiles):
            # print(kframe, ylower,yupper, xlower,xupper) # (0 143 163 150 170)
            subFrameNow = self.imageCube[kframe][ylower:yupper, xlower:xupper]
            subFrameNow[isnan(subFrameNow)]   = median(subFrameNow)

            center_asym = actr(self.imageCube[kframe], [self.yguess, self.xguess])[0]
            try:
                self.centering_LeastAsym[kframe]  = center_asym[::-1]
            except:
                self.centering_LeastAsym[kframe]  = [nan,nan]

        self.centering_df['LeastAsymmetry_Y_Centers'] = self.centering_FluxWeight.T[y]
        self.centering_df['LeastAsymmetry_X_Centers'] = self.centering_FluxWeight.T[x]

    def fit_all_centering(self):
        print('Fit for Gaussian Fitting & Gaussian Moment Centers\n')
        self.fit_gaussian_fitting_centering()
        print('Fit for Flux Weighted Centers\n')
        self.fit_flux_weighted_centering()
        print('Fit for Least Asymmetry Centers\n')
        self.fit_least_asymmetry_centering()

    def measure_effective_width(self):
        self.effective_widths = self.imageCube.sum(axis=(1,2))**2. / ((self.imageCube)**2).sum(axis=(1,2))
        self.centering_df['Effective_Widths'] = self.effective_widths

    def measure_background_circle_masked(self, aperRad=None, method='mean'):
        """
            Assigning all zeros in the mask to NaNs because the `mean` and `median`
                functions are set to `nanmean` functions, which will skip all NaNs
        """

        if aperRad is None:
            if 'wlp' in self.fitsFilenames[0].lower():
                aperRad = 100
            else:
                aperRad = 10

        medianCenter   = median(self.centering_FluxWeight, axis=0)
        aperture       = CircularAperture(medianCenter, aperRad)
        backgroundMask = abs(aperture.get_fractions(np.ones(self.imageCube[0].shape))-1)
        backgroundMask[backgroundMask == 0] = nan

        self.background_CircleMask = self.metric(self.imageCube*backgroundMask,axis=(1,2))

        self.background_df['CircleMask'] = self.background_CircleMask

    def measure_background_annular_mask(self, innerRad=None, outerRad=None, method='mean'):

        if innerRad is None:
            if 'wlp' in self.fitsFilenames[0].lower():
                innerRad = 100
            else:
                innerRad = 10

        if outerRad is None:
            if 'wlp' in self.fitsFilenames[0].lower():
                outerRad = 150
            else:
                outerRad = 15

        medianCenter  = median(self.centering_LeastAsym, axis=0)

        innerAperture = CircularAperture(medianCenter, innerRad).get_fractions(np.ones(self.imageCube[0].shape))
        outerAperture = CircularAperture(medianCenter, outerRad).get_fractions(np.ones(self.imageCube[0].shape))

        backgroundMask= abs((outerAperture - innerAperture))
        backgroundMask[backgroundMask == 0] = nan

        self.background_Annulus = self.metric(self.imageCube*backgroundMask, axis=(1,2))
        self.background_df['AnnularMask'] = self.background_Annulus

    def measure_background_median_masked(self, aperRad=None, nSig=5, method='mean'):

        if aperRad is None:
            if 'wlp' in self.fitsFilenames[0].lower():
                aperRad = 100
            else:
                aperRad = 10

        self.background_MedianMask  = np.zeros(self.nSlopeFiles)

        medianCenter   = median(self.centering_FluxWeight, axis=0)
        aperture       = CircularAperture(medianCenter, aperRad)
        backgroundMask = abs(aperture.get_fractions(np.ones(self.imageCube[0].shape))-1)

        for kframe in range(self.nSlopeFiles):
            medFrame  = median(self.imageCube[kframe])
            madFrame  = scale.mad(self.imageCube[kframe])

            medianMask= abs(self.imageCube[kframe] - medFrame) < nSig*madFrame

            maskComb  = medianMask*backgroundMask
            maskComb[maskComb == 0] = nan

            self.background_MedianMask[kframe] = self.metric(self.imageCube[kframe]*maskComb)

        self.background_df['MedianMask'] = self.background_MedianMask

    def measure_background_KDE_Mode(self, aperRad=None):

        if aperRad is None:
            if 'wlp' in self.fitsFilenames[0].lower():
                aperRad = 100
            else:
                aperRad = 10

        self.background_KDEUniv = np.zeros(self.nSlopeFiles)

        medianCenter   = median(self.centering_FluxWeight, axis=0)
        aperture       = CircularAperture(medianCenter, aperRad)
        backgroundMask = abs(aperture.get_fractions(np.ones(self.imageCube[0].shape))-1)

        for kframe in range(self.nSlopeFiles):
            frameNow = (self.imageCube[kframe]*backgroundMask).ravel()
            kdeFrame = kde.KDEUnivariate(frameNow[np.where(backgroundMask.ravel() != 0.0)])
            kdeFrame.fit()

            self.background_KDEUniv[kframe] = kdeFrame.support[kdeFrame.density.argmax()]

        self.background_df['KDEUnivMask'] = self.background_KDEUniv

    def measure_all_background(self, nSig=5):
        print('Measuring Background Using Circle Mask')
        self.measure_background_circle_masked()
        print('Measuring Background Using Annular Mask')
        self.measure_background_annular_mask()
        print('Measuring Background Using Median Mask')
        self.measure_background_median_masked(nSig=nSig)
        print('Measuring Background Using KDE Mode')
        self.measure_background_KDE_Mode()

    def compute_flux_over_time(self, aperRad=None, centering='LeastAsymmetry', background='AnnularMask'):
        y,x = 0,1

        if background not in self.background_df.columns:
            raise Exception("`background` must be in", self.background_df.columns)

        if centering not in ['Gaussian_Fit', 'Gaussian_Mom', 'FluxWeighted', 'LeastAsymmetry']:
            raise Exception("`centering` must be either 'Gaussian_Fit', 'Gaussian_Mom', 'FluxWeighted', or 'LeastAsymmetry'")

        if aperRad is None:
            if 'wlp' in self.fitsFilenames[0].lower():
                aperRad = 70
            else:
                aperRad = 3

        centering_Use = np.transpose([self.centering_df[centering + '_Y_Centers'],
                                      self.centering_df[centering + '_X_Centers']])

        background_Use= self.background_df[background]

        flux_key_now  = centering + '_' + background+'_' + 'rad' + '_' + str(aperRad)
        flux_TSO_now  = np.zeros(self.nSlopeFiles)
        noise_TSO_now = np.zeros(self.nSlopeFiles)
        for kframe in range(self.nSlopeFiles):
            frameNow  = np.copy(self.imageCube[kframe]) - background_Use[kframe]
            frameNow[np.isnan(frameNow)] = median(frameNow)

            noiseNow  = np.copy(self.noiseCube[kframe])**2.
            noiseNow[np.isnan(noiseNow)] = median(noiseNow)

            aperture  = CircularAperture([centering_Use[kframe][x], centering_Use[kframe][y]], r=aperRad)

            flux_TSO_now[kframe]  = aperture_photometry(frameNow, aperture)['aperture_sum']
            noise_TSO_now[kframe] = sqrt(aperture_photometry(noiseNow, aperture)['aperture_sum'])

        self.flux_TSO_df[flux_key_now]  = flux_TSO_now
        self.noise_TSO_df[flux_key_now] = noise_TSO_now

tm_year, tm_mon, tm_mday, tm_hour, tm_min, tm_sec, tm_wday, tm_yday, tm_isdst = localtime()
print('Completed Class Definition at ' +
      str(tm_year) + '-' + str(tm_mon) + '-' + str(tm_mday) + ' ' + \
      str(tm_hour) + 'h' + str(tm_min) + 'm' + str(tm_sec) + 's')

ppm             = 1e6
y,x             = 0,1

yguess, xguess  = 160., 167. # Specific to JWST WLP Test Data
filetype        = 'slp.fits' # Specific to JWST WLP Test Data

dataDir     = '/path/to/fits/files/main/directory/'
fitsFileDir = 'path/to/fits/subdirectories/'

loadfitsdir = dataDir + fitsFileDir

method = 'mean'
example_wanderer_mean = wanderer(fitsFileDir=loadfitsdir, filetype=filetype,
                                            yguess=yguess, xguess=xguess, method=method)

example_wanderer_mean.load_data_from_save_files(savefiledir='./SaveFiles/',
                                                     saveFileNameHeader='Example_Wanderer_Mean_', saveFileType='.pickle.save')

method = 'median'
example_wanderer_median = wanderer(fitsFileDir=loadfitsdir_ModA, filetype=filetype,
                                            yguess=yguess, xguess=xguess, method=method)

example_wanderer_median.load_data_from_save_files(savefiledir='./SaveFiles/', saveFileNameHeader='Example_Wanderer_Median_', saveFileType='.pickle.save')

dataDir     = '/path/to/fits/files/main/directory/'
fitsFileDir = 'path/to/fits/subdirectories/'

loadfitsdir = dataDir + fitsFileDir

method = 'median'

print('Initialize an instance of `wanderer` as `example_wanderer_median`\n')
example_wanderer_median = wanderer(fitsFileDir=loadfitsdir_ModB, filetype=filetype,
                                            yguess=yguess, xguess=xguess, method=method)

print('Load Data From Fits Files in ' + fitsFileDir_ModB + '\n')
example_wanderer_median.load_data_from_fits_files()

print('Skipping Load Data From Save Files in ' + fitsFileDir_ModB + '\n')
# example_wanderer_median.load_data_from_save_files()

print('Find, flag, and NaN the "Bad Pixels" Outliers' + '\n')
example_wanderer_median.find_bad_pixels()

print('Fit for All Centers: Flux Weighted, Gaussian Fitting, Gaussian Moments, Least Asymmetry' + '\n')
# example_wanderer_median.fit_gaussian_fitting_centering()
# example_wanderer_median.fit_flux_weighted_centering()
# example_wanderer_median.fit_least_asymmetry_centering()
example_wanderer_median.fit_all_centering()

print('Measure Background Estimates with All Methods: Circle Masked, Annular Masked, KDE Mode, Median Masked' + '\n')
# example_wanderer_median.measure_background_circle_masked()
# example_wanderer_median.measure_background_annular_mask()
# example_wanderer_median.measure_background_KDE_Mode()
# example_wanderer_median.measure_background_median_masked()
example_wanderer_median.measure_all_background()

print('Iterating over Background Techniques, Centering Techniques, Aperture Radii' + '\n')
background_choices = example_wanderer_median.background_df.columns
centering_choices  = ['Gaussian_Fit', 'Gaussian_Mom', 'FluxWeighted', 'LeastAsymmetry']
aperRads           = np.arange(1, 100.5,0.5)

start = time()
for bgNow in background_choices:
    for ctrNow in centering_choices:
        for aperRad in aperRads:
            print('Working on Background ' + bgNow + ' with Centering ' + ctrNow + ' and AperRad ' + str(aperRad), end=" ")
            example_wanderer_median.compute_flux_over_time(aperRad=aperRad, centering=ctrNow, background=bgNow)
            flux_key_now  = ctrNow + '_' + bgNow+'_' + 'rad' + '_' + str(aperRad)
            print(std(example_wanderer_median.flux_TSO_df[flux_key_now] / median(example_wanderer_median.flux_TSO_df[flux_key_now]))*ppm)

print('Operation took: ', time()-start)

print('Saving `example_wanderer_median` to a set of pickles for various Image Cubes and the Storage Dictionary')
example_wanderer_median.save_data_to_save_files(savefiledir='./SaveFiles/', saveFileNameHeader='Example_Wanderer_Median_', saveFileType='.pickle.save')

method = 'mean'

print('Initialize an instance of `wanderer` as `example_wanderer_mean`')
example_wanderer_mean = wanderer(fitsFileDir=loadfitsdir_ModB, filetype = filetype,
                                yguess=yguess, xguess=xguess, method=method)

print('Load Data From Fits Files in ' + loadfitsdir)
example_wanderer_mean.load_data_from_fits_files()

print('Skipping Load Data From Save Files in ' + loadfitsdir)
# example_wanderer_mean.load_data_from_save_files()

print('Find, flag, and NaN the "Bad Pixels" Outliers')
example_wanderer_mean.find_bad_pixels()

print('Fit for All Centers: Flux Weighted, Gaussian Fitting, Gaussian Moments, Least Asymmetry')
# example_wanderer_mean.fit_gaussian_fitting_centering()
# example_wanderer_mean.fit_flux_weighted_centering()
# example_wanderer_mean.fit_least_asymmetry_centering()
example_wanderer_mean.fit_all_centering()

print('Measure Background Estimates with All Methods: Circle Masked, Annular Masked, KDE Mode, Median Masked')
# example_wanderer_mean.measure_background_circle_masked()
# example_wanderer_mean.measure_background_annular_mask()
# example_wanderer_mean.measure_background_KDE_Mode()
# example_wanderer_mean.measure_background_median_masked()
example_wanderer_mean.measure_all_background()

print('Iterating over Background Techniques, Centering Techniques, Aperture Radii')
background_choices = example_wanderer_mean.background_df.columns
centering_choices  = ['Gaussian_Fit', 'Gaussian_Mom', 'FluxWeighted', 'LeastAsymmetry']
aperRads           = np.arange(1, 100.5,0.5)

start = time()
for bgNow in background_choices:
    for ctrNow in centering_choices:
        for aperRad in aperRads:
            print('Working on Background ' + bgNow + ' with Centering ' + ctrNow + ' and AperRad ' + str(aperRad), end=" ")
            example_wanderer_mean.compute_flux_over_time(aperRad=aperRad, centering=ctrNow, background=bgNow)
            flux_key_now  = ctrNow + '_' + bgNow+'_' + 'rad' + '_' + str(aperRad)
            print(std(example_wanderer_mean.flux_TSO_df[flux_key_now] / median(example_wanderer_mean.flux_TSO_df[flux_key_now]))*ppm)

print('Operation took: ', time()-start)

print('Saving `example_wanderer_mean` to a set of pickles for various Image Cubes and the Storage Dictionary')
example_wanderer_mean.save_data_to_save_files(savefiledir='./SaveFiles/', saveFileNameHeader='Example_Wanderer_Mean_', saveFileType='.pickle.save')


from .NArg import NArg
from .NKwarg import NKwarg
from .Lemma import Lemma

class Rule:
    def __init__(self, names, *tests, g_test=None):
        self.names = names.split('|')
        self.tests = tuple(map(self.parse_test, tests))
        self.size = len(tests)
        self.g_test = g_test

    def parse_test(self, test):
        if isinstance(test, str):
            return self.parse_names(test)
            # return lambda item: any(name in test.split('|') for name in item.names)
        else:
            return test

    def get_test(self, index):
        return self.tests[-1 - index]

    def finalize(self, *items):
        if self.g_test is None or self.g_test(*items):
            args = []
            kwargs = {}

            for item, test in zip(items, self.tests):
                if isinstance(test, NArg):
                    test.build(item, args, kwargs)
            # print(args, kwargs, file=sys.stderr)
            return Lemma(self, *args, **kwargs)
        else:
            return None

    def parse_names(self, names):
        if ':' in names:
            return NKwarg(names)
        else:
            return NArg(names)

    def build(self, parser, item, *items):
        if self.get_test(len(items))(item):
            # if self.tests[-1 - len(items)](item):
            if 0 <= len(items) < self.size - 1:
                for prev in parser.get_by_end(item.start):
                    for res in self.build(parser, prev, item, *items):
                        yield res
            elif len(items) == self.size - 1:
                res = self.finalize(item, *items)
                if res:
                    yield res
                # if self.g_test is None or self.g_test(item, *items):
                #     yield Lemma(self, item, *items)
            else:
                raise Exception


import tensorflow
import pickle
import numpy

def get_dataset_images(test_path_path, im_dim=32, num_channels=3):
    """
    Similar to the one used in training except that there is just a single testing binary file for testing the CIFAR10 trained models.
    """
    print("Working on testing patch")
    data_dict = unpickle_patch(test_path_path)
    images_data = data_dict[b"data"]
    dataset_array = numpy.reshape(images_data, newshape=(len(images_data), im_dim, im_dim, num_channels))
    return dataset_array, data_dict[b"labels"]

def unpickle_patch(file):
    """
    Identical to the one used in training.
    """
    patch_bin_file = open(file, 'rb')
    patch_dict = pickle.load(patch_bin_file, encoding='bytes')
    return patch_dict

#***********************************************************************************
#Dataset path containing the testing binary file to be decoded.
patches_dir = "C:\\Users\\Dell\\Downloads\\Compressed\\cifar-10-python\\cifar-10-batches-py\\"
dataset_array, dataset_labels = get_dataset_images(test_path_path=patches_dir + "test_batch", im_dim=32, num_channels=3)
print("Size of data : ", dataset_array.shape)

sess = tensorflow.Session()

#Restoring the previously saved trained model.
saved_model_path = 'C:\\Users\\Dell\\Desktop\\model\\'
saver = tensorflow.train.import_meta_graph(saved_model_path+'model.ckpt.meta')
saver.restore(sess=sess, save_path=saved_model_path+'model.ckpt')

#Initalizing the varaibales.
sess.run(tensorflow.global_variables_initializer())

graph = tensorflow.get_default_graph()

"""
Restoring previous created tensors in the training phase based on their given tensor names in the training phase.
Some of such tensors will be assigned the testing input data and their outcomes (data_tensor, label_tensor, and keep_prop).
Others are helpful in assessing the model prediction accuracy (softmax_propabilities and softmax_predictions).
"""
softmax_propabilities = graph.get_tensor_by_name(name="softmax_probs:0")
softmax_predictions = tensorflow.argmax(softmax_propabilities, axis=1)
data_tensor = graph.get_tensor_by_name(name="data_tensor:0")
label_tensor = graph.get_tensor_by_name(name="label_tensor:0")
keep_prop = graph.get_tensor_by_name(name="keep_prop:0")

#keep_prop is equal to 1 because there is no more interest to remove neurons in the testing phase.
feed_dict_testing = {data_tensor: dataset_array,
                     label_tensor: dataset_labels,
                     keep_prop: 1.0}
#Running the session to predict the outcomes of the testing samples.
softmax_propabilities_, softmax_predictions_ = sess.run([softmax_propabilities, softmax_predictions],
                                                      feed_dict=feed_dict_testing)
#Assessing the model accuracy by counting number of correctly classified samples.
correct = numpy.array(numpy.where(softmax_predictions_ == dataset_labels))
correct = correct.size
print("Correct predictions/10,000 : ", correct)

#Closing the session
sess.close()


from convert_to_file_name.convert_to_safe_file_name import convert_to_file_name
import os

from bs4 import BeautifulSoup
from urllib.request import Request, urlopen

def scrape(fchap, lchap,site):
    """
    Takes a first chapter and last chapter along with what chapter to begin scraping from.(Site is a link to that chapter)
    :param fchap:
    :param lchap:
    :param site:
    :return:
    """
    baseSite = "https://www.wuxiaworld.com"

    for x in range(fchap, lchap):
        hdr = {'User-Agent': 'Mozilla/5.0'}
        req = Request(site, headers=hdr)

        page = urlopen(req)
        soup = BeautifulSoup(page)
        folder = soup.find(class_='caption').find('h4')

        folder = convert_to_file_name(str(folder.text))

        save_path = './'+str(folder)+'/'
        if not os.path.exists(save_path):
            os.mkdir(save_path)

        "Grabs a link to the next chapter"
        nextChapter = soup.find(class_='next').find(class_='btn btn-link').get('href')

        "Finds all the chapter content of the current chapter"
        results = soup.find('div', id='chapter-content').find_all("p")

        "Gets the current chapter header"
        chapter = soup.find('div', id='chapter-outer').find('h4')

        site = baseSite + nextChapter
        textfile = "Chapter " + str(x) + ".xhtml"
        completeName = os.path.join(save_path, textfile)

        f = open(completeName, "a", encoding='utf-8')
        f.write(str(chapter))

        for y in results:

            "Checks if the content was actually scraped successfully. If not it tries again"
            if results != None:
                f.write(str(y))

            else:
                x = x - 1

        site = baseSite + nextChapter

        f.close()

if __name__ == '__main__':
    scrape(1,5,'https://www.wuxiaworld.com/novel/second-life-ranker/slr-chapter-1')


'''
Nessa aula, vamos aprender como utilizar a instrução break e os loopings infinitos a favor das nossas
estratégias de código. É muito importante saber usar o break no Python, já que em alguns casos precisamos
interromper um laço no meio do caminho.

Além disso, vamos aprender como trabalhar com as novas fstrings do Python.

cont  = 1
while cont <= 10:
    print(f'{cont}, ', end='')
    cont = cont + 1
print(f'Acabou.')

c = 0
n = 0
while c < 3:
    n = int(input(f'Informe-nos um número: '))
    c = c + 1
print(f'FIM.')

s = 0
n = 0
while n != 999:
    n = int(input(f'Informe-nos um número: '))
    s = s + n
# s = s - 999 gambiarra
print(f'A soma é {s}.')
print(f'FIM.')
'''
s = 0
n = 0
while True:
    n = int(input(f'Informe-nos um número: '))
    if n == 999:
        break
    s = s + n
#print('A some é {}'.format(s)(s))
print(f'A soma é {s}.')
print(f'FIM.')

Aulas/Aula 15 – Interrompendo repetições while.py


import socket

def main():
    host = socket.gethostbyname(socket.gethostname())
    print(host)
    s = socket.socket()
    s.connect((host, 15001))
    out = s.makefile("w")
    out.write('1 GETPOS\n')
    out.write('2 GOTOLOC VENDDOOR\n')
    out.write('3 NAVTOLOC VENDDOOR\n')
    out.write('4 NAVTOXY 74.3904 36.6048\n')
    #Should return an ERROR condition
    out.write('5 NAVTOLOC NOWHERE\n')
    s.close()

if __name__ == '__main__':
    main()


import logging
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
import pandas as pd

from ml.ml import ml
from models.metric import Metric
from models.version import Version
from utils.database import get_included_and_current_versions_filter
from utils.timeit import timeit

class BugVelocity(ml):
    """
    BugVelocity is a simple Machine Learning model (a bit naive) based on the history of
    bug velocity values. It demonstrate how you can integrate your own model into the tool.
    """

    def __init__(self, project_id, session, config):
        ml.__init__(self, project_id, session, config)
        self.name = "bugvelocity"

    @timeit
    def train(self):
        """Train the model"""
        logging.info("BugVelocity:train")

        included_versions = self.configuration.include_versions
        excluded_versions = self.configuration.exclude_versions

        releases_statement = self.session.query(Version). \
            order_by(Version.start_date.asc()) \
            .filter(Version.project_id == self.project_id) \
            .filter(Version.include_filter(included_versions)) \
            .filter(Version.exclude_filter(excluded_versions)) \
            .filter(Version.name != self.configuration.next_version_name).statement
        df = pd.read_sql(releases_statement, self.session.get_bind())
        X=df[['bug_velocity']]
        y=df[['bugs']].values.ravel()

        # Model: RandomForestRegressor
        self.model = RandomForestRegressor(n_estimators=200, random_state=1043)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=5)
        self.model.fit(X_train, y_train)
        y_pred = self.model.predict(X_test)
        rdmForest_predictions = [round(value) for value in y_pred]
        self.mse = mean_squared_error(y_test, rdmForest_predictions)
        logging.info("BugVelocity: Mean Square Error : " + str(self.mse))
        self.store()

    @timeit
    def predict(self)->int:
        """Predict the next value"""
        logging.info("BugVelocity::predict")
        self.restore()  # unpickle the model
        bug_velocity = self.session.query(Version.bug_velocity). \
            filter(Version.project_id == self.project_id). \
            filter(Version.name == self.configuration.next_version_name).scalar()
        d = {'bug_velocity': [bug_velocity]}
        X_test = pd.DataFrame(data=d)
        prediction_df = self.model.predict(X_test)
        value = round(prediction_df[0])
        return value


import matplotlib
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from mpl_toolkits.basemap import Basemap

class GIS_Map_Viz:

    def __init__(self, latitude_feature_name=None, longitude_feature_name=None,
                 gps_bounderies_dict={}):
        '''create a GIS gps map of the longitude and latitude values provided within the boundaries '''

        # Save data features by type and label
        print("GIS_Map_Viz: __init__ ... v2")

        self.latitude_feature_name = latitude_feature_name
        self.longitude_feature_name = longitude_feature_name
        self.gps_bounderies_dict = gps_bounderies_dict

    def _display_gps_map(self, df, labels, map_title):
        '''print the longitude & latitude GPS coordinates of the dataframe within the map bounderies'''

        print("GIS_Map_Viz: _display_gps_map ...")
        fig = plt.figure(figsize=(20, 10))
        plt.title(map_title)

        m = Basemap(
            projection='merc',
            llcrnrlat=self.gps_bounderies_dict['lat_min'] - 0.5,
            urcrnrlat=self.gps_bounderies_dict['lat_max'] + 0.5,
            llcrnrlon=self.gps_bounderies_dict['lon_min'] - 0.5,
            urcrnrlon=self.gps_bounderies_dict['lon_max'] + 0.5,
            resolution='i')

        # Reference: https://matplotlib.org/basemap/users/geography.html
        m.drawmapboundary(fill_color='#85A6D9')
        m.drawcoastlines(color='#6D5F47', linewidth=.8)
        m.drawrivers(color='green', linewidth=.4)
        m.shadedrelief()
        m.drawcountries()
        m.fillcontinents(lake_color='aqua')

        mycmap = matplotlib.colors.LinearSegmentedColormap.from_list(
            "", ["green", "yellow", "red"])
        longitudes = df[self.longitude_feature_name].tolist()
        latitudes = df[self.latitude_feature_name].tolist()
        if labels is None:
            labels = 'darkblue'
            wp = mpatches.Patch(color='darkblue', label='water points')
            plt.legend(handles=[wp], title='Location')
        else:
            labels = labels.cat.codes
            wp_functional = mpatches.Patch(
                color='green', label='water points: functional')
            wp_need_repair = mpatches.Patch(
                color='red', label='water points: non functional')
            wp_non_functional = mpatches.Patch(
                color='yellow', label='water points: functional needs repair')
            plt.legend(handles=[wp_functional, wp_need_repair,
                                wp_non_functional],
                       title='Location and Status')

        m.scatter(
            longitudes,
            latitudes,
            s=0.05,
            zorder=2,
            latlon=True,
            c=labels,
            cmap=mycmap)
        plt.show()


from django.core.mail import BadHeaderError, send_mail
from django.http import HttpResponse, HttpResponseRedirect

def test_send_mail(request):
    subject = request.POST.get('subject', 'send_mail')
    message = request.POST.get('message', 'mensaje enviado utilizando send_mail')
    from_email = request.POST.get('from_email', 'info@lubresrl.com.ar')
    if subject and message and from_email:
        try:
            send_mail(subject, message, from_email, ['info@lubresrl.com.ar'])
        except BadHeaderError:
            return HttpResponse('Encabezado inválido.')
        # return HttpResponseRedirect('/contact/thanks/')
        return HttpResponse('Mensaje enviado correctamente.')
    else:
        return HttpResponse('Asegúrese que todos los campos estén ingresados y sean válidos.')

from django.template import loader
from django.contrib.auth.models import User

def test_send_mail_template(request):
    subject = request.POST.get('subject', 'Gracias por usar email')
    message = request.POST.get('message', 'Mensaje enviado utilizando send_mail')
    from_email = request.POST.get('from_email', 'info@lubresrl.com.ar')
    html_message = loader.render_to_string(
                        'mails/ejemplo.html',
                        {
                            # config
                            'logo_url': 'https://lubresrl.com.ar/static/favicon.png',
                            'footer_content': '
Ruta 9 km 1306 - T4101 Los Nogales - Tucuman, Argentina
',
                            'facebook_url': 'https://www.facebook.com/lubresrl.ypfagro/',
                            'twitter_url': 'https://ar.linkedin.com/company/lubre-srl',
                            'instagram_url': 'https://www.instagram.com/lubresrl.ypfagro/',
                            'website_url': 'https://lubresrl.com.ar',
                            # css
                            'color_header_bg': '#f7f7f7',
                            'color_title': '#222222',
                            'title_size': 'h1',
                            'color_body_bg': '#ffffff',
                            'color_body': '#808080',
                            'color_body_link': '#007e9e',
                            'color_button': '#ffffff',
                            'color_button_bg': '#00add8',
                            'border_radius_button': '3',
                            'color_footer': '#ffffff',
                            'color_footer_link': '#ffffff',
                            'color_footer_bg': '#333333',
                            'color_footer_divider': '#505050',
                            # data
                            'subject': subject,
                            'title': 'Título',
                            'body': message,
                            'banner_url': 'https://lubresrl.com.ar/static/img/ypf_agro.png',
                            'button_link': 'https://lubresrl.com.ar/',
                            'button_label': 'Lubre SRL',
                            # custom
                            'user_name': 'Usuario',     # User.username,
                        }
                   )
    try:
        send_mail(subject, message, from_email, ['info@lubresrl.com.ar'],
                fail_silently=True, html_message=html_message
        )
        return HttpResponse('Mensaje enviado correctamente.')
    except BadHeaderError:
        return HttpResponse('Asegúrese que todos los campos estén ingresados y sean válidos.')

# django-templated-email
from templated_email import send_templated_mail
from decouple import config

def test_simple_mail(request):
    # template='welcome'
    template='signup'
    try:
        send_templated_mail(
            template_name=template,
            from_email='info@lubresrl.com.ar',
            recipient_list=['info@lubresrl.com.ar'],
            context={
                'username': request.user.username,
                'email': request.user.email,
                'date_joined': request.user.date_joined
            },
            # Optional:
            # cc=['cc@example.com'],
            # bcc=['bcc@example.com'],
            # headers={'My-Custom-Header':'Custom Value'},
            # template_prefix="my_emails/",
            # template_suffix="email",
        )
        return HttpResponse('Mensaje enviado correctamente.')
    except BadHeaderError:
        return HttpResponse('Asegúrese que todos los campos estén ingresados y sean válidos.')

def signup_mail(to, username, joined):
    try:
        from_email = config('DJANGO_EMAIL_HOST_USER')
        send_templated_mail(
            template_name = 'signup',
            from_email = from_email,
            recipient_list = [to],
            context={
                'username': username,
                'email': to,
                'date_joined': joined,
            },
        )
    except:
        raise


from pprint import pprint
import boto3

def put_movie(title, year, plot, rating, dynamodb=None):
    if not dynamodb:
        dynamodb = boto3.resource('dynamodb', endpoint_url="http://localhost:8000")

    table = dynamodb.Table('Movies')
    response = table.put_item(
       Item={
            'year': year,
            'title': title,
            'info': {
                'plot': plot,
                'rating': rating
            }
        }
    )
    return response

if __name__ == '__main__':
    movie_resp = put_movie("The Big New Movie", 2015,
                           "Nothing happens at all.", 0)
    print("Put movie succeeded:")
    pprint(movie_resp, sort_dicts=False)
# snippet-end:[dynamodb.python.codeexample.MoviesItemOps01]


from flask import Flask, request
from config import *
import requests
import icalendar

app = Flask(__name__)

def display(cal):
    return cal.to_ical().replace('\r\n', '\n').strip()

@app.route('/icalparser')
def icalparser():
    original = requests.get('{}'.format(url)).text
    cal = icalendar.Calendar.from_ical(original)

    new_cal = icalendar.Calendar()

    for event in cal.walk('vevent'):
        for words in blacklist:
            if all(word in event['SUMMARY'] for word in words):
                break
        else:
            new_cal.add_component(event)

    print('success')
    return (new_cal.to_ical(), 200)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)


# B - Bite Eating

N, L = map(int, input().split())
ans = 0
mn = float('inf')
for i in range(1, N + 1):
    ans += L + i - 1
    if abs(L + i - 1) < abs(mn):
        mn = L + i - 1

print(ans - mn)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import logging
import os
import pickle

from .name import Name

logger = logging.getLogger(__name__)
logger.setLevel("DEBUG")

class Structure:
    """Metaclass that creates a collection of :meth:`Name<crif_pyton_libraries.check_columns.Name>` instances

    For each dictionary inside columns list, Columns create an istance of
    :meth:`Name<crif_pyton_libraries.check_columns.Name>` and set its properties.
    Each Name instance is also treated as a Columns property.

    Attributes:
        names (list): an ordered list of all the names contained in columns list.
        domains (set): a set of available domains
    """

    def __init__(self, name_list, *args, **kwargs):
        """Constructor method for Columns

        Args:
            name_list (list): Each dictionary inside columns list
                must respect :envvar:`ordered_columns` syntax.
        """
        self.names = []
        self.domains = set()
        for name_definition in name_list:

            if name_definition["name"] in self.names:
                raise ValueError(
                    "Name {} already present".format(name_definition["name"])
                )

            self._add_name(name_definition)

            for domain in name_definition["domains"]:
                self._add_domain(domain=domain)

    def _add_domain(self, domain):
        self.domains.add(domain)

    def _add_name(self, name_definition):
        name = Name(**name_definition)
        self.names.append(name.name)
        setattr(self, name.name, name)

    def get_names(self, domain=None):
        """Select column names from self.names respecting insertion order.

        Domain and selected arguments can be used to select a sample of names.

        Args:
            domain (str): domain of expected names. Domains refers to domain
                property of :meth:`Name<crif_pyton_libraries.check_columns.Name>`

        Returns:
            selection (list): list of selected columns.

        """

        selection = []
        for name in self.names:

            target = getattr(self, name)

            if not isinstance(target, Name):
                continue

            if domain and domain not in target.domains:
                continue

            selection.append(name)

        return selection

    def __getitem__(self, name):
        return self.__dict__[name]

    def get(self, name, default=None):
        return self.__dict__.get(name, default)

    def dump(self, filename: str, overwrite=False):
        if not filename.endswith(".pkl"):
            filename += ".pkl"

        logger.info("Saving on {}".format(filename))

        if os.path.exists(filename):
            if not overwrite:
                msg = "{} ALREADY EXISTS! I'M NOT ABLE TO OVERWRITE!".format(filename)
                logger.info(msg)
                raise FileExistsError(msg)

            logger.info("Overwriting {}...".format(filename))

        with open(filename, "wb") as output:
            pickle.dump(self.__dict__, output)

    def load(self, filename):
        logger.debug("Loading from {}".format(filename))
        if not os.path.exists(filename):
            logger.error("{} DOES NOT EXISTS!".format(filename))
            raise FileNotFoundError("{} DOES NOT EXISTS!".format(filename))

        with open(filename, "rb") as input_file:
            self.__dict__ = pickle.load(input_file)


# import sys; input = sys.stdin.readline

def find(n):
    global parent
    if parent[n] != n:
        parent[n] = find(parent[n])
    return parent[n]

def union(a, b):
    global rank, parent
    if rank[a] > rank[b]:
        parent[b] = a
    else:
        parent[a] = b
        if rank[a] == rank[b]:
            rank[b] += 1

def cycle_check(a, b):
    global rank, parent
    a, b = find(a), find(b)
    if a == b:
        return True
    union(a, b)
    return False

def kruskal(N, M, adj_list):
    cost = 0
    cnt = 0
    for i in range(M):
        c, a, b = adj_list[i]
        if cycle_check(a, b): continue # 사이클이 생긴 경우
        cost += c
        cnt += 1

    if cnt == N - 1: # 건물이 N개이므로 N-1개의 도로가 있어야함
        return cost
    return -1

def main():
    # 0. 입력
    global parent, rank
    N, M = map(int, input().split())
    parent = list(range(N + 1))
    rank = [0 for _ in range(N + 1)]
    roads = []
    max_cost = 0
    for _ in range(M):
        a, b, c = map(int, input().split())
        roads.append((c, a, b))
        max_cost += c
    roads.sort()
    # 1. 최소 신장트리
    ans = kruskal(N, M, roads)
    # 2. 출력
    if ans == -1:
        print(-1)
    else:
        print(max_cost - ans)

if __name__ == "__main__":
    main()


import os
import shutil
from src.db.config import sqlite_path
from src.models.user import User
from src.models.category import Category
from src.models.product import Product
from src.models.product_info import ProductInfo
from src.models.product_picture import ProductPicture

def create_db(app, db):
    with app.app_context():
        if not os.path.isfile(sqlite_path):

            db.drop_all()
            db.create_all()

            add_user(db)
            add_category(db)
            add_product(db)

            db.session.commit()

def add_user(db):
    db.session.add(User("David"))
    db.session.add(User("Renata"))

def add_category(db):
    db.session.add(Category("Bebidas"))

def add_product(db):
    db.session.add(Product(1, "Heineken Lata 350ml", "Heineken International é uma cervejaria holandesa, fundada em 1863 por Gerard Adriaan Heineken na cidade de Amsterdã.", 100, 4.39, 0))
    db.session.add(ProductInfo(1, "Observação", "A venda e o consumo de bebidas alcoólicas são proibidos para menores de 18 anos. Beba com moderação. Se for dirigir, não beba!"))
    db.session.add(ProductInfo(1, "País de Origem", "Holanda"))
    db.session.add(ProductInfo(1, "Embalagem", "Lata verde"))
    db.session.add(ProductPicture(1, "heineken.png"))

    db.session.add(Product(1, "Original Lata 350ml", "Cerveja Pilsen Antarctica Original, harmonize suas comemorações com um sabor mais suave e refrescante!", 100, 3.49, 0))
    db.session.add(ProductInfo(2, "Observação", "A venda e o consumo de bebidas alcoólicas são proibidos para menores de 18 anos. Beba com moderação. Se for dirigir, não beba!"))
    db.session.add(ProductInfo(2, "País de Origem", "Brasil"))
    db.session.add(ProductInfo(2, "Embalagem", "Lata branca"))
    db.session.add(ProductPicture(2, "original.png"))

    db.session.add(Product(1, "Budweiser Long Neck 350ml", "Budweiser, também conhecida popularmente como Bud, é uma cerveja do tipo long americana, fabricada pela AB InBev, fundada em 1876", 100, 3.49, 0))
    db.session.add(ProductInfo(3, "Observação", "A venda e o consumo de bebidas alcoólicas são proibidos para menores de 18 anos. Beba com moderação. Se for dirigir, não beba!"))
    db.session.add(ProductInfo(3, "País de Origem", "Brasil"))
    db.session.add(ProductInfo(3, "Embalagem", "Lata branca"))
    db.session.add(ProductPicture(3, "budweiser.png"))

    db.session.add(Product(1, "Imperial Garrafa 600ml", "Cerveja ouro imperial garrafa 600ml, harmonize suas comemorações com um sabor mais suave e refrescante!", 100, 3.49, 0))
    db.session.add(ProductInfo(4, "Observação", "A venda e o consumo de bebidas alcoólicas são proibidos para menores de 18 anos. Beba com moderação. Se for dirigir, não beba!"))
    db.session.add(ProductInfo(4, "País de Origem", "Brasil"))
    db.session.add(ProductInfo(4, "Embalagem", "Lata branca"))
    db.session.add(ProductPicture(4, "imperial.png"))

02 - Flask/99 - Project/src/db/migrations.py


from Snowman3.utilities.PySignal import Signal

build_directory_changed = Signal()

part_owner_signals = dict(  # PartView
    start_set=Signal(),  # Before New Part
    start_remove=Signal(),  # Before Delete Part
    start_move=Signal(),  # Before Change Part Owner
    end_set=Signal(),  # After New Part
    end_remove=Signal(),  # After Delete Part
    end_move=Signal(),  # After Change Part Owner
)

part_hierarchy_signals = dict(  # Hierarchy View
    start_set=Signal(),  # Before New Part
    start_remove=Signal(),  # Before Delete Part
    start_move=Signal(),  # Before Change Hierarchy Parent
    end_set=Signal(),  # After New Part
    end_remove=Signal(),  # After Delete Part
    end_move=Signal(),  # After Change Hierarchy Parent
)

root_signals = dict(  # Hierarchy View
    start_change=Signal(),  # Before New/Delete Root
    end_change=Signal(),  # After New/Delete Root,
)

controller_signals = dict(  # Hierarchy View
    critical_error=Signal(),  # System breaking error
    reset=Signal()
)

maya_callback_signals = dict(
    pre_file_new_or_opened=Signal(),  # Hooks into mayas new scene callback
    selection_changed=Signal()  # Hooks into mayas selectionChanged callback
)

gui_signals = dict(
    reload_rig_builder=Signal(),  # Informs high level guis that the rig has changed in some way
    saved_time_signal=Signal(),  # Update gui with saved time
    info_notification=Signal(),  # Triggers Notification Widget to give a visual info (blue) message on the gui
    success_notification=Signal(),  # Triggers Notification Widget to give a visual success (green) message on the gui
    warning_notification=Signal(),  # Triggers Notification Widget to give a visual warning (orange) message on the gui
    error_notification=Signal(),  # Triggers Notification Widget to give a visual error (red) message on the gui
    party_notification=Signal()   # Triggers Notification Widget to give a visual party (pink) message on the gui
)
face_network_signals = dict(
    network_about_to_change=Signal(),
    network_finished_change=Signal(),
    group_start_ownership=Signal(),
    group_end_ownership=Signal(),
    group_start_disown=Signal(),
    group_end_disown=Signal(),
    corrective_start_ownership=Signal(),
    corrective_end_ownership=Signal(),
    corrective_start_disown=Signal(),
    corrective_end_disown=Signal(),
    item_changed=Signal()
)

def set_build_directory(current_build_directory):
    import Snowman3.rigger.rig_factory.environment as env  # Move env.current_build_directory to a separate module...
    env.local_build_directory = current_build_directory
    build_directory_changed.emit(current_build_directory)


class Solution:
    def minSetSize(self, arr: List[int]) -> int:
        c=0
        n=int(len(arr)/2)
        b=[]
        x=Counter(arr)
        w=sorted(x.values(),reverse=True )

        for i in range(len(w)):
            if c+w[i]>=n:
                i=i+1
                break
            c+=w[i]

        return i


#!/usr/bin/env python3

import os
import sys
import argparse
import logging
from barcode import Barcode, BarcodeTag, BarcodeStat
from fastq import FastqReader, FastqRecord, FastqFileStat
import util
from fitness import BarseqLayout, Fitness

class Context:
    BARCODE_STAT_FNAME_SUFFIX = '.bstat.tsv'
    FSCORE_BASE_FILE_NAME = 'fscore_base.tsv'
    BARSEQ_LAYOUT_OUT_FILE_NAME = 'barseq_layout.tsv'
    GSCORE_BASE_FILE_NAME = 'gscore_base.tsv'

    LOG_FILE_NAME = 'gscore.log'

    barseq_layout_fname = None
    barseq_bstat_dir = None
    bpag_fname = None
    genes_gff_fname = None
    output_dir = None
    min_time0_read_count = None

    ridge_alpha = None
    lasso_alpha = None
    enet_alpha = None
    enet_l1_ratio = None
    gene_pairs = False

    @staticmethod
    def build_context(args):
        Context.barseq_layout_fname = args.barseq_layout_fname
        Context.barseq_bstat_dir = args.input
        Context.bpag_fname = args.bpag_fname
        Context.output_dir = args.output
        Context.min_time0_read_count = args.min_time0_read_count
        Context.genes_gff_fname = args.genes_gff_fname
        Context.ridge_alpha = args.ridge_alpha
        Context.lasso_alpha = args.lasso_alpha
        Context.enet_alpha = args.enet_alpha
        Context.enet_l1_ratio = args.enet_l1_ratio
        if args.gene_pairs:
            Context.gene_pairs = True

    @staticmethod
    def gscore_base_fname():
        return os.path.join(Context.output_dir, Context.GSCORE_BASE_FILE_NAME)

    @staticmethod
    def fscore_base_fname():
        return os.path.join(Context.output_dir, Context.FSCORE_BASE_FILE_NAME)

    @staticmethod
    def barseq_layout_out_fname():
        return os.path.join(Context.output_dir, Context.BARSEQ_LAYOUT_OUT_FILE_NAME)

    @staticmethod
    def log_fname():
        return os.path.join(Context.output_dir, Context.LOG_FILE_NAME)

def parse_args():

    parser = argparse.ArgumentParser(
        description='''
        The gscore program esimates the fitness score of genes using several methods.

        First, the fintess scores of framgetms are calulcated folloing the approach
        implemnted in the fscore module of the DubSeq package (see the help for the fscore
        program).

        For all genes (both protein coding genes and rnas) listed in the gff file
        (--genes-gff-fname parameter), gene fintess score is calculated using 5 approaches:
        1. Mean score - the score of each gene is calcalted as an average of fitness
        2. CNNLS score (non-negative least squares used to caluclate noth positive and negative scores)
        3. Ridge score
        4. Lasso score
        5. Elastic Net score

        ''',
        formatter_class=util.RawDescriptionArgumentDefaultsHelpFormatter)

    parser.add_argument('-i', '--input',
                        dest='input',
                        help='path to the directory with bstat files produced by the barseq program',
                        type=str,
                        required=True
                        )

    parser.add_argument('-l', '--barseq-layout-fname',
                        dest='barseq_layout_fname',
                        help='path to a file with layout of barseq experiments',
                        type=str,
                        required=True
                        )

    parser.add_argument('-p', '--bpag-fname',
                        dest='bpag_fname',
                        help='path to a file with barcode pairs mapped to a genome using bpag program',
                        type=str,
                        required=True
                        )

    parser.add_argument('-g', '--genes-gff-fname',
                        dest='genes_gff_fname',
                        help='path to a gff file for the genome used to build dubseq library',
                        type=str,
                        required=True
                        )

    parser.add_argument('-o', '--output',
                        dest='output',
                        help='output directory',
                        type=str,
                        required=True
                        )

    parser.add_argument('-t', '--min-time0-read-count',
                        dest='min_time0_read_count',
                        help='The minimal required number of reads supporting a barcode in time zero',
                        default=10,
                        type=int
                        )

    parser.add_argument('--ridge_alpha',
                        dest='ridge_alpha',
                        help='''Regularization parameter alpha for the Ridge regression defining
                        the amount of regularization in the Ridge objective function:
                        ||Ax-y||^2_2 + alpha * ||x||^2_2 ''',
                        default=1.0,
                        type=float
                        )

    parser.add_argument('--lasso_alpha',
                        dest='lasso_alpha',
                        help='''Regularization parameter alpha for the Lasso regression defining
                        the amount of regularization in the Lasso objective function:
                        ||Ax-y||^2_2 + alpha * ||x||_1 ''',
                        default=3.35,
                        type=float
                        )

    parser.add_argument('--enet_alpha',
                        dest='enet_alpha',
                        help='''Regularization parameter alpha for the Elastic Net regression defining
                        the amount of regularization in the Elastic Net objective function:
                        ||Ax-y||^2_2 + alpha * i1_ratio * ||x||_1 + 0.5 * alpha * (1-r1_ratio) * ||x||^2_2  ''',
                        default=3.62,
                        type=float
                        )

    parser.add_argument('--enet_i1_ratio',
                        dest='enet_l1_ratio',
                        help='''Regularization parameter l1_ratio for the Elastic Net regression defining
                        the amount of regularization in the Elastic Net objective function:
                        ||Ax-y||^2_2 + alpha * i1_ratio * ||x||_1 + 0.5 * alpha * (1-r1_ratio) * ||x||^2_2  ''',
                        default=0.7,
                        type=float
                        )
    parser.add_argument('--gene_pairs',
                        dest='gene_pairs',
                        help='''If indicated, the gene paris will be added to the model as variables''',
                        action='store_true')

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    return parser.parse_args()

def check_args(args):
    pass

def main():
    Fitness.MIN_TIME0_READ_COUNT = Context.min_time0_read_count
    Fitness.RIDGE_PARAM_ALPHA = Context.ridge_alpha
    Fitness.LASSO_PARAM_ALPHA = Context.lasso_alpha
    Fitness.ELASTIC_NET_PARAM_ALPHA = Context.enet_alpha
    Fitness.ELASTIC_NET_PARAM_L1_RATIO = Context.enet_l1_ratio

    barseq_layout = BarseqLayout(Context.barseq_layout_fname)
    barseq_layout.save(Context.barseq_layout_out_fname())

    Fitness.init(barseq_layout, Context.barseq_bstat_dir,
                 Context.bpag_fname, Context.genes_gff_fname, Context.gene_pairs)
    Fitness.save_fscore_base(Context.fscore_base_fname())
    Fitness.save_gscore_base(Context.gscore_base_fname())

    for index, item in enumerate(barseq_layout.all_items):
        print('Doing %s' % item.itnum)
        ss = Fitness.get_sample(index)
        ts = Fitness.get_tzero_sample()
        fs = Fitness.build_fscores(ss, ts)

        fscore_fname = os.path.join(
            Context.output_dir, item.itnum + '.fscore.tsv')
        Fitness.save_fscores(fscore_fname, fs, ss, ts)

        gs_mean = Fitness.build_gscores(fs, Fitness.SCORE_TYPE_MEAN)
        gs_nnls = Fitness.build_gscores(fs, Fitness.SCORE_TYPE_C_NNLS)
        gs_ridge = Fitness.build_gscores(fs, Fitness.SCORE_TYPE_RIDGE)
        gs_lasso = Fitness.build_gscores(fs, Fitness.SCORE_TYPE_LASSO)
        gs_enet = Fitness.build_gscores(fs, Fitness.SCORE_TYPE_ELASTIC_NET)

        gscore_fname = os.path.join(
            Context.output_dir, item.itnum + '.gscore.tsv')
        Fitness.save_gscores(gscore_fname,
                             [Fitness.SCORE_TYPE_MEAN,
                              Fitness.SCORE_TYPE_C_NNLS,
                              Fitness.SCORE_TYPE_RIDGE,
                              Fitness.SCORE_TYPE_LASSO,
                              Fitness.SCORE_TYPE_ELASTIC_NET
                              ],
                             [gs_mean, gs_nnls, gs_ridge, gs_lasso, gs_enet])

def init_logger():
    with open(Context.log_fname(), 'w') as f:
        f.write("Parameters:\n")
        for arg, value in vars(args).items():
            f.write("\t%s=%s\n" % (arg, value))
        # f.write("Report columns:\n")
        # f.write("\t%s\n" % FastqFileStat.header(sep='\n\t'))
        # f.write("\n\n")

    logging.basicConfig(
        filename=Context.log_fname(),
        level=logging.INFO,
        format="%(asctime)s %(message)s",
        datefmt="%m/%d/%Y %I:%M:%S %p")

if __name__ == '__main__':
    args = parse_args()
    check_args(args)
    Context.build_context(args)
    init_logger()

    main()


from z3 import *
from solver import column, consecutive, exclusive, nonconsecutive, none, one

##################
# 3x3 1-colour 1 #
##################

nrows = 3
ncols = 3
puzzle = [[Bool(f'({row}, {col})') for col in range(ncols)] for row in range(nrows)]

row_0 = nonconsecutive(puzzle[0], 2)
row_1 = consecutive(puzzle[1], 2)
row_2 = one(puzzle[2])

col_0 = consecutive(column(puzzle, 0), 2)
col_1 = one(column(puzzle, 1))
col_2 = nonconsecutive(column(puzzle, 2), 2)

s = Solver()
s.add(row_0, row_1, row_2, col_0, col_1, col_2)

if s.check() == sat:
    m = s.model()
    r = [[m.evaluate(puzzle[i][j]) for j in range(ncols)] for i in range(nrows)]
    print_matrix(r)
else:
    print(unsat)

##################
# 3x3 1-colour 2 #
##################

nrows = 3
ncols = 3
puzzle = [[Bool(f'({row}, {col})') for col in range(ncols)] for row in range(nrows)]

rows = [
    consecutive(puzzle[0], 2),
    consecutive(puzzle[1], 2),
    none(puzzle[2]),
]

columns = [
    consecutive(column(puzzle, 0), 2),
    consecutive(column(puzzle, 1), 2),
    none(column(puzzle, 2)),
]

s = Solver()
s.add(rows + columns)

if s.check() == sat:
    m = s.model()
    r = [[m.evaluate(puzzle[i][j]) for j in range(ncols)] for i in range(nrows)]
    print_matrix(r)
else:
    print(unsat)

#################
# 5x5 2-colours #
#################

nrows = 5
ncols = 5
red = [[Bool(f"({row}, {col}, 'red')") for col in range(ncols)] for row in range(nrows)]
blue = [[Bool(f"({row}, {col}, 'blue')") for col in range(ncols)] for row in range(nrows)]

red_rows = [
    consecutive(red[0], 5),
    consecutive(red[1], 5),
    consecutive(red[2], 2),
    consecutive(red[3], 2),
    consecutive(red[4], 2),
]

blue_rows = [
    none(blue[0]),
    none(blue[1]),
    consecutive(blue[2], 3),
    consecutive(blue[3], 3),
    consecutive(blue[4], 3),
]

red_columns = [
    consecutive(column(red, 0), 2),
    consecutive(column(red, 1), 2),
    consecutive(column(red, 2), 2),
    consecutive(column(red, 3), 5),
    consecutive(column(red, 4), 5),
]

blue_columns = [
    consecutive(column(blue, 0), 3),
    consecutive(column(blue, 1), 3),
    consecutive(column(blue, 2), 3),
    none(column(blue, 3)),
    none(column(blue, 4)),
]

s = Solver()
s.add(red_rows + red_columns
      + blue_rows + blue_columns
      )

if s.check() == sat:
    m = s.model()
    r = [[m.evaluate(red[i][j]) for j in range(ncols)] for i in range(nrows)]
    print_matrix(r)
    r = [[m.evaluate(blue[i][j]) for j in range(ncols)] for i in range(nrows)]
    print_matrix(r)
else:
    print(unsat)

##########################
# 2x2 1-colour ambiguous #
##########################

nrows = 2
ncols = 2
puzzle = [[Bool(f'({row}, {col})') for col in range(ncols)] for row in range(nrows)]

rows = [
    one(puzzle[0]),
    one(puzzle[1])
]

columns = [
    one(column(puzzle, 0)),
    one(column(puzzle, 1))
]

s = Solver()
s.add(rows + columns)

if s.check() == sat:
    m = s.model()
    r = [[m.evaluate(puzzle[i][j]) for j in range(ncols)] for i in range(nrows)]
    print_matrix(r)
else:
    print(unsat)

#################
# 2x2 2-colour  #
#################

nrows = 2
ncols = 2
red = [[Bool(f"({row}, {col}, 'red')") for col in range(ncols)] for row in range(nrows)]
blue = [[Bool(f"({row}, {col}, 'blue')") for col in range(ncols)] for row in range(nrows)]

red_rows = [
    one(red[0]),
    one(red[1])
]

red_columns = [
    one(column(red, 0)),
    one(column(red, 1))
]

blue_rows = [
    one(blue[0]),
    none(blue[1])
]

blue_columns = [
    one(column(blue, 0)),
    none(column(blue, 1))
]

s = Solver()
s.add(red_rows + red_columns + blue_rows + blue_columns)
s.add(exclusive(red, blue))

if s.check() == sat:
    m = s.model()
    r = [[m.evaluate(red[i][j]) for j in range(ncols)] for i in range(nrows)]
    print_matrix(r)
    r = [[m.evaluate(blue[i][j]) for j in range(ncols)] for i in range(nrows)]
    print_matrix(r)
else:
    print(unsat)


# stdlib
import asyncio
from enum import Enum
import logging
import multiprocessing
import os
import platform
import signal
import subprocess  # nosec
import time
from typing import Callable
from typing import List
from typing import Tuple

# third party
from fastapi import APIRouter
from fastapi import FastAPI
import requests
from starlette.middleware.cors import CORSMiddleware
import uvicorn

# relative
from ..abstract_node import NodeSideType
from ..client.client import API_PATH
from ..util.constants import DEFAULT_TIMEOUT
from ..util.util import os_name
from .domain import Domain
from .enclave import Enclave
from .gateway import Gateway
from .node import NodeType
from .routes import make_routes

if os_name() == "macOS":
    # needed on MacOS to prevent [__NSCFConstantString initialize] may have been in
    # progress in another thread when fork() was called.
    multiprocessing.set_start_method("spawn", True)

WAIT_TIME_SECONDS = 20

def make_app(name: str, router: APIRouter) -> FastAPI:
    app = FastAPI(
        title=name,
    )

    api_router = APIRouter()

    api_router.include_router(router)
    app.include_router(api_router, prefix="/api/v2")

    app.add_middleware(
        CORSMiddleware,
        allow_origins=["*"],
        allow_credentials=True,
        allow_methods=["*"],
        allow_headers=["*"],
    )

    return app

worker_classes = {
    NodeType.DOMAIN: Domain,
    NodeType.GATEWAY: Gateway,
    NodeType.ENCLAVE: Enclave,
}

def run_uvicorn(
    name: str,
    node_type: Enum,
    host: str,
    port: int,
    reset: bool,
    dev_mode: bool,
    node_side_type: str,
    enable_warnings: bool,
):
    async def _run_uvicorn(
        name: str,
        node_type: Enum,
        host: str,
        port: int,
        reset: bool,
        dev_mode: bool,
        node_side_type: Enum,
    ):
        if node_type not in worker_classes:
            raise NotImplementedError(f"node_type: {node_type} is not supported")
        worker_class = worker_classes[node_type]
        if dev_mode:
            print(
                f"\nWARNING: private key is based on node name: {name} in dev_mode. "
                "Don't run this in production."
            )

            worker = worker_class.named(
                name=name,
                processes=0,
                reset=reset,
                local_db=True,
                node_type=node_type,
                node_side_type=node_side_type,
                enable_warnings=enable_warnings,
            )
        else:
            worker = worker_class(
                name=name,
                processes=0,
                local_db=True,
                node_type=node_type,
                node_side_type=node_side_type,
                enable_warnings=enable_warnings,
            )
        router = make_routes(worker=worker)
        app = make_app(worker.name, router=router)

        if reset:
            try:
                python_pids = find_python_processes_on_port(port)
                for pid in python_pids:
                    print(f"Stopping process on port: {port}")
                    kill_process(pid)
                    time.sleep(1)
            except Exception:  # nosec
                print(f"Failed to kill python process on port: {port}")

        log_level = "critical"
        if dev_mode:
            log_level = "info"
            logging.getLogger("uvicorn").setLevel(logging.CRITICAL)
            logging.getLogger("uvicorn.access").setLevel(logging.CRITICAL)
        config = uvicorn.Config(
            app, host=host, port=port, log_level=log_level, reload=dev_mode
        )
        server = uvicorn.Server(config)

        await server.serve()
        asyncio.get_running_loop().stop()

    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    loop.run_until_complete(
        _run_uvicorn(
            name,
            node_type,
            host,
            port,
            reset,
            dev_mode,
            node_side_type,
        )
    )
    loop.close()

def serve_node(
    name: str,
    node_type: NodeType = NodeType.DOMAIN,
    node_side_type: NodeSideType = NodeSideType.HIGH_SIDE,
    host: str = "0.0.0.0",  # nosec
    port: int = 8080,
    reset: bool = False,
    dev_mode: bool = False,
    tail: bool = False,
    enable_warnings: bool = False,
) -> Tuple[Callable, Callable]:
    server_process = multiprocessing.Process(
        target=run_uvicorn,
        args=(
            name,
            node_type,
            host,
            port,
            reset,
            dev_mode,
            node_side_type,
            enable_warnings,
        ),
    )

    def stop():
        print(f"Stopping {name}")
        server_process.terminate()
        server_process.join()

    def start():
        print(f"Starting {name} server on {host}:{port}")
        server_process.start()

        if tail:
            try:
                while True:
                    time.sleep(1)
            except KeyboardInterrupt:
                try:
                    stop()
                except SystemExit:
                    os._exit(130)
        else:
            for i in range(WAIT_TIME_SECONDS):
                try:
                    req = requests.get(
                        f"http://{host}:{port}{API_PATH}/metadata",
                        timeout=DEFAULT_TIMEOUT,
                    )
                    if req.status_code == 200:
                        print(" Done.")
                        break
                except Exception:
                    time.sleep(1)
                    if i == 0:
                        print("Waiting for server to start", end="")
                    else:
                        print(".", end="")

    return start, stop

def find_python_processes_on_port(port: int) -> List[int]:
    system = platform.system()

    if system == "Windows":
        command = f"netstat -ano | findstr :{port}"
        process = subprocess.Popen(  # nosec
            command,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
        )
        output, _ = process.communicate()
        pids = [
            int(line.strip().split()[-1]) for line in output.split("\n") if line.strip()
        ]

    else:  # Linux and MacOS
        command = f"lsof -i :{port} -sTCP:LISTEN -t"
        process = subprocess.Popen(  # nosec
            command,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
        )
        output, _ = process.communicate()
        pids = [int(pid.strip()) for pid in output.split("\n") if pid.strip()]

    python_pids = []
    for pid in pids:
        try:
            if system == "Windows":
                command = (
                    f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine"
                )
            else:
                command = f"ps -p {pid} -o pid,command"

            process = subprocess.Popen(  # nosec
                command,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
            )
            output, _ = process.communicate()
            lines = output.strip().split("\n")

            if len(lines) > 1 and "python" in lines[1].lower():
                python_pids.append(pid)

        except Exception as e:
            print(f"Error checking process {pid}: {e}")

    return python_pids

def kill_process(pid: int) -> None:
    try:
        os.kill(pid, signal.SIGTERM)
        print(f"Process {pid} terminated.")
    except Exception as e:
        print(f"Error killing process {pid}: {e}")