'''
A GRU layer capable of teacher forcing at train time, and sampling from a softmax at test time.

# Example of use:
tgru = TerminalGRU(NCHARS, rnd_seed=42, return_sequences=True, activation='softmax', temperature=0.01, name='tgru_layer')

## Overview of code: main differences from regular GRU
- Takes in two inputs:
    1. Previous layer of network (usually from a regular GRU)
    2. True original sequence input (or a dummy if running in a test phase/free running)
- Sequential output at each time step is generated in one of two ways:
    1. Teacher forcing (train time):
         Uses true input (in original space) from previous time step to update the state for the current time step.
    2. Free running (test time)
         Uses sampled output from the previous time step to calculate the current input
- State is passed as a dictionary containing the following. See step function below for details.
    - 'initial_states',  'random_cutoff_prob',  'rec_dp_mask'

    - a 'raw state' - calculated from the previous step (given by first dimension of state in step)
    - For train phase:
        - raw state is repeated twice
    - For test phase
        - a 'sampled state' - sampled using the raw state
        - This then becomes the output for the layer at this timestep
- Uses 'sampled_rnn' in order to get around state changes

## Input shape:
    - list of 2 tensors,
        1. 3D tensor (input from previous layer) : '(batch_size, timesteps, input_dim)'
        2. 3D tensor (true input sequence to model) : '(batch_size, timesteps, output_dim)'
        output_dim == units

## Output shape:
    - if 'return_sequences': 3D tensor with shape:
        '(batch_size, timesteps, output_dim)'
    - else: 2D tensor with shape '(batch_size, output_dim)'

## Other functions:
    -output_sampling(output, rand_matrix) : used to help sample the final one-hot output vector based on continuous output vector
            rand_matrix - Random matrix generated within sampled_rnn, (batch_size, )
    - get_initial_states(x)
        Generate random starting state (of zeros)

# Arguments:
    output_dim : dimensions of outputs at each time step, in each sample
            called units in Keras 2.0 recurrent layer
    teacher_force_ratio : Ratio for using teacher forcing method vs. free_running method
    temperature - Temperature for sampling  om tje GRU layer
    rnd_seed - a random seed to use (currently not being used)

This version of TerminalGRU is currrently implemented for self.implementation==0 only
Other implementations will need to be ported over from original recurrent layer.

self.implementation ==2 : gpu
self.implementation ==1 : mem
self.implementation ==0 : cpu

'''
from keras.layers.recurrent import GRU
from keras import backend as K
from keras.engine import InputSpec
import numpy as np

if K.backend() == 'tensorflow':
    from .sampled_rnn_tf import sampled_rnn
else:

    raise NotImplemented("Backend not implemented")

class TerminalGRU(GRU):
    # Heavily adapted from GRU in recurrent.py
    # Implements professor forcing

    def __init__(self, units,
                 temperature=1., rnd_seed=None, recurrent_dropout=0.0,
                 **kwargs):
        # @param: temperature - sampling temperature
        # Annealing will be handled in the callbacks
        super(TerminalGRU, self).__init__(units, **kwargs)
        self.units = units
        self.temperature = temperature
        self.rnd_seed = rnd_seed
        self.uses_learning_phase = True
        self.supports_masking = False
        self.units = units
        self.recurrent_dropout = min(1., max(0., recurrent_dropout))
        self.input_spec = [InputSpec(ndim=3),
                           InputSpec(ndim=3)]

    def build(self, input_shape):
        # all of this is copied from GRU, except for one part commented below
        if isinstance(input_shape, list):
            input_shape = input_shape[0]

        batch_size = input_shape[0] if self.stateful else None
        self.input_dim = input_shape[2]
        self.input_spec = [InputSpec(shape=(batch_size, None, self.input_dim)),
                           InputSpec(shape=(batch_size, None, self.units))]
        self.state_spec = InputSpec(shape=(batch_size, self.units))

        self.states = [None]
        if self.stateful:
            self.reset_states()

        self.kernel = self.add_weight((self.input_dim, self.units * 3),
                                      name='kernel',
                                      initializer=self.kernel_initializer,
                                      regularizer=self.kernel_regularizer,
                                      constraint=self.kernel_constraint)

        # adding an extra recurrent weight here, change from GRU layer:
        # this last recurrent weight applied to true sequence input from prev. timestep,
        #   or sampled output from prev. time step.
        self.recurrent_kernel = self.add_weight(
            (self.units, self.units * 4),
            name='recurrent_kernel',
            initializer=self.recurrent_initializer,
            regularizer=self.recurrent_regularizer,
            constraint=self.recurrent_constraint)

        if self.use_bias:
            self.bias = self.add_weight((self.units * 4,),
                                        name='bias',
                                        initializer='zero',
                                        regularizer=self.bias_regularizer,
                                        constraint=self.bias_constraint)
        else:
            self.bias = None

        self.kernel_z = self.kernel[:, :self.units]
        self.recurrent_kernel_z = self.recurrent_kernel[:, :self.units]
        self.kernel_r = self.kernel[:, self.units: self.units * 2]
        self.recurrent_kernel_r = self.recurrent_kernel[:,
                                  self.units:
                                  self.units * 2]
        self.kernel_h = self.kernel[:, self.units * 2:]
        self.recurrent_kernel_h = self.recurrent_kernel[:, self.units * 2:self.units * 3]
        self.recurrent_kernel_y = self.recurrent_kernel[:, self.units * 3:]

        if self.use_bias:
            self.bias_z = self.bias[:self.units]
            self.bias_r = self.bias[self.units: self.units * 2]
            self.bias_h = self.bias[self.units * 2: self.units * 3]
            self.bias_h = self.bias[self.units * 3:]
        else:
            self.bias_z = None
            self.bias_r = None
            self.bias_h = None
        self.built = True

    def get_initial_states(self, x):
        # build an all-zero tensor of shape [(samples, output_dim), (samples, output_dim)]
        initial_state = K.zeros_like(x)  # (samples, timesteps, input_dim)
        initial_state = K.sum(initial_state, axis=1)  # (samples, input_dim)
        reducer = K.random_uniform((self.input_dim, self.units))
        reducer = reducer / K.exp(reducer)

        initial_state = K.dot(initial_state, reducer)  # (samples, output_dim)
        initial_states = [K.stack([initial_state, initial_state]) for _ in range(len(self.states))]
        return initial_states

    def compute_mask(self, input, mask):
        # Forced to be single dimension, following behavior of Merge layer
        # not implemented
        return None

    def get_constants(self, inputs, training=None):
        constants = []
        if 0. < self.recurrent_dropout < 1.:
            ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))
            ones = K.tile(ones, (1, self.units))

            def dropped_inputs():
                return K.dropout(ones, self.recurrent_dropout)

            rec_dp_mask = [K.in_train_phase(dropped_inputs,
                                            ones,
                                            training=training) for _ in range(3)]
            constants.append(rec_dp_mask)
        else:
            constants.append([K.cast_to_floatx(1.) for _ in range(3)])
        return constants

    def call(self, inputs, mask=None):
        if type(inputs) is not list or len(inputs) != 2:
            raise Exception('terminal gru runs on list of length 2')

        X = inputs[0]
        true_seq = inputs[1]

        if self.stateful:
            initial_states = self.states
        else:
            initial_states = self.get_initial_states(X)

        # preprocessing makes input into right form for gpu/cpu settings
        # from original GRU code
        recurrent_dropout_constants = self.get_constants(X)[0]
        preprocessed_input = self.preprocess_input(X)

        #################
        ## Section for index matching of true inputs
        #################
        #  Basically, we need to add an extra timestep of just 0s for predicting the first timestep output

        axes = [1, 0] + list(range(2, K.ndim(true_seq)))

        true_seq = K.permute_dimensions(true_seq, axes)
        zeros = K.zeros_like(true_seq[:1, :, :])

        # add a column of zeros, remove last element
        true_seq = K.concatenate([zeros, true_seq[:K.int_shape(true_seq)[0] - 1, :, :]], axis=0)
        shifted_raw_inputs = K.permute_dimensions(true_seq, axes)

        ## concatenate to have same dimension as preprocessed inputs 3xoutput_dim
        # only for self.implementation = 0?
        shifted_raw_inputs = K.concatenate([shifted_raw_inputs,
                                            shifted_raw_inputs,
                                            shifted_raw_inputs], axis=2)

        all_inputs = K.stack([preprocessed_input, shifted_raw_inputs])
        num_dim = K.ndim(all_inputs)
        axes = [1, 2, 0] + list(range(3, num_dim))
        all_inputs = K.permute_dimensions(all_inputs, axes)

        # If not using true sequence, want to feed in a tensor of zeros instead.
        zeros_input_seq = K.zeros_like(preprocessed_input)
        test_phase_all_inputs = K.stack([preprocessed_input, zeros_input_seq])
        test_phase_all_inputs = K.permute_dimensions(test_phase_all_inputs, axes)

        all_inputs = K.in_train_phase(all_inputs, test_phase_all_inputs)

        last_output, outputs, states = sampled_rnn(self.step,
                                                   all_inputs,
                                                   initial_states,
                                                   self.units,
                                                   self.rnd_seed,
                                                   go_backwards=self.go_backwards,
                                                   rec_dp_constants=recurrent_dropout_constants,
                                                   mask=None)

        if self.return_sequences:
            return outputs
        else:
            return last_output

    def compute_output_shape(self, input_shape):
        # expect input_shape is a list:
        assert type(input_shape) is list

        input_shapes = input_shape

        # from original recurrent unit, can probably delete entire if this works
        if self.return_sequences:
            return input_shapes[1]
        else:
            return (input_shapes[1][0], input_shapes[1][1])

    def get_config(self):
        config = {'units': self.units,
                  'temperature': self.temperature,
                  'rnd_seed': self.rnd_seed}
        base_config = super(TerminalGRU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

    def output_sampling(self, output, rand_matrix):
        # Generates a sampled selection based on raw output state vector
        # Creates a cdf vector and compares against a randomly generated vector
        # Requires a pre-generated rand_matrix (i.e. generated outside step function)

        sampled_output = output / K.sum(output, axis=-1, keepdims=True)  # (batch_size, self.units)
        mod_sampled_output = sampled_output / K.exp(self.temperature)
        norm_exp_sampled_output = mod_sampled_output / K.sum(mod_sampled_output, axis=-1, keepdims=True)

        cdf_vector = K.cumsum(norm_exp_sampled_output, axis=-1)
        cdf_minus_vector = cdf_vector - norm_exp_sampled_output

        rand_matrix = K.stack([rand_matrix], axis=0)
        rand_matrix = K.stack([rand_matrix], axis=2)

        compared_greater_output = K.cast(K.greater(cdf_vector, rand_matrix), dtype='float32')
        compared_lesser_output = K.cast(K.less(cdf_minus_vector, rand_matrix), dtype='float32')

        final_output = compared_greater_output * compared_lesser_output
        return final_output

    def step(self, h, states):
        '''
        receives inputs for a time step
        @inp : h - [previous_layer_input, true_input_for_previous_timestep] at train time
               or  [previous_layer_input, zeros] at test time
        @inp : states - a dictionary, contains the following
            - 'initial_states' - state vector
                 - At train time, this includes the true input sequence for the given time step, in addition to the state for the previous time step.
                 - At test time,
            - 'random_cutoff_prob' - random cutoff matrix used for sampling at test time
            - 'rec_dp_mask' - for use with dropout (not tested - may break)

        @return: output - raw output, unsampled
        @return: final_output - output that has been sampled in test case

        '''

        ################
        # Parsing the states vector
        ################
        initial_states = states['initial_states']
        random_cutoff_vec = states['random_cutoff_prob']

        if self.recurrent_dropout > 0:
            rec_dp_mask = states['rec_dp_mask']
        else:
            rec_dp_mask = np.array([1., 1., 1., 1.], dtype='float32')

        h_tm1 = initial_states[0][:1, :, :]

        def teacher_forced(h, states):
            # switching from (batch_size, previous_layer_input|true_input, output_dim)
            #    to ( previous_layer_input|true_input, batch_size, output_dim)
            axes = [1, 0] + list(range(2, K.ndim(h)))
            h = K.permute_dimensions(h, axes)

            prev_layer_input = h[0:1, :, :]
            true_input = h[1:, :, :self.units]

            # this should correspond  to true input
            prev_sampled_output = true_input

            if self.implementation == 0:
                x_z = prev_layer_input[0, :, :self.units]
                x_r = prev_layer_input[0, :, self.units: 2 * self.units]
                x_h = prev_layer_input[0, :, 2 * self.units:]
            else:
                raise ValueError('Implementation type ' + self.implementation + ' is invalid')

            z = self.recurrent_activation(x_z + K.dot(h_tm1 * rec_dp_mask[0],
                                                      self.recurrent_kernel_z))
            r = self.recurrent_activation(x_r + K.dot(h_tm1 * rec_dp_mask[1],
                                                      self.recurrent_kernel_r))

            hh = self.activation(x_h +
                                 K.dot(r * h_tm1 * rec_dp_mask[2],
                                       self.recurrent_kernel_h) +
                                 K.dot(r * prev_sampled_output, self.recurrent_kernel_y))

            output = z * h_tm1 + (1. - z) * hh

            return K.stack([output, output])

        def free_running(h, states):

            prev_generated_output = initial_states[0][1:, :, :]
            prev_sampled_output = prev_generated_output

            # switching from (batch_size, previous_layer_input|true_input, output_dim)
            #    to ( previous_layer_input|true_input, batch_size, output_dim)
            axes = [1, 0] + list(range(2, K.ndim(h)))
            h = K.permute_dimensions(h, axes)

            prev_layer_input = h[0:1, :, :]

            if self.implementation == 0:
                x_z = prev_layer_input[0, :, :self.units]
                x_r = prev_layer_input[0, :, self.units: 2 * self.units]
                x_h = prev_layer_input[0, :, 2 * self.units:]

            z = self.recurrent_activation(x_z + K.dot(h_tm1 * rec_dp_mask[0],
                                                      self.recurrent_kernel_z))
            r = self.recurrent_activation(x_r + K.dot(h_tm1 * rec_dp_mask[1],
                                                      self.recurrent_kernel_r))

            hh = self.activation(x_h +
                                 K.dot(r * h_tm1 * rec_dp_mask[2],
                                       self.recurrent_kernel_h) +
                                 K.dot(r * prev_sampled_output, self.recurrent_kernel_y))

            output = z * h_tm1 + (1. - z) * hh

            final_output = self.output_sampling(output, random_cutoff_vec)

            return K.stack([output, final_output])

        output_2d_tensor = K.in_train_phase(teacher_forced(h, states),
                                            free_running(h, states))

        output_2d_tensor = K.squeeze(output_2d_tensor, 1)

        return output_2d_tensor, [output_2d_tensor]


"""manosBackend URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib import admin

#Por ver
from rest_framework import routers
from apps.apiREST.views import *
router = routers.DefaultRouter()
router.register(r'^userDOC',UserViewSet)
router.register(r'^workerDOC',WorkerViewSet)
router.register(r'^addressDOC',AddressViewSet)
router.register(r'^commentDOC',CommentViewSet)
router.register(r'^categoryDOC',CategoryViewSet)
router.register(r'^subcategoryDOC',SubcategoryViewSet)
router.register(r'^jobrequestDOC',JobRequestViewSet)
router.register(r'^photoDOC',PhotoViewSet)
router.register(r'^jobapplicationDOC',JobApplicationViewSet)

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^', include('apps.apiREST.urls')),
	url(r'^',include(router.urls)),
]


from helper import *
from data import game_data

if __name__ == "__main__":
    """
    Prints battleship info and rules
    Instantiates two player objects. If is_computer=True, that player will act as a computer
    Plays battleship between the two players with ships in data.json
    """
    print_battleship_info()

    player = Player(name=input(f"Your name: "), is_computer=False)
    computer = Player(name="Computer", is_computer=True)
    data = game_data

    play_battleship(player, computer, data)


import random

#BigO(n**2)

def bubble_sort(lista):
    n = len(lista)

    iter_i = 0
    iter_j = 0

    for i in range(n):
        iter_i += 1
        for j in range(0, n - i - 1):
            iter_j += 1
            if lista[j] > lista[j + 1]:
                lista[j], lista[j + 1]= lista[j + 1], lista[j]

    return lista, iter_i, iter_j

def main():
    tamano_de_lista = int(input('De que tamaño sera la lista? '))

    lista = [random.randint(0, 100) for i in range (tamano_de_lista)]
    print(lista)

    lista_ordenada = bubble_sort(lista)
    print(lista_ordenada[0], lista_ordenada[1], lista_ordenada[2])

if __name__ == '__main__':
    main()


import numpy as np
import pickle
import theano
import time
import constants as c
import scipy.constants as sc
import scipy.interpolate as spi
import eigen
import starry
import progressbar
import theano
import theano.tensor as tt
import mc3.stats as ms
from numba import njit

def initsystem(fit, ydeg):
    '''
    Uses a fit object to build the respective starry objects. Useful
    because starry objects cannot be pickled. Returns a tuple of
    (star, planet, system).
    '''

    cfg = fit.cfg

    star = starry.Primary(starry.Map(ydeg=1, amp=1),
                          m   =cfg.star.m,
                          r   =cfg.star.r,
                          prot=cfg.star.prot)

    planet = starry.kepler.Secondary(starry.Map(ydeg=ydeg),
                                     m    =cfg.planet.m,
                                     r    =cfg.planet.r,
                                     porb =cfg.planet.porb,
                                     prot =cfg.planet.prot,
                                     Omega=cfg.planet.Omega,
                                     ecc  =cfg.planet.ecc,
                                     w    =cfg.planet.w,
                                     t0   =cfg.planet.t0,
                                     inc  =cfg.planet.inc,
                                     theta0=180)

    system = starry.System(star, planet)

    return star, planet, system

def specint(wn, spec, filtwn_list, filttrans_list):
    """
    Integrate a spectrum over the given filters.

    Arguments
    ---------
    wn: 1D array
        Wavenumbers (/cm) of the spectrum

    spec: 1D array
        Spectrum to be integrated

    filtwn_list: list
        List of arrays of filter wavenumbers, in /cm.

    filttrans_list: list
        List of arrays of filter transmission. Same length as filtwn_list.

    Returns
    -------
    intspec: 1D array
        The spectrum integrated over each filter.
    """
    if len(filtwn_list) != len(filttrans_list):
        print("ERROR: list sizes do not match.")
        raise Exception

    intspec = np.zeros(len(filtwn_list))

    for i, (filtwn, filttrans) in enumerate(zip(filtwn_list, filttrans_list)):
        # Sort ascending
        idx = np.argsort(filtwn)

        intfunc = spi.interp1d(filtwn[idx], filttrans[idx],
                               bounds_error=False, fill_value=0)

        # Interpolate transmission
        inttrans = intfunc(wn)

        # Normalize to one
        norminttrans = inttrans / np.trapz(inttrans, wn)

        # Integrate filtered spectrum
        intspec[i] = np.trapz(spec * norminttrans, wn)

    return intspec

def vislon(planet, fit):
    """
    Determines the range of visible longitudes based on times of
    observation.

    Arguments
    ---------
    planet: starry Planet object
        Planet object

    fit: Fit object
        Fit object. Must contain observation information.

    Returns
    -------
    minlon: float
        Minimum visible longitude, in degrees

    maxlon: float
        Maximum visible longitude, in degrees
    """
    t = fit.t

    porb   = planet.porb   # days / orbit
    prot   = planet.prot   # days / rotation
    t0     = planet.t0     # days
    theta0 = planet.theta0 # degrees

    # Central longitude at each time ("sub-observer" point)
    centlon = theta0 - (t - t0) / prot * 360

    # Minimum and maximum longitudes (assuming +/- 90 degree
    # visibility)
    limb1 = centlon - 90
    limb2 = centlon + 90

    # Rescale to [-180, 180]
    limb1 = (limb1 + 180) % 360 - 180
    limb2 = (limb2 + 180) % 360 - 180

    return np.min(limb1.eval()), np.max(limb2.eval())

def readfilters(filterfiles):
    """
    Reads filter files and determines the mean wavelength.

    Arguments
    ---------
    filterfiles: list
        list of paths to filter files

    Returns
    -------
    filtmid: 1D array
        Array of mean wavelengths
    """
    filtwl_list    = []
    filtwn_list    = []
    filttrans_list = []

    wnmid = np.zeros(len(filterfiles))
    for i, filterfile in enumerate(filterfiles):
        filtwl, trans = np.loadtxt(filterfile, unpack=True)

        filtwn = 1.0 / (filtwl * c.um2cm)

        wnmid[i] = np.sum(filtwn * trans) / np.sum(trans)

        filtwl_list.append(filtwl)
        filtwn_list.append(filtwn)
        filttrans_list.append(trans)

    wlmid = 1 / (c.um2cm * wnmid)

    return filtwl_list, filtwn_list, filttrans_list, wnmid, wlmid

def visibility(t, latgrid, longrid, dlatgrid, dlongrid, theta0, prot,
               t0, rp, rs, x, y):
    """
    Calculate the visibility of a grid of cells on a planet at a specific
    time. Returns a combined visibility based on the observer's
    line-of-sight, the area of the cells, and the effect of the star.

    Arguments
    ---------
    t: float
        Time to calculate visibility.

    latgrid: 2D array
        Array of latitudes, in radians, from -pi/2 to pi/2.

    longrid: 2D array
        Array of longitudes, in radians, from -pi to pi.

    dlat: float
        Latitude resolution in radians.

    dlon: float
        Longitude resoltuion in radians.

    theta0: float
        Rotation at t0 in radians.

    prot: float
        Rotation period, the same units as t.

    t0: float
        Time of transit, same units as t.

    rp: float
        Planet radius in solar radii.

    rs: float
        Star radius in solar radii.

    x: tuple
        x position of (star, planet)

    y: tuple
        y position of (star, planet)

    Returns
    -------
    vis: 2D array
        Visibility of each grid cell. Same shape as latgrid and longrid.

    """
    if latgrid.shape != longrid.shape:
        print("Number of latitudes and longitudes do not match.")
        raise Exception

    losvis  = np.zeros(latgrid.shape)
    starvis = np.zeros(latgrid.shape)

    # Flag to do star visibility calculation (improves efficiency)
    dostar = True

    # Central longitude (observer line-of-sight)
    centlon = theta0 - (t - t0) / prot * 2 * np.pi

    # Convert relative to substellar point
    centlon = (centlon + np.pi) % (2 * np.pi) - np.pi

    xsep = x[0] - x[1]
    ysep = y[0] - y[1]
    d = np.sqrt(xsep**2 + ysep**2)

    # Visible fraction due to star
    # No grid cells visible. Return 0s
    if (d < rs - rp):
        return np.zeros(latgrid.shape)

    # All grid cells visible. No need to do star calculation.
    elif (d > rs + rp):
        starvis[:,:] = 1.0
        dostar     = False
    # Otherwise, time is during ingress/egress and we cannot simplify
    # calculation

    nlat, nlon = latgrid.shape
    for i in range(nlat):
        for j in range(nlon):
            # Angles wrt the observer
            lat  = latgrid[i,j]
            lon  = longrid[i,j]
            dlat = dlatgrid[i,j]
            dlon = dlongrid[i,j]

            phi   = lon - centlon
            theta = lat
            phimin   = phi - dlon / 2.
            phimax   = phi + dlon / 2.

            thetamin = lat - dlat / 2.
            thetamax = lat + dlat / 2.

            # Cell is not visible at this time. No need to calculate further.
            if (phimin > np.pi / 2.) or (phimax < -np.pi / 2.):
                losvis[i,j] = 0

            # Cell is visible at this time
            else:
                # Determine visible phi/theta range of the cell
                phirng   = np.array((np.max((phimin,   -np.pi / 2.)),
                                     np.min((phimax,    np.pi / 2.))))
                thetarng = np.array((np.max((thetamin, -np.pi / 2.)),
                                     np.min((thetamax,  np.pi / 2.))))

                # Visibility based on LoS
                # This is the integral of
                #
                # A(theta, phi) V(theta, phi) dtheta dphi
                #
                # where
                #
                # A = r**2 cos(theta)
                # V = cos(theta) cos(phi)
                #
                # Here we've normalized by pi*r**2, since
                # visibility will be applied to Fp/Fs where planet
                # size is already taken into account.
                losvis[i,j] = (np.diff(thetarng/2) + \
                               np.diff(np.sin(2*thetarng) / 4)) * \
                    np.diff(np.sin(phirng)) / \
                    np.pi

                # Grid cell maybe only partially visible
                if dostar:
                    thetamean = np.mean(thetarng)
                    phimean   = np.mean(phirng)
                    # Grid is "within" the star
                    if dgrid(x, y, rp, thetamean, phimean) < rs:
                        starvis[i,j] = 0.0
                    # Grid is not in the star
                    else:
                        starvis[i,j] = 1.0

    return starvis * losvis

def dgrid(x, y, rp, theta, phi):
    """
    Calculates the projected distance between a latitude (theta) and a
    longitude (phi) on a planet with radius rp to a star. Projected
    star position is (x[0], y[0]) and planet position is (x[1], y[1]).
    """
    xgrid = x[1] + rp * np.cos(theta) * np.sin(phi)
    ygrid = y[1] + rp * np.sin(theta)
    d = np.sqrt((xgrid - x[0])**2 + (ygrid - y[0])**2)
    return d

def t_dgrid():
    """
    Returns a theano function of dgrid(), with the same arguments.
    """
    print('Defining theano function.')
    arg1 = theano.tensor.dvector('x')
    arg2 = theano.tensor.dvector('y')
    arg3 = theano.tensor.dscalar('rp')
    arg4 = theano.tensor.dscalar('theta')
    arg5 = theano.tensor.dscalar('phi')

    f = theano.function([arg1, arg2, arg3, arg4, arg5],
                        dgrid(arg1, arg2, arg3, arg4, arg5))
    return f

def mapintensity(map, lat, lon, amp):
    """
    Calculates a grid of intensities, multiplied by the amplitude given.
    """
    grid = map.intensity(lat=lat.flatten(), lon=lon.flatten()).eval()
    grid *= amp
    grid = grid.reshape(lat.shape)
    return grid

def hotspotloc_driver(fit, map):
    """
    Calculates a distribution of hotspot locations based on the MCMC
    posterior distribution.

    Note that this function assumes the first ncurves parameters
    in the posterior are associated with eigencurves. This will not
    be true if some eigencurves are skipped over, as MC3 does not
    include fixed parameters in the posterior.

    Inputs
    ------
    fit: Fit instance

    map: Map instance (not starry Map)

    Returns
    -------
    hslocbest: tuple
        Best-fit hotspot location (lat, lon), in degrees.

    hslocstd: tuple
        Standard deviation of the hotspot location posterior distribution
        as (lat, lon)

    hspot: tuple
        Marginalized posterior distributions of latitude and longitude
    """

    post = map.post[map.zmask]

    nsamp, nfree = post.shape

    ntries     =  5
    oversample =  1

    if fit.cfg.twod.ncalc > nsamp:
        print("Warning: ncalc reduced to match burned-in sample.")
        ncalc = nsamp
    else:
        ncalc = fit.cfg.twod.ncalc

    hslon = np.zeros(ncalc)
    hslat = np.zeros(ncalc)
    thinning = nsamp // ncalc

    bounds = None
    bounds = (-90, 90),(-360, 360)
    smap = starry.Map(ydeg=map.lmax)
    # Function defined in this way to avoid passing non-numeric arguments
    def hotspotloc(yval):
        smap[1:,:] = yval
        lat, lon, val = smap.minimize(oversample=oversample,
                                      ntries=ntries, bounds=bounds)
        return lat, lon, val

    arg1 = tt.dvector()
    t_hotspotloc = theano.function([arg1], hotspotloc(arg1))

    # Note the maps created here do not include the correct uniform
    # component because that does not affect the location of the
    # hotspot. Also note that the eigenvalues are negated because
    # we want to maximize, not minize, but starry only includes
    # a minimize method.
    pbar = progressbar.ProgressBar(max_value=ncalc)
    for i in range(0, ncalc):
        ipost = i * thinning
        yval = np.zeros((map.lmax+1)**2-1)
        for j in range(map.ncurves):
            yval += -1 * post[ipost,j] * map.eigeny[j,1:]

        hslat[i], hslon[i], _ = t_hotspotloc(yval)
        pbar.update(i+1)

    star, planet, system = initsystem(fit, map.lmax)
    planet.map[1:,:] = 0.0
    for j in range(map.ncurves):
        planet.map[1:,:] += -1 * map.bestp[j] * map.eigeny[j,1:]
    hslatbest, hslonbest, _ = planet.map.minimize(oversample=oversample,
                                                  bounds=bounds,
                                                  ntries=ntries)
    hslonbest = hslonbest.eval()
    hslatbest = hslatbest.eval()

    # Constrain longitudes to [-180, 180]
    hslonbest = (hslonbest + 180.) % 360. - 180.
    hslon     = (hslon     + 180.) % 360. - 180.
    hslatbest = (hslatbest +  90.) % 180. -  90.
    hslat     = (hslat     +  90.) % 180. -  90.

    hslonstd = np.std(hslon)
    hslatstd = np.std(hslat)

    # Two-sided errors
    pdf, xpdf, hpdmin = ms.cred_region(hslon)
    crlo = np.amin(xpdf[pdf>hpdmin])
    crhi = np.amax(xpdf[pdf>hpdmin])
    hsloncrlo = crlo - hslonbest
    hsloncrhi = crhi - hslonbest

    pdf, xpdf, hpdmin = ms.cred_region(hslat)
    crlo = np.amin(xpdf[pdf>hpdmin])
    crhi = np.amax(xpdf[pdf>hpdmin])
    hslatcrlo = crlo - hslatbest
    hslatcrhi = crhi - hslatbest

    hslocbest  = (hslatbest, hslonbest)
    hslocstd   = (hslatstd,  hslonstd)
    hslocpost  = (hslat,     hslon)
    hsloctserr = ((hslatcrhi, hslatcrlo), (hsloncrhi, hsloncrlo))

    return hslocbest, hslocstd, hslocpost, hsloctserr

def tmappost(fit, map):
    post = map.post[map.zmask]

    nsamp, nfree = post.shape
    ncurves = map.ncurves

    if fit.cfg.twod.ncalc > nsamp:
        print("Warning: ncalc reduced to match burned-in sample.")
        ncalc = nsamp
    else:
        ncalc = fit.cfg.twod.ncalc

    thinning = nsamp // ncalc

    fmaps = np.zeros((ncalc, fit.cfg.twod.nlat, fit.cfg.twod.nlon))
    tmaps = np.zeros((ncalc, fit.cfg.twod.nlat, fit.cfg.twod.nlon))

    star, planet, system = initsystem(fit, map.lmax)

    def calcfmap(yval, unifamp):
        planet.map[1:,:] = 0.0
        amp = unifamp - 1
        fmap = planet.map.intensity(lat=fit.lat.flatten(),
                                    lon=fit.lon.flatten()) * amp

        planet.map[1:,:] = yval
        fmap += planet.map.intensity(lat=fit.lat.flatten(),
                                     lon=fit.lon.flatten())

        return fmap

    arg1 = tt.dvector()
    arg2 = tt.dscalar()
    t_calcfmap = theano.function([arg1, arg2], calcfmap(arg1, arg2))

    pbar = progressbar.ProgressBar(max_value=ncalc)
    for i in range(ncalc):
        ipost = i * thinning
        yval = np.zeros((map.lmax+1)**2-1)
        for j in range(map.ncurves):
            yval += post[ipost,j] * map.eigeny[j,1:]

        fmaps[i] = t_calcfmap(yval, post[ipost, ncurves]).reshape(fit.lat.shape)
        tmaps[i] = fmap_to_tmap(fmaps[i], map.wlmid*1e-6,
                                fit.cfg.planet.r, fit.cfg.star.r,
                                fit.cfg.star.t, post[ipost,ncurves+1])

        pbar.update(i+1)

    return fmaps, tmaps

def fmap_to_tmap(fmap, wl, rp, rs, ts, scorr):
    '''
    Convert flux map to brightness temperatures.
    See Rauscher et al., 2018, eq. 8
    '''
    ptemp = (sc.h * sc.c) / (wl * sc.k)
    sfact = 1 + scorr
    tmap = ptemp / np.log(1 + (rp / rs)**2 *
                          (np.exp(ptemp / ts) - 1) /
                          (np.pi * fmap * sfact))
    return tmap

def ess(chain):
    '''
    Calculates the Steps Per Effectively-Independent Sample and
    Effective Sample Size (ESS) of a chain from an MCMC posterior
    distribution.

    Adapted from some code I wrote for MC3 many years ago, and
    the SPEIS/ESS calculation in BART.
    '''
    nciter, npar = chain.shape

    speis = np.zeros(npar)
    ess   = np.zeros(npar)

    for i in range(npar):
        mean     = np.mean(chain[:,i])
        autocorr = np.correlate(chain[:,i] - mean,
                                chain[:,i] - mean,
                                mode='full')
        # Keep lags >= 0 and normalize
        autocorr = autocorr[np.size(autocorr) // 2:] / np.max(autocorr)
        # Sum adjacent pairs (Geyer, 1993)
        pairsum = autocorr[:-1:2] + autocorr[1::2]
        # Find where the sum goes negative, or use the whole thing
        if np.any(pairsum < 0):
            idx = np.where(pairsum < 0)[0][0]
        else:
            idx = len(pairsum)
            print("WARNING: parameter {} did not decorrelate!"
                  "Do not trust ESS/SPEIS!".format(i))
        # Calculate SPEIS
        speis[i] = -1 + 2 * np.sum(pairsum[:idx])
        ess[i]   = nciter / speis[i]

    return speis, ess

def crsig(ess, cr=0.683):
    '''
    Calculates the absolute error on an estimate of a credible region
    of a given percentile based on the effective sample size.

    See Harrington et al, 2021.

    Arguments
    ---------
    ess: int
        Effective Sample Size

    cr: float
        Credible region percentile to calculate error on. E.g.,
        for a 1-sigma region, use 0.683 (the default).

    Returns
    -------
    crsig: float
        The absolute error on the supplied credible region.
    '''
    return (cr * (1 - cr) / (ess + 3))**0.5

@njit
def fast_linear_interp(a, b, x):
    return (b[1] - a[1]) / (b[0] - a[0]) * (x - a[0]) + a[1]

@njit
def blackbody(T, wn):
    '''
    Calculates the Planck function for a grid of temperatures and
    wavenumbers. Wavenumbers must be in /cm.
    '''
    nt  = len(T)
    nwn = len(wn)
    bb = np.zeros((nt, nwn))

    # Convert from /cm to /m
    wn_m = wn * 1e2
    for i in range(nt):
        bb[i] = (2.0 * sc.h * sc.c**2 * wn_m**3) \
            * 1/(np.exp(sc.h * sc.c * wn_m / sc.k / T[i]) - 1.0)

    return bb


#!/usr/bin/env python3
import os
import copy
import argparse

import pickle
import numpy as np

import multiprocessing
from fourdvel import fourdvel
from display import display

import matplotlib.pyplot as plt
import collections

def createParser():

    parser = argparse.ArgumentParser( description='driver of fourdvel')

    parser.add_argument('-p','--param_file', dest='param_file', type=str, help='parameter file', required=True)

    parser.add_argument('--output_mode', dest='output_mode', type=str, help='output_mode (estimation or analysis)', default='estimation')

    # needs to be consistent with what is used for running
    parser.add_argument('-t','--task_name', dest='task_name', type=str, help='task_name', required=True)

    # This is for further narrow down the output quantity
    parser.add_argument('-o', '--output_names', dest='output_names', type=str, help='output_names e.g., true,est', default=None)

    parser.add_argument('-q','--quant_list_name', dest='quant_list_name', type=str, help='quant_list_name', default=None)

    parser.add_argument('--npc','--no_phase_correction', dest='no_phase_correction', action='store_true')

    return parser

def cmdLineParse(iargs = None):
    parser = createParser()
    return parser.parse_args(args=iargs)

class output(fourdvel):

    def __init__(self, inps):

        param_file = inps.param_file
        super(output,self).__init__(param_file = param_file)

        self.get_grid_set_v2()
        self.get_grid_set_velo()
        test_id = self.test_id

        self.estimation_dir = os.path.join(self.estimations_dir,str(test_id))

        self.display = display(param_file)

        self.output_mode = inps.output_mode

        self.output_names = inps.output_names

        self.quant_list_name = inps.quant_list_name

        self.task_name = inps.task_name

        self.no_phase_correction = inps.no_phase_correction

        self.get_shelf_points()

    def run_output_residual(self):

        this_result_folder = self.estimation_dir
        test_id = self.test_id

        #with open(this_result_folder + '/'
        #            + str(test_id) + '_' + 'grid_set_resid_of_secular.pkl','rb') as f:
        #    self.grid_set_resid_of_secular = pickle.load(f)

        with open(this_result_folder + '/'
                    + str(test_id) + '_' + 'grid_set_resid_of_tides.pkl','rb') as f:
            self.grid_set_resid_of_tides = pickle.load(f)

        # compare id
        compare_id = 202010565141

        with open(self.estimations_dir + '/' + str(compare_id) + '/' +
                    str(compare_id) + '_' + 'grid_set_resid_of_tides.pkl','rb') as f:
            self.grid_set_resid_of_tides_compare = pickle.load(f)

        # Set the compare set

        grid_sets = {}
        #grid_sets['resid_of_secular'] = self.grid_set_resid_of_secular
        grid_sets['resid_of_tides'] = self.grid_set_resid_of_tides

        state = 'est'
        comps = ['range','azimuth']

        for misfit_name in grid_sets.keys():
            for comp in comps:

                quant_name = '_'.join([misfit_name, comp])

                print('Output quantity name: ', quant_name)
                grid_set_quant = {}
                grid_set_quant_compare = {}

                # The two grid sets
                this_grid_set = grid_sets[misfit_name]
                compare_grid_set = self.grid_set_resid_of_tides_compare

                output_keys = this_grid_set.keys()

                # For all available points in grid_set.
                for point in output_keys:

                    # Four entries: range_mean(0), range_rms(1), azimuth_mean(2), azimuth_rms(3)
                    quant = this_grid_set[point]

                    try:
                        quant_compare = compare_grid_set[point]
                    except:
                        quant_compare = [np.nan]*10

                    # Output the rms
                    if comp == 'range':
                        grid_set_quant[point] = quant[1]
                        #print(quant[1])
                        #print(quant_compare[1])
                        grid_set_quant_compare[point] = quant[1] - quant_compare[1]

                    elif comp == 'azimuth':
                        grid_set_quant[point] = quant[3]
                        grid_set_quant_compare[point] = quant[3] - quant_compare[3]

                # Write to xyz file.
                xyz_name = os.path.join(this_result_folder, str(test_id) + '_' + state + '_' + quant_name + '.xyz')
                self.display.write_dict_to_xyz(grid_set_quant, xyz_name = xyz_name)

                # Write the compare to xyz file.
                xyz_name = os.path.join(this_result_folder, str(test_id) + '_' + state + '_' + quant_name + '_compare' + '.xyz')

                self.display.write_dict_to_xyz(grid_set_quant_compare, xyz_name = xyz_name)

        return 0

    def load_master_model(self,num,prefix='est'):

        this_result_folder = self.estimation_dir
        # Load all the results.
        if prefix == 'true':
            filename = self.estimations_dir +'/'+str(num)+'/'+str(num)+'_grid_set_true_tide_vec.pkl'
        else:
            filename = self.estimations_dir +'/'+str(num)+'/'+str(num)+'_grid_set_tide_vec.pkl'

        with open(filename,'rb') as f:
            self.grid_set_master_model_tide_vec = pickle.load(f)
        return 0

    def load_slave_model(self,num,prefix='est'):

        this_result_folder = self.estimation_dir
        # Load all the results.

        if prefix == 'true':
            filename = self.estimations_dir + '/' + str(num)+'/'+str(num)+'_grid_set_true_tide_vec.pkl'
        else:
            filename = self.estimations_dir + '/' + str(num)+'/'+str(num)+'_grid_set_tide_vec.pkl'

        with open(filename,'rb') as f:
            self.grid_set_slave_model_tide_vec = pickle.load(f)
        return 0

    def load_everything(self):

        this_result_folder = self.estimation_dir
        test_id = self.test_id

        # Load all the results.
        with open(this_result_folder + '/'
                    + str(test_id) + '_' + 'grid_set_true_tide_vec.pkl','rb') as f:
            self.grid_set_true_tide_vec = pickle.load(f)

        with open(this_result_folder + '/'
                    + str(test_id) + '_' + 'grid_set_tide_vec.pkl','rb') as f:
            self.grid_set_tide_vec = pickle.load(f)

        with open(this_result_folder + '/'
                    + str(test_id) + '_' + 'grid_set_tide_vec_uq.pkl','rb') as f:
            self.grid_set_tide_vec_uq = pickle.load(f)

        return 0

    def run_output_difference(self, compare_id, compare_prefix):

        print('Ouput difference...')

        this_result_folder = self.estimation_dir
        test_id = self.test_id

        self.load_master_model(test_id)
        self.load_slave_model(num=compare_id,prefix=compare_prefix)

        #self.load_true_est_uq()

        quant_list = [ 'secular_horizontal_velocity_difference' ]

        for quant_name in quant_list:

            print('Output quantity name: ', quant_name)
            grid_set_quant = {}

            if quant_name == 'secular_horizontal_velocity_difference':
                grid_set_slave = self.grid_set_slave_model_tide_vec
                grid_set_master = self.grid_set_master_model_tide_vec

                for point in grid_set_master.keys():
                    if not np.isnan(grid_set_master[point][0,0]) and point in grid_set_slave:
                        quant_master = self.tide_vec_to_quantity(input_tide_vec = grid_set_master[point], quant_name = "secular_horizontal_velocity_EN")
                        quant_slave = self.tide_vec_to_quantity(input_tide_vec = grid_set_slave[point], quant_name = 'secular_horizontal_velocity_EN')
                        grid_set_quant[point] = np.linalg.norm(quant_master - quant_slave, 2)

                # Write to xyz file.
                state = 'est'

                xyz_name = os.path.join(this_result_folder, '_'.join([str(test_id), state, quant_name, str(compare_id), compare_prefix]) + '.xyz')
                self.display.write_dict_to_xyz(grid_set_quant, xyz_name = xyz_name)

                xyz_name_2 = os.path.join(this_result_folder, '_'.join([str(test_id), state, quant_name]) + '.xyz')
                self.display.write_dict_to_xyz(grid_set_quant, xyz_name = xyz_name_2)

        return 0

    def run_output_others(self):

        this_result_folder = self.estimation_dir
        test_id = self.test_id

        # load others results
        with open(this_result_folder + '/'
                    + str(test_id) + '_' + 'grid_set_others.pkl','rb') as f:
            this_grid_set = pickle.load(f)

        # load up amplitude scaling output at estimation stage
        if self.task_name == 'tides_3':
            up_amp_scale_file = this_result_folder + '/' + str(test_id) + '_est_up_amplitude_scaling.xyz'
            up_amp_scale_grid_set = self.read_xyz_into_dict(up_amp_scale_file)
            #print(list(up_amp_scale_grid_set.keys())[0])
            #print(list(this_grid_set.keys())[0])
            #print(stop)

        # Set quant_list for others
        quant_list = [  'secular_east_north_velocity_corr',
                        'secular_east_up_velocity_corr',
                        'secular_north_up_velocity_corr',
                        'height',
                        'max_num_of_csk_offset_pairs',
                        'num_of_csk_offset_pairs',
                        'ratio_of_valid_csk_offset_pairs',
                        'max_num_of_s1_offset_pairs',
                        'num_of_s1_offset_pairs',
                        'ratio_of_valid_s1_offset_pairs',
                        ]

        if self.task_name == 'tides_3':
            # credible interval needs to before grounding level for filtering purposes
            #quant_list = quant_list + ["up_scale", "grounding_level_credible_interval", "optimal_grounding_level_prescale", "optimal_grounding_level"]

            #quant_list = quant_list + ["up_scale", "grounding_level_credible_interval", "optimal_grounding_level_prescale", "optimal_grounding_level"]
            # up_scale is deprecated
            quant_list = quant_list + ["grounding_level_credible_interval", "optimal_grounding_level_prescale", "optimal_grounding_level"]
            # 2022.09.17
            quant_list = quant_list + ["lowest_tide_height"]

            #quant_list=["up_scale", "grounding_level_credible_interval", "optimal_grounding_level_prescale", "optimal_grounding_level", "grounding_duration", "height"]

            #quant_list_for_bias = ['optimal_grounding_level', 'up_scale']
            # up_scale is deprecated
            quant_list_for_bias = ['optimal_grounding_level']

        elif self.task_name == 'tides_1':
            quant_list_for_bias = []

        else:
            raise ValueError()

        print("others list: ", quant_list)

        states = ['true', 'est']

        # Used for save the results from true and est to get bias
        saved_grid_set_quant_results = {}

        for state in states:
            for quant_name in quant_list:
                print('Output quantity name: ', quant_name)
                grid_set_quant = {}

                output_keys = this_grid_set.keys()

                for point in output_keys:

                    ###########################################################
                    # The ad hoc treatments, to skip some invalid values

                    # The point should be tuple. However, there are exceptions for the key, e.g. current_auto_enum_stage
                    if not isinstance(point, tuple):
                        continue

                    if self.shelf_points_dict is None:
                        model_up = self.grid_set_velo[point][2]
                    else:
                        # If not exists, then the value is 0
                        model_up = self.shelf_points_dict.get(point, 0)

                    if quant_name.startswith("optimal_grounding_level"):

                        quant_name_orig = 'optimal_grounding_level'

                        # If the data does not exist
                        if not quant_name_orig in this_grid_set[point].keys():
                            continue

                        # Remove the stagnent points
                        #if self.grid_set_velo[point][2]<=0.4:
                        #if self.grid_set_velo[point][2]<=0.2:
                        #if self.grid_set_velo[point][2]<=0.1:
                        #if self.grid_set_velo[point][2]<=0.05:
                        if model_up == 0:
                            continue

                        if this_grid_set[point][quant_name_orig]=='external':
                            continue

                        if state == 'est':
                            optimal_grounding_level = this_grid_set[point][quant_name_orig]/(10**6)

                            # Remove very low grounding level, note the value is integer
                            if self.proj == 'Rutford' and optimal_grounding_level <= -2.8:
                                continue

                            #if self.proj == 'Evans' and optimal_grounding_level <= -3.0:
                            #    continue

                            #if self.proj == 'Evans' and optimal_grounding_level <= -2.2:
                            #    continue

                            # Determine credible interval size
                            if self.proj == 'Rutford':
                                #gl_ci_thres = 100
                                #gl_ci_thres = 0.4
                                #gl_ci_thres = 0.5
                                # 2022.03.11: I use 0.6 for the manuscripts
                                #gl_ci_thres = 0.4
                                #gl_ci_thres = 0.6
                                gl_ci_thres = 0.8
                                #gl_ci_thres = 1.0
                                #gl_ci_thres = 1.1
                                #gl_ci_thres = 1.2
                                #gl_ci_thres = 1.5

                            elif self.proj == 'Evans':
                                #gl_ci_thres = 0.5
                                # 2022.03.11: I use 0.6 for manuscripts
                                gl_ci_thres = 0.6
                                #gl_ci_thres = 1.0

                            else:
                                raise ValueError()

                            # Remove based obtained credible level
                            gl_ci = grid_set_gl_ci.get(point, np.nan)
                            if np.isnan(gl_ci) or gl_ci>=gl_ci_thres:
                                continue

                            # Remove points if the lowest_tide_height is too close to the obtained value
                            lowest_tide_height = this_grid_set[point]['lowest_tide_height']
                            #print('state: ', state)
                            #print('lowest_tide_height: ', lowest_tide_height)
                            #print('obtained ogl: ', optimal_grounding_level)
                            #if optimal_grounding_level - lowest_tide_height <= 0.1:
                            #    continue

                            # For Evans project, remove points with csk coverage but only have ascending tracks
                            # For real data only!!
                            if self.proj == 'Evans' and self.s1_data_mode == 3:
                                csk_exists = False
                                csk_desc_exists = False
                                for track in self.grid_set[point]:
                                    if track[3] == 'csk':
                                        csk_exists = True

                                    if track[3] == 'csk' and track[0]>=11:
                                        csk_desc_exists = True

                                if csk_exists == True and csk_desc_exists == False:
                                    continue

                    ###################################################################################
                    # Record everything, including np.nan
                    # np.nan is filtered in write_dict_to_xyz
                    # up_scale
                    if quant_name in ["up_scale"]:
                        if state == "true":
                            grid_set_quant[point] = this_grid_set[point].get("true_" + quant_name, np.nan)

                        elif state == 'est':
                            grid_set_quant[point] = this_grid_set[point].get(quant_name, np.nan)

                    # optimal grounding level
                    elif quant_name.startswith("optimal_grounding_level"):

                        if state == "true":
                            grid_set_quant[point] = this_grid_set[point].get("true_" + "optimal_grounding_level", np.nan)

                        elif state == 'est':
                            optimal_grounding_level_int = this_grid_set[point].get("optimal_grounding_level", np.nan)
                            optimal_grounding_level = optimal_grounding_level_int / 10**6

                            crsp_up_scale = up_amp_scale_grid_set.get(point, np.nan)

                            # unscaled
                            if quant_name.endswith('prescale'):
                                grid_set_quant[point] = optimal_grounding_level

                            # scaled
                            else:
                                if "grounding_level_up_scale" in this_grid_set[point]:
                                    grid_set_quant[point] = optimal_grounding_level * crsp_up_scale
                                else:
                                    raise ValueError()

                        else:
                            raise ValueError()

                    elif quant_name in ["grounding_level_credible_interval"]:
                        if state == "true":
                            grid_set_quant[point] = np.nan

                        elif state == 'est':
                            ci = this_grid_set[point].get("grounding_level_credible_interval", (np.nan, np.nan))
                            grid_set_quant[point] = ci[1] - ci[0]

                            # clip every thing outside ice-shelf
                            if model_up == 0:
                                grid_set_quant[point] = np.nan
                                #grid_set_quant[point] = 10

                        else:
                            raise ValueError()

                    elif quant_name in ["height"]:
                        if state == "true":
                            grid_set_quant[point] = this_grid_set[point].get(quant_name, np.nan)
                        elif state == 'est':
                            grid_set_quant[point] = np.nan

                        else:
                            raise ValueError()

                        if grid_set_quant[point] == 0:
                            grid_set_quant[point] = np.nan

                    elif quant_name in ["max_num_of_csk_offset_pairs", "max_num_of_s1_offset_pairs", "num_of_csk_offset_pairs", "num_of_s1_offset_pairs"]:
                        if state == "true":
                            grid_set_quant[point] = np.nan
                        elif state == 'est':
                            grid_set_quant[point] = this_grid_set[point].get(quant_name, np.nan)
                        else:
                            raise ValueError()

                        if grid_set_quant[point] == 0:
                            grid_set_quant[point] = np.nan

                    elif quant_name in ["ratio_of_valid_csk_offset_pairs"]:
                        if state == "true":
                            grid_set_quant[point] = np.nan

                        elif state == 'est':
                            if grid_set_max_num_of_csk_offset_pairs.get(point, np.nan)>0:
                                grid_set_quant[point] = grid_set_num_of_csk_offset_pairs.get(point, np.nan) / grid_set_max_num_of_csk_offset_pairs.get(point, np.nan)
                            else:
                                grid_set_quant[point] = np.nan
                        else:
                            raise ValueError()

                    elif quant_name in ["ratio_of_valid_s1_offset_pairs"]:
                        if state == "true":
                            grid_set_quant[point] = np.nan

                        elif state == 'est':
                            if grid_set_max_num_of_s1_offset_pairs.get(point, np.nan)>0:
                                grid_set_quant[point] = grid_set_num_of_s1_offset_pairs.get(point, np.nan) / grid_set_max_num_of_s1_offset_pairs.get(point, np.nan)
                            else:
                                grid_set_quant[point] = np.nan
                        else:
                            raise ValueError()

                    elif quant_name in ["secular_east_north_velocity_corr"]:
                        if state == 'true':
                            grid_set_quant[point] = 0

                        elif state == 'est':
                            try:
                                grid_set_quant[point] = this_grid_set[point]['secular_corr'][0]
                            except:
                                grid_set_quant[point] = np.nan

                        else:
                            raise ValueError()

                    elif quant_name in ["secular_east_up_velocity_corr"]:
                        if state == 'true':
                            grid_set_quant[point] = 0

                        elif state == 'est':
                            try:
                                grid_set_quant[point] = this_grid_set[point]['secular_corr'][1]
                            except:
                                grid_set_quant[point] = np.nan

                        else:
                            raise ValueError()

                    elif quant_name in ["secular_north_up_velocity_corr"]:
                        if state == 'true':
                            grid_set_quant[point] = 0

                        elif state == 'est':
                            try:
                                grid_set_quant[point] = this_grid_set[point]['secular_corr'][2]
                            except:
                                grid_set_quant[point] = np.nan

                        else:
                            raise ValueError()

                    elif quant_name in ["lowest_tide_height"]:
                        if state == 'true':
                            grid_set_quant[point] = this_grid_set[point]["lowest_tide_height"]

                            if grid_set_quant[point] is None:
                                grid_set_quant[point] = np.nan

                            # only ice shelf
                            model_up = self.grid_set_velo[point][2]
                            if model_up == 0:
                                grid_set_quant[point] = np.nan

                        elif state == 'est':
                            grid_set_quant[point] = np.nan

                    else:
                        print(quant_name, "is not found")
                        raise ValueError()

                # Save some results for processing other quantities
                if state == 'est' and quant_name in ["grounding_level_credible_interval"]:
                    grid_set_gl_ci = grid_set_quant

                # Save some results for processing other quantities
                if state == 'est' and quant_name in ["max_num_of_csk_offset_pairs"]:
                    grid_set_max_num_of_csk_offset_pairs = grid_set_quant

                if state == 'est' and quant_name in ["num_of_csk_offset_pairs"]:
                    grid_set_num_of_csk_offset_pairs = grid_set_quant

                if state == 'est' and quant_name in ["max_num_of_s1_offset_pairs"]:
                    grid_set_max_num_of_s1_offset_pairs = grid_set_quant

                if state == 'est' and quant_name in ["num_of_s1_offset_pairs"]:
                    grid_set_num_of_s1_offset_pairs = grid_set_quant

                #if state == 'est' and quant_name in ["lowest_tide_height"]:
                #    grid_set_lowest_tide_height = grid_set_quant

                # Write to xyz file.
                xyz_name = os.path.join(this_result_folder, str(test_id) + '_' + state + '_' + 'others' + '_' + quant_name + '.xyz')
                print("Saving: ", xyz_name)

                self.display.write_dict_to_xyz(grid_set_quant, xyz_name = xyz_name)

                # Save the results
                saved_grid_set_quant_results[(state, quant_name)] = grid_set_quant

        # bias
        state = 'bias'
        for quant_name in quant_list_for_bias:

            print('Output quantity name: ', quant_name)
            grid_set_quant = {}

            output_keys = this_grid_set.keys()

            true_grid_set_quant = saved_grid_set_quant_results[("true", quant_name)]
            est_grid_set_quant = saved_grid_set_quant_results[("est", quant_name)]

            for point in output_keys:

                if not isinstance(point, tuple):
                    continue

                grid_set_quant[point] = est_grid_set_quant.get(point,np.nan) - true_grid_set_quant.get(point,np.nan)

            # Write to xyz file.
            xyz_name = os.path.join(this_result_folder, str(test_id) + '_' + state + '_' + 'others' + '_' + quant_name + '.xyz')

            #print(grid_set_quant)
            #print(sorted(grid_set_quant.keys()))
            #print(len(grid_set_quant.keys()))
            #print(stop)

            self.display.write_dict_to_xyz(grid_set_quant, xyz_name = xyz_name)

        return 0

    def run_output_analysis(self):

        this_result_folder = self.estimation_dir
        test_id = self.test_id

        # residual_vs_tide_height
        if self.task_name == 'residual_vs_tide_height':
            pkl_name = '_'.join([str(test_id), 'grid_set_analysis', self.task_name, self.analysis_name])  + '.pkl'

            # Load the pickle file
            with open(this_result_folder + '/' + pkl_name,'rb') as f:
                this_grid_set = pickle.load(f)

            # Set the quantities for output
            state='est'

            quant_names = ['best_slr_results', 'best_slr_data_stats','lowest_tide']

            subquant_names = {}
            subquant_names['best_slr_results'] = ['slope','intercept','r_value','p_value','min_proxy_tide','track_num']
            subquant_names['best_slr_data_stats']=['data_mean','data_median','data_std','picked_data_mean','picked_data_median','picked_data_std']
            subquant_names['lowest_tide']=['height','track_num']

            print('subquant_names: ', subquant_names)
            #print(stop)
            for quant_name in quant_names:

                for subquant_name in subquant_names[quant_name]:

                    print('Output quantity name: ', quant_name +'_' + subquant_name)

                    grid_set_quant = {}
                    output_keys = this_grid_set.keys()

                    for point in output_keys:

                        point_values = this_grid_set[point]

                        # Valid result
                        if len(point_values)<20:
                            #print(point_values)
                            point_quant_values = point_values[quant_name]

                            # This is not an empty dictionary
                            if len(point_quant_values)>0:
                                grid_set_quant[point] = point_quant_values[subquant_name]
                            else:
                                grid_set_quant[point] = np.nan
                        else:
                            grid_set_quant[point] = np.nan

                    # Write to xyz file.
                    xyz_name = os.path.join(this_result_folder, '_'.join((str(test_id), state, self.analysis_name, quant_name, subquant_name)) + '.xyz')
                    self.display.write_dict_to_xyz(grid_set_quant, xyz_name = xyz_name)

        # residual analysis
        elif self.task_name == 'residual_analysis':
            pkl_name = '_'.join((str(test_id), 'grid_set_analysis', self.task_name))  + '.pkl'

            with open(this_result_folder + '/' + pkl_name,'rb') as f:
                residual_set_by_point = pickle.load(f)

            print(residual_set_by_point)

            # Rearrange the dictionary to generate residual_set_by_track
            residual_set_by_obs = collections.defaultdict(dict)

            for point in residual_set_by_point.keys():
                for obs in residual_set_by_point[point]:
                    print(point, obs, residual_set_by_point[point][obs])
                    residual_set_by_obs[obs][point] = residual_set_by_point[point][obs]

            # Output the residuals
            print(residual_set_by_obs)
            for obs in residual_set_by_obs.keys():

                (obs_sate, obs_track_num), obs_vec = obs
                xyz_name = os.path.join(this_result_folder, '_'.join(['residual',obs_sate, str(obs_track_num), obs_vec]) + '.xyz')

                grid_set_quant = residual_set_by_obs[obs]

                print(xyz_name)
                self.display.write_dict_to_xyz(grid_set_quant, xyz_name = xyz_name)

        else:
            raise ValueError()

        return 0

    def output_estimations(self, output_states):

        modeling_tides = self.modeling_tides
        n_modeling_tide = self.n_modeling_tides

        this_result_folder = self.estimation_dir
        test_id = self.test_id

        quant_list_name = self.quant_list_name
        task_name = self.task_name

        self.load_everything()

        quant_list = [  'secular_horizontal_speed',
                        'secular_east_velocity',
                        'secular_north_velocity',
                        'Msf_horizontal_displacement_amplitude',
                        'Msf_east_displacement_amplitude',
                        'Msf_north_displacement_amplitude',
                        'M2_up_displacement_amplitude',
                        'O1_up_displacement_amplitude']

        quant_list = [  'secular_horizontal_speed',
                        'secular_east_velocity',
                        'secular_north_velocity',
                        'secular_up_velocity',
                        'secular_horizontal_velocity',
                        'Msf_horizontal_displacement_amplitude',
                        'Msf_north_displacement_amplitude',
                        'Msf_north_displacement_phase',
                        'M2_up_displacement_amplitude',
                        'M2_up_displacement_phase',
                        'O1_up_displacement_amplitude',
                        'O1_up_displacement_phase']

        quant_list = [  'secular_horizontal_speed',
                        'secular_east_velocity',
                        'secular_north_velocity',
                        'secular_up_velocity',
                        'secular_horizontal_velocity',

                        'Msf_horizontal_displacement_amplitude',
                        'Msf_north_displacement_amplitude',
                        'Msf_north_displacement_phase',

                        'Mf_horizontal_displacement_amplitude',
                        'Mf_north_displacement_amplitude',
                        'Mf_north_displacement_phase',

                        'M2_up_displacement_amplitude',
                        'M2_up_displacement_phase',
                        'O1_up_displacement_amplitude',
                        'O1_up_displacement_phase']

        quant_list = [  'secular_horizontal_speed',
                        'secular_east_velocity',
                        'secular_north_velocity',
                        'secular_up_velocity',
                        'secular_horizontal_velocity',

                        'M2_up_displacement_amplitude',
                        'M2_up_displacement_phase',
                        'O1_up_displacement_amplitude',
                        'O1_up_displacement_phase',
                        'N2_up_displacement_amplitude',
                        'N2_up_displacement_phase',
                        'Q1_up_displacement_amplitude',
                        'Q1_up_displacement_phase',

                        # Msf
                        "Msf_horizontal_displacement_group",
                        "Msf_up_displacement_amplitude",
                        "Msf_up_displacement_phase",

                        # Msf east and north
                        "Msf_east_displacement_amplitude",
                        "Msf_east_displacement_phase",

                        "Msf_north_displacement_amplitude",
                        "Msf_north_displacement_phase",

                        # Mf
                        'Mf_horizontal_displacement_amplitude',
                        "Mf_up_displacement_amplitude",
                        "Mf_up_displacement_phase",

                        # M2
                        'M2_horizontal_displacement_amplitude',

                        # O1
                        'O1_horizontal_displacement_amplitude'
                        ]

        #quant_list = [      'secular_east_velocity_variation',
        #                    'secular_north_velocity_variation',
        #                    'secular_up_velocity_variation']

        if self.task_name == 'tides_1':
            quant_list = quant_list + [ 'M2_up_displacement_amplitude_norm',
                                        'O1_up_displacement_amplitude_norm',
                                        'N2_up_displacement_amplitude_norm']

        if quant_list_name == "BM_2017":
            quant_list = [  'secular_horizontal_speed',
                            'secular_east_velocity',
                            'secular_north_velocity',
                            'secular_up_velocity',
                            'secular_horizontal_velocity',

                            'M2_up_displacement_amplitude',
                            'M2_up_displacement_phase',
                            'O1_up_displacement_amplitude',
                            'O1_up_displacement_phase',

                            # Msf
                            "Msf_horizontal_displacement_group"
                            ]

        sub_quant_names_for_groups = {}
        sub_quant_names_for_groups["Msf_horizontal_displacement_group"] = [ "Msf_along_flow_displacement_amplitude",
                                                                            "Msf_along_flow_displacement_phase",
                                                                            "Msf_along_flow_displacement_phase_in_deg",

                                                                            "Msf_alf_speed_div_secular_speed",
                                                                            "Msf_crf_speed_div_secular_speed",

                                                                            "Msf_cross_flow_displacement_amplitude",
                                                                            "Msf_cross_flow_displacement_phase",

                                                                            "Msf_horizontal_displacement_amplitude"]

        # Additional parameters
        if task_name == 'tides_3':
            quant_list.append('up_amplitude_scaling')

        if self.est_topo_resid == True:
            quant_list.append('topo_resid')

        print("quant_list: ", quant_list)

        ### End with quant list ###

        states = {}
        states['true'] = self.grid_set_true_tide_vec
        states['est'] = self.grid_set_tide_vec

        # This is from model_posterior_to_uncertainty, which converts Cm_p to tide_vec_uq
        states['uq'] = self.grid_set_tide_vec_uq

        # Look through the sets
        phase_center = {}

        # Used for save the results from true and est to get bias
        saved_grid_set_quant_results = {}

        # Create a file to save mean phase
        if 'true' in output_states or 'est' in output_states:
            f_mp  = open(self.estimation_dir + '/' + 'mean_phase.txt','w')

        print("output states: ", output_states)
        for state in output_states:

            print("\n")
            print("###########################")
            print("current state: ", state)

            if state in ["true","est","uq"]:
                this_grid_set = states[state]

            # Loop through the quantities.
            for quant_name in quant_list:

                ## Derive the point set
                # normal states
                if state in ["true", "est", "uq"]:

                    # down sample for velocity vector.
                    if quant_name == 'secular_horizontal_velocity':
                        output_keys = []

                        for point in this_grid_set.keys():

                            lon, lat = point
                            lon_ind = np.round(lon/self.lon_step_int)
                            lat_ind = np.round(lat/self.lat_step_int)

                            if self.resolution == 100:
                                downsample = 50
                            elif self.resolution == 500:
                                downsample = 10
                            elif self.resolution == 1000:
                                downsample = 5
                            elif self.resolution == 2000:
                                downsample = 5
                            else:
                                raise Exception()

                            #downsample = 1
                            if lon_ind % downsample==0 and lat_ind % downsample==0:
                                output_keys.append((lon,lat))

                        output_keys = set(output_keys)

                    else:
                        output_keys = this_grid_set.keys()
                        print("size of output point set: ", len(output_keys))

                    # Note that: For "true", there is no output_keys in data_mode 3.
                    print('Output quantity name: ', quant_name)

                    # Initialization
                    grid_set_quant = {}
                    # no phase correction
                    grid_set_quant_npc = {}

                    # Check if this is a single or group quant_name
                    # group mode
                    if quant_name.endswith("group"):
                        print("group name")
                        if quant_name == "Msf_horizontal_displacement_group":
                            sub_quant_names = sub_quant_names_for_groups[quant_name]

                            for sub_quant_name in sub_quant_names:
                                grid_set_quant[sub_quant_name] = {}
                                grid_set_quant_npc[sub_quant_name] = {}

                        else:
                            raise Exception("Undefined group name")

                        for point in output_keys:
                            # The vector is not nan
                            if not np.isnan(this_grid_set[point][0,0]):

                                # when processing uq, need to accompany it with est
                                if state in ['true', 'est']:
                                    input_tide_vec = this_grid_set[point]
                                elif state in ['uq']:
                                    input_tide_vec = (states['est'][point], this_grid_set[point])
                                else:
                                    raise Exception()

                                quant_group = self.tide_vec_to_quantity(input_tide_vec = input_tide_vec, quant_name = quant_name, point = point, state=state)

                                # Save it into grid_set_quant
                                for sub_quant_name in sub_quant_names:
                                    try:
                                        grid_set_quant[sub_quant_name][point] = quant_group[sub_quant_name]
                                    except:
                                        print(quant_group.keys())
                                        print(grid_set_quant.keys())
                                        print(stop)

                    # Normal single mode
                    else:
                        sub_quant_names = [quant_name]
                        grid_set_quant[quant_name] = {}
                        grid_set_quant_npc[quant_name] = {}

                        # Loop through each point in the output point set
                        # If state is true and data_mode is 3, there is simply no key
                        for point in output_keys:

                            # Only record points where inverse problem can be done, Cm_p exists.
                            if not np.isnan(this_grid_set[point][0,0]):

                                # when processing uq, need to accompany it with est
                                if state in ['true', 'est']:
                                    input_tide_vec = this_grid_set[point]
                                elif state in ['uq']:
                                    input_tide_vec = (states['est'][point], this_grid_set[point])
                                else:
                                    raise Exception()

                                # It is possible that some tides are not in the model. This is taken care of in the called method.
                                quant = self.tide_vec_to_quantity(input_tide_vec = input_tide_vec, quant_name = quant_name, point=point, state=state)

                                # Here we record everything, if Cm_p exists, including nan futher filtered by tide_vec_to_quantity.
                                grid_set_quant[quant_name][point] = quant

                    ######################################################
                    # Some additonal work on removing bad values

                    # 1. Only keep data with at least two tracks
                    for sub_quant_name in sub_quant_names:
                        for point in grid_set_quant[sub_quant_name].keys():
                            # Need at least two track
                            if len(self.grid_set[point])<=1:
                                grid_set_quant[sub_quant_name][point] = np.nan

                    ########    End of extraction   #############

                    ##### Do phase correction for mean phase #################
                    do_correction = True
                    if self.no_phase_correction:
                        do_correction = False

                    ## Do phase correction with the mean phase of true model
                    do_correction_with_true = False

                    # If S1 is not used. Remove wrong estimation where there is only one csk track
                    for sub_quant_name in sub_quant_names:
                        if self.proj == 'Evans' and self.s1_data_date_option == 'no_data':
                            for point in grid_set_quant[sub_quant_name].keys():
                                csk_tracks = [track_info[0] for track_info in self.grid_set[point] if track_info[3] == 'csk']
                                # Need at least two track and cannot be both asc or desc
                                if len(csk_tracks)<=1 or len(csk_tracks) == len([ i for i in csk_tracks if i<=10]) or len(csk_tracks) == len([i for i in csk_tracks if i>=11]):
                                    grid_set_quant[sub_quant_name][point] = np.nan

                    for sub_quant_name in sub_quant_names:
                        if (state=='true' or state=='est') and 'phase' in sub_quant_name:
                            ### Some post processing on phase
                            # Remove wrong phase where there is only one track (this is supposed to be done in the lines above)
                            for point in grid_set_quant[sub_quant_name].keys():
                                # Need at least two track
                                if len(self.grid_set[point])<=1:
                                    grid_set_quant[sub_quant_name][point] = np.nan

                            # Make a copy of the quant before phase correction
                            grid_set_quant_npc[(state, sub_quant_name)] = grid_set_quant[sub_quant_name].copy()

                            # Get all the values
                            values = np.asarray(list(grid_set_quant[sub_quant_name].values()))

                            # Count the number of non-zero values
                            count = np.count_nonzero(~np.isnan(values))

                            # If there are at least one valid value
                            if count>0:
                                # this is turned off ususally
                                if do_correction_with_true ==True and sub_quant_name in phase_center and state == "est":
                                    print("In phase center: ", sub_quant_name)
                                    center = phase_center[sub_quant_name]
                                # default version
                                else:
                                    print("Calculated the mean phase")
                                    center = np.nansum(values) /count

                                # Fix the center for certain componenets
                                if self.proj == "Evans" and state == 'est'  and sub_quant_name == "Msf_along_flow_displacement_phase":
                                    center = -3.647

                                if self.proj == "Evans" and state == 'est'  and sub_quant_name == "Msf_cross_flow_displacement_phase":
                                    center = -3.0

                                if self.proj == "Rutford" and state == 'est'  and sub_quant_name == "Msf_along_flow_displacement_phase":
                                    center = -4.68

                                if self.proj == "Rutford" and state == 'est'  and sub_quant_name == "Msf_cross_flow_displacement_phase":
                                    center = -1.6

                                # Save the mean phase
                                f_mp.write(state + '_' + sub_quant_name + ' ' + str(center)+'\n')

                                # Do correction
                                # Forced no correction for "in_deg" (ad hoc)
                                if do_correction and not "in_deg" in sub_quant_name:
                                    print("Do mean phase shift")
                                    print("The mean phase is: ", center)
                                    for point in grid_set_quant[sub_quant_name].keys():
                                        grid_set_quant[sub_quant_name][point] -= center
                                else:
                                    print("Skip mean phase shift: ", sub_quant_name)
                                    print("The mean phase is: ", center)

                                if state=="true":
                                    print("Give the mean phase of true model to phase center dictionary")
                                    phase_center[sub_quant_name] = center

                            # Mean phase is not defined
                            else:
                                # Save the mean phase
                                f_mp.write(state + '_' + sub_quant_name + ' ' + 'NaN' + '\n')

                    ######## End of mean phase correction   #####

                elif state == "bias":
                    # Note that: For "true", there is no output_keys in data_mode 3.
                    print('Output quantity name: ', quant_name)

                    # Initialization
                    grid_set_quant = {}
                    # If this is a group
                    if quant_name.endswith("group"):
                        print("group name")
                        if quant_name == "Msf_horizontal_displacement_group":
                            sub_quant_names = sub_quant_names_for_groups[quant_name]
                            for sub_quant_name in sub_quant_names:
                                grid_set_quant[sub_quant_name] = {}
                        else:
                            raise Exception("Undefined group name")

                    # If this is a normal quant_name
                    else:
                        sub_quant_names = [quant_name]
                        grid_set_quant[quant_name] = {}

                    # Find the difference betweeen true and est (without phase correction)
                    for sub_quant_name in sub_quant_names:
                        true_grid_set_quant = saved_grid_set_quant_results[("true", sub_quant_name)]
                        est_grid_set_quant = saved_grid_set_quant_results[("est", sub_quant_name)]

                        for point in true_grid_set_quant.keys():
                            if point in est_grid_set_quant:

                                if isinstance(true_grid_set_quant[point], float):
                                    grid_set_quant[sub_quant_name][point] = est_grid_set_quant[point] - true_grid_set_quant[point]

                                elif isinstance(true_grid_set_quant[point], tuple):
                                    grid_set_quant[sub_quant_name][point] = tuple([est_grid_set_quant[point][j]-true_grid_set_quant[point][j] for j in range(len(true_grid_set_quant[point]))])
                                else:
                                    raise Exception("Unknown type of true_grid_set_quant: ", type(true_grid_set_quant[point]))
                else:
                    raise ValueError('Unknown state')

                #### Write to xyz file #####
                for sub_quant_name in sub_quant_names:
                    xyz_name = os.path.join(this_result_folder, str(test_id) + '_' + state + '_' + sub_quant_name + '.xyz')

                    processed_grid_set_quant = self.process_grid_set_quant(state, sub_quant_name, grid_set_quant[sub_quant_name])

                    # Write the processed grid_set_quant to xyz
                    self.display.write_dict_to_xyz(processed_grid_set_quant, xyz_name = xyz_name)

                    # Save the results
                    # If this is phase, then phase correction may be done.
                    # Record the phase values with no correction
                    if (state=='true' or state=='est') and 'phase' in sub_quant_name:
                        saved_grid_set_quant_results[(state, sub_quant_name)] = grid_set_quant_npc[(state, sub_quant_name)]

                    # In other cases, just record the values
                    else:
                        saved_grid_set_quant_results[(state, sub_quant_name)] = grid_set_quant[sub_quant_name]

        # close the mean phase file
        if 'true' in output_states or 'est' in output_states:
            f_mp.close()

        return 0

    def process_grid_set_quant(self, state, quant_name, grid_set_quant_orig):

        # along-flow phase
        if self.proj == 'Evans' and state == 'est' and quant_name == "Msf_along_flow_displacement_phase":
            print('Processing Evans est Msf_along_flow_displacement_phase')

            grid_set_quant_med = copy.deepcopy(grid_set_quant_orig)
            points_to_keep = set()

            print("total number of points in original grid_set_quant: ", len(grid_set_quant_orig))

            # First, set values outside the range to be np.nan
            minval = -3
            maxval = 3
            for point in grid_set_quant_orig.keys():
                if grid_set_quant_orig[point] < minval or grid_set_quant_orig[point] > maxval:
                    grid_set_quant_med[point] = np.nan

            # Second, remove positive values outside the ice-shelf, west of -77.5
            for point in grid_set_quant_orig.keys():
                lon_int, lat_int = point

                model_up = self.grid_set_velo[point][2]

                if not self.shelf_points_dict is None:
                    model_up = self.shelf_points_dict.get(point, 0)

                if self.int5d_to_float(lon_int) < -77.5 and model_up == 0 and grid_set_quant_orig[point]>0:
                    grid_set_quant_med[point] = np.nan

            # Third, do BFS to find the chunk
            start_lon = -75.6
            start_lat = -77

            points_to_keep = set()
            bfs_q = []
            bfs_q.append((self.round_int_5dec(start_lon), self.round_int_5dec(start_lat)))

            lon_step_int = self.lon_step_int
            lat_step_int = self.lat_step_int

            count = 0
            while len(bfs_q)>0:
                count+=1
                #print(count)

                # Get the head point
                curr_point = bfs_q.pop(0)

                curr_lon, curr_lat = curr_point

                # Four directions
                for shift in ((-1,0),(1,0),(0,-1),(0,1)):
                    ne_lon, ne_lat = curr_lon + lon_step_int * shift[0], curr_lat + lat_step_int * shift[1]
                    ne_point = (ne_lon, ne_lat)

                    if not ne_point in points_to_keep and not np.isnan(grid_set_quant_med.get(ne_point, np.nan)):
                        points_to_keep.add(ne_point)
                        bfs_q.append(ne_point)

            processed_grid_set_quant = {}
            for point in points_to_keep:
                processed_grid_set_quant[point] = grid_set_quant_med[point]

            print("Done with BFS")
            print("total number of points to keep: ", len(processed_grid_set_quant))
            self.saved_points_for_evans_est_Msf_along_flow_disp_phase = points_to_keep

        # cross-flow phase
        elif self.proj == 'Evans' and state == 'est' and quant_name == "Msf_cross_flow_displacement_phase":

            if hasattr(self, 'saved_points_for_evans_est_Msf_along_flow_disp_phase'):
                processed_grid_set_quant = {}
                for point in self.saved_points_for_evans_est_Msf_along_flow_disp_phase:
                    processed_grid_set_quant[point] = grid_set_quant_orig.get(point, np.nan)
            else:
                processed_grid_set_quant = grid_set_quant_orig

        # msf alf speed / secular speed
        elif self.proj == 'Evans' and state == 'est' and quant_name in ["Msf_alf_speed_div_secular_speed", "Msf_crf_speed_div_secular_speed"]:

            if hasattr(self, 'saved_points_for_evans_est_Msf_along_flow_disp_phase'):
                processed_grid_set_quant = {}
                for point in self.saved_points_for_evans_est_Msf_along_flow_disp_phase:
                    processed_grid_set_quant[point] = grid_set_quant_orig.get(point, np.nan)
            else:
                processed_grid_set_quant = grid_set_quant_orig

        # Normalized M2, N2 and O1 amplitude
        elif state == 'est' and quant_name.endswith('up_displacement_amplitude_norm'):

            # Read in the values at the reference point

            # Get the ref point
            if self.proj == 'Rutford':
                data_prefix = 'RIS1'
                ref_point = self.get_ref_point(data_prefix)

            elif self.proj == 'Evans':
                data_prefix = 'EIS'
                ref_point = self.get_ref_point(data_prefix)

            else:
                raise ValueError()

            #print('ref_point: ', ref_point)
            #print(quant_name)

            # Get the filename
            file_name = os.path.join(self.estimation_dir, str(self.test_id) + '_est_' + quant_name[:-5] + '.xyz')

            #print(file_name)

            # Get the winsize
            winsize = (11,11)

            ref_value = self.read_point_data_from_xyz(ref_point, file_name, self.proj, winsize = winsize)

            processed_grid_set_quant = {}
            for point in grid_set_quant_orig:
                processed_grid_set_quant[point] = grid_set_quant_orig[point]/ref_value

        # no processing
        else:

            processed_grid_set_quant = grid_set_quant_orig

        return processed_grid_set_quant

def main(iargs=None):

    inps = cmdLineParse(iargs)

    out = output(inps)

    if out.task_name in ['residual_analysis']:
        out.output_mode = 'analysis'

    print('output_mode: ', out.output_mode)

    # output estimation
    if out.output_mode == 'estimation':

        # If output_names is provided, we need to override the default output names
        if out.output_names is not None:
            # override the settings from param file
            out.output_true = False
            out.output_est = False
            out.output_uq = False
            out.output_resid = False
            out.output_difference = False
            out.output_analysis = False
            out.output_others = False

            for output_name in out.output_names.split(','):
                # Set this output name to be True
                setattr(out, 'output_' + output_name, True)

        ### Prepare the outout ###
        # Start to output
        # First output others
        if out.output_others:
            out.run_output_others()

        # Then output estimations
        output_states = []
        if out.output_true: output_states.append("true")
        if out.output_est:  output_states.append("est")
        if out.output_uq:   output_states.append("uq")

        # bias
        if 'true' in output_states and 'est' in output_states:
            output_states.append("bias")

        ### Do the output jobs ###
        out.output_estimations(output_states)

        # Output residual
        if out.output_resid:
            out.run_output_residual()

        if out.output_difference:
            if out.proj=="Evans":
                # Evans
                #out.run_output_difference(compare_id=620, compare_prefix='true')
                #out.run_output_difference(compare_id=20211514, compare_prefix='true')
                out.run_output_difference(compare_id=20211525, compare_prefix='true')

            elif out.proj == "Rutford":
                # Rutford
                out.run_output_difference(compare_id=202007042, compare_prefix='true')
            else:
                raise Exception()

    elif out.output_mode == 'analysis':

        out.run_output_analysis()

    else:
        raise ValueError()

    # End of main

if __name__=='__main__':
    main()


from urllib.request import urlopen
import requests
from bs4 import BeautifulSoup
import urllib.request

class Sandro:
    def get_info(self, page_url):
        response = requests.get(page_url)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        name = soup.select_one("#title").text
        #price = soup.select_one("span.price-sales").text.strip()
        images = soup.select("img", attrs={"class": "productthumbnail lazyload loaded"})

        return {'name': name, 'images': images}

    def save_images(self, info):
        img_list =[]
        for img in info['images']:
            if "data-hires" in img.attrs:
                img_url = img["data-hires"]
                img_list.append(img_url)
            else:
                pass

        for index, url in enumerate(img_list):
            with urlopen(url) as f:
                with open(f"{info['name']}_{str(index)}.jpg", 'wb') as h:
                    img = f.read()
                    h.write(img)

class DIOR:
    def get_info(self, page_url):
        response = requests.get(page_url)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        name = soup.select_one('span.multiline-text.Titles_title__PAVsd').text
        images = soup.select('img')
        reference = soup.select_one('p.Titles_ref__7LPN1').text.split(':')[1].strip()

        return {'name': name, 'images': images, 'reference': reference}

    def save_images(self, info):
        img_list =[]
        for img in info['images']:
            if info['reference'] in img['src']:
                try:
                    img_list.append(img['src'])
                except:
                    pass

        for index, url in enumerate(img_list):
            with urlopen(url) as f:
                with open(f"{info['name']}_{str(index)}.jpg", 'wb') as h:
                    img = f.read()
                    h.write(img)

class CELINE:
    def get_info(self, page_url):
        response = requests.get(page_url)
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')

        name = soup.select_one('span.o-product__title-truncate.f-body--em').text
        images = soup.find_all('button','m-thumb-carousel__img')
        reference = soup.select_one('span.product-id.d-none').text

        return {'name': name, 'images': images, 'reference': reference}

    def save_images(self, info):
        n = 1
        for img in info['images']:
            imgUrl = img.find('img')['data-src-zoom']
            urllib.request.urlretrieve(imgUrl, info['name'] + str(n)+'.jpg')
            n += 1

        # for index, url in enumerate(img_list):
        #     with urlopen(url) as f:
        #         with open(f"./results/{info['name']}_{str(index)}.jpg", 'wb') as h:
        #             img = f.read()
        #             h.write(img)