from math import floor, ceil

# known
# 40% of the harvest is used for wine
grapes_per_liter = 2.5

# user input
winery_area_cub_met = int(input())
grapes_per_cub_met = float(input())
needed_wine = int(input())
workers_number = int(input())

# logic
grapes_produced = winery_area_cub_met * grapes_per_cub_met
grapes_for_wine = grapes_produced * 0.4
wine_produced = grapes_for_wine / grapes_per_liter

# code output
if wine_produced < needed_wine:
    print(f"It will be a tough winter! More {floor(needed_wine - wine_produced)} liters wine needed.")
else:
    print(f"Good harvest this year! Total wine: {floor(wine_produced)} liters.")
    print(f"{ceil(wine_produced - needed_wine)} liters left -> "
          f"{ceil((wine_produced - needed_wine) / workers_number)} liters per person.")


# This checks that the loadflow results are in limits.
#
# All you really want to do with this is call
#   limitChecker = Limits()
#   limitsChecker.check(loadflow_stdout)

import csv
import string

def skiplines(infile, n):
    for x in range(n):
        infile.readline()

def cleanup_output(infile):
    skiplines(infile, 5)
    result = []

    while True:
        line = infile.readline()
        cols = string.split(line)
        if len(cols) == 0:
            break

        Vm = float(cols[6])
        result.append(["B", cols[1], Vm])

    skiplines(infile, 2)

    while True:
        line = infile.readline()
        cols = string.split(line)
        if len(cols) == 0:
            break

        P1 = float(cols[len(cols) - 4])
        P2 = float(cols[len(cols) - 2])
        P = max(abs(P1), abs(P2))

        result.append(["L", cols[0], cols[1], P])

    return result

class Limits:
    """min_volt, max_volt, line_limit_type, line_limits"""

    def __init__(self, limitsfile, min_volt=0.9, max_volt=1.1, line_limit_type=1):
        self.min_volt = min_volt
        self.max_volt = max_volt
        self.line_limit_type = line_limit_type
        self.readlimits(limitsfile)

    def readlimits(self, limitsfile):
        self.line_limit = {}
        self.gen_limit = {}
        reader = csv.reader(limitsfile)
        for row in reader:
            if row[0].strip() == "line":
                self.line_limit[(row[2].strip(), row[3].strip())] = row[4:]
            elif row[0].strip() == "gen":
                self.gen_limit[row[1].strip()] = row[2].strip()
            else:
                raise "expected 'line' got '" + row[0] + "'"

    def businlimit(self, row):
        if not (self.min_volt < float(row[2]) < self.max_volt):
            # print "BUS-LIMIT, " , row[1] , ", " , row[2]
            return False
        return True

    def lineinlimit(self, row):
        name = (row[1], row[2])
        if name in self.line_limit:
            lim = self.line_limit[name]
            limit = float(lim[self.line_limit_type])
            P = float(row[3])

            if P > limit:
                # print "LINE-LIMIT, ", row[1], ", ", row[2], ", ", P, ", ", limit
                return False
        else:
            pass  # it's probably a generator interconnect
        return True

    def check(self, loadflow_output):
        cleaned_loadflow_output = cleanup_output(loadflow_output)
        failcount = 0
        for row in cleaned_loadflow_output:
            if row[0] == "B":
                if not self.businlimit(row):
                    failcount += 1
            elif row[0] == "L":
                if not self.lineinlimit(row):
                    failcount += 1
            else:
                raise "expected 'L' or 'B' got '" + row[0] + "'"
        return failcount


from tir import Webapp
import unittest

class ATFA490(unittest.TestCase):

    @classmethod
    def setUpClass(inst):

        inst.oHelper = Webapp()

        inst.oHelper.Setup("SIGAATF", "01/03/2016", "T1", "D MG 01 ", "01")
        inst.oHelper.Program("ATFA490")

        # Para que o Tipo de Saldo seja considerado nas pesquisas e validações relacionados

        #inst.oHelper.AddParameter("MV_CTBCTG ","", ".T.")
        # inst.oHelper.SetParameters()

# ==================================================================================
#                 Copiar Provisão
# Kanoah "https://jiraproducao.totvs.com.br/secure/Tests.jspa#/testCase/GTSER-T532"
# ==================================================================================
    def test_ATFA490_001(self):
            ##COPIA
        self.oHelper.SearchBrowse("D MG 01 00000000060001")
        self.oHelper.SetButton("Outras Ações", "Copiar")
        self.oHelper.SetValue("FNU_COD","TIR01")
        self.oHelper.SetValue("FNU_DESCR","Copia TIR")

        self.oHelper.SetButton("Confirmar")
        self.oHelper.SetButton("Fechar")
        self.oHelper.AssertTrue()

# ==================================================================================
#                 Revisar Provisão
# Kanoah "https://jiraproducao.totvs.com.br/secure/Tests.jspa#/testCase/GTSER-T551"
# ==================================================================================
    def test_ATFA490_002(self):
        self.oHelper.SearchBrowse("D MG 01 00000000070001")

        self.oHelper.SetButton("Outras Ações", "Revisar")

        self.oHelper.SetValue("Mostra Lanc. Contabeis ?","Não")
        self.oHelper.SetValue("Aglut Lançamentos ?","Não")
        self.oHelper.SetValue("Contabiliza Online ?","Não")
        self.oHelper.SetButton("OK")

        self.oHelper.SetButton("Confirmar")
        self.oHelper.CheckHelp("Deseja gravar a operação, bloqueando a revisão anterior ?","Sim")

        self.oHelper.AssertTrue()

# ==================================================================================
#                 Desbloqueio de Provisão
# Kanoah "https://jiraproducao.totvs.com.br/secure/Tests.jspa#/testCase/GTSER-T537"
# ==================================================================================
    def test_ATFA490_003(self):
        self.oHelper.SearchBrowse("D MG 01 00000000080001")
        self.oHelper.SetButton("Outras Ações", "Bloqueio/Desbloq")
        self.oHelper.SetValue("FNU_MSBLQL","2 - Não")

        self.oHelper.SetButton("Confirmar")
        self.oHelper.CheckHelp("Deseja Realizar o bloqueio/desbloqueio do controle de provisão ?","Sim")
        self.oHelper.SetButton("Fechar")

        self.oHelper.SetButton("Visualizar")
        self.oHelper.CheckResult("FNU_MSBLQL","2 - Não")
        self.oHelper.SetButton("Fechar")
        self.oHelper.AssertTrue()

    @classmethod
    def tearDownClass(inst):
        inst.oHelper.TearDown()

if __name__ == '__main__':
    unittest.main()


import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# Load dataset
recipes_df = pd.read_csv('indian_dishes.csv')

# Define vectorizer object
vectorizer = CountVectorizer()

# Create ingredient list string for each recipe
recipes_df['ingredients'] = recipes_df.apply(lambda x: ','.join([str(i) for i in x[1:]]), axis=1)

# Create ingredient count matrix
ingredient_count_matrix = vectorizer.fit_transform(recipes_df['ingredients'])

# Define function to get top n similar recipes
def get_similar_recipes(input_dishes, n, recipes_df):
    # Create ingredient list string for input dishes
    input_dishes_list = ','.join(input_dishes)

    # Add input dishes to dataset
    new_row = {'name': 'input_dishes', 'ingredients': input_dishes_list}
    recipes_df = recipes_df.append(new_row, ignore_index=True)

    # Create ingredient count matrix
    ingredient_count_matrix = vectorizer.fit_transform(recipes_df['ingredients'])

    # Get cosine similarity scores between input dishes and all recipes
    cosine_similarities = cosine_similarity(ingredient_count_matrix)
    cosine_scores = list(enumerate(cosine_similarities[-1]))

    # Sort cosine scores in descending order
    sorted_cosine_scores = sorted(cosine_scores, key=lambda x:x[1], reverse=True)

    # Get top n similar recipes
    top_n_similar_recipes = []
    for i in range(1,n+1):
        top_n_similar_recipes.append(recipes_df.iloc[sorted_cosine_scores[i][0]]['name'])

    # Remove input dishes from dataset
    recipes_df.drop(recipes_df.tail(1).index, inplace=True)

    return top_n_similar_recipes

# Example usage
input_dishes = ['rice', 'chicken', 'tomato', 'onion', 'ginger', 'garlic']
n = 5

similar_recipes = get_similar_recipes(input_dishes, n, recipes_df)

print(similar_recipes)


from rsf.proj import *

#==============================================================
# job running parameters

xv0=12.192
zv0=0

par = dict(
    ns=1376,
    dt=0.0008,

    isep=0,
    ihomo=0,
    itaper=1,
#    tapertype="DaveHale",
#    tapertype="Sinc",
#    tapertype="Cosine",
    tapertype="TTI",
#    tapertype="None",
    nstep=2,
    nxv=1400, ox=xv0, lx='x',  ux='km',
    nzv=1320, oz=zv0, lz='z',  uz='km',
    )
# =============================================================
# download Hess VTI models from 'ftp://software.seg.org'
# Depth grid: 1501
# Horizontal grid: 3617
# Vertical and horizontal spacing are both 20ft.

from rsf.gallery import hessvti

#hessvti.get_smallmodel('vp vs epsilon delta')
hessvti.get_model('vp vs epsilon delta')

ft2km = 0.0003048
dx = 20*ft2km
dz = 20*ft2km

for mod in Split('vp vs epsilon delta'):
	Flow(mod+'1',mod,
	     '''
             put d1=%g d2=%g unit1=km label1=Depth unit2=km label2=Distance unit=m/s |
             window n1=%d min1=%g n2=%d min2=%g
             ''' % (dx, dz,
		    par['nzv'],par['oz'],
		    par['nxv'],par['ox']))

Flow('vs_hess','vp_hess',
     '''
     math output="input*0.5"
     ''')

# =============================================================
# produce vp0 & vs0 model by scaling vp model
Flow('vp0','vp1',
     '''
     math output="input*1000" | smooth rect1=5 rect2=5
     ''')

Flow('vs0','vs1',
     '''
     math output="input*1000" | smooth rect1=5 rect2=5
     ''')

Flow('epsi','epsilon1',
     '''
     smooth rect1=3 rect2=3
     ''')

Flow('del','delta1',
     '''
     smooth rect1=3 rect2=3
     ''')

# =============================================================

# produce vp0 & vs0 model by scaling vp model
Flow('hessvp0','vp1',
     '''
     math output="input*1000"
     ''')

Flow('hessvs0','vs1',
     '''
     math output="input*1000"
     ''')

Flow('hessepsilon','epsilon1',
     '''
     math output="input"
     ''')

Flow('hessdelta','delta1',
     '''
     math output="input"
     ''')

name0='''
hessvp0 hessvs0
'''

name00='''
hessepsilon hessdelta
'''

Result('vp0',
	'''
	grey color=g scalebar=y screenht=4 screenwd=10 barreverse=y wanttitle=n}
	''')

for ff in Split(name0):
        Result(ff,
	'''
	grey color=g scalebar=y screenht=4 screenwd=10 barreverse=y wanttitle=n}
	''')

for gg in Split(name00):
        Result(gg,
	'''
	grey color=g scalebar=y screenht=4 screenwd=10 barreverse=y wanttitle=n}
	''')
#	grey color=j scalebar=y screenratio=0.47 allpos=n barreverse=y wanttitle=n}
################################################################
# Stage I:Forward modeling with real SV0
################################################################
name1='''
PseudoPurePx PseudoPurePz PseudoPureP
'''

Flow(['PseudoPurePx',  'PseudoPurePz',  'PseudoPureP'],
         'vp0  vs0  epsi del',
         '''
         vti2dpseudop
         vp0=${SOURCES[0]}
         vs0=${SOURCES[1]}
         epsi=${SOURCES[2]}
         del=${SOURCES[3]}
         PseudoPurePz=${TARGETS[1]}
         PseudoPureP=${TARGETS[2]}
         ns=%d
         dt=%g
         isep=%d
         ihomo=%d
         itaper=%d
         tapertype=%s
         nstep=%d
         ''' % (par['ns'],par['dt'],par['isep'],par['ihomo'],par['itaper'],par['tapertype'],par['nstep'])
    )

for qq in Split(name1):
        Result(qq,
	'''
	grey polarity=n scalebar=n screenratio=1 wanttitle=n
	''')

################################################################
# Stage II:Forward modeling with finite SV0
################################################################
name2='''
PseudoPurePxFsv0 PseudoPurePzFsv0 PseudoPurePFsv0
'''

Flow(['PseudoPurePxFsv0',  'PseudoPurePzFsv0',  'PseudoPurePFsv0', 'Fvs0'],
         'vp0  epsi del',
         '''
         vti2dpseudopfvs0
         vp0=${SOURCES[0]}
         epsi=${SOURCES[1]}
         del=${SOURCES[2]}
         PseudoPurePz=${TARGETS[1]}
         PseudoPureP=${TARGETS[2]}
         Fvs0=${TARGETS[3]}
         ns=%d
         dt=%g
         isep=%d
         ihomo=%d
         itaper=%d
         tapertype=%s
         nstep=%d
         ''' % (par['ns'],par['dt'],par['isep'],par['ihomo'],par['itaper'],par['tapertype'],par['nstep'])
    )

for qq in Split(name2):
        Result(qq,
	'''
	grey polarity=n scalebar=n screenratio=1 wanttitle=n
	''')
################################################################
# Stage III: Coverted the RTM (cjb-SU) data to RSF format and plot
################################################################
dx=6.096*2/1000
dz=6.096*2/1000
nx0=3617

# Not really reproducible, just display

from rsf.recipes.beg import server

Fetch('hessrtmillum.su','hess',server)

Flow('hessrtm','hessrtmillum.su',
     '''
     datasucjb2rsf2d fn=$SOURCE
     | window n2=1809 min2=0
     | put d1=0.012192 d2=0.012192 unit1=km label1=Depth unit2=km label2=Distance
     ''')

Result('hessrtm',
     '''
     grey pclip=96 color=e polarity=n scalebar=n screenht=6 screenwd=10 wanttitle=n
     ''')

End()


#Sua função vai receber uma string nos parâmetros e retornar de acordo com a tabela abaixo:

#Chave	                Valor
#"Jabroni"	            "Patron Tequila"
#"School Counselor"	    "Anything with Alcohol"
#"Programmer"	        "Hipster Craft Beer"
#"Bike Gang Member"	    "Moonshine"
#"Politican"	        "Your tax dollars"
#"Rapper"	            "Cristal"
#qualquer outra coisa	"Beer"

#Nota: qualquer outra coisa é um caso base: se a string passada por parâmetro não existir na nossa tabela, retornar 'Beer'.

#Atente-se em cobrir os casos em que certas palavras não vêm com a formatação correta.
#Exemplo: o parâmetro "pOLitiCIaN" ainda deve retornar "Your tax dollars".

def get_drink_by_profession(param):
    drinks = {
        "jabroni": "Patron Tequila",
        "school counselor": "Anything with Alcohol",
        "programmer": "Hipster Craft Beer",
        "bike gang member": "Moonshine",
        "politician": "Your tax dollars",
        "rapper": "Cristal"
    }
    param = param.lower()
    for key, value in drinks.items():
        if key == param:
            return value
    return "Beer"


import sys

def memoized_cut_rod(price, length, MaxRev, CutCost, NumCuts, LargestIndexArray) -> int:
    #if MaxRev[length] is already calculated, just return it
    if (int(MaxRev[length]) > 0):
        return MaxRev[length]
    #My base case, if length equals 0 then just return 0
    if (int(length) == 0):
        NumCuts[0] = 0
        return 0
    count = 1
    largestIndex = -1
    largest = 0
    cuts = 0
    while (count <= length):
        # if count equals length, then that means we are not cutting the rod at all, which is why it does not include  "- (CutCost * (1 + NumCuts[length - count]))"
        if (count == length):
            temp = int(price[count]) + int(memoized_cut_rod(price, (length-count), MaxRev, CutCost, NumCuts, LargestIndexArray))
        else:
            temp = int(price[count]) + int(memoized_cut_rod(price, (length-count), MaxRev, CutCost, NumCuts, LargestIndexArray)) - (CutCost * (1 + NumCuts[length - count]))
        if (int(temp) > largest):
            largestIndex = count
            largest = temp
        count = count + 1
    if (largestIndex == length):
        cuts = 0
    else:
        cuts = 1 + NumCuts[length - largestIndex]
    #updates arrays before returning
    MaxRev[length] = largest
    NumCuts[length] = cuts
    LargestIndexArray[length] = largestIndex
    return largest

#Checks to make sure program received file name
if (len(sys.argv) < 2):
    sys.exit("No command line argument")
FileName = sys.argv[1]
File = open(FileName, "r")

#Makes sure file opens properly
if (File.mode != 'r'):
    print("Error opening file")

#reads in data from file into its proper variable
length = (int(File.readline().rstrip()))
CutCost = (int(File.readline().rstrip()))
temp = (File.readline().rstrip())

#define all variables that are needed
count = 0
price = []
start = 0 # variable used to translate file string to int
MaxRev = [] # array that maps rod length to the optimal revenue
NumCuts = [] #array that maps rod length to the number of cuts needed to get optimal revenue
LargestIndexArray = [] #array that maps rod length to the index i, which gives the largest value to the equation price[i] + MaxRev[length - i]

# both while loops below initializes my arrays values
price.append(0)
while (count < len(temp)):
    if (temp[count] == ','):
        price.append(temp[start:count])
        start = count+1
    count = count + 1
price.append(temp[start:count])
count = 0
while (count <= len(price)):
    MaxRev.append(0)
    NumCuts.append(-1)
    LargestIndexArray.append(-1)
    count = count + 1

#call to my function that calculates everything
MaxRev[length] = memoized_cut_rod(price, length, MaxRev, CutCost, NumCuts, LargestIndexArray)

#prints out results
print("My name is Ronnie Clark, and the optimal revenue for rod at length ", end = str(length))
print(" is " , end = str(MaxRev[length]))
print(", and the number of cuts for optimal revenue is ", end = str(NumCuts[length]))
print(", giving you 1 rod at length ", end = str(LargestIndexArray[length]))
count = length - LargestIndexArray[length]
while (count > 0):
    print(", plus 1 rod at length ", end = str(count))
    count = count - LargestIndexArray[count]


import copy
import os
from collections import OrderedDict
from logging import getLogger
from time import time

import numpy as num
import theano.tensor as tt
from pymc3 import Deterministic, Uniform
from pyrocko.gf import LocalEngine, RectangularSource
from theano import config as tconfig
from theano import shared
from theano.printing import Print

from beat import config as bconfig
from beat import covariance as cov
from beat import heart, theanof, utility
from beat.ffi import get_gf_prefix, load_gf_library
from beat.interseismic import geo_backslip_synthetics, seperate_point
from beat.models.base import (
    Composite,
    ConfigInconsistentError,
    FaultGeometryNotFoundError,
    get_hypervalue_from_point,
)
from beat.models.distributions import multivariate_normal_chol

logger = getLogger("geodetic")

km = 1000.0

__all__ = [
    "GeodeticGeometryComposite",
    "GeodeticInterseismicComposite",
    "GeodeticDistributerComposite",
]

class GeodeticComposite(Composite):
    """
    Comprises data structure of the geodetic composite.

    Parameters
    ----------
    gc : :class:`config.GeodeticConfig`
        configuration object containing seismic setup parameters
    project_dir : str
        directory of the model project, where to find the data
    events : list
        of :class:`pyrocko.model.Event`
        contains information of reference event(s), coordinates of reference
        point(s) and source time(s)
    hypers : boolean
        if true initialise object for hyper parameter optimization
    """

    _hierarchicalnames = None
    weights = None

    def __init__(self, gc, project_dir, events, hypers=False):

        super(GeodeticComposite, self).__init__(events)

        logger.debug("Setting up geodetic structure ...\n")
        self.name = "geodetic"
        self._like_name = "geo_like"

        geodetic_data_path = os.path.join(project_dir, bconfig.geodetic_data_name)

        self.datasets = utility.load_objects(geodetic_data_path)
        logger.info("Number of geodetic datasets: %i " % self.n_t)

        # init geodetic targets
        self.targets = heart.init_geodetic_targets(
            datasets=self.datasets,
            earth_model_name=gc.gf_config.earth_model_name,
            interpolation=gc.interpolation,
            crust_inds=[gc.gf_config.reference_model_idx],
            sample_rate=gc.gf_config.sample_rate,
        )

        # merge geodetic data to calculate residuals on single array
        datasets, los_vectors, odws, self.Bij = heart.concatenate_datasets(
            self.datasets
        )
        logger.info("Number of geodetic data points: %i " % self.Bij.ordering.size)

        self.sdata = shared(datasets, name="geodetic_data", borrow=True)
        self.slos_vectors = shared(los_vectors, name="los_vecs", borrow=True)
        self.sodws = shared(odws, name="odws", borrow=True)

        self.noise_analyser = cov.GeodeticNoiseAnalyser(
            config=gc.noise_estimator, events=self.events
        )

        if gc.corrections_config.has_enabled_corrections:
            correction_configs = gc.corrections_config.iter_corrections()
            logger.info("Initialising corrections ...")
            for data in self.datasets:
                data.setup_corrections(
                    event=self.event, correction_configs=correction_configs
                )

        self.config = gc

        if hypers:
            self._llks = []
            for t in range(self.n_t):
                self._llks.append(
                    shared(num.array([1.0]), name="geo_llk_%i" % t, borrow=True)
                )

    def init_weights(self):
        self.weights = []
        for i, data in enumerate(self.datasets):
            if int(data.covariance.data.sum()) == data.ncoords:
                logger.warning(
                    "Data covariance is identity matrix! Please double check!!!"
                )

            choli = data.covariance.chol_inverse
            self.weights.append(shared(choli, name="geo_weight_%i" % i, borrow=True))
            data.covariance.update_slog_pdet()

    @property
    def n_t(self):
        return len(self.datasets)

    def get_all_dataset_names(self, hp_name):
        """
        Return unique GNSS stations and radar acquisitions.
        """
        names = []
        for dataset in self.datasets:
            if dataset.typ == hp_name.split("_")[1]:
                if isinstance(dataset, heart.DiffIFG):
                    names.append(dataset.name)
                elif isinstance(dataset, heart.GNSSCompoundComponent):
                    names.append(dataset.component)
                else:
                    TypeError(
                        'Geodetic Dataset of class "%s" not '
                        "supported" % dataset.__class__.__name__
                    )

        return names

    def analyse_noise(self, tpoint=None):
        """
        Analyse geodetic noise in datasets and set
        data-covariance matrixes accordingly.
        """
        if self.config.noise_estimator.structure == "non-toeplitz":
            results = self.assemble_results(tpoint)
        else:
            results = [None] * len(self.datasets)

        if len(self.datasets) != len(results):
            raise ValueError("Number of datasets and results need to be equal!")

        for dataset, result in zip(self.datasets, results):
            logger.info(
                'Retrieving geodetic data-covariances with structure "%s" '
                "for %s ..." % (self.config.noise_estimator.structure, dataset.name)
            )

            cov_d_geodetic = self.noise_analyser.get_data_covariance(
                dataset, result=result
            )

            if dataset.covariance is None:
                dataset.covariance = heart.Covariance(data=cov_d_geodetic)
            else:
                dataset.covariance.data = cov_d_geodetic

            if int(dataset.covariance.data.sum()) == dataset.ncoords:
                logger.warning(
                    "Data covariance is identity matrix! Please double check!!!"
                )

    def get_hypersize(self, hp_name=""):
        """
        Return size of the hyperparameter

        Parameters
        ----------
        hp_name: str
            of hyperparameter name

        Returns
        -------
        int
        """
        n_datasets = len(self.get_all_dataset_names(hp_name))
        if n_datasets == 0:
            raise ConfigInconsistentError(
                'Found no data for hyperparameter "%s". Please either load'
                " the data or remove it from types dictionary!" % hp_name,
                params="hypers",
            )
        elif self.config.dataset_specific_residual_noise_estimation:
            return n_datasets
        else:
            return 1

    def assemble_results(self, point, **kwargs):
        """
        Assemble geodetic data for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters

        Returns
        -------
        List with :class:`heart.GeodeticResult`
        """

        logger.debug("Assembling geodetic data ...")

        processed_synts = self.get_synthetics(point, outmode="stacked_arrays")

        results = []
        for i, data in enumerate(self.datasets):
            res = data.displacement - processed_synts[i]

            results.append(
                heart.GeodeticResult(
                    point=point,
                    processed_obs=data.displacement,
                    processed_syn=processed_synts[i],
                    processed_res=res,
                )
            )

        return results

    def export(
        self,
        point,
        results_path,
        stage_number,
        fix_output=False,
        force=False,
        update=False,
    ):

        from kite.scene import Scene, UserIOWarning
        from pyrocko.guts import dump

        from beat.plotting import map_displacement_grid

        def save_covs(datasets, cov_mat="pred_v"):
            """
            Save covariance matrixes of given attribute
            """

            covs = {
                dataset.name: getattr(dataset.covariance, cov_mat)
                for dataset in datasets
            }

            outname = os.path.join(results_path, "%s_C_%s" % ("geodetic", cov_mat))
            logger.info('"geodetic covariances" to: %s', outname)
            num.savez(outname, **covs)

        gc = self.config

        results = self.assemble_results(point)

        def get_filename(attr, ending="csv"):
            return os.path.join(
                results_path,
                "{}_{}_{}.{}".format(
                    os.path.splitext(dataset.name)[0], attr, stage_number, ending
                ),
            )

        # export for gnss
        for typ, config in gc.types.items():
            if "GNSS" == typ:
                from pyrocko.model import gnss

                logger.info("Exporting GNSS data ...")
                campaigns = config.load_data(campaign=True)

                for campaign in campaigns:
                    model_camp = gnss.GNSSCampaign(
                        stations=copy.deepcopy(campaign.stations),
                        name="%s_model" % campaign.name,
                    )

                    dataset_to_result = {}
                    for dataset, result in zip(self.datasets, results):
                        if dataset.typ == "GNSS":
                            dataset_to_result[dataset] = result

                    for dataset, result in dataset_to_result.items():
                        for ista, sta in enumerate(model_camp.stations):
                            comp = getattr(sta, dataset.component)
                            comp.shift = result.processed_syn[ista]
                            comp.sigma = 0.0

                    outname = os.path.join(
                        results_path, "gnss_synths_%i.yaml" % stage_number
                    )

                    dump(model_camp, filename=outname)

            elif "SAR" == typ:
                logger.info("Exporting SAR data ...")
                for dataset, result in zip(self.datasets, results):
                    if dataset.typ == "SAR":
                        try:
                            scene_path = os.path.join(config.datadir, dataset.name)
                            logger.info(
                                "Loading full resolution kite scene: %s" % scene_path
                            )
                            scene = Scene.load(scene_path)
                        except UserIOWarning:
                            logger.warning(
                                "Full resolution data could not be"
                                " loaded! Skipping ..."
                            )
                            continue

                        for attr in ["processed_obs", "processed_syn", "processed_res"]:

                            filename = get_filename(attr, ending="csv")
                            displacements = getattr(result, attr)
                            dataset.export_to_csv(filename, displacements)
                            logger.info("Stored CSV file to: %s" % filename)

                            filename = get_filename(attr, ending="yml")
                            vals = map_displacement_grid(displacements, scene)
                            scene.displacement = vals
                            scene.save(filename)
                            logger.info("Stored kite scene to: %s" % filename)

        # export stdz residuals
        self.analyse_noise(point)
        if update:
            logger.info("Saving velocity model covariance matrixes...")
            self.update_weights(point)
            save_covs(self.datasets, "pred_v")

        logger.info("Saving data covariance matrixes...")
        save_covs(self.datasets, "data")

    def init_hierarchicals(self, problem_config):
        """
        Initialize hierarchical parameters.
        Ramp estimation in azimuth and range direction of a radar scene and/or
        Rotation of GNSS stations around an Euler pole
        """
        hierarchicals = problem_config.hierarchicals
        self._hierarchicalnames = []
        for number, corr in enumerate(
            self.config.corrections_config.iter_corrections()
        ):
            logger.info(
                "Evaluating config for %s corrections " "for datasets..." % corr.feature
            )
            if corr.enabled:
                for data in self.datasets:
                    if data.name in corr.dataset_names:
                        hierarchical_names = corr.get_hierarchical_names(
                            name=data.name, number=number
                        )
                    else:
                        hierarchical_names = []

                    for hierarchical_name in hierarchical_names:
                        if not corr.enabled and hierarchical_name in hierarchicals:

                            raise ConfigInconsistentError(
                                "%s %s disabled, but they are defined"
                                " in the problem configuration"
                                " (hierarchicals)!" % (corr.feature, data.name)
                            )

                        if (
                            corr.enabled
                            and hierarchical_name not in hierarchicals
                            and data.name in corr.dataset_names
                        ):
                            raise ConfigInconsistentError(
                                "%s %s corrections enabled, but they are"
                                " not defined in the problem configuration!"
                                " (hierarchicals)" % (corr.feature, data.name)
                            )

                        param = hierarchicals[hierarchical_name]
                        if hierarchical_name not in self.hierarchicals:
                            if not num.array_equal(param.lower, param.upper):
                                kwargs = dict(
                                    name=param.name,
                                    shape=param.dimension,
                                    lower=param.lower,
                                    upper=param.upper,
                                    testval=param.testvalue,
                                    transform=None,
                                    dtype=tconfig.floatX,
                                )

                                try:
                                    self.hierarchicals[hierarchical_name] = Uniform(
                                        **kwargs
                                    )
                                except TypeError:
                                    kwargs.pop("name")
                                    self.hierarchicals[
                                        hierarchical_name
                                    ] = Uniform.dist(**kwargs)

                                self._hierarchicalnames.append(hierarchical_name)
                            else:
                                logger.info(
                                    "not solving for %s, got fixed at %s"
                                    % (
                                        param.name,
                                        utility.list2string(param.lower.flatten()),
                                    )
                                )
                                self.hierarchicals[hierarchical_name] = param.lower
            else:
                logger.info("No %s correction!" % corr.feature)

        logger.info("Initialized %i hierarchical parameters." % len(self.hierarchicals))

    def apply_corrections(self, residuals, point=None, operation="-"):
        """
        Apply all the configured correction terms e.g. SAR orbital ramps,
        GNSS Euler pole rotations etc...
        """
        for i, dataset in enumerate(self.datasets):
            if dataset.has_correction:
                for corr in dataset.corrections:
                    correction = corr.get_displacements(self.hierarchicals, point=point)
                    # correction = Print('corr')(correction)

                    if operation == "-":
                        residuals[i] -= correction  # needs explicit assignment!
                    elif operation == "+":
                        residuals[i] += correction

        return residuals

    def update_llks(self, point):
        """
        Update posterior likelihoods (in place) of the composite w.r.t.
        one point in the solution space.

        Parameters
        ----------
        point : dict
            with numpy array-like items and variable name keys
        """
        results = self.assemble_results(point)
        for l, result in enumerate(results):
            choli = self.datasets[l].covariance.chol_inverse
            tmp = choli.dot(result.processed_res)
            _llk = num.asarray([num.dot(tmp, tmp)])
            self._llks[l].set_value(_llk)

    def get_variance_reductions(self, point, results=None, weights=None):
        """
        Parameters
        ----------
        point : dict
            with parameters to point in solution space to calculate
            variance reductions

        Returns
        -------
        dict of floats,
            keys are nslc_ids
        """
        if results is None:
            results = self.assemble_results(point)

        ndatasets = len(self.datasets)

        assert len(results) == ndatasets

        if weights is None:
            self.analyse_noise(point)
            self.update_weights(point)
            weights = self.weights

        nweights = len(weights)
        assert nweights == ndatasets

        logger.debug("n weights %i , n datasets %i" % (nweights, ndatasets))

        assert nweights == ndatasets

        logger.debug("Calculating variance reduction for solution ...")

        counter = utility.Counter()
        hp_specific = self.config.dataset_specific_residual_noise_estimation

        var_reds = OrderedDict()
        for dataset, weight, result in zip(self.datasets, weights, results):

            hp = get_hypervalue_from_point(
                point, dataset, counter, hp_specific=hp_specific
            )
            icov = dataset.covariance.inverse(num.exp(hp * 2.0))

            data = result.processed_obs
            residual = result.processed_res

            nom = residual.T.dot(icov).dot(residual)
            denom = data.T.dot(icov).dot(data)

            logger.debug("nom %f, denom %f" % (float(nom), float(denom)))
            var_red = 1 - (nom / denom)

            logger.debug("Variance reduction for %s is %f" % (dataset.name, var_red))

            if 0:
                from matplotlib import pyplot as plt

                fig, ax = plt.subplots(1, 1)
                im = ax.imshow(dataset.covariance.data)
                plt.colorbar(im)
                plt.show()

            var_reds[dataset.name] = var_red

        return var_reds

    def get_standardized_residuals(self, point, results=None, weights=None):
        """
        Parameters
        ----------
        point : dict
            with parameters to point in solution space to calculate
            standardized residuals

        Returns
        -------
        dict of arrays of standardized residuals,
            keys are nslc_ids
        """
        if results is None:
            results = self.assemble_results(point)

        if weights is None:
            self.update_weights(point)

        counter = utility.Counter()
        hp_specific = self.config.dataset_specific_residual_noise_estimation

        stdz_residuals = OrderedDict()
        for dataset, result in zip(self.datasets, results):
            hp = get_hypervalue_from_point(
                point, dataset, counter, hp_specific=hp_specific
            )
            choli = num.linalg.inv(dataset.covariance.chol(num.exp(hp * 2.0)))
            stdz_residuals[dataset.name] = choli.dot(result.processed_res)

        return stdz_residuals

class GeodeticSourceComposite(GeodeticComposite):
    """
    Comprises how to solve the non-linear geodetic forward model.

    Parameters
    ----------
    gc : :class:`config.GeodeticConfig`
        configuration object containing seismic setup parameters
    project_dir : str
        directory of the model project, where to find the data
    sources : list
        of :class:`pyrocko.gf.seismosizer.Source`
    events : list
        of :class:`pyrocko.model.Event`
        contains information of reference event, coordinates of reference
        point and source time
    hypers : boolean
        if true initialise object for hyper parameter optimization
    """

    def __init__(self, gc, project_dir, sources, events, hypers=False):

        super(GeodeticSourceComposite, self).__init__(
            gc, project_dir, events, hypers=hypers
        )

        self.engine = LocalEngine(store_superdirs=[gc.gf_config.store_superdir])

        self.sources = sources

    def __getstate__(self):
        self.engine.close_cashed_stores()
        return self.__dict__.copy()

    def point2sources(self, point):
        """
        Updates the composite source(s) (in place) with the point values.
        """
        tpoint = copy.deepcopy(point)
        tpoint.update(self.fixed_rvs)
        tpoint = utility.adjust_point_units(tpoint)

        # remove hyperparameters from point
        hps = self.config.get_hypernames()

        for hyper in hps:
            if hyper in tpoint:
                tpoint.pop(hyper)

        source_params = list(self.sources[0].keys())
        for param in list(tpoint.keys()):
            if param not in source_params:
                tpoint.pop(param)

        source_points = utility.split_point(tpoint)
        for i, source in enumerate(self.sources):
            utility.update_source(source, **source_points[i])
            # reset source time may result in store error otherwise
            source.time = 0.0

    def get_pyrocko_events(self, point=None):
        """
        Transform sources to pyrocko events.

        Returns
        -------
        events : list
            of :class:`pyrocko.model.Event`
        """

        if point is not None:
            self.point2sources(point)

        events = []
        target = self.targets[0]
        store = self.engine.get_store(target.store_id)
        for source in self.sources:
            events.append(source.pyrocko_event(store=store, target=target))

        return events

    def get_formula(self, input_rvs, fixed_rvs, hyperparams, problem_config):
        """
        Get geodetic likelihood formula for the model built. Has to be called
        within a with model context.
        Part of the pymc3 model.

        Parameters
        ----------
        input_rvs : dict
            of :class:`pymc3.distribution.Distribution`
        fixed_rvs : dict
            of :class:`numpy.array`
        hyperparams : dict
            of :class:`pymc3.distribution.Distribution`
        problem_config : :class:`config.ProblemConfig`

        Returns
        -------
        posterior_llk : :class:`theano.tensor.Tensor`
        """
        hp_specific = self.config.dataset_specific_residual_noise_estimation
        tpoint = problem_config.get_test_point()

        self.input_rvs = input_rvs
        self.fixed_rvs = fixed_rvs

        logger.info(
            "Geodetic optimization on: \n " "%s" % ", ".join(self.input_rvs.keys())
        )

        self.input_rvs.update(fixed_rvs)

        t0 = time()
        disp = self.get_synths(self.input_rvs)
        t1 = time()
        logger.debug("Geodetic forward model on test model takes: %f" % (t1 - t0))

        los_disp = (disp * self.slos_vectors).sum(axis=1)

        residuals = self.Bij.srmap(
            tt.cast((self.sdata - los_disp) * self.sodws, tconfig.floatX)
        )

        self.init_hierarchicals(problem_config)
        self.analyse_noise(tpoint)
        self.init_weights()
        if self.config.corrections_config.has_enabled_corrections:
            logger.info("Applying corrections! ...")
            residuals = self.apply_corrections(residuals, operation="-")

        logpts = multivariate_normal_chol(
            self.datasets, self.weights, hyperparams, residuals, hp_specific=hp_specific
        )

        llk = Deterministic(self._like_name, logpts)
        return llk.sum()

class GeodeticGeometryComposite(GeodeticSourceComposite):
    def __init__(self, gc, project_dir, sources, events, hypers=False):

        super(GeodeticGeometryComposite, self).__init__(
            gc, project_dir, sources, events, hypers=hypers
        )

        if not hypers:
            # synthetics generation
            logger.debug("Initialising synthetics functions ... \n")
            self.get_synths = theanof.GeoSynthesizer(
                engine=self.engine, sources=self.sources, targets=self.targets
            )

    def get_synthetics(self, point, **kwargs):
        """
        Get synthetics for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters
        kwargs especially to change output of the forward model

        Returns
        -------
        list with :class:`numpy.ndarray` synthetics for each target
        """
        self.point2sources(point)

        displacements = heart.geo_synthetics(
            engine=self.engine, targets=self.targets, sources=self.sources, **kwargs
        )

        synths = []
        for disp, data in zip(displacements, self.datasets):
            los_d = (disp * data.los_vector).sum(axis=1)
            synths.append(los_d)

        if self.config.corrections_config.has_enabled_corrections:
            synths = self.apply_corrections(synths, point=point, operation="+")

        return synths

    def update_weights(self, point, n_jobs=1, plot=False):
        """
        Updates weighting matrixes (in place) with respect to the point in the
        solution space.

        Parameters
        ----------
        point : dict
            with numpy array-like items and variable name keys
        """
        gc = self.config

        if not self.weights:
            self.init_weights()

        self.point2sources(point)

        # update data covariances in case model dependent non-toeplitz
        if self.config.noise_estimator.structure == "non-toeplitz":
            logger.info("Updating data-covariances ...")
            self.analyse_noise(point)

        crust_inds = range(*gc.gf_config.n_variations)
        thresh = 5
        if len(crust_inds) > thresh:
            logger.info("Updating geodetic velocity model-covariances ...")
            if self.config.noise_estimator.structure == "non-toeplitz":
                logger.warning(
                    "Non-toeplitz estimation in combination with model "
                    "prediction covariances is still EXPERIMENTAL and results"
                    " should be interpreted with care!!"
                )

            for i, data in enumerate(self.datasets):
                crust_targets = heart.init_geodetic_targets(
                    datasets=[data],
                    earth_model_name=gc.gf_config.earth_model_name,
                    interpolation=gc.interpolation,
                    crust_inds=crust_inds,
                    sample_rate=gc.gf_config.sample_rate,
                )

                logger.debug("Track %s" % data.name)
                cov_pv = cov.geodetic_cov_velocity_models(
                    engine=self.engine,
                    sources=self.sources,
                    targets=crust_targets,
                    dataset=data,
                    plot=plot,
                    event=self.event,
                    n_jobs=1,
                )

                cov_pv = utility.ensure_cov_psd(cov_pv)
                data.covariance.pred_v = cov_pv
                choli = data.covariance.chol_inverse

                self.weights[i].set_value(choli)
                data.covariance.update_slog_pdet()
        else:
            logger.info(
                "Not updating geodetic velocity model-covariances because "
                "number of model variations is too low! < %i" % thresh
            )

class GeodeticInterseismicComposite(GeodeticSourceComposite):
    def __init__(self, gc, project_dir, sources, events, hypers=False):

        super(GeodeticInterseismicComposite, self).__init__(
            gc, project_dir, sources, events, hypers=hypers
        )

        for source in sources:
            if not isinstance(source, RectangularSource):
                raise TypeError("Sources have to be RectangularSources!")

        if not hypers:
            self._lats = self.Bij.l2a([data.lats for data in self.datasets])
            self._lons = self.Bij.l2a([data.lons for data in self.datasets])

            self.get_synths = theanof.GeoInterseismicSynthesizer(
                lats=self._lats,
                lons=self._lons,
                engine=self.engine,
                targets=self.targets,
                sources=sources,
                reference=self.event,
            )

    def get_synthetics(self, point, **kwargs):
        """
        Get synthetics for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters
        kwargs especially to change output of the forward model

        Returns
        -------
        list with :class:`numpy.ndarray` synthetics for each target
        """
        tpoint = copy.deepcopy(point)
        tpoint.update(self.fixed_rvs)
        spoint, bpoint = seperate_point(tpoint)

        self.point2sources(spoint)

        synths = []
        for target, data in zip(self.targets, self.datasets):
            disp = geo_backslip_synthetics(
                engine=self.engine,
                sources=self.sources,
                targets=[target],
                lons=target.lons,
                lats=target.lats,
                reference=self.event,
                **bpoint
            )
            synths.append((disp * data.los_vector).sum(axis=1))

        return synths

    def update_weights(self, point, n_jobs=1, plot=False):

        if not self.weights:
            self.init_weights()

        logger.warning("Not implemented yet!")
        raise NotImplementedError("Not implemented yet!")

class GeodeticDistributerComposite(GeodeticComposite):
    """
    Comprises how to solve the geodetic (static) linear forward model.
    Distributed slip
    """

    def __init__(self, gc, project_dir, events, hypers=False):

        super(GeodeticDistributerComposite, self).__init__(
            gc, project_dir, events, hypers=hypers
        )

        self.gfs = {}
        self.gf_names = {}

        self._mode = "ffi"
        self.gfpath = os.path.join(project_dir, self._mode, bconfig.linear_gf_dir_name)

        self.fault = None

    def get_gflibrary_key(self, crust_ind, wavename, component):
        return "%i_%s_%s" % (crust_ind, wavename, component)

    def load_gfs(self, crust_inds=None, make_shared=True):
        """
        Load Greens Function matrixes for each variable to be inverted for.
        Updates gfs and gf_names attributes.

        Parameters
        ----------
        crust_inds : list
            of int to indexes of Green's Functions
        make_shared : bool
            if True transforms gfs to :class:`theano.shared` variables
        """

        if crust_inds is None:
            crust_inds = range(*self.config.gf_config.n_variations)

        if not isinstance(crust_inds, list):
            raise TypeError("crust_inds need to be a list!")

        for crust_ind in crust_inds:
            gfs = {}
            for var in self.slip_varnames:
                gflib_name = get_gf_prefix(
                    datatype=self.name,
                    component=var,
                    wavename="static",
                    crust_ind=crust_ind,
                )
                gfpath = os.path.join(self.gfpath, gflib_name)

                gfs = load_gf_library(directory=self.gfpath, filename=gflib_name)

                if make_shared:
                    gfs.init_optimization()

                key = self.get_gflibrary_key(
                    crust_ind=crust_ind, wavename="static", component=var
                )

                self.gf_names[key] = gfpath
                self.gfs[key] = gfs

    def load_fault_geometry(self):
        """
        Load fault-geometry, i.e. discretized patches.

        Returns
        -------
        :class:`heart.FaultGeometry`
        """
        try:
            return utility.load_objects(
                os.path.join(self.gfpath, bconfig.fault_geometry_name)
            )[0]
        except Exception:
            raise FaultGeometryNotFoundError()

    def point2sources(self, point):
        """
        Returns the fault source patche(s) with the point values updated.

        Parameters
        ----------
        point : dict
            with random variables from solution space
        """
        tpoint = copy.deepcopy(point)
        # tpoint.update(self.fixed_rvs)   if vars are fixed GFS are not calculated

        if self.nevents == 1:
            events = [self.event]  # single event
        else:
            events = self.events  # multi event

        if self.fault is None:
            self.fault = self.load_fault_geometry()

        return self.fault.point2sources(tpoint, events=events)

    def get_formula(self, input_rvs, fixed_rvs, hyperparams, problem_config):
        """
        Formulation of the distribution problem for the model built. Has to be
        called within a with-model-context.

        Parameters
        ----------
        input_rvs : list
            of :class:`pymc3.distribution.Distribution`
        hyperparams : dict
            of :class:`pymc3.distribution.Distribution`

        Returns
        -------
        llk : :class:`theano.tensor.Tensor`
            log-likelihood for the distributed slip
        """
        logger.info("Loading %s Green's Functions" % self.name)
        self.load_gfs(
            crust_inds=[self.config.gf_config.reference_model_idx], make_shared=False
        )

        tpoint = problem_config.get_test_point()
        self.analyse_noise(tpoint)
        for gfs in self.gfs.values():
            gfs.init_optimization()

        self.init_weights()

        hp_specific = self.config.dataset_specific_residual_noise_estimation

        self.input_rvs = input_rvs
        self.fixed_rvs = fixed_rvs
        ref_idx = self.config.gf_config.reference_model_idx

        mu = tt.zeros((self.Bij.ordering.size), tconfig.floatX)
        for var in self.slip_varnames:
            key = self.get_gflibrary_key(
                crust_ind=ref_idx, wavename="static", component=var
            )
            mu += self.gfs[key].stack_all(slips=input_rvs[var])

        residuals = self.Bij.srmap(
            tt.cast((self.sdata - mu) * self.sodws, tconfig.floatX)
        )

        self.init_hierarchicals(problem_config)
        if self.config.corrections_config.has_enabled_corrections:
            residuals = self.apply_corrections(residuals)

        logpts = multivariate_normal_chol(
            self.datasets, self.weights, hyperparams, residuals, hp_specific=hp_specific
        )

        llk = Deterministic(self._like_name, logpts)

        return llk.sum()

    def get_synthetics(self, point, outmode="data"):
        """
        Get synthetics for given point in solution space.

        Parameters
        ----------
        point : :func:`pymc3.Point`
            Dictionary with model parameters
        kwargs especially to change output of the forward model

        Returns
        -------
        list with :class:`numpy.ndarray` synthetics for each target
        """

        ref_idx = self.config.gf_config.reference_model_idx
        if len(self.gfs) == 0:
            self.load_gfs(crust_inds=[ref_idx], make_shared=False)

        for gfs in self.gfs.values():
            gfs.set_stack_mode("numpy")

        tpoint = copy.deepcopy(point)

        hps = self.config.get_hypernames()

        for hyper in hps:
            if hyper in tpoint:
                tpoint.pop(hyper)

        mu = num.zeros((self.Bij.ordering.size))
        for var in self.slip_varnames:
            key = self.get_gflibrary_key(
                crust_ind=ref_idx, wavename="static", component=var
            )
            mu += self.gfs[key].stack_all(slips=point[var])

        synths = self.Bij.a2l(mu)

        if self.config.corrections_config.has_enabled_corrections:
            synths = self.apply_corrections(synths, point=point, operation="+")

        return synths

    def update_weights(self, point, n_jobs=1, plot=False):
        """
        Updates weighting matrixes (in place) with respect to the point in the
        solution space.

        Parameters
        ----------
        point : dict
            with numpy array-like items and variable name keys
        """
        gc = self.config

        if not self.weights:
            self.init_weights()

        # update data covariances in case model dependent non-toeplitz
        if self.config.noise_estimator.structure == "non-toeplitz":
            logger.info("Updating data-covariances ...")
            self.analyse_noise(point)

        crust_inds = range(*gc.gf_config.n_variations)
        thresh = 5
        if len(crust_inds) > thresh:
            logger.info("Updating geodetic velocity model-covariances ...")
            if self.config.noise_estimator.structure == "non-toeplitz":
                logger.warning(
                    "Non-toeplitz estimation in combination with model "
                    "prediction covariances is still EXPERIMENTAL and results"
                    " should be interpreted with care!!"
                )

            crust_inds = list(range(*self.config.gf_config.n_variations))
            n_variations = len(crust_inds)
            if len(self.gfs) != n_variations:
                logger.info("Loading geodetic linear GF matrixes ...")
                self.load_gfs(crust_inds=crust_inds, make_shared=False)

            crust_displacements = num.zeros((n_variations, self.Bij.ordering.size))
            for i, crust_ind in enumerate(crust_inds):
                mu = num.zeros((self.Bij.ordering.size))
                for var in self.slip_varnames:
                    key = self.get_gflibrary_key(
                        crust_ind=crust_ind, wavename="static", component=var
                    )
                    mu += self.gfs[key].stack_all(slips=point[var])

                crust_displacements[i, :] = mu

            crust_synths = self.Bij.a_nd2l(crust_displacements)
            if len(crust_synths) != self.n_t:
                raise ValueError(
                    "Number of datasets %i and number of synthetics %i "
                    "inconsistent!" % (self.n_t, len(crust_synths))
                )

            for i, data in enumerate(self.datasets):
                logger.debug("Track %s" % data.name)
                cov_pv = num.cov(crust_synths[i], rowvar=0)

                cov_pv = utility.ensure_cov_psd(cov_pv)
                data.covariance.pred_v = cov_pv
                choli = data.covariance.chol_inverse

                self.weights[i].set_value(choli)
                data.covariance.update_slog_pdet()
        else:
            logger.info(
                "Not updating geodetic velocity model-covariances because "
                "number of model variations is too low! < %i" % thresh
            )


"""Python process Django Models."""
import json
import os
from pathlib import Path
from typing import Any, Dict, List, MutableMapping, Optional, Type

from .communicator import communicator
from .fields import (
    BooleanField,
    DateTimeField,
    Field,
    IntegerField,
    JsonField,
    ListField,
    RelationDescriptor,
    StringField,
    fields_from_schema,
)

# Get the user model label. First try the standard Django way.
# If it fails (no Django installed) try to get it from the listener.
try:
    from django.contrib.auth import get_user_model

    USER_MODEL_LABEL = get_user_model()._meta.label
except ModuleNotFoundError:
    USER_MODEL_LABEL = communicator.get_user_model_label()

DATA_ID = int(os.getenv("DATA_ID", "-1"))
USER_MODEL_APP, USER_MODEL_NAME = USER_MODEL_LABEL.split(".")

class ModelField(Field):
    """Field representing a model.

    ForeignKey relation is modeled with this field.
    """

    field_type = "model"

    def __init__(self, full_model_name: str, *args, **kwargs):
        """Initialize."""
        super().__init__(*args, **kwargs)
        self.full_model_name = full_model_name
        self.app_name, self.related_model_name = full_model_name.split(".")
        self.label = full_model_name
        self.known_models = RegisteredModels()

    def to_python(self, value):
        """Return the python object representing the field."""
        if isinstance(value, Model):
            return value
        if isinstance(value, int):
            model_class = self.known_models.get_model(self.full_model_name)
            return model_class(value)
        return super().to_python(value)

    def to_output(self, value):
        """Convert the value to output format."""
        if isinstance(value, int):
            return value
        if isinstance(value, Model):
            return value.id
        if self.related_model_name == "Storage" and isinstance(value, str):
            return value
        return super().to_output(value)

FIELDS_MAP = {
    "BigAutoField": IntegerField,
    "AutoField": IntegerField,
    "VersionField": StringField,
    "CharField": StringField,
    "TextField": StringField,
    "DateTimeField": DateTimeField,
    "ForeignKey": ModelField,
    "ManyToManyField": ListField,
    "PositiveIntegerField": IntegerField,
    "PositiveSmallIntegerField": IntegerField,
    "ArrayField": Field,
    "JSONField": JsonField,
    "BigIntegerField": IntegerField,
    "BooleanField": BooleanField,
    "SlugField": StringField,
}

_hydrate_cache: Dict[int, str] = dict()

def hydrate_if_needed(value, model_instance, field_name, field):
    """Hydrate path if needed."""
    # Only hidrate paths on outputs of data objects that we are not processing.
    if (
        model_instance._model_name == "Data"
        and model_instance.id != DATA_ID
        and field_name == "output"
        and field.get_field_type()
        in ("basic:file", "list:basic:file", "basic:dir", "list:basic:dir")
    ):
        if model_instance.id not in _hydrate_cache:
            _hydrate_cache[model_instance.id] = Path(
                communicator.resolve_data_path(model_instance.id)
            )
        base_path = _hydrate_cache[model_instance.id] / str(model_instance.location_id)
        if field.get_field_type().startswith("basic"):
            value.path = os.fspath(base_path / value.path)
        else:
            for entry in value:
                entry.path = os.fspath(base_path / entry.path)
    return value

class JSONDescriptor(MutableMapping[str, Any]):
    """JSON field."""

    def __init__(
        self,
        model: "Model",
        field_name: str,
        field_schema: List[Dict],
        read_only=False,
        cache: Optional[Dict[str, Any]] = None,
    ):
        """Initialize."""
        self._model = model
        self._model_name = self._model._model_name
        self._app_name = self._model._app_name
        self._field_name = field_name
        self._pk = self._model._pk
        self._cache: Dict[str, Any] = dict()
        self._read_only = read_only
        self._fields: Dict[str, Field] = dict()
        for field_name, field in fields_from_schema(field_schema).items():
            # JSON fields in schema are special since they represent the
            # storage model.
            if field.get_field_type() == "basic:json":
                field = ModelField(full_model_name="flow.Storage")
            field.contribute_to_class(self, self._fields, field_name)
        # Create cache if initial data is given.
        if cache is not None:
            self.refresh_cache(cache)

    def freeze(self):
        """Freeze fields."""
        self._read_only = True
        return self

    def refresh_cache(self, json_data: Optional[dict] = None):
        """Get the entire JSON field content from the server.

        TODO: read only required parts when Django supports it.
        """
        json_data = (
            json_data
            or communicator.get_model_fields(
                self._app_name, self._model_name, self._pk, [self._field_name]
            )[self._field_name]
        )
        for field_name, field in self._fields.items():
            if field_name in json_data:
                self._cache[field_name] = hydrate_if_needed(
                    field.clean(json_data[field_name]),
                    self._model,
                    self._field_name,
                    field,
                )

    def __getattr__(self, key):
        """Allow dot syntax."""
        return self.__getitem__(key)

    def __setattr__(self, key, value):
        """Set the value for the given key."""
        if key.startswith("_"):
            super().__setattr__(key, value)
        else:
            self.__setitem__(key, value)

    def __getitem__(self, key):
        """Get the value for the given key."""
        if self._fields is not None and key not in self._fields:
            raise AttributeError(f"Value {key} must be in the fields")
        if key not in self._cache:
            self.refresh_cache()
        # Return None even if key is not known.
        return self._cache.get(key)

    def __setitem__(self, key, value):
        """Set the value for the given key."""
        if self._read_only:
            raise ValueError("Cannot change read-only mapping.")
        if key not in self._fields:
            raise AttributeError(f"No field named {key}")
        to_output = None
        field = self._fields[key]
        if getattr(field, "related_model_name", None) == "Storage":
            # Check if JSON must be read from the file.
            if isinstance(value, str):
                json_file = Path(value)
                if json_file.is_file():
                    value = json.loads(json_file.read_text())
            storage = getattr(self, key)
            if storage is None:
                storage = Storage.create(
                    json=value,
                    name="Storage for data id {}".format(self._pk),
                    contributor=self._model.contributor,
                )
                storage.data += [self._model]
                value = storage
                to_output = storage.id
            else:
                storage.json = value
        else:
            value = field.clean(value)
            to_output = field.to_output(value)

        if to_output is not None:
            communicator.update_model_fields(
                self._app_name,
                self._model_name,
                self._pk,
                {self._field_name: {key: to_output}},
            )
        self._cache[key] = value

    def __len__(self) -> int:
        """Return the number of known fields."""
        if self._fields is not None:
            return len(self._fields)
        else:
            return len(self._cache)

    def __delitem__(self, name):
        """Delete the item from JSON field."""
        raise NotImplementedError("Delete is not implemented")

    def __iter__(self):
        """Return iterator."""
        return iter(self._cache)

    def __str__(self):
        """Return string representation."""
        return f"JSONDescriptor({self._model}, {self._field_name})"

class ModelMetaclass(type):
    """Construct fields for the Django class."""

    def __new__(mcs, name, bases, namespace, **kwargs):
        """Create new Model class."""
        model: Type[Model] = type.__new__(mcs, name, bases, namespace)
        fields: Dict[str, Field] = {}
        if name != "Model" and communicator is not None:
            app_name = model._app_name
            fields_details = communicator.get_model_fields_details(app_name, name)
            for field_name in fields_details:
                field_type, required, related_model_name = fields_details[field_name]
                if field_type in FIELDS_MAP:
                    kwargs = {"required": required}
                    if field_type == "ForeignKey":
                        id_field = IntegerField()
                        id_field.contribute_to_class(model, fields, f"{field_name}_id")
                        setattr(model, f"{field_name}_id", id_field)
                        kwargs["full_model_name"] = related_model_name
                    if field_type == "ManyToManyField":
                        kwargs["inner"] = ModelField(related_model_name)
                    field_class = FIELDS_MAP[field_type]
                    field = field_class(**kwargs)
                    field.contribute_to_class(model, fields, field_name)
                    setattr(model, field_name, field)

            model.fields = fields
        return model

class JSONModelEncoder(json.JSONEncoder):
    """Encode Model to JSON.

    When instance of Model class is serialized to JSON it should be serialized
    to its id property.
    """

    def default(self, o: Any):
        """Override default method."""
        if isinstance(o, Model):
            return o.id
        return json.JSONEncoder.default(self, o)

class Model(metaclass=ModelMetaclass):
    """Base django model."""

    _app_name = "App name"
    _model_name = "Model name"
    # A default list of extra fields that are returned on filter call.
    _filter_response_fields = []

    def __init__(self, pk: int):
        """Initialize."""
        self._pk = pk
        self._cache: Dict[str, Any] = {"id": pk}

    @property
    def full_model_name(self):
        """Return the full model name."""
        return f"{self._app_name}.{self._model_name}"

    @classmethod
    def filter(cls, **filters: Dict[str, Any]) -> List["Model"]:
        """Return a list of all objects that fit criteria."""
        # Make sure attributes have 'id' in the first place.
        attributes = filters.pop("__fields", None)
        attributes = attributes or cls._filter_response_fields
        attributes = ["id"] + [
            attribute for attribute in attributes if attribute != "id"
        ]

        objects = communicator.filter_objects(
            cls._app_name, cls._model_name, filters, attributes
        )
        models = []
        for entry in objects:
            model = cls(entry[0])
            for field_name, value in zip(attributes[1:], entry[1:]):
                field = model.fields[field_name]
                model._cache[field_name] = field.clean(value)
            models.append(model)
        return models

    @classmethod
    def exists(cls, **filters: Dict[str, Any]) -> List[int]:
        """Check if objects that fit criteria exists.

        If no such object exists empty list is returned.
        Else list of ids that fit the criteria is returned.
        """
        return [
            e[0]
            for e in communicator.filter_objects(
                cls._app_name, cls._model_name, filters, ["id"]
            )
        ]

    @classmethod
    def get(cls, **filters: Dict[str, Any]) -> "Model":
        """Get a single model based on filters.

        :raises RuntimeError: when different than one objects match the given
            criteria.
        """
        pks = communicator.filter_objects(
            cls._app_name, cls._model_name, filters, ["id"]
        )
        if len(pks) > 1:
            raise RuntimeError(
                f"Exactly one object should match the given criteria, received {len(pks)}."
            )
        elif len(pks) == 0:
            raise RuntimeError(
                "No objects match the given criteria or no permission to read object."
            )
        return cls(pks[0][0])

    @classmethod
    def create(cls, **object_data: Dict[str, Any]) -> int:
        """Create object with the given data.

        If creation was successfull return its id.
        """
        mappings = []
        for field_name in object_data:
            field = cls.fields.get(field_name)
            if field is not None and field.get_field_type() == "model":
                mappings.append(
                    (field_name, f"{field_name}_id", object_data[field_name].id)
                )
        for old_name, new_name, new_value in mappings:
            del object_data[old_name]
            object_data[new_name] = new_value
        communicator.encoder = JSONModelEncoder
        return cls(
            communicator.create_object(cls._app_name, cls._model_name, object_data)
        )

    def __str__(self):
        """Return a string representation."""
        return f"{self._model_name}(pk={self._pk})"

    def _set_field_data(self, field: Field, value: Any):
        """Set the value of the field."""
        self._cache[field.name] = value
        communicator.update_model_fields(
            self._app_name,
            self._model_name,
            self._pk,
            {field.name: field.to_output(value)},
        )

    def _get_field_data(self, field: Field) -> Any:
        """Get data for the given field.

        It is used by the fields (descriptors) to retrieve the data from the
        server. The cache is used if applicable.

        :raises RuntimeError: when no data is received.
        """
        if field.name not in self._cache:
            result = communicator.get_model_fields(
                self._app_name, self._model_name, self._pk, [field.name]
            )
            if len(result) > 1:
                result = [e[field.name] for e in result]
            elif len(result) == 1:
                result = result[field.name]
            else:
                raise RuntimeError(
                    (
                        f"No data received for property '{field.name}' of "
                        f"model {self._model_name} with id {self._pk}. Check "
                        "property name and permissions."
                    )
                )
            self._cache[field.name] = field.clean(result)
        return self._cache[field.name]

    @classmethod
    def __init_subclass__(cls: Type["Model"], **kwargs):
        """Register class."""
        super().__init_subclass__(**kwargs)
        RegisteredModels().register_model(cls)

class RegisteredModels:
    """Registered Python process models."""

    #  A single instance of this class.
    __instance = None
    _known_models: Dict[str, Type["Model"]] = dict()

    @classmethod
    def __new__(cls, *args, **kwargs):
        """Create or return singleton."""
        if RegisteredModels.__instance is None:
            RegisteredModels.__instance = super().__new__(cls)

        return RegisteredModels.__instance

    def get_model(self, full_model_name: str) -> Type["Model"]:
        """Get the registered model."""
        return self._known_models.get(full_model_name, Model)

    def register_model(self, model_class: Type["Model"]):
        """Add new model class.

        :raises AssertionError: when model is already registered.
        """
        full_model_name = f"{model_class._app_name}.{model_class._model_name}"
        assert (
            full_model_name not in self._known_models
        ), f"Model named {full_model_name} already registered."
        self._known_models[full_model_name] = model_class

class Process(Model):
    """Process model."""

    _app_name = "flow"
    _model_name = "Process"

    @classmethod
    def get_latest(cls, slug: str) -> "Model":
        """Get a latest version of the Process with the given slug.

        :raises RuntimeError: when no object can be retrieved.
        """
        pks = communicator.filter_objects(
            cls._app_name, cls._model_name, {"slug": slug}, ["-version"], ["id"]
        )
        if len(pks) == 0:
            raise RuntimeError(
                "No objects match the given criteria or no permission to read object."
            )
        else:
            return cls(pks[0][0])

class Entity(Model):
    """Entity model."""

    _app_name = "flow"
    _model_name = "Entity"

class Storage(Model):
    """Storage model."""

    _app_name = "flow"
    _model_name = "Storage"

class User(Model):
    """Django user model."""

    _app_name = USER_MODEL_APP
    _model_name = USER_MODEL_NAME

class DescriptorSchema(Model):
    """User model."""

    _app_name = "flow"
    _model_name = "DescriptorSchema"

    @classmethod
    def get_latest(cls, slug: str) -> "Model":
        """Get a latest version of the DescriptorSchema with the given slug.

        :raises RuntimeError: when no object can be retrieved.
        """
        pks = communicator.filter_objects(
            cls._app_name, cls._model_name, {"slug": slug}, ["-version"], ["id"]
        )
        if len(pks) == 0:
            raise RuntimeError(
                "No objects match the given criteria or no permission to read object."
            )
        else:
            return cls(pks[0][0])

class Collection(Model):
    """Collection model."""

    _app_name = "flow"
    _model_name = "Collection"

    @property
    def relations(self):
        """Get relation."""
        if "__relations" not in self._cache:
            self._cache["__relations"] = communicator.get_relations(self.id)
        return self._cache["__relations"]

class Data(Model):
    """Data object."""

    _app_name = "flow"
    _model_name = "Data"

    @classmethod
    def from_slug(cls, slug: str) -> "Data":
        """Get Data object from slug."""
        return Data(communicator.get_data_by_slug(slug))

    @property
    def type(self) -> str:
        """Get the process type for this data object.

        DEPRECATED: compatibility with old DataDescriptor class.
        """
        return self.process.type

    @property
    def entity_name(self):
        """Entity name.

        Causes error when entity is not defined.

        DEPRECATED: compatibility with old DataDescriptor class.
        """
        return self.entity.name

    @property
    def relations(self):
        """Get relations for this data object in its collection."""
        relations = set()
        for relation in self.collection.relations:
            for partition in relation["partitions"]:
                if partition["entity_id"] == self.entity.id:
                    relations.add(RelationDescriptor.from_dict(relation))
        return list(relations)


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from einops import rearrange, repeat
from torch import einsum

def compute_imitation_loss(input, target, weights):
    target = torch.where(torch.isnan(target), input, target)  # ignore nan targets
    diff = input - target
    loss = 0.5 * diff ** 2

    assert weights.shape == loss.shape[:-1]
    weights = weights.unsqueeze(-1)
    assert len(loss.shape) == len(weights.shape)
    loss = loss * weights
    return loss

def compute_backbone_l1_loss(features_preds, features_targets, target):
    feature_ditill_loss = 0.0
    if isinstance(features_preds, list):
        for i in range(len(features_preds)):
            downsample_ratio = 2 ** (i + 3)

            feature_target = features_targets[i].detach()
            feature_pred = features_preds[i]

            mask = calculate_box_mask(feature_pred, target, downsample_ratio)  # mask [B, H, W]

            feature_pred = feature_pred.permute(0, *range(2, len(feature_pred.shape)), 1)
            feature_target = feature_target.permute(0, *range(2, len(feature_target.shape)), 1)
            batch_size = int(feature_pred.shape[0])
            positives = feature_pred.new_ones(*feature_pred.shape[:3])
            positives = positives * torch.any(feature_target != 0, dim=-1).float()
            positives = positives * mask.cuda()

            reg_weights = positives.float()
            pos_normalizer = positives.sum().float()
            reg_weights /= pos_normalizer

            pos_inds = reg_weights > 0
            pos_feature_preds = feature_pred[pos_inds]
            pos_feature_targets = feature_target[pos_inds]

            imitation_loss_src = compute_imitation_loss(pos_feature_preds,
                                                          pos_feature_targets,
                                                          weights=reg_weights[pos_inds])  # [N, M]

            imitation_loss = imitation_loss_src.mean(-1)
            imitation_loss = imitation_loss.sum() / batch_size
            feature_ditill_loss = feature_ditill_loss + imitation_loss

    else:
        raise NotImplementedError

    return feature_ditill_loss

def compute_backbone_resize_affinity_loss(features_preds, features_targets):
    feature_ditill_loss = 0.0
    resize_shape = features_preds[-1].shape[-2:]
    if isinstance(features_preds, list):
        for i in range(len(features_preds)):  # 1/8   1/16   1/32
            feature_target = features_targets[i].detach()
            feature_pred = features_preds[i]

            B, C, H, W = feature_pred.shape
            feature_pred_down = F.interpolate(feature_pred, size=resize_shape, mode="bilinear")
            feature_target_down = F.interpolate(feature_target, size=resize_shape, mode="bilinear")

            feature_target_down = feature_target_down.reshape(B, C, -1)
            depth_affinity = torch.bmm(feature_target_down.permute(0, 2, 1), feature_target_down)

            feature_pred_down = feature_pred_down.reshape(B, C, -1)
            rgb_affinity = torch.bmm(feature_pred_down.permute(0, 2, 1), feature_pred_down)

            feature_ditill_loss = feature_ditill_loss + F.l1_loss(rgb_affinity, depth_affinity, reduction='mean') / B

    else:
        raise NotImplementedError

    return feature_ditill_loss

def compute_backbone_local_affinity_loss(features_preds, features_targets):
    feature_ditill_loss = 0.0
    local_shape = features_preds[-1].shape[-2:]
    if isinstance(features_preds, list):
        for i in range(len(features_preds)):  # 1/8   1/16   1/32
            feature_target = features_targets[i].detach()
            feature_pred = features_preds[i]

            B, _, H, W = feature_pred.shape
            feature_pred_q = rearrange(feature_pred, 'b c (h p1) (w p2) -> (b h w) (p1 p2) c', p1=local_shape[0], p2=local_shape[1])
            feature_pred_k = rearrange(feature_pred, 'b c (h p1) (w p2) -> (b h w) (p1 p2) c', p1=local_shape[0], p2=local_shape[1])

            rgb_affinity = einsum('b i d, b j d -> b i j', feature_pred_q, feature_pred_k)

            feature_target_q = rearrange(feature_target, 'b c (h p1) (w p2) -> (b h w) (p1 p2) c', p1=local_shape[0], p2=local_shape[1])
            feature_target_k = rearrange(feature_target, 'b c (h p1) (w p2) -> (b h w) (p1 p2) c', p1=local_shape[0], p2=local_shape[1])

            depth_affinity = einsum('b i d, b j d -> b i j', feature_target_q, feature_target_k)

            feature_ditill_loss = feature_ditill_loss + F.l1_loss(rgb_affinity, depth_affinity, reduction='mean') / B

    else:
        raise NotImplementedError

    return feature_ditill_loss

def calculate_box_mask(features_preds, target, downsample_ratio):
    B, C, H, W = features_preds.shape
    gt_mask = torch.zeros((B, H, W))

    for i in range(B):
        for j in range(target['obj_num'][i]):
            bbox2d_gt = target['box2d_gt'][i, j] / downsample_ratio

            left_top = bbox2d_gt[:2]
            right_bottom = bbox2d_gt[2:]

            left_top_x = int(left_top[0].item())
            left_top_y = int(left_top[1].item())

            right_bottom_x = int(right_bottom[0].item())
            right_bottom_y = int(right_bottom[1].item())

            gt_mask[i, left_top_y:right_bottom_y, left_top_x:right_bottom_x] = 1
    return gt_mask


from PIL import Image
import numpy as np
from scipy.ndimage import filters
import matplotlib.pyplot as plt

# Not complete yet

##Apply the label() function to a thresholded image of your choice. Use histograms
##and the resulting label image to plot the distribution of object sizes in the image.

def label(image = None,threshold = None):
    # get all index with pixel > than threshold
    idx = image[:,:] < threshold
    image[idx] *= 0
    return image
infile = '../Images/baboon.png'
im = np.array(Image.open(infile).convert('L'))
(nr,nc) = im.shape
(dx,dy) = np.gradient(np.double(im))
orientation = np.arctan(np.divide(dx,dy))
(x,y) = np.meshgrid(im.shape[0],im.shape[1])

orientationThreshold = label(orientation, 1)
u = dx
v = dy
plt.figure()
plt.imshow(orientationThreshold)
plt.title('Out')
##plt.quiver(x,y,u,v)
plt.show()

# Here

CHAPTER1-Basic Image Handling and Processing/ex6.py


import spin_scale as m1
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['figure.facecolor'] = '0.2'

f = lambda scalex, scaley: m1.transformation(m1.scaleM(250, 250, scalex, scaley))
f2 = lambda mx, my: m1.transformation(m1.translationM(mx, my))
f3 = lambda angle: m1.transformation(m1.rotateM(200,200, angle))
f4 = lambda angle: m1.transformation(m1.rotateM(250, 250, angle))

C = np.full((500,500), 0, 'int')
rect = [(40,40), (260,40), (260, 260)]
#hex = [(20, 80), (30, 70), (50, 70), (60, 80), (50, 90), (30, 90)]
#hex2 = m1.hexagon(C, *list(map(f3(np.pi), hex)))
#print(hex2)

#m1.rectangle(C, *rect)

# plt.matshow(C, origin='lower', cmap='gray')
# plt.show()

def recImg(mx, my, angle, scalex = .5, scaley = .5):
    C = np.full((500,500), 0, 'int')
    #m1.rectangle(C, *list(map(f(scale, scale), rect)))
    #m1.rectangle(C, *list(map(f2(mx,my), rect)))
    #m1.rectangle(C, *list(map(f3(angle), rect)))
    m1.rectangle(C, *list(map(f(scalex, scaley), map(f4(angle), rect))))
    #m1.hexagon(C, *list(map(f3(np.pi/2), map(f2(mx,my), hex))))
    return C.copy()

steps = 100
images = [recImg(i,20,np.pi *i /steps) for i in range(steps)]

from PIL import Image

image_sequence = []

for frame in images:
    image = Image.fromarray(frame)
    image_sequence.append(image)

image_sequence[0].save('animation3.gif', save_all=True, append_images=image_sequence[1:], duration=20, loop=0)