import os.path as osp
import torch
from torch import nn
import torch.nn.functional as F
from torch_geometric.nn import GCNConv
from examples.core.base_model import BaseModel

class GCNConvLinkPredNet(BaseModel):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.save_hyperparameters()

        assert kwargs["num_layers"] >= 2

        self.convs = nn.ModuleList()

        self.convs.append(
            GCNConv(
                kwargs["num_features"],
                kwargs["hidden_channels"],
                cached=kwargs["cached"],
                normalize=not kwargs["use_gdc"],
            )
        )

        for idx in range(kwargs["num_layers"] - 2):
            self.convs.append(
                GCNConv(
                    kwargs["hidden_channels"],
                    kwargs["hidden_channels"],
                    cached=kwargs["cached"],
                    normalize=not kwargs["use_gdc"],
                )
            )

        self.convs.append(
            GCNConv(
                kwargs["hidden_channels"],
                kwargs["embedding_dim"],
                cached=kwargs["cached"],
                normalize=not kwargs["use_gdc"],
            )
        )

    def forward(self, batch):
        x = batch.x.squeeze()
        for idx, conv in enumerate(self.convs):
            x = F.relu(conv(x, batch.pos_edge_index.squeeze(), batch.pos_edge_attr))
        total_edge_index = torch.cat([batch.pos_edge_index.squeeze(), batch.neg_edge_index.squeeze()], dim=-1)
        x_j = torch.index_select(x, 0, total_edge_index[0])
        x_i = torch.index_select(x, 0, total_edge_index[1])
        return torch.einsum("ef, ef -> e", x_i, x_j), 0


import typing as t
import math

import torch
from torch import nn
from torch.nn import functional as F

__all__ = [
    'get_activation',
    'mlp_fn'
]

def get_activation(name, *args, **kwargs):
    """Get activation function by name"""
    name = name.lower()
    if name == 'relu':
        return nn.ReLU(*args, **kwargs)
    if name == 'elu':
        return nn.ELU(*args, **kwargs)
    if name == 'selu':
        return nn.SELU(*args, **kwargs)
    raise ValueError('Activation not implemented')

# def mlp_fn(
#     x: torch.Tensor,
#     cond: torch.Tensor,
#     mlp_func: t.Callable,
#     chunk_sizes: t.List[int] = [1, 2]
# ) -> t.Tuple[torch.Tensor]:
#     x = mlp_func(torch.cat(x, cond), dim=-1)
#     mu, var = torch.split(x, chunk_sizes, -1)
#     var = F.softplus(var) / math.log(2)
#     return mu, var

def mlp_fn(
    mlp_func: t.Callable,
    x: torch.Tensor,
    cond: torch.Tensor,
    seg_ids
    # # num_in_feats: int,
    # # num_mid_feats: int,
    # num_out_feats: int,
    # chunk_sizes: t.List[int]=[1, 2]
):
    def mlp(x: torch.Tensor, cond: torch.Tensor, seg_ids):
        x = torch.repeat_interleave(x, torch.tensor(seg_ids), dim=0)
        x_ = mlp_func(torch.cat((x, cond), dim=-1), seg_ids)
        mu, var = torch.split(x_, x_.size(-1) // 2, -1)
        var = F.softplus(var) / math.log(2)
        return mu, var
    return mlp


# 给出一个非抢占单线程CPU的 n 个函数运行日志，找到函数的独占时间。
#
#  每个函数都有一个唯一的 Id，从 0 到 n-1，函数可能会递归调用或者被其他函数调用。
#
#  日志是具有以下格式的字符串：function_id：start_or_end：timestamp。例如："0:start:0" 表示函数 0 从 0 时刻
# 开始运行。"0:end:0" 表示函数 0 在 0 时刻结束。
#
#  函数的独占时间定义是在该方法中花费的时间，调用其他函数花费的时间不算该函数的独占时间。你需要根据函数的 Id 有序地返回每个函数的独占时间。
#
#  示例 1:
#
#  输入:
# n = 2
# logs =
# ["0:start:0",
#  "1:start:2",
#  "1:end:5",
#  "0:end:6"]
# 输出:[3, 4]
# 说明：
# 函数 0 在时刻 0 开始，在执行了  2个时间单位结束于时刻 1。
# 现在函数 0 调用函数 1，函数 1 在时刻 2 开始，执行 4 个时间单位后结束于时刻 5。
# 函数 0 再次在时刻 6 开始执行，并在时刻 6 结束运行，从而执行了 1 个时间单位。
# 所以函数 0 总共的执行了 2 +1 =3 个时间单位，函数 1 总共执行了 4 个时间单位。
#
#
#  说明：
#
#
#  输入的日志会根据时间戳排序，而不是根据日志Id排序。
#  你的输出会根据函数Id排序，也就意味着你的输出数组中序号为 0 的元素相当于函数 0 的执行时间。
#  两个函数不会在同时开始或结束。
#  函数允许被递归调用，直到运行结束。
#  1 <= n <= 100
#
from typing import List

class Solution:
    def exclusiveTime(self, n: int, logs: List[str]) -> List[int]:
        result = []
        # 当前正在运行的函数id
        stack = []
        # 记录函数运行时间
        dict = {}
        last_time = 0
        for x in logs:
            log = x.split(":")
            if not stack:
                stack.append(log)
                dict.setdefault(log[0], 0)
                last_time = int(log[2])
                continue
            tmp = stack.pop()
            time = dict[tmp[0]]
            if log[1] != "end":
                dict[tmp[0]] = time + int(log[2]) - last_time
                dict.setdefault(log[0], 0)
                stack.append(tmp)
                stack.append(log)
                last_time = int(log[2])
            else:
                dict[tmp[0]] = time + int(log[2]) - last_time + 1
                last_time = int(log[2]) + 1

        for i in range(n):
            result.append(dict[str(i)])
        return result

s = Solution()
print(s.exclusiveTime(3,["0:start:0","0:end:0","1:start:1","1:end:1","2:start:2","2:end:2","2:start:3","2:end:3"]))
print(s.exclusiveTime(1,["0:start:0","0:start:2","0:end:5","0:start:6","0:end:6","0:end:7"]))

栈/函数的独占时间.py


# 1000 dahil 1 ile 1000 arasındaki sayiları ekrana yazdıran metod yazıdnız

def Say():
    for i in range(1000,0,-1):
        print(i)
Say()

# ya da

def Saydirici():
    i = 1000
    while (i>=1):
        print(i)
        i -= 1
Saydirici()


import torch
#from distributions import Normal
from torch.autograd import Variable

class NoiseLoss(torch.nn.Module):
  # need the scale for noise standard deviation
  # scale = noise  std
  def __init__(self, params, scale=None, observed=None):
    super(NoiseLoss, self).__init__()
    # initialize the distribution for each parameter
    #self.distributions = []
    self.noises = []
    for param in params:
      noise = 0*param.data.cuda() # will fill with normal at each forward
      self.noises.append(noise)
    if scale is not None:
      self.scale = scale
    else:
      self.scale = 1.
    self.observed = observed

  def forward(self, params, scale=None, observed=None):
    # scale should be sqrt(2*alpha/eta)
    # where eta is the learning rate and alpha is the strength of drag term
    if scale is None:
      scale = self.scale
    if observed is None:
      observed = self.observed

    assert scale is not None, "Please provide scale"
    noise_loss = 0.0
    for noise, var in zip(self.noises, params):
      # This is scale * z^T*v
      # The derivative wrt v will become scale*z
      _noise = noise.normal_(0,1)
      noise_loss += scale*torch.sum(Variable(_noise)*var)
    noise_loss /= observed
    return noise_loss

class PriorLoss(torch.nn.Module):
  # negative log Gaussian prior
  def __init__(self, prior_std=1., observed=None):
    super(PriorLoss, self).__init__()
    self.observed = observed
    self.prior_std = prior_std

  def forward(self, params, observed=None):
    if observed is None:
      observed = self.observed
    prior_loss = 0.0
    for var in params:
      prior_loss += torch.sum(var*var/(self.prior_std*self.prior_std))
    prior_loss /= observed
    return prior_loss


from __future__ import with_statement, unicode_literals
import sys
import os
import re
import time
import logging

from ..config import config
if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3:
    from bsddb3 import db
else:
    from bsddb import db

#-------------------------------------------------------------------------
#
# Gramps modules
#
#-------------------------------------------------------------------------
from ..const import GRAMPS_LOCALE as glocale
_ = glocale.translation.gettext
from ..constfunc import cuni, UNITYPE, handle2internal
from ..lib.markertype import MarkerType
from ..lib.nameorigintype import NameOriginType
from ..lib.place import Place
from ..lib.placeref import PlaceRef
from ..lib.placetype import PlaceType
from ..lib.eventtype import EventType
from ..lib.tag import Tag
from ..utils.file import create_checksum
from ..utils.id import create_id
from . import BSDDBTxn
from .write import _mkname, SURNAMES
from .dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY,
                      MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY)
from gramps.gui.dialog import (InfoDialog)

LOG = logging.getLogger(".upgrade")

def gramps_upgrade_17(self):
    """
    Upgrade database from version 16 to 17.

    1. This upgrade adds tags to event, place, repository, source and
       citation objects.
    2. Data of Source becomes SourceAttributes Secondary Object
    3. Create a place hierarchy.
    4. Add checksum field to media objects.
    5. Rebuild list of custom events.
    """
    length = (len(self.event_map) + len(self.place_map) +
              len(self.repository_map) + len(self.source_map) +
              len(self.citation_map) + len(self.media_map))
    self.set_total(length)

    # ---------------------------------
    # Modify Event
    # ---------------------------------
    # Add new tag_list field.
    self.event_names = set()
    for handle in self.event_map.keys():
        event = self.event_map[handle]
        new_event = list(event)
        event_type = EventType()
        event_type.unserialize(new_event[2])
        if event_type.is_custom():
            self.event_names.add(str(event_type))
        new_event = new_event[:11] + [[]] + new_event[11:]
        new_event = tuple(new_event)
        with BSDDBTxn(self.env, self.event_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_event)
        self.update()

    # ---------------------------------
    # Modify Place
    # ---------------------------------
    # Convert to hierarchical structure and add new tag_list field.
    locations = {}
    self.max_id = 0
    index = re.compile('[0-9]+')
    for handle in self.place_map.keys():
        place = self.place_map[handle]
        match = index.search(place[1])
        if match:
            if self.max_id <= int(match.group(0)):
                self.max_id = int(match.group(0))
        if place[5] is not None:
            locations[get_location(place[5])] = handle

    for handle in list(self.place_map.keys()):
        place = self.place_map[handle]
        new_place = list(place)

        zip_code = ''
        if new_place[5]:
            zip_code = new_place[5][0][6]

        # find title and type
        main_loc = get_location(new_place[5])
        for level, name in enumerate(main_loc):
            if name:
                break

        loc = list(main_loc[:])
        loc[level] = ''

        # find top parent
        parent_handle = None
        for n in range(7):
            if loc[n]:
                tup = tuple([''] * n + loc[n:])
                parent_handle = locations.get(tup, None)
                if parent_handle:
                    break

        # create nodes
        if parent_handle:
            n -= 1
        while n > level:
            if loc[n]:
                title = ', '.join([item for item in loc[n:] if item])
                parent_handle = add_place(self, loc[n], n, parent_handle, title)
                locations[tuple([''] * n + loc[n:])] = parent_handle
            n -= 1

        if parent_handle is not None:
            placeref = PlaceRef()
            placeref.ref = handle2internal(parent_handle)
            placeref_list = [placeref.serialize()]
        else:
            placeref_list = []

        type_num = 7 - level if name else PlaceType.UNKNOWN
        new_place = new_place[:5] + [placeref_list, name, [],
                    PlaceType(type_num).serialize(), zip_code] + \
                    new_place[6:12] + [[]] + new_place[12:]
        new_place = tuple(new_place)
        with BSDDBTxn(self.env, self.place_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_place)
        self.update()

    # ---------------------------------
    # Modify Repository
    # ---------------------------------
    # Add new tag_list field.
    for handle in self.repository_map.keys():
        repository = self.repository_map[handle]
        new_repository = list(repository)
        new_repository = new_repository[:8] + [[]] + new_repository[8:]
        new_repository = tuple(new_repository)
        with BSDDBTxn(self.env, self.repository_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_repository)
        self.update()

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    # Add new tag_list field.
    for handle in self.source_map.keys():
        source = self.source_map[handle]
        new_source = list(source)
        new_source = new_source[:11] + [[]] + new_source[11:]
        new_source = tuple(new_source)
        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    # ---------------------------------
    # Modify Citation
    # ---------------------------------
    # Add new tag_list field.
    for handle in self.citation_map.keys():
        citation = self.citation_map[handle]
        new_citation = list(citation)
        new_citation = new_citation[:10] + [[]] + new_citation[10:]
        new_citation = tuple(new_citation)
        with BSDDBTxn(self.env, self.citation_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_citation)
        self.update()

    # -------------------------------------------------------
    # Upgrade Source and Citation datamap to SrcAttributeBase
    # -------------------------------------------------------
    for handle in self.source_map.keys():
        source = self.source_map[handle]
        (handle, gramps_id, title, author, pubinfo,
            notelist, medialist, abbrev, change, datamap, reporef_list,
            taglist, private) = source
        srcattributelist = upgrade_datamap_17(datamap)
        new_source = (handle, gramps_id, title, author, pubinfo,
            notelist, medialist, abbrev, change, srcattributelist, reporef_list,
            taglist, private)
        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    for handle in self.citation_map.keys():
        citation = self.citation_map[handle]
        (handle, gramps_id, datelist, page, confidence, source_handle,
            notelist, medialist, datamap, change, taglist, private) = citation
        srcattributelist = upgrade_datamap_17(datamap)
        new_citation = (handle, gramps_id, datelist, page, confidence, source_handle,
            notelist, medialist, srcattributelist, change, taglist, private)
        with BSDDBTxn(self.env, self.citation_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_citation)
        self.update()

    # ---------------------------------
    # Modify Media
    # ---------------------------------
    # Add new checksum field.
    base_path = self.metadata.get(b'mediapath', '')
    for handle in self.media_map.keys():
        media = self.media_map[handle]
        new_media = list(media)
        if os.path.isabs(new_media[2]):
            full_path = new_media[2]
        else:
            full_path = os.path.join(base_path, new_media[2])
        checksum = create_checksum(full_path)
        new_media = new_media[:5] + [checksum] + new_media[5:]
        new_media = tuple(new_media)
        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 17)

def get_location(loc):
    # (street, locality, parish, city, county, state, country)
    if loc is None:
        location = ('',) * 7
    else:
        location = loc[0][:2] + (loc[1],) + loc[0][2:6]
    return location

def add_place(self, name, level, parent, title):
    handle = create_id()
    place = Place()
    place.handle = handle
    self.max_id += 1
    place.gramps_id = self.place_prefix % self.max_id
    place.set_name(name)
    place.set_title(title)
    place.set_type(PlaceType(7-level))
    if parent is not None:
        placeref = PlaceRef()
        placeref.ref = handle2internal(parent)
        place.set_placeref_list([placeref])
    with BSDDBTxn(self.env, self.place_map) as txn:
        if isinstance(handle, UNITYPE):
            handle = handle.encode('utf-8')
        txn.put(handle, place.serialize())
    return handle

def upgrade_datamap_17(datamap):
    """
    In version 16 key value pairs are stored in source and citation. These become
    SrcAttribute
    """
    new_srcattr_list = []
    private = False
    from ..lib.srcattrtype import SrcAttributeType
    for (key, value) in datamap.items():
        the_type = SrcAttributeType(key).serialize()
        new_srcattr_list.append((private, the_type, value))
    return new_srcattr_list

def gramps_upgrade_16(self):
    """
    Upgrade database from version 15 to 16. This upgrade converts all
    SourceRef child objects to Citation Primary objects.

    For each primary object that has a sourceref, what we have to do is:

        (1) create each citation
        (2) update the object to reference the Citations
        (3) remove backlinks for references from object to Source
        (4) add backlinks for references from object to Citations
        (5) add backlinks for references from Citation to Source

    the backlinks are all updated at the end by calling
    :py:meth:`reindex_reference_map <.write.DbBsddb.reindex_reference_map>`
    """
    # Only People, Families, Events, Media Objects, Places, Sources and
    # Repositories need to be updated, because these are the only primary
    # objects that can have source citations.
    length = (len(self.person_map) +
              len(self.event_map) + len(self.family_map) +
              len(self.repository_map) + len(self.media_map) +
              len(self.place_map) + len(self.source_map))
    self.set_total(length)

    # Setup data for upgrade statistics information dialogue
    keyorder = [PERSON_KEY, FAMILY_KEY, EVENT_KEY, MEDIA_KEY,
                PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY]
    key2data = {
            PERSON_KEY : 0,
            FAMILY_KEY : 1,
            EVENT_KEY: 2,
            MEDIA_KEY: 3,
            PLACE_KEY: 4,
            REPOSITORY_KEY: 5,
            SOURCE_KEY : 6,
            }
    key2string = {
        PERSON_KEY      : _('%6d  People        upgraded with %6d citations in %6d secs\n'),
        FAMILY_KEY      : _('%6d  Families      upgraded with %6d citations in %6d secs\n'),
        EVENT_KEY       : _('%6d  Events        upgraded with %6d citations in %6d secs\n'),
        MEDIA_KEY       : _('%6d  Media Objects upgraded with %6d citations in %6d secs\n'),
        PLACE_KEY       : _('%6d  Places        upgraded with %6d citations in %6d secs\n'),
        REPOSITORY_KEY  : _('%6d  Repositories  upgraded with %6d citations in %6d secs\n'),
        SOURCE_KEY      : _('%6d  Sources       upgraded with %6d citations in %6d secs\n'),
        }
    data_upgradeobject = [0] * 7

    # Initialise the citation gramps ID number
    self.cmap_index = 0

    # ---------------------------------
    # Modify Person
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for person_handle in self.person_map.keys():
        person = self.person_map[person_handle]
        try:
            # The parameters are evaluated before deciding whether logging is on
            # or not. Since the retrieval of names is so complex, I think it is
            # safer to protect this with a try except block, even though it
            # seems to work for names being present and not.
            LOG.debug("upgrade person %s %s" % (person[3][4],
                      " ".join([name[0] for name in person[3][5]])))
        except:
            pass
        (handle, gramps_id, gender, primary_name, alternate_names,
         death_ref_index, birth_ref_index, event_ref_list, family_list,
         parent_family_list, media_list, address_list, attribute_list,
         urls, lds_seal_list, source_list, note_list, change, tag_list,
         private, person_ref_list) = person
        if primary_name:
            primary_name = upgrade_name_16(self, primary_name)
        if alternate_names:
            alternate_names = upgrade_name_list_16(
                                    self, alternate_names)
        if address_list:
            address_list = upgrade_address_list_16(
                                    self, address_list)
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if attribute_list:
            attribute_list = upgrade_attribute_list_16(
                                    self, attribute_list)
        if lds_seal_list:
            lds_seal_list = upgrade_lds_seal_list_16(
                                    self, lds_seal_list)
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if person_ref_list:
            person_ref_list = upgrade_person_ref_list_16(
                                    self, person_ref_list)
        if event_ref_list:
            event_ref_list = upgrade_event_ref_list_16(self, event_ref_list)
        if primary_name or alternate_names  or address_list or \
           media_list or attribute_list or lds_seal_list or source_list or \
           person_ref_list or event_ref_list:
            new_person = (handle, gramps_id, gender, primary_name,
                          alternate_names, death_ref_index,
                          birth_ref_index, event_ref_list, family_list,
                          parent_family_list, media_list, address_list,
                          attribute_list, urls, lds_seal_list,
                          new_citation_list, note_list, change, tag_list,
                          private, person_ref_list)
            LOG.debug("      upgrade new_person %s" % [new_person])
            with BSDDBTxn(self.env, self.person_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_person)
        self.update()

    LOG.debug("%d persons upgraded with %d citations in %d seconds. " %
              (len(list(self.person_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[PERSON_KEY]] = (len(list(self.person_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Media
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for media_handle in self.media_map.keys():
        media = self.media_map[media_handle]
        LOG.debug("upgrade media object %s" % media[4])
        (handle, gramps_id, path, mime, desc,
         attribute_list, source_list, note_list, change,
         date, tag_list, private) = media
        new_citation_list = convert_source_list_to_citation_list_16(
                                   self, source_list)
        new_attribute_list = upgrade_attribute_list_16(
                                   self, attribute_list)

        new_media = (handle, gramps_id, path, mime, desc,
                     new_attribute_list, new_citation_list, note_list,
                     change, date, tag_list, private)
        LOG.debug("      upgrade new_media %s" % [new_media])
        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    LOG.debug("%d media objects upgraded with %d citations in %d seconds" %
              (len(self.media_map.keys()),
               self.cmap_index - start_num_citations,
               int(time.time() - start_time)))
    data_upgradeobject[key2data[MEDIA_KEY]] = (len(list(self.media_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Places
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for place_handle in self.place_map.keys():
        place = self.place_map[place_handle]
        LOG.debug("upgrade place %s" % place[2])
        (handle, gramps_id, title, longi, lat,
         main_loc, alt_loc, urls, media_list, source_list, note_list,
         change, private) = place
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if source_list or media_list:
            new_place = (handle, gramps_id, title,
                         longi, lat, main_loc, alt_loc, urls,
                         media_list, new_citation_list, note_list,
                         change, private)
            LOG.debug("      upgrade new_place %s" % [new_place])
            with BSDDBTxn(self.env, self.place_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_place)
        self.update()

    LOG.debug("%d places upgraded with %d citations in %d seconds. " %
              (len(list(self.place_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[PLACE_KEY]] = (len(list(self.place_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Families
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for family_handle in self.family_map.keys():
        family = self.family_map[family_handle]
        LOG.debug("upgrade family (gramps_id) %s" % family[1])
        (handle, gramps_id, father_handle, mother_handle,
         child_ref_list, the_type, event_ref_list, media_list,
         attribute_list, lds_seal_list, source_list, note_list,
         change, tag_list, private) = family
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if child_ref_list:
            child_ref_list = upgrade_child_ref_list_16(
                                    self, child_ref_list)
        if lds_seal_list:
            lds_seal_list = upgrade_lds_seal_list_16(
                                    self, lds_seal_list)
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if attribute_list:
            attribute_list = upgrade_attribute_list_16(
                                    self, attribute_list)
        if event_ref_list:
            event_ref_list = upgrade_event_ref_list_16(self, event_ref_list)
        if source_list or media_list or child_ref_list or \
            attribute_list or lds_seal_list or event_ref_list:
            new_family = (handle, gramps_id, father_handle, mother_handle,
                          child_ref_list, the_type, event_ref_list, media_list,
                          attribute_list, lds_seal_list, new_citation_list,
                          note_list, change, tag_list, private)
            LOG.debug("      upgrade new_family %s" % [new_family])
            with BSDDBTxn(self.env, self.family_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_family)
        self.update()

    LOG.debug("%d families upgraded with %d citations in %d seconds. " %
              (len(list(self.family_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[FAMILY_KEY]] = (len(list(self.family_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)
    # ---------------------------------
    # Modify Events
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for event_handle in self.event_map.keys():
        event = self.event_map[event_handle]
        LOG.debug("upgrade event %s" % event[4])
        (handle, gramps_id, the_type, date, description, place,
         source_list, note_list, media_list, attribute_list,
         change, private) = event
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if attribute_list:
            attribute_list = upgrade_attribute_list_16(
                                    self, attribute_list)
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if source_list or attribute_list or media_list:
            new_event = (handle, gramps_id, the_type, date, description, place,
                         new_citation_list, note_list, media_list,
                         attribute_list,
                         change, private)
            LOG.debug("      upgrade new_event %s" % [new_event])
            with BSDDBTxn(self.env, self.event_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_event)
        self.update()

    LOG.debug("%d events upgraded with %d citations in %d seconds. " %
              (len(self.event_map.keys()),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[EVENT_KEY]] = (len(list(self.event_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Repositories
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for repository_handle in self.repository_map.keys():
        repository = self.repository_map[repository_handle]
        LOG.debug("upgrade repository %s" % repository[3])
        (handle, gramps_id, the_type, name, note_list,
         address_list, urls, change, private) = repository
        if address_list:
            address_list = upgrade_address_list_16(
                                    self, address_list)
        if address_list:
            new_repository = (handle, gramps_id, the_type, name, note_list,
                              address_list, urls, change, private)
            LOG.debug("      upgrade new_repository %s" % [new_repository])
            with BSDDBTxn(self.env, self.repository_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_repository)
        self.update()

    LOG.debug("%d repositories upgraded with %d citations in %d seconds. " %
              (len(list(self.repository_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[REPOSITORY_KEY]] = (len(list(self.repository_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for source_handle in self.source_map.keys():
        source = self.source_map[source_handle]
        LOG.debug("upgrade source %s" % source[2])
        (handle, gramps_id, title, author,
         pubinfo, note_list, media_list,
         abbrev, change, datamap, reporef_list,
         private) = source
        if media_list:
            media_list = upgrade_media_list_16(
                                        self, media_list)

        new_source = (handle, gramps_id, title, author,
                      pubinfo, note_list, media_list,
                      abbrev, change, datamap, reporef_list,
                      private)
        LOG.debug("      upgrade new_source %s" % [new_source])
        with BSDDBTxn(self.env, self.source_map) as txn:
            txn.put(str(handle), new_source)
        self.update()

    LOG.debug("%d sources upgraded with %d citations in %d seconds" %
              (len(self.source_map.keys()),
               self.cmap_index - start_num_citations,
               int(time.time() - start_time)))
    data_upgradeobject[key2data[SOURCE_KEY]] = (len(self.source_map.keys()),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------

    # ---------------------------------
    # Example database from repository took:
    # 3403 events upgraded with 8 citations in 23 seconds. Backlinks took 1071 seconds
    # actually 4 of these citations were from:
    # Media upgrade 4 citations upgraded in 4 seconds
    # by only doing the backlinks when there might be something to do,
    # improved to:
    # 3403 events upgraded with 8 citations in 19 seconds. Backlinks took 1348 seconds
    # further improved by skipping debug logging:
    # 3403 events upgraded with 8 citations in 2 seconds. Backlinks took 167 seconds

    #Number of new objects upgraded:
#  2090  People        upgraded with   2092 citations in 2148 secs
#   734  Families      upgraded with    735 citations in 768 secs
#  3403  Events        upgraded with      4 citations in 212 secs
#     7  Media Objects upgraded with      4 citations in 3 secs
#   852  Places        upgraded with      0 citations in 39 secs

# with reduced diagnostics
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     74 secs
#    35  Families      upgraded with     36 citations in     31 secs
#  3403  Events        upgraded with      4 citations in      7 secs
#     7  Media Objects upgraded with      4 citations in      3 secs
#   852  Places        upgraded with      0 citations in      1 secs

# without doing any backlinks
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     43 secs
#    35  Families      upgraded with     36 citations in     24 secs
#  3403  Events        upgraded with      4 citations in      6 secs
#     7  Media Objects upgraded with      4 citations in      2 secs
#   852  Places        upgraded with      0 citations in      1 secs

# another run about the same code:
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     48 secs
#    35  Families      upgraded with     36 citations in     21 secs
#  3403  Events        upgraded with      4 citations in      9 secs
#     7  Media Objects upgraded with      4 citations in      4 secs
#   852  Places        upgraded with      0 citations in      1 secs

# another run
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     36 secs
#    35  Families      upgraded with     36 citations in     18 secs
#  3403  Events        upgraded with      4 citations in      9 secs
#     7  Media Objects upgraded with      4 citations in      2 secs
#   852  Places        upgraded with      0 citations in      1 secs

# without incorrect nestetd tranaction structure:
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in      0 secs
#    35  Families      upgraded with     36 citations in      0 secs
#  3403  Events        upgraded with      4 citations in      0 secs
#     7  Media Objects upgraded with      4 citations in      0 secs
#   852  Places        upgraded with      0 citations in      0 secs

#[[(73, 76, 0.12430405616760254), (35, 36, 0.042523860931396484), (3403, 4, 0.52303886413574219), (7, 4, 0.058229923248291016), (852, 0, 0.14816904067993164)]]

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 16)

    LOG.debug([data_upgradeobject])
    txt = _("Number of new objects upgraded:\n")
    for key in keyorder:
        try:
            txt += key2string[key] % data_upgradeobject[key2data[key]]
        except:
            txt += key2string[key]
    txt += _("\n\nYou may want to run\n"
             "Tools -> Family Tree Processing -> Merge\n"
             "in order to merge citations that contain similar\n"
             "information")
    InfoDialog(_('Upgrade Statistics'), txt, monospaced=True)

def upgrade_media_list_16(self, media_list):
    new_media_list = []
    for media in media_list:
        (privacy, source_list, note_list, attribute_list, ref, rect) = media
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_attribute_list = upgrade_attribute_list_16(
                                        self, attribute_list)
        new_media = (privacy, new_citation_list, note_list, new_attribute_list,
                     ref, rect)
        new_media_list.append((new_media))
    return new_media_list

def upgrade_attribute_list_16(self, attribute_list):
    new_attribute_list = []
    for attribute in attribute_list:
        (privacy, source_list, note_list, the_type,
         value) = attribute
        new_citation_list = convert_source_list_to_citation_list_16(
                                self, source_list)
        new_attribute = (privacy, new_citation_list, note_list,
                         the_type, value)
        new_attribute_list.append((new_attribute))
    return new_attribute_list

def upgrade_child_ref_list_16(self, child_ref_list):
    new_child_ref_list = []
    for child_ref in child_ref_list:
        (privacy, source_list, note_list, ref, frel, mrel) = child_ref
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_child_ref = (privacy, new_citation_list, note_list, ref, frel, mrel)
        new_child_ref_list.append((new_child_ref))
    return new_child_ref_list

def upgrade_lds_seal_list_16(self, lds_seal_list):
    new_lds_seal_list = []
    for lds_seal in lds_seal_list:
        (source_list, note_list, date, type, place,
         famc, temple, status, private) = lds_seal
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_lds_seal = (new_citation_list, note_list, date, type, place,
                        famc, temple, status, private)
        new_lds_seal_list.append((new_lds_seal))
    return new_lds_seal_list

def upgrade_address_list_16(self, address_list):
    new_address_list = []
    for address in address_list:
        (privacy, source_list, note_list, date, location) = address
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_address = (privacy, new_citation_list, note_list, date, location)
        new_address_list.append((new_address))
    return new_address_list

def upgrade_name_list_16(self, name_list):
    new_name_list = []
    for name in name_list:
        new_name = upgrade_name_16(self, name)
        new_name_list.append((new_name))
    return new_name_list

def upgrade_name_16(self, name):
    (privacy, source_list, note, date, first_name, surname_list, suffix,
     title, name_type, group_as, sort_as, display_as, call, nick,
     famnick) = name
    new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
    new_name = (privacy, new_citation_list, note, date, first_name,
                surname_list, suffix, title, name_type, group_as, sort_as,
                display_as, call, nick, famnick)
    return new_name

def upgrade_person_ref_list_16(self, person_ref_list):
    new_person_ref_list = []
    for person_ref in person_ref_list:
        (privacy, source_list, note_list, ref, rel) = person_ref
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_person_ref = (privacy, new_citation_list, note_list, ref, rel)
        new_person_ref_list.append((new_person_ref))
    return new_person_ref_list

def upgrade_event_ref_list_16(self, event_ref_list):
    new_event_ref_list = []
    for event_ref in event_ref_list:
        (privacy, note_list, attribute_list, ref, role) = event_ref
        new_attribute_list = upgrade_attribute_list_16(
                                        self, attribute_list)
        new_event_ref = (privacy, note_list, new_attribute_list, ref, role)
        new_event_ref_list.append((new_event_ref))
    return new_event_ref_list

def convert_source_list_to_citation_list_16(self, source_list):
    citation_list = []
    for source in source_list:
        (date, private, note_list, confidence, ref, page) = source
        new_handle = create_id()
        new_media_list = []
        new_data_map = {}
        new_change = time.time()
        new_gramps_id = self.citation_prefix % self.cmap_index
        new_citation = (new_handle, new_gramps_id,
                        date, page, confidence, ref, note_list, new_media_list,
                        new_data_map, new_change, private)
        with BSDDBTxn(self.env, self.citation_map) as txn:
            if isinstance(new_handle, UNITYPE):
                new_handle = new_handle.encode('utf-8')
            txn.put(new_handle, new_citation)
        self.cmap_index += 1
        citation_list.append((new_handle))
    return citation_list

def gramps_upgrade_15(self):
    """
    Upgrade database from version 14 to 15. This upgrade adds:

        * tagging
        * surname list
        * remove marker
    """
    length = (len(self.note_map) + len(self.person_map) +
              len(self.event_map) + len(self.family_map) +
              len(self.repository_map) + len(self.media_map) +
              len(self.place_map) + len(self.source_map)) + 10
    self.set_total(length)
    self.tags = {}

    # ---------------------------------
    # Modify Person
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.person_map.keys():
        person = self.person_map[handle]

        (junk_handle,        #  0
         gramps_id,          #  1
         gender,             #  2
         primary_name,       #  3
         alternate_names,    #  4
         death_ref_index,    #  5
         birth_ref_index,    #  6
         event_ref_list,     #  7
         family_list,        #  8
         parent_family_list, #  9
         media_list,         # 10
         address_list,       # 11
         attribute_list,     # 12
         urls,               # 13
         ord_list,           # 14
         psource_list,       # 15
         pnote_list,         # 16
         change,             # 17
         marker,             # 18
         pprivate,           # 19
         person_ref_list,    # 20
         ) = person

        tag_handle = convert_marker(self, marker)
        if tag_handle:
            tags = [tag_handle]
        else:
            tags = []
        address_list = list(map(convert_address, address_list))
        new_primary_name = convert_name_15(primary_name)
        new_alternate_names = list(map(convert_name_15, alternate_names))
        new_person = (junk_handle,        #  0
                      gramps_id,          #  1
                      gender,             #  2
                      new_primary_name,   #  3
                      new_alternate_names,#  4
                      death_ref_index,    #  5
                      birth_ref_index,    #  6
                      event_ref_list,     #  7
                      family_list,        #  8
                      parent_family_list, #  9
                      media_list,         # 10
                      address_list,       # 11
                      attribute_list,     # 12
                      urls,               # 13
                      ord_list,           # 14
                      psource_list,       # 15
                      pnote_list,         # 16
                      change,             # 17
                      tags,               # 18
                      pprivate,           # 19
                      person_ref_list     # 20
                      )

        with BSDDBTxn(self.env, self.person_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_person)
        self.update()
    #surname is now different, remove secondary index with names
    _db = db.DB(self.env)
    try:
        _db.remove(_mkname(self.full_name, SURNAMES), SURNAMES)
    except db.DBNoSuchFileError:
        pass

    # ---------------------------------
    # Modify Family
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.family_map.keys():
        family = self.family_map[handle]
        new_family = list(family)
        tag_handle = convert_marker(self, new_family[13])
        if tag_handle:
            new_family[13] = [tag_handle]
        else:
            new_family[13] = []
        new_family = tuple(new_family)
        with BSDDBTxn(self.env, self.family_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_family)
        self.update()

    # ---------------------------------
    # Modify Note
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.note_map.keys():
        note = self.note_map[handle]
        new_note = list(note)
        tag_handle = convert_marker(self, new_note[6])
        if tag_handle:
            new_note[6] = [tag_handle]
        else:
            new_note[6] = []
        new_note = tuple(new_note)
        with BSDDBTxn(self.env, self.note_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_note)
        self.update()

    # ---------------------------------
    # Modify Media object
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.media_map.keys():
        media = self.media_map[handle]
        new_media = list(media)
        new_media[10] = []
        new_media = tuple(new_media)
        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    # ---------------------------------
    # Modify Event
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.event_map.keys():
        event = self.event_map[handle]
        new_event = list(event)
        new_event = new_event[:11] + new_event[12:]
        #new_event[11] = []
        new_event = tuple(new_event)
        with BSDDBTxn(self.env, self.event_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_event)
        self.update()

    # ---------------------------------
    # Modify Place
    # ---------------------------------
    # Remove the old marker field, set new locality.
    for handle in self.place_map.keys():
        place = self.place_map[handle]
        new_place = list(place)
        if new_place[5] is not None:
            new_place[5] = convert_location(new_place[5])
        new_place[6] = list(map(convert_location, new_place[6]))
        new_place = new_place[:12] + new_place[13:]
        new_place = tuple(new_place)
        with BSDDBTxn(self.env, self.place_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_place)
        self.update()

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    # Remove the old marker field.
    for handle in self.source_map.keys():
        source = self.source_map[handle]
        new_source = list(source)
        new_source = new_source[:11] + new_source[12:]
        new_source = tuple(new_source)
        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    # ---------------------------------
    # Modify Repository
    # ---------------------------------
    # Remove the old marker field, set new locality.
    for handle in self.repository_map.keys():
        repository = self.repository_map[handle]
        new_repository = list(repository)
        new_repository = new_repository[:8] + new_repository[9:]
        new_repository[5] = list(map(convert_address, new_repository[5]))
        new_repository = tuple(new_repository)
        with BSDDBTxn(self.env, self.repository_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_repository)
        self.update()

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 15)

def convert_marker(self, marker_field):
    """Convert a marker into a tag."""
    marker = MarkerType()
    marker.unserialize(marker_field)
    tag_name = cuni(marker)

    if tag_name != '':
        if tag_name not in self.tags:
            tag = Tag()
            handle = create_id()
            tag.set_handle(handle)
            tag.set_change_time(time.time())
            tag.set_name(tag_name)
            tag.set_priority(len(self.tags))
            with BSDDBTxn(self.env, self.tag_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, tag.serialize())
            self.tags[tag_name] = handle
        return self.tags[tag_name]
    else:
        return None

def convert_locbase(loc):
    """Convert location base to include an empty locality field."""
    return tuple([loc[0], ''] + list(loc[1:]))

def convert_location(loc):
    """Convert a location into the new format."""
    return (convert_locbase(loc[0]), loc[1])

def convert_address(addr):
    """Convert an address into the new format."""
    return (addr[0], addr[1], addr[2], addr[3], convert_locbase(addr[4]))

def convert_name_15(name):
    (privacy, source_list, note_list, date,
     first_name, surname, suffix, title,
     name_type, prefix, patronymic,
     group_as, sort_as, display_as, call) = name

    connector = ""
    origintype = (NameOriginType.NONE, "")
    patorigintype = (NameOriginType.PATRONYMIC, "")

    if patronymic.strip() == "":
        #no patronymic, create a single surname
        surname_list = [(surname, prefix, True, origintype, connector)]
    else:
        #a patronymic, if no surname or equal as patronymic, a single surname
        if (surname.strip() == "") or (surname == patronymic and prefix == ""):
            surname_list = [(patronymic, prefix, True, patorigintype, connector)]
        else:
            #two surnames, first patronymic, then surname which is primary
            surname_list = [(patronymic, "", False, patorigintype, ""),
                            (surname, prefix, True, origintype, connector)]

    #return new value, add two empty strings for nick and family nick
    return (privacy, source_list, note_list, date,
         first_name, surname_list, suffix, title, name_type,
         group_as, sort_as, display_as, call, "", "")

def gramps_upgrade_14(self):
    """Upgrade database from version 13 to 14."""
    # This upgrade modifies notes and dates
    length = (len(self.note_map) + len(self.person_map) +
              len(self.event_map) + len(self.family_map) +
              len(self.repository_map) + len(self.media_map) +
              len(self.place_map) + len(self.source_map))
    self.set_total(length)

    # ---------------------------------
    # Modify Notes
    # ---------------------------------
    # replace clear text with StyledText in Notes
    for handle in self.note_map.keys():
        note = self.note_map[handle]
        (junk_handle, gramps_id, text, format, note_type,
         change, marker, private) = note
        styled_text = (text, [])
        new_note = (handle, gramps_id, styled_text, format, note_type,
                    change, marker, private)
        with BSDDBTxn(self.env, self.note_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_note)
        self.update()

    # ---------------------------------
    # Modify Event
    # ---------------------------------
    # update dates with newyear
    for handle in self.event_map.keys():
        event = self.event_map[handle]
        (junk_handle, gramps_id, the_type, date, description, place,
         source_list, note_list, media_list, attribute_list,
         change, marker, private) = event
        new_date = convert_date_14(date)
        new_source_list = new_source_list_14(source_list)
        new_media_list = new_media_list_14(media_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_event = (junk_handle, gramps_id, the_type, new_date,
                     description, place, new_source_list, note_list,
                     new_media_list, new_attribute_list, change,marker,private)
        with BSDDBTxn(self.env, self.event_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_event)
        self.update()

    # ---------------------------------
    # Modify Person
    # ---------------------------------
    # update dates with newyear
    for handle in self.person_map.keys():
        person = self.person_map[handle]
        (junk_handle,        #  0
         gramps_id,          #  1
         gender,             #  2
         primary_name,       #  3
         alternate_names,    #  4
         death_ref_index,    #  5
         birth_ref_index,    #  6
         event_ref_list,     #  7
         family_list,        #  8
         parent_family_list, #  9
         media_list,         # 10
         address_list,       # 11
         attribute_list,     # 12
         urls,               # 13
         lds_ord_list,       # 14
         psource_list,       # 15
         pnote_list,         # 16
         change,             # 17
         marker,             # 18
         pprivate,           # 19
         person_ref_list,    # 20
         ) = person

        new_address_list = []
        for address in address_list:
            (privacy, asource_list, anote_list, date, location) = address
            new_date = convert_date_14(date)
            new_asource_list = new_source_list_14(asource_list)
            new_address_list.append((privacy, new_asource_list, anote_list,
                                     new_date, location))
        new_ord_list = []
        for ldsord in lds_ord_list:
            (lsource_list, lnote_list, date, type, place,
             famc, temple, status, lprivate) = ldsord
            new_date = convert_date_14(date)
            new_lsource_list = new_source_list_14(lsource_list)
            new_ord_list.append( (new_lsource_list, lnote_list, new_date, type,
                                  place, famc, temple, status, lprivate))

        new_primary_name = convert_name_14(primary_name)

        new_alternate_names = [convert_name_14(name) for name
                               in alternate_names]

        new_media_list = new_media_list_14(media_list)
        new_psource_list = new_source_list_14(psource_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_person_ref_list = new_person_ref_list_14(person_ref_list)

        new_person = (junk_handle,        #  0
                      gramps_id,          #  1
                      gender,             #  2
                      new_primary_name,       #  3
                      new_alternate_names,    #  4
                      death_ref_index,    #  5
                      birth_ref_index,    #  6
                      event_ref_list,     #  7
                      family_list,        #  8
                      parent_family_list, #  9
                      new_media_list,         # 10
                      new_address_list,       # 11
                      new_attribute_list,     # 12
                      urls,               # 13
                      new_ord_list,          # 14
                      new_psource_list,       # 15
                      pnote_list,         # 16
                      change,             # 17
                      marker,             # 18
                      pprivate,           # 19
                      new_person_ref_list,    # 20
                      )

        with BSDDBTxn(self.env, self.person_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_person)
        self.update()

    # ---------------------------------
    # Modify Family
    # ---------------------------------
    # update dates with newyear
    for handle in self.family_map.keys():
        family = self.family_map[handle]
        (junk_handle, gramps_id, father_handle, mother_handle,
         child_ref_list, the_type, event_ref_list, media_list,
         attribute_list, lds_seal_list, source_list, note_list,
         change, marker, private) = family
        new_child_ref_list = new_child_ref_list_14(child_ref_list)
        new_media_list = new_media_list_14(media_list)
        new_source_list = new_source_list_14(source_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_seal_list = []
        for ldsord in lds_seal_list:
            (lsource_list, lnote_list, date, type, place,
             famc, temple, status, lprivate) = ldsord
            new_date = convert_date_14(date)
            new_lsource_list = new_source_list_14(lsource_list)
            new_seal_list.append( (new_lsource_list, lnote_list, new_date, type,
                                   place, famc, temple, status, lprivate))

        new_family = (junk_handle, gramps_id, father_handle, mother_handle,
                      new_child_ref_list, the_type, event_ref_list, new_media_list,
                      new_attribute_list, new_seal_list, new_source_list, note_list,
                      change, marker, private)

        with BSDDBTxn(self.env, self.family_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_family)
        self.update()

    # ---------------------------------
    # Modify Repository
    # ---------------------------------
    # update dates with newyear
    for handle in self.repository_map.keys():
        repository = self.repository_map[handle]
        # address
        (junk_handle, gramps_id, the_type, name, note_list,
         address_list, urls, change, marker, private) = repository

        new_address_list = []
        for address in address_list:
            (privacy, asource_list, anote_list, date, location) = address
            new_date = convert_date_14(date)
            new_asource_list = new_source_list_14(asource_list)
            new_address_list.append((privacy, new_asource_list, anote_list,
                                     new_date, location))

        new_repository = (junk_handle, gramps_id, the_type, name, note_list,
                          new_address_list, urls, change, marker, private)

        with BSDDBTxn(self.env, self.repository_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_repository)
        self.update()

    # ---------------------------------
    # Modify Media
    # ---------------------------------
    for media_handle in self.media_map.keys():
        media = self.media_map[media_handle]
        (handle, gramps_id, path, mime, desc,
         attribute_list, source_list, note_list, change,
         date, marker, private) = media
        new_source_list = new_source_list_14(source_list)
        new_date = convert_date_14(date)
        new_media = (handle, gramps_id, path, mime, desc,
                     attribute_list, new_source_list, note_list, change,
                     new_date, marker, private)

        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    # ---------------------------------
    # Modify Place
    # ---------------------------------
    for place_handle in self.place_map.keys():
        place = self.place_map[place_handle]
        (handle, gramps_id, title, longi, lat,
         main_loc, alt_loc, urls, media_list, source_list, note_list,
         change, marker, private) = place
        new_media_list = new_media_list_14(media_list)
        new_source_list = new_source_list_14(source_list)
        new_place = (handle, gramps_id, title, longi, lat,
                     main_loc, alt_loc, urls, new_media_list,
                     new_source_list, note_list, change, marker, private)

        with BSDDBTxn(self.env, self.place_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_place)
        self.update()

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    for source_handle in self.source_map.keys():
        source = self.source_map[source_handle]
        (handle, gramps_id, title, author,
         pubinfo, note_list, media_list,
         abbrev, change, datamap, reporef_list,
         marker, private) = source
        new_media_list = new_media_list_14(media_list)
        new_source = (handle, gramps_id, title, author,
                      pubinfo, note_list, new_media_list,
                      abbrev, change, datamap, reporef_list,
                      marker, private)

        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 14)

def new_source_list_14(source_list):
    new_source_list = []
    for source in source_list:
        (date, private, note_list, confidence, ref, page) = source
        new_date = convert_date_14(date)
        new_source_list.append((new_date, private, note_list, confidence, ref, page))
    return new_source_list

def new_attribute_list_14(attribute_list):
    new_attribute_list = []
    for attribute in attribute_list:
        (private, asource_list, note_list, the_type, value) = attribute
        new_asource_list = new_source_list_14(asource_list)
        new_attribute_list.append((private, new_asource_list, note_list, the_type, value))
    return new_attribute_list

def new_media_list_14(media_list):
    # ---------------------------------
    # Event Media list
    # ---------------------------------
    new_media_list = []
    for media in media_list:
        (private, source_list, note_list,attribute_list,ref,role) = media
        new_source_list = new_source_list_14(source_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_media_list.append((private, new_source_list, note_list, new_attribute_list, ref, role))
    return new_media_list

def new_person_ref_list_14(person_ref_list):
    new_person_ref_list = []
    for person_ref in person_ref_list:
        (private, source_list, note_list, ref, rel) = person_ref
        new_source_list = new_source_list_14(source_list)
        new_person_ref_list.append((private, new_source_list, note_list, ref, rel))
    return new_person_ref_list

def new_child_ref_list_14(child_ref_list):
    new_child_ref_list = []
    for data in child_ref_list:
        (private, source_list, note_list, ref, frel, mrel) = data
        new_source_list = new_source_list_14(source_list)
        new_child_ref_list.append((private, new_source_list, note_list, ref, frel, mrel))
    return new_child_ref_list

def convert_date_14(date):
    if date:
        (calendar, modifier, quality, dateval, text, sortval) = date
        return (calendar, modifier, quality, dateval, text, sortval, 0)
    else:
        return None

def convert_name_14(name):
    (privacy, source_list, note_list, date,
     first_name, surname, suffix, title,
     name_type, prefix, patronymic,
     group_as, sort_as, display_as, call) = name
    new_date = convert_date_14(date)
    new_source_list = new_source_list_14(source_list)
    return (privacy, new_source_list, note_list, new_date,
            first_name, surname, suffix, title,
            name_type, prefix, patronymic,
            group_as, sort_as, display_as, call)


from conans import ConanFile, CMake, tools
import os

class mklDynamic(ConanFile):
    name = "mkl-shared"
    version = "2019.4"
    url = "https://github.com/shellshocked2003/mkl-shared"
    homepage = "https://anaconda.org/anaconda/mkl"
    author = "Michael Gardner <mhgardner@berkeley.edu>"
    license = "Intel Simplified Software License"
    settings = {"os": None, "arch": ["x86_64"]}
    options = {"single_lib" : [True, False]}
    default_options = {"single_lib": False}
    description = "Intel Math Kernel Library Shared Binaries"
    exports_sources = ["CMakeLists.txt"]
    generators = "cmake"
    build_policy = "missing"

    # Custom attributes for Bincrafters recipe conventions
    _source_subfolder = "source_subfolder"
    _build_subfolder = "build_subfolder"

    def _configure_cmake(self):
        cmake = CMake(self)
        cmake.configure(build_folder=self._build_subfolder)
        return cmake

    def build(self):
        if self.settings.os == "Windows":
            url = ("https://anaconda.org/intel/mkl-devel/2019.4/download/win-64/mkl-devel-2019.4-intel_245.tar.bz2")
            bin_url = ("https://anaconda.org/anaconda/mkl/2019.4/download/win-64/mkl-2019.4-245.tar.bz2")
            tools.get(bin_url, destination=self._source_subfolder)
        elif self.settings.os == "Macos":
            url = ("https://anaconda.org/anaconda/mkl/2019.4/download/osx-64/mkl-2019.4-233.tar.bz2")
        elif self.settings.os == "Linux":
            url = ("https://anaconda.org/anaconda/mkl/2019.4/download/linux-64/mkl-2019.4-243.tar.bz2")
        else:
            raise Exception("Binary does not exist for these settings")
        tools.get(url, destination=self._source_subfolder)

    def package(self):
        self.copy("LICENSE.txt", dst="licenses", src=self._source_subfolder + "/info")
        if self.settings.os == "Windows":
            self.copy("*", dst="lib", src=self._source_subfolder + "/Library/lib")
            self.copy("*", dst="bin", src=self._source_subfolder + "/Library/bin")
        else:
            self.copy("*", dst="lib", src=self._source_subfolder + "/lib")

    def package_info(self):
        if "single_lib" in self.options is False:
            self.cpp_info.libs = tools.collect_libs(self)
        else :
            self.cpp_info.libs = ["mkl_rt"]

        self.cpp_info.libdirs = ['lib']  # Directories where libraries can be found
        self.cpp_info.bindirs = ['bin', 'lib']  # Directories where executables and shared libs can be found
        self.env_info.PATH.append(os.path.join(self.package_folder, "bin"))
        self.env_info.PATH.append(os.path.join(self.package_folder, "lib"))
        self.env_info.LD_LIBRARY_PATH.append(os.path.join(self.package_folder, "lib"))
        self.env_info.DYLD_LIBRARY_PATH.append(os.path.join(self.package_folder, "lib"))


# test_IM_conNewsDisable
import json
import logging
import random

import allure
import pytest

from Common.getConsoleLogin import getConsoleLogin_token
from Common.get_time_stamp import get_time_stamp13, TimeTostamp

from Common.sign import get_sign

from Common.requests_library import Requests
from Common.tools.read_write_json import get_json

from glo import console_JSON, console_HTTP, BASE_DIR

@pytest.mark.skip(reason="调试中 ")
@allure.feature('资讯console_删除恢复资讯')
class TestIMConNewsDisable():
    @classmethod
    def setup_class(cls) -> None:
        cls.session = Requests().get_session()

    def tearDown(self) -> None:
        Requests(self.session).close_session()

    # @pytest.mark.skip(reason="调试中 ")
    @pytest.mark.parametrize('info',
                             get_json(BASE_DIR + r"/TestData/testIMData/test_IM_conNewsDisable.json"))
    def test_IM_conNewsDisable(self,info):
        url = console_HTTP + "/api/con_news/v1/list"
        header = console_JSON

        # 拼装参数
        paylo = {
            "pageSize": 20,
            "currentPage": 1,
            "startTime": TimeTostamp(),
            "endTime": get_time_stamp13()
        }
        # paylo = info
        # print(paylo)
        sign1 = {"sign": get_sign(paylo)}  # 把参数签名后通过sign1传出来
        payload1 = {}
        payload1.update(paylo)
        payload1.update(sign1)
        headers = {}
        headers.update(header)
        # print(token)
        # print(type(token))
        token = {"token": getConsoleLogin_token()}
        headers.update(token)  # 将token更新到headers
        # print(headers)
        payload = json.dumps(dict(payload1))

        r = Requests(self.session).post(
            url=url, headers=headers, data=payload, title="查询资讯列表"
        )

        j = r.json()
        # print(j)
        # print(j.get('data').get('list')[0].get('newsId'))
        url_disable = console_HTTP + "/api/con_news/v1/disable"
        paylo_disable = {
            "newsId": f"{j.get('data').get('list')[0].get('newsId')}"
        }
        paylo_disable1 = info
        sign1 = {"sign": get_sign(paylo)}  # 把参数签名后通过sign1传出来
        payload1 = {}
        payload1.update(paylo_disable)
        payload1.update(paylo_disable1)
        payload1.update(sign1)
        payload2 = json.dumps(dict(payload1))

        r = Requests(self.session).post(
            url=url_disable, headers=headers, data=payload2, title="删除恢复资讯"
        )
        # print(r.json())
        assert r.status_code == 200
        assert r.json().get("code") == "000000"
        assert r.json().get("msg") == "ok"
        assert r.json().get("data") == True


# -*- coding: utf-8 -*-
# © 2016 Oihane Crucelaegui - AvanzOSC
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from openerp.tests.common import TransactionCase

class TestProjectClaims(TransactionCase):

    def setUp(self):
        super(TestProjectClaims, self).setUp()
        self.claim_model = self.env['crm.claim']
        self.project = self.env['project.project'].create({
            'name': 'Test Project',
        })
        self.task = self.env['project.task'].create({
            'name': 'Test Task',
            'project_id': self.project.id,
        })
        self.task2 = self.env['project.task'].create({
            'name': 'Test Task 2',
        })
        self.claim = self.claim_model.create({
            'name': 'Test Claim',
        })

    def test_claim_onchange_project(self):
        claim = self.claim_model.new()
        claim.project_id = self.project
        claim.task_id = self.task2
        self.assertEquals(claim.task_id, self.task2)
        claim.onchange_project_id()
        self.assertFalse(claim.task_id)
        self.assertEquals(claim.project_id, self.project)

    def test_claim_onchange_task(self):
        claim = self.claim_model.new()
        claim.task_id = self.task
        claim.onchange_task_id()
        self.assertEqual(claim.project_id, self.project)
        self.assertEquals(claim.task_id, self.task)

    def test_claim_write_project(self):
        self.claim.write({
            'project_id': self.project.id,
        })
        self.assertEquals(self.claim.ref, self.project)

    def test_claim_write_task(self):
        self.claim.write({
            'project_id': self.task.project_id.id,
            'task_id': self.task.id,
        })
        self.assertEquals(self.claim.ref, self.task)

    def test_claim_write_ref_project(self):
        self.claim.ref = self.project
        self.assertEquals(self.claim.project_id, self.project)
        self.assertEquals(self.project.claim_count,
                          len(self.project.claim_ids))
        self.assertEquals(self.project.claim_count, 1)

    def test_claim_write_ref_task(self):
        self.claim.ref = self.task
        self.assertEquals(self.claim.task_id, self.task)
        self.assertEquals(self.task.claim_count,
                          len(self.task.claim_ids))
        self.assertEquals(self.task.claim_count, 1)
        self.assertEquals(self.claim.project_id, self.task.project_id)
        self.assertEquals(self.claim.project_id, self.project)
        self.assertEquals(self.project.claim_count,
                          len(self.project.claim_ids))
        self.assertEquals(self.project.claim_count, 1)


# Recursive Functions -  a functon that call itself is known as recursive function
# a function that calls itself is known as recursive function.

'''Eg : factorial(3)= 3 * factorial(2)
                 = 3 * 2 * factorial(1)
                 = 3 * 2 * 1 * factorial(0)
                 = 3 * 2 * 1 * 0
                 = 6

                 factorial(n) = n * factorial(n-1)
The Main Advantages of recursive function are
1. we can reduce lenth of code and improves readablity.
2. we can solve complex problems very easily.
'''
# Q) Write a Python Function to find Factorial of given Number with Recursion

def factorial(n):
    if n==0:
        result=1
    else:
        result= n * factorial(n-1)
    return result
print('factorial of 3 is',factorial(3))
print('factorial of 4 is',factorial(4))


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 27 14:28:46 2020

@author: Three
"""

# Definition for singly-linked list.
class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None
class Solution:
    def reverseList(self, head):
        stack = []
        while head is not None:
            stack.append(head.val)
            head = head.next
        if not stack: # 异常情况要多多考虑
            return None
        res = init = ListNode(stack.pop()) # 学会这种next下去的方法
        while stack:
            res.next = ListNode(stack.pop()) # 一步步next
            res = res.next
        return init

input = [1,2,3,4,5]
head2 = head = ListNode(input[0])
for i in range(1,len(input)):
    head2.next = ListNode(input[i])
    head2 = head2.next
res = Solution()
print(res.reverseList(head))

# 思路就是先把数据倒进堆栈，再利用LIFO放进链表
# 但是放进链表比输出难，因为放入的时候要考虑下一个next不是none，不然报错
# 但是输出的时候不存在这种问题


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 23 18:53:47 2023

@author: jenskremkow
"""
# %%
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rcParams

# %%

# experiment_name = '2022-12-19_16-00-17'
experiment_name = '2022-12-19_18-53-46'
# experiment_name = '2022-12-20_15-49-59'
# experiment_name = '2022-12-20_16-51-54'
# experiment_name = '2022-12-22_13-52-57'
# experiment_name = '2022-12-22_16-24-43'

data_dir = 'data/' + experiment_name + '/'

fn_pupil = data_dir + 'pupil_dlc_data.npy'
fn_ttl_cont = data_dir + 'opto_ttl_cont.npy'

data_pupil = np.load(fn_pupil, encoding='latin1', allow_pickle=True).item()
data_opto = np.load(fn_ttl_cont, encoding='latin1', allow_pickle=True).item()

# %%
opto_right_analog = data_opto['ttl_opto_right']
opto_left_analog = data_opto['ttl_opto_left']

# %% event triggered average
opto_right_tmp = opto_right_analog
diff_ttl = np.diff(opto_right_tmp)
gaps_idx = np.where(diff_ttl > 50000)[0]

opto_right_starts = np.append(opto_right_tmp[0], opto_right_tmp[gaps_idx+1])

opto_left_tmp = opto_left_analog
diff_ttl = np.diff(opto_left_tmp)
gaps_idx = np.where(diff_ttl > 50000)[0]

opto_left_starts = np.append(opto_left_tmp[0], opto_left_tmp[gaps_idx+1])

if experiment_name == '2022-12-22_16-24-43':
    opto_left_starts = np.delete(opto_left_starts, 12)

timestamps_cont = data_opto['timestamps']
cont_opto_right = data_opto['cont_opto_right']
cont_opto_left = data_opto['cont_opto_left']

# %%
channel_states = np.load(data_dir+'TTL_1/channel_states.npy')
timestamps = np.load(data_dir+'TTL_1/timestamps.npy')

events = {}

for i in range(8):
    events[i] = timestamps[channel_states == i]

# %% video ttls

video_ttls_tmp = events[2]

diff_ttl = np.diff(video_ttls_tmp)
gaps_idx = np.where(diff_ttl > 5000)[0]

video_starts = np.append(video_ttls_tmp[0], video_ttls_tmp[gaps_idx+1])
video_stops = np.append(video_ttls_tmp[gaps_idx], video_ttls_tmp[-1])

video_ttls = {}
for i in range(len(video_starts)):
    idx = (video_ttls_tmp >= video_starts[i]) & (
        video_ttls_tmp <= video_stops[i])
    video_ttls[i] = video_ttls_tmp[idx]
    print(video_ttls[i].shape)
    # print(left_pupil_data[i]['com_x'].shape)

if experiment_name == '2022-12-20_15-49-59':
    # we need to skip the first video, it is short and probbably wrong
    video_ttls[0] = video_ttls[1]
    video_ttls[1] = video_ttls[2]
    video_ttls.pop(2)

plt.figure(8)
plt.clf()

ts = video_ttls_tmp
plt.plot(ts, np.ones(ts.shape)*i, '.')

ts = video_starts
plt.plot(ts, np.ones(ts.shape)*i, '.m')

ts = video_stops
plt.plot(ts, np.ones(ts.shape)*i, '.c')

for vid in video_ttls.keys():
    ts = video_ttls[vid]

    plt.plot(ts, np.ones(ts.shape)*3, '.')

ts = opto_left_starts
plt.plot(ts, np.ones(ts.shape)*4, '.')

ts = opto_right_starts
plt.plot(ts, np.ones(ts.shape)*5, '.')

ts = opto_left_starts[12]
plt.plot(ts, np.ones(ts.shape)*4, 'o')

# %%
# %%
right_pupil_data = data_pupil['right_pupil_data']
left_pupil_data = data_pupil['left_pupil_data']

thr = 0.9
com_x_re_ma = {}
com_x_le_ma = {}

com_y_re_ma = {}
com_y_le_ma = {}

x_rp_ma = {}
x_lp_ma = {}
x_nose_ma = {}

for vid in video_ttls.keys():
    # right eye
    com_x = right_pupil_data[vid]['com_x']
    com_y = right_pupil_data[vid]['com_y']
    com_p = right_pupil_data[vid]['com_p']

    mask = com_p <= thr
    com_x_re_ma[vid] = np.ma.masked_array(com_x, mask)
    com_y_re_ma[vid] = np.ma.masked_array(com_y, mask)

    # left eye
    com_x = left_pupil_data[vid]['com_x']
    com_y = left_pupil_data[vid]['com_y']
    com_p = left_pupil_data[vid]['com_p']

    mask = com_p <= thr
    com_x_le_ma[vid] = np.ma.masked_array(com_x, mask)
    com_y_le_ma[vid] = np.ma.masked_array(com_y, mask)

# %%

events_to_average = {'opto_right': opto_right_starts,
                     'opto_left': opto_left_starts}

com_x_events_re_all_events = {}
com_x_events_le_all_events = {}
com_y_events_re_all_events = {}
com_y_events_le_all_events = {}

pixy_angle_events = {}
pixy_movement_events = {}

opto_right_cont_events = {}
opto_left_cont_events = {}

x_lp_all_events = {}
x_rp_all_events = {}
x_nose_all_events = {}

for i, key in enumerate(events_to_average):

    #    ts = events[i]
    # ts = opto_right_starts
    # ts = opto_left_starts
    # ts = opto_left_starts
    ts = events_to_average[key]

    start_s = -2
    stop_s = 15
    frames = np.arange(start_s*150, stop_s*150, 1)
    n_frames = len(frames)
    com_x_events_re = np.ma.zeros([n_frames, len(ts)])
    com_x_events_le = np.ma.zeros([n_frames, len(ts)])
    com_y_events_re = np.ma.zeros([n_frames, len(ts)])
    com_y_events_le = np.ma.zeros([n_frames, len(ts)])

    x_events_rp = np.ma.zeros([n_frames, len(ts)])
    x_events_lp = np.ma.zeros([n_frames, len(ts)])
    x_events_nose = np.ma.zeros([n_frames, len(ts)])

    for j, ttl in enumerate(ts):
        print(ttl)

        # we have to go through the videos
        for vid in video_ttls.keys():
            video_ttls_tmp = video_ttls[vid]

            search_idx = np.searchsorted(video_ttls_tmp, ttl)

            if search_idx == 0:
                continue
            if search_idx == len(video_ttls_tmp):
                continue
            print(str(vid)+' - '+str(search_idx))

            # we cutout data
            frames_tmp = frames + search_idx

            com_x_events_re[:, j] = com_x_re_ma[vid][frames_tmp]
            com_x_events_le[:, j] = com_x_le_ma[vid][frames_tmp]
            com_y_events_re[:, j] = com_y_re_ma[vid][frames_tmp]
            com_y_events_le[:, j] = com_y_le_ma[vid][frames_tmp]

            # x_events_rp[:,j] = x_rp_ma[vid][frames_tmp]
            # x_events_lp[:,j] = x_lp_ma[vid][frames_tmp]
            # x_events_nose[:,j] = x_nose_ma[vid][frames_tmp]

    com_x_events_re_all_events[key] = com_x_events_re
    com_x_events_le_all_events[key] = com_x_events_le
    com_y_events_re_all_events[key] = com_y_events_re
    com_y_events_le_all_events[key] = com_y_events_le

    x_lp_all_events[key] = x_events_lp
    x_rp_all_events[key] = x_events_rp
    x_nose_all_events[key] = x_events_nose

    # # pixy stuff
    sr_raw = 30000
    sr_subsampled = np.int64(1/(np.mean(np.diff(timestamps_cont))/sr_raw))
    resolution_s = 0.001
    resolution = np.int64(resolution_s * sr_subsampled)
    data_idxs = np.arange(start_s*sr_subsampled,
                          stop_s*sr_subsampled, resolution)
    n_data = len(data_idxs)

    # pixy_angle_tmp = np.ma.zeros([n_data,len(ts)])
    # pixy_movement_tmp = np.ma.zeros([n_data,len(ts)])
    opto_right_cont_tmp = np.ma.zeros([n_data, len(ts)])
    opto_left_cont_tmp = np.ma.zeros([n_data, len(ts)])

    for j, ttl in enumerate(ts):
        # print(ttl)
        ttl_cont_index = np.searchsorted(timestamps_cont, ttl)

        # we cutout data
        data_idxs_tmp = data_idxs + ttl_cont_index
        # pixy_angle_tmp[:,j] = pixy_angle[data_idxs_tmp]
        # pixy_movement_tmp[:,j] = pixy_movement[data_idxs_tmp]

        opto_right_cont_tmp[:, j] = cont_opto_right[data_idxs_tmp]
        opto_left_cont_tmp[:, j] = cont_opto_left[data_idxs_tmp]

    # pixy_angle_events[key] = pixy_angle_tmp
    # pixy_movement_events[key] = pixy_movement_tmp

    opto_right_cont_events[key] = opto_right_cont_tmp
    opto_left_cont_events[key] = opto_left_cont_tmp

data_time_ms = data_idxs  # / 30.

# %%
video_sr = 1./150
frames_s = frames * video_sr
frames_ms = frames_s * 1000

idx_onset_values = (frames_ms > -20) & (frames_s <= 0)

# %
fig = plt.figure(524)
# fig.set_size_inches(7,10)
fig.set_size_inches(8, 7)
plt.clf()

key = 'opto_left'
# right eye, opto left
ax = plt.subplot(3, 2, 1)
tmp = com_x_events_re_all_events[key]
for j in range(tmp.shape[1]):
    y = tmp[:, j].copy()
    y -= y[idx_onset_values].mean()
    plt.plot(frames_ms, y+j*10)

# plt.plot(frames_ms,np.mean(tmp,1),'k',lw=2)
plt.title('Right Eye, Left Opto')
plt.plot([0, 0], plt.ylim(), 'k:')
plt.ylabel('Eye X position')

# left eye, opto left
plt.subplot(3, 2, 2, sharex=ax, sharey=ax)
tmp = com_x_events_le_all_events[key]
for j in range(tmp.shape[1]):
    y = tmp[:, j].copy()
    y -= y[idx_onset_values].mean()
    plt.plot(frames_ms, y+j*10)
# plt.plot(frames_ms,np.mean(tmp,1),'k',lw=2)
plt.title('Left Eye, Left Opto')
plt.plot([0, 0], plt.ylim(), 'k:')
plt.ylabel('Eye X position')

key = 'opto_right'
# right eye, opto right
plt.subplot(3, 2, 3, sharex=ax, sharey=ax)
tmp = com_x_events_re_all_events[key]
for j in range(tmp.shape[1]):
    y = tmp[:, j].copy()
    y -= y[idx_onset_values].mean()
    plt.plot(frames_ms, y+j*10)
# plt.plot(frames_ms,np.mean(tmp,1),'k',lw=2)
plt.title('Right Eye, Right Opto')
plt.plot([0, 0], plt.ylim(), 'k:')
plt.ylabel('Eye X position')

# left eye, opto right
plt.subplot(3, 2, 4, sharex=ax, sharey=ax)
tmp = com_x_events_le_all_events[key]
for j in range(tmp.shape[1]):
    y = tmp[:, j].copy()
    y -= y[idx_onset_values].mean()
    plt.plot(frames_ms, y+j*10)
plt.title('Left Eye, Right Opto')
plt.plot([0, 0], plt.ylim(), 'k:')

plt.ylabel('Eye X position')
plt.ylim([-50, tmp.shape[1]*12])

plt.subplot(3, 2, 5, sharex=ax)
key = 'opto_left'
# right eye, opto left
tmp = com_x_events_re_all_events[key]
tmp_mean = np.mean(tmp, 1)
tmp_mean -= tmp_mean[idx_onset_values].mean()
plt.plot(frames_ms, tmp_mean)

key = 'opto_right'
# right eye, opto right
tmp = com_x_events_re_all_events[key]
tmp_mean = np.mean(tmp, 1)
tmp_mean -= tmp_mean[idx_onset_values].mean()
plt.plot(frames_ms, tmp_mean)
plt.xlabel('Time (ms)')
plt.legend(['Opto left', 'Opto right'])
plt.title('Right Eye Mean')
plt.ylabel('Temporal - Nasal')
plt.plot([0, 0], plt.ylim(), 'k:')

plt.subplot(3, 2, 6, sharex=ax)
key = 'opto_left'
# left eye, opto left
tmp = com_x_events_le_all_events[key]
tmp_mean = np.mean(tmp, 1)
tmp_mean -= tmp_mean[idx_onset_values].mean()
plt.plot(frames_ms, tmp_mean)

key = 'opto_right'
# left eye, opto right
tmp = com_x_events_le_all_events[key]
tmp_mean = np.mean(tmp, 1)
tmp_mean -= tmp_mean[idx_onset_values].mean()
plt.plot(frames_ms, tmp_mean)

plt.legend(['Opto left', 'Opto right'])

plt.xlabel('Time (ms)')
plt.title('Left Eye Mean')
plt.ylabel('Nasal - Temporal')

plt.xlim([-1000, 5000])
plt.plot([0, 0], plt.ylim(), 'k:')

fig.tight_layout()
plt.draw()

plt.savefig(experiment_name+'_all_trial_left_right_opto.png', dpi=200)

# %%
data_time_ms = data_idxs  # < / 30.

video_sr = 1./150
frames_s = frames * video_sr
frames_ms = frames_s * 1000

idx_onset_values = (frames_ms > -20) & (frames_s <= 0)

key = 'opto_left'
# key = 'opto_right'

for key in ['opto_left', 'opto_right']:

    n_trials = opto_right_cont_events[key].shape[1]

    # trial = 27

    for trial in range(n_trials):
        idx_onset_values_frames = (frames_ms > -20) & (frames_s <= 0)
        idx_onset_values_cont = (data_time_ms > -20) & (data_time_ms <= 0)

        fig = plt.figure(212)
        fig.set_size_inches(5, 7)
        plt.clf()

        ax = plt.subplot(3, 1, 1)
        tmp = opto_right_cont_events[key][:, trial]
        plt.plot(data_time_ms, tmp, 'm')
        tmp = opto_left_cont_events[key][:, trial]
        plt.plot(data_time_ms, tmp, 'g')

        plt.legend(['opto right', 'opto left'])
        plt.title(experiment_name+', '+key + ' trial: '+str(trial))

        # tmp = com_x_events_re_all_events[key][:,trial]
        # tmp -= tmp[idx_onset_values_frames].mean()
        # plt.plot(frames_ms,tmp,'b')
        # plt.title(key + ' trial: '+str(trial))
        # plt.ylim([-30,30])
        # plt.plot([0,0],plt.ylim(),'k:')
        # plt.ylabel('Pupil x')

        plt.subplot(3, 1, 2, sharex=ax)
        tmp = com_x_events_le_all_events[key][:, trial]
        tmp -= tmp[idx_onset_values_frames].mean()
        plt.plot(frames_ms, tmp, 'r')
        tmp = com_x_events_re_all_events[key][:, trial]
        tmp -= tmp[idx_onset_values_frames].mean()
        plt.plot(frames_ms, tmp, 'b')
        plt.ylim([-50, 50])
        plt.plot([0, 0], plt.ylim(), 'k:')
        plt.ylabel('Pupil x')
        plt.legend(['LE', 'RE'])

        plt.subplot(3, 1, 3, sharex=ax)
        tmp = com_y_events_le_all_events[key][:, trial]
        tmp -= tmp[idx_onset_values_frames].mean()
        plt.plot(frames_ms, tmp, 'r')
        tmp = com_y_events_re_all_events[key][:, trial]
        tmp -= tmp[idx_onset_values_frames].mean()
        plt.plot(frames_ms, tmp, 'b')
        # plt.title(key + ' trial: '+str(trial))
        plt.ylim([-50, 50])
        plt.plot([0, 0], plt.ylim(), 'k:')
        plt.ylabel('Pupil y')
        plt.legend(['LE', 'RE'])

        # plt.subplot(5,1,4,sharex=ax)
        # tmp = pixy_movement_events[key][:,trial]
        # tmp -= tmp[idx_onset_values].mean()
        # plt.plot(data_time_ms,tmp,color=[0.5,0.5,0.5])
        # plt.ylim([-5000,5000])
        # plt.plot([0,0],plt.ylim(),'k:')
        # plt.ylabel('Pixy Movement')

        # plt.subplot(5,1,5,sharex=ax)
        # tmp = pixy_angle_events[key][:,trial]
        # tmp -= tmp[idx_onset_values].mean()
        # plt.plot(data_time_ms,tmp,color=[0.5,0.5,0.5])
        # plt.plot([0,0],plt.ylim(),'k:')
        # plt.ylabel('Pixy Angle')

        plt.xlabel('Time (ms)')

        plt.xlim([-1000, 5000])

        fig.tight_layout()
        plt.draw()

        plt.savefig('figures/'+experiment_name+'_' + key +
                    '_trial_' + str(trial).zfill(3) + '.png', dpi=200)


from karel.stanfordkarel import *

"""
File: MidpointKarel.py
----------------------
When you finish writing it, MidpointKarel should
leave a beeper on the corner closest to the center of 1st Street
(or either of the two central corners if 1st Street has an even
number of corners).  Karel can put down additional beepers as it
looks for the midpoint, but must pick them up again before it
stops.  The world may be of any size, but you are allowed to
assume that it is at least as tall as it is wide.
"""

"""
---------------------------- LOGIC ----------------------------
In this code, Karel puts beepers alternately from both the sides until the entire row is filled with beepers. After
that, it puts an extra beeper on the middle corner. Then Karel moves to any one of the end points and picks the beepers
of the entire row, again alternatively from both sides. As there are two beepers on the centre corner, after picking up
all the beepers, effectively there will be one beeper present at the centre corner if it is an odd-sided world. For
worlds of even side, Karel puts a beeper at the end as all beepers get picked up in the above process. No commands other
than the basic Karel commands and conditions used in this program.
"""
def main():

    put_beepers_on_end_points() # Put one beeper on both the end points.

    while(no_beepers_present()):   # To complete the entire filling up of the row. If this while() block is not
        put_beeper_alternately() # present, and only put_beepers_alternately() is present, it will put beepers
                                   # only on two opposite ends of the row, other middle blocks i.e effectively the
                                   # middle block will be without a beeper.

    put_extra_beeper_on_the_centre_corner() # Put one extra beeper at the centre of the row.

    pick_beepers_from_end_points() # Pick one beeper from both the end points.

    if(front_is_clear()):
        while(beepers_present()):       # The while loop logic is similar to the above while loop.
            pick_beeper_alternately() # Picks up beepers alternatively from both the end points.

    odd_even_world_size_equalling() # For even sized world all beepers get picked up so 1 extra beeper needs to be put.

"""
Pre-condition: None.
Post-condition: It equalizes the required condition of Karel to be on a beeper at the midpoint for odd or even world.
"""
def odd_even_world_size_equalling(): # If odd world, do nothing; if even world place a beeper for size of world > 2.
    round_about()
    if beepers_present(): # This (if block) is executed for ODD LENGTH OF WORLD, i.e after picking beepers up, Karel
        pass              # ends up on the required(middle) beeper only.

    else:                       # This (else block) is executed for EVEN LENGTH OF WORLD, because it picks all beepers.

        if front_is_blocked():  # This (if block) is specially for world of (2 * N), as it has wall nearer to it and to
            round_about()       # lay beeper on only one of the middle corner(either 1st or 2nd).
        else:
            put_beeper()        # This (else block) is for all even worlds greater than 2*2, as in even world Karel ends
                                # up at any one of the middle corner but picking all the beepers laid so one beeper
                                # should be put.

"""
Pre-condition: None.
Post-condition: Karel puts one extra beeper on the centre corner.
"""
def put_extra_beeper_on_the_centre_corner():
    round_about()      # The put_beeper() before move_ahead() in the function is for placing 2 beepers on the middle
    put_beeper()       # corner, so that when all beepers are picked up, there would be effectively one beeper on the
    move_ahead()       # centre corner.
    turn_around()

"""
Pre-condition: None
Post-condition: Karel puts one beeper on both the end points of the row and takes a U-turn.
"""
def put_beepers_on_end_points(): # Put beepers on end-points of the row.
    put_beeper()
    move_ahead()
    put_beeper()
    round_about()

"""
Pre-condition: There should be at least one beeper on both the end-points of the row.
Post-condition: Karel picks one beeper from both the end points of the row and takes a U-turn.
"""
def pick_beepers_from_end_points(): # Pick beepers from the end-points of the row.
    pick_beeper()
    move_ahead()
    pick_beeper()
    round_about()

"""
Pre-condition: No beepers should be present on the initial starting block.
Post-condition: Karel puts beepers in a row alternatively for each end only once. To use this function to fill up the
                entire row, include this function in a while loop as written in the main func.
"""
def put_beeper_alternately(): # Karel puts beepers alternately from either side only once.
    while no_beepers_present():
        move()
    round_about() # For pre and post condition of put_beeper_alternatively() to be valid, when in a while loop.
    put_beeper()
    move()

"""
Pre-condition: Beepers should be present at then initial starting block.
Post-condition: Karel picks beepers in a row alternatively for each end only once. To use this function to pick beepers
                from the entire row, include this function in a while loop as written in the main func.
"""
def pick_beeper_alternately(): # Karel picks beepers alternatively from either side only once.
    while beepers_present():
        move()
    round_about() # For pre and post condition of pick_beeper_alternatively() to be valid, when in a while loop.
    pick_beeper()
    move()

"""
Pre-condition: None.
Post-condition: Karel faces opposite direction to whichever direction Karel was facing previously.
"""
def turn_around(): # Turn Karel in opposite direction at the same position.
    for i in range(2):
        turn_left()

"""
Pre-condition: Karel should be in a position to move 1 block backwards.
Post-condition: Karel takes a U-turn and moves one block forward.(i.e. effectively one block backwards)
"""
def round_about(): # Take a U-turn and move 1 block ahead if possible.
    turn_around()
    if front_is_clear():
        move()

"""
Pre-condition: None
Post-condition: Karel moves ahead in the initial direction it is facing, as long as its front is clear.
"""
def move_ahead(): # Move till front is clear.
    while front_is_clear():
        move()

# There is no need to edit code beyond this point

if __name__ == "__main__":
    run_karel_program()


"""Version information about documenteer."""

__all__ = ["__version__", "version_info"]

from importlib.metadata import PackageNotFoundError, version

__version__: str
"""The version string of Documenteer (PEP 440 / SemVer compatible)."""

try:
    __version__ = version("documenteer")
except PackageNotFoundError:
    # package is not installed
    __version__ = "0.0.0"

version_info = __version__.split(".")
"""The decomposed version, split across "``.``."

Use this for version comparison.
"""