"""DICOM cardiac MRI image training pipeline.

Usage:
  dicompipeline [--log <level>] (--data-dir <data_dir>)
  dicompipeline (-h | --help)
  dicompipeline --version

Options:
  --log <level>          Specify the log level to use, one of "info",
                         "warning", or "debug".
                         intermediate files are generated.
  --data-dir <data_dir>  Use the given data directory for the source data set.
  -h --help              Show this screen.
  --version              Show version.

Exit Codes:
  0 if no errors occurred.
  1 on user error.
  2 on an unexpected error, e.g. lack of memory, disk, bug, etc.
"""
import asyncio
import logging
import os
import sys

from docopt import docopt
from dicompipeline.dataset import Dataset
from dicompipeline.pipeline import Pipeline
from dicompipeline.version import get_version
from traceback import format_exc

def main(argv=None):
  if argv is None:
    # When not invoked by tests or from code, get argv from how we were
    # invoked on the command-line.
    from sys import argv

  arguments = docopt(__doc__,
                     version=get_version(),
                     options_first=True,
                     help=True,
                     argv=argv[1:])

  log_level = arguments["--log"]
  if log_level is None:
    log_level = "info"
  numeric_level = getattr(logging, log_level.upper(), None)
  if not isinstance(numeric_level, int):
    logging.error("Invalid log level {}".format(logLevel))
    sys.exit(1)
  logging.basicConfig(level=numeric_level)

  data_dir = arguments["--data-dir"]
  if not os.path.isdir(data_dir):
    # Note: docopt ensures that if we are here then "data_dir" is not None
    # because "--data-dir" is mandatory per the docstring.
    logging.error("The specified data directory '{}' does not exist.".format(data_dir))
    sys.exit(1)

  try:
    dicom_dir = os.path.join(data_dir, "dicoms")
    i_contour_dir = os.path.join(data_dir, "contourfiles")
    links_filename = os.path.join(data_dir, "link.csv")

    dataset = Dataset.load_dataset(
      dicom_dir,
      i_contour_dir,
      links_filename)

    if dataset.size() == 0:
      logging.error("No input images and contour masks were found in the data directory.")
      logging.error("This could happen if no contour files match any of the DICOM files even if there are images and contour files in the data directory.")
      sys.exit(1)

    loop = asyncio.get_event_loop()
    pipeline = Pipeline(dataset, loop=loop)
    pipeline.train()

    sys.exit(0)
  except Exception as e:
    logging.error("An unexpected error occurred.")
    logging.error(str(e))
    logging.error(format_exc())
    sys.exit(2)


import sys

def progress_bar(current, total, prefix=''):

    len = 20

    filled_width = int(round(len * current / total))
    percentage = round(100.0 * current / total, 1)
    bar = '⬜' * filled_width + '⬛' * (len - filled_width)

    sys.stdout.write(f'\r{prefix} |{bar}| {percentage}%')
    sys.stdout.flush()

    if current == total:
        sys.stdout.write('\n')
        sys.stdout.flush()

# Test progress_bar
if __name__ == '__main__':
    import time

    for i in range(100):
        progress_bar(i + 1, 100, prefix=f'Progress: {i + 1} / 100')
        time.sleep(0.1)


# njit decorator
def njit(*args, **kwargs):
    try:
        import numba

        return numba.njit(*args, **kwargs)

    except:
        warning_msg = "".join(
            [
                "Could not import numba. ",
                "Install numba to use JITed implementations of backend ",
                "functions for speed up of baseline removal algorithms",
            ]
        )

        from warnings import warn

        warn(warning_msg)

        def no_decorator(fn):
            return fn

        return no_decorator


from aiogram import Bot, Dispatcher, executor, types
from config import token
from questions import *
start_text = '''Приветствую тебя в тесте
на знание языка программирования Python😁

Ты сможешь оценить свои знания и возможно расширить их🎓'''

balls = 0

bot = Bot(token)
dp = Dispatcher(bot)

start_markup = types.InlineKeyboardMarkup()
button1 = types.InlineKeyboardButton('Начать тест' , callback_data='старт')
start_markup.add(button1)

q1_markup = types.InlineKeyboardMarkup()
button1 = types.InlineKeyboardButton('Да' , callback_data='1-Да')
button2 = types.InlineKeyboardButton('Нет' , callback_data='1-Нет')
button3 = types.InlineKeyboardButton('Ошибку' , callback_data='1-Err')
q1_markup.add(button1,button2,button3)

q2_markup = types.InlineKeyboardMarkup()
button1 = types.InlineKeyboardButton('5' , callback_data='2-5')
button2 = types.InlineKeyboardButton('10' , callback_data='2-10')
button3 = types.InlineKeyboardButton('Ошибку' , callback_data='2-Err')
q2_markup.add(button1,button2,button3)

q3_markup = types.InlineKeyboardMarkup()
button0 = types.InlineKeyboardButton('True' , callback_data='3-f')
button1 = types.InlineKeyboardButton('34' , callback_data='3-f')
button2 = types.InlineKeyboardButton('45' , callback_data='3-t')
button3 = types.InlineKeyboardButton('весь список' , callback_data='3-f')
q3_markup.add(button0,button1)
q3_markup.add(button2,button3)

q4_markup = types.InlineKeyboardMarkup()
button0 = types.InlineKeyboardButton('ошибка' , callback_data='4-f')
button1 = types.InlineKeyboardButton('2' , callback_data='4-t')
button2 = types.InlineKeyboardButton('True' , callback_data='4-f')
button3 = types.InlineKeyboardButton('False' , callback_data='4-f')
q4_markup.add(button0,button1)
q4_markup.add(button2,button3)

q5_markup = types.InlineKeyboardMarkup()
button0 = types.InlineKeyboardButton('начнется вечный цикл', callback_data='5-t')
button1 = types.InlineKeyboardButton('4', callback_data='5-f')
button2 = types.InlineKeyboardButton('5', callback_data='5-f')
button3 = types.InlineKeyboardButton('3', callback_data='5-f')
q5_markup.add(button0)
q5_markup.add(button1,button2,button3)

finish_markup = types.InlineKeyboardMarkup()
button0 = types.InlineKeyboardButton('Начать заново', callback_data='гг')
finish_markup.add(button0)

@dp.message_handler(commands=['start'])
async def hello(message: types.Message):
    await bot.send_message(message.chat.id, start_text ,parse_mode='HTML', reply_markup=start_markup)

@dp.callback_query_handler()
async def callback(callback:types.CallbackQuery):
    global balls
    if callback.data == 'старт':
        await callback.answer('START')
        await callback.bot.send_message(callback.message.chat.id , question1 , reply_markup=q1_markup, parse_mode='HTML')
    elif callback.data[0] == '1':
        if callback.data == '1-Да':
            await callback.answer('+ балл 🔥')
            balls+=1
        else:
            await callback.answer('Ответ не правильный🫤')
        await callback.bot.send_message(callback.message.chat.id, question2, reply_markup=q2_markup, parse_mode='HTML')
    elif callback.data[0] == '2':
        if callback.data == '2-10':
            await callback.answer('+ балл 🔥')
            balls+=1
        else:
            await callback.answer('Ответ не правильный🫤')
        await callback.bot.send_message(callback.message.chat.id, question3, reply_markup=q3_markup, parse_mode='HTML')
    elif callback.data[0] == '3':
        if callback.data[2] == 't':
            await callback.answer('+ балл 🔥')
            balls+=1
        else:
            await callback.answer('Ответ не правильный🫤')
        await callback.bot.send_message(callback.message.chat.id, question4, reply_markup=q4_markup, parse_mode='HTML')
    elif callback.data[0] == '4':
        if callback.data[2] == 't':
            await callback.answer('+ балл 🔥')
            balls += 1
        else:
            await callback.answer('Ответ не правильный🫤')
        await callback.bot.send_message(callback.message.chat.id, question5, reply_markup=q5_markup, parse_mode='HTML')
    elif callback.data[0] == '5':
        if callback.data[2] == 't':
            await callback.answer('+ балл 🔥')
            balls += 1
        else:
            await callback.answer('Ответ не правильный🫤')
        await callback.bot.send_message(callback.message.chat.id, f'ОГО, ты набрал целых {balls} баллов🤩\n Продолжай заниматься и тогда ты станешь отличным программистом👨🏫' ,
                                        parse_mode='HTML',
                                        reply_markup=finish_markup)
        balls = 0
    elif callback.data == 'гг':
        await callback.answer('START')
        await callback.bot.send_message(callback.message.chat.id, question1, reply_markup=q1_markup, parse_mode='HTML')
executor.start_polling(dp)


# _*_ coding:utf-8 _*_
import xml.etree.ElementTree as ET
import pickle
import os
from os import listdir, getcwd
from os.path import join
import cv2
import logging
from glob import glob
from glob import glob
from random import randint
import time

'''
YOLO v5
xml -> txt
'''
class2id = {"fire": 0,"big_fire":1,"smoke":2}

def convert(size, box):
    dw = 1. / (size[0])
    dh = 1. / (size[1])
    x = (box[0] + box[1]) / 2.0 - 1
    y = (box[2] + box[3]) / 2.0 - 1
    w = box[1] - box[0]
    h = box[3] - box[2]
    x = x * dw
    w = w * dw
    y = y * dh
    h = h * dh
    x = min(x, 1.0)
    y = min(y, 1.0)
    w = min(w, 1.0)
    h = min(h, 1.0)

    return (x, y, w, h)

def convert_annotation(image_path):
    in_file = open(image_path.replace('.xml', '.xml'), encoding="utf-8")
    out_file = open(image_path.replace('.xml', '.txt'), 'w')
    # print(in_file)
    tree = ET.parse(in_file)
    root = tree.getroot()
    size = root.find('size')
    w = int(size.find('width').text)
    h = int(size.find('height').text)

    # img = cv2.imread("./datasets/VOCdevkit/VOC2007/JPEGImages/"+image_id+".jpg")
    # sp = img.shape

    for obj in root.iter('object'):
        name = obj.find('name').text
        cls_id = class2id[name]
        xmlbox = obj.find('bndbox')
        b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),
             float(xmlbox.find('ymax').text))
        bb = convert((w, h), b, )
        out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')

if __name__ == '__main__':
    files=[]
    file = glob('/project/train/src_repo/trainval/*.xml')
    for i in file:
        files.append(i)
    file = glob('/project/train/src_repo/trainval/820/*.xml')
    for i in file:
        files.append(i)
    file = glob('/project/train/src_repo/trainval/599/*.xml')
    for i in file:
        files.append(i)
    for file in files:
        convert_annotation(file)

    files=[]
    file = glob('/project/train/src_repo/trainval/*.txt')
    for i in file:
        files.append(i)
    file = glob('/project/train/src_repo/trainval/820/*.txt')
    for i in file:
        files.append(i)
    file = glob('/project/train/src_repo/trainval/599/*.txt')
    for i in file:
        files.append(i)

    train = open("/project/train/src_repo/train.txt", 'w')
    val = open("/project/train/src_repo/val.txt", 'w')

    for i in files:
        num = randint(1, 10)
        name = i.replace('.txt', '.jpg')
        if num < 8:
            train.write(name + '\n')
        else:
            val.write(name + '\n')
    train.close()
    val.close()


from cctbx import crystal, miller, sgtbx
from cctbx.array_family import flex
import numpy as np
from itertools import izip

def get_val_at_hkl(hkl, val_map, sg=sgtbx.space_group(" P 4nw 2abw")):
    """
    given a miller index, find its value in the miller dictionary array (val_map)

    :param hkl:  tuple
    :param val_map:  miller data dictionary
    :param sg: space group
    :return: miller value
    """
    poss_equivs = [i.h() for i in miller.sym_equiv_indices(sg, hkl).indices()]

    in_map = False
    for hkl2 in poss_equivs:
        if hkl2 in val_map:  # fast lookup
            in_map = True
            break
    if in_map:
        return hkl2, val_map[hkl2]
    else:
        return (None, None, None), None

def generate_table(complex_data, indices, numpy_args=False, anom=True):
    """

    :param complex_data: structure factors
    :param indices: miller indices in a list, Nx3
    :param numpy_args: are the complex data and indices numpy type, if not
        assume flex
    :param anom: return a miller array with +H and -H ?
    :return: dictionary whose keys are miller index tuple and values are structure fact
    """
    sg = sgtbx.space_group(" P 4nw 2abw")
    Symm = crystal.symmetry(unit_cell=(79, 79, 38, 90, 90, 90), space_group=sg)
    if numpy_args:
        assert type(indices) == tuple
        assert (type(indices[0]) == tuple)
        indices = flex.miller_index(indices)

    mil_set = miller.set(crystal_symmetry=Symm, indices=indices, anomalous_flag=anom)
    if numpy_args:
        complex_data = flex.complex_double(np.ascontiguousarray(complex_data))
    mil_ar = miller.array(mil_set, data=complex_data)

    mil_dict = {h: val for h, val in izip(mil_ar.indices(), mil_ar.data())}
    return mil_dict


#
# @lc app=leetcode id=800 lang=python3
#
# [800] Similar RGB Color
#

# @lc code=start
class Solution:
    def similarRGB(self, color):
        """
        :type color: str
        :rtype: str
        """
        def getClosest(s):
            m = ['00','11','22','33','44','55','66','77','88','99','aa','bb','cc','dd','ee','ff']
            return min(m, key = lambda x: abs(int(x, 16)- int(s,16)))

        res = ''
        color = color[1:]
        for i in range(3):
            s = color[2*i:2*i+2]
            res += getClosest(s)
        return '#' +res

# @lc code=end


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# by @aetsu
import socket
import sys
import os
import signal
import random
import time
from multiprocessing import Process

interface = 'mon0'

def channel_hopper():
    """
    Channel hopper
    """
    while True:
        try:
            channel = random.randrange(1, 13)  # 13 canales legales en europa
            os.system("iw dev %s set channel %d" % (interface, channel))
            time.sleep(1)
        except KeyboardInterrupt:
            break

def signal_handler(signal, frame):
    """
    Captures CTRL+C interruption
    """
    p.terminate()
    p.join()
    sys.exit(0)

if __name__ == "__main__":
    p = Process(target=channel_hopper)
    p.start()

    signal.signal(signal.SIGINT, signal_handler)

    sniff = socket.socket(
        socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0003)
    )

    sniff.bind((interface, 0x0003))
    aps = []

    while True:
        fm = sniff.recvfrom(4069)[0]

        if fm[36] == "\x80":  # beacon frame x80
            ssidLen = ord(fm[73])  # SSID length
            if fm[46:52] not in aps and ssidLen > 0:
                aps.append(fm[46:52])  # mac is added to list

                print(
                    "ESSID: %s \t BSSID: %s \t CHANNEL: %d" %
                    (fm[74:74 + ssidLen],
                        fm[46:52].encode('hex'),
                        (ord(fm[86 + ssidLen])))
                )


from datalytics.data_analysis.models import ShortTermAnalyser, IndoorTemperatureAnalyser, HumphreyTemperatureAnalyser
from datalytics.data_analysis.data_unpackers import TupleDataUnpacker

from datalytics.settings_controller import settings

from datalytics.models import Room

class DataAnalyseInterface:
    unpacker_class = None

    def __init__(self, data_unpacker: TupleDataUnpacker=None, data_analyser:ShortTermAnalyser=None):
        self.data_unpacker = data_unpacker or self.unpacker_class()
        self.data_analysers = data_analyser

    def inform_real_time(self, measurement_data):
        if isinstance(self.data_analysers, list):
            analysers = self.data_analysers
        else:
            analysers = [self.data_analysers]

        unpacked_data = self.data_unpacker.unpack(measurement_data)

        for analyser in analysers:
            analyser.update(*unpacked_data)

class AnalyserFront:

    rooms = None
    highest_id = 0

    def __init__(self):
        self.rooms = {}

        # prep storage
        self.storage = settings.load('storage').interface
        self._load_rooms()

    def _load_rooms(self):
        for room in self.storage.rooms.load_all():
            self.highest_id = max(self.highest_id, room.id)
            self.rooms[str(room.id)] = room

            # Load in the measurement values
            measurements = self.storage.measurements.load_for_room(room)
            for m in measurements:
                try:
                    sensor = room.add_sensor(m['type'])
                except KeyError:
                    # Duplicates encountered, for some reason. So take that into account.
                    print(f"Duplicate measurement data has been detected on the server on {room.name} - {m['type']}")
                    sensor = room.sensors[m['type']]
                    if m['timestamp'] > sensor.last_update:
                        sensor.last_update = m['timestamp']
                        sensor.last_value = m['value']
                else:
                    sensor.last_update = m['timestamp']
                    sensor.last_value = m['value']
            # Load active messages and set them to the relevant analysers
            msg_dict = {}
            for msg in self.storage.messages.load_active_messages(room):
                msg_dict[msg.code] = msg

            for sensor in room.sensors.values():
                for analyser in sensor.analysers:
                    if analyser.code in msg_dict.keys():
                        analyser.active_message = msg_dict.pop(analyser.code)
            for key in msg_dict.keys():
                print(f"Code {key} could not be attributed to the correct analyser upon launch.")

    def add_room(self, room_type, name, sensor_types=None):
        """
        Adds a room to the system
        :param room_type: The type of room
        :param name: The name of the room
        :param sensor_types: The types of sensors active in the room
        :return:
        """
        room = Room(room_type, name, initial_sensor_types=sensor_types)
        new_id = self.highest_id + 1
        room.id = new_id
        self.rooms[str(new_id)] = room

        self.storage.rooms.add(room)

        return room

    def get_room(self, name=None, room_id=None, fall_back_as_id=False) -> Room:
        """
        Returns the room with the given name or id
        :param name: The name of the room (room_id should be None)
        :param room_id: The id  of the room (name should  be None)
        :param fall_back_as_id: search id if name does not yield results
        :return: The room or None
        """
        assert not (name and room_id)
        assert name or room_id
        if name:
            for room in self.rooms.values():
                if room.name == name:
                    return room
            if fall_back_as_id:
                return self.rooms.get(str(name))
            return None
        else:
            return self.rooms.get(str(room_id))

class TempDataAnalyser(DataAnalyseInterface):
    unpacker_class = TupleDataUnpacker
    analysis_classes = [IndoorTemperatureAnalyser, HumphreyTemperatureAnalyser]

    def __init__(self, external_temperature_data=None, **kwargs):
        super(TempDataAnalyser, self).__init__(**kwargs)

        self.data_analysers = [
            IndoorTemperatureAnalyser(),
            HumphreyTemperatureAnalyser(external_temperature_data),
        ]


import os
def myPattern(string1,string2,f1,f2):
    fin1 = open (f1)
    fin2 = open (f2,'w')
    for line in fin1:
        if string1 in line:
            flash=line.replace(string1,string2)
            fin2.write(flash)
        fin2.write(line)
    return fin2

myPattern('172','192','running-config.cfg','new-config.cfg')


"""
给你一个大小为 rows x cols 的矩阵 mat，其中 mat[i][j] 是 0 或 1，请返回 矩阵 mat 中特殊位置的数目 。

特殊位置 定义：如果 mat[i][j] == 1 并且第 i 行和第 j 列中的所有其他元素均为 0（行和列的下标均 从 0 开始 ），则位置 (i, j) 被称为特殊位置。
"""

def numSpecial(mat):
    matchrow = []
    matchcol = []
    row, col = len(mat), len(mat[0])
    for i in range(row):
        if sum(mat[i]) == 1:
            matchrow.append(i)
    for i, j in enumerate(zip(*mat)):
        if sum(j) == 1:
            matchcol.append(i)
    res = 0
    for i in matchrow:
        for j in matchcol:
            if mat[i][j] == 1:
                res += 1
    return res

print(numSpecial(mat = [[1,0,0],
            [0,0,1],
            [1,0,0]]))
print(numSpecial(mat = [[1,0,0],
            [0,1,0],
            [0,0,1]]))
print(numSpecial(mat = [[0,0,0,1],
[1,0,0,0],
[0,1,1,0],
[0,0,0,0]]))
print(numSpecial(mat = [[0,0,0,0,0],
[1,0,0,0,0],
[0,1,0,0,0],
[0,0,1,0,0],
[0,0,0,1,1]]))

1582. 二进制矩阵中的特殊位置.py

1582. 二进制矩阵中的特殊位置.py


import argparse
import json
import os
import sys

import numpy
import torch
from matplotlib import image as mImage

from cnn.versions.ADNet_v3 import ADNetV3

DEFAULT_MODEL_DIR = "pretrained_model"
DEFAULT_MODEL_NAME = "model.pt"

AD_NET_VERSIONS = {
    1: {
        'model': None,
        'description': '1th version'
    },
    2: {
        'model': None,
        'description': '2nd version'
    },
    3: {
        'model': ADNetV3(),
        'description': '3th version'
    }
}

def predict(pretrained_model_path,
            image_path,
            version):
    # do some checks
    if not os.path.exists(image_path):
        err = {'errorMessage': f'Image not found, path {image_path}'}
        print(json.dumps(err))
        exit(-1)
    if not os.path.exists(pretrained_model_path):
        err = {'errorMessage': f'Model not found, path {pretrained_model_path}'}
        print(json.dumps(err))
        exit(-1)
    if AD_NET_VERSIONS.get(version) is None:
        err = {'errorMessage': f'Version {version} does not exist. Available: [{list(AD_NET_VERSIONS.keys())}]'}
        print(json.dumps(err))
        exit(-1)

    final_json = []

    # load state
    net = AD_NET_VERSIONS.get(version)['model']
    loaded_state = torch.load(pretrained_model_path, map_location='cpu')['model_state']

    with torch.no_grad():
        net.load_state_dict(loaded_state)
        net.eval()

        if os.path.isdir(args.image):
            images = os.listdir(args.image)
            for idx, val in enumerate(images):
                output = proceed(image_path + '/' + val, net)
                interpret_prediction(output, val, final_json)
        else:
            output = proceed(image_path, net)
            interpret_prediction(output, image_path, final_json)

    print(json.dumps(final_json))

def proceed(path_to_image,
            net):
    img = mImage.imread(path_to_image, 'pgm')
    img_array = numpy.asarray(img, dtype='float32')
    img_array = numpy.expand_dims(img_array, axis=0)
    img_array = numpy.expand_dims(img_array, axis=0)
    img_array = torch.from_numpy(img_array)
    output = net(img_array)

    return output

def interpret_prediction(output,
                         image_name,
                         history):
    prediction = output.max(1, keepdim=True)
    prediction_tensor = prediction[1]

    cover_percentage, stego_percentage = output[0].numpy()
    if prediction_tensor[0].numpy()[0] == 0:
        history.append({
            'imageName': os.path.basename(image_name),
            'probability': f'{cover_percentage:.4f}/{stego_percentage:.4f}',
            'result': 'COVER'
        })
        # print(f'Image {image_name}\t[{cover_percentage:.4f}/{stego_percentage:.4f}] \tCOVER')
    else:
        history.append({
            'imageName': os.path.basename(image_name),
            'probability': f'{cover_percentage:.4f}/{stego_percentage:.4f}',
            'result': 'STEGO'
        })
        # print(f'Image {image_name}\t[{cover_percentage:.4f}/{stego_percentage:.4f}] \tSTEGO')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Get prediction for provided image.',
        usage="python ad_predict.py --image ./stego_lena.pgm",
    )
    parser.add_argument(
        '-i', '--image',
        help='path to image to check',
        required=True
    )
    parser.add_argument(
        '-v', '--version',
        help='ADNet cnn version',
        required=False,
        default=max(list(AD_NET_VERSIONS.keys()))
    )
    parser.add_argument(
        '-m', '--model',
        help='path to model',
        required=False,
        default=DEFAULT_MODEL_DIR + '/' + DEFAULT_MODEL_NAME
    )

    args = parser.parse_args()

    try:
        predict(args.model, args.image, args.version)
    except Exception as e:
        error = {'errorMessage': e.__str__()}
        sys.stderr.write(json.dumps(error))
        sys.stderr.flush()
        exit(-1)


#!/usr/bin/python3
"""
def roman_to_int(roman_string):
    from functools import reduce
    rd = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
    if roman_string == "":
        return 0
    if roman_string is None or type(roman_string) is not str:
        return 0
    match_list = list(roman_string.upper())
    match_values = list(map(lambda x: rd[x], match_list))
    i, j = 0, 1
    while j < len(match_values):
        if match_values[i] < match_values[j]:
            match_values[i] = match_values[i] * -1
        i += 1
        j += 1
    return reduce(lambda x, y: x + y if x > y else y - x, match_values)
"""

def roman_to_int(roman_string):
    from functools import reduce
    rd = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
    if not roman_string or not isinstance(roman_string, str):
        return 0

    match_val = list(map(lambda x: rd.get(x, 0), list(roman_string.upper())))
    i, j = 0, 1
    while j < len(match_val):
        if match_val[i] < match_val[j]:
            match_val[i] *= -1
        i += 1
        j += 1
    return reduce(lambda x, y: x + y, match_val)


import os
import glob
import shutil
import datetime
import numpy as np
import xml.etree.ElementTree as ET

import isce, isceobj
from isceobj.Constants import SPEED_OF_LIGHT
from isceobj.Alos2Proc.runSwathOffset import swathOffset
from isceobj.Alos2Proc.runFrameOffset import frameOffset
from isceobj.Alos2Proc.runIonSubband import defineIonDir

from StackPulic import loadTrack
from StackPulic import createObject
from StackPulic import stackDateStatistics
from StackPulic import acquisitionModesAlos2

def runIonSubband(self, referenceTrack, idir, dateReferenceStack, dateReference, dateSecondary):
    '''create subband interferograms
    '''
    #catalog = isceobj.Catalog.createCatalog(self._insar.procDoc.name)
    #self.updateParamemetersFromUser()

    #if not self.doIon:
    #    catalog.printToLog(logger, "runIonSubband")
    #    self._insar.procDoc.addAllFromCatalog(catalog)
    #    return

    #referenceTrack = self._insar.loadTrack(reference=True)
    #secondaryTrack = self._insar.loadTrack(reference=False)

    #using 1/3, 1/3, 1/3 band split
    radarWavelength = referenceTrack.radarWavelength
    rangeBandwidth = referenceTrack.frames[0].swaths[0].rangeBandwidth
    rangeSamplingRate = referenceTrack.frames[0].swaths[0].rangeSamplingRate
    radarWavelengthLower = SPEED_OF_LIGHT/(SPEED_OF_LIGHT / radarWavelength - rangeBandwidth / 3.0)
    radarWavelengthUpper = SPEED_OF_LIGHT/(SPEED_OF_LIGHT / radarWavelength + rangeBandwidth / 3.0)
    subbandRadarWavelength = [radarWavelengthLower, radarWavelengthUpper]
    subbandBandWidth = [rangeBandwidth / 3.0 / rangeSamplingRate, rangeBandwidth / 3.0 / rangeSamplingRate]
    subbandFrequencyCenter = [-rangeBandwidth / 3.0 / rangeSamplingRate, rangeBandwidth / 3.0 / rangeSamplingRate]

    subbandPrefix = ['lower', 'upper']

    '''
    ionDir = {
        ionDir['swathMosaic'] : 'mosaic',
        ionDir['insar'] : 'insar',
        ionDir['ion'] : 'ion',
        ionDir['subband'] : ['lower', 'upper'],
        ionDir['ionCal'] : 'ion_cal'
        }
    '''
    #define upper level directory names
    ionDir = defineIonDir()

    #self._insar.subbandRadarWavelength = subbandRadarWavelength

    ############################################################
    # STEP 1. create directories
    ############################################################
    #create and enter 'ion' directory
    #after finishing each step, we are in this directory
    os.makedirs(ionDir['ion'], exist_ok=True)
    os.chdir(ionDir['ion'])

    #create insar processing directories
    for k in range(2):
        subbandDir = ionDir['subband'][k]
        for i, frameNumber in enumerate(self._insar.referenceFrames):
            frameDir = 'f{}_{}'.format(i+1, frameNumber)
            for j, swathNumber in enumerate(range(self._insar.startingSwath, self._insar.endingSwath + 1)):
                swathDir = 's{}'.format(swathNumber)
                fullDir = os.path.join(subbandDir, frameDir, swathDir)
                os.makedirs(fullDir, exist_ok=True)

    #create ionospheric phase directory
    os.makedirs(ionDir['ionCal'], exist_ok=True)

    ############################################################
    # STEP 2. create subband interferograms
    ############################################################
    #import numpy as np
    #import stdproc
    #from iscesys.StdOEL.StdOELPy import create_writer
    #from isceobj.Alos2Proc.Alos2ProcPublic import readOffset
    #from contrib.alos2proc.alos2proc import rg_filter
    from StackPulic import formInterferogram

    for i, frameNumber in enumerate(self._insar.referenceFrames):
        frameDir = 'f{}_{}'.format(i+1, frameNumber)
        for j, swathNumber in enumerate(range(self._insar.startingSwath, self._insar.endingSwath + 1)):
            swathDir = 's{}'.format(swathNumber)

            #skip this time consuming process, if interferogram already exists
            if os.path.isfile(os.path.join(ionDir['subband'][0], frameDir, swathDir, self._insar.interferogram)) and \
               os.path.isfile(os.path.join(ionDir['subband'][0], frameDir, swathDir, self._insar.interferogram+'.vrt')) and \
               os.path.isfile(os.path.join(ionDir['subband'][0], frameDir, swathDir, self._insar.interferogram+'.xml')) and \
               os.path.isfile(os.path.join(ionDir['subband'][0], frameDir, swathDir, self._insar.amplitude)) and \
               os.path.isfile(os.path.join(ionDir['subband'][0], frameDir, swathDir, self._insar.amplitude+'.vrt')) and \
               os.path.isfile(os.path.join(ionDir['subband'][0], frameDir, swathDir, self._insar.amplitude+'.xml')) and \
               os.path.isfile(os.path.join(ionDir['subband'][1], frameDir, swathDir, self._insar.interferogram)) and \
               os.path.isfile(os.path.join(ionDir['subband'][1], frameDir, swathDir, self._insar.interferogram+'.vrt')) and \
               os.path.isfile(os.path.join(ionDir['subband'][1], frameDir, swathDir, self._insar.interferogram+'.xml')) and \
               os.path.isfile(os.path.join(ionDir['subband'][1], frameDir, swathDir, self._insar.amplitude)) and \
               os.path.isfile(os.path.join(ionDir['subband'][1], frameDir, swathDir, self._insar.amplitude+'.vrt')) and \
               os.path.isfile(os.path.join(ionDir['subband'][1], frameDir, swathDir, self._insar.amplitude+'.xml')):
                print('interferogram already exists at swath {}, frame {}'.format(swathNumber, frameNumber))
                continue

            # #filter reference and secondary images
            # for slcx in [self._insar.referenceSlc, self._insar.secondarySlc]:
            #     slc = os.path.join('../', frameDir, swathDir, slcx)
            #     slcLower = os.path.join(ionDir['subband'][0], frameDir, swathDir, slcx)
            #     slcUpper = os.path.join(ionDir['subband'][1], frameDir, swathDir, slcx)
            #     rg_filter(slc, 2,
            #         [slcLower, slcUpper],
            #         subbandBandWidth,
            #         subbandFrequencyCenter,
            #         257, 2048, 0.1, 0, 0.0)
            #resample
            for k in range(2):
                os.chdir(os.path.join(ionDir['subband'][k], frameDir, swathDir))
                slcReference = os.path.join('../../../../', idir, dateReference, frameDir, swathDir, dateReference+'_{}.slc'.format(ionDir['subband'][k]))
                slcSecondary = os.path.join('../../../../', idir, dateSecondary, frameDir, swathDir, dateSecondary+'_{}.slc'.format(ionDir['subband'][k]))
                formInterferogram(slcReference, slcSecondary, self._insar.interferogram, self._insar.amplitude, self._insar.numberRangeLooks1, self._insar.numberAzimuthLooks1)
                os.chdir('../../../')

    ############################################################
    # STEP 3. mosaic swaths
    ############################################################
    from isceobj.Alos2Proc.runSwathMosaic import swathMosaic
    from isceobj.Alos2Proc.Alos2ProcPublic import create_xml

    #log output info
    log  = 'mosaic swaths in {} at {}\n'.format(os.path.basename(__file__), datetime.datetime.now())
    log += '================================================================================================\n'

    for k in range(2):
        os.chdir(ionDir['subband'][k])
        for i, frameNumber in enumerate(self._insar.referenceFrames):
            frameDir = 'f{}_{}'.format(i+1, frameNumber)
            os.chdir(frameDir)

            mosaicDir = ionDir['swathMosaic']
            os.makedirs(mosaicDir, exist_ok=True)
            os.chdir(mosaicDir)

            if not (self._insar.endingSwath-self._insar.startingSwath >= 1):
                import shutil
                swathDir = 's{}'.format(referenceTrack.frames[i].swaths[0].swathNumber)

                # if not os.path.isfile(self._insar.interferogram):
                #     os.symlink(os.path.join('../', swathDir, self._insar.interferogram), self._insar.interferogram)
                # shutil.copy2(os.path.join('../', swathDir, self._insar.interferogram+'.vrt'), self._insar.interferogram+'.vrt')
                # shutil.copy2(os.path.join('../', swathDir, self._insar.interferogram+'.xml'), self._insar.interferogram+'.xml')
                # if not os.path.isfile(self._insar.amplitude):
                #     os.symlink(os.path.join('../', swathDir, self._insar.amplitude), self._insar.amplitude)
                # shutil.copy2(os.path.join('../', swathDir, self._insar.amplitude+'.vrt'), self._insar.amplitude+'.vrt')
                # shutil.copy2(os.path.join('../', swathDir, self._insar.amplitude+'.xml'), self._insar.amplitude+'.xml')

                os.rename(os.path.join('../', swathDir, self._insar.interferogram), self._insar.interferogram)
                os.rename(os.path.join('../', swathDir, self._insar.interferogram+'.vrt'), self._insar.interferogram+'.vrt')
                os.rename(os.path.join('../', swathDir, self._insar.interferogram+'.xml'), self._insar.interferogram+'.xml')
                os.rename(os.path.join('../', swathDir, self._insar.amplitude), self._insar.amplitude)
                os.rename(os.path.join('../', swathDir, self._insar.amplitude+'.vrt'), self._insar.amplitude+'.vrt')
                os.rename(os.path.join('../', swathDir, self._insar.amplitude+'.xml'), self._insar.amplitude+'.xml')

                #no need to update frame parameters here
                os.chdir('../')
                #no need to save parameter file here
                os.chdir('../')

                continue

            #choose offsets
            numberOfFrames = len(referenceTrack.frames)
            numberOfSwaths = len(referenceTrack.frames[i].swaths)
            # if self.swathOffsetMatching:
            #     #no need to do this as the API support 2-d list
            #     #rangeOffsets = (np.array(self._insar.swathRangeOffsetMatchingReference)).reshape(numberOfFrames, numberOfSwaths)
            #     #azimuthOffsets = (np.array(self._insar.swathAzimuthOffsetMatchingReference)).reshape(numberOfFrames, numberOfSwaths)
            #     rangeOffsets = self._insar.swathRangeOffsetMatchingReference
            #     azimuthOffsets = self._insar.swathAzimuthOffsetMatchingReference

            # else:
            #     #rangeOffsets = (np.array(self._insar.swathRangeOffsetGeometricalReference)).reshape(numberOfFrames, numberOfSwaths)
            #     #azimuthOffsets = (np.array(self._insar.swathAzimuthOffsetGeometricalReference)).reshape(numberOfFrames, numberOfSwaths)
            #     rangeOffsets = self._insar.swathRangeOffsetGeometricalReference
            #     azimuthOffsets = self._insar.swathAzimuthOffsetGeometricalReference

            # rangeOffsets = rangeOffsets[i]
            # azimuthOffsets = azimuthOffsets[i]

            #compute swath offset using reference stack
            #geometrical offset is enough now
            offsetReferenceStack = swathOffset(referenceTrack.frames[i], dateReference+'.slc', 'swath_offset_' + dateReference + '.txt',
                               crossCorrelation=False, numberOfAzimuthLooks=10)
            #we can faithfully make it integer.
            #this can also reduce the error due to floating point computation
            rangeOffsets = [float(round(x)) for x in offsetReferenceStack[0]]
            azimuthOffsets = [float(round(x)) for x in offsetReferenceStack[1]]

            #list of input files
            inputInterferograms = []
            inputAmplitudes = []
            #phaseDiff = [None]
            swathPhaseDiffIon = [self.swathPhaseDiffLowerIon, self.swathPhaseDiffUpperIon]
            phaseDiff = swathPhaseDiffIon[k]
            if swathPhaseDiffIon[k] is None:
                phaseDiff = None
            else:
                phaseDiff = swathPhaseDiffIon[k][i]
                phaseDiff.insert(0, None)

            for j, swathNumber in enumerate(range(self._insar.startingSwath, self._insar.endingSwath + 1)):
                swathDir = 's{}'.format(swathNumber)
                inputInterferograms.append(os.path.join('../', swathDir, self._insar.interferogram))
                inputAmplitudes.append(os.path.join('../', swathDir, self._insar.amplitude))

                # #compute phase needed to be compensated using startingRange
                # if j >= 1:
                #     #phaseDiffSwath1 = -4.0 * np.pi * (referenceTrack.frames[i].swaths[j-1].startingRange - secondaryTrack.frames[i].swaths[j-1].startingRange)/subbandRadarWavelength[k]
                #     #phaseDiffSwath2 = -4.0 * np.pi * (referenceTrack.frames[i].swaths[j].startingRange - secondaryTrack.frames[i].swaths[j].startingRange)/subbandRadarWavelength[k]
                #     phaseDiffSwath1 = +4.0 * np.pi * referenceTrack.frames[i].swaths[j-1].startingRange * (1.0/radarWavelength - 1.0/subbandRadarWavelength[k]) \
                #                       -4.0 * np.pi * secondaryTrack.frames[i].swaths[j-1].startingRange * (1.0/radarWavelength - 1.0/subbandRadarWavelength[k])
                #     phaseDiffSwath2 = +4.0 * np.pi * referenceTrack.frames[i].swaths[j].startingRange * (1.0/radarWavelength - 1.0/subbandRadarWavelength[k]) \
                #                       -4.0 * np.pi * secondaryTrack.frames[i].swaths[j].startingRange * (1.0/radarWavelength - 1.0/subbandRadarWavelength[k])
                #     if referenceTrack.frames[i].swaths[j-1].startingRange - secondaryTrack.frames[i].swaths[j-1].startingRange == \
                #        referenceTrack.frames[i].swaths[j].startingRange - secondaryTrack.frames[i].swaths[j].startingRange:
                #         #phaseDiff.append(phaseDiffSwath2 - phaseDiffSwath1)
                #         #if reference and secondary versions are all before or after version 2.025 (starting range error < 0.5 m),
                #         #it should be OK to do the above.
                #         #see results in neom where it meets the above requirement, but there is still phase diff
                #         #to be less risky, we do not input values here
                #         phaseDiff.append(None)
                #     else:
                #         phaseDiff.append(None)

            #note that frame parameters are updated after mosaicking, here no need to update parameters
            #mosaic amplitudes
            swathMosaic(referenceTrack.frames[i], inputAmplitudes, self._insar.amplitude,
                rangeOffsets, azimuthOffsets, self._insar.numberRangeLooks1, self._insar.numberAzimuthLooks1, resamplingMethod=0)
            #mosaic interferograms
            #These are for ALOS-2, may need to change for ALOS-4!
            phaseDiffFixed = [0.0, 0.4754024578084084, 0.9509913179406437, 1.4261648478671614, 2.179664007520499, 2.6766909968024932, 3.130810857]

            #if (referenceTrack.frames[i].processingSoftwareVersion == '2.025' and secondaryTrack.frames[i].processingSoftwareVersion == '2.023') or \
            #   (referenceTrack.frames[i].processingSoftwareVersion == '2.023' and secondaryTrack.frames[i].processingSoftwareVersion == '2.025'):

            #    #               changed value                number of samples to estimate new value            new values estimate area
            #    ###########################################################################################################################
            #    #  2.6766909968024932-->2.6581660335779866                    1808694                               d169-f2850, north CA
            #    #  2.179664007520499 -->2.204125866652153                      131120                               d169-f2850, north CA

            #    phaseDiffFixed = [0.0, 0.4754024578084084, 0.9509913179406437, 1.4261648478671614, 2.204125866652153, 2.6581660335779866, 3.130810857]

            snapThreshold = 0.2

            #the above preparetions only applies to 'self._insar.modeCombination == 21'
            #looks like it also works for 31 (scansarNominalModes-stripmapModes)
            # if self._insar.modeCombination != 21:
            #     phaseDiff = None
            #     phaseDiffFixed = None
            #     snapThreshold = None

            #whether snap for each swath
            if self.swathPhaseDiffSnapIon == None:
                snapSwath = [[True for jjj in range(numberOfSwaths-1)] for iii in range(numberOfFrames)]
            else:
                snapSwath = self.swathPhaseDiffSnapIon
                if len(snapSwath) != numberOfFrames:
                    raise Exception('please specify each frame for parameter: swath phase difference snap to fixed values')
                for iii in range(numberOfFrames):
                    if len(snapSwath[iii]) != (numberOfSwaths-1):
                       raise Exception('please specify correct number of swaths for parameter: swath phase difference snap to fixed values')

            (phaseDiffEst, phaseDiffUsed, phaseDiffSource, numberOfValidSamples) = swathMosaic(referenceTrack.frames[i], inputInterferograms, self._insar.interferogram,
                rangeOffsets, azimuthOffsets, self._insar.numberRangeLooks1, self._insar.numberAzimuthLooks1, updateFrame=False,
                phaseCompensation=True, phaseDiff=phaseDiff, phaseDiffFixed=phaseDiffFixed, snapThreshold=snapThreshold, snapSwath=snapSwath[i], pcRangeLooks=1, pcAzimuthLooks=4,
                filt=False, resamplingMethod=1)

            #the first item is meaningless for all the following list, so only record the following items
            if phaseDiff == None:
                phaseDiff = [None for iii in range(self._insar.startingSwath, self._insar.endingSwath + 1)]
            #catalog.addItem('frame {} {} band swath phase diff input'.format(frameNumber, ionDir['subband'][k]), phaseDiff[1:], 'runIonSubband')
            #catalog.addItem('frame {} {} band swath phase diff estimated'.format(frameNumber, ionDir['subband'][k]), phaseDiffEst[1:], 'runIonSubband')
            #catalog.addItem('frame {} {} band swath phase diff used'.format(frameNumber, ionDir['subband'][k]), phaseDiffUsed[1:], 'runIonSubband')
            #catalog.addItem('frame {} {} band swath phase diff used source'.format(frameNumber, ionDir['subband'][k]), phaseDiffSource[1:], 'runIonSubband')
            #catalog.addItem('frame {} {} band swath phase diff samples used'.format(frameNumber, ionDir['subband'][k]), numberOfValidSamples[1:], 'runIonSubband')

            log += 'frame {} {} band swath phase diff input: {}\n'.format(frameNumber, ionDir['subband'][k], phaseDiff[1:])
            log += 'frame {} {} band swath phase diff estimated: {}\n'.format(frameNumber, ionDir['subband'][k], phaseDiffEst[1:])
            log += 'frame {} {} band swath phase diff used: {}\n'.format(frameNumber, ionDir['subband'][k], phaseDiffUsed[1:])
            log += 'frame {} {} band swath phase diff used source: {}\n'.format(frameNumber, ionDir['subband'][k], phaseDiffSource[1:])
            log += 'frame {} {} band swath phase diff samples used: {}\n'.format(frameNumber, ionDir['subband'][k], numberOfValidSamples[1:])

            #check if there is value around 3.130810857, which may not be stable
            phaseDiffUnstableExist = False
            for xxx in phaseDiffUsed:
                if abs(abs(xxx) - 3.130810857) < 0.2:
                    phaseDiffUnstableExist = True
            #catalog.addItem('frame {} {} band swath phase diff unstable exists'.format(frameNumber, ionDir['subband'][k]), phaseDiffUnstableExist, 'runIonSubband')
            log += 'frame {} {} band swath phase diff unstable exists: {}\n'.format(frameNumber, ionDir['subband'][k], phaseDiffUnstableExist)
            log += '\n'

            create_xml(self._insar.amplitude, referenceTrack.frames[i].numberOfSamples, referenceTrack.frames[i].numberOfLines, 'amp')
            create_xml(self._insar.interferogram, referenceTrack.frames[i].numberOfSamples, referenceTrack.frames[i].numberOfLines, 'int')

            #update secondary frame parameters here, here no need to update parameters
            os.chdir('../')
            #save parameter file, here no need to save parameter file
            os.chdir('../')
        os.chdir('../')

    ############################################################
    # STEP 4. mosaic frames
    ############################################################
    from isceobj.Alos2Proc.runFrameMosaic import frameMosaic
    from isceobj.Alos2Proc.Alos2ProcPublic import create_xml

    log += 'mosaic frames in {} at {}\n'.format(os.path.basename(__file__), datetime.datetime.now())
    log += '================================================================================================\n'

    spotlightModes, stripmapModes, scansarNominalModes, scansarWideModes, scansarModes = acquisitionModesAlos2()
    for k in range(2):
        os.chdir(ionDir['subband'][k])

        mosaicDir = ionDir['insar']
        os.makedirs(mosaicDir, exist_ok=True)
        os.chdir(mosaicDir)

        numberOfFrames = len(referenceTrack.frames)
        if numberOfFrames == 1:
            import shutil
            frameDir = os.path.join('f1_{}/mosaic'.format(self._insar.referenceFrames[0]))
            # if not os.path.isfile(self._insar.interferogram):
            #     os.symlink(os.path.join('../', frameDir, self._insar.interferogram), self._insar.interferogram)
            # #shutil.copy2() can overwrite
            # shutil.copy2(os.path.join('../', frameDir, self._insar.interferogram+'.vrt'), self._insar.interferogram+'.vrt')
            # shutil.copy2(os.path.join('../', frameDir, self._insar.interferogram+'.xml'), self._insar.interferogram+'.xml')
            # if not os.path.isfile(self._insar.amplitude):
            #     os.symlink(os.path.join('../', frameDir, self._insar.amplitude), self._insar.amplitude)
            # shutil.copy2(os.path.join('../', frameDir, self._insar.amplitude+'.vrt'), self._insar.amplitude+'.vrt')
            # shutil.copy2(os.path.join('../', frameDir, self._insar.amplitude+'.xml'), self._insar.amplitude+'.xml')

            os.rename(os.path.join('../', frameDir, self._insar.interferogram), self._insar.interferogram)
            os.rename(os.path.join('../', frameDir, self._insar.interferogram+'.vrt'), self._insar.interferogram+'.vrt')
            os.rename(os.path.join('../', frameDir, self._insar.interferogram+'.xml'), self._insar.interferogram+'.xml')
            os.rename(os.path.join('../', frameDir, self._insar.amplitude), self._insar.amplitude)
            os.rename(os.path.join('../', frameDir, self._insar.amplitude+'.vrt'), self._insar.amplitude+'.vrt')
            os.rename(os.path.join('../', frameDir, self._insar.amplitude+'.xml'), self._insar.amplitude+'.xml')

            #update track parameters, no need to update track parameters here

        else:
            # #choose offsets
            # if self.frameOffsetMatching:
            #     rangeOffsets = self._insar.frameRangeOffsetMatchingReference
            #     azimuthOffsets = self._insar.frameAzimuthOffsetMatchingReference
            # else:
            #     rangeOffsets = self._insar.frameRangeOffsetGeometricalReference
            #     azimuthOffsets = self._insar.frameAzimuthOffsetGeometricalReference

            if referenceTrack.operationMode in scansarModes:
                matchingMode=0
            else:
                matchingMode=1

            #geometrical offset is enough
            offsetReferenceStack = frameOffset(referenceTrack, dateReference+'.slc', 'frame_offset_' + dateReference + '.txt',
                                       crossCorrelation=False, matchingMode=matchingMode)

            #we can faithfully make it integer.
            #this can also reduce the error due to floating point computation
            rangeOffsets = [float(round(x)) for x in offsetReferenceStack[0]]
            azimuthOffsets = [float(round(x)) for x in offsetReferenceStack[1]]

            #list of input files
            inputInterferograms = []
            inputAmplitudes = []
            for i, frameNumber in enumerate(self._insar.referenceFrames):
                frameDir = 'f{}_{}'.format(i+1, frameNumber)
                inputInterferograms.append(os.path.join('../', frameDir, 'mosaic', self._insar.interferogram))
                inputAmplitudes.append(os.path.join('../', frameDir, 'mosaic', self._insar.amplitude))

            #note that track parameters are updated after mosaicking
            #mosaic amplitudes
            frameMosaic(referenceTrack, inputAmplitudes, self._insar.amplitude,
                rangeOffsets, azimuthOffsets, self._insar.numberRangeLooks1, self._insar.numberAzimuthLooks1,
                updateTrack=False, phaseCompensation=False, resamplingMethod=0)
            #mosaic interferograms
            (phaseDiffEst, phaseDiffUsed, phaseDiffSource, numberOfValidSamples) = frameMosaic(referenceTrack, inputInterferograms, self._insar.interferogram,
                rangeOffsets, azimuthOffsets, self._insar.numberRangeLooks1, self._insar.numberAzimuthLooks1,
                updateTrack=False, phaseCompensation=True, resamplingMethod=1)

            create_xml(self._insar.amplitude, referenceTrack.numberOfSamples, referenceTrack.numberOfLines, 'amp')
            create_xml(self._insar.interferogram, referenceTrack.numberOfSamples, referenceTrack.numberOfLines, 'int')

            #if multiple frames, remove frame amplitudes/inteferograms to save space
            for x in inputAmplitudes:
                os.remove(x)
                os.remove(x+'.vrt')
                os.remove(x+'.xml')

            for x in inputInterferograms:
                os.remove(x)
                os.remove(x+'.vrt')
                os.remove(x+'.xml')

            #catalog.addItem('{} band frame phase diff estimated'.format(ionDir['subband'][k]), phaseDiffEst[1:], 'runIonSubband')
            #catalog.addItem('{} band frame phase diff used'.format(ionDir['subband'][k]), phaseDiffUsed[1:], 'runIonSubband')
            #catalog.addItem('{} band frame phase diff used source'.format(ionDir['subband'][k]), phaseDiffSource[1:], 'runIonSubband')
            #catalog.addItem('{} band frame phase diff samples used'.format(ionDir['subband'][k]), numberOfValidSamples[1:], 'runIonSubband')

            log += '{} band frame phase diff estimated: {}\n'.format(ionDir['subband'][k], phaseDiffEst[1:])
            log += '{} band frame phase diff used: {}\n'.format(ionDir['subband'][k], phaseDiffUsed[1:])
            log += '{} band frame phase diff used source: {}\n'.format(ionDir['subband'][k], phaseDiffSource[1:])
            log += '{} band frame phase diff samples used: {}\n'.format(ionDir['subband'][k], numberOfValidSamples[1:])
            log += '\n'

            #update secondary parameters here, no need to update secondary parameters here

        os.chdir('../')
        #save parameter file, no need to save parameter file here
        os.chdir('../')

    ############################################################
    # STEP 5. clear frame processing files
    ############################################################
    import shutil
    from isceobj.Alos2Proc.Alos2ProcPublic import runCmd

    for k in range(2):
        os.chdir(ionDir['subband'][k])
        for i, frameNumber in enumerate(self._insar.referenceFrames):
            frameDir = 'f{}_{}'.format(i+1, frameNumber)
            #keep subswath interferograms
            #shutil.rmtree(frameDir)
            #cmd = 'rm -rf {}'.format(frameDir)
            #runCmd(cmd)
        os.chdir('../')

    ############################################################
    # STEP 6. create differential interferograms
    ############################################################
    import numpy as np
    from isceobj.Alos2Proc.Alos2ProcPublic import runCmd

    for k in range(2):
        os.chdir(ionDir['subband'][k])

        insarDir = ionDir['insar']
        os.makedirs(insarDir, exist_ok=True)
        os.chdir(insarDir)

        rangePixelSize = self._insar.numberRangeLooks1 * referenceTrack.rangePixelSize
        radarWavelength = subbandRadarWavelength[k]

        ml1 = '_{}rlks_{}alks'.format(self._insar.numberRangeLooks1, self._insar.numberAzimuthLooks1)
        if dateReference == dateReferenceStack:
            rectRangeOffset = os.path.join('../../../', idir, dateSecondary, 'insar', dateSecondary + ml1 + '_rg_rect.off')
            cmd = "imageMath.py -e='a*exp(-1.0*J*b*4.0*{}*{}/{})*(b!=0)' --a={} --b={} -o {} -t cfloat".format(np.pi, rangePixelSize, radarWavelength, self._insar.interferogram, rectRangeOffset, self._insar.differentialInterferogram)
        elif dateSecondary == dateReferenceStack:
            rectRangeOffset = os.path.join('../../../', idir, dateReference, 'insar', dateReference + ml1 + '_rg_rect.off')
            cmd = "imageMath.py -e='a*exp(1.0*J*b*4.0*{}*{}/{})*(b!=0)' --a={} --b={} -o {} -t cfloat".format(np.pi, rangePixelSize, radarWavelength, self._insar.interferogram, rectRangeOffset, self._insar.differentialInterferogram)
        else:
            rectRangeOffset1 = os.path.join('../../../', idir, dateReference, 'insar', dateReference + ml1 + '_rg_rect.off')
            rectRangeOffset2 = os.path.join('../../../', idir, dateSecondary, 'insar', dateSecondary + ml1 + '_rg_rect.off')
            cmd = "imageMath.py -e='a*exp(1.0*J*(b-c)*4.0*{}*{}/{})*(b!=0)*(c!=0)' --a={} --b={} --c={} -o {} -t cfloat".format(np.pi, rangePixelSize, radarWavelength, self._insar.interferogram, rectRangeOffset1, rectRangeOffset2, self._insar.differentialInterferogram)
        runCmd(cmd)

        os.chdir('../../')

    os.chdir('../')

    return log

def cmdLineParse():
    '''
    command line parser.
    '''
    import sys
    import argparse

    parser = argparse.ArgumentParser(description='create subband interferograms for ionospheric correction')
    parser.add_argument('-idir', dest='idir', type=str, required=True,
            help = 'input directory where resampled data of each date (YYMMDD) is located. only folders are recognized')
    parser.add_argument('-ref_date_stack', dest='ref_date_stack', type=str, required=True,
            help = 'reference date of stack. format: YYMMDD')
    parser.add_argument('-ref_date', dest='ref_date', type=str, required=True,
            help = 'reference date of this pair. format: YYMMDD')
    parser.add_argument('-sec_date', dest='sec_date', type=str, required=True,
            help = 'reference date of this pair. format: YYMMDD')
    parser.add_argument('-nrlks1', dest='nrlks1', type=int, default=1,
            help = 'number of range looks 1. default: 1')
    parser.add_argument('-nalks1', dest='nalks1', type=int, default=1,
            help = 'number of azimuth looks 1. default: 1')
    # parser.add_argument('-nrlks_ion', dest='nrlks_ion', type=int, default=1,
    #         help = 'number of range looks ion. default: 1')
    # parser.add_argument('-nalks_ion', dest='nalks_ion', type=int, default=1,
    #         help = 'number of azimuth looks ion. default: 1')
    parser.add_argument('-snap', dest='snap', type=int, nargs='+', action='append', default=None,
            help='swath phase difference snap to fixed values. e.g. you have 3 swaths and 2 frames. specify this parameter as: -snap 1 1 -snap 1 0, where 0 means no snap, 1 means snap')
    parser.add_argument('-phase_diff_lower', dest='phase_diff_lower', type=str, nargs='+', action='append', default=None,
            help='swath phase difference lower band. e.g. you have 3 swaths and 2 frames. specify this parameter as: -snap -1.3 2.37 -snap 0.1 None, where None means no user input phase difference value')
    parser.add_argument('-phase_diff_upper', dest='phase_diff_upper', type=str, nargs='+', action='append', default=None,
            help='swath phase difference upper band. e.g. you have 3 swaths and 2 frames. specify this parameter as: -snap -1.3 2.37 -snap 0.1 None, where None means no user input phase difference value')

    if len(sys.argv) <= 1:
        print('')
        parser.print_help()
        sys.exit(1)
    else:
        return parser.parse_args()

if __name__ == '__main__':

    inps = cmdLineParse()

    #get user parameters from input
    idir = inps.idir
    dateReferenceStack = inps.ref_date_stack
    dateReference = inps.ref_date
    dateSecondary = inps.sec_date
    numberRangeLooks1 = inps.nrlks1
    numberAzimuthLooks1 = inps.nalks1
    #numberRangeLooksIon = inps.nrlks_ion
    #numberAzimuthLooksIon = inps.nalks_ion
    swathPhaseDiffSnapIon = inps.snap
    swathPhaseDiffLowerIon = inps.phase_diff_lower
    swathPhaseDiffUpperIon = inps.phase_diff_upper
    #######################################################

    pair = '{}-{}'.format(dateReference, dateSecondary)
    ms = pair

    ml1 = '_{}rlks_{}alks'.format(numberRangeLooks1, numberAzimuthLooks1)

    dateDirs,   dates,   frames,   swaths,   dateIndexReference = stackDateStatistics(idir, dateReferenceStack)
    nframe = len(frames)
    nswath = len(swaths)

    trackReferenceStack = loadTrack('./', dates[dateIndexReference])
    #trackReference = loadTrack('./', dateReference)
    #trackSecondary = loadTrack('./', dateSecondary)

    self = createObject()
    self._insar = createObject()
    self._insar.referenceFrames = frames
    self._insar.startingSwath = swaths[0]
    self._insar.endingSwath = swaths[-1]

    self._insar.numberRangeLooks1 = numberRangeLooks1
    self._insar.numberAzimuthLooks1 = numberAzimuthLooks1

    self._insar.interferogram = ms + ml1 + '.int'
    self._insar.amplitude = ms + ml1 + '.amp'
    self._insar.differentialInterferogram = 'diff_' + ms + ml1 + '.int'

    #set self.swathPhaseDiffSnapIon, self.swathPhaseDiffLowerIon, self.swathPhaseDiffUpperIon
    if swathPhaseDiffSnapIon is not None:
        swathPhaseDiffSnapIon = [[True if x==1 else False for x in y] for y in swathPhaseDiffSnapIon]
        if len(swathPhaseDiffSnapIon) != nframe:
            raise Exception('please specify each frame for parameter: -snap')
        for i in range(nframe):
            if len(swathPhaseDiffSnapIon[i]) != (nswath-1):
               raise Exception('please specify correct number of swaths for parameter: -snap')

    if swathPhaseDiffLowerIon is not None:
        swathPhaseDiffLowerIon = [[float(x) if x.upper() != 'NONE' else None for x in y] for y in swathPhaseDiffLowerIon]
        if len(swathPhaseDiffLowerIon) != nframe:
            raise Exception('please specify each frame for parameter: -phase_diff_lower')
        for i in range(nframe):
            if len(swathPhaseDiffLowerIon[i]) != (nswath-1):
               raise Exception('please specify correct number of swaths for parameter: -phase_diff_lower')

    if swathPhaseDiffUpperIon is not None:
        swathPhaseDiffUpperIon = [[float(x) if x.upper() != 'NONE' else None for x in y] for y in swathPhaseDiffUpperIon]
        if len(swathPhaseDiffUpperIon) != nframe:
            raise Exception('please specify each frame for parameter: -phase_diff_upper')
        for i in range(nframe):
            if len(swathPhaseDiffUpperIon[i]) != (nswath-1):
               raise Exception('please specify correct number of swaths for parameter: -phase_diff_upper')

    self.swathPhaseDiffSnapIon = swathPhaseDiffSnapIon
    self.swathPhaseDiffLowerIon = swathPhaseDiffLowerIon
    self.swathPhaseDiffUpperIon = swathPhaseDiffUpperIon

    log = runIonSubband(self, trackReferenceStack, idir, dateReferenceStack, dateReference, dateSecondary)

    logFile = 'process.log'
    with open(logFile, 'a') as f:
        f.write(log)


# Crea una función que te regrese el último día de un mes recibido por valor, se recibe como parámetro mes y año.
import calendar
from datetime import datetime

now = datetime.now()

mes = input("Ingresa el mes: ")
anio = input("Ingresa el anio: ")

mes = int(mes)
anio = int(anio)

# now_anio = now.year
# now_mes = now.month

if mes < 1 or anio < 1 or mes > 12:
    print("INGRESE DATOS CORRECTOS")

# elif anio > now_anio or mes > now_mes:
#     print("INGRESE DATOS CORRECTOS")

else:
    ultimo_dia = calendar.monthrange(anio, mes)[1]
    print(f"El ultimo dia del mes {mes} en el año {anio} es: ", ultimo_dia)


from django.shortcuts import render
from django.views.generic import ListView, CreateView
from .models import Events, EventTime
from .forms import EventForm, EventTimeForm
from datetime import datetime
# Create your views here.

class EventsListView(ListView):
    model = Events
    template_name = "eventsapp/list.html"

    def get(self, request):
        events = self.get_queryset(request)
        return render(request, self.template_name, {'events': events})

    def get_queryset(self, request):
        qs = super().get_queryset()
        results = set()
        if 'weekday' in request.GET:
            for q in qs:
                c_all = q.event_times.all()
                for c in c_all:
                    if str(c.event_start_time.weekday()) == (request.GET['weekday']):
                        results.add(q)
            return results
        elif'yyyy-mm' in request.GET:
            year, month = request.GET['yyyy-mm'].split('-')
            qs = set(qs.filter(event_times__event_start_time__year=year, event_times__event_start_time__month=month))
        elif 'yyyy-mm-dd' in request.GET:
            year, month, date = request.GET['yyyy-mm-dd'].split('-')
            qs = set(qs.filter(event_times__event_start_time__year=year, event_times__event_start_time__month=month, event_times__event_start_time__day=date))
        return qs

class EventsCreateView(CreateView):
    def get(self, request):
        eform = EventForm()
        tform = EventTimeForm()
        template_name = "eventsapp/createview.html"
        return render(request, template_name, {'eform': eform, 'tform': tform, })

    def post(self, request):
        template_name = "eventsapp/createview.html"
        eform = EventForm(request.POST)
        if eform.is_valid():
            eform.title = eform.cleaned_data['title']
            eform.place = eform.cleaned_data['place']
            eform.tags = eform.cleaned_data['tags']
            eform.created_by = eform.cleaned_data['created_by']
            e = eform.save()
        tform = EventTime()
        event_start_time = request.POST['event_start_time_0'] + " " + request.POST['event_start_time_1']
        event_start_time = datetime.strptime(event_start_time, '%Y-%m-%d %H:%M:%S')
        tform.event_start_time = event_start_time
        event_end_time = request.POST['event_end_time_0'] + " " + request.POST['event_end_time_1']
        event_end_time = datetime.strptime(event_end_time, '%Y-%m-%d %H:%M:%S')
        tform.event_end_time = event_end_time
        tform.event = Events.objects.get(pk=e.pk)
        tform.save()
        return render(request, template_name, {})


import numpy as np
np.warnings.filterwarnings('ignore')
from PIL import Image

def complex_matrix(xmin, xmax, ymin, ymax, pixel_density):
    real = np.linspace(xmin, xmax, int((xmax-xmin)*pixel_density))
    imagenary = np.linspace(ymin, ymax, int((ymax-ymin)*pixel_density))
    return real[np.newaxis, :] + imagenary[:, np.newaxis]*1j

def is_stable(c: complex, num_iterations: int) -> bool:
    z = 0
    for _ in range(num_iterations):
        z = z ** 2 + c
    return abs(z) <= 2

c = complex_matrix(-2, 0.5, -1.5, 1.5, pixel_density=512)
image = Image.fromarray(~is_stable(c, num_iterations=20))
image.show()


import pandas as pd

# File paths
json_books_path = r"C:\Users\Anthony\Desktop\book-recommendation-ml\dataset\bookCategory.json"
csv_books_path = r"C:\Users\Anthony\Desktop\book-recommendation-ml\dataset\initialData\Books.csv"

# Load JSON books data
with open(json_books_path, 'r') as file:
    json_books_data = pd.read_json(file)

# Load CSV books data
csv_books_df = pd.read_csv(csv_books_path, converters={'ISBN': str}, low_memory=False)

# Find unique ISBNs in both datasets
unique_isbn_json = set(json_books_data['ISBN'].unique())
unique_isbn_csv = set(csv_books_df['ISBN'].unique())

# Find books that are in CSV but not in JSON
books_in_csv_not_json = unique_isbn_csv - unique_isbn_json
books_to_add = csv_books_df[csv_books_df['ISBN'].isin(books_in_csv_not_json)]

# Add the 'Category' attribute with an empty string value
books_to_add['Category'] = ''

# Append the missing books to the original JSON data
updated_books_data = pd.concat([json_books_data, books_to_add], ignore_index=True)

# Save the updated data back to bookCategory.json
updated_books_data.to_json(json_books_path, orient='records')

print(f"Added {len(books_in_csv_not_json)} books with 'Category' set to an empty string to {json_books_path}.")


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import re
import csv

driver = webdriver.Chrome("chromedriver.exe")
driver.get("https://www.youtube.com/watch?v=QJzSJhM9blI")
time.sleep(2)  # give the page some time to load

# --------------- FETCH TITLE ---------------
# get the video's title
title = driver.find_element_by_xpath('//*[@id="container"]/h1/yt-formatted-string').text
print(title)

# --------------- LOAD ALL COMMENTS ---------------
# defining the numbers here so we can reference and easily change them
SCROLL_PAUSE_TIME = 2
CYCLES = 100

# we know there's always exactly one HTML element, so let's access it
html = driver.find_element_by_tag_name('html')
# first time needs to not jump to the very end in order to start
html.send_keys(Keys.PAGE_DOWN)  # doing it twice for good measure
html.send_keys(Keys.PAGE_DOWN)  # one time sometimes wasn't enough
# adding extra time for initial comments to load
# if they fail (because too little time allowed), the whole script breaks
time.sleep(SCROLL_PAUSE_TIME * 3)
# and now for loading the hidden comments by scrolling down and up
for i in range(CYCLES):
    html.send_keys(Keys.END)
    time.sleep(SCROLL_PAUSE_TIME)
    # might not be necessary; try out without it.
    # html.send_keys(Keys.PAGE_UP)
    # time.sleep(SCROLL_PAUSE_TIME)

# --------------- GETTING THE COMMENT TEXTS ---------------
name_elems=driver.find_elements_by_xpath('//*[@id="author-text"]')
comment_elems = driver.find_elements_by_xpath('//*[@id="content-text"]')
num_of_names = len(name_elems)

# --------------- WRITING TO OUTPUT FILE ---------------
with open("ytb_comment.csv", "w", newline='', encoding='utf-8') as f:
    writer = csv.writer(f, delimiter=',')
    writer.writerow(['Username', 'Comment'])
    for i in range(num_of_names):
        name = name_elems[i].text
        text = comment_elems[i].text
        text = str(text).replace('\n', '')

        # writer.writerow(['Username', 'Comment'])
        writer.writerow([name, text])

driver.close()


from __future__ import unicode_literals
import yt_dlp as ytdl

from celery import Celery
import redis

from hashlib import md5

# initialize redis connection
redis = redis.Redis(host="redis", port="6379")

# define celery app
app = Celery("tasks", broker="redis://redis")

# utilities for youtube_dl
class MyLogger(object):
    def debug(self, msg):
        # debug left empty to prevent message spam
        # print(msg) to get output
        pass

    def warning(self, msg):
        print(msg)

    def error(self, msg):
        print(msg)

def progress_hook(d):
    """
    Hook to process data from download callbacks during download
    """
    video_id = d["filename"].split("/")[1]
    if d["status"] == "downloading":
        if "eta" not in d or "_percent_str" not in d:
            return
        redis.hmset(video_id,
            {
                "eta": d["_eta_str"],
                "percent": d["_percent_str"].strip(),
                "state": "downloading",
            },
        )
    elif d["status"] == "finished":
        redis.hmset(video_id, {"eta": "0", "percent": "100%", "state": "processing"})

def post_hook(d):
    """
    Hook to process data when processing (e.g. converting) during download
    """
    if d["status"] == "started":
        video_id = d["info_dict"]["_filename"].split("/")[1]
        redis.hmset(
            video_id, {"eta": "unknown", "percent": "unknown", "state": "processing"}
        )
    elif d["status"] == "finished":
        video_id = d["info_dict"]["_filename"].split("/")[1]
        redis.hmset(video_id, {"eta": "unknown", "percent": "unknown", "state": "done"})

# urls:
# test url: https://www.youtube.com/watch?v=BaW_jenozKc
# long url: https://www.youtube.com/watch?v=ddRMOKFDgos
# short ur: https://www.youtube.com/watch?v=RKmw9oS__MM

# Main task definition
@app.task
def download_request(url, dl_format):
    # config for youtube download
    ydl_opts = {
        "format": "bestvideo+bestaudio",
        "postprocessors": [],
        "logger": MyLogger(),
        "progress_hooks": [progress_hook],
        "postprocessor_hooks": [post_hook],
        #"outtmpl": "%(title)s-%(id)s.%(ext)s", # drop the title to prevent filename errors
        "outtmpl": "%(id)s.%(ext)s",
        "paths": {"home": "tmp/"},
        "verbose": True,
    }

    ydl_audio_postprocessor = {
        "key": "FFmpegExtractAudio",
        "when": "post_process",
        "preferredcodec": "mp3",
        "preferredquality": "192",
    }

    # job id used to track progress
    job_id = md5((url+"-"+dl_format).encode("ascii")).hexdigest()
    # store files by <id>/<format> (more to come later when more options are added)
    ydl_opts["paths"]["home"] = "tmp/" + job_id
    # add additional post processor if audio-only requested
    if dl_format == "audio_only":
        ydl_opts["postprocessors"].append(ydl_audio_postprocessor)
        ydl_opts["format"] = "bestaudio"
    elif dl_format == "video_only":
        ydl_opts["format"] = "bestvideo"

    # start download process
    try:
        with ytdl.YoutubeDL(ydl_opts) as ydl:
            ydl.download([url])
    except Exception as e:
        print("Error downloading")
        redis.hmset(job_id, {"state": "error", "eta": "", "percent": "", "message": str(e)})
        return


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Sat Jun 10 19:53:32 2017

@author: Bene

This file creates a fwd-model for the TESCAN Q-PHASE under
multiple-scattering. It is majorly derived from  "LEarning approach for optical tomography"
U. S. Kamilov, BIG, EPFL, 2014.89    """
# %load_ext autoreload
import tensorflow as tf
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import time
import os
import NanoImagingPack as nip

# change the following to %matplotlib notebook for interactive plotting
# %matplotlib inline

# Optionally, tweak styles.
mpl.rc('figure',  figsize=(6, 4))
mpl.rc('image', cmap='gray')
#plt.switch_backend('agg')

# load own functions
import src.simulations as experiments
import src.model as mus
import src.tf_generate_object as tf_go
import src.data as data
import src.MyParameter as paras
import src.tf_helper as tf_helper

os.environ["CUDA_VISIBLE_DEVICES"]='0'
os.environ["TF_CUDNN_USE_AUTOTUNE"]="0"

'''Define some stuff related to infrastructure'''
savepath = experiments.mysavepath

# Create directory
try:
    os.mkdir(savepath)
except(FileExistsError):
    print('Folder exists already')

''' Define parameters '''
is_padding = False # better don't do it, some normalization is probably incorrect
is_display = True
is_optimization = False
is_optimization_psf = False
is_flip = False
is_measurement = False

'''Choose between Born (BORN) or BPM (BPM)'''
psf_model =  'BORN' # MultiSlice
#psf_model =  '3QDPC' # MultiSlice
#psf_model =  'BPM' # 1st Born

# Generate Test-Object
''' File which stores the experimental parameters from the Q-PHASE setup
    2.) Read in the parameters of the dataset '''
myparams = paras.MyParameter()
myparams.loadExperiment(experiments)
myparams.print()

# reset default graph
tf.reset_default_graph()

''' Create the Model'''
muscat = mus.MuScatModel(myparams, is_optimization=is_optimization)

''' Create a 3D Refractive Index Distributaton as a artificial sample'''
mydiameter = 1
objtype = 'sphere'# 'cheek100'#'cheek100' # 'sphere', 'twosphere', 'slphantom'
if(objtype == 'sphere'):
    obj_real= tf_go.generateObject(mysize=myparams.mysize, obj_dim=np.array((myparams.dz, myparams.dx, myparams.dy)), obj_type ='sphere', diameter = mydiameter, dn = experiments.dn, nEmbb = myparams.nEmbb)#)dn)
    obj_absorption= tf_go.generateObject(mysize=myparams.mysize, obj_dim=np.array((myparams.dz, myparams.dx, myparams.dy)), obj_type ='sphere', diameter = mydiameter, dn = .01, nEmbb = 0.)#)dn)
elif(objtype == 'cheek100'):
    # Fake Cheek-Cell
    matlab_val_file = './Data/PHANTOM/HeLa_cell_mat_obj_100.mat'; matname='HeLa_cell_mat'
    obj_real = data.import_realdata_h5(filename = matlab_val_file, matname=matname)
    obj_absorption = obj_real*0

obj = (obj_real + 1j*obj_absorption)
obj = np.roll(obj,5,0)
#obj = np.roll(obj, shift=5, axis=0)

# introduce zernike factors here
muscat.zernikefactors = experiments.zernikefactors
muscat.zernikemask = experiments.zernikefactors*0

print('Start the TF-session')
sess = tf.Session()#config=tf.ConfigProto(log_device_placement=True))

''' Compute the systems model'''
if psf_model == 'BPM':
    # Define 'BPM' model
    tf_fwd = muscat.compute_bpm(obj,is_padding=is_padding, mysubsamplingIC=experiments.mysubsamplingIC)

elif psf_model == 'BORN':
    # Define Born Model
    tf_fwd = muscat.compute_born(obj, is_padding=is_padding, mysubsamplingIC=experiments.mysubsamplingIC, is_precompute_psf=True)

''' Evaluate the model '''
print('Initiliaze Variables')
sess.run(tf.global_variables_initializer())

# The first call is -unfortunately- very expensive...
print('Compute Result')
start = time.time()
myfwd = sess.run(tf_fwd)
end = time.time()
print(end - start)

#nip.v5(muscat.A_input)
#%% display the results
centerslice = myfwd.shape[0]//2
#myfwd = myfwd - 1j
if(psf_model == 'BORN' or psf_model == '3QDPC'):
    #% write Freq-Support to disk
    tf_helper.plot_ASF_ATF(savepath, muscat.myATF, muscat.myASF)
    tf_helper.plot_obj_fft(savepath, myfwd)
#%%
#%% Display Apertures
plt.subplot(131), plt.title('Ic'),plt.imshow(muscat.Ic), plt.colorbar()#, plt.show()
plt.subplot(132), plt.title('Abs Po'),plt.imshow(np.abs(muscat.Po)), plt.colorbar()#, plt.show()
plt.subplot(133), plt.title('Angle Po'),plt.imshow(np.angle(muscat.Po)), plt.colorbar()# plt.show()

#%%
plt.figure()
plt.subplot(231), plt.title('real XZ'),plt.imshow(np.real(myfwd)[:,myfwd.shape[1]//2,:]), plt.colorbar()#, plt.show()
plt.subplot(232), plt.title('real YZ'),plt.imshow(np.real(myfwd)[:,:,myfwd.shape[2]//2]), plt.colorbar()#, plt.show()
plt.subplot(233), plt.title('real XY'),plt.imshow(np.real(myfwd)[centerslice,:,:]), plt.colorbar()# plt.show()
plt.subplot(234), plt.title('imag XZ'),plt.imshow(np.imag(myfwd)[:,myfwd.shape[1]//2,:]), plt.colorbar()#, plt.show()
plt.subplot(235), plt.title('imag YZ'),plt.imshow(np.imag(myfwd)[:,:,myfwd.shape[2]//2]), plt.colorbar()#, plt.show()
plt.subplot(236), plt.title('imag XY'),plt.imshow(np.imag(myfwd)[centerslice,:,:]), plt.colorbar()
plt.savefig(savepath+'fwd_realimag'+psf_model+'.png'), plt.show()

plt.subplot(231), plt.title('abs XZ'),plt.imshow(np.abs(myfwd)[:,myfwd.shape[1]//2,:]), plt.colorbar()#, plt.show()
plt.subplot(232), plt.title('abs YZ'),plt.imshow(np.abs(myfwd)[:,:,myfwd.shape[2]//2]), plt.colorbar()#, plt.show()
plt.subplot(233), plt.title('abs XY'),plt.imshow(np.abs(myfwd)[centerslice,:,:]), plt.colorbar()# plt.show()
plt.subplot(234), plt.title('angle XZ'),plt.imshow(np.angle(myfwd)[:,myfwd.shape[1]//2,:]), plt.colorbar()#, plt.show()
plt.subplot(235), plt.title('angle YZ'),plt.imshow(np.angle(myfwd)[:,:,myfwd.shape[2]//2]), plt.colorbar()#, plt.show()
plt.subplot(236), plt.title('angle XY'),plt.imshow(np.angle(myfwd)[centerslice,:,:]), plt.colorbar()
plt.savefig(savepath+'fwd_absphase_'+psf_model+'.png'), plt.show()

# plot the object RI distribution
if(0):
    plt.figure()
    plt.subplot(231), plt.title('obj - real XZ'),plt.imshow(np.real(obj)[:,obj.shape[1]//2,:]), plt.colorbar()#, plt.show()
    plt.subplot(232), plt.title('obj - real YZ'),plt.imshow(np.real(obj)[:,:,obj.shape[2]//2]), plt.colorbar()#, plt.show()
    plt.subplot(233), plt.title('obj - real XY'),plt.imshow(np.real(obj)[centerslice,:,:]), plt.colorbar()# plt.show()
    plt.subplot(234), plt.title('obj - imag XZ'),plt.imshow(np.imag(obj)[:,obj.shape[1]//2,:]), plt.colorbar()#, plt.show()
    plt.subplot(235), plt.title('obj - imag YZ'),plt.imshow(np.imag(obj)[:,:,obj.shape[2]//2]), plt.colorbar()#, plt.show()
    plt.subplot(236), plt.title('obj - imag XY'),plt.imshow(np.imag(obj)[centerslice,:,:]), plt.colorbar()
    plt.savefig(savepath+'obj_'+psf_model+'.png'), plt.show()

#%% save the resultsl
np.save(savepath+'allAmp_simu_'+psf_model+'.npy', myfwd)
np.save(savepath+'myObj_simu_'+psf_model+'.npy', obj)
np.save(savepath+'myPar_simu_'+psf_model+'.npy', myparams)

data.export_realdatastack_h5(savepath+'/obj'+psf_model+'.h5', 'phase, abs',
                        np.stack((np.real(obj),np.imag(obj)), axis=0))
data.export_realdatastack_h5(savepath+'/myfwd'+psf_model+'.h5', 'real, imag',
                        np.stack((np.real(myfwd),
                                  np.imag(myfwd)), axis=0))


#!/usr/bin/env python

from setuptools import setup, find_packages

__version__ = '1.0'

__build__ = ''

setup(name='callingcard',
      version=__version__ + __build__,
      description='twilio voice call forwarding',
      author='Eric Siegel',
      author_email='siegel.eric@gmail.com',
      url='http://www.esiegel.com',
      packages=find_packages(exclude=['*.tests']),
      setup_requires=[
          'nose>=1.0',
      ],
      install_requires=[
          'Flask>=0.8',
      ],
      tests_require=[
      ],
      test_suite='callingcard.tests',
      entry_points={
          'console_scripts': [
              'development = callingcard.development:main',
          ]
      },
      include_package_data=True,
      )


import os
import glob
import sys
import math
import json
import index
from nltk import FreqDist
import text_proc

def model(json_request):

	data = json.loads(json_request)

	if data['action'] == 'build':
		request_build(data['path'])
	elif data['action'] == 'query':
		return request_query(data['query'], data['count'], data['similarity_techniques'])

# TODO: proceamiento de texto
def request_build(path):

	data = []

	if 'win' in sys.platform:
		_path = path + "\**"
	else:
		_path = path + "/**"

	if os.path.exists(path):
		# pdf_docs = glob.glob(os.path.join(_path, "*.pdf"), recursive=True)
		# docx_docs = glob.glob(os.path.join(_path, "*.docx"), recursive=True)
		txt_docs = glob.glob(os.path.join(_path, "*.txt"), recursive=True)
		docs =  txt_docs #+ docx_docs + pdf_docs

		docs_text = [(doc_path,read_doc(doc_path),0) for doc_path in docs]
		vector = get_vector(docs_text)

		#crear la estructura 'docs_text' que se utiliza para poder calcular los minterms d cada documento
		for index_doc in range(len(docs_text)):

			json_process = json.dumps({'action':'process', 'data':docs_text[index_doc][1]})
			terms = json.loads(text_proc.text_work(json_process))['terms']

			minterm_of_doc = get_minterm_of_document(vector, terms)
			docs_text[index_doc] = (docs_text[index_doc][0], terms, minterm_of_doc)

		#Creando el indice
		for doc in docs_text:
			terms = doc[1]
			doc_name = os.path.basename(doc[0])
			append_terms(doc_name, terms, data, minterm_doc=doc[2], vector= vector )

		append_idfs(data, len(docs))
		append_weights(data)

		json_process = json.dumps({'action':'create', 'data':data})
		index.start(json_process)

	else: raise Exception("The path is not correct")

# TODO: proceamiento de texto
#TODO poner las otras medidas
#TODO quitar palabras que no son de ningun doc
def request_query(query, count_doc, similarity_techniques):

	json_process = json.dumps({'action': 'process', 'data': query})
	query_terms = json.loads(text_proc.text_work(json_process))['terms']

	query_data = []
	vector = {query_terms[index_term]:index_term for index_term in range(len(query_terms)) }

	#Construir el indice de la query
	append_terms('query',query_terms,query_data,"0",vector)
	append_idfs(query_data, 1)
	append_weights(query_data)

	docs = get_docs_from_query(query_terms)
	dicc_ranking = similarity_cosine(query_data, docs) if similarity_techniques == 0 else similarity_vectorial_model(query_data, docs)

	result= list(zip(dicc_ranking.keys(), dicc_ranking.values()))
	result.sort(key=lambda x: x[1], reverse=True)
	result = result[:count_doc]

	return json.dumps({'action':'report', 'sucess':True,'results':[{'document':dc, 'match':val} for dc, val in result]})

##########################################################
#################### CONSTRUIR INDEX #####################
##########################################################

def append_terms(doc, terms, data, minterm_doc,vector):

	tf = FreqDist(terms)
	max_tf = max(tf.values()if len(tf)>0 else [0])

	for term in tf.keys():
		normalize_tf = tf[term] / max_tf
		new_doc = {'tf': normalize_tf,
				   'weight': 0,
				   'minterm':minterm_doc}
		in_data = False
		for term_data in data:
			if term == term_data['key']:

				#update
				term_data['value']['documents'][doc] = new_doc
				in_data = True
				break

		if not in_data:
			# add
			data.append({'key': term,
						 'value': {'idf':0,
									'documents': {doc:new_doc},
									'index_in_vector': vector[term]}})

def append_idfs(data, N):
	for term in data:
		term['value']['idf'] = math.log10((N / len(term['value']['documents'])) + 1)

def append_weights(data):

	for term in data:
		for doc in term['value']['documents'].keys():
			term['value']['documents'][doc]['weight'] = get_weight_term_doc(data, term, (doc,term['value']['documents'][doc]))

#TODO: Index doc[terms]
#TODO values = value
def get_weight_term_doc(data, term, doc):
	# json_get_request = index.start(json.dumps({'action':'get', 'key':term}))
	# dic_get_request = json.loads(json_get_request)

	term = term['value']

	weight = doc[1]['tf'] * term['idf']

	ter_doc = [tm for tm in data if doc[0] in tm['value']['documents'].keys()]
	list_for_sum = [(tm['value']['idf'] * tm['value']['documents'][dc]['tf']) ** 2 for tm in ter_doc for  dc in tm['value']['documents'] if
					dc == doc[0]]

	doc_norm = math.sqrt(sum(list_for_sum))

	return weight / doc_norm

#TODO: proceamiento de texto
def get_vector(docs_text):

	all_docs_together = ""
	vector_of_terms = []
	i=0
	for doc in docs_text:
		print(os.path.basename(doc[0]))
		i+=1
		json_process = json.dumps({'action': 'process', 'data': doc[1]})
		json_request =json.loads(text_proc.text_work(json_process))
		vector_of_terms = vector_of_terms + json_request['terms']
	vector_of_terms = list(set(vector_of_terms))
	return  {vector_of_terms[index_term]: index_term for index_term in range(len(vector_of_terms))}

def get_minterm_of_document(vector, doc_terms):
	minterm = ""
	for tm in vector.keys():
		if tm in doc_terms:
			minterm += "1"
		else:
			minterm += "0"

	return minterm

##########################################################

##########################################################
############### PROCESAMIENTO DOCUMENTOS #################
##########################################################

def get_docs_from_query(query_terms):

	docs = {}
	for term in query_terms:
		json_respons = json.loads(index.start(json.dumps({'action':'get', 'key':term})))
		if json_respons['success']:
			term_value = json_respons['value']
			for doc in term_value['documents'].keys():
				if doc not in docs:
					docs[doc] = {'terms': {term: 0}, 'weights': [term_value['documents'][doc]['weight']]}
				else:
					docs[doc]['terms'][term] = len(docs[doc]['weights'])
					docs[doc]['weights'].append(term_value['documents'][doc]['weight'])

	return docs

def read_doc(doc_path):
	return read_txt(doc_path) if doc_path.endswith('.txt') else read_pdf(doc_path) if doc_path.endswith('.pdf') else\
		read_html(doc_path) if doc_path.endswith('.html') else read_docx(doc_path)

def read_txt(txt_path):
	with open(txt_path, "r", encoding="utf8", errors="ignore") as f:
		document = f.read()
		f.close()
		return document

# TODO: PDF
def read_pdf(pdf_path):
	pass

#TODO: HTML
def read_html(html_path):
	pass

#TODO: DOCX
def read_docx(docx_path):
	pass

##########################################################

##########################################################
###################### FORMULAS MVG ######################
##########################################################

#TODO:EXPLICAR
def term_i_Dot_term_j(term_i, term_j):
	result = 0
	analized_minterm = []
	for doc_tm_i in term_i['documents']:
		if term_i['documents'][doc_tm_i]['minterm'] not in analized_minterm and \
				term_i['documents'][doc_tm_i]['minterm'][term_i['index_in_vector']] == '1' and term_i['documents'][doc_tm_i]['minterm'][term_j['index_in_vector']] == '1':
			result += ci_r(term_i,term_i['documents'][doc_tm_i]['minterm'])*ci_r(term_j,term_i['documents'][doc_tm_i]['minterm'])
			analized_minterm.append(term_i['documents'][doc_tm_i]['minterm'])

	for  doc_tm_j in term_j['documents']:
		if term_j['documents'][doc_tm_j]['minterm'] not in analized_minterm and \
				term_j['documents'][doc_tm_j]['minterm'][term_i['index_in_vector']] == '1' and term_j['documents'][doc_tm_j]['minterm'][term_j['index_in_vector']] == '1':
			result += ci_r(term_i,term_j['documents'][doc_tm_j]['minterm'])*ci_r(term_j,term_j['documents'][doc_tm_j]['minterm'])
			analized_minterm.append(term_j['documents'][doc_tm_j]['minterm'])

	return result

def ci_r(term_i, minterm_r):
	return sum([term_i['documents'][doc]['weight'] for  doc in term_i['documents'] if term_i['documents'][doc]['minterm'] == minterm_r ])

##########################################################

##########################################################
################### SIMILARITY METRICS ###################
##########################################################

def similarity_cosine(query_data, docs):
	dicc_ranking = {}

	for doc in docs.keys():
		numerador = 0
		sum_weight_in_doc =0
		sum_weight_in_query =0
		for tm_query_i in query_data:
			for tm_query_j in query_data:
				if tm_query_i['key'] in docs[doc]['terms'].keys():
					weight_query_i = tm_query_i['value']['documents']['query']['weight']
					index_weight_data = docs[doc]['terms'][tm_query_i['key']]
					weight_doc_j = docs[doc]['weights'][index_weight_data]
					tmI = {}
					tmJ = {}
					tm = get_from_index(tm_query_i['key'])
					if tm['success']:
						tmI = tm['value']
					tm = get_from_index(tm_query_j['key'])

					if tm['success']:
						tmJ = tm['value']
					tmI_dot_tmJ = term_i_Dot_term_j(tmI, tmJ)

					numerador += weight_query_i * weight_doc_j * tmI_dot_tmJ

			if tm_query_i['key'] in docs[doc]['terms'].keys():

				weight_query = tm_query_i['value']['documents']['query']['weight']
				sum_weight_in_query += weight_query **2

		json_request = json.dumps({'action':'terms', 'key':doc})
		json_responsive = index.start(json_request)
		result = json.loads(json_responsive)
		terms_in_doc = result['terms']

		for tm_doc in terms_in_doc:
			tm = get_from_index(tm_doc)['value']
			for d in tm['documents']:
				if d == doc:
					sum_weight_in_doc += tm['documents'][d]['weight']**2

		denominador = math.sqrt(sum_weight_in_doc*sum_weight_in_query)

		dicc_ranking[doc] = numerador/denominador

	return dicc_ranking

def similarity_vectorial_model(query_data, docs):
	dicc_ranking = {}

	for doc in docs.keys():
		for tm_query in query_data:
			if tm_query['key'] in docs[doc]['terms'].keys():

				weight_query = tm_query['value']['documents']['query']['weight']
				index_weight_data = docs[doc]['terms'][tm_query['key']]
				weight_data = docs[doc]['weights'][index_weight_data]
				if doc not in dicc_ranking:
					dicc_ranking[doc] = weight_query * weight_data
				else:
					dicc_ranking[doc] += weight_query * weight_data
	return dicc_ranking

##########################################################

def get_from_index(term):
	return json.loads(index.start(json.dumps({'action':'get', 'key':term})))


'''
Given a list of integers S and a target number k, write a function that returns a subset of S that adds up to k. If such a subset cannot be made, then return null.
Integers can appear more than once in the list. You may assume all numbers in the list are positive.
For example, given S = [12, 1, 61, 5, 9, 2] and k = 24, return [12, 9, 2, 1] since it sums up to 24.

Written By Austin Bursey
Created On: 7/16/2020
Updated On: 7/16/2020
'''
import numpy as np
def sum_to_k(S,k):

    n = len(S)
    dp_array = np.full((len(S)+1,k+1),False,dtype=bool)

    for i in range(len(S)+1) :
        dp_array[i][0] = True
    for i in range(1,len(S)+1):
        for j in range(1,k+1):
            val = S[i-1]
            if  j >= S[i-1] :
                dp_array[i][j] = dp_array[i-1][j] or dp_array[i-1][j-S[i-1]]
            else :
                dp_array[i][j] = dp_array[i-1][j]

    solution = np.zeros(0,dtype= int)
    find_the_answer(dp_array, n, k , solution, S)

def find_the_answer(dp_array, i , k , solution_set,S):
    if (i == 0 and k == 0 ) :
        print("{0}".format(solution_set))

        return
    elif(i == 0 ) :
        print("No Subsets")
        return
    if (dp_array[i-1][k]) :
        find_the_answer(dp_array, i-1 , k , solution_set,S)

    if(k >= S[i-1] and dp_array[i][k-S[i-1]]):
        solution_set = np.insert(solution_set,0,S[i-1])
        find_the_answer(dp_array,i-1, k-S[i-1] , solution_set, S )

if __name__ == "__main__":
    array = [1,2,3,4,5]
    k = 10

    sum_to_k(array, k )


from slavdict.dictionary.models import civilrus_convert
from slavdict.dictionary.models import CollocationGroup

LETTER = 'в'
OUTPUT_FILENAME = '/root/collogroups_without_meanings.tsv'

register = []
for cg in CollocationGroup.objects.all():
    if cg.host_entry.civil_equivalent.lower().startswith(LETTER):
        if not cg.all_meanings or all(
                not m.meaning.strip() and not m.gloss.strip()
                for m in cg.all_meanings):
            entry = cg.host_entry
            author = entry.authors.first().last_name
            register.append((author, entry, cg))

def sort_key(x):
    a, entry, cg = x
    e = (entry.civil_equivalent, entry.homonym_order)
    c = '; '.join(c.collocation for c in cg.collocations)
    c = civilrus_convert(c)
    return a, e, c

register.sort(key=sort_key)

with open(OUTPUT_FILENAME, 'w') as f:
    f.write('Автор\tСтатья\tСловосочетание\n')
    for (a, entry, collogroup) in register:
        e = '{0}{1} {2}'.format(entry.civil_equivalent,
                {1: '¹', 2: '²', 3: '³'}.get(entry.homonym_order, ''),
                entry.get_part_of_speech_display())
        cg = '; '.join(c.collocation for c in collogroup.collocations)
        f.write('{0}\t{1}\t{2}\n'.format(a, e, cg))


import re
from htd_utilities import *
from htd_collaterals import *
from htd_te_shared import *
import csv
import atexit

# -----------------------
#
# ------------------------

class htd_action_statistics(object):
    # ----------------------
    def __init__(self):
        self.csvfile = open('htd_statistics.csv', 'w')
        self.csvwriter = csv.writer(self.csvfile)
        self.row_db = []
        self.__saved = 0
        self.header = []

    description = "The htd_statistics class collects statistical information per action "
    author = "alexse"

    # ---------------------
    def append_new_column(self, name):
        if (name not in self.header):
            self.header.append(name)
            # --------------------------------

    def capture_statistics(self, action_ptr):
        if (action_ptr.is_inner_action()):
            return
        self.append_new_column("action_name")
        self.append_new_column("action_type")
        for arg in action_ptr.get_declared_arguments():
            if (arg not in self.header):
                self.header.append(arg)
                for r in self.row_db:
                    r.append("-")
        # ------------------------------
        self.interactive = 0
        self.non_interactive = 0
        self.determine_if_interactive(action_ptr)
        self.append_new_column("interactive")
        self.append_new_column("non_interactive")
        # ------------------------------
        self.append_new_column("start_time")
        self.append_new_column("end_time")
        self.append_new_column("start_model_time")
        self.append_new_column("end_model_time")
        # -----------------------------------------
        self.append_new_column("documented")  # --The information of register is matched user requirest
        self.append_new_column("implicit_rtl_nodes_exists")  # --The explicit field rtl node - user specified
        self.append_new_column(
            "explicit_rtl_nodes_exists")  # --The implicit field rtl node - complimentory  to user specified
        self.append_new_column("ready4rmw")  # --   implicit_rtl_nodes_status +
        self.append_new_column("ready4check")  # --   implicit_rtl_nodes_status +  explicit_rtl_nodes_status
        self.append_new_column("ready4injection")  # --   explicit_rtl_nodes_status
        # ---------------------------------
        self.append_new_column("documented_details")
        self.append_new_column("implicit_rtl_details")
        self.append_new_column("explicit_rtl_details")
        # -----------------------

        new_row = []
        for h in self.header:
            if (h == "action_name"):
                new_row.append(action_ptr.get_action_name())
            elif (h == "action_type"):
                new_row.append(action_ptr.get_action_type())
            elif (h == "interactive"):
                new_row.append(self.interactive)
            elif (h == "non_interactive"):
                new_row.append(self.non_interactive)
            elif (h == "start_time"):
                new_row.append(str(action_ptr.action_time_start))
            elif (h == "end_time"):
                new_row.append(str(action_ptr.action_time_end))
            elif (h == "start_model_time"):
                new_row.append(str(action_ptr.action_model_time_start))
            elif (h == "end_model_time"):
                new_row.append(str(action_ptr.action_model_time_end))
            # ---------------
            elif (h == "documented"):
                new_row.append(action_ptr.documented)
            elif (h == "implicit_rtl_nodes_exists"):
                new_row.append(action_ptr.implicit_rtl_nodes_exists if action_ptr.documented else "0")
            elif (h == "explicit_rtl_nodes_exists"):
                new_row.append(action_ptr.explicit_rtl_nodes_exists if action_ptr.documented else "0")
            elif (h == "ready4rmw"):
                new_row.append(action_ptr.implicit_rtl_nodes_exists if action_ptr.documented else "0")
            elif (h == "ready4check"):
                new_row.append((
                               action_ptr.implicit_rtl_nodes_exists and action_ptr.explicit_rtl_nodes_exists) if action_ptr.documented else "0")
            elif (h == "ready4injection"):
                new_row.append(action_ptr.explicit_rtl_nodes_exists if action_ptr.documented else "0")
            elif (h == "documented_details"):
                new_row.append(action_ptr.documented_details)
            elif (h == "implicit_rtl_details"):
                new_row.append(action_ptr.implicit_rtl_details)
            elif (h == "explicit_rtl_details"):
                new_row.append(action_ptr.explicit_rtl_details)
            else:
                if (h not in list(action_ptr.arguments.arg_l.keys())):
                    new_row.append("-")
                else:
                    new_row.append(str(action_ptr.arguments.get_argument(h)))
        self.row_db.append(new_row)

    # ------------------------
    def snoop_csv_file(self):
        if (not self.__saved):
            self.row_db.insert(0, self.header)
            self.csvwriter.writerows(self.row_db)
            self.csvfile.close()
            self.__saved = 1

    def determine_if_interactive(self, action_ptr):
        if action_ptr.argument_exists('silent_mode') and action_ptr.argument_exists('dummy'):
            self.interactive = (
                not action_ptr.get_action_argument('silent_mode') and not action_ptr.get_action_argument(
                    'dummy'))  # if not dummy and not silent mode --> this is an action which is enabled for simulation
            self.non_interactive = (
                action_ptr.get_action_argument('silent_mode') and not action_ptr.get_action_argument(
                    'dummy'))  # if not dummy and silent mode --> action runs non_interactively, but hasn't neen enabled for interactive run (sim/emu)

# -------------------
HTD_STATISTICS_MGR = htd_action_statistics()

def snoop_statistics_file():
    HTD_STATISTICS_MGR.snoop_csv_file()

# define the at exit handler to properly close statistics file upon exit
atexit.register(snoop_statistics_file)


import subprocess
import datetime
import speech
def make_note(text):
    date = datetime.datetime.now()
    file_name = str(date).replace(":", "-") + "-note.txt"
    with open(file_name, "w") as f:
        f.write(text)

    subprocess.Popen(["notepad.exe", file_name])

    speech.speech("I've made a note of that.")