import smbus
import time

bus = smbus.SMBus(1)

Arduino_uno1 = 7
Arduino_uno2 = 8

def read_wire():
    return bus.read_i2c_block_data(Arduino_uno1, 1)

def write_wire(addr, value):
    bus.read_i2c_block_data(addr, value, 0)

while 1:
    try:
        write_wire(Arduino_uno2, read_wire())
        time.sleep(0.5)

    except KeyboardInterrupt:
        x = 0


import collections
import numpy as np
import h5py

dGBInfoDSName = '++info++'

def openFile( filenm, mode ):
  """ Opens and ensures and optimize access to the hdf5 file

  Parameters:
    * filenm (str): HDF5 file name
    * mode (str): options to read, write or edit to the file (r, r+, w, w-, x, a)

  Returns:
    * obj: A HDF5 file object (in mode selected)
  """

  return h5py.File( filenm, mode, libver=('v110', 'latest') )

def getInfoDataSet( h5file_or_grp ):
  """ Gets info on HDF5 object

  Parameters:
    * h5file_or_grp (object): hdf5 file object

  Returns:
    * obj: A h5 dataset object created by OpendTect,
           Input error is generated if HDF5 file is not created by OpendTect

  Notes:
    * KeyError (HDF5 key not found) is returned if dataset has no attribute passed
  """

  try:
    return h5file_or_grp[dGBInfoDSName]
  except KeyError:
    print( "Input Error: HDF5 file probably not created by OpendTect (no ++info++ attribute)" )
    raise

def ensureHasDataset( h5file_or_grp ):
  """ Ensures HDF5 object has info file (++info++) created by OpendTect

  Parameters:
    * h5file_or_grp (obj): HDF5 file object

  Returns:
    * obj: A h5 dataset object created by OpendTect
  """

  if dGBInfoDSName in h5file_or_grp:
    return getInfoDataSet( h5file_or_grp )
  return h5file_or_grp.create_dataset(dGBInfoDSName,dtype='S1')

def getAttr( dataset, ky ):
  """ Gets attribute value from HDF5 Dataset

  Parameters:
    * dataset (obj): HDF5 file object
    * ky (str): HDF5 attribute name

  Returns:
    bytes: Attribute value

  Notes:
    * KeyError (HDF5 key not found) is returned if dataset has no attribute passed
  """

  if not ky in dataset.attrs:
    print( "HDF5 key not found: '"+ky+"'. Available keys:\n")
    print( list(dataset.attrs.keys()) )
    raise KeyError

  attrib = dataset.attrs[ky]
  if isinstance(attrib,np.ndarray):
    return attrib[0]
  return np.bytes_( attrib ).decode().rstrip().split("\x00")[0]

def setAttr( dataset, ky, str ):
  """ Sets HDF5 dataset attribute

  Parameters:
    * dataset (obj): HDF5 file object
    * ky (str): attribute name
    * str (str): attribute value

  Returns:
    * Nothing (sets dataset attribute)
  """

  dataset.attrs[ky] = np.chararray.encode( str )

def hasAttr( dataset, ky ):
  """ Checks if HDF5 object has attribute

  Parameters:
    * dataset (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * bool: True if attribute is present, False if otherwise
  """

  return ky in dataset.attrs

def getAttrs( dataset, ky ):
  """ Gets attribute value(s) from HDF5 dataset

  Parameters:
    * dataset (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * list: of atrribute value(s)

  Notes:
    * KeyError (HDF5 key not found) is returned if dataset has no attribute passed
      List of available attributes is displayed
  """

  return getAttr( dataset, ky ).split( "`" )

def getText( infos, ky ):
  """ Gets attribute text value from HDF5 dataset

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * str: attribute value
  """

  ret = getAttr( infos, ky )
  if "`" in ret:
    return ret.split( "`" )
  return ret

def setArray( infos, ky, arr ):
  """ Sets array values for dataset attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name
    * arr (list, tup, int, float, str): attribute value(s)

  Returns:
    * Nothing (sets dataset attribute value(s))
  """

  try:
    arrstr = '`'.join( map(str,arr) )
  except TypeError:
    arrstr = str(arr)
  setAttr( infos, ky, arrstr )

def getBoolValue( infos, ky ):
  """ Gets boolean value

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    bool: True if attribute key evaluates to 'Yes', False if otherwise
  """

  return getAttr( infos, ky ) == "Yes"

def getIArray_( valsstr ):
  """ Provides integer equivalence of input. Converts input or element of input into integers

  Parameters:
    * valsstr (int, float, list of int/float)

  Returns:
    * int, list of int
  """

  if isinstance(valsstr,list) and len(valsstr) > 1:
    ret = list()
    for valstr in valsstr:
      ret.append( int(valstr) )
    return ret
  try:
    return int(valsstr)
  except ValueError:
    return int(float(valsstr))

def getIntValue( infos, ky ):
  """ Gets integer value of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * int: attribute value

  Notes:
    * Returns ValueError if attribute value is text
  """

  return getIArray_( getText(infos,ky) )

def getIInterval( infos, ky ):
  """ Gets interval int values of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * list: interval int values

  Notes:
    * Returns ValueError if attribute value is text
  """

  return getIArray_( getText(infos,ky) )

def getIStepInterval( infos, ky ):
  """ Gets interval step int values of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * tuple: interval step int values

  Notes:
    * Returns ValueError if attribute value is text
  """

  return getIArray_( getText(infos,ky) )

def getIArray( infos, ky ):
  """ Gets array int values of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * list: attribute int values

  Notes:
    * Returns ValueError if attribute value is text
  """

  return getIArray_( getText(infos,ky) )

def getDArray_( valsstr ):
  """ Provides float equivalence of input. Converts input or element of input into floats

  Parameters:
    * valsstr (int, float, list of int/float)

  Returns:
    * float, list of float
  """

  if isinstance(valsstr,list) and len(valsstr) > 1:
    ret = list()
    for valstr in valsstr:
      ret.append( float(valstr) )
    return ret
  return float(valsstr)

def getDValue( infos, ky ):
  """ Gets float value of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * float: attribute value

  Notes:
    * Returns ValueError if attribute value is text
  """
  return getDArray_( getText(infos,ky) )

def getDInterval( infos, ky ):
  """ Gets interval float values of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * list: interval float values

  Notes:
    * Returns ValueError if attribute value is text
  """

  return getDArray_( getText(infos,ky) )

def getDStepInterval( infos, ky ):
  """ Gets float interval step values of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * tuple: interval step float values

  Notes:
    * Returns ValueError if attribute value is text
  """

  return getDArray_( getText(infos,ky) )

def getDArray( infos, ky ):
  """ Gets array float values of attribute

  Parameters:
    * infos (obj): HDF5 file object
    * ky (str): attribute name

  Returns:
    * list: attribute float values

  Notes:
    * Returns ValueError if attribute value is text
  """

  return getDArray_( getText(infos,ky) )

def getAttribInfo( filenm ):
  h5file = openFile( filenm, 'r' )
  infods = getInfoDataSet( h5file )

  blocksversion = getIntValue(infods,"Blocks.Version")
  zdomain  = getText(infods,"ZDomain")
  datasetnm = getText(infods,"Name.Cube")
  compnms  = getText(infods,"Components")
  datachar = getText(infods,"Data storage")
  inlinerg    = getIInterval(infods,"In-line range")
  crosslinerg = getIInterval(infods,"Cross-line range")
  zrg      = getDStepInterval(infods,"Z range")
  inl_step = getIntValue(infods,"Step In-line")
  crl_step = getIntValue(infods,"Step Cross-line")
  blocksdim = getIInterval(infods,"Blocks.Max Dimensions")
  blocksinlrg = getIInterval(infods,"Blocks.Inl ID Range")
  blockscrlrg = getIInterval(infods,"Blocks.Crl ID Range")
  h5file.close()
  if zdomain == "TWT":
    zrg = np.multiply(zrg,1000).astype("int32")
    zrg = [zrg[0],zrg[1],zrg[2]]
  return collections.OrderedDict({
    'name': datasetnm,
    'attributes': compnms,
    'version': blocksversion,
    'zdomain': zdomain,
    'storage': datachar,
    'range': collections.OrderedDict({
      'Inline': [inlinerg[0],inlinerg[1],inl_step],
      'Crossline': [crosslinerg[0],crosslinerg[1],crl_step],
      'Z': zrg
    }),
    'block': {
      'size': blocksdim,
      'range': collections.OrderedDict({
        'Inline': blocksinlrg,
        'Crossline': blockscrlrg
      })
    }
  })

# Survey transformation (X-Y) to (Inl-Crl)
# Given an Inl-Crl pair: ic_pos = [inl,crl]
# the following returns the corresponding X,Y location:
# xy_pos = transform['origin'] + np.dot( transform['rotation'], ic_pos )
def getTransform( info ):
  coordsxbid = getDStepInterval(info,"Coord-X-BinID")
  coordsybid = getDStepInterval(info,"Coord-Y-BinID")
  return collections.OrderedDict({
    'origin':   [coordsxbid[0],coordsybid[0]],
    'rotation': [[coordsxbid[1],coordsxbid[2]],[coordsybid[1],coordsybid[2]]]
  })

def getSurveyInfo( filenm ):
  """ Gets survey information

  Parameters:
    * filenm (str): file name

  Returns:
    * orderedDict: containing information on survey name, survey ranges (inline, crossline, x, y), transform
  """

  h5file = openFile( filenm, 'r' )
  infods = getInfoDataSet( h5file )
  surveyname = getText(infods,"Name.Survey")
  inlstart = getIntValue(infods,"First In-line")
  inlstop  = getIntValue(infods,"Last In-line")
  crlstart = getIntValue(infods,"First Cross-line")
  crlstop  = getIntValue(infods,"Last Cross-line")
  xrg = getDInterval(infods,"X range")
  yrg = getDInterval(infods,"Y range")
  transform = getTransform( infods )
  h5file.close()
  return collections.OrderedDict({
    'name': surveyname,
    'range': collections.OrderedDict({
      'Inline': [inlstart,inlstop],
      'Crossline': [crlstart,crlstop],
      'X': xrg,
      'Y': yrg
    }),
    'transform': transform
  })


import sys

data = sys.stdin.read().splitlines()
print(data)

# 0=rock, 1=paper, 2=scissors
# N loses to N+1 (with wrap-around, so 3=rock)
# N beats N-1 (with wrap-around, so -1=scissors)
# (wrap-around meand modulo 3)
SYMBOLS = 'RPS'

def get_outcome(my, their):
    # 0 - lose (my == their-1)
    # 3 - draw (my == their)
    # 6 - win  (my == their+1)
    return (my - their + 1) % 3 * 3

def get_score_and_log(my, their):
    outcome = get_outcome(my, their)
    print(f"{SYMBOLS[my]}/{SYMBOLS[their]} {my+1:+} {outcome:+}")
    return (my + 1) + outcome

score = 0
for row in data:
    their, my = row.split()
    my = 'XYZ'.index(my)
    their = 'ABC'.index(their)
    score += get_score_and_log(my, their)

print('*** part 1:', score)

score = 0
move_scores = {m:s for s, m in enumerate('RPS', start=1)}
for row in data:
    their, my = row.split()
    their = 'ABC'.index(their)
    my = (their + 'XYZ'.index(my) - 1) % 3
    score += get_score_and_log(my, their)

print('*** part 2:', score)


from RPA.Browser.Selenium import Selenium
from datetime import datetime, timedelta
from SeleniumLibrary.errors import ElementNotFound
from selenium.common.exceptions import StaleElementReferenceException, ElementClickInterceptedException
from functions.logger import logger
from functions.utils import Utils

class NyTimes:

    def __init__(
        self,
        months: str,
        search_term: str,
        categories: str
    ):

        self.months = months
        self.search_term = search_term
        self.categories: categories
        self.utils = Utils()

    def open_browser(self) -> None:
        browser = Selenium()
        browser.open_available_browser()
        browser.maximize_browser_window()
        self.browser = browser
        logger.info("browser opened successfully")

    def get_date_range(self):
    # The code block you provided is a function called `get_date_range` that calculates the start and end
    # dates based on the number of months provided.
        number_of_months = int(self.months)
        current_date = datetime.now()
        start_current_day = current_date

        if (number_of_months == 1 or number_of_months == 0):
            number_of_months = 0
            current_day = current_date.today().day-1
            current_day
            start_current_day = current_date - timedelta(days=current_day)

        start_date = start_current_day - timedelta(days=number_of_months * 30)

        end_date = current_date

        start_date_str = start_date.strftime('%Y%m%d')
        end_date_str = end_date.strftime('%Y%m%d')
        self.start_date_str = start_date_str
        self.end_date_str = end_date_str

    def close_updated_terms(self) -> None :

        #Finals
        terms_button = '//div[@class="css-hqisq1"]//button'

        #Process
        try:
            if self.browser.find_element(terms_button) :
                self.browser.click_element(terms_button)
                logger.info("close update terms was closed")
        except:
            logger.warning("close updated terms was not found")

    def click_cookies(self) -> None:
        """
        This function accepts cookies on a webpage using RPA Selenium.

        :param browser: An instance of the RPA Browser Selenium class, which is used to automate web browsers
        for testing and web scraping purposes. It allows the script to interact with the web page and perform
        actions such as clicking buttons, filling out forms, and navigating through pages.
        :type browser: RPA.Browser.Selenium.Selenium
        """
        accept_button_xpath = "//button[@data-testid='GDPR-accept']"
        try:
            self.browser.wait_until_page_contains_element(accept_button_xpath)
            self.browser.click_element(accept_button_xpath)
            self.browser.reload_page()
            logger.info("click_cookies was clicked")
        except Exception as e:
            logger.error("Can't find or interact with cookies button: %s", str(e))

    def show_more(self) -> None:
        """
        The function `show_more` uses a while loop to continuously click a "show more" button on a web page
        until the button is no longer visible or an error occurs.

        :param browser: The parameter "browser" is of type Selenium, which is likely a reference to a
        Selenium WebDriver object. This object is used to automate web browser actions, such as navigating
        to URLs, interacting with elements on a webpage, and waiting for certain conditions to be met
        :type browser: Selenium
        """
        #Finals
        btn_show_more = "//button[@data-testid='search-show-more-button']"
        condition = True
        current_url = self.browser.get_location()
        while condition:
            if self.browser.get_location() == current_url:
                try:
                    self.browser.wait_until_element_is_visible(btn_show_more, 6)
                    self.browser.set_focus_to_element(btn_show_more)
                    self.browser.click_button(btn_show_more)
                except AssertionError:
                    logger.error("Error")
                    condition = False
                except ElementNotFound:
                    logger.warning("btn already not found")
                except StaleElementReferenceException:
                    logger.error("stale element reference")
                    self.browser.go_to(current_url)
                except ElementClickInterceptedException:
                    self.utils.screenshot(self.browser)
                    logger.error("Other element would receive the click")
                    self.browser.wait_until_element_is_visible(btn_show_more, 2)
                    self.click_cookies()
                    continue
            else:
                self.browser.go_to(current_url)

    def open_site(self)  -> Selenium :

        #Process
        # This code is using the RPA.Browser.Selenium library to open a web browser and navigate to a
        # specific URL.
        # The code is calling the `get_date_range` function to obtain the start and end dates based on the
        # number of months provided. It then uses these dates to construct a URL for a search query on the
        # New York Times website. The URL includes the start and end dates as query parameters, along with
        # the search term and sorting option.
        self.get_date_range()
        url = f"https://www.nytimes.com/search?dropmab=false&endDate={self.end_date_str}&query={self.search_term}&sort=best&startDate={self.start_date_str}"
        self.browser.go_to(url)
        self.close_updated_terms()
        self.click_cookies()
        logger.info("site opened successfully")
        return self.browser


from pprint import pprint
import random

# this won't work
try:
    from app import Game
except ModuleNotFoundError:
    print("Doesn't Work")

# Hacky solution!!
import sys
from pathlib import Path

def get_module_directory():
    # get the parent directory
    print(__file__)
    print(Path(__file__).parent.parent)
    package_dir = Path(__file__).parent.parent
    return str(package_dir)

module_dir = get_module_directory()
print("--------This is how sys.path is before we touch it---------------")
pprint(sys.path)
sys.path.append(module_dir)
print("--------We've now added our package_dir string to sys.path--------------")
pprint(sys.path)
# This now works!  But it's hacky
from app import Game


from msa_sdk.variables import Variables
from msa_sdk.msa_api import MSA_API
from msa_sdk.order import Order
from msa_sdk import util
import json

dev_var = Variables()
dev_var.add('name', var_type='String')
dev_var.add('device.0.target', var_type='Device')
dev_var.add('version', var_type='String')
dev_var.add('additional_device', var_type='String')
dev_var.add('additional_version', var_type='String')
context = Variables.task_call(dev_var)

process_id = context['SERVICEINSTANCEID']
matched_devices = dict()
error_devices = dict()

# GET the list of Devices that needs be updated (ie versionc an be imported and not the same as the target version)
devices = context['device']
for i in range(len(devices)):
#  util.log_to_process_file(process_id, devices[i]['target'])
  devicelongid = devices[i]['target'][-3:]
  order = Order(devicelongid)
  order.command_execute('IMPORT', {"Apache_Version":"0"})
  order.command_objects_instances("Apache_Version")
  if json.loads(order.content):
    version = json.loads(order.content)[0]
    order.command_objects_instances_by_id("Apache_Version", version)
    ver = json.loads(order.content)['Apache_Version'][version]['object_id']
#    util.log_to_process_file(process_id, ver)
    if ver != context['version']:
      matched_devices[devicelongid] = list()
      context['matched'] = list()
      matched_devices[devicelongid].append({'id': devicelongid})
      context['matched'].append({'id': devicelongid})
  else:
    version = "NULL"
    error_devices[devicelongid] = list()
    error_devices[devicelongid].append({'id': devicelongid})
#  util.log_to_process_file(process_id, version)

if context['additional_device']:
  devicelongid = context['additional_device'][-3:]
  order = Order(devicelongid)
  order.command_execute('IMPORT', {"Apache_Version":"0"})
  order.command_objects_instances("Apache_Version")
  if json.loads(order.content):
    version = json.loads(order.content)[0]
    order.command_objects_instances_by_id("Apache_Version", version)
    ver = json.loads(order.content)['Apache_Version'][version]['object_id']
#    util.log_to_process_file(process_id, ver)
    if ver != context['version']:
      matched_devices[devicelongid] = list()
      matched_devices[devicelongid].append({'id': devicelongid})
  else:
    version = "NULL"
    error_devices[devicelongid] = list()
    error_devices[devicelongid].append({'id': devicelongid})
#  util.log_to_process_file(process_id, version)

ret = MSA_API.process_content('ENDED', f'Apache will be updated on {matched_devices}. Errors happened on {error_devices}', context, True)
print(ret)


"""
URL configuration for sistema_coder project.

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path

from control_estudios.views import listar_albums, eliminar_canciones, filtrar_albums, filtrar_canciones, eliminar_albums, buscar_albums, buscar_canciones, crear_canciones, crear_albums, listar_canciones, listar_artistas, crear_artistas, eliminar_artistas,\
    buscar_artistas

urlpatterns = [
    path("albums/", listar_albums, name="lista_albums"),
    path("artistas/", listar_artistas, name="lista_artistas"),
    path("canciones/", listar_canciones, name="lista_canciones"),
    path("crear-artistas/", crear_artistas, name="crear_artistas"),
    path("crear-albums/", crear_albums, name="crear_albums"),
    path("crear-canciones/", crear_canciones, name="crear_canciones"),
    path("eliminar-artistas/", eliminar_artistas, name="eliminar_artistas"),
    path("eliminar-albums/", eliminar_albums, name="eliminar_albums"),
    path("eliminar-canciones/", eliminar_canciones, name="eliminar_canciones"),
    path("buscar-artistas/", buscar_artistas, name="buscar_artistas"),
    path("buscar-albums/", buscar_albums, name="buscar_albums"),
    path("buscar-canciones/", buscar_canciones, name="buscar_canciones"),
    path("filtrar-canciones/", filtrar_canciones, name="filtrar_canciones"),
    path("filtrar-albums/", filtrar_albums, name="filtrar_albums")
]


import os
import json
from tqdm import tqdm
import numpy as np
from PIL import Image
np.random.seed(2019)

YELP_CLASSES = {
	"inside" : 0,
	"outside": 1,
	"food"   : 2,
	"drink"  : 3,
	"menu"   : 4
}

meta_file = "data/raw/photo.json"
image_dir = "data/raw/photos"
save_path = "data/preprocessed/yelp_photos_{}.json"

metadata = open(meta_file).readlines()
n = len(metadata)
split_idx = n * np.array([0, 0.80, 1.0])
split_idx = split_idx.astype('int32')
idx = np.arange(n)
np.random.shuffle(idx)

for i, split in enumerate(tqdm(['train', 'test'])):

	json_data = []
	for j in tqdm(idx[split_idx[i]:split_idx[i+1]]):
		data = json.loads(metadata[j])
		image_path = os.path.join(image_dir, data["photo_id"] + '.jpg')
		image = Image.open(image_path)
		w, h = image.size

		json_data.append({
			'image_path': image_path,
			"label":YELP_CLASSES[data["label"]]
		})

	with open(save_path.format(split), mode='w') as json_file:
		json.dump(json_data, json_file)


import json
import csv

def main():
    dictobject = {}
    state_list = []
    year_list = []

    for num in xrange(1990, 2011):
        dictobject[str(num)] = {}

    with open('generation_annual.csv', 'rU') as generationcsv:
        generationreader = csv.reader(generationcsv)
        for row in generationreader:
            if (row[0] != 'YEAR'):
                year_list.append(row[0])
            if ((row[1] != '  ') and (row[1] != 'STATE_CODE')):
                state_list.append(row[1])

    year_set = sorted(set(year_list))
    state_set = sorted(set(state_list))

    for year in year_set:
        dictobject[year] = {}
        for state in state_set:
            dictobject[year][state] = {}

    with open('generation_annual.csv', 'rU') as generationcsv:
        generationreader = csv.reader(generationcsv)
        for row in generationreader:
            if ((row[0] != 'YEAR') and
                    (row[1] != '  ') and
                    (row[1] != 'STATE_CODE') and
                    (row[2] == 'Total Electric Power Industry')):
                dictobject[row[0]][row[1]].update({row[3]: row[4]})

    with open('generation_annual.json', 'w') as generationjson:
        json.dump(dictobject, generationjson, sort_keys=True, indent=4, separators=(',', ': '))
        generationjson.write('\n')

    generationcsv.close()
    generationjson.close()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

import sqlite3
from env import *
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
import datetime

#Connect to database

conn = sqlite3.connect('db_file')
cursor = conn.cursor()

#Create tables

# cursor.execute('''
# DROP TABLE transactions
# ''')
# conn.commit()

cursor.execute('''CREATE TABLE transactions (
transactions_id INTEGER PRIMARY KEY,
account INTEGER,
trade_date TEXT,
settlement_date TEXT,
trans_type TEXT,
investment_name TEXT,
category TEXT,
ticker TEXT,
shares REAL,
share_price REAL,
amount REAL,
principal_amount REAL);
''')
conn.commit()

# Connect and read from the spreadsheet

wb = load_workbook(f'{path}{csv_filename}')
ws = wb.active
wb.iso_dates = True

for row in range(2, 88):
    data = []
    data.append(row-1)
    for col in range(1,12):
        char = get_column_letter(col)
        cell_value = ws[f'{char}{row}'].value
        if type(cell_value) == str and '\xa0' in cell_value:
            cell_value = cell_value.replace('\xa0', ' ')
        if type(cell_value) == datetime.datetime:
            cell_value = str(cell_value)
        data.append(cell_value)
    data = tuple(data)
    print(data)
    cursor.execute(f'''INSERT INTO transactions VALUES{data}''')

conn.commit()

# #Simple SQLite shell:

# buffer = ""
#
# print("Enter your SQL commands to execute in sqlite3.")
# print("Enter a blank line to exit.")
#
# while True:
#     line = input()
#     if line == "":
#         break
#     buffer += line
#     if sqlite3.complete_statement(buffer):
#         try:
#             buffer = buffer.strip()
#             cursor.execute(buffer)
#             if buffer.lstrip().upper().startswith("SELECT"):
#                 result = cursor.fetchall()
#                 if result is []:
#                     print('Result set is empty')
#                 else:
#                     for i in result:
#                         print(i)
#         except sqlite3.Error as e:
#             print("An error occurred:", e.args[0])
#         buffer = ""

#Print out the table

# cursor.execute('''
# SELECT * FROM transactions
# WHERE Category = 'Balanced';
# ''')
# result = cursor.fetchall()
# col_names = []
# for i in cursor.description:
#     col_names.append(i[0])
# print(col_names)
# if result is []:
#     print('Result set is empty')
# else:
#     for i in result:
#         print(i)

#Check rows affected and close the db connection

def check_rows():
    if cursor.rowcount == -1:
        return 0
    else:
        return cursor.rowcount

print(f'Number of rows modified: {check_rows()}')
conn.close()


"""
Describes the abstract Corpus class that all interfaces to corpora should
subclass.
"""

from collections import namedtuple
import logging.config
import os
from pathlib import Path
import pickle
import random
import subprocess
from typing import Any, List, Callable, Optional, Set, Sequence, Tuple, Type, TypeVar

import numpy as np

from .config import ENCODING
from .preprocess import feat_extract
from . import utils
from .exceptions import PersephoneException
from .exceptions import LabelMismatchException
from .preprocess import elan, wav
from . import utterance
from .utterance import Utterance
from .preprocess.labels import LabelSegmenter

logger = logging.getLogger(__name__) # type: ignore

CorpusT = TypeVar("CorpusT", bound="Corpus")

def ensure_no_set_overlap(train: Sequence[str], valid: Sequence[str], test: Sequence[str]) -> None:
    """ Ensures no test set data has creeped into the training set."""

    logger.debug("Ensuring that the training, validation and test data sets have no overlap")
    train_s = set(train)
    valid_s = set(valid)
    test_s = set(test)

    if train_s & valid_s:
        logger.warning("train and valid have overlapping items: {}".format(train_s & valid_s))
        raise PersephoneException("train and valid have overlapping items: {}".format(train_s & valid_s))
    if train_s & test_s:
        logger.warning("train and test have overlapping items: {}".format(train_s & test_s))
        raise PersephoneException("train and test have overlapping items: {}".format(train_s & test_s))
    if valid_s & test_s:
        logger.warning("valid and test have overlapping items: {}".format(valid_s & test_s))
        raise PersephoneException("valid and test have overlapping items: {}".format(valid_s & test_s))

def find_untranscribed_wavs(wav_path: Path, transcription_path: Path, label_type: str) -> List[str]:
    """Find the prefixes for all the wav files that do not have an associated transcription
    Args:
        wav_path: Path to search for wav files in
        transcription_path: Path to search for transcriptions in
        label_type: The type of labels for transcriptions. Eg "phonemes" "phonemes_and_tones"
    Returns:
        A list of all untranscribed prefixes
    """
    audio_files = wav_path.glob("**/*.wav")
    transcription_files = transcription_path.glob("**/*.{}".format(label_type))

    transcription_file_prefixes = [t_file.stem for t_file in transcription_files]

    untranscribed_prefixes = [] # type: List[str]
    for a_file in audio_files:
        if a_file.stem not in transcription_file_prefixes:
            untranscribed_prefixes.append(a_file.stem)
    return untranscribed_prefixes

def get_untranscribed_prefixes_from_file(target_directory: Path) -> List[str]:
    """
    The file "untranscribed_prefixes.txt" will specify prefixes which
    do not have an associated transcription file if placed in the target directory.

    This will fetch those prefixes from that file and will return an empty
    list if that file does not exist.

    See find_untranscribed_wavs function for finding untranscribed prefixes in an
    experiment directory.

    Returns:
        A list of all untranscribed prefixes as specified in the file
    """

    untranscribed_prefix_fn = target_directory / "untranscribed_prefixes.txt"
    if untranscribed_prefix_fn.exists():
        with untranscribed_prefix_fn.open() as f:
            prefixes = f.readlines()

        return [prefix.strip() for prefix in prefixes]
    else:
        #logger.warning("Attempting to get untranscribed prefixes but the file ({})"
        #                " that should specify these does not exist".format(untranscribed_prefix_fn))
        pass
    return []

class Corpus:
    """ Represents a preprocessed corpus that is ready to be used in model
    training.

    Construction of a `Corpus` instance involves preprocessing data if the data has
    not previously already been preprocessed. The extent of the preprocessing
    depends on which constructor is used. If the default constructor,
    `__init__()` is used, transcriptions are assumed to already be preprocessed
    and only speech feature extraction from WAV files is performed. In other
    constructors such as `from_elan()`, preprocessing of the transcriptions is
    performed. See the documentation of the relevant constructors for more
    information.

    Once a Corpus object is created it should be considered immutable. At this
    point feature extraction from WAVs will have been performed, with feature
    files in `tgt_dir/feat/`.  Transcriptions will have been segmented into
    appropriate tokens (labels) and will be stored in `tgt_dir/label/`.

    """

    def __init__(self, feat_type: str, label_type: str, tgt_dir: Path,
                 *,
                 labels: Optional[Set[str]] = None,
                 max_samples: int=1000,
                 speakers: Optional[Sequence[str]] = None) -> None:
        """ Construct a `Corpus` instance from preprocessed data.

        Assumes that the corpus data has been preprocessed and is
        structured as follows: (1) WAVs for each utterance are found in
        `<tgt_dir>/wav/` with the filename `<prefix>.wav`, where `prefix` is
        some string uniquely identifying the utterance; (2) For each WAV file,
        there is a corresponding transcription found in `<tgt_dir>/label/` with
        the filename `<prefix>.<label_type>`, where `label_type` is some string
        describing the type of label used (for example, "phonemes" or "tones").

        If the data is found in the format, WAV normalization and speech
        feature extraction will be performed during `Corpus` construction, and
        the utterances will be randomly divided into training, validation and
        test_sets. If you would like to define these datasets yourself, include
        files named `train_prefixes.txt`, `valid_prefixes.txt` and
        `test_prefixes.txt` in `<tgt_dir>`. Each file should be a list of
        prefixes (utterance IDs), one per line. If these are found during
        `Corpus` construction, those sets will be used instead.

        Args:
            feat_type: A string describing the input speech features. For
                       example, "fbank" for log Mel filterbank features.
            label_type: A string describing the transcription labels. For example,
                         "phonemes" or "tones".
            labels: A set of strings representing labels (tokens) used in
                    transcription. For example: {"a", "o", "th", ...}.
                    If this parameter is not provided the experiment directory is
                    scanned for labels present in the transcription files.
            max_samples: The maximum number of samples an utterance in the
                corpus may have. If an utterance is longer than this, it is not
                included in the corpus.

        """

        if speakers:
            raise NotImplementedError("Speakers not implemented")

        logger.debug("Creating a new Corpus object with feature type {}, label type {},"
                     "target directory {}, label set {}, max_samples {}, speakers {}".format(
                    feat_type, label_type, labels, tgt_dir, max_samples, speakers)
        )

        # In case path is supplied as a string, make it a Path
        self.tgt_dir = Path(tgt_dir)

        #: A string representing the type of speech feature (eg. "fbank"
        #: for log filterbank energies).
        self.feat_type = feat_type

        #: An arbitrary string representing the transcription tokenization
        #: used (eg. "phonemes", "tones", "joint", or "characters").
        self.label_type = label_type

        # Setting up directories
        # Set the directory names
        self.feat_dir = self.get_feat_dir()
        self.wav_dir = self.get_wav_dir()
        self.label_dir = self.get_label_dir()

        logger.debug("Setting up directories for this Corpus object at %s", self.tgt_dir)
        self.set_and_check_directories(self.tgt_dir)

        # Label-related stuff
        if labels is not None:
            self.labels = labels
            found_labels = determine_labels(self.tgt_dir, label_type)
            if found_labels != self.labels:
                raise LabelMismatchException("User specified labels, {}, do"
                    " not match those automatically found, {}.".format(labels,
                    found_labels))
        else:
            self.labels = determine_labels(self.tgt_dir, label_type)
        self.vocab_size = len(self.labels)
        self.LABEL_TO_INDEX, self.INDEX_TO_LABEL = self.initialize_labels(self.labels)
        logger.info("Corpus label set: \n\t{}".format(self.labels))

        # This is a lazy function that assumes wavs are already in the WAV dir
        # but only creates features if necessary
        logger.debug("Preparing features")
        self.prepare_feats()
        self._num_feats = None

        self.train_prefixes = [] # type: List[str]
        self.valid_prefixes = [] # type: List[str]
        self.test_prefixes = [] # type: List[str]
        # This is also lazy if the {train,valid,test}_prefixes.txt files exist.
        self.make_data_splits(max_samples=max_samples)

        # Sort the training prefixes by size for more efficient training
        logger.debug("Training prefixes")
        self.train_prefixes = utils.sort_by_size(
            self.feat_dir, self.train_prefixes, feat_type)

        # Ensure no overlap between training and test sets
        try:
            ensure_no_set_overlap(
                self.get_train_fns()[0],
                self.get_valid_fns()[0],
                self.get_test_fns()[0]
            )
        except PersephoneException:
            logger.error("Got overlap between train, valid and test data sets")
            raise

        untranscribed_from_file = self.get_untranscribed_prefixes()
        untranscribed_found = find_untranscribed_wavs(self.get_wav_dir(), self.get_label_dir(), self.label_type)

        self.untranscribed_prefixes = list(set(untranscribed_from_file) | set(untranscribed_found))

        # TODO Need to contemplate whether Corpus objects have Utterance
        # objects or # not. Some of the TestBKW tests currently rely on this
        # for testing.
        self.utterances = []  # type: List[Utterance]

        self.pickle()

    @classmethod
    def from_elan(cls: Type[CorpusT], org_dir: Path, tgt_dir: Path,
                  feat_type: str = "fbank", label_type: str = "phonemes",
                  *,
                  utterance_filter: Callable[[Utterance], bool] = None,
                  label_segmenter: Optional[LabelSegmenter] = None,
                  speakers: List[str] = None, lazy: bool = True,
                  tier_prefixes: Tuple[str, ...] = ("xv", "rf")) -> CorpusT:
        """ Construct a `Corpus` from ELAN files.

        Args:
            org_dir: A path to the directory containing the unpreprocessed
                data.
            tgt_dir: A path to the directory where the preprocessed data will
                be stored.
            feat_type: A string describing the input speech features. For
                       example, "fbank" for log Mel filterbank features.
            label_type: A string describing the transcription labels. For example,
                         "phonemes" or "tones".
            utterance_filter: A function that returns False if an utterance
                should not be included in the corpus and True otherwise. This
                can be used to remove undesirable utterances for training, such as
                codeswitched utterances.
            label_segmenter: An object that has an attribute `segment_labels`,
                which is creates new `Utterance` instances from old ones,
                by segmenting the tokens in their `text` attribute. Note,
                `LabelSegmenter` might be better as a function, the only issue
                is it needs to carry with it a list of labels. This could
                potentially be a function attribute.
            speakers: A list of speakers to filter for. If `None`, utterances
                from all speakers are included.
            tier_prefixes: A collection of strings that prefix ELAN tiers to
                filter for. For example, if this is `("xv", "rf")`, then tiers
                named "xv", "xv@Mark", "rf@Rose" would be extracted if they
                existed.

        """
        # This currently bails out if label_segmenter is not provided
        if not label_segmenter:
            raise ValueError("A label segmenter must be provided via label_segmenter")

        # In case path is supplied as a string, make it a Path
        if isinstance(tgt_dir, str):
            tgt_dir = Path(tgt_dir)

        # Read utterances from org_dir.
        utterances = elan.utterances_from_dir(org_dir,
                                              tier_prefixes=tier_prefixes)

        # Filter utterances based on some criteria (such as codeswitching).
        if utterance_filter:
            utterances = [utter for utter in utterances if utterance_filter(utter)]
        utterances = utterance.remove_duplicates(utterances)

        # Segment the labels in the utterances appropriately
        if label_segmenter:
            utterances = [label_segmenter.segment_labels(utter) for utter in utterances]

        # Remove utterances without transcriptions.
        utterances = utterance.remove_empty_text(utterances)

        # Remove utterances with exceptionally short wav_files that are too
        # short for CTC to work.
        utterances = utterance.remove_too_short(utterances)

        tgt_dir.mkdir(parents=True, exist_ok=True)

        # TODO A lot of these methods aren't ELAN-specific. preprocess.elan was
        # only used to get the utterances. There could be another Corpus
        # factory method that takes Utterance objects. the fromElan and
        # fromPangloss constructors could call this.

        # Writes the transcriptions to the tgt_dir/label/ dir
        utterance.write_transcriptions(utterances, (tgt_dir / "label"),
                               label_type, lazy=lazy)
        # Extracts utterance level WAV information from the input file.
        wav.extract_wavs(utterances, (tgt_dir / "wav"), lazy=lazy)

        corpus = cls(feat_type, label_type, tgt_dir,
                     labels=label_segmenter.labels, speakers=speakers)
        corpus.utterances = utterances
        return corpus

    def get_wav_dir(self) -> Path:
        return self.tgt_dir / "wav"

    def get_feat_dir(self) -> Path:
        return self.tgt_dir / "feat"

    def get_label_dir(self) -> Path:
        return self.tgt_dir / "label"

    @property
    def train_prefix_fn(self) -> Path:
        return self.tgt_dir / "train_prefixes.txt"

    @property
    def valid_prefix_fn(self) -> Path:
        return self.tgt_dir / "valid_prefixes.txt"

    @property
    def test_prefix_fn(self) -> Path:
        return self.tgt_dir / "test_prefixes.txt"

    def set_and_check_directories(self, tgt_dir: Path) -> None:
        """
        Make sure that the required directories exist in the target directory.
        set variables accordingly.
        """
        logger.info("Setting up directories for corpus in %s", tgt_dir)

        # Check directories exist.
        if not tgt_dir.is_dir():
            raise FileNotFoundError(
                "The directory {} does not exist.".format(tgt_dir))
        if not self.wav_dir.is_dir():
            raise PersephoneException(
                "The supplied path requires a 'wav' subdirectory.")
        self.feat_dir.mkdir(parents=True, exist_ok=True)
        if not self.label_dir.is_dir():
            raise PersephoneException(
                "The supplied path requires a 'label' subdirectory.")

    def initialize_labels(self, labels: Set[str]) -> Tuple[dict, dict]:
        """Create mappings from label to index and index to label"""
        logger.debug("Creating mappings for labels")

        label_to_index = {label: index for index, label in enumerate(
                                 ["pad"] + sorted(list(labels)))}
        index_to_label = {index: phn for index, phn in enumerate(
                                 ["pad"] + sorted(list(labels)))}

        return label_to_index, index_to_label

    def prepare_feats(self) -> None:
        """ Prepares input features"""

        logger.debug("Preparing input features")
        self.feat_dir.mkdir(parents=True, exist_ok=True)

        should_extract_feats = False
        for path in self.wav_dir.iterdir():
            if not path.suffix == ".wav":
                logger.info("Non wav file found in wav directory: %s", path)
                continue
            prefix = os.path.basename(os.path.splitext(str(path))[0])
            mono16k_wav_path = self.feat_dir / "{}.wav".format(prefix)
            feat_path = self.feat_dir / "{}.{}.npy".format(prefix, self.feat_type)
            if not feat_path.is_file():
                # Then we should extract feats
                should_extract_feats = True
                if not mono16k_wav_path.is_file():
                    feat_extract.convert_wav(path, mono16k_wav_path)

        # TODO Should be extracting feats on a per-file basis. Right now we
        # check if any feats files don't exist and then do all the feature
        # extraction.
        if should_extract_feats:
            feat_extract.from_dir(self.feat_dir, self.feat_type)

    def make_data_splits(self, max_samples: int) -> None:
        """ Splits the utterances into training, validation and test sets."""

        train_f_exists = self.train_prefix_fn.is_file()
        valid_f_exists = self.valid_prefix_fn.is_file()
        test_f_exists = self.test_prefix_fn.is_file()

        if train_f_exists and valid_f_exists and test_f_exists:
            logger.debug("Split for training, validation and tests specified by files")
            self.train_prefixes = self.read_prefixes(self.train_prefix_fn)
            self.valid_prefixes = self.read_prefixes(self.valid_prefix_fn)
            self.test_prefixes = self.read_prefixes(self.test_prefix_fn)
            return

        # Otherwise we now need to load prefixes for other cases addressed
        # below
        prefixes = self.determine_prefixes()
        prefixes = utils.filter_by_size(
            self.feat_dir, prefixes, self.feat_type, max_samples)

        if not train_f_exists and not valid_f_exists and not test_f_exists:
            logger.debug("No files supplied to define the split for training, validation"
                         " and tests. Using default.")
            train_prefixes, valid_prefixes, test_prefixes = self.divide_prefixes(prefixes)
            self.train_prefixes = train_prefixes
            self.valid_prefixes = valid_prefixes
            self.test_prefixes = test_prefixes
            self.write_prefixes(train_prefixes, self.train_prefix_fn)
            self.write_prefixes(valid_prefixes, self.valid_prefix_fn)
            self.write_prefixes(test_prefixes, self.test_prefix_fn)
        elif not train_f_exists and valid_f_exists and test_f_exists:
            # Then we just make all other prefixes training prefixes.
            self.valid_prefixes = self.read_prefixes(self.valid_prefix_fn)
            self.test_prefixes = self.read_prefixes(self.test_prefix_fn)
            train_prefixes = list(
                set(prefixes) - set(self.valid_prefixes))
            self.train_prefixes = list(
                set(train_prefixes) - set(self.test_prefixes))
            self.write_prefixes(self.train_prefixes, self.train_prefix_fn)
        else:
            raise NotImplementedError(
                "The following case has not been implemented:" +
                "{} exists - {}\n".format(self.train_prefix_fn, train_f_exists) +
                "{} exists - {}\n".format(self.valid_prefix_fn, valid_f_exists) +
                "{} exists - {}\n".format(self.test_prefix_fn, test_f_exists))

    @staticmethod
    def read_prefixes(prefix_fn: Path) -> List[str]:
        if not prefix_fn.is_file():
            logger.critical("Expected a prefix file at path {}, but this path is"
                           " not a file".format(prefix_fn))
        assert prefix_fn.is_file(), "Path {} was not a file".format(prefix_fn)
        with prefix_fn.open() as prefix_f:
            prefixes = [line.strip() for line in prefix_f]
        if not prefixes:
            raise PersephoneException(
                "Empty prefix file {}. Either delete it\
                or put something in it".format(prefix_fn))
        return prefixes

    @staticmethod
    def write_prefixes(prefixes: List[str], prefix_fn: Path) -> None:
        if not prefixes:
            raise PersephoneException(
                "No prefixes. Will not write {}".format(prefix_fn))
        with prefix_fn.open("w") as prefix_f:
            for prefix in prefixes:
                print(prefix, file=prefix_f)

    @staticmethod
    def divide_prefixes(prefixes: List[str], *, seed:int=0) -> Tuple[List[str], List[str], List[str]]:
        """Divide data into training, validation and test subsets"""
        if len(prefixes) < 3:
            raise PersephoneException(
                "{} cannot be split into 3 groups as it only has {} items".format(prefixes, len(prefixes))
            )
        Ratios = namedtuple("Ratios", ["train", "valid", "test"])
        ratios=Ratios(.90, .05, .05)
        train_end = int(ratios.train*len(prefixes))
        valid_end = int(train_end + ratios.valid*len(prefixes))

        # We must make sure that at least one element exists in test
        if valid_end == len(prefixes):
            valid_end -= 1

        # If train_end and valid_end are the same we end up with no valid_prefixes
        # so we must ensure at least one prefix is placed in this category
        if train_end == valid_end:
            train_end -= 1

        random.seed(seed)
        random.shuffle(prefixes)

        train_prefixes = prefixes[:train_end]
        valid_prefixes = prefixes[train_end:valid_end]
        test_prefixes = prefixes[valid_end:]

        assert train_prefixes, "Got empty set for training data"
        assert valid_prefixes, "Got empty set for validation data"
        assert test_prefixes, "Got empty set for testing data"

        return train_prefixes, valid_prefixes, test_prefixes

    def indices_to_labels(self, indices: Sequence[int]) -> List[str]:
        """ Converts a sequence of indices into their corresponding labels."""

        return [(self.INDEX_TO_LABEL[index]) for index in indices]

    def labels_to_indices(self, labels: Sequence[str]) -> List[int]:
        """ Converts a sequence of labels into their corresponding indices."""

        return [self.LABEL_TO_INDEX[label] for label in labels]

    @property
    def num_feats(self):
        """ The number of features per time step in the corpus. """
        if not self._num_feats:
            filename = self.get_train_fns()[0][0]
            feats = np.load(filename)
            # pylint: disable=maybe-no-member
            if len(feats.shape) == 3:
                # Then there are multiple channels of multiple feats
                self._num_feats = feats.shape[1] * feats.shape[2]
            elif len(feats.shape) == 2:
                # Otherwise it is just of shape time x feats
                self._num_feats = feats.shape[1]
            else:
                raise ValueError(
                    "Feature matrix of shape %s unexpected" % str(feats.shape))
        return self._num_feats

    def prefixes_to_fns(self, prefixes: List[str]) -> Tuple[List[str], List[str]]:
        """ Fetches the file paths to the features files and labels files
        corresponding to the provided list of features"""
        # TODO Return pathlib.Paths
        feat_fns = [str(self.feat_dir / ("%s.%s.npy" % (prefix, self.feat_type)))
                    for prefix in prefixes]
        label_fns = [str(self.label_dir / ("%s.%s" % (prefix, self.label_type)))
                      for prefix in prefixes]
        return feat_fns, label_fns

    def get_train_fns(self) -> Tuple[List[str], List[str]]:
        """ Fetches the training set of the corpus.

        Outputs a Tuple of size 2, where the first element is a list of paths
        to input features files, one per utterance. The second element is a list
        of paths to the transcriptions.
        """
        return self.prefixes_to_fns(self.train_prefixes)

    def get_valid_fns(self) -> Tuple[List[str], List[str]]:
        """ Fetches the validation set of the corpus."""
        return self.prefixes_to_fns(self.valid_prefixes)

    def get_test_fns(self) -> Tuple[List[str], List[str]]:
        """ Fetches the test set of the corpus."""
        return self.prefixes_to_fns(self.test_prefixes)

    def get_untranscribed_prefixes(self) -> List[str]:
        """
        The file "untranscribed_prefixes.txt" will specify prefixes which
        do not have an associated transcription file if placed in the target directory.

        This will fetch those prefixes from that file and will return an empty
        list if that file does not exist.

        See find_untranscribed_wavs function for finding untranscribed prefixes in an
        experiment directory.
        """
        return get_untranscribed_prefixes_from_file(self.tgt_dir)

    def get_untranscribed_fns(self) -> List[str]:
        feat_fns = [os.path.join(str(self.feat_dir), "%s.%s.npy" % (prefix, self.feat_type))
                    for prefix in self.untranscribed_prefixes]
        return feat_fns

    def determine_prefixes(self) -> List[str]:
        label_prefixes = [str(path.relative_to(self.label_dir).with_suffix(""))
                          for path in
                          self.label_dir.glob("**/*.{}".format(self.label_type))]
        wav_prefixes = [str(path.relative_to(self.wav_dir).with_suffix(""))
                          for path in
                          self.wav_dir.glob("**/*.{}".format("wav"))]

        # Take the intersection; sort for determinism.
        prefixes = sorted(list(set(label_prefixes) & set(wav_prefixes)))

        if not prefixes:
            raise PersephoneException("""WARNING: Corpus object has no data. Are you sure
            it's in the correct directories? WAVs should be in {} and
            transcriptions in {} with the extension .{}""".format(
                self.wav_dir, self.label_dir, self.label_type))

        return prefixes

    def review(self) -> None:
        """ Used to play the WAV files and compare with the transcription. """

        for prefix in self.determine_prefixes():
            print("Utterance: {}".format(prefix))
            wav_fn = self.feat_dir / "{}.wav".format(prefix)
            label_fn = self.label_dir / "{}.{}".format(prefix,self.label_type)
            with label_fn.open() as f:
                transcript = f.read().strip()
            print("Transcription: {}".format(transcript))
            subprocess.run(["play", str(wav_fn)])

    def pickle(self) -> None:
        """ Pickles the Corpus object in a file in tgt_dir. """

        pickle_path = self.tgt_dir / "corpus.p"
        logger.debug("pickling %r object and saving it to path %s", self, pickle_path)
        with pickle_path.open("wb") as f:
            pickle.dump(self, f)

    @classmethod
    def from_pickle(cls: Type[CorpusT], tgt_dir: Path) -> CorpusT:
        pickle_path = tgt_dir / "corpus.p"
        logger.debug("Creating Corpus object from pickle file path %s", pickle_path)
        with pickle_path.open("rb") as f:
            return pickle.load(f)

def determine_labels(target_dir: Path, label_type: str) -> Set[str]:
    """ Returns a set of all phonemes found in the corpus. Assumes that WAV files and
    label files are split into utterances and segregated in a directory which contains a
    "wav" subdirectory and "label" subdirectory.

    Arguments:
        target_dir: A `Path` to the directory where the corpus data is found
        label_type: The type of label we are creating the label set from. For example
                    "phonemes" would only search for labels for that type.
    """
    logger.info("Finding phonemes of type %s in directory %s", label_type, target_dir)

    label_dir = target_dir / "label/"
    if not label_dir.is_dir():
        raise FileNotFoundError(
            "The directory {} does not exist.".format(target_dir))

    phonemes = set() # type: Set[str]
    for fn in os.listdir(str(label_dir)):
        if fn.endswith(str(label_type)):
            with (label_dir / fn).open("r", encoding=ENCODING) as f:
                try:
                    line_phonemes = set(f.readline().split())
                except UnicodeDecodeError:
                    logger.error("Unicode decode error on file %s", fn)
                    print("Unicode decode error on file {}".format(fn))
                    raise
                phonemes = phonemes.union(line_phonemes)
    return phonemes


#######################################################################################################################
#   file name: PSGD
#
#   description:
#   this file builds a grid-world environment.
#   then runs PSGD to solve the linear COIRL problem.
#######################################################################################################################
# imports:
#######################################################################################################################
import sys
sys.path.append('../../../include')
import numpy as np
from ICMDP import *
from domains import *
import pickle
import random
import os

#######################################################################################################################
# data savings
#######################################################################################################################
valuesindex = 0
if not os.path.exists('obj'):
    os.mkdir('obj')

def save_obj(obj, name):
    with open('obj/'+ name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(name):
    with open('obj/' + name + '.pkl', 'rb') as f:
        return pickle.load(f)

#######################################################################################################################
# HYPER PARAMS
#######################################################################################################################
gamma = 0.9
test_size = 100
max_iters = 1000
context_nums = np.asarray([100,200,300,400,500])
dims_vec = [(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10)]
repeats = 5
tol = 1e-5
RUN_TEST = True
batch_size = 12

#######################################################################################################################
# Train & Evaluation
#######################################################################################################################
for dims in dims_vec:
    print("####################################################################")
    print("%%%%%%%%%%%%% running with dimensions: " + str(dims[0]) + "X" + str(dims[1]))
    print("####################################################################")
    d = {}
    d["context_nums"] = np.asarray(context_nums)
    states = np.arange(dims[0]*dims[1])
    dim_contexts = len(states)

    all_contexts = np.load("../../../data/grid_world/rand_contexts_" + str(dims[0]) + "X" + str(dims[1]) + ".npy")
    test_contexts = all_contexts[all_contexts.shape[0] - test_size: all_contexts.shape[0]]

    mdp = build_domain('GRID', [dims[0], dims[1], False])

    real_W = np.eye(dim_contexts)
    test_expert_value = 0
    test_expert_value_ar = []
    policy_expert = []
    for context in test_contexts:
        feat_expert_i = mdp.solve_MDP(gamma, tol=tol, w=(context @ real_W), flag='uniform')
        policy_expert.append(feat_expert_i.policy)
        feat_expert = feat_expert_i.M
        test_expert_value_ar.append(context @ real_W @ feat_expert)
    test_expert_value = np.asarray(test_expert_value_ar).mean()
    d["test_value_expert"] = test_expert_value
    print("Expert test value: ", str(test_expert_value))

    d["train_value_expert"] = []
    for trainset in range(repeats):
        run_i = trainset
        for num_contexts in context_nums:
            contexts = all_contexts[trainset*context_nums[-1]:trainset*context_nums[-1] + num_contexts]
            training_features = []
            training_policy = []
            train_expert_value_ar = []
            real_W = np.eye(dim_contexts)
            for ctt in contexts:
                soll = mdp.solve_MDP(gamma,tol=tol,w=(ctt @ real_W),flag='uniform')
                training_policy.append(soll.policy)
                training_features.append(soll.M)
                train_expert_value_ar.append(ctt @ real_W @ soll.M)
            train_expert_value = np.asarray(train_expert_value_ar).mean()
            d["train_value_expert"].append(train_expert_value)
            W = np.ones([len(states), len(states)])
            W /= np.sum(W.flatten())
            W_mean = W.copy()
            for iter in range(max_iters):
                rand_sample = random.choices(list(range(num_contexts)), k=batch_size)
                r_est = contexts[rand_sample[0]] @ W / np.linalg.norm(W.flatten(), np.inf)
                features_agent = mdp.solve_MDP(gamma,tol=tol,w=r_est,flag='uniform')
                batch_outer = np.outer(contexts[rand_sample[0]], features_agent.M - training_features[rand_sample[0]])
                for sample in rand_sample[1:]:
                    r_est = contexts[sample] @ W / np.linalg.norm(W.flatten(), np.inf)
                    features_agent = mdp.solve_MDP(gamma, tol=tol, w=r_est, flag='uniform')
                    batch_outer += np.outer(contexts[sample], features_agent.M - training_features[sample])

                batch_outer /= batch_size

                W = W - 0.1/ np.sqrt(2 * dim_contexts * len(states) * (iter + 1)) * batch_outer
                W /= max((np.linalg.norm(np.reshape(W, W.size), np.inf)), 1.0)
                W_mean = (W_mean * iter + W) / (iter + 1)

            W_sol = [W_mean]
            train_agent_value_ar = []
            train_agent_accuracy_ar = []
            for cont in range(len(contexts)):
                solution = mdp.solve_MDP(gamma, tol=tol, w=contexts[cont] @ W_sol[0], flag='uniform')
                curr_policy = training_policy[cont]
                train_agent_value_ar.append(contexts[cont] @ real_W @ solution.M)
                train_agent_accuracy_ar.append(100.0 * np.asarray(np.sum(curr_policy == solution.policy) / float(len(curr_policy))).mean())

            train_agent_accuracy = np.asarray(train_agent_accuracy_ar).mean()
            train_agent_value = np.asarray(train_agent_value_ar).mean()
            print(" == Accuracy on train: ", str(train_agent_accuracy), "value on train: ", train_agent_value)
            d[trainset, "train_value", num_contexts] = train_agent_value
            d[trainset, "train_accuracy", num_contexts] = train_agent_accuracy

            save_obj(d, "coirl_values"+str(valuesindex))
            agent_vals = []
            agent_accurs = []
            test_agent_value_ar = []
            test_agent_accuracy_ar = []
            for cont in range(len(test_contexts)):
                solution = mdp.solve_MDP(gamma,tol=tol,w=test_contexts[cont] @ W_sol[0],flag='uniform')
                policy_expert_i = policy_expert[cont]
                test_agent_value_ar.append(test_contexts[cont] @ real_W @ solution.M)
                test_agent_accuracy_ar.append(100.0 * np.asarray(np.sum(policy_expert_i == solution.policy) / float(len(policy_expert_i))).mean())

            test_agent_accuracy = np.asarray(test_agent_accuracy_ar).mean()
            agent_accurs.append(test_agent_accuracy)
            test_agent_value = np.asarray(test_agent_value_ar).mean()
            agent_vals.append(test_agent_value)
            print(" == Generalization for ",str(num_contexts), "contexts: ", ", accuracy: ",str(test_agent_accuracy), " value: ",str(test_agent_value))
            d[trainset,"test_value",num_contexts] = agent_vals
            d[trainset,"accuracy",num_contexts] = agent_accurs

    save_obj(d, "coirl_values"+str(valuesindex))
    valuesindex += 1


import time
import selenium
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
from browser_alone import WebDriverFactory
import re
from page_methods import wait_for_open
from wait_element import *
from file_work import list_books
from locators import *
from after_download import *

url = "https://audio-joiner.com/ru/"
browser = WebDriverFactory().driver

def get_url(link):
    browser.get(link)
    if wait_for_open():
        return True
    else:
        return False

def drag_and_drop_mp3file(pathed):
    if wait_presence(*Locators.add):
        browser.find_element(*Locators.add).send_keys(pathed)  # field_drag_and_drop

def cross_off():
    list_cross = browser.find_elements(*Locators.cross_on)  # cross_off
    for i in list_cross:
        browser.find_element(i).click()

def check_exists_by_xpath(how, what):
    try:
        browser.find_element(how, what)
    except NoSuchElementException:
        return False
    return True

def main(how_much=3):
    path = ''
    # print("Input path folder:")
    # input(path)
    # print("How much files?")
    # input(int(how_much))
    list_book = list_books(r"C:\Users\Serafim\Downloads\man_who_laughs")
    if get_url("https://audio-joiner.com/ru/"):
        for i in range(0, how_much):
            drag_and_drop_mp3file(list_book[i])
        browser.find_element(*Locators.save_button).click()
        wait_download_button(*Locators.download_button)
        browser.find_element(*Locators.download_button).click()
        while True:
            find_the_file()
            if re.search("audio-joiner", find_the_file()) and not re.search(".download", find_the_file()):
                break
        replace_file(find_the_file(), rename("test"), '1.mp3')
    browser.quit()
    print("File already downloaded")

main()


import unittest

from pypika import Columns, Table, Tables, VerticaQuery

class VerticaQueryTests(unittest.TestCase):
    table_abc = Table("abc")

    def test_select_query_with_hint(self):
        q = VerticaQuery.from_("abc").select("*").hint("test_hint")

        self.assertEqual('SELECT /*+label(test_hint)*/ * FROM "abc"', str(q))

    def test_insert_query_with_hint(self):
        query = VerticaQuery.into(self.table_abc).insert(1).hint("test_hint")

        self.assertEqual('INSERT /*+label(test_hint)*/ INTO "abc" VALUES (1)', str(query))

    def test_update_query_with_hint(self):
        q = VerticaQuery.update(self.table_abc).set("foo", "bar").hint("test_hint")

        self.assertEqual('UPDATE /*+label(test_hint)*/ "abc" SET "foo"=\'bar\'', str(q))

    def test_delete_query_with_hint(self):
        q = VerticaQuery.from_("abc").delete().hint("test_hint")

        self.assertEqual('DELETE /*+label(test_hint)*/ FROM "abc"', str(q))

class CopyCSVTests(unittest.TestCase):
    table_abc = Table("abc")

    def test_copy_from_file(self):
        with self.subTest("with table as string"):
            q = VerticaQuery.from_file("/path/to/file").copy_("abc")

            self.assertEqual(
                "COPY \"abc\" FROM LOCAL '/path/to/file' PARSER fcsvparser(header=false)",
                str(q),
            )

        with self.subTest("with table"):
            q = VerticaQuery.from_file("/path/to/file").copy_(self.table_abc)

            self.assertEqual(
                "COPY \"abc\" FROM LOCAL '/path/to/file' PARSER fcsvparser(header=false)",
                str(q),
            )

class CreateTemporaryTableTests(unittest.TestCase):
    new_table, existing_table = Tables("abc", "efg")
    foo, bar = Columns(("a", "INT"), ("b", "VARCHAR(100)"))
    select = VerticaQuery.from_(existing_table).select(existing_table.foo, existing_table.bar)

    def test_create_local_temporary_table(self):
        with self.subTest("with columns"):
            q = VerticaQuery.create_table(self.new_table).temporary().local().columns(self.foo, self.bar)

            self.assertEqual('CREATE LOCAL TEMPORARY TABLE "abc" ("a" INT,"b" VARCHAR(100))', str(q))

        with self.subTest("with select"):
            q = VerticaQuery.create_table(self.new_table).temporary().local().as_select(self.select)

            self.assertEqual(
                'CREATE LOCAL TEMPORARY TABLE "abc" AS (SELECT "foo","bar" FROM "efg")',
                str(q),
            )

    def test_create_local_table_without_temporary_raises_error(self):
        with self.assertRaises(AttributeError):
            VerticaQuery.create_table(self.new_table).local()

    def test_create_temporary_table_preserve_rows(self):
        with self.subTest("with columns"):
            q = VerticaQuery.create_table(self.new_table).temporary().preserve_rows().columns(self.foo, self.bar)

            self.assertEqual(
                'CREATE TEMPORARY TABLE "abc" ("a" INT,"b" VARCHAR(100)) ON COMMIT PRESERVE ROWS',
                str(q),
            )

        with self.subTest("with select"):
            q = VerticaQuery.create_table(self.new_table).temporary().preserve_rows().as_select(self.select)

            self.assertEqual(
                'CREATE TEMPORARY TABLE "abc" ON COMMIT PRESERVE ROWS AS (SELECT "foo","bar" FROM "efg")',
                str(q),
            )

    def test_create_table_preserve_rows_without_temporary_raises_error(self):
        with self.assertRaises(AttributeError):
            VerticaQuery.create_table(self.new_table).preserve_rows()


# -*- coding: utf-8 -*-
from appium import webdriver
from elements_id import *
from config import *
from appium_lib import *
__author__ = 'liaoben'

def search_clue(driver,content):
    # try:
    #     content = content.decode('utf-8')
    # except:
    #     pass
    if not element_exsist(driver,'id',SEARCH_EDITTEXT):
        get_to_search(driver,MODE)
    el = driver.find_element_by_id(SEARCH_EDITTEXT)
    el.send_keys(content)
    driver.press_keycode(66)
    driver.find_element_by_name(CLUESEARCHTABTXT).click()
    result = element_exsist(driver,'id',HEAD_CONTENT,)
    if result:
        return True
    else:
        return False


# -*- coding: utf-8 -*-
import os
from typing import Tuple

import torch
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
import numpy as np

import postproc
import util
from dataset import ExcerptDataset, BaseDataset
import metric
from plot import Plotter

class Evaluator:
    """
    Evaluator class, created to bundle all evaluation and helper functions into a single class.
    Reduces parameter count by storing important values as class variables.
    """

    def __init__(self, feature_type: str, scene: str, hyper_params: dict, network_params: dict, fft_params: dict,
                 model_path: str, device: torch.device, writer: SummaryWriter, plotter: Plotter):
        self.feature_type = feature_type
        self.scene = scene
        self.classes = util.get_scene_classes(scene)
        self.hyper_params = hyper_params
        self.network_params = network_params
        self.fft_params = fft_params
        self.device = device
        self.writer = writer
        self.plotter = plotter
        self.loss_fn = torch.nn.BCELoss()
        self.net = torch.load(model_path, map_location='cpu').to(device)

    def evaluate(self) -> None:
        # create datasets and loaders
        dev_dataset = BaseDataset(self.feature_type, self.scene, self.hyper_params, self.fft_params)
        eval_dataset = BaseDataset(self.feature_type, self.scene, self.hyper_params, self.fft_params,
                                   data_path=os.path.join('data', 'eval'))
        dev_set = ExcerptDataset(dev_dataset, self.feature_type, self.classes, self.hyper_params['excerpt_size'],
                                 self.fft_params, overlap_factor=1, rnd_augment=False)
        eval_set = ExcerptDataset(eval_dataset, self.feature_type, self.classes, self.hyper_params['excerpt_size'],
                                  self.fft_params, overlap_factor=1, rnd_augment=False)
        dev_loader = DataLoader(dev_set, batch_size=1, shuffle=False, num_workers=0)
        eval_loader = DataLoader(eval_set, batch_size=1, shuffle=False, num_workers=0)

        # evaluate on all individual files for both dev and eval set
        eval_loss, metrics_eval, metrics_pp_eval = self.evaluate_model_on_files(eval_loader)
        dev_loss, metrics_dev, metrics_pp_dev = self.evaluate_model_on_files(dev_loader)

        # write results to files and log parameters
        self.write_losses(dev_loss, eval_loss)
        self.write_metrics(metrics_dev, metrics_eval, metrics_pp_dev, metrics_pp_eval)
        filtered_results = self.log_params(metrics_eval, metrics_pp_eval)
        print(f'final eval ER: {filtered_results["final_metric/segment_based/overall/ER"]}')
        print(f'final eval F: {filtered_results["final_metric/segment_based/overall/F"]}')
        print(f'final eval ER (post-processed): {filtered_results["final_metric_pp/segment_based/overall/ER"]}')
        print(f'final eval F (post-processed): {filtered_results["final_metric_pp/segment_based/overall/F"]}')

    @staticmethod
    def write_losses(dev_loss, eval_loss):
        with open(os.path.join('results', 'final_losses.txt'), 'w') as f:
            print(f"Scores:", file=f)
            print(f"evaluation set loss: {eval_loss}", file=f)
            print(f"development set loss: {dev_loss}", file=f)

    @staticmethod
    def write_metrics(metrics_dev, metrics_eval, metrics_pp_dev, metrics_pp_eval):
        metrics_path = os.path.join('results', 'final', 'metrics')
        metrics_pp_path = os.path.join('results', 'final_pp', 'metrics')
        metric.write_dcase_metrics_to_file(metrics_eval, metrics_path, 'eval_average')
        metric.write_dcase_metrics_to_file(metrics_dev, metrics_path, 'dev_average')
        metric.write_dcase_metrics_to_file(metrics_pp_eval, metrics_pp_path, 'eval_average')
        metric.write_dcase_metrics_to_file(metrics_pp_dev, metrics_pp_path, 'dev_average')

    def log_params(self, metrics_eval, metrics_pp_eval):
        # combine parameter dictionaries into a single one
        all_params = {}
        for key in self.hyper_params.keys():
            all_params[key] = self.hyper_params[key]
        for key in self.network_params.keys():
            all_params[key] = self.network_params[key]
        for key in self.fft_params.keys():
            all_params[key] = self.fft_params[key]
        # flatten metric dictionary and filter out important metrics
        flat_results = util.flatten_dict(metrics_eval, 'final_metric')
        flat_results_pp = util.flatten_dict(metrics_pp_eval, 'final_metric_pp')
        filtered_results = metric.filter_metrics_dict(flat_results)
        filtered_results_pp = metric.filter_metrics_dict(flat_results_pp)
        # log hyper parameters and metrics to tensorboard
        for key in filtered_results_pp.keys():
            filtered_results[key] = filtered_results_pp[key]
        self.writer.add_hparams(all_params, filtered_results)
        return filtered_results

    def evaluate_model_on_files(self, dataloader: torch.utils.data.DataLoader) -> Tuple[torch.Tensor, dict, dict]:
        plot_per_file = True  # set to False to significantly reduce evaluation time
        plot_path = os.path.join('results', 'final', 'plots')
        metrics_path = os.path.join('results', 'final', 'metrics')
        plot_pp_path = os.path.join('results', 'final_pp', 'plots')
        metrics_pp_path = os.path.join('results', 'final_pp', 'metrics')
        loss = torch.tensor(0., device=self.device)
        file_targets, file_predictions, file_pp_predictions = [], [], []
        all_targets, all_predictions, all_pp_predictions = [], [], []
        curr_file = None
        with torch.no_grad():
            # loop over all excerpts
            for data in tqdm(dataloader, desc='evaluating', position=0):
                inputs, targets, audio_file, idx = data
                audio_file = audio_file[0]
                if curr_file is None:
                    curr_file = audio_file
                elif audio_file != curr_file:  # if filename changes:
                    # combine all targets and predictions from current file
                    concat_targets = np.concatenate(file_targets, axis=2)
                    concat_predictions = np.concatenate(file_predictions, axis=2)
                    concat_pp_predictions = np.concatenate(file_pp_predictions, axis=2)
                    all_targets.append(concat_targets)
                    all_predictions.append(concat_predictions)
                    all_pp_predictions.append(concat_pp_predictions)
                    # plot and compute metrics for current file
                    filename = os.path.split(curr_file)[-1]
                    if plot_per_file:
                        self.plotter.plot(concat_targets, concat_predictions, plot_path, filename, True)
                        self.plotter.plot(concat_targets, concat_pp_predictions, plot_pp_path, filename, True)
                    metrics = metric.compute_dcase_metrics([concat_targets], [concat_predictions], self.classes)
                    metric.write_dcase_metrics_to_file(metrics, metrics_path, filename)
                    metrics_pp = metric.compute_dcase_metrics([concat_targets], [concat_pp_predictions], self.classes)
                    metric.write_dcase_metrics_to_file(metrics_pp, metrics_pp_path, filename)
                    # set up variables for next file
                    curr_file = audio_file
                    file_targets.clear()
                    file_predictions.clear()
                    file_pp_predictions.clear()

                inputs = inputs.to(self.device, dtype=torch.float32)
                targets = targets.to(self.device, dtype=torch.float32)
                predictions = self.net(inputs)
                # accumulate data
                loss += self.loss_fn(predictions, targets)
                file_targets.append(targets.detach().cpu().numpy())
                detach_predictions = predictions.detach().cpu().numpy()
                file_predictions.append(detach_predictions)
                file_pp_predictions.append(postproc.post_process_predictions(detach_predictions))
            loss /= len(dataloader)
            # compute final metrics for all prediction on all files
            metrics = metric.compute_dcase_metrics(all_targets, all_predictions, self.classes)
            metrics_pp = metric.compute_dcase_metrics(all_targets, [postproc.post_process_predictions(pred) for pred in
                                                                    all_predictions], self.classes)
        return loss, metrics, metrics_pp


# -*- coding: utf-8 -*-

import base64

from Crypto.Cipher import DES
from crypto_comm import padding

class DESCrypt:
    """DES/DES3加解密功能封装"""

    def __init__(self, key_bytes, mode='cbc', padmode='zero', iv=None):
        self.CRType = DES  # 记录加密器类型
        if mode == 'ecb':
            emode = DES.MODE_ECB
        elif mode == 'cbc':
            emode = DES.MODE_CBC
        else:
            emode = DES.MODE_CBC

        # DES要求密钥长度必须为8
        key_bytes = key_bytes[:8]

        if iv is None:  # 根据是否有初始化向量决定如何构造加密器对象
            self.cryptor = self.CRType.new(key_bytes, emode)
        else:
            self.cryptor = self.CRType.new(key_bytes, emode, iv)

        self.padding = padding(self.CRType.block_size, padmode)

    def encrypt(self, bytes):
        """加密指定的字节数据"""
        pln = self.padding.pad(bytes)
        return self.cryptor.encrypt(pln)

    def decrypt(self, bytes):
        """解密指定的字节数据"""
        pln = self.cryptor.decrypt(bytes)
        return self.padding.unpad(pln)

def des_base64_decode(dat, key, iv=None, pad='pkcs7', mode='ecb'):
    try:
        key = key.encode('latin-1')
        if iv:
            iv = iv.encode('latin-1')

        des = DESCrypt(key, mode=mode, padmode=pad, iv=iv)
        chp = base64.b64decode(dat)
        edat = des.decrypt(chp)
        return edat.decode('utf-8'), ''
    except Exception as e:
        return '', str(e)