"""
file: wordData.py
description: This is a supporting module which includes class definitions and utility functions relevant to all tasks
language: python3
author: Calvin Wu, cwx7054@rit.edu
date: 12/04/2015
"""

from rit_lib import *

class YearCount(struct):
    _slots=((int,'year'),(int,'count'))

class WordTrend(struct):
    _slots=((str,'word'),(float,'trend'))

def readWordFile(fileName):
    """
    Creates a dictionary mapping words to lists of YearCount objects given the name of a unigram data file
    :param fileName: text file
    :return: dict
    """
    wordDict={}
    wordList=[]
    file=open('data/'+fileName)
    for line in file:
        line=line.strip()
        if line[0].isalpha():
            currWord=line
            toggle=False
            wordList=[]
        else:
            if toggle==True:
                wordList.append(YearCount(int(line[:4]),int(line[5:])))
            if toggle==False:
                wordList.append(YearCount(int(line[:4]),int(line[5:])))
                wordDict[currWord]=wordList
                toggle=True

    return wordDict

def totalOccurrences(word,words):
    """
    Returns the total occurrences that a word has appeared in print given a dictionary
    :param word: str
    :param words: dict
    :return: int
    """
    total=0
    for i in words[word]:
        total+=i.count
    return 'Total occurrences of ' + word + ': ' + str(total)

if __name__ == '__main__':
    words=readWordFile(input('Enter word file: '))
    print(totalOccurrences((input('Enter word: ')),words))


#Author: Ty Vareka
#Date: 1/24/2020
#Description:Program takes a positive integer and returns the number at that position of the Fibonacci sequence
def fib(n):
    a = 0
    b = 1
    if n==1:
        return a+b
    else:
        for val in range (2,n+1):
            c = a + b
            a = b
            b = c
        return c


import pandas as pd
import numpy as np
from qstock_searcher import get_stock_df
import schemas
import datetime
MEAN_PRICE_LONG_PERIOD=30
MEAN_PRICE_SHORT_PERIOD=10

class StockAnalyzer(object):

    def __init__(self,):
        return

    def get_stock_fund_info(self):
        pass

    def get_mean_prices(self, df):
        periods = [MEAN_PRICE_SHORT_PERIOD, MEAN_PRICE_LONG_PERIOD]
        for period in periods:
            col_name = f'{period}日均价'
            price_roll_window = df['close'].rolling(window=period)
            df[col_name] = price_roll_window.mean()
            df = df.fillna(0)

        df[f'{periods[0]}日均线上穿{periods[1]}日均线'] = (df[f'{periods[0]}日均价'].shift(1) < df[f'{periods[1]}日均价'].shift(1)) & (df[f'{periods[0]}日均价'] > df[f'{periods[1]}日均价'])
        df[f'{periods[0]}日均线下穿{periods[1]}日均线'] = (df[f'{periods[0]}日均价'].shift(1) > df[f'{periods[1]}日均价'].shift(1)) & (df[f'{periods[0]}日均价'] < df[f'{periods[1]}日均价'])
        return df

    def bollinger_bands(self, data, window, num_std):
        """
        计算布林带
        :param data: 股票价格数据
        :param window: 计算布林带所需的时间窗口
        :param num_std: 用于计算上下轨的标准差倍数
        :return: 中轨，上轨，下轨
        """
        rolling_mean = data.rolling(window=window).mean()
        rolling_std = data.rolling(window=window).std()

        upper_band = rolling_mean + (rolling_std * num_std)
        lower_band = rolling_mean - (rolling_std * num_std)

        return rolling_mean, upper_band, lower_band

    def get_bollinger_bands(self, df):
        df['Middle_Band'], df['Upper_Band'], df['Lower_Band'] = self.bollinger_bands(data=df['close'], window=20, num_std=2)
        df['Reach_Bollinger_Low'] = df['close'] <= df['Lower_Band']
        df['Reach_Bollinger_Up'] = df['close'] >= df['Upper_Band']
        df['Middle_Band_Trend'] = df['Middle_Band'] - df['Middle_Band'].shift(1)
        return df

    def macd(self, data, short_window, long_window, signal_window):
        """
        计算平滑异同移动平均线（MACD）
        :param data: 股票价格数据
        :param short_window: 短期EMA窗口
        :param long_window: 长期EMA窗口
        :param signal_window: MACD信号线窗口
        :return: MACD, Signal
        """
        ema_short = data.ewm(span=short_window).mean()
        ema_long = data.ewm(span=long_window).mean()
        macd = ema_short - ema_long
        signal = macd.ewm(span=signal_window).mean()

        return macd, signal

    def cal_macd(self, df):
        # 计算MACD
        short_window = 12
        long_window = 26
        signal_window = 9
        df['MACD'], df['Signal'] = self.macd(df['close'], short_window, long_window, signal_window)
        df['Histogram'] = df['MACD'] - df['Signal']
        # 计算交叉点
        df['Cross'] = np.where(df['MACD'] > df['Signal'], 1, -1)
        # 查找买入点和卖出点
        df['up_cross'] = df['Cross'].diff() == 2
        df['down_cross']= df['Cross'].diff() == -2
        return df

    def generate_tech_infos(self, code:str,date:datetime = None ) -> schemas.StockTechInfo:
        df= get_stock_df(code,date=date)
        df.to_csv('stock_data.csv')
        df = self.get_bollinger_bands(df)
        df = self.cal_macd(df)
        df = self.get_mean_prices(df)
        last_row = df.tail(1)
        price_info = schemas.StockPriceInfo(
            sample_frequency = schemas.SampleFrequency.daily.value,
            price = df.tail(15)['close'].values.tolist(),
            volume=df.tail(15)['volume'].values.tolist()
        )
        if last_row['Reach_Bollinger_Up'].iloc[0]:
            boll_status = schemas.BollStatus.reach_up.value
        elif last_row['Reach_Bollinger_Low'].iloc[0]:
            boll_status = schemas.BollStatus.reach_down.value
        else:
            boll_status =  schemas.BollStatus.normal.value

        boll_info = schemas.BollingerInfo(
            boll_status =  boll_status,
            boll_trend = schemas.BollTrend.up.value if last_row['Middle_Band_Trend'].iloc[0] >= 0 else schemas.BollTrend.down.value,
        )
        if last_row['up_cross'].iloc[0]:
            macd_cross_status = schemas.MACDCrossStatus.gold
        elif last_row['down_cross'].iloc[0]:
            macd_cross_status = schemas.MACDCrossStatus.dead
        else:
            macd_cross_status = schemas.MACDCrossStatus.normal
        macd_info=schemas.MACD(
            crossover= macd_cross_status,
            histogram=round(last_row['Histogram'].iloc[0],2)
        )
        if last_row[f'{MEAN_PRICE_SHORT_PERIOD}日均线上穿{MEAN_PRICE_LONG_PERIOD}日均线'].iloc[0]:
            mean_price_cross = schemas.MeanPriceStatus.up_cross
        elif last_row[f'{MEAN_PRICE_SHORT_PERIOD}日均线下穿{MEAN_PRICE_LONG_PERIOD}日均线'].iloc[0]:
            mean_price_cross = schemas.MeanPriceStatus.down_cross
        else:
            mean_price_cross = schemas.MeanPriceStatus.normal

        mean_price_info = schemas.MeanPriceInfo(
            sample_frequency = schemas.SampleFrequency.daily.value,
            long_period=MEAN_PRICE_LONG_PERIOD,
            short_period=MEAN_PRICE_SHORT_PERIOD,
            cross=mean_price_cross
        )

        # print(price_info.to_prompt())
        # print(boll_info.to_prompt())
        # print(macd_info.to_prompt())
        # print(mean_price_info.to_prompt())
        stock_tech_info=schemas.StockTechInfo(
            price_infos=[price_info],
            mean_price_infos=[mean_price_info],
            boll_info=boll_info,
            macd_info=macd_info
        )
        # print(stock_tech_info.to_prompt())
        return stock_tech_info

stock_analyzer = StockAnalyzer()
if __name__ == '__main__':
    analyzer = StockAnalyzer()
    # df= get_stock_df('000155')
    # print(df.columns)
    # # df = analyzer.get_mean_prices(df)
    # df = analyzer.get_bollinger_bands(df)
    # df = analyzer.cal_macd(df)
    # print(df)

    analyzer.generate_tech_infos('603778')


import pandas as pd
import numpy as np
import urllib.parse
import urllib.request
import os
import xmlschema
from .errors import MetricLookupException

class UniprotLookupError(MetricLookupException): pass

class UniprotLookup:
    """
    The score of a mutation is based on which uniprot features it is part of.
    This code can also be used to annotate mutations with the uniprot features (see uniprot_exploration).

    The features to use can be selected by listing the feature_types.
    To further filter the features, the description_contains argument can be used (this should be a list the same length
    as the feature_types).

    Example:
        u = UniprotLookup(
                feature_types=['topological domain', 'repeat'],
                description_contains=['Cytoplasmic', None],
            )
    This lookup will score a mutation 1 if it is in a topological domain with 'Cytoplasmic' in the description, or any
    'repeat'. All other mutations will be scored zero.

    """

    # Features generally have a single residue, or include a set of residues between the begin and end positions
    # However, disulfide bonds have a begin and end position but do not include the residues in between
    SPLIT_FEATURES = {'disulfide bond'}
    MATCH_VARIANT_FEATURES = {'sequence variant'}

    REQUIRED_COLS = ('position', 'begin_position', 'end_position', 'description')

    def __init__(self, uniprot_directory=None, store_xml=False, feature_types=None,
                 description_contains=None, uniprot_upload_lists='https://www.uniprot.org/uploadlists/',
                 uniprot_xml_url="https://www.uniprot.org/uniprot/{}.xml",
                 schema_location='https://www.uniprot.org/docs/uniprot.xsd', transcript_uniprot_mapping=None,
                 force_download=False, match_variant_change=True, name='Uniprot', verbose=False):
        """

        :param uniprot_directory: A directory to store xml files downloaded from uniprot.
        Will use files from this directory if they already exist and force_download=False.
        :param store_xml: If true, will store the xml files downloaded from Uniprot in the uniprot_directory.
        :param feature_types: List of the types to test. They include 'signal peptide', 'chain', 'topological domain',
         'transmembrane region', 'domain', 'repeat', 'region of interest', 'metal ion-binding site', 'site',
         'modified residue', 'glycosylation site', 'disulfide bond', 'cross-link', 'splice variant', 'mutagenesis site',
         'sequence conflict', 'helix', 'strand'.
        :param description_contains: Text that the description must contain. E.g. you might want only topological domains
        with "Cytoplasmic" in the description
        :param uniprot_upload_lists: URL for downloading the Uniprot xml files using a transcript_id.  FROM 25/05/22 THIS
        IS REPLACED TO A REQUEST TO https://rest.uniprot.org/uniprotkb/
        :param uniprot_xml_url: URL for downloading the Uniprot xml files from a Uniprot accession number.
        :param schema_location: Location of the Uniprot xml schema.
        :param transcript_uniprot_mapping: Sometimes uniprot may not know which entry matches the transcript, or will
        not return the desired match. For these cases, you can provide a dictionary like {ENST00000123456: P12345},
        or a file with lines like "ENST00000123456 P12345".
        A file in that format can be downloaded from ensembl biomart by selection the "Transcript stable ID" and
        "UniProtKB/Swiss-Prot ID". This can be useful for GRCh37.
        Any transcripts not in the given file/dictionary will be matched using the uniprot mapping as usual.
        :param force_download: Will download new data from Uniprot, even if a previous file exists.
        :param match_variant_change: Set to false to just match position of sequence variants.
        :param name: Name of the lookup to appear on plot axes.
        :param verbose: When running, will print some information about the Uniprot accession used.
        """
        self.verbose=verbose
        self.uniprot_upload_lists = uniprot_upload_lists
        self.uniprot_xml_url = uniprot_xml_url
        self.schema = xmlschema.XMLSchema(schema_location)
        self.transcript_uniprot_mapping = None
        if isinstance(transcript_uniprot_mapping, dict):
            self.transcript_uniprot_mapping = transcript_uniprot_mapping
        elif isinstance(transcript_uniprot_mapping, str):
            self.transcript_uniprot_mapping = {}
            with open(transcript_uniprot_mapping) as fh:
                for line in fh:
                    try:
                        transcript, uniprot = line.strip().split()
                    except ValueError as e:
                        # Skip any rows with wrong number of entries (e.g. where uniprot id is missing).
                        continue
                    self.transcript_uniprot_mapping[transcript] = uniprot
        elif transcript_uniprot_mapping is not None:
            raise UniprotLookupError('transcript_uniprot_mapping must be dictionary or file path')

        self.uniprot_directory = uniprot_directory
        self.store_xml = store_xml
        self.force_download = force_download
        if feature_types is not None:
            if isinstance(feature_types, str):
                self.feature_types = [feature_types]
            else:
                self.feature_types = feature_types
        else:
            self.feature_types = 'all'

        if description_contains is not None:
            if isinstance(description_contains, str):
                description_contains = [description_contains]
            if len(description_contains) != len(self.feature_types):
                raise ValueError("Must be same number of description matches as there are features")
            self.description_contains = description_contains
        else:
            self.description_contains = None

        self.match_variant_change = match_variant_change  # Set to false to just match position of sequence variants

        self.name = name  # Will appear on some plot axes

    def __call__(self, seq_object):
        return self._get_scores(seq_object.null_mutations, seq_object.transcript_id)

    def setup_project(self, project):
        if project.verbose and not self.verbose:
            self.verbose=True

    def get_uniprot_xml_from_acc(self, acc):
        url = self.uniprot_xml_url.format(acc)
        req = urllib.request.Request(url)
        with urllib.request.urlopen(req) as f:
            response = f.read()
        return response.decode('utf-8')

    def get_uniprot_xml_from_transcript_id(self, transcript_id):
        """
        From 25/05/22, the request to convert from ENSEMBL_TRS_ID to ACC no longer works without the version number on
        the transcript id. This may be made more flexible in the future.

        In the meantime, replacing with a request to rest.uniprot.org/uniprotkb/.

        :param transcript_id:
        :return:
        """
        # params = {
        #     'from': 'ENSEMBL_TRS_ID',
        #     'to': 'ACC',
        #     'format': 'xml',
        #     'query': transcript_id
        # }

        # data = urllib.parse.urlencode(params)
        # data = data.encode('utf-8')
        BASE_XML_URL = "https://rest.uniprot.org/uniprotkb/stream?format=xml&query=%28xref%3Aensembl-{}%29"
        req = urllib.request.Request(BASE_XML_URL.format(transcript_id))
        with urllib.request.urlopen(req) as f:
            response = f.read()
        return response.decode('utf-8')

    def get_pdb_structures_for_transcript(self, transcript_id):
        xml_dict = self.get_uniprot_xml_dict(transcript_id)
        return self._get_pdb_structures_from_xml_dict(xml_dict)

    def _float_resolution_entry(self, value):
        if isinstance(value, str):
            if value.endswith(" A"):
                return float(value[:-2])
        return float(value)

    def _float_resolution_column(self, df):
        df['resolution'] = df['resolution'].apply(self._float_resolution_entry)
        return df

    def _get_pdb_structures_from_xml_dict(self, xml_dict):
        """
        Returns a list of PDB IDs.
        :param xml_dict:
        :return:
        """
        try:
            pdb_structures = []
            for p in xml_dict['entry'][0]['dbReference']:
                if p['@type'] == 'PDB':
                    d = {'pdb_id': p['@id']}
                    for property in p['property']:
                        if property['@type'] == 'chains':
                            d['chains'] = property['@value'].split("=")[0]  # Ignore the residues here, can use SIFTS
                        else:
                            d[property['@type']] = property['@value']
                    pdb_structures.append(d)

        except KeyError as e:
            # No features for this uniprot gene
            return UniprotLookupError('No pdb structures found in xml_dict')
        pdb_structures = pd.DataFrame(pdb_structures)
        if 'resolution' in pdb_structures.columns:
            pdb_structures = self._float_resolution_column(pdb_structures)
        return pdb_structures

    def get_features_from_xml(self, xml_dict):
        try:
            features = xml_dict['entry'][0]['feature']
        except KeyError as e:
            # No features for this uniprot gene
            return None  # UniprotLookupError('No features found in xml_dict')

        # Need to unpack the feature dictionaries and make a pandas dataframe
        new_dicts = []
        for f in features:
            d = {}
            for k, v in f.items():
                if k.startswith("@"):
                    d[k[1:]] = v
                elif k == 'location':
                    # This is a dictionary with more dictionaries as the values
                    for k2, v2 in v.items():
                        if k2 == 'position':
                            d['position'] = v2['@position']
                            d['position_status'] = v2['@status']
                        elif k2 == '@sequence':
                            d['location_sequence'] = v2
                        else:
                            # try:
                            for k3, v3 in v2.items():
                                d[k2 + '_' + k3[1:]] = v3
                            # except AttributeError as e:
                            #     print(v2)
                                # raise e
                else:
                    d[k] = v

            new_dicts.append(d)

        d = pd.DataFrame(new_dicts)
        for col in self.REQUIRED_COLS:
            if col not in d.columns:
                d[col] = np.nan

        return d

    def get_uniprot_xml_dict(self, transcript_id):
        xml_dict = None
        uniprot_id = None
        if self.transcript_uniprot_mapping is not None:
            uniprot_id = self.transcript_uniprot_mapping.get(transcript_id)

        if self.uniprot_directory is not None and not self.force_download:  # Look for an already downloaded version of the uniprot entry
            if uniprot_id is not None:
                # First look for a file with the given accession number
                xml_path = os.path.join(self.uniprot_directory, transcript_id + '_' + uniprot_id + ".xml")
                if os.path.exists(xml_path):
                    xml_dict = self.schema.to_dict(xml_path)

            if xml_dict is None:
                xml_path = os.path.join(self.uniprot_directory, transcript_id + ".xml")
                if os.path.exists(xml_path):
                    xml_dict = self.schema.to_dict(xml_path)

                if xml_dict is not None and uniprot_id is not None:
                    # Check if the existing file matches the given accession id
                    accessions = xml_dict['entry'][0]['accession']
                    if uniprot_id not in accessions:
                        # Need to download the requested accession
                        xml = self.get_uniprot_xml_from_acc(uniprot_id)
                        xml_dict = self.schema.to_dict(xml)
                        if self.store_xml and self.uniprot_directory is not None:
                            with open(os.path.join(self.uniprot_directory,
                                                   transcript_id + '_' + uniprot_id + ".xml"), 'w') as fh:
                                fh.writelines(xml)

        if xml_dict is None:  #  No version already downloaded or forcing new download. Get the data from the uniprot api
            if uniprot_id is not None:
                # Get the requested uniprot entry
                xml = self.get_uniprot_xml_from_acc(uniprot_id)
                if len(xml) == 0:
                    # Failed to find the uniprot data
                    raise UniprotLookupError('Failed to download uniprot xml with accession id {}'.format(uniprot_id))
            else:
                # Use uniprot to find the correct entry using the transcript id
                xml = self.get_uniprot_xml_from_transcript_id(transcript_id)
                if len(xml) == 0:
                    # Failed to find the uniprot data
                    raise UniprotLookupError('Failed to find and download uniprot xml for transcript {}'.format(
                        transcript_id))

            xml_dict = self.schema.to_dict(xml)
            if self.store_xml and self.uniprot_directory is not None:
                if uniprot_id is not None:
                    xml_path = os.path.join(self.uniprot_directory,
                                            transcript_id + '_' + uniprot_id + ".xml")
                else:
                    xml_path = os.path.join(self.uniprot_directory, transcript_id + ".xml")
                with open(xml_path, 'w') as fh:
                    fh.writelines(xml)

        if self.verbose:
            if uniprot_id is None:
                uniprot_id = xml_dict['entry'][0]['accession'][0]
            print("Using uniprot accession {} for transcript id {}".format(uniprot_id, transcript_id))
            print("https://www.uniprot.org/uniprot/{}".format(uniprot_id))
        return xml_dict

    def get_uniprot_data(self, transcript_id):
        xml_dict = self.get_uniprot_xml_dict(transcript_id)

        return self.get_features_from_xml(xml_dict)

    def _add_annotation(self, value, new_text, sep):
        if pd.isnull(new_text):
            new_text = 'TRUE'
        if pd.isnull(value):
            return new_text
        else:
            return value + sep + new_text

    def annotate_dataframe(self, df, transcript_id, sep='|||', return_feature_columns=False):
        feature_columns = []  # List the columns used for the annotating of the mutations
        transcript_features = self.get_uniprot_data(transcript_id)
        if transcript_features is not None:
            if 'location_sequence' in transcript_features.columns:
                # Entry defined on an alternative isoform. Remove here as it could match to the wrong residues.
                transcript_features = transcript_features[pd.isnull(transcript_features['location_sequence'])]
            if self.feature_types == 'all':
                transcript_feature_types = transcript_features['type'].unique()
            else:
                transcript_feature_types = self.feature_types
            if self.description_contains is None:
                transcript_desc_contains = [None] * len(transcript_feature_types)
            else:
                transcript_desc_contains = self.description_contains

            df['score'] = np.nan
            for f, d in zip(transcript_feature_types, transcript_desc_contains):
                if d is None or d == "":
                    col = f
                    feature_rows = transcript_features[transcript_features['type'] == f]
                else:
                    col = f + '_' + d
                    feature_rows = transcript_features[(transcript_features['type'] == f) &
                                                   transcript_features['description'].str.contains(d)]

                df[col] = np.nan
                feature_columns.append(col)

                if f in self.SPLIT_FEATURES:  # Just matching the start and end position, not any residues in between
                    for i, row in feature_rows.iterrows():
                        df.loc[df['residue'].isin([row['begin_position'], row['end_position']]), 'score'] = 1
                        df.loc[df['residue'].isin([row['begin_position'], row['end_position']]), col] = \
                            df.loc[df['residue'].isin([row['begin_position'], row['end_position']]), col].apply(
                                lambda x: self._add_annotation(x,
                                                               row['description'],
                                                               sep))
                elif f in self.MATCH_VARIANT_FEATURES and self.match_variant_change:  # Also need to match the amino acid change of the variant
                    for i, row in feature_rows.iterrows():
                        pos = row['position']
                        ref_aa = row['original']
                        alt_aa = row['variation']
                        if not pd.isnull(alt_aa):
                            if len(alt_aa) > 1:
                                print(alt_aa)
                                print(len(alt_aa))
                                print(type(alt_aa))
                                print(row)
                            for aa in alt_aa:
                                df.loc[(df['residue'] == pos) & (df['aaref'] == ref_aa) &
                                       (df['aamut'] == aa), 'score'] = 1
                                df.loc[(df['residue'] == pos) & (df['aaref'] == ref_aa) &
                                       (df['aamut'] == aa), col] = df.loc[(df['residue'] == pos) & (df['aaref'] == ref_aa) &
                                       (df['aamut'] == aa), col].apply(lambda x: self._add_annotation(x, row['description'],
                                                                                sep))
                else:
                    for i, row in feature_rows.iterrows():
                        if pd.isnull(row['position']):  # Region of protein
                            start = row['begin_position']
                            end = row['end_position']
                        else:  # Single residue
                            start = row['position']
                            end = row['position']
                        df.loc[(df['residue'] >= start) & (df['residue'] <= end), 'score'] = 1
                        df.loc[(df['residue'] >= start) & (df['residue'] <= end), col] = \
                            df.loc[(df['residue'] >= start) &
                                   (df['residue'] <= end), col].apply(lambda x: self._add_annotation(x,
                                                                                row[
                                                                                'description'],
                                                                                sep))

        if return_feature_columns:
            return df, feature_columns
        return df

    def _get_scores(self, df, transcript_id):
        try:
            df = self.annotate_dataframe(df, transcript_id)
        except UniprotLookupError as e:
            print(type(e).__name__, e)
            return None

        return df['score'].fillna(value=0).values


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 21 14:49:28 2020

@author: Annoy
"""
import scrapy
import json
class QuotesSpider(scrapy.Spider):
    name = "css"
    start_urls = [
        'http://quotes.toscrape.com/page/1/']

    def parse(self, response):
        for quote in response.css('div.quote'):
            filename=(
                'C:\\Users\\Annoy\\Desktop\\Spyder\\Test\\Example\\Example\\spiders\\css-scraper-results.json'
                )
            data=json.dumps({
                    'text': quote.css('span.text::text').get(),
                    'author': quote.css('small.author::text').get(),
                    'tags': quote.css('div.tags a.tag::text').getall()
                })
            with open(filename,'a') as f:
                f.write(data)
        next_page = response.css('li.next a::attr(href)').get()
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)


import heapq
import sys

INF = sys.maxsize

n, m = map(int, sys.stdin.readline().split())
graph = {i: [] for i in range(1, n+1)}

for _ in range(m):
    u, v, w = map(int, sys.stdin.readline().split())
    graph[u].append((w, v))
    graph[v].append((w, u))

x, z = map(int, sys.stdin.readline().split())
p = int(sys.stdin.readline())
middle = set(list(map(int, sys.stdin.readline().split())))

def dijkstra(start):
    distance = {i: INF for i in range(1, n+1)}
    distance[start] = 0

    q = []
    heapq.heappush(q, (distance[start], start))

    while q:
        current_distance, current_node = heapq.heappop(q)

        if distance[current_node] < current_distance: continue

        for next_distance, next_node in graph[current_node]:
            weighted_distance = current_distance + next_distance
            if weighted_distance < distance[next_node]:
                distance[next_node] = weighted_distance
                heapq.heappush(q, (distance[next_node], next_node))

    return distance

result_x = dijkstra(x)
result_z = dijkstra(z)

answer = INF
for i in middle:
    answer = min(answer, result_x[i] + result_z[i])

if answer == INF:
    print(-1)
else:
    print(answer)


import io
from random import randrange
import numpy as np
import math

naranja = [[5,5.5],[4.5,5],[5.2,4.9]]
toronja = [[6.5,6.8],[5.9,6],[5.7,5.5]]
arrayNaraja = []
arrayToronja = []
x1 = [0,0]
y1 = [0,0]
desX1 = [0,0]
desY1 = [0,0]

for x in range(2):
    for y in range(3):
        x1[x] = x1[x] + naranja[y][x]
        y1[x] = y1[x] + toronja[y][x]

    x1[x] = x1[x] / 3
    y1[x] = y1[x] / 3

np.random.uniform(low=4.9, high=6.6)

maxLength= 100
resultado = []
for x in range(maxLength):
    numRanfom1 = np.random.uniform(low=4.9, high=6.6)
    numRanfom2 = np.random.uniform(low=4.9, high=6.6)
    numRanfom3 = np.random.uniform(low=4.9, high=6.6)
    numRanfom4 = np.random.uniform(low=4.9, high=6.6)
    arrayNaraja.append([numRanfom1,numRanfom2])
    arrayToronja.append([numRanfom3,numRanfom4])

rn = 0
rt = 0
naranjasf = []
toronjaf = []
for y in range(maxLength):
    rn = math.sqrt(math.pow((arrayNaraja[y][0]- x1[0]),2) + math.pow((x1[1] - arrayNaraja[y][1]),2))
    rt = math.sqrt(math.pow((arrayToronja[y][0]-y1[0]),2) + math.pow((y1[1] - arrayToronja[y][1]),2))

    if rn < rt:
        resultado.append([rn,rt,'Naranja'])

    else:
        resultado.append([rn,rt,'Toronja'])

print(resultado)


'''
满减凑单问题

假设双十一购物，有满xx减xx的活动（例如满100-30），我现在购物车中有充足的商品，商品价格各不相同，请帮我利用购物车内商品凑单，
每个商品只能用一次，使我获得最大优惠（即凑单商品总价大于等于100但最近接近100）

输入：商品价格数组，凑单金额
输出：总价超过凑单金额但最接近的金额

例如
输入 [90, 52, 30, 65, 20] 100
输出 102
'''
class Solution:
    def function(self, coins, capacity):
        # DP
        dp = [coins[i] for i in range(len(coins))]

        dp = [[0 for i in range(len(coins) + 1)] for _ in range(capacity + 1)]
        for i in range(1, len(coins) + 1):
            for j in range(1, capacity + 1):
                # if dp[i-1][j]
                dp[i][j] = dp[i-1][j] or dp[i-1][j-coins[i]]
        # BFS
        queue = [(0, coins[0])]
        visited = [False for i in range(len(coins))]
        visited[0] = True
        self.res = float("inf")
        while queue:
            cur, step = queue.pop(0)
            if step >= capacity:
                self.res = min(self.res, step)
                continue
            for i in range(len(coins)):
                if not visited[i]:
                    new_cur, new_step = i, step + coins[i]
                    queue.append((new_cur, new_step))
        return self.res if self.res != float("inf") else -1

        dp = [coins[i] for i in range(len(coins))]
        for i in range(len(coins)):
            for j in range(i):
                if dp[i] >= capacity and dp[j] + coins[i] >= capacity:
                    dp[i] = min(dp[i], dp[j] + coins[i])
                elif dp[i] < capacity and dp[j] + coins[i] < capacity:
                    dp[i] = max(dp[i], dp[j] + coins[i])
                # elif
                dp[i] = dp[j] + coins[i] if abs(dp[j] + coins[i] - capacity) < abs(dp[i] - capacity) else dp[i]
        # for i in dp:

03_DP问题/背包问题零钱问题/头条满减.py


#coding: utf-8

from random import random

import model

class BlockFreeController:

	def __init__(self, parent):
		self.parent = parent
		self.block = False

	def update(self):
		block_s = self.parent.block_s

		if(self.block == False):
			pos = ( int(random() * (self.parent.size[0] / block_s)) * block_s , int(random() * (self.parent.size[1] / block_s)) * block_s )
			self.block = model.BlockFree(False, pos)

			# while(self.block.collidelist(self.parent.stage_blocks) != -1):
			# 	pos = ( int(random() * (self.parent.size[0] / block_s)) * block_s , int(random() * (self.parent.size[1] / block_s)) * block_s )
			# 	self.block = model.BlockFree(False, pos)

		if(self.block != False):
			self.block.paint(self.parent.screenBuffer)


import re
from os import path, getcwd
from pandas import concat, ExcelFile, ExcelWriter, DataFrame
from tkinter import messagebox
from numpy import nan, setdiff1d

from .converter import convert_dict

def create_messagebox(text, is_error=True):
    """
    Display a message box with the given `text`.

    Parameters
    ----------
    text : str
        text message to be displayed.
    is_error : bool
        if True, an error message box will be shown.
        Otherwise, displays an information message box.

    Returns
    -------
    None.

    """
    if is_error:
        messagebox.showerror(title='Saving Request', message=text)
    else:
        messagebox.showinfo(title='Saving Request', message=text)

def on_enter(event):
    """Change the background color of a widget to blue."""
    event.widget['background'] = '#33E6FF'

def on_leave(event):
    """Change the background color of a widget to green."""
    event.widget['background'] = '#3DC70D'

def read_sheet(excel_file, sheet_name, skip_columns, use_rows):
    """Read one sheet of an Excel file."""
    df = excel_file.parse(
        sheet_name=sheet_name, skiprows=4,
        skipfooter=2, header=None)
    if df.at[0, 0] is nan:
        if df.at[1, 0] == 'Маҳсулот номи':
            skip_rows = 5
        else:
            skip_rows = 3
        df = excel_file.parse(
            sheet_name=sheet_name, skiprows=skip_rows,
            skipfooter=2, header=None)

    df = df.replace('Туманлар', '')
    df.iloc[0:2] = df.iloc[0:2].fillna('')
    df.columns = df.iloc[0:2].apply(
        lambda column: ''.join([row for row in column]), axis=0)
    df = df.drop(skip_columns, axis=1, errors='ignore')
    df = df.iloc[2:]
    try:
        df.set_index('Маҳсулот номи', inplace=True)
        df = df.loc[use_rows, :]
        df = df.transpose()
        return df
    except Exception as excp:
        create_messagebox(f'{sheet_name}: ' + str(excp))
        return DataFrame()

def read_file(filename):
    """Read an Excel file containing price information."""
    skip_columns = ['Респуб-лика бўйича ўртача', 'Вилоят бўйича ўртача', 'Шаҳар бўйича ўртача']
    use_rows = [
        'Мол гўшти', 'Сут, 1 литр', 'Тухум, 10 донаси',
        'Картошка', 'Ўсимлик ёғи', 'Гуруч', 'Буғдой уни', 'Шакар'
    ]
    excel_file = ExcelFile(filename)
    df_list = []
    for sheet in excel_file.sheet_names:
        if sheet not in ['laroux', '1700']:
            df_sheet = read_sheet(excel_file, sheet, skip_columns, use_rows)
            if not df_sheet.empty:
                df_list.append(df_sheet)
            else:
                return DataFrame()

    df_merged = concat(df_list)
    df_merged = df_merged.astype('float64', copy=True, errors='raise')
    temp_index = df_merged.index.to_series().replace(
        {re.compile(r'(.*) \*'): r'\1', re.compile(r'(.*)\*'): r'\1'}, regex=True
        )
    df_merged.index = temp_index.replace(convert_dict)
    df_merged = df_merged[df_merged.index.notnull()]
    return df_merged

def calculate_difference(old_filename, new_filename):
    """Calculate price changes for eight basic products."""
    extension_pattern = '.xlsx'
    if not re.search(extension_pattern, old_filename):
        old_filename += extension_pattern
    if not re.search(extension_pattern, new_filename):
        new_filename += extension_pattern

    date_pattern = r'\d{2}\.\d{2}\.\d{4}'
    old_date = re.search(date_pattern, old_filename)
    new_date = re.search(date_pattern, new_filename)
    if not old_date:
        create_messagebox('Old filename is invalid.')
        return
    elif not new_date:
        create_messagebox('New filename is invalid.')
        return
    elif not path.isfile(old_filename):
        create_messagebox(f'Old file doesn\'t exist in {getcwd()}')
        return
    elif not path.isfile(new_filename):
        create_messagebox(f'New file doesn\'t exist in {getcwd()}')
        return

    old_file = read_file(old_filename)
    if old_file.empty:
        create_messagebox(f'Look for the problem in {old_filename}', False)
        return
    new_file = read_file(new_filename)
    if new_file.empty:
        create_messagebox(f'Look for the problem in {new_filename}', False)
        return

    old_date = old_date.group()
    new_date = new_date.group()
    non_matches = setdiff1d(new_file.index, old_file.index)
    if non_matches.size > 0:
        create_messagebox('non-matching names: ' + str(non_matches))
        return

    percentage = new_file.subtract(old_file).div(old_file)
    excel_name = f'{old_date}-{new_date}-inflation.xlsx'
    writer = ExcelWriter(excel_name, engine='xlsxwriter')
    new_file.to_excel(writer, sheet_name='new_data',
                      encoding='utf-8', index_label='place')
    for column in percentage.columns:
        series_sorted = percentage[column].nlargest(10, keep='first')
        series_sorted.to_excel(writer, sheet_name=column,
                               encoding='utf-8', index_label='place')
    writer.save()
    create_messagebox(f'{excel_name} has been created.', False)


import csv
from flask import Flask, jsonify, request

app = Flask(__name__, static_url_path="")

@app.route("/", methods=["GET"])
def read_csv():
    employee_list = []

    with open('data/employees.csv', 'r') as file_csv:
        content = csv.DictReader(file_csv)

        for employee in content:
            employee_list.append(employee)

    return jsonify(employee_list)

if __name__ == "__main__":
    app.run(debug=True, host="0.0.0.0", port = 5050)


import math
import re

def text_preprocessing(text):
    # Setting every word to lower
    text = text.lower()

    # Removing punctuations
    text = re.sub(r'[()\[\]{}!-/–;:\'",<>./?@#$%^&*_“~\\]', ' ', text)

    # Removing digits
    text = re.sub(r'\d', '', text)

    # Removing sequential whitespaces
    text = re.sub(r'\s+', ' ', text)

    # Removing leading and trailing whitespaces
    text = text.strip()

    return text

def tokenization(text):
    text = text.split()
    text = list(set(text))
    text.sort()

    return text

def create_domain_count(text, domain) -> dict:
    domain_count = {}
    for word in text:
        # Split used to avoid count every character in a word   (EX: avoid count 'a' in great)
        split_sentence = domain.split()
        domain_count[word] = split_sentence.count(word)

    return domain_count

def create_domain_tf(domain_count) -> dict:
    total = sum(domain_count.values())
    domain_tf = {}
    for word, count in domain_count.items():
        domain_tf[word] = count / total

    return domain_tf

def create_idf(domain1_count, domain2_count, num_of_domains=2):
    idf = {}

    for (word, count1), (count2) in zip(domain1_count.items(), domain2_count.values()):
        # Calculate how many docs has the word
        total_docs_count = 0
        total_docs_count += 1 if count1 > 0 else 0
        total_docs_count += 1 if count2 > 0 else 0

        idf[word] = math.log(num_of_domains / total_docs_count, 10)

    return idf

def create_tf_idf(domain1_tf, idf):
    tf_idf = {}

    for (word, tf), (idf_value) in zip(domain1_tf.items(), idf.values()):
        tf_idf[word] = tf * idf_value

    return tf_idf


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
from future.builtins import *  # NOQA

import fnmatch
import inspect
import io
import os
import re
import unittest
import warnings

import obspy
from obspy.core.util import AttribDict
from obspy.core.inventory import (Inventory, Network, ResponseStage)
from obspy.core.util.base import NamedTemporaryFile
from lxml import etree
import obspy.io.stationxml.core

class StationXMLTestCase(unittest.TestCase):
    """
    """

    def setUp(self):
        self.maxDiff = 10000
        # Most generic way to get the actual data directory.
        self.data_dir = os.path.join(os.path.dirname(os.path.abspath(
            inspect.getfile(inspect.currentframe()))), "data")

    def _assert_station_xml_equality(self, xml_file_buffer,
                                     expected_xml_file_buffer):
        """
        Helper function comparing two BytesIO buffers contain Station XML
        files.
        """
        # utf-8 only needed PY2
        new_lines = [_i.decode('utf-8').strip().replace("'", '"')
                     for _i in xml_file_buffer.read().splitlines()]
        # utf-8 only needed PY2
        org_lines = [_i.decode('utf-8').strip().replace("'", '"')
                     for _i in expected_xml_file_buffer.read().splitlines()]

        # Remove the module lines from the original file.
        org_lines = [_i.strip() for _i in org_lines
                     if not _i.strip().startswith("<Module")]

        for new_line, org_line in zip(new_lines, org_lines):
            regex = "<(.*?) (.*?)/?>"

            def callback(pattern):
                part2 = " ".join(sorted(pattern.group(2).split(" ")))
                return "<%s %s>" % (pattern.group(1), part2)

            # resort attributes alphabetically
            org_line = re.sub(regex, callback, org_line, count=1)
            new_line = re.sub(regex, callback, new_line, count=1)
            self.assertEqual(org_line, new_line)

        # Assert the line length at the end to find trailing non-equal lines.
        # If it is done before the line comparison it is oftentimes not very
        # helpful as you do not know which line is missing.
        self.assertEqual(len(new_lines), len(org_lines))

    def test_is_stationxml(self):
        """
        Tests the _is_stationxml() function.
        """
        # Check positives.
        stationxmls = [os.path.join(self.data_dir, "minimal_station.xml")]
        for stat in stationxmls:
            self.assertTrue(obspy.io.stationxml.core._is_stationxml(stat))

        # Check some negatives.
        not_stationxmls = [
            "Variations-FDSNSXML-SEED.txt",
            "fdsn-station+availability-1.0.xsd", "fdsn-station-1.0.xsd"]
        not_stationxmls = [
            os.path.join(self.data_dir, os.path.pardir,
                         os.path.pardir, "data", _i) for _i in not_stationxmls]
        for stat in not_stationxmls:
            self.assertFalse(obspy.io.stationxml.core._is_stationxml(
                stat))

    def test_different_write_levels(self):
        """
        Tests different levels of writing
        """
        filename = os.path.join(self.data_dir, "stationxml_BK.CMB.__.LKS.xml")
        inv = obspy.read_inventory(filename)

        # Write to network level
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", level="network")
        file_buffer.seek(0, 0)

        network_inv = obspy.read_inventory(file_buffer)

        self.assertTrue(len(network_inv.networks) == len(inv.networks))

        for net in network_inv.networks:
            self.assertTrue(len(net.stations) == 0)

        # Write to station level
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", level="station")
        file_buffer.seek(0, 0)

        station_inv = obspy.read_inventory(file_buffer)

        for net in station_inv.networks:
            self.assertTrue(len(net.stations) == len(inv[0].stations))
            for sta in net.stations:
                self.assertTrue(len(sta.channels) == 0)

        # Write to channel level
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", level="channel")
        file_buffer.seek(0, 0)

        channel_inv = obspy.read_inventory(file_buffer)

        for net in channel_inv.networks:
            self.assertTrue(len(net.stations) == len(inv[0].stations))
            for sta in net.stations:
                self.assertTrue(len(sta.channels) == len(inv[0][0].channels))
                for cha in sta.channels:
                    self.assertTrue(cha.response is None)

    def test_read_and_write_minimal_file(self):
        """
        Test that writing the most basic StationXML document possible works.
        """
        filename = os.path.join(self.data_dir, "minimal_station.xml")
        inv = obspy.read_inventory(filename)

        # Assert the few values that are set directly.
        self.assertEqual(inv.source, "OBS")
        self.assertEqual(inv.created, obspy.UTCDateTime(2013, 1, 1))
        self.assertEqual(len(inv.networks), 1)
        self.assertEqual(inv.networks[0].code, "PY")

        # Write it again. Also validate it to get more confidence. Suppress the
        # writing of the ObsPy related tags to ease testing.
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", validate=True,
                  _suppress_module_tags=True)
        file_buffer.seek(0, 0)

        with open(filename, "rb") as open_file:
            expected_xml_file_buffer = io.BytesIO(open_file.read())
        expected_xml_file_buffer.seek(0, 0)

        self._assert_station_xml_equality(file_buffer,
                                          expected_xml_file_buffer)

    def test_subsecond_read_and_write_minimal_file(self):
        """
        Test reading and writing of sub-second time in datetime field,
        using creation time

        """
        filename = os.path.join(self.data_dir,
                                "minimal_station_with_microseconds.xml")
        inv = obspy.read_inventory(filename)

        # Write it again. Also validate it to get more confidence. Suppress the
        # writing of the ObsPy related tags to ease testing.
        file_buffer = io.BytesIO()

        inv.write(file_buffer, format="StationXML", validate=True,
                  _suppress_module_tags=True)
        file_buffer.seek(0, 0)

        with open(filename, "rb") as open_file:
            expected_xml_file_buffer = io.BytesIO(open_file.read())
        expected_xml_file_buffer.seek(0, 0)

        self._assert_station_xml_equality(file_buffer,
                                          expected_xml_file_buffer)

    def test_read_and_write_full_file(self):
        """
        Test that reading and writing of a full StationXML document with all
        possible tags works.
        """
        filename = os.path.join(self.data_dir, "full_random_stationxml.xml")
        inv = obspy.read_inventory(filename)

        # Write it again. Also validate it to get more confidence. Suppress the
        # writing of the ObsPy related tags to ease testing.
        file_buffer = io.BytesIO()

        inv.write(file_buffer, format="StationXML", validate=True,
                  _suppress_module_tags=True)
        file_buffer.seek(0, 0)

        with open(filename, "rb") as open_file:
            expected_xml_file_buffer = io.BytesIO(open_file.read())
        expected_xml_file_buffer.seek(0, 0)

        self._assert_station_xml_equality(file_buffer,
                                          expected_xml_file_buffer)

    def test_writing_module_tags(self):
        """
        Tests the writing of ObsPy related tags.
        """
        net = Network(code="UL")
        inv = Inventory(networks=[net], source="BLU")

        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", validate=True)
        file_buffer.seek(0, 0)
        lines = file_buffer.read().decode().splitlines()
        module_line = [_i.strip() for _i in lines if _i.strip().startswith(
            "<Module>")][0]
        self.assertTrue(fnmatch.fnmatch(module_line,
                                        "<Module>ObsPy *</Module>"))
        module_uri_line = [_i.strip() for _i in lines if _i.strip().startswith(
            "<ModuleURI>")][0]
        self.assertEqual(module_uri_line,
                         "<ModuleURI>https://www.obspy.org</ModuleURI>")

    def test_reading_other_module_tags(self):
        """
        Even though the ObsPy Tags are always written, other tags should be
        able to be read.
        """
        filename = os.path.join(
            self.data_dir,
            "minimal_with_non_obspy_module_and_sender_tags_station.xml")
        inv = obspy.read_inventory(filename)
        self.assertEqual(inv.module, "Some Random Module")
        self.assertEqual(inv.module_uri, "http://www.some-random.site")

    def test_reading_and_writing_full_root_tag(self):
        """
        Tests reading and writing a full StationXML root tag.
        """
        filename = os.path.join(
            self.data_dir,
            "minimal_with_non_obspy_module_and_sender_tags_station.xml")
        inv = obspy.read_inventory(filename)
        self.assertEqual(inv.source, "OBS")
        self.assertEqual(inv.created, obspy.UTCDateTime(2013, 1, 1))
        self.assertEqual(len(inv.networks), 1)
        self.assertEqual(inv.networks[0].code, "PY")
        self.assertEqual(inv.module, "Some Random Module")
        self.assertEqual(inv.module_uri, "http://www.some-random.site")
        self.assertEqual(inv.sender, "The ObsPy Team")

        # Write it again. Do not write the module tags.
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", validate=True,
                  _suppress_module_tags=True)
        file_buffer.seek(0, 0)

        with open(filename, "rb") as open_file:
            expected_xml_file_buffer = io.BytesIO(open_file.read())
        expected_xml_file_buffer.seek(0, 0)

        self._assert_station_xml_equality(
            file_buffer, expected_xml_file_buffer)

    def test_reading_and_writing_full_network_tag(self):
        """
        Tests the reading and writing of a file with a more or less full
        network tag.
        """
        filename = os.path.join(self.data_dir,
                                "full_network_field_station.xml")
        inv = obspy.read_inventory(filename)

        # Assert all the values...
        self.assertEqual(len(inv.networks), 1)
        net = inv.networks[0]
        self.assertEqual(net.code, "PY")
        self.assertEqual(net.start_date, obspy.UTCDateTime(2011, 1, 1))
        self.assertEqual(net.end_date, obspy.UTCDateTime(2012, 1, 1))
        self.assertEqual(net.restricted_status, "open")
        self.assertEqual(net.alternate_code, "PYY")
        self.assertEqual(net.historical_code, "YYP")
        self.assertEqual(net.description, "Some Description...")
        self.assertEqual(len(net.comments), 2)

        comment_1 = net.comments[0]
        self.assertEqual(comment_1.value, "Comment number 1")
        self.assertEqual(comment_1.begin_effective_time,
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(comment_1.end_effective_time,
                         obspy.UTCDateTime(2008, 2, 3))
        self.assertEqual(len(comment_1.authors), 1)
        authors = comment_1.authors[0]
        self.assertEqual(len(authors.names), 2)
        self.assertEqual(authors.names[0], "This person")
        self.assertEqual(authors.names[1], "has multiple names!")
        self.assertEqual(len(authors.agencies), 3)
        self.assertEqual(authors.agencies[0], "And also")
        self.assertEqual(authors.agencies[1], "many")
        self.assertEqual(authors.agencies[2], "many Agencies")
        self.assertEqual(len(authors.emails), 4)
        self.assertEqual(authors.emails[0], "email1@mail.com")
        self.assertEqual(authors.emails[1], "email2@mail.com")
        self.assertEqual(authors.emails[2], "email3@mail.com")
        self.assertEqual(authors.emails[3], "email4@mail.com")
        self.assertEqual(len(authors.phones), 2)
        self.assertEqual(authors.phones[0].description, "phone number 1")
        self.assertEqual(authors.phones[0].country_code, 49)
        self.assertEqual(authors.phones[0].area_code, 123)
        self.assertEqual(authors.phones[0].phone_number, "456-7890")
        self.assertEqual(authors.phones[1].description, "phone number 2")
        self.assertEqual(authors.phones[1].country_code, 34)
        self.assertEqual(authors.phones[1].area_code, 321)
        self.assertEqual(authors.phones[1].phone_number, "129-7890")

        comment_2 = net.comments[1]
        self.assertEqual(comment_2.value, "Comment number 2")
        self.assertEqual(comment_2.begin_effective_time,
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(comment_1.end_effective_time,
                         obspy.UTCDateTime(2008, 2, 3))
        self.assertEqual(len(comment_2.authors), 3)
        for _i, author in enumerate(comment_2.authors):
            self.assertEqual(len(author.names), 1)
            self.assertEqual(author.names[0], "Person %i" % (_i + 1))
            self.assertEqual(len(author.agencies), 1)
            self.assertEqual(author.agencies[0], "Some agency")
            self.assertEqual(len(author.emails), 1)
            self.assertEqual(author.emails[0], "email@mail.com")
            self.assertEqual(len(author.phones), 1)
            self.assertEqual(author.phones[0].description, None)
            self.assertEqual(author.phones[0].country_code, 49)
            self.assertEqual(author.phones[0].area_code, 123)
            self.assertEqual(author.phones[0].phone_number, "456-7890")

        # Now write it again and compare to the original file.
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", validate=True,
                  _suppress_module_tags=True)
        file_buffer.seek(0, 0)

        with open(filename, "rb") as open_file:
            expected_xml_file_buffer = io.BytesIO(open_file.read())
        expected_xml_file_buffer.seek(0, 0)

        self._assert_station_xml_equality(
            file_buffer,
            expected_xml_file_buffer)

    def test_reading_and_writing_full_station_tag(self):
        """
        Tests the reading and writing of a file with a more or less full
        station tag.
        """
        filename = os.path.join(self.data_dir,
                                "full_station_field_station.xml")
        inv = obspy.read_inventory(filename)

        # Assert all the values...
        self.assertEqual(len(inv.networks), 1)
        self.assertEqual(inv.source, "OBS")
        self.assertEqual(inv.module, "Some Random Module")
        self.assertEqual(inv.module_uri, "http://www.some-random.site")
        self.assertEqual(inv.sender, "The ObsPy Team")
        self.assertEqual(inv.created, obspy.UTCDateTime(2013, 1, 1))
        self.assertEqual(len(inv.networks), 1)
        network = inv.networks[0]
        self.assertEqual(network.code, "PY")

        # Now assert the station specific values.
        self.assertEqual(len(network.stations), 1)
        station = network.stations[0]
        self.assertEqual(station.code, "PY")
        self.assertEqual(station.start_date, obspy.UTCDateTime(2011, 1, 1))
        self.assertEqual(station.end_date, obspy.UTCDateTime(2012, 1, 1))
        self.assertEqual(station.restricted_status, "open")
        self.assertEqual(station.alternate_code, "PYY")
        self.assertEqual(station.historical_code, "YYP")
        self.assertEqual(station.description, "Some Description...")
        self.assertEqual(len(station.comments), 2)
        comment_1 = station.comments[0]
        self.assertEqual(comment_1.value, "Comment number 1")
        self.assertEqual(comment_1.begin_effective_time,
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(comment_1.end_effective_time,
                         obspy.UTCDateTime(2008, 2, 3))
        self.assertEqual(len(comment_1.authors), 1)
        authors = comment_1.authors[0]
        self.assertEqual(len(authors.names), 2)
        self.assertEqual(authors.names[0], "This person")
        self.assertEqual(authors.names[1], "has multiple names!")
        self.assertEqual(len(authors.agencies), 3)
        self.assertEqual(authors.agencies[0], "And also")
        self.assertEqual(authors.agencies[1], "many")
        self.assertEqual(authors.agencies[2], "many Agencies")
        self.assertEqual(len(authors.emails), 4)
        self.assertEqual(authors.emails[0], "email1@mail.com")
        self.assertEqual(authors.emails[1], "email2@mail.com")
        self.assertEqual(authors.emails[2], "email3@mail.com")
        self.assertEqual(authors.emails[3], "email4@mail.com")
        self.assertEqual(len(authors.phones), 2)
        self.assertEqual(authors.phones[0].description, "phone number 1")
        self.assertEqual(authors.phones[0].country_code, 49)
        self.assertEqual(authors.phones[0].area_code, 123)
        self.assertEqual(authors.phones[0].phone_number, "456-7890")
        self.assertEqual(authors.phones[1].description, "phone number 2")
        self.assertEqual(authors.phones[1].country_code, 34)
        self.assertEqual(authors.phones[1].area_code, 321)
        self.assertEqual(authors.phones[1].phone_number, "129-7890")
        comment_2 = station.comments[1]
        self.assertEqual(comment_2.value, "Comment number 2")
        self.assertEqual(comment_2.begin_effective_time,
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(comment_1.end_effective_time,
                         obspy.UTCDateTime(2008, 2, 3))
        self.assertEqual(len(comment_2.authors), 3)
        for _i, author in enumerate(comment_2.authors):
            self.assertEqual(len(author.names), 1)
            self.assertEqual(author.names[0], "Person %i" % (_i + 1))
            self.assertEqual(len(author.agencies), 1)
            self.assertEqual(author.agencies[0], "Some agency")
            self.assertEqual(len(author.emails), 1)
            self.assertEqual(author.emails[0], "email@mail.com")
            self.assertEqual(len(author.phones), 1)
            self.assertEqual(author.phones[0].description, None)
            self.assertEqual(author.phones[0].country_code, 49)
            self.assertEqual(author.phones[0].area_code, 123)
            self.assertEqual(author.phones[0].phone_number, "456-7890")

        self.assertEqual(station.latitude, 10.0)
        self.assertEqual(station.longitude, 20.0)
        self.assertEqual(station.elevation, 100.0)

        self.assertEqual(station.site.name, "Some site")
        self.assertEqual(station.site.description, "Some description")
        self.assertEqual(station.site.town, "Some town")
        self.assertEqual(station.site.county, "Some county")
        self.assertEqual(station.site.region, "Some region")
        self.assertEqual(station.site.country, "Some country")

        self.assertEqual(station.vault, "Some vault")
        self.assertEqual(station.geology, "Some geology")

        self.assertEqual(len(station.equipments), 2)
        self.assertEqual(station.equipments[0].resource_id, "some_id")
        self.assertEqual(station.equipments[0].type, "Some type")
        self.assertEqual(station.equipments[0].description, "Some description")
        self.assertEqual(station.equipments[0].manufacturer,
                         "Some manufacturer")
        self.assertEqual(station.equipments[0].vendor, "Some vendor")
        self.assertEqual(station.equipments[0].model, "Some model")
        self.assertEqual(station.equipments[0].serial_number, "12345-ABC")
        self.assertEqual(station.equipments[0].installation_date,
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(station.equipments[0].removal_date,
                         obspy.UTCDateTime(1999, 5, 5))
        self.assertEqual(station.equipments[0].calibration_dates[0],
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(station.equipments[0].calibration_dates[1],
                         obspy.UTCDateTime(1992, 5, 5))
        self.assertEqual(station.equipments[1].resource_id, "something_new")
        self.assertEqual(station.equipments[1].type, "Some type")
        self.assertEqual(station.equipments[1].description, "Some description")
        self.assertEqual(station.equipments[1].manufacturer,
                         "Some manufacturer")
        self.assertEqual(station.equipments[1].vendor, "Some vendor")
        self.assertEqual(station.equipments[1].model, "Some model")
        self.assertEqual(station.equipments[1].serial_number, "12345-ABC")
        self.assertEqual(station.equipments[1].installation_date,
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(station.equipments[1].removal_date,
                         obspy.UTCDateTime(1999, 5, 5))
        self.assertEqual(station.equipments[1].calibration_dates[0],
                         obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(station.equipments[1].calibration_dates[1],
                         obspy.UTCDateTime(1992, 5, 5))

        self.assertEqual(len(station.operators), 2)
        self.assertEqual(station.operators[0].agencies[0], "Agency 1")
        self.assertEqual(station.operators[0].agencies[1], "Agency 2")
        self.assertEqual(station.operators[0].contacts[0].names[0],
                         "This person")
        self.assertEqual(station.operators[0].contacts[0].names[1],
                         "has multiple names!")
        self.assertEqual(len(station.operators[0].contacts[0].agencies), 3)
        self.assertEqual(station.operators[0].contacts[0].agencies[0],
                         "And also")
        self.assertEqual(station.operators[0].contacts[0].agencies[1], "many")
        self.assertEqual(station.operators[0].contacts[0].agencies[2],
                         "many Agencies")
        self.assertEqual(len(station.operators[0].contacts[0].emails), 4)
        self.assertEqual(station.operators[0].contacts[0].emails[0],
                         "email1@mail.com")
        self.assertEqual(station.operators[0].contacts[0].emails[1],
                         "email2@mail.com")
        self.assertEqual(station.operators[0].contacts[0].emails[2],
                         "email3@mail.com")
        self.assertEqual(station.operators[0].contacts[0].emails[3],
                         "email4@mail.com")
        self.assertEqual(len(station.operators[0].contacts[0].phones), 2)
        self.assertEqual(
            station.operators[0].contacts[0].phones[0].description,
            "phone number 1")
        self.assertEqual(
            station.operators[0].contacts[0].phones[0].country_code, 49)
        self.assertEqual(
            station.operators[0].contacts[0].phones[0].area_code, 123)
        self.assertEqual(
            station.operators[0].contacts[0].phones[0].phone_number,
            "456-7890")
        self.assertEqual(
            station.operators[0].contacts[0].phones[1].description,
            "phone number 2")
        self.assertEqual(
            station.operators[0].contacts[0].phones[1].country_code, 34)
        self.assertEqual(station.operators[0].contacts[0].phones[1].area_code,
                         321)
        self.assertEqual(
            station.operators[0].contacts[0].phones[1].phone_number,
            "129-7890")
        self.assertEqual(station.operators[0].contacts[1].names[0], "Name")
        self.assertEqual(station.operators[0].contacts[1].agencies[0],
                         "Agency")
        self.assertEqual(station.operators[0].contacts[1].emails[0],
                         "email@mail.com")
        self.assertEqual(
            station.operators[0].contacts[1].phones[0].description,
            "phone number 1")
        self.assertEqual(
            station.operators[0].contacts[1].phones[0].country_code, 49)
        self.assertEqual(
            station.operators[0].contacts[1].phones[0].area_code, 123)
        self.assertEqual(
            station.operators[0].contacts[1].phones[0].phone_number,
            "456-7890")
        self.assertEqual(station.operators[0].website, "http://www.web.site")

        self.assertEqual(station.operators[1].agencies[0], "Agency")
        self.assertEqual(station.operators[1].contacts[0].names[0], "New Name")
        self.assertEqual(station.operators[1].contacts[0].agencies[0],
                         "Agency")
        self.assertEqual(station.operators[1].contacts[0].emails[0],
                         "email@mail.com")
        self.assertEqual(
            station.operators[1].contacts[0].phones[0].description,
            "phone number 1")
        self.assertEqual(
            station.operators[1].contacts[0].phones[0].country_code, 49)
        self.assertEqual(station.operators[1].contacts[0].phones[0].area_code,
                         123)
        self.assertEqual(
            station.operators[1].contacts[0].phones[0].phone_number,
            "456-7890")
        self.assertEqual(station.operators[1].website, "http://www.web.site")

        self.assertEqual(station.creation_date, obspy.UTCDateTime(1990, 5, 5))
        self.assertEqual(station.termination_date,
                         obspy.UTCDateTime(2009, 5, 5))
        self.assertEqual(station.total_number_of_channels, 100)
        self.assertEqual(station.selected_number_of_channels, 1)

        self.assertEqual(len(station.external_references), 2)
        self.assertEqual(station.external_references[0].uri,
                         "http://path.to/something")
        self.assertEqual(station.external_references[0].description,
                         "Some description")
        self.assertEqual(station.external_references[1].uri,
                         "http://path.to/something/else")
        self.assertEqual(station.external_references[1].description,
                         "Some other description")

        # Now write it again and compare to the original file.
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML", validate=True,
                  _suppress_module_tags=True)
        file_buffer.seek(0, 0)

        with open(filename, "rb") as open_file:
            expected_xml_file_buffer = io.BytesIO(open_file.read())
        expected_xml_file_buffer.seek(0, 0)

        self._assert_station_xml_equality(file_buffer,
                                          expected_xml_file_buffer)

    def test_reading_and_writing_channel_with_response(self):
        """
        Test the reading and writing of a single channel including a
        multi-stage response object.
        """
        filename = os.path.join(self.data_dir,
                                "IRIS_single_channel_with_response.xml")
        inv = obspy.read_inventory(filename)
        self.assertEqual(inv.source, "IRIS-DMC")
        self.assertEqual(inv.sender, "IRIS-DMC")
        self.assertEqual(inv.created, obspy.UTCDateTime("2013-04-16T06:15:28"))
        # Assert that precisely one channel object has been created.
        self.assertEqual(len(inv.networks), 1)
        self.assertEqual(len(inv.networks[0].stations), 1)
        self.assertEqual(len(inv.networks[0].stations[0].channels), 1)
        network = inv.networks[0]
        station = network.stations[0]
        channel = station.channels[0]
        # Assert some fields of the network. This is extensively tested
        # elsewhere.
        self.assertEqual(network.code, "IU")
        self.assertEqual(network.start_date,
                         obspy.UTCDateTime("1988-01-01T00:00:00"))
        self.assertEqual(network.end_date,
                         obspy.UTCDateTime("2500-12-12T23:59:59"))
        self.assertEqual(network.description,
                         "Global Seismograph Network (GSN - IRIS/USGS)")
        # Assert a few fields of the station. This is extensively tested
        # elsewhere.
        self.assertEqual(station.code, "ANMO")
        self.assertEqual(station.latitude, 34.94591)
        self.assertEqual(station.longitude, -106.4572)
        self.assertEqual(station.elevation, 1820.0)
        self.assertEqual(station.site.name, "Albuquerque, New Mexico, USA")
        # Start to assert the channel reading.
        self.assertEqual(channel.code, "BHZ")
        self.assertEqual(channel.location_code, "10")
        self.assertEqual(channel.start_date,
                         obspy.UTCDateTime("2012-03-13T08:10:00"))
        self.assertEqual(channel.end_date,
                         obspy.UTCDateTime("2599-12-31T23:59:59"))
        self.assertEqual(channel.restricted_status, "open")
        self.assertEqual(channel.latitude, 34.945913)
        self.assertEqual(channel.longitude, -106.457122)
        self.assertEqual(channel.elevation, 1759.0)
        self.assertEqual(channel.depth, 57.0)
        self.assertEqual(channel.azimuth, 0.0)
        self.assertEqual(channel.dip, -90.0)
        self.assertEqual(channel.types, ["CONTINUOUS", "GEOPHYSICAL"])
        self.assertEqual(channel.sample_rate, 40.0)
        self.assertEqual(channel.clock_drift_in_seconds_per_sample, 0.0)
        self.assertEqual(channel.sensor.type,
                         "Guralp CMG3-T Seismometer (borehole)")
        # Check the response.
        response = channel.response
        sensitivity = response.instrument_sensitivity
        self.assertEqual(sensitivity.value, 3.31283E10)
        self.assertEqual(sensitivity.frequency, 0.02)
        self.assertEqual(sensitivity.input_units, "M/S")
        self.assertEqual(sensitivity.input_units_description,
                         "Velocity in Meters Per Second")
        self.assertEqual(sensitivity.output_units, "COUNTS")
        self.assertEqual(sensitivity.output_units_description,
                         "Digital Counts")
        # Assert that there are three stages.
        self.assertEqual(len(response.response_stages), 3)

    def test_stationxml_with_availability(self):
        """
        A variant of StationXML has support for availability information.
        Make sure this works.
        """
        filename = os.path.join(self.data_dir,
                                "stationxml_with_availability.xml")
        inv = obspy.read_inventory(filename, format="stationxml")
        channel = inv[0][0][0]
        self.assertEqual(channel.data_availability.start,
                         obspy.UTCDateTime("1998-10-26T20:35:58"))
        self.assertEqual(channel.data_availability.end,
                         obspy.UTCDateTime("2014-07-21T12:00:00"))

        # Now write it again and compare to the original file.
        file_buffer = io.BytesIO()
        inv.write(file_buffer, format="StationXML",
                  _suppress_module_tags=True)
        file_buffer.seek(0, 0)

        with open(filename, "rb") as open_file:
            expected_xml_file_buffer = io.BytesIO(open_file.read())
        expected_xml_file_buffer.seek(0, 0)

        self._assert_station_xml_equality(file_buffer,
                                          expected_xml_file_buffer)

    def test_parse_file_with_no_default_namespace(self):
        """
        Tests that reading a file with no default namespace works fine.

        See #1060.
        """
        filename = os.path.join(self.data_dir, "no_default_namespace.xml")
        inv = obspy.read_inventory(filename)
        # Very small file with almost no content.
        self.assertEqual(len(inv.networks), 1)
        self.assertEqual(inv[0].code, "XX")

    def test_parse_file_with_schema_2(self):
        """
        Reading a StationXML file version 2.0
        """
        filename = os.path.join(self.data_dir, "version20.xml")

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', UserWarning)
            inv = obspy.read_inventory(filename)
        self.assertEqual(len(w), 1)
        self.assertTrue('StationXML file has version 2.0' in str(w[0].message))

        # Very small file with almost no content.
        self.assertEqual(len(inv.networks), 1)
        self.assertEqual(inv[0].code, "XX")

    def test_numbers_are_written_to_poles_and_zeros(self):
        """
        Poles and zeros have a number attribute. Make sure this is written,
        even if set with a custom complex list.
        """
        # Read default inventory and cut down to a single channel.
        inv = obspy.read_inventory()
        inv.networks = inv[:1]
        inv[0].stations = inv[0][:1]
        inv[0][0].channels = inv[0][0][:1]

        # Manually set the poles and zeros - thus these are cast to our
        # custom classes but number are not yet set.
        inv[0][0][0].response.response_stages[0].poles = [0 + 1j, 2 + 3j]
        inv[0][0][0].response.response_stages[0].zeros = [0 + 1j, 2 + 3j]

        with io.BytesIO() as buf:
            inv.write(buf, format="stationxml", validate=True)
            buf.seek(0, 0)
            data = buf.read().decode()

        # Ugly test - remove all whitespace and make sure the four following
        # lines are part of the written output.
        data = re.sub(r'\s+', ' ', data)

        self.assertIn(
            '<Zero number="0"> <Real>0.0</Real> '
            '<Imaginary>1.0</Imaginary> </Zero>', data)
        self.assertIn(
            '<Zero number="1"> <Real>2.0</Real> '
            '<Imaginary>3.0</Imaginary> </Zero>', data)
        self.assertIn(
            '<Pole number="0"> <Real>0.0</Real> '
            '<Imaginary>1.0</Imaginary> </Pole>', data)
        self.assertIn(
            '<Pole number="1"> <Real>2.0</Real> '
            '<Imaginary>3.0</Imaginary> </Pole>', data)

    def test_write_with_extra_tags_namespace_redef(self):
        """
        Tests the exceptions are raised when namespaces
        are redefined.
        """
        filename = os.path.join(
            self.data_dir, "stationxml_with_availability.xml")
        # read the StationXML with availability
        inv = obspy.read_inventory(filename)
        with NamedTemporaryFile() as tf:
            # manually add custom namespace definition
            tmpfile = tf.name
            # assert that namespace prefix of xsi raises ValueError
            mynsmap = {'xsi': 'http://bad.custom.ns/'}
            self.assertRaises(
                ValueError, inv.write, path_or_file_object=tmpfile,
                format="STATIONXML", nsmap=mynsmap)
            # assert that namespace prefix of None raises ValueError
            mynsmap = {None: 'http://bad.custom.ns/'}
            self.assertRaises(
                ValueError, inv.write, path_or_file_object=tmpfile,
                format="STATIONXML", nsmap=mynsmap)

    def test_write_with_extra_tags_without_read_extra(self):
        """
        Tests that a Inventory object that was instantiated with
        custom namespace tags and attributes is written correctly.
        """
        # read the default inventory
        inv = obspy.read_inventory()
        # manually add extra to the dictionary
        network = inv[0]
        network.extra = {}
        ns = 'http://test.myns.ns/'
        # manually add a new custom namespace tag and attribute to the
        # inventory
        network.extra['mynsNetworkTag'] = AttribDict({
            'value': 'mynsNetworkTagValue',
            'namespace': ns})
        network.extra['mynsNetworkAttrib'] = AttribDict({
            'value': 'mynsNetworkAttribValue',
            'namespace': ns,
            'type': 'attribute'})
        station = inv[0][0]
        station.extra = {}
        station.extra['mynsStationTag'] = AttribDict({
            'value': 'mynsStationTagValue',
            'namespace': ns})
        station.extra['mynsStationAttrib'] = AttribDict({
            'value': 'mynsStationAttribValue',
            'namespace': ns,
            'type': 'attribute'})
        channel = inv[0][0][0]
        # add data availability to inventory
        channel.data_availability = AttribDict({
            'start': obspy.UTCDateTime('1998-10-26T20:35:58+00:00'),
            'end': obspy.UTCDateTime('2014-07-21T12:00:00+00:00')})
        channel.extra = {}
        channel.extra['mynsChannelTag'] = AttribDict({
            'value': 'mynsChannelTagValue', 'namespace': ns})
        channel.extra['mynsChannelAttrib'] = AttribDict({
            'value': 'mynsChannelAttribValue',
            'namespace': ns,
            'type': 'attribute'})
        # add nested tags
        nested_tag = AttribDict()
        nested_tag.namespace = ns
        nested_tag.value = AttribDict()
        # add two nested tags
        nested_tag.value.my_nested_tag1 = AttribDict()
        nested_tag.value.my_nested_tag1.namespace = ns
        nested_tag.value.my_nested_tag1.value = 1.23E+10
        nested_tag.value.my_nested_tag2 = AttribDict()
        nested_tag.value.my_nested_tag2.namespace = ns
        nested_tag.value.my_nested_tag2.value = True
        nested_tag.value.my_nested_tag2.attrib = {'{%s}%s' % (
            ns, 'nestedAttribute1'): 'nestedAttributeValue1'}
        channel.extra['nested'] = nested_tag
        with NamedTemporaryFile() as tf:
            # manually add custom namespace definition
            tmpfile = tf.name
            # set namespace map to include only valid custom namespaces
            mynsmap = {'myns': ns}
            # write file with manually defined namespace map
            inv.write(tmpfile, format="STATIONXML", nsmap=mynsmap)
            # check contents
            with open(tmpfile, "rb") as fh:
                # enforce reproducible attribute orders through write_c14n
                obj = etree.fromstring(fh.read()).getroottree()
                buf = io.BytesIO()
                obj.write_c14n(buf)
                buf.seek(0, 0)
                content = buf.read()
            # check namespace definitions in root element
            expected = [
                b'xmlns="http://www.fdsn.org/xml/station/1"',
                b'xmlns:myns="http://test.myns.ns/"',
                b'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"']
            for line in expected:
                self.assertIn(line, content)
            # check additional tags
            expected = [
                b'<myns:mynsNetworkTag>' +
                b'mynsNetworkTagValue' +
                b'</myns:mynsNetworkTag>',
                b'myns:mynsNetworkAttrib="mynsNetworkAttribValue"',
                b'<myns:mynsStationTag>' +
                b'mynsStationTagValue' +
                b'</myns:mynsStationTag>',
                b'myns:mynsStationAttrib="mynsStationAttribValue"',
                b'<myns:mynsChannelTag>' +
                b'mynsChannelTagValue' +
                b'</myns:mynsChannelTag>',
                b'myns:mynsChannelAttrib="mynsChannelAttribValue"',
                b'<myns:nested>',
                b'<myns:my_nested_tag1>' +
                b'12300000000.0' +
                b'</myns:my_nested_tag1>',
                b'<myns:my_nested_tag2 ' +
                b'myns:nestedAttribute1="nestedAttributeValue1">' +
                b'True' +
                b'</myns:my_nested_tag2>',
                b'</myns:nested>'
            ]
            for line in expected:
                self.assertIn(line, content)

    def test_write_with_extra_tags_and_read(self):
        """
        First tests that a StationXML file with additional
        custom "extra" tags gets written correctly. Then
        tests that when reading the written file again the
        extra tags are parsed correctly.
        """
        filename = os.path.join(
            self.data_dir, "IRIS_single_channel_with_response_custom_tags.xml")

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            inv = obspy.read_inventory(filename)
            self.assertEqual(len(w), 0)
        with NamedTemporaryFile() as tf:
            tmpfile = tf.name
            # write file
            inv.write(tmpfile, format="STATIONXML")
            # check contents
            with open(tmpfile, "rb") as fh:
                # enforce reproducible attribute orders through write_c14n
                obj = etree.fromstring(fh.read()).getroottree()
                buf = io.BytesIO()
                obj.write_c14n(buf)
                buf.seek(0, 0)
                content = buf.read()
            # check namespace definitions in root element
            expected = [b'xmlns="http://www.fdsn.org/xml/station/1"',
                        b'xmlns:test="http://just.a.test/xmlns/1"'
                        ]
            for line in expected:
                self.assertIn(line, content)
            # check custom tags, nested custom tags, and attributes
            # at every level of the StationXML hierarchy
            expected = [
                # root
                b'test:customRootAttrib="testRootAttribute"',
                b'<test:CustomRootTag>testRootTag</test:CustomRootTag>',
                b'<test:CustomNestedRootTag>',
                b'<test:NestedTag1 nestedTagAttrib="testNestedAttribute">' +
                b'nestedRootTag1' +
                b'</test:NestedTag1>',
                b'<test:NestedTag2>nestedRootTag2</test:NestedTag2>',
                b'</test:CustomNestedRootTag>',
                # network
                b'test:customNetworkAttrib="testNetworkAttribute"',
                b'<test:CustomNetworkTag>' +
                b'testNetworkTag' +
                b'</test:CustomNetworkTag>',
                b'<test:CustomNestedNetworkTag>',
                b'<test:NestedTag1>nestedNetworkTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedNetworkTag2</test:NestedTag2>',
                b'</test:CustomNestedNetworkTag>',
                # station
                b'test:customStationAttrib="testStationAttribute"',
                b'<test:CustomStationTag>' +
                b'testStationTag' +
                b'</test:CustomStationTag>',
                b'<test:CustomNestedStationTag>',
                b'<test:NestedTag1>nestedStationTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedStationTag2</test:NestedTag2>',
                b'</test:CustomNestedStationTag>',
                # comment
                b'test:customCommentAttrib="testCommentAttribute"',
                b'<test:CustomCommentTag>' +
                b'testCommentTag' +
                b'</test:CustomCommentTag>',
                b'<test:CustomNestedCommentTag>',
                b'<test:NestedTag1>nestedCommentTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedCommentTag2</test:NestedTag2>',
                b'</test:CustomNestedCommentTag>',
                # person
                b'test:customPersonAttrib="testPersonAttribute"',
                b'<test:CustomPersonTag>testPersonTag</test:CustomPersonTag>',
                b'<test:CustomNestedPersonTag>',
                b'<test:NestedTag1>nestedPersonTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedPersonTag2</test:NestedTag2>',
                b'</test:CustomNestedPersonTag>',
                # phone
                b'test:customPhoneAttrib="testPhoneAttribute"',
                b'<test:CustomPhoneTag>testPhoneTag</test:CustomPhoneTag>',
                b'<test:CustomNestedPhoneTag>',
                b'<test:NestedTag1>nestedPhoneTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedPhoneTag2</test:NestedTag2>',
                b'</test:CustomNestedPhoneTag>',
                # site
                b'test:customSiteAttrib="testSiteAttribute"',
                b'<test:CustomSiteTag>testSiteTag</test:CustomSiteTag>',
                b'<test:CustomNestedSiteTag>',
                b'<test:NestedTag1>nestedSiteTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedSiteTag2</test:NestedTag2>',
                b'</test:CustomNestedSiteTag>',
                # equipment
                b'test:customEquipmentAttrib="testEquipmentAttribute"',
                b'<test:CustomEquipmentTag>' +
                b'testEquipmentTag' +
                b'</test:CustomEquipmentTag>',
                b'<test:CustomNestedEquipmentTag>',
                b'<test:NestedTag1>nestedEquipmentTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedEquipmentTag2</test:NestedTag2>',
                b'</test:CustomNestedEquipmentTag>',
                # operator
                b'test:customOperatorAttrib="testOperatorAttribute"',
                b'<test:CustomOperatorTag>' +
                b'testOperatorTag' +
                b'</test:CustomOperatorTag>',
                b'<test:CustomNestedOperatorTag>',
                b'<test:NestedTag1>nestedOperatorTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedOperatorTag2</test:NestedTag2>',
                b'</test:CustomNestedOperatorTag>',
                # external reference
                b'test:customExtRefAttrib="testExtRefAttribute"',
                b'<test:CustomExtRefTag>testExtRefTag</test:CustomExtRefTag>',
                b'<test:CustomNestedExtRefTag>',
                b'<test:NestedTag1>nestedExtRefTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedExtRefTag2</test:NestedTag2>',
                b'</test:CustomNestedExtRefTag>',
                # channel
                b'test:customChannelAttrib="testChannelAttribute"',
                b'<test:CustomChannelTag>' +
                b'testChannelTag' +
                b'</test:CustomChannelTag>',
                b'<test:CustomNestedChannelTag>',
                b'<test:NestedTag1>nestedChannelTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedChannelTag2</test:NestedTag2>',
                b'</test:CustomNestedChannelTag>',
                # response
                b'test:customResponseAttrib="testResponseAttribute"',
                b'<test:CustomResponseTag>' +
                b'testResponseTag' +
                b'</test:CustomResponseTag>',
                b'<test:CustomNestedResponseTag>',
                b'<test:NestedTag1>nestedResponseTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedResponseTag2</test:NestedTag2>',
                b'</test:CustomNestedResponseTag>',
                # data availability
                b'test:customDAAttrib="testDAAttribute"',
                b'<test:CustomDATag>testDATag</test:CustomDATag>',
                b'<test:CustomNestedDATag>',
                b'<test:NestedTag1>nestedDATag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedDATag2</test:NestedTag2>',
                b'</test:CustomNestedDATag>',
                # response stage (PolesZeros response stage)
                b'test:customStagePZAttrib="testStagePZAttribute"',
                b'<test:CustomStagePZTag>' +
                b'testStagePZTag' +
                b'</test:CustomStagePZTag>',
                b'<test:CustomNestedStagePZTag>',
                b'<test:NestedTag1>nestedStagePZTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedStagePZTag2</test:NestedTag2>',
                b'</test:CustomNestedStagePZTag>',
                # response stage (Coefficients response stage)
                b'test:customStageCoefAttrib="testStageCoefAttribute"',
                b'<test:CustomStageCoefTag>' +
                b'testStageCoefTag' +
                b'</test:CustomStageCoefTag>',
                b'<test:CustomNestedStageCoefTag>',
                b'<test:NestedTag1>nestedStageCoefTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedStageCoefTag2</test:NestedTag2>',
                b'</test:CustomNestedStageCoefTag>',
                # instrument sensitivity
                b'test:customSensitivityAttrib="testSensitivityAttribute"',
                b'<test:CustomSensitivityTag>' +
                b'testSensitivityTag' +
                b'</test:CustomSensitivityTag>',
                b'<test:CustomNestedSensitivityTag>',
                b'<test:NestedTag1>nestedSensitivityTag1</test:NestedTag1>',
                b'<test:NestedTag2>nestedSensitivityTag2</test:NestedTag2>',
                b'</test:CustomNestedSensitivityTag>'
            ]
            for line in expected:
                self.assertIn(line, content)
            # now, read again to test if it's parsed correctly..
            inv = obspy.read_inventory(tmpfile)

    def test_reading_file_with_empty_channel_object(self):
        """
        Tests reading a file with an empty channel object. This is strictly
        speaking not valid but we are forgiving.
        """
        filename = os.path.join(self.data_dir, "empty_channel.xml")
        inv = obspy.read_inventory(filename)
        self.assertEqual(
            inv.get_contents(),
            {'networks': ['IV'], 'stations': ['IV.LATE (Latera)'],
             'channels': []})

    def test_reading_channel_without_coordinates(self):
        """
        Tests reading a file with an empty channel object. This is strictly
        speaking not valid but we are forgiving.
        """
        filename = os.path.join(self.data_dir,
                                "channel_without_coordinates.xml")
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            inv = obspy.read_inventory(filename)

        # Should raise a warning that it could not read the channel without
        # coordinates.
        self.assertEqual(len(w), 1)
        self.assertEqual(
            w[0].message.args[0],
            "Channel 00.BHZ of station LATE does not have a complete set of "
            "coordinates and thus it cannot be read. It will not be part of "
            "the final inventory object.")

        self.assertEqual(
            inv.get_contents(),
            {'networks': ['IV'], 'stations': ['IV.LATE (Latera)'],
             'channels': []})

    def test_units_during_identity_stage(self):
        """
        """
        t = obspy.UTCDateTime(2017, 1, 1)
        inv = obspy.read_inventory().select(station="RJOB", channel="EHZ",
                                            time=t)
        response = inv.get_response("BW.RJOB..EHZ", t)
        response.response_stages[0].input_units_description = "M/S"
        response.response_stages[0].output_units_description = "Volts"
        rstage_2 = ResponseStage(2, 1, 1, "V", "V",
                                 input_units_description="Volts",
                                 output_units_description="Volts")
        rstage_3 = ResponseStage(3, 1, 1, "V", "V",
                                 input_units_description="Volts",
                                 output_units_description="Volts")
        response.response_stages.insert(1, rstage_2)
        response.response_stages.insert(2, rstage_3)
        for i, rstage in enumerate(response.response_stages[3:]):
            rstage.stage_sequence_number = i + 4

        with io.BytesIO() as buf:
            inv.write(buf, format="stationxml", validate=True)
            buf.seek(0, 0)
            inv_2 = obspy.read_inventory(buf)

        response_2 = inv_2.get_response("BW.RJOB..EHZ", t)

        self.assertEqual(response, response_2)
        self.assertEqual(response_2.response_stages[1].input_units, "V")
        self.assertEqual(response_2.response_stages[1].output_units, "V")
        self.assertEqual(
            response_2.response_stages[1].input_units_description, "Volts")
        self.assertEqual(
            response_2.response_stages[1].output_units_description, "Volts")
        self.assertEqual(response_2.response_stages[2].input_units, "V")
        self.assertEqual(response_2.response_stages[2].output_units, "V")
        self.assertEqual(
            response_2.response_stages[2].input_units_description, "Volts")
        self.assertEqual(
            response_2.response_stages[2].output_units_description, "Volts")

        # Also try from the other side.
        inv = obspy.read_inventory().select(station="RJOB", channel="EHZ",
                                            time=t)
        response = inv.get_response("BW.RJOB..EHZ", t)
        response.response_stages[0].input_units = None
        response.response_stages[0].output_units = None
        response.response_stages[0].input_units_description = None
        response.response_stages[0].output_units_description = None
        rstage_2 = ResponseStage(2, 1, 1, "V", "V",
                                 input_units_description="Volts",
                                 output_units_description="Volts")
        rstage_3 = ResponseStage(3, 1, 1, "V", "V",
                                 input_units_description="Volts",
                                 output_units_description="Volts")
        response.response_stages.insert(1, rstage_2)
        response.response_stages.insert(2, rstage_3)
        for i, rstage in enumerate(response.response_stages[3:]):
            rstage.stage_sequence_number = i + 4

        with io.BytesIO() as buf:
            inv.write(buf, format="stationxml")
            buf.seek(0, 0)
            inv_2 = obspy.read_inventory(buf)

        response_2 = inv_2.get_response("BW.RJOB..EHZ", t)
        # Set these to None manually as the autocorrection during parsing will
        # set it.
        response_2.response_stages[0].input_units = None
        response_2.response_stages[0].input_units_description = None

        self.assertEqual(response, response_2)
        self.assertEqual(response_2.response_stages[1].input_units, "V")
        self.assertEqual(response_2.response_stages[1].output_units, "V")
        self.assertEqual(
            response_2.response_stages[1].input_units_description, "Volts")
        self.assertEqual(
            response_2.response_stages[1].output_units_description, "Volts")
        self.assertEqual(response_2.response_stages[2].input_units, "V")
        self.assertEqual(response_2.response_stages[2].output_units, "V")
        self.assertEqual(
            response_2.response_stages[2].input_units_description, "Volts")
        self.assertEqual(
            response_2.response_stages[2].output_units_description, "Volts")

def suite():
    return unittest.makeSuite(StationXMLTestCase, "test")

if __name__ == '__main__':
    unittest.main(defaultTest='suite')


import openpyxl

from classes.TableBlockConstructor import TableBlockConstructor

book = openpyxl.open('CYR_GP.xlsx')
sheet = book.active
# sheet['C3'].value = 'rere'
# sheet['B7'].value = 1
# book.save('rere.xlsx')
years = (2022, 2021, 2020, 2016, 2017)
plan_fact = (40, 40, 40, 5, 5, 0)
fact_plan = (60, 70, 56, 5, 6, 0)

# (self, indicator, event_id, event, main_event_id, main_event, subprogram_id, subprogram,
#                        type, unit, years, plan_marks, fact_marks, comment)

TableBlockConstructor.fill_CYR_sample('x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x')
TableBlockConstructor.fill_GP_sample('x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', years,
                                     plan_fact, fact_plan, '2005', 'gsfsrhf')


'''
Milone Technologies:  https://milonetech.com/products/standard-etape-assembly
Micropython:          https://docs.micropython.org/en/latest/esp32/quickref.html#adc-analog-to-digital-conversion

Brandon Gant
Created: 2020-06-08

Sump Pit Level:
This script uses a Milone eTape with Volage Divider to monitor the level
of water in a home sump pit.

Pinout:
   Red   --> 3.3V
   Black --> GND
   White --> GPIO32  (any ADC pin 32-39 should work)
'''

from time import sleep
from machine import Pin, ADC

adc = ADC(Pin(32))
adc.atten(ADC.ATTN_11DB)   # 0V to 3.3V range
adc.width(ADC.WIDTH_10BIT) # 0  to 1023 bits read

# Single reading from eTape
def read():
    return adc.read()

# Instead of a single reading let's take the average of multiple readings (throwing out highs and lows)
def average():
    adc_sample = []
    for x in range(40):
        adc_sample.append(adc.read())
        sleep(0.01)
    remove_high_low = sorted(adc_sample)[15:-5]
    average_adc = sum(remove_high_low) / len(remove_high_low)
    return round(average_adc)

# Re-maps a number from one range to another.
# Python equivalent to Arduino map() function.
# Source: https://www.arduino.cc/reference/en/language/functions/math/map/
def range_map(x, in_min, in_max, out_min, out_max):
    return (x-in_min) * (out_max-out_min) / (in_max-in_min) + out_min

def inches():
    # Based on manual measurements of etape ADC values at half-inch increments
    # Min ADC 480 is  1.5 inches or less (1.0 and less is 472)
    # Max ADC 620 is 10.0 inches
    #
    # Sump switches  ON at about 7 inches
    # Sump switches OFF at about 3 inches and fills with water

    in_min = 480
    in_max = 620

    out_min = 1.5
    out_max = 10.0
    inches = range_map(average(), in_min, in_max, out_min, out_max)

    # bottom of etape is about a half inch above the base of the sump pit
    #inches += 0.5

    return round(inches, 1)


import numpy as np
import pandas as pd
import time
from scipy.spatial import distance

contentMode = "cbvis"
visLayers = ["FC8","FC7","PROB"] #"FC7","PROB"
requestedRecommendations = 7
suspectToCandidateImportanceRatio = 0.25

dst = np.zeros((4652,4652))
for visLayer in visLayers:
    vec = pd.read_csv("processData/"+contentMode+visLayer+"_test_embeddings.csv", sep=';', header=None)
    fids = pd.read_csv("processData/"+contentMode+visLayer+"_test_ids.csv", sep=";", header=None, dtype="str")
    fidList = fids[0].tolist()
    v = np.asarray(vec)
    print(v.shape)
    partDst = distance.cdist(v, v, 'cosine')
    partDst[partDst < 1e-14] = 1.0 # deal with identical objects
    dst = dst + partDst[0:4652, 0:4652]
print(dst.shape)

lineupIDs = ["8061205130408", "8100303170706", "10301206130703", "13430524130707", "8121007141903", "8300518161902", "8510203150012", "10150425165030"]
for lid in lineupIDs:
    t1 = time.time()
    print(t1)
    idx = fids.index[fids[0] == lid].tolist()[0]
    arr = np.repeat(dst[idx,:],1)
    recsSim = []
    recsIDs = []
    recsI = [idx]
    for rID in range(requestedRecommendations):
        recIdx = arr.argsort()[0:50]
        for i in recIdx:
            if (min(dst[recsI,i]) < 1e-14) or (max(dst[recsI,i]) > 0.999):
                print(min(dst[recsI,i]), max(dst[recsI,i]))
                continue #items same as already found ones
            recsSim.append(arr[i])
            recsIDs.append(fidList[i])
            recsI.append(i)
            addedDST = dst[i,:] * suspectToCandidateImportanceRatio
            arr = arr + addedDST
            break

    t2 = time.time()
    print(t2)
    print(t2 - t1)

    print(idx)
    print(recsI)
    print(recsSim)
    print(recsIDs)

    row = "cnn_"+contentMode+"COMBINED_UNIFORMITY;"+lid+";;"+",".join(recsIDs)+"\n"
    print(row)
    with open("testOutput/cnnResults.csv", "a") as fp:
        fp.write(row)

#np.savetxt("processData/"+contentMode+visLayer+"_cosineDistance.csv", dst, delimiter=';')