#!/usr/bin/env python
# -*- coding:utf-8 -*-
###
# File: /home/hky/github/Identification_CR/P00_readMCdata/CRMCDataset.py
# Project: /home/hky/github/Identification_CR/P00_readMCdata
# Created Date: 2022-06-16 14:55:54
# Author: Hu Kongyi
# Email:hukongyi@ihep.ac.cn
# -----
# Last Modified: 2022-06-22 15:38:13
# Modified By: Hu Kongyi
# -----
# HISTORY:
# Date      	By      	Comments
# ----------	--------	----------------------------------------------------
# 2022-06-22	K.Y.Hu		add number of class
# 2022-06-19	K.Y.Hu		realize should not use InMemoryDataset
# 2022-06-16	K.Y.Hu		Create this file
###
import os.path as osp
from typing import Callable, Optional

import torch
from torch_geometric.data import Data, Dataset
import numpy as np
from tqdm import tqdm
from torch_geometric.loader import DataLoader

# can not use InMemoryDataset, need Dataset
class CRMCDataset(Dataset):

    def __init__(self,
                 root: str,
                 calibfile: str,
                 kneighbors: int = 4,
                 nchmin: int = 4,
                 x_range: int = 150,
                 y_range: int = 150,
                 transform: Optional[Callable] = None,
                 pre_transform: Optional[Callable] = None,
                 pre_filter: Optional[Callable] = None):
        self.root = root
        self.kneighbors = kneighbors
        self.nchmin = nchmin
        self.calibfile = calibfile
        self.x_range = x_range
        self.y_range = y_range

        self.num_classes = 6
        data_number_path = osp.join(self.processed_dir, 'data_number')
        if not osp.exists(data_number_path):
            self.number = 0
        else:
            with open(data_number_path, 'r') as f:
                self.number = int(f.read())
        super().__init__(root, transform, pre_transform, pre_filter)
        if not osp.exists(data_number_path):
            with open(data_number_path, 'w') as f:
                f.write(str(self.number))

    @property
    def raw_dir(self) -> str:
        return '/home/hky/Data/chenxu/'

    @property
    def processed_dir(self) -> str:
        return osp.join(
            self.root,
            f'processed_class_{self.num_classes}_kneighbors_{self.kneighbors}_nchmin_{self.nchmin}_{self.x_range}_{self.y_range}'
        )

    @property
    def raw_file_names(self) -> str:
        return 'MCmodelBany4.npz'

    @property
    def processed_file_names(self) -> str:
        return [f'data_{i}.pt' for i in range(self.number)]

    def download(self):
        pass

    def process(self):

        loc_x = list()
        loc_y = list()
        loc_z = list()

        with open(self.calibfile, "r") as f:
            for line in f:
                loc_x.append(float(line.split()[1]))
                loc_y.append(float(line.split()[2]))
                loc_z.append(float(line.split()[3]))
        loc_x = np.array(loc_x)
        loc_y = np.array(loc_y)
        loc_z = np.array(loc_z)

        orgin_data = np.load(self.raw_paths[0])

        # pri_e_num = orgin_data['pri_e_num']
        pri_id = orgin_data['pri_id']
        pri_e = orgin_data['pri_e']
        pri_theta = orgin_data['pri_theta']
        pri_phi = orgin_data['pri_phi']
        pri_ne = orgin_data['pri_ne']
        pri_core_x = orgin_data['pri_core_x']
        pri_core_y = orgin_data['pri_core_y']
        pri_sump = orgin_data['pri_sump']
        pri_n_hit = orgin_data['pri_n_hit']
        Tibetevent = orgin_data['Tibetevent']
        Tibet = orgin_data['Tibet']
        MDevent = orgin_data['MDevent']
        MD = orgin_data['MD']

        particle_id = list(set(pri_id))
        particle_id.sort()
        pri_kind = np.zeros(len(pri_id), dtype=int)
        for j, i in enumerate(particle_id):
            if j == 0:
                pri_kind[pri_id == i] = 0
            elif j == 1:
                pri_kind[pri_id == i] = 1
            elif j == 2 or j == 3 or j == 4:
                pri_kind[pri_id == i] = 2
            elif j == 5 or j == 6 or j == 7:
                pri_kind[pri_id == i] = 3
            elif j == 8 or j == 9 or j == 10:
                pri_kind[pri_id == i] = 4
            elif j == 11:
                pri_kind[pri_id == i] = 5

        for i in range(self.num_classes):
            print(i, len(pri_kind[pri_kind == i]))

        for i in tqdm(range(len(pri_id))):
            if pri_n_hit[i] >= self.nchmin:
                data = get_one_trigger_from_numpy(
                    indices=i,
                    Tibetevent=Tibetevent,
                    Tibet=Tibet,
                    MDevent=MDevent,
                    MD=MD,
                    kneighbors=self.kneighbors,
                    loc_x=loc_x,
                    loc_y=loc_y,
                    loc_z=loc_z,
                    pri_id=pri_id,
                    pri_theta=pri_theta,
                    pri_phi=pri_phi,
                    pri_core_x=pri_core_x,
                    pri_core_y=pri_core_y,
                    pri_e=pri_e,
                    pri_ne=pri_ne,
                    pri_sump=pri_sump,
                    y=pri_kind,
                )
                if self.pre_filter(data, self.nchmin, self.x_range, self.y_range):
                    torch.save(data, osp.join(self.processed_dir, f'data_{self.number}.pt'))
                    self.number += 1
        # if self.pre_filter is not None:
        #     data_list = [data for data in data_list if self.pre_filter(data)]

        # if self.pre_transform is not None:
        #     data_list = [self.pre_transform(data) for data in data_list]

        # data, slices = self.collate(data_list)
        # torch.save((data, slices), self.processed_paths[0])

    def len(self):
        return self.number

    def get(self, idx):
        data = torch.load(osp.join(self.processed_dir, f'data_{idx}.pt'))
        return data

def pre_filter(data, nchmin, x_range, y_range):
    return (data.num_nodes >= nchmin) and (abs(data.pri_core_x) < x_range) and (abs(data.pri_core_y)
                                                                                < y_range)

def get_one_trigger_from_numpy(indices: int, Tibetevent: np.ndarray, Tibet: np.ndarray,
                               MDevent: np.ndarray, MD: np.ndarray, kneighbors: int,
                               loc_x: np.ndarray, loc_y: np.ndarray, loc_z: np.ndarray,
                               pri_id: np.ndarray, pri_theta: np.ndarray, pri_phi: np.ndarray,
                               pri_core_x: np.ndarray, pri_core_y: np.ndarray, pri_e: np.ndarray,
                               pri_ne: np.ndarray, pri_sump: np.ndarray, y: np.ndarray) -> Data:
    x = list()
    edge_index = list()
    if indices < len(Tibetevent):
        data_range_Tibet = Tibet[Tibetevent[indices]:Tibetevent[indices + 1], :]
        data_range_MD = MD[MDevent[indices]:MDevent[indices + 1], :]

    else:
        data_range_Tibet = Tibet[Tibetevent[indices]:, :]
        data_range_MD = MD[MDevent[indices], :]
    for i in data_range_Tibet:
        # !only use the \pm 300 ns data
        if (i[3] <= 300) and (i[3] >= -300) and (i[2] > 0.6):
            x.append([loc_x[int(i[1])], loc_y[int(i[1])], loc_z[int(i[1])], i[3], i[2]])

    x = np.array(x)
    if len(x) - 1 <= kneighbors:
        for i in range(len(x)):
            for j in range(len(x)):
                if i != j:
                    edge_index.append([i, j])
    else:
        for i in range(len(x)):
            # + or -
            distance = np.sum(
                (x[i, :3] - x[:, :3]) ** 2, axis=1) + (x[i, 3] - x[:, 3]) ** 2 * 0.3 ** 2
            # print(len(distance), kneighbors)
            min5 = np.argpartition(distance, kneighbors)[:kneighbors]
            for j in min5:
                if i != j:
                    if [i, j] not in edge_index:
                        edge_index.append([i, j])
                    if [j, i] not in edge_index:
                        edge_index.append([j, i])

    MD_tmp = np.zeros([4 * 16 * 4])
    for i in data_range_MD:
        MD_tmp[int(i[1])] = i[2]
    MD_tmp = MD_tmp.reshape(4, 4, 4, 4)
    MD_new = np.zeros([12, 12])
    MD_new[8:, :4] = MD_tmp[0, :, :, 0]
    MD_new[:4, :4] = MD_tmp[1, :, :, 0]
    MD_new[:4, 8:] = MD_tmp[2, :, :, 0]
    MD_new[8:, 8:] = MD_tmp[3, :, :, 0]

    data = Data(
        x=torch.tensor(x, dtype=torch.float),
        edge_index=torch.tensor(edge_index, dtype=torch.long).t().contiguous(),
        y=torch.tensor([y[indices]], dtype=torch.int64),
        # MD=torch.tensor(MD_tmp[:, :, :, 0], dtype=torch.float),
        MD=torch.tensor(MD_new, dtype=torch.float),
        # MD0=torch.tensor(MD0, dtype=torch.float),
        # MD1=torch.tensor(MD1, dtype=torch.float),
        # MD2=torch.tensor(MD2, dtype=torch.float),
        # MD3=torch.tensor(MD3, dtype=torch.float),
        pri_id=torch.tensor([pri_id[indices]], dtype=torch.int),
        pri_theta=torch.tensor([pri_theta[indices]], dtype=torch.float),
        pri_phi=torch.tensor([pri_phi[indices]], dtype=torch.float),
        pri_core_x=torch.tensor([pri_core_x[indices]], dtype=torch.float),
        pri_core_y=torch.tensor([pri_core_y[indices]], dtype=torch.float),
        pri_e=torch.tensor([pri_e[indices]], dtype=torch.float),
        pri_ne=torch.tensor([pri_ne[indices]], dtype=torch.float),
        pri_sump=torch.tensor([pri_sump[indices]], dtype=torch.float),
    )

    return data

if __name__ == '__main__':
    dataset = CRMCDataset(
        root="/home/hky/Data/chenxu/pygDataset/",
        calibfile="/home/hky/for_mengy3/src16_1.8/userfile/mccaliblmzhai_20190312.cal",
        kneighbors=8,
        nchmin=16,
        x_range=100,
        y_range=100,
        pre_filter=pre_filter)
    # print(len(dataset))
    print(dataset[0])
    print(dataset.num_classes)
    # loader = DataLoader(dataset, batch_size=32, shuffle=True)
    # for batch in loader:
    #     pass


def my_code(obj, key, list1=None):
    if list1 is None:
        list1 = []
    list1.append(key)  # добавляю в лист вершину с которой работаю
    if key in obj.keys():
        if isinstance(obj, dict):
            for k, v in obj.items():
                if isinstance(v, (list)):
                    for v1 in v:
                        if (v1 in obj[key] and k==key): #условия смежной вершины: 1) значение есть в словаре под номером родительской
                                                            # 2) номер совпадает с номером родительской
                            if v1 not in obj.keys() or len(obj[v1])==1: #убеждение, что это вершина крайняя или что связана лишь с одной дочерней вершиной
                                obj[key].remove(v1)
                            return my_code(obj, v1, list1)
    else: # пошел заново с первой вершины другие пути искать
        return my_code(obj, list1[0], list1)
    return print_answer(list1)

def print_answer(l): #привожу к формату вывода, так как в лист складываются все маршруты к конечным вершинам
    answer = []
    [answer.append(x) for x in l if x not in answer]
    print(*answer, sep='\n')

data = {
 1: [2, 3],
 2: [4]
}
data1 = {
 1: [2, 3],
 2: [3, 4],
 4: [1]
}
my_code(data, 1)
my_code(data1, 1)


"""
The main `Echo Nest`_ `Remix API`_ module for manipulating audio files and
their associated `Echo Nest`_ `Analyze API`_ analyses.

AudioData, and getpieces by Robert Ochshorn
on 2008-06-06.  Some refactoring and everything else by Joshua Lifton
2008-09-07.  Refactoring by Ben Lacker 2009-02-11. Other contributions
by Adam Lindsay.

:group Base Classes: AudioAnalysis, AudioRenderable, AudioData, AudioData32
:group Audio-plus-Analysis Classes: AudioFile, LocalAudioFile, LocalAnalysis
:group Building Blocks: AudioQuantum, AudioSegment, AudioQuantumList, ModifiedRenderable
:group Effects: AudioEffect, LevelDB, AmplitudeFactor, TimeTruncateFactor, TimeTruncateLength, Simultaneous
:group Exception Classes: FileTypeError, EchoNestRemixError

:group Audio helper functions: getpieces, mix, assemble, megamix
:group ffmpeg helper functions: ffmpeg, settings_from_ffmpeg, ffmpeg_error_check
:group Utility functions: chain_from_mixed, _dataParser, _attributeParser, _segmentsParser

.. _Analyze API: http://developer.echonest.com/pages/overview?version=2
.. _Remix API: http://code.google.com/p/echo-nest-remix/
.. _Echo Nest: http://the.echonest.com/
"""

__version__ = "$Revision: 0 $"
# $Source$

import hashlib
import numpy
import os
import sys
import StringIO
import struct
import subprocess
import tempfile
import wave

from pyechonest import track
import pyechonest.util
import echonest.selection as selection
import pyechonest.config as config
from echonest.support import stupidxml

class AudioAnalysis(track.Track) :
    """
    This class wraps `echonest.web` to allow transparent caching of the
    audio analysis of an audio file.

    For example, the following script will display the bars of a track
    twice::

        from echonest import *
        a = audio.AudioAnalysis('YOUR_TRACK_ID_HERE')
        a.bars
        a.bars

    The first time `a.bars` is called, a network request is made of the
    `Echo Nest`_ `Analyze API`_.  The second time time `a.bars` is called, the
    cached value is returned immediately.

    An `AudioAnalysis` object can be created using an existing ID, as in
    the example above, or by specifying the audio file to upload in
    order to create the ID, as in::

        a = audio.AudioAnalysis(filename='FULL_PATH_TO_AUDIO_FILE')

    .. _Analyze API: http://developer.echonest.com/pages/overview?version=2
    .. _Echo Nest: http://the.echonest.com/
    """

    def __init__( self, audio):
        """
        Constructor.  If the argument is a valid local path or a URL,
        the track ID is generated by uploading the file to the `Echo Nest`_
        `Analyze API`_\.  Otherwise, the argument is assumed to be
        the track ID.

        :param audio: A string representing either a path to a local
            file, a valid URL, or the ID of a file that has already
            been uploaded for analysis.

        .. _Analyze API: http://developer.echonest.com/pages/overview?version=2
        .. _Echo Nest: http://the.echonest.com/
        """

        if type(audio) is not str:
            # Argument is invalid.
            raise TypeError("Argument 'audio' must be a string representing either a filename, track ID, or MD5.")
        super(AudioAnalysis, self).__init__(audio)
        self.source = None

        self._bars = None
        self._beats = None
        self._tatums = None
        self._sections = None
        self._segments = None

    @property
    def bars(self):
        if self._bars is None:
            self._bars = _dataParser('bar', super(AudioAnalysis, self).bars)
            self._bars.attach(self)
        return self._bars

    @property
    def beats(self):
        if self._beats is None:
            self._beats = _dataParser('beat', super(AudioAnalysis, self).beats)
            self._beats.attach(self)
        return self._beats

    @property
    def tatums(self):
        if self._tatums is None:
            self._tatums = _dataParser('tatum', super(AudioAnalysis, self).tatums)
            self._tatums.attach(self)
        return self._tatums

    @property
    def sections(self):
        if self._sections is None:
            self._sections = _attributeParser('section', super(AudioAnalysis, self).sections)
            self._sections.attach(self)
        return self._sections

    @property
    def segments(self):
        if self._segments is None:
            self._segments = _segmentsParser(super(AudioAnalysis, self).segments)
            self._segments.attach(self)
        return self._segments

    def __getstate__(self):
        """
        Eliminates the circular reference for pickling.
        """
        dictclone = self.__dict__.copy()
        del dictclone['source']
        return dictclone

    def __setstate__(self, state):
        """
        Recreates circular references after unpickling.
        """
        self.__dict__.update(state)
        for cached_var in AudioAnalysis.CACHED_VARIABLES:
            if type(object.__getattribute__(self, cached_var)) == AudioQuantumList:
                object.__getattribute__(self, cached_var).attach(self)

class AudioRenderable(object):
    """
    An object that gives an `AudioData` in response to a call to its `render`\()
    method.
    Intended to be an abstract class that helps enforce the `AudioRenderable`
    protocol. Picked up a couple of convenience methods common to many descendants.

    Every `AudioRenderable` must provide three things:

    render()
        A method returning the `AudioData` for the object. The rhythmic duration (point
        at which any following audio is appended) is signified by the `endindex` accessor,
        measured in samples.
    source
        An accessor pointing to the `AudioData` that contains the original sample data of
        (a superset of) this audio object.
    duration
        An accessor returning the rhythmic duration (in seconds) of the audio object.
    """
    def resolve_source(self, alt):
        """
        Given an alternative, fallback `alt` source, return either `self`'s
        source or the alternative. Throw an informative error if no source
        is found.

        Utility code that ended up being replicated in several places, so
        it ended up here. Not necessary for use in the RenderableAudioObject
        protocol.
        """
        if hasattr(self, 'source'):
            source = self.source
        else:
            if isinstance(alt, AudioData):
                source = alt
            else:
                print >> sys.stderr, self.__repr__()
                raise EchoNestRemixError("%s has no implicit or explicit source during rendering." %
                                                (self.__class__.__name__, ))
        return source

    @staticmethod
    def init_audio_data(source, num_samples):
        """
        Convenience function for rendering: return a pre-allocated, zeroed
        `AudioData`.
        """
        if source.numChannels > 1:
            newchans = source.numChannels
            newshape = (num_samples, newchans)
        else:
            newchans = 1
            newshape = (num_samples,)
        return AudioData32(shape=newshape, sampleRate=source.sampleRate, numChannels=newchans, defer=False)

    def encode(self, filename):
        """
        Shortcut function that takes care of the need to obtain an `AudioData`
        object first, through `render`.
        """
        self.render().encode(filename)

class AudioData(AudioRenderable):
    """
    Handles audio data transparently. A smart audio container
    with accessors that include:

    sampleRate
        samples per second
    numChannels
        number of channels
    data
        a `numpy.array`_

    .. _numpy.array: http://docs.scipy.org/doc/numpy/reference/generated/numpy.array.html
    """
    def __init__(self, filename=None, ndarray = None, shape=None, sampleRate=None, numChannels=None, defer=False, verbose=True):
        """
        Given an input `ndarray`, import the sample values and shape
        (if none is specified) of the input `numpy.array`.

        Given a `filename` (and no input ndarray), use ffmpeg to convert
        the file to wave, then load the file into the data,
        auto-detecting the sample rate, and number of channels.

        :param filename: a path to an audio file for loading its sample
            data into the AudioData.data
        :param ndarray: a `numpy.array`_ instance with sample data
        :param shape: a tuple of array dimensions
        :param sampleRate: sample rate, in Hz
        :param numChannels: number of channels

        .. _numpy.array: http://docs.scipy.org/doc/numpy/reference/generated/numpy.array.html
        """
        self.verbose = verbose
        if (filename is not None) and (ndarray is None) :
            if sampleRate is None or numChannels is None:
                # force sampleRate and numChannels to 44100 hz, 2
                sampleRate, numChannels = 44100, 2
                parsestring = ffmpeg(filename, overwrite=False, verbose=self.verbose)
                ffmpeg_error_check(parsestring[1])
                sampleRate, numChannels = settings_from_ffmpeg(parsestring[1])
        self.defer = defer
        self.filename = filename
        self.sampleRate = sampleRate
        self.numChannels = numChannels
        self.convertedfile = None
        self.endindex = 0
        if shape is None and isinstance(ndarray, numpy.ndarray) and not self.defer:
            self.data = numpy.zeros(ndarray.shape, dtype=numpy.int16)
        elif shape is not None and not self.defer:
            self.data = numpy.zeros(shape, dtype=numpy.int16)
        elif not self.defer and self.filename:
            self.data = None
            self.load()
        else:
            self.data = None
        if ndarray is not None and self.data is not None:
            self.endindex = len(ndarray)
            self.data[0:self.endindex] = ndarray

    def load(self):
        if isinstance(self.data, numpy.ndarray):
            return
        if self.filename.lower().endswith(".wav") and (self.sampleRate, self.numChannels) == (44100, 2):
            file_to_read = self.filename
        elif self.convertedfile:
            file_to_read = self.convertedfile
        else:
            foo, self.convertedfile = tempfile.mkstemp(".wav")
            result = ffmpeg(self.filename, self.convertedfile, overwrite=True,
                numChannels=self.numChannels, sampleRate=self.sampleRate, verbose=self.verbose)
            ffmpeg_error_check(result[1])
            file_to_read = self.convertedfile

        w = wave.open(file_to_read, 'r')
        numFrames = w.getnframes()
        raw = w.readframes(numFrames)
        sampleSize = numFrames * self.numChannels
        data = numpy.frombuffer(raw, dtype="<h", count=sampleSize)
        ndarray = numpy.array(data, dtype=numpy.int16)
        if self.numChannels > 1:
            ndarray.resize((numFrames, self.numChannels))
        self.data = numpy.zeros(ndarray.shape, dtype=numpy.int16)
        self.endindex = 0
        if ndarray is not None:
            self.endindex = len(ndarray)
            self.data = ndarray

    def __getitem__(self, index):
        """
        Fetches a frame or slice. Returns an individual frame (if the index
        is a time offset float or an integer sample number) or a slice if
        the index is an `AudioQuantum` (or quacks like one).
        """
        if not isinstance(self.data, numpy.ndarray) and self.defer:
            self.load()
        if isinstance(index, float):
            index = int(index*self.sampleRate)
        elif hasattr(index, "start") and hasattr(index, "duration"):
            index =  slice(index.start, index.start+index.duration)

        if isinstance(index, slice):
            if ( hasattr(index.start, "start") and
                 hasattr(index.stop, "duration") and
                 hasattr(index.stop, "start") ) :
                index = slice(index.start.start, index.stop.start+index.stop.duration)

        if isinstance(index, slice):
            return self.getslice(index)
        else:
            return self.getsample(index)

    def getslice(self, index):
        "Help `__getitem__` return a new AudioData for a given slice"
        if not isinstance(self.data, numpy.ndarray) and self.defer:
            self.load()
        if isinstance(index.start, float):
            index = slice(int(index.start*self.sampleRate), int(index.stop*self.sampleRate), index.step)
        return AudioData(None, self.data[index], sampleRate=self.sampleRate, numChannels=self.numChannels, defer=False)

    def getsample(self, index):
        """
        Help `__getitem__` return a frame (all channels for a given
        sample index)
        """
        if not isinstance(self.data, numpy.ndarray) and self.defer:
            self.load()
        if isinstance(index, int):
            return self.data[index]
        else:
            #let the numpy array interface be clever
            return AudioData(None, self.data[index], defer=False)

    def pad_with_zeros(self, num_samples):
        if num_samples > 0:
            if self.numChannels == 1:
                extra_shape = (num_samples,)
            else:
                extra_shape = (num_samples, self.numChannels)
            self.data = numpy.append(self.data,
                                     numpy.zeros(extra_shape, dtype=numpy.int16), axis=0)

    def append(self, as2):
        "Appends the input to the end of this `AudioData`."
        extra = len(as2.data) - (len(self.data) - self.endindex)
        self.pad_with_zeros(extra)
        self.data[self.endindex:self.endindex+len(as2)] += as2.data
        self.endindex += as2.endindex

    def sum(self, as2):
        extra = len(as2.data) - len(self.data)
        self.pad_with_zeros(extra)
        compare_limit = min(len(as2.data), len(self.data)) - 1
        self.data[:compare_limit] += as2.data[:compare_limit]

    def add_at(self, time, as2):
        offset = int(time * self.sampleRate)
        extra = offset + len(as2.data) - len(self.data)
        self.pad_with_zeros(extra)
        self.data[offset:offset+len(as2.data)] += as2.data

    def __len__(self):
        if self.data is not None:
            return len(self.data)
        else:
            return 0

    def encode(self, filename=None, mp3=None):
        """
        Outputs an MP3 or WAVE file to `filename`.
        Format is determined by `mp3` parameter.
        """
        if not mp3 and filename.lower().endswith('.wav'):
            mp3 = False
        else:
            mp3 = True
        if mp3:
            foo, tempfilename = tempfile.mkstemp(".wav")
        else:
            tempfilename = filename
        fid = open(tempfilename, 'wb')
        # Based on Scipy svn
        # http://projects.scipy.org/pipermail/scipy-svn/2007-August/001189.html
        fid.write('RIFF')
        fid.write(struct.pack('i',0)) # write a 0 for length now, we'll go back and add it later
        fid.write('WAVE')
        # fmt chunk
        fid.write('fmt ')
        if self.data.ndim == 1:
            noc = 1
        else:
            noc = self.data.shape[1]
        bits = self.data.dtype.itemsize * 8
        sbytes = self.sampleRate*(bits / 8)*noc
        ba = noc * (bits / 8)
        fid.write(struct.pack('ihHiiHH', 16, 1, noc, self.sampleRate, sbytes, ba, bits))
        # data chunk
        fid.write('data')
        fid.write(struct.pack('i', self.data.nbytes))
        self.data.tofile(fid)
        # Determine file size and place it in correct
        # position at start of the file.
        size = fid.tell()
        fid.seek(4)
        fid.write(struct.pack('i', size-8))
        fid.close()
        if not mp3:
            return tempfilename
        # now convert it to mp3
        if not filename.lower().endswith('.mp3'):
            filename = filename + '.mp3'
        try:
            bitRate = config.MP3_BITRATE
        except NameError:
            bitRate = 128
        parsestring = ffmpeg(tempfilename, filename, bitRate=bitRate, verbose=self.verbose)
        ffmpeg_error_check(parsestring[1])
        if tempfilename != filename:
            if self.verbose:
                print >> sys.stderr, "Deleting: %s" % tempfilename
            os.remove(tempfilename)
        return filename

    def unload(self):
        self.data = None
        if self.convertedfile:
            if self.verbose:
                print >> sys.stderr, "Deleting: %s" % self.convertedfile
            os.remove(self.convertedfile)
            self.convertedfile = None

    def render(self):
        return self

    @property
    def duration(self):
        return float(self.endindex) / self.sampleRate

    @property
    def source(self):
        return self

class AudioData32(AudioData):
    """A 32-bit variant of AudioData, intended for data collection on
    audio rendering with headroom."""
    def __init__(self, filename=None, ndarray = None, shape=None, sampleRate=None, numChannels=None, defer=False, verbose=True):
        """
        Special form of AudioData to allow for headroom when collecting samples.
        """
        self.verbose = verbose
        if (filename is not None) and (ndarray is None) :
            if sampleRate is None or numChannels is None:
                # force sampleRate and numChannels to 44100 hz, 2
                sampleRate, numChannels = 44100, 2
                parsestring = ffmpeg(filename, overwrite=False, verbose=self.verbose)
                ffmpeg_error_check(parsestring[1])
                sampleRate, numChannels = settings_from_ffmpeg(parsestring[1])
        self.defer = defer
        self.filename = filename
        self.sampleRate = sampleRate
        self.numChannels = numChannels
        self.convertedfile = None
        self.normalized = None
        if shape is None and isinstance(ndarray, numpy.ndarray) and not self.defer:
            self.data = numpy.zeros(ndarray.shape, dtype=numpy.int32)
        elif shape is not None and not self.defer:
            self.data = numpy.zeros(shape, dtype=numpy.int32)
        elif not self.defer and self.filename:
            self.load()
        else:
            self.data = None
        self.endindex = 0
        if ndarray is not None and self.data is not None:
            self.endindex = len(ndarray)
            self.data[0:self.endindex] = ndarray

    def load(self):
        if isinstance(self.data, numpy.ndarray):
            return
        if self.filename.lower().endswith(".wav") and (self.sampleRate, self.numChannels) == (44100, 2):
            file_to_read = self.filename
        elif self.convertedfile:
            file_to_read = self.convertedfile
        else:
            foo, self.convertedfile = tempfile.mkstemp(".wav")
            result = ffmpeg(self.filename, self.convertedfile, overwrite=True,
                numChannels=self.numChannels, sampleRate=self.sampleRate, verbose=self.verbose)
            ffmpeg_error_check(result[1])
            file_to_read = self.convertedfile

        w = wave.open(file_to_read, 'r')
        numFrames = w.getnframes()
        raw = w.readframes(numFrames)
        sampleSize = numFrames * self.numChannels
        data = numpy.frombuffer(raw, dtype="<h", count=sampleSize)
        ndarray = numpy.array(data, dtype=numpy.int16)
        if self.numChannels > 1:
            ndarray.resize((numFrames, self.numChannels))
        self.data = numpy.zeros(ndarray.shape, dtype=numpy.int32)
        self.endindex = 0
        if ndarray is not None:
            self.endindex = len(ndarray)
            self.data[0:self.endindex] = ndarray

    def encode(self, filename=None, mp3=None):
        """
        Outputs an MP3 or WAVE file to `filename`.
        Format is determined by `mp3` parameter.
        """
        self.normalize()
        if not mp3 and filename.lower().endswith('.wav'):
            mp3 = False
        else:
            mp3 = True
        if mp3:
            foo, tempfilename = tempfile.mkstemp(".wav")
        else:
            tempfilename = filename
        fid = open(tempfilename, 'wb')
        # Based on Scipy svn
        # http://projects.scipy.org/pipermail/scipy-svn/2007-August/001189.html
        fid.write('RIFF')
        fid.write(struct.pack('i',0)) # write a 0 for length now, we'll go back and add it later
        fid.write('WAVE')
        # fmt chunk
        fid.write('fmt ')
        if self.normalized.ndim == 1:
            noc = 1
        else:
            noc = self.normalized.shape[1]
        bits = self.normalized.dtype.itemsize * 8
        sbytes = self.sampleRate*(bits / 8)*noc
        ba = noc * (bits / 8)
        fid.write(struct.pack('ihHiiHH', 16, 1, noc, self.sampleRate, sbytes, ba, bits))
        # data chunk
        fid.write('data')
        fid.write(struct.pack('i', self.normalized.nbytes))
        self.normalized.tofile(fid)
        # Determine file size and place it in correct
        # position at start of the file.
        size = fid.tell()
        fid.seek(4)
        fid.write(struct.pack('i', size-8))
        fid.close()
        self.normalized = None
        if not mp3:
            return tempfilename
        # now convert it to mp3
        if not filename.lower().endswith('.mp3'):
            filename = filename + '.mp3'
        try:
            bitRate = config.MP3_BITRATE
        except NameError:
            bitRate = 128
        parsestring = ffmpeg(tempfilename, filename, bitRate=bitRate, verbose=self.verbose)
        ffmpeg_error_check(parsestring[1])
        if tempfilename != filename:
            if self.verbose:
                print >> sys.stderr, "Deleting: %s" % tempfilename
            os.remove(tempfilename)
        return filename

    def normalize(self):
        """Return to 16-bit for encoding."""
        if self.numChannels == 1:
            self.normalized = numpy.zeros((self.data.shape[0],), dtype=numpy.int16)
        else:
            self.normalized = numpy.zeros((self.data.shape[0], self.data.shape[1]), dtype=numpy.int16)

        factor = 32767.0 / numpy.max(numpy.absolute(self.data.flatten()))
        # If the max was 32768, don't bother scaling:
        if factor < 1.000031:
            self.normalized[:len(self.data)] += self.data * factor
        else:
            self.normalized[:len(self.data)] += self.data

    def pad_with_zeros(self, num_samples):
        if num_samples > 0:
            if self.numChannels == 1:
                extra_shape = (num_samples,)
            else:
                extra_shape = (num_samples, self.numChannels)
            self.data = numpy.append(self.data,
                                     numpy.zeros(extra_shape, dtype=numpy.int32), axis=0)

def ffmpeg(infile, outfile=None, overwrite=True, bitRate=None, numChannels=None, sampleRate=None, verbose=True):
    """
    Executes ffmpeg through the shell to convert or read media files.
    """
    command = "en-ffmpeg"
    if overwrite:
        command += " -y"
    command += " -i \"" + infile + "\""
    if bitRate is not None:
        command += " -ab " + str(bitRate) + "k"
    if numChannels is not None:
        command += " -ac " + str(numChannels)
    if sampleRate is not None:
        command += " -ar " + str(sampleRate)
    if outfile is not None:
        command += " " + outfile
    if verbose:
        print >> sys.stderr, command
    p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return p.communicate()

def settings_from_ffmpeg(parsestring):
    """
    Parses the output of ffmpeg to determine sample rate and frequency of
    an audio file.
    """
    parse = parsestring.split('\n')
    freq, chans = 44100, 2
    for line in parse:
        if "Stream #0" in line and "Audio" in line:
            segs = line.split(", ")
            for s in segs:
                if "Hz" in s:
                    #print "Found: "+str(s.split(" ")[0])+"Hz"
                    freq = int(s.split(" ")[0])
                elif "stereo" in s:
                    #print "stereo"
                    chans = 2
                elif "mono" in s:
                    #print "mono"
                    chans = 1
    return freq, chans

def ffmpeg_error_check(parsestring):
    "Looks for known errors in the ffmpeg output"
    parse = parsestring.split('\n')
    for num, line in enumerate(parse):
        if "Unknown format" in line or "error occur" in line:
            raise RuntimeError("ffmpeg conversion error:\n\t" + "\n\t".join(parse[num:]))

def getpieces(audioData, segs):
    """
    Collects audio samples for output.
    Returns a new `AudioData` where the new sample data is assembled
    from the input audioData according to the time offsets in each
    of the elements of the input segs (commonly an `AudioQuantumList`).

    :param audioData: an `AudioData` object
    :param segs: an iterable containing objects that may be accessed
        as slices or indices for an `AudioData`
    """
    #calculate length of new segment
    audioData.data = None
    audioData.load()
    dur = 0
    for s in segs:
        dur += int(s.duration*audioData.sampleRate)
    # if I wanted to add some padding to the length, I'd do it here

    #determine shape of new array
    if len(audioData.data.shape) > 1:
        newshape = (dur, audioData.data.shape[1])
        newchans = audioData.data.shape[1]
    else:
        newshape = (dur,)
        newchans = 1

    #make accumulator segment
    newAD = AudioData(shape=newshape,sampleRate=audioData.sampleRate, numChannels=newchans, defer=False)

    #concatenate segs to the new segment
    for s in segs:
        newAD.append(audioData[s])
    # audioData.unload()
    return newAD

def assemble(audioDataList, numChannels=1, sampleRate=44100):
    """
    Collects audio samples for output.
    Rerturns a new `AudioData` object assembled
    by concatenating all the elements of audioDataList.

    :param audioDatas: a list of `AudioData` objects
    """
    if numChannels == 1:
        new_shape = (sum([len(x.data) for x in audioDataList]),)
    else:
        new_shape = (sum([len(x.data) for x in audioDataList]),numChannels)
    new_data = AudioData(shape=new_shape, numChannels=numChannels, sampleRate=sampleRate, defer=False)
    for ad in audioDataList:
        if not isinstance(ad, AudioData):
            raise TypeError('Encountered something other than an AudioData')
        new_data.append(ad)
    return new_data

def mix(dataA,dataB,mix=0.5):
    """
    Mixes two `AudioData` objects. Assumes they have the same sample rate
    and number of channels.

    Mix takes a float 0-1 and determines the relative mix of two audios.
    i.e., mix=0.9 yields greater presence of dataA in the final mix.
    """
    if dataA.endindex > dataB.endindex:
        newdata = AudioData(ndarray=dataA.data, sampleRate=dataA.sampleRate, numChannels=dataA.numChannels, defer=False)
        newdata.data *= float(mix)
        newdata.data[:dataB.endindex] += dataB.data[:] * (1 - float(mix))
    else:
        newdata = AudioData(ndarray=dataB.data, sampleRate=dataB.sampleRate, numChannels=dataB.numChannels, defer=False)
        newdata.data *= 1 - float(mix)
        newdata.data[:dataA.endindex] += dataA.data[:] * float(mix)
    return newdata

def megamix(dataList):
    """
    Mix together any number of `AudioData` objects. Keep the shape of
    the first one in the list. Assume they all have the same sample rate
    and number of channels.
    """
    if not isinstance(dataList, list):
        raise TypeError('input must be a list of AudioData objects')
    newdata = AudioData(shape=dataList[0].data.shape, sampleRate=dataList[0].sampleRate,
                            numChannels=dataList[0].numChannels, defer=False)
    for adata in dataList:
        if not isinstance(adata, AudioData):
            raise TypeError('input must be a list of AudioData objects')
        if len(adata) > len(newdata):
            newseg = AudioData(ndarray=adata[:newdata.endindex].data,
                                numChannels=newdata.numChannels,
                                sampleRate=newdata.sampleRate, defer=False)
            newseg.endindex = newdata.endindex
        else:
            newseg = AudioData(ndarray=adata.data,
                                numChannels=newdata.numChannels,
                                sampleRate=newdata.sampleRate, defer=False)
            newseg.endindex = adata.endindex
        newdata.data[:newseg.endindex] += newseg.data / float(len(dataList))
    newdata.endindex = len(newdata)
    return newdata

class AudioFile(AudioData) :
    """
    The basic do-everything class for remixing. Acts as an `AudioData`
    object, but with an added `analysis` selector which is an
    `AudioAnalysis` object.
    """
    def __init__(self, filename, verbose=True, defer=False):
        """
        :param filename: path to a local MP3 file
        """
        if verbose:
            print >> sys.stderr, "Uploading file for analysis..."
        AudioData.__init__(self, filename=filename, verbose=verbose, defer=defer)
        self.analysis = AudioAnalysis(filename)
        self.analysis.source = self

    @property
    def duration(self):
        """
        Since we consider `AudioFile` to be an evolved version of
        `AudioData`, we return the measured duration from the analysis.
        """
        return self.analysis.duration

    def __setstate__(self, state):
        """
        Recreates circular reference after unpickling.
        """
        self.__dict__.update(state)
        self.analysis.source = weakref.proxy(self)

class LocalAudioFile(AudioFile):
    """
    Like `AudioFile`, but with conditional upload: recommended. If a file
    is already known to the Analyze API, then it does not bother uploading
    the file.
    """
    def __init__(self, filename, verbose=True, defer=False):
        """
        :param filename: path to a local MP3 file
        """
        trackID = hashlib.md5(file(filename, 'rb').read()).hexdigest()
        if verbose:
            print >> sys.stderr, "Computed MD5 of file is " + trackID
        try:
            if verbose:
                print >> sys.stderr, "Probing for existing analysis"
            tempanalysis = AudioAnalysis(trackID)
            tempanalysis.metadata
            self.analysis = AudioAnalysis(trackID)
            if verbose:
                print >> sys.stderr, "Analysis found. No upload needed."
        except pyechonest.util.EchoNestAPIThingIDError:
            if verbose:
                print >> sys.stderr, "Analysis not found. Uploading..."
            self.analysis = AudioAnalysis(filename)
        AudioData.__init__(self, filename=filename, verbose=verbose, defer=defer)
        self.analysis.source = self

    def toxml(self, context=None):
        track = stupidxml.Node("trackinfo")
        track.attributes.id = self.analysis.id
        track.attributes.filename = self.filename
        metadata = stupidxml.Node("metadata")
        metadata.attributes.update(self.analysis.metadata)
        track.append(metadata)
        return track

class LocalAnalysis(object):
    """
    Like `LocalAudioFile`, it conditionally uploads the file with which
    it was initialized. Unlike `LocalAudioFile`, it is not a subclass of
    `AudioData`, so contains no sample data.
    """
    def __init__(self, filename, verbose=True):
        """
        :param filename: path to a local MP3 file
        """
        trackID = hashlib.md5(file(filename, 'rb').read()).hexdigest()
        if verbose:
            print >> sys.stderr, "Computed MD5 of file is " + trackID
        try:
            if verbose:
                print >> sys.stderr, "Probing for existing analysis"
            tempanalysis = AudioAnalysis(trackID)
            tempanalysis.metadata
            self.analysis = AudioAnalysis(trackID)
            if verbose:
                print >> sys.stderr, "Analysis found. No upload needed."
        except pyechonest.util.EchoNestAPIThingIDError:
            if verbose:
                print >> sys.stderr, "Analysis not found. Uploading..."
            self.analysis = AudioAnalysis(filename)
        # no AudioData.__init__()

class AudioQuantum(AudioRenderable) :
    """
    A unit of musical time, identified at minimum with a start time and
    a duration, both in seconds. It most often corresponds with a `section`,
    `bar`, `beat`, `tatum`, or (by inheritance) `segment` obtained from an Analyze
    API call.

    Additional properties include:

    end
        computed time offset for convenience: `start` + `duration`
    container
        a circular reference to the containing `AudioQuantumList`,
        created upon creation of the `AudioQuantumList` that covers
        the whole track
    """
    def __init__(self, start=0, duration=0, kind=None, confidence=None, source=None) :
        """
        Initializes an `AudioQuantum`.

        :param start: offset from the start of the track, in seconds
        :param duration: length of the `AudioQuantum`
        :param kind: string containing what kind of rhythm unit it came from
        :param confidence: float between zero and one
        """
        self.start = start
        self.duration = duration
        self.kind = kind
        self.confidence = confidence
        self._source = source

    def get_end(self):
        return self.start + self.duration

    end = property(get_end, doc="""
    A computed property: the sum of `start` and `duration`.
    """)

    def get_source(self):
        "Returns itself or its parent."
        if self._source:
            return self._source
        else:
            source = None
            try:
                source = self.container.source
            except AttributeError:
                source = None
            return source

    def set_source(self, value):
        if isinstance(value, AudioData):
            self._source = value
        else:
            raise TypeError("Source must be an instance of echonest.audio.AudioData")

    source = property(get_source, set_source, doc="""
    The `AudioData` source for the AudioQuantum.
    """)

    def sources(self):
        return set([self.source])

    def parent(self):
        """
        Returns the containing `AudioQuantum` in the rhythm hierarchy:
        a `tatum` returns a `beat`, a `beat` returns a `bar`, and a `bar` returns a
        `section`.
        """
        pars = {'tatum': 'beats',
                'beat':  'bars',
                'bar':   'sections'}
        try:
            uppers = getattr(self.container.container, pars[self.kind])
            return uppers.that(selection.overlap(self))[0]
        except LookupError:
            # Might not be in pars, might not have anything in parent.
            return None

    def children(self):
        """
        Returns an `AudioQuantumList` of the AudioQuanta that it contains,
        one step down the hierarchy. A `beat` returns `tatums`, a `bar` returns
        `beats`, and a `section` returns `bars`.
        """
        chils = {'beat':    'tatums',
                 'bar':     'beats',
                 'section': 'bars'}
        try:
            downers = getattr(self.container.container, chils[self.kind])
            return downers.that(selection.are_contained_by(self))
        except LookupError:
            return None

    def group(self):
        """
        Returns the `children`\() of the `AudioQuantum`\'s `parent`\().
        In other words: 'siblings'. If no parent is found, then return the
        `AudioQuantumList` for the whole track.
        """
        if self.parent():
            return self.parent().children()
        else:
            return self.container

    def prev(self, step=1):
        """
        Step backwards in the containing `AudioQuantumList`.
        Returns `self` if a boundary is reached.
        """
        group = self.container
        try:
            loc = group.index(self)
            new = max(loc - step, 0)
            return group[new]
        except:
            return self

    def next(self, step=1):
        """
        Step forward in the containing `AudioQuantumList`.
        Returns `self` if a boundary is reached.
        """
        group = self.container
        try:
            loc = group.index(self)
            new = min(loc + step, len(group))
            return group[new]
        except:
            return self

    def __str__(self):
        """
        Lists the `AudioQuantum`.kind with start and
        end times, in seconds, e.g.::

            "segment (20.31 - 20.42)"
        """
        return "%s (%.2f - %.2f)" % (self.kind, self.start, self.end)

    def __repr__(self):
        """
        A string representing a constructor, including kind, start time,
        duration, and (if it exists) confidence, e.g.::

            "AudioQuantum(kind='tatum', start=42.198267, duration=0.1523394)"
        """
        if self.confidence is not None:
            return "AudioQuantum(kind='%s', start=%f, duration=%f, confidence=%f)" % (self.kind, self.start, self.duration, self.confidence)
        else:
            return "AudioQuantum(kind='%s', start=%f, duration=%f)" % (self.kind, self.start, self.duration)

    def local_context(self):
        """
        Returns a tuple of (*index*, *length*) within rhythm siblings, where
        *index* is the (zero-indexed) position within its `group`\(), and
        *length* is the number of siblings within its `group`\().
        """
        group = self.group()
        count = len(group)
        try:
            loc  = group.index(self)
        except: # seem to be some uncontained beats
            loc = 0
        return (loc, count,)

    def absolute_context(self):
        """
        Returns a tuple of (*index*, *length*) within the containing
        `AudioQuantumList`, where *index* is the (zero-indexed) position within
        its container, and *length* is the number of siblings within the
        container.
        """
        group = self.container
        count = len(group)
        loc = group.index(self)
        return (loc, count,)

    def context_string(self):
        """
        Returns a one-indexed, human-readable version of context.
        For example::

            "bar 4 of 142, beat 3 of 4, tatum 2 of 3"
        """
        if self.parent() and self.kind != "bar":
            return "%s, %s %i of %i" % (self.parent().context_string(),
                                  self.kind, self.local_context()[0] + 1,
                                  self.local_context()[1])
        else:
            return "%s %i of %i" % (self.kind, self.absolute_context()[0] + 1,
                                  self.absolute_context()[1])

    def __getstate__(self):
        """
        Eliminates the circular reference for pickling.
        """
        dictclone = self.__dict__.copy()
        del dictclone['container']
        return dictclone

    def toxml(self, context=None):
        attributedict = {'duration': self.duration,
                         'start': self.start}
        try:
            if not(hasattr(context, 'source') and self.source == context.source):
                attributedict['source'] = self.source.analysis.id
        except:
            pass
        return stupidxml.Node(self.kind, **attributedict)

    def render(self, start=0.0, to_audio=None, with_source=None):
        if not to_audio:
            source = self.resolve_source(with_source)
            return source[self]
        if with_source != self.source:
            return
        to_audio.add_at(start, with_source[self])
        return

class AudioSegment(AudioQuantum):
    """
    Subclass of `AudioQuantum` for the data-rich segments returned by
    the Analyze API.
    """
    def __init__(self, start=0., duration=0., pitches=[], timbre=[],
                 loudness_begin=0., loudness_max=0., time_loudness_max=0.,
                 loudness_end=None, kind='segment', source=None):
        """
        Initializes an `AudioSegment`.

        :param start: offset from start of the track, in seconds
        :param duration: duration of the `AudioSegment`, in seconds
        :param pitches: a twelve-element list with relative loudnesses of each
                pitch class, from C (pitches[0]) to B (pitches[11])
        :param timbre: a twelve-element list with the loudness of each of a
                principal component of time and/or frequency profile
        :param kind: string identifying the kind of AudioQuantum: "segment"
        :param loudness_begin: loudness in dB at the start of the segment
        :param loudness_max: loudness in dB at the loudest moment of the
                segment
        :param time_loudness_max: time (in sec from start of segment) of
                loudest moment
        :param loudness_end: loudness at end of segment (if it is given)
        """
        self.start = start
        self.duration = duration
        self.pitches = pitches
        self.timbre = timbre
        self.loudness_begin = loudness_begin
        self.loudness_max = loudness_max
        self.time_loudness_max = time_loudness_max
        if loudness_end:
            self.loudness_end = loudness_end
        self.kind = kind
        self.confidence = None
        self._source = source

class ModifiedRenderable(AudioRenderable):
    """Class that contains any AudioRenderable, but overrides the
    render() method with nested effects, called sequentially on the
    result of the preceeding effect."""
    def __init__(self, original, effects=[]):
        if isinstance(original, ModifiedRenderable):
            self._original = original._original
            self._effects = original._effects + effects
        else:
            self._original = original
            self._effects = effects

    @property
    def duration(self):
        dur = self._original.duration
        for effect in self._effects:
            if hasattr(effect, 'duration'):
                dur = effect.duration(dur)
        return dur

    @property
    def source(self):
        return self._original.source

    @property
    def sources(self):
        return self._original.sources

    def render(self, start=0.0, to_audio=None, with_source=None):
        if not to_audio:
            source = self.resolve_source(with_source)
            base = self._original.render(with_source=with_source)
            copy = AudioData32(ndarray=base.data, sampleRate=base.sampleRate, numChannels=base.numChannels, defer=False)
            for effect in self._effects:
                copy = effect.modify(copy)
            return copy
        if with_source != self.source:
            return
        base = self._original.render(with_source=with_source)
        copy = AudioData32(ndarray=base.data, shape=base.data.shape, sampleRate=base.sampleRate, numChannels=base.numChannels, defer=False)
        for effect in self._effects:
            copy = effect.modify(copy)
        to_audio.add_at(start, copy)
        return

class AudioEffect(object):
    def __call__(self, aq):
        return ModifiedRenderable(aq, [self])

class LevelDB(AudioEffect):
    def __init__(self, change):
        self.change = change

    def modify(self, adata):
        adata.data *= pow(10.,self.change/20.)
        return adata

class AmplitudeFactor(AudioEffect):
    def __init__(self, change):
        self.change = change

    def modify(self, adata):
        adata.data *= self.change
        return adata

class TimeTruncateFactor(AudioEffect):
    def __init__(self, factor):
        self.factor = factor

    def duration(self, old_duration):
        return old_duration * self.factor

    def modify(self, adata):
        endindex = int(self.factor * len(adata))
        if self.factor > 1:
            adata.pad_with_zeros(endindex - len(adata))
        adata.endindex = endindex
        return adata[:endindex]

class TimeTruncateLength(AudioEffect):
    def __init__(self, duration):
        self.new_duration = duration

    def duration(self, old_duration):
        return self.new_duration

    def modify(self, adata):
        endindex = int(self.new_duration * adata.sampleRate)
        if self.new_duration > adata.duration:
            adata.pad_with_zeros(endindex - len(adata))
        adata.endindex = endindex
        return adata[:endindex]

class AudioQuantumList(list, AudioRenderable):
    """
    A container that enables content-based selection and filtering.
    A `List` that contains `AudioQuantum` objects, with additional methods
    for manipulating them.

    When an `AudioQuantumList` is created for a track via a call to the
    Analyze API, `attach`\() is called so that its container is set to the
    containing `AudioAnalysis`, and the container of each of the
    `AudioQuantum` list members is set to itself.

    Additional accessors now include AudioQuantum elements such as
    `start`, `duration`, and `confidence`, which each return a List of the
    corresponding properties in the contained AudioQuanta. A special name
    is `kinds`, which returns a List of the `kind` of each `AudioQuantum`.
    If `AudioQuantumList.kind` is "`segment`", then `pitches`, `timbre`,
    `loudness_begin`, `loudness_max`, `time_loudness_max`, and `loudness_end`
    are available.
    """
    def __init__(self, initial = None, kind = None, container = None, source = None):
        """
        Initializes an `AudioQuantumList`. All parameters are optional.

        :param initial: a `List` type with the initial contents
        :param kind: a label for the kind of `AudioQuantum` contained
            within
        :param container: a reference to the containing `AudioAnalysis`
        :param source: a reference to the `AudioData` with the corresponding samples
            and time base for the contained AudioQuanta
        """
        list.__init__(self)
        self.kind = None
        self._source = None
        if isinstance(initial, AudioQuantumList):
            self.kind = initial.kind
            self.container = initial.container
            self._source = initial.source
        if kind:
            self.kind = kind
        if container:
            self.container = container
        if source:
            self._source = source
        if initial:
            self.extend(initial)

    def get_many(attribute):
        def fun(self):
            """
            Returns a list of %s for each `AudioQuantum`.
            """ % attribute
            return [getattr(x, attribute) for x in list.__iter__(self)]
        return fun

    def get_many_if_segment(attribute):
        def fun(self):
            """
            Returns a list of %s for each `Segment`.
            """ % attribute
            if self.kind == 'segment':
                return [getattr(x, attribute) for x in list.__iter__(self)]
            else:
                raise AttributeError("<%s> only accessible for segments" % (attribute,))
        return fun

    def get_duration(self):
        return sum(self.durations)
        #return sum([x.duration for x in self])

    def get_source(self):
        "Returns its own or its parent's source."
        if self._source:
            return self._source
        else:
            try:
                source = self.container.source
            except AttributeError:
                source = self[0].source
            return source

    def set_source(self, value):
        "Checks input to see if it is an `AudioData`."
        if isinstance(value, AudioData):
            self._source = value
        else:
            raise TypeError("Source must be an instance of echonest.audio.AudioData")

    durations  = property(get_many('duration'))
    kinds      = property(get_many('kind'))
    start      = property(get_many('start'))
    confidence = property(get_many('confidence'))

    pitches           = property(get_many_if_segment('pitches'))
    timbre            = property(get_many_if_segment('timbre'))
    loudness_begin    = property(get_many_if_segment('loudness_begin'))
    loudness_max      = property(get_many_if_segment('loudness_max'))
    time_loudness_max = property(get_many_if_segment('time_loudness_max'))
    loudness_end      = property(get_many_if_segment('loudness_end'))

    source = property(get_source, set_source, doc="""
    The `AudioData` source for the `AudioQuantumList`.
    """)

    duration = property(get_duration, doc="""
    Total duration of the `AudioQuantumList`.
    """)

    def sources(self):
        ss = set()
        for aq in list.__iter__(self):
            ss.update(aq.sources())
        return ss

    def that(self, filt):
        """
        Method for applying a function to each of the contained
        `AudioQuantum` objects. Returns a new `AudioQuantumList`
        of the same `kind` containing the `AudioQuantum` objects
        for which the input function is true.

        See `echonest.selection` for example selection filters.

        :param filt: a function that takes one `AudioQuantum` and returns
            a `True` value `None`

        :change: experimenting with a filter-only form
        """
        out = AudioQuantumList(kind=self.kind)
        out.extend(filter(filt, self))
        return out

    def ordered_by(self, function, descending=False):
        """
        Returns a new `AudioQuantumList` of the same `kind` with the
        original elements, but ordered from low to high according to
        the input function acting as a key.

        See `echonest.sorting` for example ordering functions.

        :param function: a function that takes one `AudioQuantum` and returns
            a comparison key
        :param descending: when `True`, reverses the sort order, from
            high to low
        """
        out = AudioQuantumList(kind=self.kind)
        out.extend(sorted(self, key=function, reverse=descending))
        return out

    def beget(self, source, which=None):
        """
        There are two basic forms: a map-and-flatten and an converse-that.

        The basic form, with one `function` argument, returns a new
        `AudioQuantumList` so that the source function returns
        `None`, one, or many AudioQuanta for each `AudioQuantum` contained within
        `self`, and flattens them, in order. ::

            beats.beget(the_next_ones)

        A second form has the first argument `source` as an `AudioQuantumList`, and
        a second argument, `which`, is used as a filter for the first argument, for
        *each* of `self`. The results are collapsed and accordianned into a flat
        list.

        For example, calling::

            beats.beget(segments, which=overlap)

        Gets evaluated as::

            for beat in beats:
                return segments.that(overlap(beat))

        And all of the `AudioQuantumList`\s that return are flattened into
        a single `AudioQuantumList`.

        :param source: A function of one argument that is applied to each
            `AudioQuantum` of `self`, or an `AudioQuantumList`, in which case
            the second argument is required.
        :param which: A function of one argument that acts as a `that`\() filter
            on the first argument if it is an `AudioQuantumList`, or as a filter
            on the output, in the case of `source` being a function.
        """
        out = AudioQuantumList()
        if isinstance(source, AudioQuantumList):
            if not which:
                raise TypeError("'beget' requires a second argument, 'which'")
            out.extend(chain_from_mixed([source.that(which(x)) for x in self]))
        else:
            out.extend(chain_from_mixed(map(source, self)))
            if which:
                out = out.that(which)
        return out

    def attach(self, container):
        """
        Create circular references to the containing `AudioAnalysis` and for the
        contained `AudioQuantum` objects.
        """
        self.container = container
        for i in self:
            i.container = self

    def __getstate__(self):
        """
        Eliminates the circular reference for pickling.
        """
        dictclone = self.__dict__.copy()
        del dictclone['container']
        return dictclone

    def toxml(self, context=None):
        xml = stupidxml.Node("sequence")
        xml.attributes.duration = self.duration
        if not context:
            xml.attributes.source = self.source.analysis.id
            for s in self.sources():
                xml.append(s.toxml())
        elif self._source:
            try:
                if self.source != context.source:
                    xml.attributes.source = self.source.analysis.id
            except:
                pass
        for x in list.__iter__(self):
            xml.append(x.toxml(context=self))
        return xml

    def render(self, start=0.0, to_audio=None, with_source=None):
        if not to_audio:
            dur = 0
            tempsource = self.source or list.__getitem__(self, 0).source
            for aq in list.__iter__(self):
                dur += int(aq.duration * tempsource.sampleRate)
            to_audio = self.init_audio_data(tempsource, dur)
        if not hasattr(with_source, 'data'):
            for tsource in self.sources():
                this_start = start
                for aq in list.__iter__(self):
                    aq.render(start=this_start, to_audio=to_audio, with_source=tsource)
                    this_start += aq.duration
                if tsource.defer: tsource.unload()
            return to_audio
        else:
            if with_source not in self.sources():
                return
            for aq in list.__iter__(self):
                aq.render(start=start, to_audio=to_audio, with_source=with_source)
                start += aq.duration

class Simultaneous(AudioQuantumList):
    """
    Stacks all contained AudioQuanta atop one another, adding their respective
    samples. The rhythmic length of the segment is the duration of the first
    `AudioQuantum`, but there can be significant overlap caused by the longest
    segment.

    Sample usage::
        Simultaneous(a.analysis.bars).encode("my.mp3")
    """
    def __init__(self, *args, **kwargs):
         AudioQuantumList.__init__(self, *args, **kwargs)

    def get_duration(self):
        try:
            return self[0].duration
        except:
            return 0.

    duration = property(get_duration, doc="""
        Rhythmic duration of the `Simultaneous` AudioQuanta: the
        same as the duration of the first in the list.
        """)

    def toxml(self, context=None):
        xml = stupidxml.Node("parallel")
        xml.attributes.duration = self.duration
        if not context:
            xml.attributes.source = self.source.analysis.id
        elif self.source != context.source:
            try:
                xml.attributes.source = self.source.analysis.id
            except:
                pass
        for x in list.__iter__(self):
            xml.append(x.toxml(context=self))
        return xml

    def render(self, start=0.0, to_audio=None, with_source=None):
        if not to_audio:
            tempsource = self.source or list.__getitem__(self, 0).source
            dur = int(max(self.durations) * tempsource.sampleRate)
            to_audio = self.init_audio_data(tempsource, dur)
        if not hasattr(with_source, 'data'):
            for source in self.sources():
                for aq in list.__iter__(self):
                    aq.render(start=start, to_audio=to_audio, with_source=source)
                if source.defer: source.unload()
            return to_audio
        else:
            if with_source not in self.sources():
                return
            else:
                for aq in list.__iter__(self):
                    aq.render(start=start, to_audio=to_audio, with_source=with_source)

def _dataParser(tag, nodes):
    out = AudioQuantumList(kind=tag)
    for n in nodes:
        out.append(AudioQuantum(start=n['start'], kind=tag, confidence=n['confidence']))
    if len(out) > 1:
        for i in range(len(out) - 1) :
            out[i].duration = out[i+1].start - out[i].start
        out[-1].duration = out[-2].duration
    return out

def _attributeParser(tag, nodes):
    out = AudioQuantumList(kind=tag)
    for n in nodes :
        out.append(AudioQuantum(n['start'], n['duration'], tag))
    return out

def _segmentsParser(nodes):
    out = AudioQuantumList(kind='segment')
    for n in nodes:
        out.append(AudioSegment(start=n['start'], duration=n['duration'],
                                pitches=n['pitches'], timbre=n['timbre'],
                                loudness_begin=n['loudness_begin'],
                                loudness_max=n['loudness_max'],
                                time_loudness_max=n['time_loudness_max'],
                                loudness_end=n['loudness_end']))
    return out

def chain_from_mixed(iterables):
    """
    Helper function to flatten a list of elements and lists
    into a list of elements.
    """
    for y in iterables:
        try:
            iter(y)
            for element in y:
                yield element
        except:
            yield y

class FileTypeError(Exception):
    def __init__(self, filename, message):
        self.filename = filename
        self.message = message

    def __str__(self):
        return self.message+': '+self.filename

class EchoNestRemixError(Exception):
    """
    Error raised by the Remix API.
    """
    pass


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 25 21:25:24 2019

@group: Monday 13-15 pm Group 4

@Tutor name: Anirban Saha

@Authors: Chandan Bhat(229746), Rashmi Koparde(230322), Suraj Shashidhar(230052)

Description: The program has to parse input data along with the expected results.
    It should come up with its own implementation of multi variable linear equation.
    It should calculate the appropriate weights to reduce the errors in predicted results.

    The program should be able to run on any dataset and should be able to take
    learning rate and error threshold from the command line.

    User can then tune the learning threshold and threshold from command line.

Run_Format:
python3 linearregr.py --data "/Users/surajshashidhar/Desktop/ovgu/semester_1/machine_learning/programming/linreg/random.csv" --learningRate "0.0001" --threshold "0.0001"

python3 linearregr.py --data "/Users/surajshashidhar/Desktop/ovgu/semester_1/machine_learning/programming/linreg/yacht.csv --learningRate "0.0001" --threshold "0.0001"
"""

import numpy as np
import pandas as pd
import argparse
import os
"""
Importing Numpy for array calculation, Importing pandas for file I/O, Import argparse for argument parsing
os for getting base path for output location
"""
ap = argparse.ArgumentParser()

# Add the arguments to the parser and parse the arguments from command line
ap.add_argument( "--data", required=True, help="first operand")
ap.add_argument("--learningRate", required=True, help="second operand")
ap.add_argument("--threshold", required=True, help="third operand")
#ap.add_argument("--output", required=True, help="fourth operand")
args = vars(ap.parse_args())

filepath = args["data"]
learning_rate = float(args["learningRate"])
threshold = float(args["threshold"])
#output_filepath = args["output"]

"""
Function name: readInputFile
input: filepath
output: dataframe with features and expected value , with number of weights
"""
def readInputFile(filepath):
    tmpDf = pd.read_csv(filepath);
    print("Record count is: " + str(tmpDf.shape[0]))
    numOfCols = tmpDf.shape[1]
    collist = []
    base_columnname = "x"
    t = base_columnname

    for i in range(1,numOfCols):
        t = "x"
        t = t + str(i)
        collist.append(t)

    collist.append("expected_value")

    print(*collist)
    tmpDf = pd.read_csv(filepath,  index_col = None, names = collist);
    tmpDf.insert(0, "x0", 1.0)
    return numOfCols, tmpDf;

# get the input dataframe along with number of weights need to be calculated
numOfWeights, inputDf = readInputFile(filepath)

#convert dataframe into an numpy array
inputNpArray = inputDf.rename_axis('ID').values

# extract the features and expected values into separate Numpy arrays
featureNpArray = inputNpArray[:,:-1]
expectedvalueNpArray = inputNpArray[:,-1:]
record_count = expectedvalueNpArray.shape[0]

#Function takes record_count and numOfWeights as ip and returns
#initilized weights and predicted array
def initialize_required_arrays(record_count,numOfWeights):
    weightsNpArray = np.zeros(shape=(1, numOfWeights));
    predictedvalueNpArray = np.zeros(shape=(record_count,1));
    errorNpArray =  np.zeros(shape=(record_count,1));
    gradientNpArray = np.zeros(shape=(1, numOfWeights));
    return weightsNpArray, predictedvalueNpArray, errorNpArray, gradientNpArray

#initializing the wights and predicted value numpy arrays
weightsNpArray, predictedvalueNpArray, errorNpArray, gradientNpArray = initialize_required_arrays(record_count,numOfWeights);

#Calculate the predicted value for example: f(x) = w0 + w1x1 + w2x2 + w3x3
def calculate_predicted_value(weightsNpArray, featureNpArray):
    return np.dot(featureNpArray,weightsNpArray.T);

"""
Function name: update_weights
input: earning_rate, weightsNpArray, featureNpArray, expectedvalueNpArray
output: numpy array of weights

Function takes input parameters, First it calculates predicted value
error = expected value - predicted value
gradient = error * feature
new weight = old weight + learning rate * gradient
"""
def update_weights(learning_rate, weightsNpArray, featureNpArray, expectedvalueNpArray):
    predictedvalueNpArray = calculate_predicted_value(weightsNpArray, featureNpArray);
    errorNpArray = expectedvalueNpArray - predictedvalueNpArray;
    tmpNpArray = errorNpArray*featureNpArray
    gradientNpArray = np.sum(tmpNpArray, axis = 0)
    weightsNpArray = weightsNpArray + learning_rate * gradientNpArray
    return weightsNpArray

# This function calculates cost by first squaring entire error array and summing the array to single value
def calculate_costfunction(featureNpArray, expectedvalueNpArray, weightsNpArray):
    predicted_values = calculate_predicted_value(weightsNpArray, featureNpArray)
    error = expectedvalueNpArray - predicted_values;
    sq_error = np.array(error)**2
    return round(sq_error.sum(),4)

#initialization for job run
output_weight_lst = []
output_cost_lst = []
cnt = 0
prev_error = 0.0
flag = 0

#Start the updation process
"""
First calculate the error / cost
use the error to update the weights for the next batch
check if the threshold is met or we have already reached the minima and cannot further minimize the error
if thats the case then break from loop
else use the new weights and again do the above things
"""
while(1):
    cost = calculate_costfunction(featureNpArray, expectedvalueNpArray, weightsNpArray);
    output_cost_lst.append(cost)
    weightsNpArray = update_weights(learning_rate, weightsNpArray, featureNpArray, expectedvalueNpArray)
    output_weight_lst.append(weightsNpArray.tolist())
    cnt = cnt + 1
    if(abs(prev_error - cost) < threshold ):
        break;
    prev_error = cost

print("\n ---------------------------------------------------- \n")

#Process to just collect the updated weights and append it into a dataframe.
#After dataframe has been created just print its head and tail and create a output file from it
output_lst = []

for i in range(cnt):
    tmp_lst = []
    for j in range(len(output_weight_lst[0])):
        for k in range(len(output_weight_lst[0][0])):
            tmp_lst.append(output_weight_lst[i][j][k])
    tmp_lst.append(output_cost_lst[i])
    output_lst.append(tmp_lst)

columnlist = []
for i in range(numOfWeights):
    basename = "weight"
    columnname = "weight" + str(i)
    columnlist.append(columnname)

columnlist.append("sum_of_squared_errors")

output_df = pd.DataFrame(output_lst, columns = columnlist, index = None)
output_df.index.name = "iteration_number"

print(output_df.head(5))
print(output_df.tail(5))
print(output_df.shape)

#Creating a output file consisting of weights and sum of squared errors from dataframe

#print(os.path.basename(filepath))
output_filename = os.path.basename(filepath)
output_filename = "p01_" + output_filename
output_df.to_csv(output_filename, sep=',', encoding='utf-8')


from collections import defaultdict
from collections import deque
def solution(n, edge):
    D = defaultdict(list)
    for a, b in edge:
        D[a].append(b)
        D[b].append(a)

    S = [20000] * (n+1)
    S[0] = S[1] = 0
    Q = deque();
    Q.append(1)
    while Q:
        idx = Q.popleft()
        for v in D[idx]:
            if S[v] > S[idx] + 1:
                S[v] = S[idx] + 1
                Q.append(v)
    return S.count(max(S))

#Test 1 〉	통과 (0.01ms, 10.3MB)
#Test 2 〉	통과 (0.02ms, 10.2MB)
#Test 3 〉	통과 (0.04ms, 10.3MB)
#Test 4 〉	통과 (0.24ms, 10.2MB)
#Test 5 〉	통과 (0.98ms, 10.6MB)
#Test 6 〉	통과 (2.68ms, 10.8MB)
#Test 7 〉	통과 (24.39ms, 17.5MB)
#Test 8 〉	통과 (40.20ms, 20.8MB)
#Test 9 〉	통과 (35.80ms, 20.5MB)

# T/C 8 번만 시간초과
def solution2(n, edge):
    S = [0] * n
    S[0] = 0
    while True:
        changed = False
        for a, b in edge:
            if S[b-1] > S[a-1]+1:
                S[b-1] = S[a-1]+1
                changed = True
            elif S[a-1] > S[b-1]+1:
                S[a-1] = S[b-1]+1
                changed = True
        if not changed:
            break
    return S.count(max(S))

# 노드의 개수 n은 2 이상 20,000 이하입니다.
# 간선은 양방향이며 총 1개 이상 50,000개 이하의 간선이 있습니다.
# vertex 배열 각 행 [a, b]는 a번 노드와 b번 노드 사이에 간선이 있다는 의미입니다.
print(solution(6, [[3, 6], [4, 3], [3, 2], [1, 3], [1, 2], [2, 4], [5, 2]]), 3)
print(solution(5, []), 4)
print(solution(5, [[1,2]]), 3)
print(solution(5, [[1,1]]), 4)


#To change file extension of files in the folder, useful for image projects
import os

path = 'C:\\Users\\Name\\Desktop\\folder'  #type in your file path with \\
folder= os.chdir(path)

os.listdir(folder)
i=0
count=0
for file in os.listdir(folder):
    print(file) # to check on the existing files in the folder
    filename, extension = os.path.splitext(file)
    new_extension='.png' #type in your desired filetype, example '.jpg','.tif','.png' etc
    #new_filename="myfile{}".format(i)  #use this to change filename
    #i=i+1
    #new=os.rename(file, new_filename + new_extension) #to change both filename and extension together
    new=os.rename(file, filename + new_extension) #change this to comment if filename renaming cmd above is used.
    count=count+1

print("There are", (count),"files in this folder.\nThank you!")

path = os.path.realpath(path)
os.startfile(path)

Auto change file extension for all files in the folder.py


import numpy as np, random, scipy.stats as ss

def majority_vote_fast(votes):
    mode, count = ss.mstats.mode(votes)
    return mode

def distance(p1, p2):
    return np.sqrt(np.sum(np.power(p2 - p1, 2)))

def find_nearest_neighbors(p, points, k=5):
    distances = np.zeros(points.shape[0])
    for i in range(len(distances)):
        distances[i] = distance(p, points[i])
    ind = np.argsort(distances)
    return ind[:k]

def knn_predict(p, points, outcomes, k=5):
    ind = find_nearest_neighbors(p, points, k)
    return majority_vote_fast(outcomes[ind])[0]

import pandas as pd

data = pd.read_csv("https://courses.edx.org/asset-v1:HarvardX+PH526x+2T2019+type@asset+block@wine.csv", index_col = 0)

data['is_red'] = (data["color"] == "red").astype(int)

numeric_data = data.drop('color' ,axis = 1)
numeric_data.groupby('is_red').count()

import sklearn.preprocessing as sp

scaled_data = sp.scale(numeric_data)
numeric_data = pd.DataFrame(scaled_data, columns = numeric_data.columns)

import sklearn.decomposition as sd

pca = sd.PCA()
principal_components = pca.fit_transform(numeric_data)
print(principal_components.shape)

"""EX 4"""
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from matplotlib.backends.backend_pdf import PdfPages
observation_colormap = ListedColormap(['red', 'blue'])
x = principal_components[:,0]
y = principal_components[:,1]

plt.title("Principal Components of Wine")
plt.scatter(x, y, alpha = 0.2,
    c = data['high_quality'], cmap = observation_colormap, edgecolors = 'none')
plt.xlim(-8, 8); plt.ylim(-8, 8)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.show()

"""
EX 5
"""

import numpy as np
np.random.seed(1)

x = np.random.randint(0, 2, 1000)
y = np.random.randint(0, 2, 1000)

def accuracy(predictions, outcomes):
    return 100*np.mean(predictions == outcomes)

accuracy(x,y)

"""
Ex 6
"""
accuracy(0, data["high_quality"])

"""
Ex 7
"""

from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors = 5)
knn.fit(numeric_data, data['high_quality'])

library_predictions = knn.predict(numeric_data)

round(accuracy(library_predictions, data["high_quality"]))

"""
EX 8
"""

n_rows = data.shape[0]
random.seed(123)
selection = random.sample(range(n_rows), 10)

case study part 1/case study 3/final3.py


#  统计类别信息及各类样本数目
import os
import random
import pandas as pd
data_root = r'C:\Users\sherlock\Desktop\ML_faceMardk\small_dataset'

# # 数据统计
# info = []
# classes_name = os.listdir(data_root)
# classes_num = len(classes_name)
# for i in classes_name:
#     pic_root = os.path.join(data_root, i)
#     samples_name = os.listdir(pic_root)
#     samples_num = len(samples_name)
#     a = [i, samples_num]
#     info.append(a)
# df = pd.DataFrame(info)
# df.to_csv('info2.csv')

# # 数据增强
# from torchvision import transforms as tfs
# from PIL import Image
# num = 5
# classes_name = os.listdir(data_root)
# classes_num = len(classes_name)
# for i in classes_name:
#     pic_root = os.path.join(data_root, i)
#     samples_name = os.listdir(pic_root)
#     samples_num = len(samples_name)
#     # 随机改变图像的亮度
#     brightness_change = tfs.ColorJitter(brightness=0.5)
#     # 随机改变图像的对比度
#     contrast_change = tfs.ColorJitter(contrast=0.5)
#     rotation1 = tfs.RandomRotation((10, 45))
#     rotation2 = tfs.RandomRotation((10, 45))
#     transforms = [brightness_change, contrast_change, rotation2, rotation1]
#     if samples_num > num:
#         for j in range(samples_num - num):
#             os.remove(os.path.join(pic_root, samples_name[j]))
#     if samples_num < num:
#         im = Image.open(os.path.join(pic_root, samples_name[0]))
#         f_flag = 1
#         for j in range(num - samples_num + 1):
#             p = random.random()
#             if p < 0.5 and f_flag:
#                 f_flag = 0
#                 new_img = tfs.RandomHorizontalFlip(p=1)(im)
#             else:
#                 transfor = random.choice(transforms)
#                 new_img = transfor(im)
#             index = str(samples_num + j)
#             index = index.zfill(4)
#             img_name = i + "_" + index + '.jpg'
#             save_path = os.path.join(pic_root, img_name)
#             new_img.save(save_path)

# # 人脸特征提取 dlib
# import cv2
# import dlib
# import json
# model_path = r'C:\Users\sherlock\Desktop\ML_faceMardk\shape_predictor_68_face_landmarks.dat'
# detector = dlib.get_frontal_face_detector()
# predictor = dlib.shape_predictor(model_path)
# num = 5
# classes_name = os.listdir(data_root)
# classes_num = len(classes_name)
# feature_dic = {}
# for i in classes_name:
#     pic_root = os.path.join(data_root, i)
#     samples_name = os.listdir(pic_root)
#     samples_num = len(samples_name)
#     tmp_dic = {}
#     for j in samples_name:
#         img_path = os.path.join(pic_root, j)
#         img = cv2.imread(img_path)
#         gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#         faces = detector(gray_img, 1)
#         try:
#             shape = predictor(img, faces[0])
#             feature = shape.parts()
#             feature = [[i.x, i.y] for i in feature]
#             tmp_dic[j] = feature
#         except:
#             pass
#     feature_dic[i] = tmp_dic
#
# with open('features.json', 'w') as f:
#     json.dump(feature_dic, f)

# 人脸特征提取 insightface
import cv2
import numpy as np
import insightface
from insightface.app import FaceAnalysis
from insightface.data import get_image as ins_get_image
from  multiprocessing import Process, Pool, Manager
app = FaceAnalysis(providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
app.prepare(ctx_id=0, det_size=(640, 640))
handler = insightface.model_zoo.get_model('./model.onnx')
handler.prepare(ctx_id=0)

num = 5
classes_name = os.listdir(data_root)
classes_num = len(classes_name)
save_root = r'C:\Users\sherlock\Desktop\ML_faceMardk\new_dataset'
t = 0
for i in classes_name:
    pic_root = os.path.join(data_root, i)
    picsave_root = os.path.join(save_root, i)
    samples_name = os.listdir(pic_root)
    samples_num = len(samples_name)
    os.makedirs(picsave_root)
    for j in samples_name:
        img_path = os.path.join(pic_root, j)
        img_save = os.path.join(picsave_root, j)
        img = cv2.imread(img_path)
        try:
            faces = app.get(img)
            feature = handler.get(img, faces[0])
            save_path, ext = os.path.splitext(img_save)
            save_path = save_path + '.npy'
            np.save(save_path, feature)
        except:
            t += 1
            print('no face finded!!!', t)

# img_path = r'C:\Users\sherlock\Desktop\ML_faceMardk\small_dataset\Aaron_Eckhart\Aaron_Eckhart_0001.jpg'
# img = cv2.imread(img_path)
# faces = app.get(img)
# feature = handler.get(img, faces[0])
# print(feature)


import sys
sys.stdin = open('input.txt','r')

t = int(input())

for _ in range(t):
    n = int(input())
    candidate = []
    for i in range(n):
        test,interview = map(int,input().split())
        candidate.append([test,interview])

    candidate = sorted(candidate,key=lambda x:(x[0],x[1]),reverse=True)
    print(candidate)


import pygame
from typing import TYPE_CHECKING

from domain.utils import colors
from domain.services.drawer import Drawer
from domain.utils.math_utillity import multiply_tuple_infix as t_mul
# if TYPE_CHECKING:
#     from game import Game

class Button:
    def __init__(self, size, pos, callback, **kwargs):

        self.size = size
        self.pos = pos
        self.callback = callback
        self.rect = pygame.Rect(pos, size)
        self.drawer = Drawer()

        self.text = kwargs.pop('text', "Click Me!")
        self.text_color = kwargs.pop('text_color', colors.WHITE)
        self.background_color = kwargs.pop('background_color', colors.PASTEL_GREEN)
        self.border_width = kwargs.pop('border_width', 5)
        self.border_color = kwargs.pop('border_color', colors.PASTEL_DARK_GREEN)
        self.border_radius = kwargs.pop('border_radius', 15)
        self.font_obj = kwargs.pop('font_obj', pygame.font.SysFont('Calibri', 30,True))
        self.padding = kwargs.pop('padding', (18,18))
        self.hover_callback = kwargs.pop('hover_callback', self.default_hover)
        self.hovering = False

        self.defaults = {
            'size': self.size,
            'pos': self.pos,
            'rect': self.rect,
            'text_color': self.text_color,
            'background_color': self.background_color,
            'border_width': self.border_width,
            'border_color': self.border_color,
            'font_obj': self.font_obj,
            'padding': self.padding,
            'hover_callback': self.hover_callback,
        }

    def mouse_in_bounds(self):
        mouse_pos = pygame.mouse.get_pos()
        return self.rect.left < mouse_pos[0] < self.rect.right and\
            self.rect.top < mouse_pos[1] < self.rect.bottom

    def update(self, events):

        if self.mouse_in_bounds():
            self.hovering = True
            self.hover_callback(self)
        else:
            self.hovering = False
            self.hover_out()

        for ev in events:
            if ev.type == pygame.MOUSEBUTTONDOWN and self.mouse_in_bounds():
                self.callback(self)

    def draw(self, screen: pygame.Surface):
        pygame.draw.rect(screen, self.background_color, self.rect, border_radius=int(self.border_radius))
        pygame.draw.rect(screen, self.border_color, self.rect, self.border_width,self.border_radius)

        _txt_surface = self.drawer.get_text_surface(self.text, self.text_color, self.font_obj, 0)
        _txt_rect = _txt_surface.get_rect()
        _txt_rect.center = self.rect.center
        screen.blit(_txt_surface, _txt_rect.topleft)

    def default_hover(self, btn: 'Button'):
        btn.background_color = colors.PASTEL_LIGHT_GREEN
        btn.border_color = colors.PASTEL_GREEN

    def hover_out(self):
        for prop in self.defaults.items():
            setattr(self, prop[0], prop[1])


# 題目希望 O(n) 而且不用除法。那就不應該「全部乘來/nums[i]」
# 還好題目有暗示，就用 prefix 及 suffix 來做吧
# The product of any prefix or suffix of nums is guaranteed to fit in a 32-bit integer.
class Solution:
    def productExceptSelf(self, nums: List[int]) -> List[int]:
        N = len(nums)
        prefix = [0]*N # prefix[i] 表示 nums[0] * ... * nums[i]
        suffix = [0]*N # suffix[i] 表示 nums[i] * ... * nums[N-1]

        prefix[0] = nums[0]
        suffix[N-1] = nums[N-1]
        for i in range(1,N):
            prefix[i] = nums[i] * prefix[i-1] # 往右更新
            suffix[N-1-i] = nums[N-1-i] * suffix[N-1-i+1] # 往左更新
        # print(prefix)
        # print(suffix)

        ans = [0]*N
        ans[0] = suffix[1]
        ans[N-1] = prefix[N-2]
        for i in range(1,N-1):
            ans[i] = prefix[i-1] * suffix[i+1] # 就避開了 nums[i]
        return ans


##Criar um arquivo txt na pasta. Adicione algumas linhas no arquivo e faça
#um script python ler esse arquivo

file1 = open("Files/exercicio.txt","r")

print(file1.read())

file1.close()

##Abra o arquivo criado e sobrescreva o arquivo e adicione 2 linhas novas
file1 = open("Files/exercicio.txt","w+")
file1.write("Escrever um conteudo\n")
file1.write("Escrever dois conteudos\n")
file1.seek(0,0)
print(file1.read())
file1.close()

##No mesmo arquivo criado adicione uma nova linha sem sobrescrever o conteúdo do mesmo.
file1 = open("Files/exercicio.txt","a+")
file1.write("Adiciona mais uma linha")
file1.seek(0,0)
print(file1.read())
#file1.close()

##Copie o conteúdo do arquivo criado anteriormente para um novo arquivo.
#Utilize o gerenciador de contexto de arquivos para isso

outroFile = str(file1.read())

with open("Files/exercicio2.txt", "w+") as ftxt:
    ftxt.write(outroFile)
    ftxt.seek(0,0)
    print(ftxt.read())
    file1.close()


import unittest
class Node():
    def __init__(self, value):
        self.value = value
        self.next = None

def partition(node, x):
    less_than_x_head = None
    less_than_x_tail = None
    greater_equal_x_head = None
    greater_equal_x_tail = None

    while node:
        #keep track of node.next
        next = node.next
        #set current node to point to None
        node.next = None
        if node.value<x:
            if less_than_x_head == None:
                less_than_x_head = node
                less_than_x_tail = node
            else:
                less_than_x_tail.next = node
                less_than_x_tail = node
        else:
            if greater_equal_x_head == None:
                greater_equal_x_head = node
                greater_equal_x_tail = node
            else:
                greater_equal_x_tail.next = node
                greater_equal_x_tail = node
        node = next
        #check if less_than_h == None
    if less_than_x_head == None:
        return greater_equal_x_head
    else:
        less_than_x_tail.next = greater_equal_x_head
        less_than_x_tail = greater_equal_x_head
        return less_than_x_head
n1 = Node(3)
n2 = Node(5)
n3 = Node(8)
n4 = Node(5)
n5 = Node(10)
n6 = Node(2)
n7 = Node(1)
n1.next = n2
n2.next = n3
n3.next = n4
n4.next = n5
n5.next = n6
n6.next = n7

class Test(unittest.TestCase):
    def test_partition(self):
        result = partition(n1,5)#to jak polaczylam te nody razem to znaczy ze do funkcji jak podam n1 to jest lista nodow? jakos mi to wyglada ze n1 w funkcji partition jest rowny 3, czyli pojedynczemu nodowi?
        self.assertEqual(self.node_to_list(result),[3,2,1,5,8,5,10])

    def node_to_list(self,node):
        nodelist = []
        current = node
        while current:
            nodelist.append(current.value)
            current = current.next
        return nodelist

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-

class Solution:
    def maxCoins(self, nums):
        def _maxCoins(i, j):
            if i == j:
                return 0

            if cache[i * n + j] == -1:
                prod = 1
                if i > 0:
                    prod *= nums[i - 1]
                if j < len(nums):
                    prod *= nums[j]

                cache[i * n + j] = max(
                    _maxCoins(i, k) + prod * nums[k] + _maxCoins(k + 1, j)
                    for k in range(i, j)
                )

            return cache[i * n + j]

        n = len(nums) + 1
        cache = [-1 for i in range(n) for j in range(n)]
        return _maxCoins(0, len(nums))

if __name__ == "__main__":
    solution = Solution()

    assert 167 == solution.maxCoins([3, 1, 5, 8])


velocidade_carro = 110
velocidade_caminhao = 80
distancia_total = 100
tempo_pedagio = 10 # tempo em minutos

distancia_carro = ((velocidade_carro * ((distancia_total/2))) / (velocidade_carro + velocidade_caminhao)) + ((velocidade_carro * (tempo_pedagio/60)))

distancia_restante_carro = distancia_total - distancia_carro

distancia_caminhao = (velocidade_caminhao * ((distancia_total/2)-(tempo_pedagio/60))) / (velocidade_caminhao + velocidade_carro) + ((velocidade_caminhao * (tempo_pedagio/60)))

distancia_restante_caminhao = distancia_total - distancia_caminhao

print("O carro estará a", distancia_restante_carro, "km de Ribeirão Preto quando os veículos se cruzarem.")

print("O caminhão estará a", distancia_restante_caminhao, "km de Ribeirão Preto quando os veículos se cruzarem.")

if (distancia_restante_caminhao < distancia_restante_carro):
    print("Caminhão está mais perto")
else:
    print("Carro está mais perto")


from fpf_common.common.pff_common_exception import PFFCommonException
from pf_sqlalchemy.db.orm import database, Base
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from pf_sqlalchemy.crud.pfs_crud_exception import parse_integrity_error

class PfsCrudService:

    def _handle_exception(self, exception: Exception):
        if isinstance(exception, IntegrityError):
            raise PFFCommonException(parse_integrity_error(exception))
        raise PFFCommonException(str(exception))

    def save(self, model: Base):
        try:
            database.session.add(model)
            database.session.commit()
        except Exception as e:
            self._handle_exception(e)

    def save_all(self, models: list):
        try:
            database.session.add_all(models)
            database.session.commit()
        except Exception as e:
            self._handle_exception(e)

    def delete(self, model: Base):
        try:
            database.session.delete(model)
            database.session.commit()
        except Exception as e:
            self._handle_exception(e)

    def raw_query(self, sql):
        try:
            connection = database.engine.connect()
            return connection.execute(text(sql))
        except Exception as e:
            self._handle_exception(e)


import random,time

print("Choose one from ROCK-PAPER-SCISSOR while computer selects its own...")
user = input()
arr = ['ROCK','PAPER','SCISSOR']

computer = random.randint(1,3)
computer = arr[computer-1]
print("Computer chose: ",computer)
result = 0  # 0:no result, 1:user wins, 2:computer wins

if user == 'ROCK':
    if computer == 'SCISSOR':
        result = 1
    elif computer == 'PAPER':
        result = 2
elif user == 'PAPER':
    if computer == 'ROCK':
        result = 2
    elif computer == 'SCISSOR':
        result = 2
else:
    if computer == 'ROCK':
        result = 2
    elif computer == 'PAPER':
        result = 1

print("Winner is...", end='')
time.sleep(1)
if result == 0:
    print('DRAW')
elif result == 1:
    print("USER")
else:
    print("COMPUTER")


import os
import tornado.ioloop
import tornado.web

class MyFileHandler(tornado.web.StaticFileHandler):
    def initialize(self, path):
        self.dirname, self.filename = os.path.split(path)
        super(MyFileHandler, self).initialize(self.dirname)
    def get(self, path = None, include_body = True):
        self.set_header('Content-Type', 'text/javascript')
        super(MyFileHandler, self).get(self.filename, include_body)

app = tornado.web.Application([
    (r'/final\.json', MyFileHandler, {'path': './data/worldCupFinals.json'}),
    (r'/winner\.json', MyFileHandler, {'path': './data/worldCupMatches.json'})
])

app.listen(8888)
tornado.ioloop.IOLoop.current().start()


import sys
sys.setrecursionlimit(10**5)
N = int(input())
arr = []
dp = [[0] * N for _ in range(N)]
dp2 = [[0] * N for _ in range(N)]
normal = 0
greenGuy = 0
mv = [[1, 0], [-1, 0], [0, 1], [0, -1]]
for i in range(N):
    arr.append(list(str(input())))

def dfs(i, j, color):
    if dp[i][j]:
        return
    dp[i][j] = 1
    for [dx, dy] in mv:
        tX = dx + i
        tY = dy + j
        if 0 <= tX < N and 0 <= tY < N:
            if arr[tX][tY] == color:
                dfs(tX, tY, color)

def dfs2(i, j, color):
    if dp2[i][j]:
        return
    dp2[i][j] = 1
    for [dx, dy] in mv:
        tX = dx + i
        tY = dy + j
        if 0 <= tX < N and 0 <= tY < N:
            if color == "B" and arr[tX][tY] == color:
                dfs2(tX, tY, color)
            elif (color == 'R' or color == 'G') and (arr[tX][tY] == 'R' or arr[tX][tY] == 'G'):
                dfs2(tX, tY, color)

for i in range(N):
    for j in range(N):
        if not dp[i][j]:
            dfs(i, j, arr[i][j])
            normal += 1
        if not dp2[i][j]:
            dfs2(i, j, arr[i][j])
            greenGuy += 1
print(normal, greenGuy)